').appendTo($('#div_semgrex'));\n renderSemgrex(data);\n },\n error: function(data) {\n var alertDiv = $('
').addClass('alert').addClass('alert-danger').addClass('alert-dismissible').addClass('semgrex_error').attr('role', 'alert')\n var button = $('
').text(data.responseText);\n button.appendTo(alertDiv);\n message.appendTo(alertDiv);\n alertDiv.appendTo($('#div_semgrex'));\n }\n });\n });\n\n $('#form_tregex').submit( function (e) {\n // Don't actually submit the form\n e.preventDefault();\n // Get text\n if ($('#tregex_search').val().trim() == '') {\n $('#tregex_search').val('NP < NN=animal');\n }\n var pattern = $('#tregex_search').val();\n // Remove existing annotation\n $('#tregex').remove();\n // Add missing required annotators\n var requiredAnnotators = annotators().split(',');\n if (requiredAnnotators.indexOf('parse') < 0) {\n requiredAnnotators.push('parse');\n }\n // Make ajax call\n $.ajax({\n type: 'POST',\n url: serverAddress + '/tregex?pattern=' + encodeURIComponent(\n pattern.replace(\"&\", \"\\\\&\").replace('+', '\\\\+')) +\n '&properties=' + encodeURIComponent(\n '{\"annotators\": \"' + requiredAnnotators.join(',') + '\", \"date\": \"' + date() + '\"}') +\n '&pipelineLanguage=' + encodeURIComponent($('#language').val()),\n data: encodeURIComponent(currentQuery),\n success: function(data) {\n $('.tregex_error').remove(); // Clear error messages\n $('
').appendTo($('#div_tregex'));\n renderTregex(data);\n },\n error: function(data) {\n var alertDiv = $('
').addClass('alert').addClass('alert-danger').addClass('alert-dismissible').addClass('tregex_error').attr('role', 'alert')\n var button = $('
× ');\n var message = $('
').text(data.responseText);\n button.appendTo(alertDiv);\n message.appendTo(alertDiv);\n alertDiv.appendTo($('#div_tregex'));\n }\n });\n });\n\n});\n"
},
{
"path": "stanza/pipeline/demo/stanza-parseviewer.js",
"content": "//'use strict';\n\n//d3 || require('d3');\n//var dagreD3 = require('dagre-d3');\n//var jquery = require('jquery');\n//var $ = jquery;\n\nvar ParseViewer = function(params) {\n // Container in which the scene template is displayed\n this.selector = params.selector;\n this.container = $(this.selector);\n this.fitToGraph = true;\n this.onClickNodeCallback = params.onClickNodeCallback;\n this.onHoverNodeCallback = params.onHoverNodeCallback;\n this.init();\n return this;\n};\n\nParseViewer.MIN_WIDTH = 100;\nParseViewer.MIN_HEIGHT = 100;\n\nParseViewer.prototype.constructor = ParseViewer;\n\nParseViewer.prototype.getAutoWidth = function () {\n return Math.max(ParseViewer.MIN_WIDTH, this.container.width());\n};\n\nParseViewer.prototype.getAutoHeight = function () {\n return Math.max(ParseViewer.MIN_HEIGHT, this.container.height() - 20);\n};\n\nParseViewer.prototype.init = function () {\n var canvasWidth = this.getAutoWidth();\n var canvasHeight = this.getAutoHeight();\n this.parseElem = d3.select(this.selector)\n .append('svg')\n .attr({'width': canvasWidth, 'height': canvasHeight})\n .style({'width': canvasWidth, 'height': canvasHeight});\n console.log(this.parseElem);\n this.graph = null;\n this.graphRendered = false;\n\n this.controls = $('
');\n this.container.append(this.controls);\n};\n\nvar GraphBuilder = function(roots) {\n // Create the input graph\n this.graph = new dagreD3.graphlib.Graph()\n .setGraph({})\n .setDefaultEdgeLabel(function () {\n return {};\n });\n this.visitIndex = 0;\n //console.log('building graph', roots);\n for (var i = 0; i < roots.length; i++) {\n this.build(roots[i]);\n }\n};\n\nGraphBuilder.prototype.build = function(node) {\n console.log(node);\n // Track my visit index\n this.visitIndex++;\n node.visitIndex = this.visitIndex;\n\n // Add a node\n var nodeData = node; // TODO: replace with semantic data\n var nodeLabel = node.label;\n var nodeIndex = node.visitIndex;\n var nodeClass = 'parse-RULE';\n\n this.graph.setNode(nodeIndex, { label: nodeLabel, class: nodeClass, data: nodeData });\n if (node.parent) {\n this.graph.setEdge(node.parent.visitIndex, nodeIndex, {\n class: 'parse-EDGE'\n });\n }\n\n if (node.isTerminal) {\n this.visitIndex++;\n nodeIndex = this.visitIndex;\n nodeLabel = node.text;\n nodeClass = 'parse-TERMINAL';\n\n this.graph.setNode(nodeIndex, { label: nodeLabel, class: nodeClass, data: nodeData });\n this.graph.setEdge(node.visitIndex, nodeIndex, {\n class: 'parse-EDGE'\n });\n } else if (node.children) {\n for (var i = 0; i < node.children.length; i++) {\n this.build(node.children[i]);\n }\n }\n};\n\nParseViewer.prototype.updateGraphPosition = function (svg, g, minWidth, minHeight) {\n if (this.fitToGraph) {\n minWidth = g.graph().width;\n minHeight = this.getAutoHeight();\n }\n adjustGraphPositioning(svg, g, minWidth, minHeight);\n};\n\nfunction adjustGraphPositioning(svg, g, minWidth, minHeight) {\n // Resize svg\n var newWidth = Math.max(minWidth, g.graph().width);\n var newHeight = Math.max(minHeight, g.graph().height + 40);\n svg.attr({'width': newWidth, 'height': newHeight});\n svg.style({'width': newWidth, 'height': newHeight});\n // Center the graph\n var svgGroup = svg.select('g');\n var xCenterOffset = (svg.attr('width') - g.graph().width) / 2;\n svgGroup.attr('transform', 'translate(' + xCenterOffset + ', 20)');\n svg.attr('height', g.graph().height + 40);\n svg.style('height', g.graph().height + 40);\n}\n\nParseViewer.prototype.renderGraph = function (svg, g, parse) {\n // Create the renderer\n var render = new dagreD3.render();\n // Run the renderer. This is what draws the final graph.\n var svgGroup = svg.select('g');\n render(svgGroup, g);\n\n var scope = this;\n var nodes = svgGroup.selectAll('g.node');\n nodes.on('click',\n function (d) {\n var v = d;\n var node = g.node(v);\n if (scope.onClickNodeCallback) {\n scope.onClickNodeCallback(node.data);\n }\n console.log(g.node(v));\n }\n );\n\n nodes.on('mouseover',\n function (d) {\n var v = d;\n var node = g.node(v);\n if (scope.onHoverNodeCallback) {\n scope.onHoverNodeCallback(node.data);\n }\n }\n );\n\n this.updateGraphPosition(svg, g, svg.attr('width'), svg.attr('height'));\n this.graphRendered = true;\n};\n\nParseViewer.prototype.showParse = function (root) {\n this.showParses([root]);\n};\n\nParseViewer.prototype.showParses = function (roots) {\n // Take parse and create a graph\n var gb = new GraphBuilder(roots);\n var g = gb.graph;\n\n g.nodes().forEach(function (v) {\n var node = g.node(v);\n // Round the corners of the nodes\n node.rx = node.ry = 5;\n });\n\n var svg = this.parseElem;\n svg.selectAll('*').remove();\n var svgGroup = svg.append('g');\n this.graph = g;\n this.parse = roots;\n if (this.container.is(':visible')) {\n if (roots.length > 0) {\n this.renderGraph(svg, this.graph, this.parse);\n }\n } else {\n this.graphRendered = false;\n }\n};\n\nParseViewer.prototype.showAnnotation = function (annotation) {\n var parses = [];\n for (var i = 0; i < annotation.sentences.length; i++) {\n var s = annotation.sentences[i];\n if (s && s.parseTree) {\n parses.push(s.parseTree);\n }\n }\n this.showParses(parses);\n};\n\nParseViewer.prototype.onResize = function () {\n var canvasWidth = this.getAutoWidth();\n var canvasHeight = this.getAutoHeight();\n var svg = this.parseElem;\n\n // Center the graph\n var svgGroup = svg.select('g');\n if (svgGroup && this.graph) {\n if (!this.graphRendered) {\n svg.attr({'width': canvasWidth, 'height': canvasHeight});\n svg.style({'width': canvasWidth, 'height': canvasHeight});\n this.renderGraph(svg, this.graph, this.parse);\n } else {\n this.updateGraphPosition(svg, this.graph, canvasWidth, canvasHeight);\n }\n } else {\n svg.attr({'width': canvasWidth, 'height': canvasHeight});\n svg.style({'width': canvasWidth, 'height': canvasHeight});\n }\n};\n\n// Exports\n//module.exports = ParseViewer;\n"
},
{
"path": "stanza/pipeline/depparse_processor.py",
"content": "\"\"\"\nProcessor for performing dependency parsing\n\"\"\"\n\nimport torch\n\nfrom stanza.models.common import doc\nfrom stanza.models.common.utils import unsort\nfrom stanza.models.common.vocab import VOCAB_PREFIX\nfrom stanza.models.depparse.data import DataLoader\nfrom stanza.models.depparse.trainer import Trainer\nfrom stanza.pipeline._constants import *\nfrom stanza.pipeline.processor import UDProcessor, register_processor\n\n# these imports trigger the \"register_variant\" decorations\nfrom stanza.pipeline.external.corenlp_converter_depparse import ConverterDepparse\n\nDEFAULT_SEPARATE_BATCH=150\n\n@register_processor(name=DEPPARSE)\nclass DepparseProcessor(UDProcessor):\n\n # set of processor requirements this processor fulfills\n PROVIDES_DEFAULT = set([DEPPARSE])\n # set of processor requirements for this processor\n REQUIRES_DEFAULT = set([TOKENIZE, POS, LEMMA])\n\n def __init__(self, config, pipeline, device):\n self._pretagged = None\n super().__init__(config, pipeline, device)\n\n def _set_up_requires(self):\n self._pretagged = self._config.get('pretagged')\n if self._pretagged:\n self._requires = set()\n else:\n self._requires = self.__class__.REQUIRES_DEFAULT\n\n def _set_up_model(self, config, pipeline, device):\n self._trainer = config.get('trainer')\n if self._trainer is not None:\n return\n\n self._pretrain = pipeline.foundation_cache.load_pretrain(config['pretrain_path']) if 'pretrain_path' in config else None\n args = {'charlm_forward_file': config.get('forward_charlm_path', None),\n 'charlm_backward_file': config.get('backward_charlm_path', None)}\n self._trainer = Trainer(args=args, pretrain=self.pretrain, model_file=config['model_path'], device=device, foundation_cache=pipeline.foundation_cache)\n\n def get_known_relations(self):\n \"\"\"\n Return a list of relations which this processor can produce\n \"\"\"\n keys = [k for k in self.vocab['deprel']._unit2id.keys() if k not in VOCAB_PREFIX]\n return keys\n\n def process(self, document):\n if hasattr(self, '_variant'):\n return self._variant.process(document)\n\n if any(word.upos is None and word.xpos is None for sentence in document.sentences for word in sentence.words):\n raise ValueError(\"POS not run before depparse!\")\n try:\n batch = DataLoader(document, self.config['batch_size'], self.config, self.pretrain, vocab=self.vocab, evaluation=True,\n sort_during_eval=self.config.get('sort_during_eval', True),\n min_length_to_batch_separately=self.config.get('min_length_to_batch_separately', DEFAULT_SEPARATE_BATCH))\n with torch.no_grad():\n preds = []\n for i, b in enumerate(batch):\n preds += self.trainer.predict(b)\n if batch.data_orig_idx is not None:\n preds = unsort(preds, batch.data_orig_idx)\n batch.doc.set((doc.HEAD, doc.DEPREL), [y for x in preds for y in x])\n # build dependencies based on predictions\n for sentence in batch.doc.sentences:\n sentence.build_dependencies()\n return batch.doc\n except RuntimeError as e:\n if str(e).startswith(\"CUDA out of memory. Tried to allocate\"):\n new_message = str(e) + \" ... You may be able to compensate for this by separating long sentences into their own batch with a parameter such as depparse_min_length_to_batch_separately=150 or by limiting the overall batch size with depparse_batch_size=400.\"\n raise RuntimeError(new_message) from e\n else:\n raise\n"
},
{
"path": "stanza/pipeline/external/__init__.py",
"content": ""
},
{
"path": "stanza/pipeline/external/corenlp_converter_depparse.py",
"content": "\"\"\"\nA depparse processor which converts constituency trees using CoreNLP\n\"\"\"\n\nfrom stanza.pipeline._constants import TOKENIZE, CONSTITUENCY, DEPPARSE\nfrom stanza.pipeline.processor import ProcessorVariant, register_processor_variant\nfrom stanza.server.dependency_converter import DependencyConverter\n\n@register_processor_variant(DEPPARSE, 'converter')\nclass ConverterDepparse(ProcessorVariant):\n # set of processor requirements for this processor\n REQUIRES_DEFAULT = set([TOKENIZE, CONSTITUENCY])\n\n def __init__(self, config):\n if config['lang'] != 'en':\n raise ValueError(\"Constituency to dependency converter only works for English\")\n\n # TODO: get classpath from config\n # TODO: close this when finished?\n # a more involved approach would be to turn the Pipeline into\n # a context with __enter__ and __exit__\n # __exit__ would try to free all resources, although some\n # might linger such as GPU allocations\n # maybe it isn't worth even trying to clean things up on account of that\n self.converter = DependencyConverter(classpath=\"$CLASSPATH\")\n self.converter.open_pipe()\n\n def process(self, document):\n return self.converter.process(document)\n"
},
{
"path": "stanza/pipeline/external/jieba.py",
"content": "\"\"\"\nProcessors related to Jieba in the pipeline.\n\"\"\"\n\nimport re\nimport warnings\n\nfrom stanza.models.common import doc\nfrom stanza.pipeline._constants import TOKENIZE\nfrom stanza.pipeline.processor import ProcessorVariant, register_processor_variant\n\ndef check_jieba():\n \"\"\"\n Import necessary components from Jieba to perform tokenization.\n \"\"\"\n try:\n import jieba\n except ImportError:\n raise ImportError(\n \"Jieba is used but not installed on your machine. Go to https://pypi.org/project/jieba/ for installation instructions.\"\n )\n return True\n\n@register_processor_variant(TOKENIZE, 'jieba')\nclass JiebaTokenizer(ProcessorVariant):\n def __init__(self, config):\n \"\"\" Construct a Jieba-based tokenizer by loading the Jieba pipeline.\n\n Note that this tokenizer uses regex for sentence segmentation.\n \"\"\"\n if config['lang'] not in ['zh', 'zh-hans', 'zh-hant']:\n raise Exception(\"Jieba tokenizer is currently only allowed in Chinese (simplified or traditional) pipelines.\")\n\n # Surpress a DeprecationWarning about pkg_resource from jieba.\n with warnings.catch_warnings():\n warnings.filterwarnings(\"ignore\", category=DeprecationWarning, module=\"jieba\")\n check_jieba()\n import jieba\n\n self.nlp = jieba\n self.no_ssplit = config.get('no_ssplit', False)\n\n def process(self, document):\n \"\"\" Tokenize a document with the Jieba tokenizer and wrap the results into a Doc object.\n \"\"\"\n if isinstance(document, doc.Document):\n text = document.text\n else:\n text = document\n if not isinstance(text, str):\n raise Exception(\"Must supply a string or Stanza Document object to the Jieba tokenizer.\")\n tokens = self.nlp.cut(text, cut_all=False)\n\n sentences = []\n current_sentence = []\n offset = 0\n for token in tokens:\n if re.match(r'\\s+', token):\n offset += len(token)\n continue\n\n token_entry = {\n doc.TEXT: token,\n doc.MISC: f\"{doc.START_CHAR}={offset}|{doc.END_CHAR}={offset+len(token)}\"\n }\n current_sentence.append(token_entry)\n offset += len(token)\n\n if not self.no_ssplit and token in ['。', '！', '？', '!', '?']:\n sentences.append(current_sentence)\n current_sentence = []\n\n if len(current_sentence) > 0:\n sentences.append(current_sentence)\n\n return doc.Document(sentences, text)\n"
},
{
"path": "stanza/pipeline/external/pythainlp.py",
"content": "\"\"\"\nProcessors related to PyThaiNLP in the pipeline.\n\nGitHub Home: https://github.com/PyThaiNLP/pythainlp\n\"\"\"\n\nfrom stanza.models.common import doc\nfrom stanza.pipeline._constants import TOKENIZE\nfrom stanza.pipeline.processor import ProcessorVariant, register_processor_variant\n\ndef check_pythainlp():\n \"\"\"\n Import necessary components from pythainlp to perform tokenization.\n \"\"\"\n try:\n import pythainlp\n except ImportError:\n raise ImportError(\n \"The pythainlp library is required. \"\n \"Try to install it with `pip install pythainlp`. \"\n \"Go to https://github.com/PyThaiNLP/pythainlp for more information.\"\n )\n return True\n\n@register_processor_variant(TOKENIZE, 'pythainlp')\nclass PyThaiNLPTokenizer(ProcessorVariant):\n def __init__(self, config):\n \"\"\" Construct a PyThaiNLP-based tokenizer.\n\n Note that we always uses the default tokenizer of PyThaiNLP for sentence and word segmentation.\n Currently this is a CRF model for sentence segmentation and a dictionary-based model (newmm) for word segmentation.\n \"\"\"\n if config['lang'] != 'th':\n raise Exception(\"PyThaiNLP tokenizer is only allowed in Thai pipeline.\")\n\n check_pythainlp()\n from pythainlp.tokenize import sent_tokenize as pythai_sent_tokenize\n from pythainlp.tokenize import word_tokenize as pythai_word_tokenize\n\n self.pythai_sent_tokenize = pythai_sent_tokenize\n self.pythai_word_tokenize = pythai_word_tokenize\n self.no_ssplit = config.get('no_ssplit', False)\n \n def process(self, document):\n \"\"\" Tokenize a document with the PyThaiNLP tokenizer and wrap the results into a Doc object.\n \"\"\"\n if isinstance(document, doc.Document):\n text = document.text\n else:\n text = document\n if not isinstance(text, str):\n raise Exception(\"Must supply a string or Stanza Document object to the PyThaiNLP tokenizer.\")\n\n sentences = []\n current_sentence = []\n offset = 0\n\n if self.no_ssplit:\n # skip sentence segmentation\n sent_strs = [text]\n else:\n sent_strs = self.pythai_sent_tokenize(text, engine='crfcut')\n for sent_str in sent_strs:\n for token_str in self.pythai_word_tokenize(sent_str, engine='newmm'):\n # by default pythainlp will output whitespace as a token\n # we need to skip these tokens to be consistent with other tokenizers\n if token_str.isspace():\n offset += len(token_str)\n continue\n \n # create token entry\n token_entry = {\n doc.TEXT: token_str,\n doc.MISC: f\"{doc.START_CHAR}={offset}|{doc.END_CHAR}={offset+len(token_str)}\"\n }\n current_sentence.append(token_entry)\n offset += len(token_str)\n \n # finish sentence\n sentences.append(current_sentence)\n current_sentence = []\n\n if len(current_sentence) > 0:\n sentences.append(current_sentence)\n\n return doc.Document(sentences, text)"
},
{
"path": "stanza/pipeline/external/spacy.py",
"content": "\"\"\"\nProcessors related to spaCy in the pipeline.\n\"\"\"\n\nfrom stanza.models.common import doc\nfrom stanza.pipeline._constants import TOKENIZE\nfrom stanza.pipeline.processor import ProcessorVariant, register_processor_variant\n\ndef check_spacy():\n \"\"\"\n Import necessary components from spaCy to perform tokenization.\n \"\"\"\n try:\n import spacy\n except ImportError:\n raise ImportError(\n \"spaCy is used but not installed on your machine. Go to https://spacy.io/usage for installation instructions.\"\n )\n return True\n\n@register_processor_variant(TOKENIZE, 'spacy')\nclass SpacyTokenizer(ProcessorVariant):\n def __init__(self, config):\n \"\"\" Construct a spaCy-based tokenizer by loading the spaCy pipeline.\n \"\"\"\n if config['lang'] != 'en':\n raise Exception(\"spaCy tokenizer is currently only allowed in English pipeline.\")\n\n try:\n import spacy\n from spacy.lang.en import English\n except ImportError:\n raise ImportError(\n \"spaCy 2.0+ is used but not installed on your machine. Go to https://spacy.io/usage for installation instructions.\"\n )\n\n # Create a Tokenizer with the default settings for English\n # including punctuation rules and exceptions\n self.nlp = English()\n # by default spacy uses dependency parser to do ssplit\n # we need to add a sentencizer for fast rule-based ssplit\n if spacy.__version__.startswith(\"2.\"):\n self.nlp.add_pipe(self.nlp.create_pipe(\"sentencizer\"))\n else:\n self.nlp.add_pipe(\"sentencizer\")\n self.no_ssplit = config.get('no_ssplit', False)\n\n def process(self, document):\n \"\"\" Tokenize a document with the spaCy tokenizer and wrap the results into a Doc object.\n \"\"\"\n if isinstance(document, doc.Document):\n text = document.text\n else:\n text = document\n if not isinstance(text, str):\n raise Exception(\"Must supply a string or Stanza Document object to the spaCy tokenizer.\")\n spacy_doc = self.nlp(text)\n\n sentences = []\n for sent in spacy_doc.sents:\n tokens = []\n for tok in sent:\n token_entry = {\n doc.TEXT: tok.text,\n doc.MISC: f\"{doc.START_CHAR}={tok.idx}|{doc.END_CHAR}={tok.idx+len(tok.text)}\"\n }\n tokens.append(token_entry)\n sentences.append(tokens)\n\n # if no_ssplit is set, flatten all the sentences into one sentence\n if self.no_ssplit:\n sentences = [[t for s in sentences for t in s]]\n\n return doc.Document(sentences, text)\n"
},
{
"path": "stanza/pipeline/external/sudachipy.py",
"content": "\"\"\"\nProcessors related to SudachiPy in the pipeline.\n\nGitHub Home: https://github.com/WorksApplications/SudachiPy\n\"\"\"\n\nimport re\n\nfrom stanza.models.common import doc\nfrom stanza.pipeline._constants import TOKENIZE\nfrom stanza.pipeline.processor import ProcessorVariant, register_processor_variant\n\ndef check_sudachipy():\n \"\"\"\n Import necessary components from SudachiPy to perform tokenization.\n \"\"\"\n try:\n import sudachipy\n import sudachidict_core\n except ImportError:\n raise ImportError(\n \"Both sudachipy and sudachidict_core libraries are required. \"\n \"Try install them with `pip install sudachipy sudachidict_core`. \"\n \"Go to https://github.com/WorksApplications/SudachiPy for more information.\"\n )\n return True\n\n@register_processor_variant(TOKENIZE, 'sudachipy')\nclass SudachiPyTokenizer(ProcessorVariant):\n def __init__(self, config):\n \"\"\" Construct a SudachiPy-based tokenizer.\n\n Note that this tokenizer uses regex for sentence segmentation.\n \"\"\"\n if config['lang'] != 'ja':\n raise Exception(\"SudachiPy tokenizer is only allowed in Japanese pipelines.\")\n\n check_sudachipy()\n from sudachipy import tokenizer\n from sudachipy import dictionary\n\n self.tokenizer = dictionary.Dictionary().create()\n self.no_ssplit = config.get('no_ssplit', False)\n\n def process(self, document):\n \"\"\" Tokenize a document with the SudachiPy tokenizer and wrap the results into a Doc object.\n \"\"\"\n if isinstance(document, doc.Document):\n text = document.text\n else:\n text = document\n if not isinstance(text, str):\n raise Exception(\"Must supply a string or Stanza Document object to the SudachiPy tokenizer.\")\n\n # we use the default sudachipy tokenization mode (i.e., mode C)\n # more config needs to be added to support other modes\n\n tokens = self.tokenizer.tokenize(text)\n\n sentences = []\n current_sentence = []\n for token in tokens:\n token_text = token.surface()\n # by default sudachipy will output whitespace as a token\n # we need to skip these tokens to be consistent with other tokenizers\n if token_text.isspace():\n continue\n start = token.begin()\n end = token.end()\n\n token_entry = {\n doc.TEXT: token_text,\n doc.MISC: f\"{doc.START_CHAR}={start}|{doc.END_CHAR}={end}\"\n }\n current_sentence.append(token_entry)\n\n if not self.no_ssplit and token_text in ['。', '！', '？', '!', '?']:\n sentences.append(current_sentence)\n current_sentence = []\n\n if len(current_sentence) > 0:\n sentences.append(current_sentence)\n\n return doc.Document(sentences, text)\n"
},
{
"path": "stanza/pipeline/langid_processor.py",
"content": "\"\"\"\nProcessor for determining language of text.\n\"\"\"\n\nimport emoji\nimport re\nimport stanza\nimport torch\n\nfrom stanza.models.common.doc import Document\nfrom stanza.models.langid.model import LangIDBiLSTM\nfrom stanza.pipeline._constants import *\nfrom stanza.pipeline.processor import UDProcessor, register_processor\n\n\n@register_processor(name=LANGID)\nclass LangIDProcessor(UDProcessor):\n \"\"\"\n Class for detecting language of text.\n \"\"\"\n\n # set of processor requirements this processor fulfills\n PROVIDES_DEFAULT = set([LANGID])\n\n # set of processor requirements for this processor\n REQUIRES_DEFAULT = set([])\n\n # default max sequence length\n MAX_SEQ_LENGTH_DEFAULT = 1000\n\n def _set_up_model(self, config, pipeline, device):\n batch_size = config.get(\"batch_size\", 64)\n self._model = LangIDBiLSTM.load(path=config[\"model_path\"], device=device,\n batch_size=batch_size, lang_subset=config.get(\"lang_subset\"))\n self._char_index = self._model.char_to_idx\n self._clean_text = config.get(\"clean_text\")\n\n def _text_to_tensor(self, docs):\n \"\"\"\n Map list of strings to batch tensor. Assumed all docs are same length.\n \"\"\"\n\n device = next(self._model.parameters()).device\n all_docs = []\n for doc in docs:\n doc_chars = [self._char_index.get(c, self._char_index[\"UNK\"]) for c in list(doc)]\n all_docs.append(doc_chars)\n return torch.tensor(all_docs, device=device, dtype=torch.long)\n\n def _id_langs(self, batch_tensor):\n \"\"\"\n Identify languages for each sequence in a batch tensor\n \"\"\"\n predictions = self._model.prediction_scores(batch_tensor)\n prediction_labels = [self._model.idx_to_tag[prediction] for prediction in predictions]\n\n return prediction_labels\n\n # regexes for cleaning text\n http_regex = re.compile(r\"https?:\\/\\/t\\.co/[a-zA-Z0-9]+\")\n handle_regex = re.compile(\"@[a-zA-Z0-9_]+\")\n hashtag_regex = re.compile(\"#[a-zA-Z]+\")\n punctuation_regex = re.compile(\"[!.]+\")\n all_regexes = [http_regex, handle_regex, hashtag_regex, punctuation_regex]\n\n @staticmethod\n def clean_text(text):\n \"\"\"\n Process text to improve language id performance. Main emphasis is on tweets, this method removes shortened\n urls, hashtags, handles, and punctuation and emoji.\n \"\"\"\n\n for regex in LangIDProcessor.all_regexes:\n text = regex.sub(\" \", text)\n\n text = emoji.emojize(text)\n text = emoji.replace_emoji(text, replace=' ')\n\n if text.strip():\n text = text.strip()\n\n return text\n\n def _process_list(self, docs):\n \"\"\"\n Identify language of list of strings or Documents\n \"\"\"\n\n if len(docs) == 0:\n # TO DO: what standard do we want for bad input, such as empty list?\n # TO DO: more handling of bad input\n return\n\n if isinstance(docs[0], str):\n docs = [Document([], text) for text in docs]\n\n docs_by_length = {}\n for doc in docs:\n text = LangIDProcessor.clean_text(doc.text) if self._clean_text else doc.text\n doc_length = len(text)\n if doc_length not in docs_by_length:\n docs_by_length[doc_length] = []\n docs_by_length[doc_length].append((doc, text))\n\n for doc_length in docs_by_length:\n inputs = [doc[1] for doc in docs_by_length[doc_length]]\n predictions = self._id_langs(self._text_to_tensor(inputs))\n for doc, lang in zip(docs_by_length[doc_length], predictions):\n doc[0].lang = lang\n\n return docs\n\n def process(self, doc):\n \"\"\"\n Handle single str or Document\n \"\"\"\n\n wrapped_doc = [doc]\n return self._process_list(wrapped_doc)[0]\n\n def bulk_process(self, docs):\n \"\"\"\n Handle list of strings or Documents\n \"\"\"\n\n return self._process_list(docs)\n\n"
},
{
"path": "stanza/pipeline/lemma_processor.py",
"content": "\"\"\"\nProcessor for performing lemmatization\n\"\"\"\n\nfrom itertools import compress\n\nimport torch\n\nfrom stanza.models.common import doc\nfrom stanza.models.lemma.data import DataLoader\nfrom stanza.models.lemma.trainer import Trainer\nfrom stanza.pipeline._constants import *\nfrom stanza.pipeline.processor import UDProcessor, register_processor\n\nWORD_TAGS = [doc.TEXT, doc.UPOS]\n\n@register_processor(name=LEMMA)\nclass LemmaProcessor(UDProcessor):\n\n # set of processor requirements this processor fulfills\n PROVIDES_DEFAULT = set([LEMMA])\n # set of processor requirements for this processor\n # pos will be added later for non-identity lemmatizerx\n REQUIRES_DEFAULT = set([TOKENIZE])\n # default batch size\n DEFAULT_BATCH_SIZE = 5000\n\n def __init__(self, config, pipeline, device):\n # run lemmatizer in identity mode\n self._use_identity = None\n self._pretagged = None\n super().__init__(config, pipeline, device)\n\n @property\n def use_identity(self):\n return self._use_identity\n\n def _set_up_model(self, config, pipeline, device):\n if config.get('use_identity') in ['True', True]:\n self._use_identity = True\n self._config = config\n self.config['batch_size'] = LemmaProcessor.DEFAULT_BATCH_SIZE\n else:\n # the lemmatizer only looks at one word when making\n # decisions, not the surrounding context\n # therefore, we can save some time by remembering what\n # we did the last time we saw any given word,pos\n # since a long running program will remember everything\n # (unless we go back and make it smarter)\n # we make this an option, not the default\n # TODO: need to update the cache to skip the contextual lemmatizer\n self.store_results = config.get('store_results', False)\n self._use_identity = False\n args = {'charlm_forward_file': config.get('forward_charlm_path', None),\n 'charlm_backward_file': config.get('backward_charlm_path', None)}\n lemma_classifier_args = dict(args)\n lemma_classifier_args['wordvec_pretrain_file'] = config.get('pretrain_path', None)\n self._trainer = Trainer(args=args, model_file=config['model_path'], device=device, foundation_cache=pipeline.foundation_cache, lemma_classifier_args=lemma_classifier_args)\n\n def _set_up_requires(self):\n self._pretagged = self._config.get('pretagged', None)\n if self._pretagged:\n self._requires = set()\n elif self.config.get('pos') and not self.use_identity:\n self._requires = LemmaProcessor.REQUIRES_DEFAULT.union(set([POS]))\n else:\n self._requires = LemmaProcessor.REQUIRES_DEFAULT\n\n def process(self, document):\n if not self.use_identity:\n batch = DataLoader(document, self.config['batch_size'], self.config, vocab=self.vocab, evaluation=True, expand_unk_vocab=True)\n else:\n batch = DataLoader(document, self.config['batch_size'], self.config, evaluation=True, conll_only=True)\n if self.use_identity:\n preds = [word.text for sent in batch.doc.sentences for word in sent.words]\n elif self.config.get('dict_only', False):\n preds = self.trainer.predict_dict(batch.doc.get([doc.TEXT, doc.UPOS]))\n else:\n if self.config.get('ensemble_dict', False):\n # skip the seq2seq model when we can\n skip = self.trainer.skip_seq2seq(batch.doc.get([doc.TEXT, doc.UPOS]))\n # although there is no explicit use of caseless or lemma_caseless in this processor,\n # it shows up in the config which gets passed to the DataLoader,\n # possibly affecting its results\n seq2seq_batch = DataLoader(document, self.config['batch_size'], self.config, vocab=self.vocab,\n evaluation=True, skip=skip, expand_unk_vocab=True)\n else:\n seq2seq_batch = batch\n\n with torch.no_grad():\n preds = []\n edits = []\n for i, b in enumerate(seq2seq_batch):\n ps, es = self.trainer.predict(b, self.config['beam_size'], seq2seq_batch.vocab)\n preds += ps\n if es is not None:\n edits += es\n\n if self.config.get('ensemble_dict', False):\n word_tags = batch.doc.get(WORD_TAGS)\n words = [x[0] for x in word_tags]\n preds = self.trainer.postprocess([x for x, y in zip(words, skip) if not y], preds, edits=edits)\n if self.store_results:\n new_word_tags = compress(word_tags, map(lambda x: not x, skip))\n new_predictions = [(x[0], x[1], y) for x, y in zip(new_word_tags, preds)]\n self.trainer.train_dict(new_predictions, update_word_dict=False)\n # expand seq2seq predictions to the same size as all words\n i = 0\n preds1 = []\n for s in skip:\n if s:\n preds1.append('')\n else:\n preds1.append(preds[i])\n i += 1\n preds = self.trainer.ensemble(word_tags, preds1)\n else:\n preds = self.trainer.postprocess(batch.doc.get([doc.TEXT]), preds, edits=edits)\n\n if self.trainer.has_contextual_lemmatizers():\n preds = self.trainer.update_contextual_preds(batch.doc, preds)\n\n # map empty string lemmas to '_'\n preds = [max([(len(x), x), (0, '_')])[1] for x in preds]\n batch.doc.set([doc.LEMMA], preds)\n return batch.doc\n"
},
{
"path": "stanza/pipeline/morphseg_processor.py",
"content": "from stanza.pipeline.core import UnsupportedProcessorError\nfrom stanza.pipeline.processor import UDProcessor, register_processor\nfrom stanza.pipeline._constants import MORPHSEG, TOKENIZE\n\n\n@register_processor(name=MORPHSEG)\nclass MorphSegProcessor(UDProcessor):\n PROVIDES_DEFAULT = {MORPHSEG}\n REQUIRES_DEFAULT = {TOKENIZE}\n\n def __init__(self, config, pipeline, device):\n self._config = config\n self._pipeline = pipeline\n self._set_up_requires()\n self._set_up_provides()\n self._set_up_model(config, pipeline, device)\n\n def _set_up_model(self, config, pipeline, device):\n try:\n from morphseg import MorphemeSegmenter\n except ImportError:\n raise ImportError(\n \"morphseg is required for morpheme segmentation. \"\n \"Install it with: pip install morphseg\"\n )\n\n lang = config.get('lang', 'en')\n model_path = config.get('morphseg_model_path', None)\n\n if model_path:\n self._segmenter = MorphemeSegmenter(\n lang=lang,\n load_pretrained=False,\n model_filepath=model_path,\n is_local=True\n )\n else:\n self._segmenter = MorphemeSegmenter(\n lang=lang,\n load_pretrained=True\n )\n if self._segmenter.sequence_labeller is None:\n raise UnsupportedProcessorError(\"morphseg\", lang)\n\n def process(self, document):\n # Collect all words from all sentences\n all_words = []\n word_mapping = [] # Track which sentence and word index each prediction belongs to\n\n for sent_idx, sent in enumerate(document.sentences):\n if not sent.words:\n continue\n for word_idx, word in enumerate(sent.words):\n all_words.append(word.text)\n word_mapping.append((sent_idx, word_idx))\n\n if not all_words:\n return document\n\n # Prepare input for morphseg (it expects normalized, lowercased character lists)\n word_char_lists = [\n list(self._segmenter.normalize_for_morphology(word))\n for word in all_words\n ]\n\n # Batch predict using the internal sequence_labeller\n predictions = self._segmenter.sequence_labeller.predict(sources=word_char_lists)\n\n # Extract segmentations from predictions\n from morphseg.training.oracle import rules2sent\n segmentations = [\n rules2sent(\n source=[align_pos.symbol for align_pos in pred.alignment],\n actions=pred.prediction\n ).split(' @@') # Split by morphseg's default delimiter\n for pred in predictions\n ]\n\n # Assign segmentations back to words\n for (sent_idx, word_idx), seg in zip(word_mapping, segmentations):\n document.sentences[sent_idx].words[word_idx].morphemes = seg\n\n return document\n"
},
{
"path": "stanza/pipeline/multilingual.py",
"content": "\"\"\"\nClass for running multilingual pipelines\n\"\"\"\n\nfrom collections import OrderedDict\nimport copy\nimport logging\nfrom typing import Union\n\nfrom stanza.models.common.doc import Document\nfrom stanza.models.common.utils import default_device\nfrom stanza.pipeline.core import Pipeline, DownloadMethod\nfrom stanza.pipeline._constants import *\nfrom stanza.resources.common import DEFAULT_MODEL_DIR, get_language_resources, load_resources_json\n\nlogger = logging.getLogger('stanza')\n\nclass MultilingualPipeline:\n \"\"\"\n Pipeline for handling multilingual data. Takes in text, detects language, and routes request to pipeline for that\n language.\n\n You can specify options to individual language pipelines with the lang_configs field.\n For example, if you want English pipelines to have NER, but want to turn that off for French, you can do:\n lang_configs = {\"en\": {\"processors\": \"tokenize,pos,lemma,depparse,ner\"},\n \"fr\": {\"processors\": \"tokenize,pos,lemma,depparse\"}}\n pipeline = MultilingualPipeline(lang_configs=lang_configs)\n\n You can also pass in a defaultdict created in such a way that it provides default parameters for each language.\n For example, in order to only get tokenization for each language:\n (remembering that the Pipeline will automagically add MWT to a language which uses MWT):\n from collections import defaultdict\n lang_configs = defaultdict(lambda: dict(processors=\"tokenize\"))\n pipeline = MultilingualPipeline(lang_configs=lang_configs)\n\n download_method can be set as in Pipeline to turn off downloading\n of the .json config or turn off downloading of everything\n \"\"\"\n\n def __init__(self,\n model_dir: str = DEFAULT_MODEL_DIR,\n lang_id_config: dict = None,\n lang_configs: dict = None,\n ld_batch_size: int = 64,\n max_cache_size: int = 10,\n use_gpu: bool = None,\n restrict: bool = False,\n device: str = None,\n download_method: DownloadMethod = DownloadMethod.DOWNLOAD_RESOURCES,\n # python 3.6 compatibility - maybe want to update to 3.7 at some point\n processors: Union[str, list] = None,\n ):\n # set up configs and cache for various language pipelines\n self.model_dir = model_dir\n self.lang_id_config = {} if lang_id_config is None else copy.deepcopy(lang_id_config)\n self.lang_configs = {} if lang_configs is None else copy.deepcopy(lang_configs)\n self.max_cache_size = max_cache_size\n # OrderedDict so we can use it as a LRU cache\n # most recent Pipeline goes to the end, pop the oldest one\n # when we run out of space\n self.pipeline_cache = OrderedDict()\n if processors is None:\n self.default_processors = None\n elif isinstance(processors, str):\n self.default_processors = [x.strip() for x in processors.split(\",\")]\n else:\n self.default_processors = list(processors)\n\n self.download_method = download_method\n if 'download_method' not in self.lang_id_config:\n self.lang_id_config['download_method'] = self.download_method\n\n # if lang is not in any of the lang_configs, update them to\n # include the lang parameter. otherwise, the default language\n # will always be used...\n for lang in self.lang_configs:\n if 'lang' not in self.lang_configs[lang]:\n self.lang_configs[lang]['lang'] = lang\n\n if restrict and 'langid_lang_subset' not in self.lang_id_config:\n known_langs = sorted(self.lang_configs.keys())\n if known_langs == 0:\n logger.warning(\"MultilingualPipeline asked to restrict to lang_configs, but lang_configs was empty. Ignoring...\")\n else:\n logger.debug(\"Restricting MultilingualPipeline to %s\", known_langs)\n self.lang_id_config['langid_lang_subset'] = known_langs\n\n # set use_gpu\n if device is None:\n if use_gpu is None or use_gpu == True:\n device = default_device()\n else:\n device = 'cpu'\n self.device = device\n \n # build language id pipeline\n self.lang_id_pipeline = Pipeline(dir=self.model_dir, lang='multilingual', processors=\"langid\", \n device=self.device, **self.lang_id_config)\n # load the resources so that we can refer to it later when building a new pipeline\n # note that it was either downloaded or not based on download_method when building the lang_id_pipeline\n self.resources = load_resources_json(self.model_dir)\n\n def _update_pipeline_cache(self, lang):\n \"\"\"\n Do any necessary updates to the pipeline cache for this language. This includes building a new\n pipeline for the lang, and possibly clearing out a language with the old last access date.\n \"\"\"\n\n # update request history\n if lang in self.pipeline_cache:\n self.pipeline_cache.move_to_end(lang, last=True)\n\n # update language configs\n # try/except to allow for a defaultdict\n try:\n lang_config = self.lang_configs[lang]\n except KeyError:\n lang_config = {'lang': lang}\n self.lang_configs[lang] = lang_config\n\n # if a defaultdict is passed in, the defaultdict might not contain 'lang'\n # so even though we tried adding 'lang' in the constructor, we'll check again here\n if 'lang' not in lang_config:\n lang_config['lang'] = lang\n\n if 'download_method' not in lang_config:\n lang_config['download_method'] = self.download_method\n\n if 'processors' not in lang_config:\n if self.default_processors:\n lang_resources = get_language_resources(self.resources, lang)\n lang_processors = [x for x in self.default_processors if x in lang_resources]\n if lang_processors != self.default_processors:\n logger.info(\"Not all requested processors %s available for %s. Loading %s instead\", self.default_processors, lang, lang_processors)\n lang_config['processors'] = \",\".join(lang_processors)\n\n if 'device' not in lang_config:\n lang_config['device'] = self.device\n\n # update pipeline cache\n if lang not in self.pipeline_cache:\n logger.debug(\"Loading unknown language in MultilingualPipeline: %s\", lang)\n # clear least recently used lang from pipeline cache\n if len(self.pipeline_cache) == self.max_cache_size:\n self.pipeline_cache.popitem(last=False)\n self.pipeline_cache[lang] = Pipeline(dir=self.model_dir, **self.lang_configs[lang])\n\n def process(self, doc):\n \"\"\"\n Run language detection on a string, a Document, or a list of either, route to language specific pipeline\n \"\"\"\n\n # only return a list if given a list\n singleton_input = not isinstance(doc, list)\n if singleton_input:\n docs = [doc]\n else:\n docs = doc\n\n if docs and isinstance(docs[0], str):\n docs = [Document([], text=text) for text in docs]\n\n # run language identification\n docs_w_langid = self.lang_id_pipeline.process(docs)\n\n # create language specific batches, store global idx with each doc\n lang_batches = {}\n for doc_idx, doc in enumerate(docs_w_langid):\n logger.debug(\"Language for document %d: %s\", doc_idx, doc.lang)\n if doc.lang not in lang_batches:\n lang_batches[doc.lang] = []\n lang_batches[doc.lang].append(doc)\n\n # run through each language, submit a batch to the language specific pipeline\n for lang in lang_batches.keys():\n self._update_pipeline_cache(lang)\n self.pipeline_cache[lang](lang_batches[lang])\n\n # only return a list if given a list\n if singleton_input:\n return docs_w_langid[0]\n else:\n return docs_w_langid\n\n def __call__(self, doc):\n doc = self.process(doc)\n return doc\n\n"
},
{
"path": "stanza/pipeline/mwt_processor.py",
"content": "\"\"\"\nProcessor for performing multi-word-token expansion\n\"\"\"\n\nimport io\n\nimport torch\n\nfrom stanza.models.mwt.data import DataLoader\nfrom stanza.models.mwt.trainer import Trainer\nfrom stanza.pipeline._constants import *\nfrom stanza.pipeline.processor import UDProcessor, register_processor\n\n@register_processor(MWT)\nclass MWTProcessor(UDProcessor):\n\n # set of processor requirements this processor fulfills\n PROVIDES_DEFAULT = set([MWT])\n # set of processor requirements for this processor\n REQUIRES_DEFAULT = set([TOKENIZE])\n\n def _set_up_model(self, config, pipeline, device):\n self._trainer = Trainer(model_file=config['model_path'], device=device)\n\n def build_batch(self, document):\n return DataLoader(document, self.config['batch_size'], self.config, vocab=self.vocab, evaluation=True, expand_unk_vocab=True)\n\n def process(self, document):\n batch = self.build_batch(document)\n\n # process the rest\n expansions = batch.doc.get_mwt_expansions(evaluation=True)\n if len(batch) > 0:\n # decide trainer type and run eval\n if self.config['dict_only']:\n preds = self.trainer.predict_dict(expansions)\n else:\n with torch.no_grad():\n preds = []\n for i, b in enumerate(batch.to_loader()):\n preds += self.trainer.predict(b, never_decode_unk=True, vocab=batch.vocab)\n\n if self.config.get('ensemble_dict', False):\n preds = self.trainer.ensemble(expansions, preds)\n else:\n # skip eval if dev data does not exist\n preds = []\n\n batch.doc.set_mwt_expansions(preds, process_manual_expanded=False)\n return batch.doc\n\n def bulk_process(self, docs):\n \"\"\"\n MWT processor counts some statistics on the individual docs, so we need to separately redo those stats\n \"\"\"\n docs = super().bulk_process(docs)\n for doc in docs:\n doc._count_words()\n return docs\n"
},
{
"path": "stanza/pipeline/ner_processor.py",
"content": "\"\"\"\nProcessor for performing named entity tagging.\n\"\"\"\n\nimport torch\n\nimport logging\n\nfrom stanza.models.common import doc\nfrom stanza.models.common.exceptions import ForwardCharlmNotFoundError, BackwardCharlmNotFoundError\nfrom stanza.models.common.utils import unsort\nfrom stanza.models.ner.data import DataLoader\nfrom stanza.models.ner.trainer import Trainer\nfrom stanza.models.ner.utils import merge_tags\nfrom stanza.pipeline._constants import *\nfrom stanza.pipeline.processor import UDProcessor, register_processor\n\nlogger = logging.getLogger('stanza')\n\n@register_processor(name=NER)\nclass NERProcessor(UDProcessor):\n\n # set of processor requirements this processor fulfills\n PROVIDES_DEFAULT = set([NER])\n # set of processor requirements for this processor\n REQUIRES_DEFAULT = set([TOKENIZE])\n\n def _get_dependencies(self, config, dep_name):\n dependencies = config.get(dep_name, None)\n if dependencies is not None:\n dependencies = dependencies.split(\";\")\n dependencies = [x if x else None for x in dependencies]\n else:\n dependencies = [x.get(dep_name) for x in config.get('dependencies', [])]\n return dependencies\n\n def _set_up_model(self, config, pipeline, device):\n # set up trainer\n model_paths = config.get('model_path')\n if isinstance(model_paths, str):\n model_paths = model_paths.split(\";\")\n\n charlm_forward_files = self._get_dependencies(config, 'forward_charlm_path')\n charlm_backward_files = self._get_dependencies(config, 'backward_charlm_path')\n pretrain_files = self._get_dependencies(config, 'pretrain_path')\n\n # allow predict_tagset to be specified as an int\n # (which only applies to the first model)\n # or as a string \";\" separated list of ints\n self._predict_tagset = {}\n predict_tagset = config.get('predict_tagset', None)\n if predict_tagset:\n if isinstance(predict_tagset, int):\n self._predict_tagset[0] = predict_tagset\n else:\n predict_tagset = predict_tagset.split(\";\")\n for piece_idx, piece in enumerate(predict_tagset):\n if piece:\n self._predict_tagset[piece_idx] = int(piece)\n\n self.trainers = []\n for (model_path, pretrain_path, charlm_forward, charlm_backward) in zip(model_paths, pretrain_files, charlm_forward_files, charlm_backward_files):\n logger.debug(\"Loading %s with pretrain %s, forward charlm %s, backward charlm %s\", model_path, pretrain_path, charlm_forward, charlm_backward)\n pretrain = pipeline.foundation_cache.load_pretrain(pretrain_path) if pretrain_path else None\n args = {'charlm_forward_file': charlm_forward,\n 'charlm_backward_file': charlm_backward}\n\n predict_tagset = self._predict_tagset.get(len(self.trainers), None)\n if predict_tagset is not None:\n args['predict_tagset'] = predict_tagset\n\n try:\n trainer = Trainer(args=args, model_file=model_path, pretrain=pretrain, device=device, foundation_cache=pipeline.foundation_cache)\n except ForwardCharlmNotFoundError as e:\n raise ForwardCharlmNotFoundError(\"Could not find the forward charlm %s. Please specify the correct path with ner_forward_charlm_path\" % e.filename, e.filename) from None\n except BackwardCharlmNotFoundError as e:\n raise BackwardCharlmNotFoundError(\"Could not find the backward charlm %s. Please specify the correct path with ner_backward_charlm_path\" % e.filename, e.filename) from None\n self.trainers.append(trainer)\n\n self._trainer = self.trainers[0]\n self.model_paths = model_paths\n\n def _set_up_final_config(self, config):\n \"\"\" Finalize the configurations for this processor, based off of values from a UD model. \"\"\"\n # set configurations from loaded model\n if len(self.trainers) == 0:\n raise RuntimeError(\"Somehow there are no models loaded!\")\n self._vocab = self.trainers[0].vocab\n self.configs = []\n for trainer in self.trainers:\n loaded_args = trainer.args\n # filter out unneeded args from model\n loaded_args = {k: v for k, v in loaded_args.items() if not UDProcessor.filter_out_option(k)}\n loaded_args.update(config)\n self.configs.append(loaded_args)\n self._config = self.configs[0]\n\n def __str__(self):\n return \"NERProcessor(%s)\" % \";\".join(self.model_paths)\n\n def mark_inactive(self):\n \"\"\" Drop memory intensive resources if keeping this processor around for reasons other than running it. \"\"\"\n super().mark_inactive()\n self.trainers = None\n\n def process(self, document):\n with torch.no_grad():\n all_preds = []\n for trainer, config in zip(self.trainers, self.configs):\n # set up a eval-only data loader and skip tag preprocessing\n batch = DataLoader(document, config['batch_size'], config, vocab=trainer.vocab, evaluation=True, preprocess_tags=False, bert_tokenizer=trainer.model.bert_tokenizer)\n preds = []\n for i, b in enumerate(batch):\n preds += trainer.predict(b)\n all_preds.append(preds)\n # for each sentence, gather a list of predictions\n # merge those predictions into a single list\n # earlier models will have precedence\n preds = [merge_tags(*x) for x in zip(*all_preds)]\n batch.doc.set([doc.NER], [y for x in preds for y in x], to_token=True)\n batch.doc.set([doc.MULTI_NER], [tuple(y) for x in zip(*all_preds) for y in zip(*x)], to_token=True)\n # collect entities into document attribute\n total = len(batch.doc.build_ents())\n logger.debug(f'{total} entities found in document.')\n return batch.doc\n\n def bulk_process(self, docs):\n \"\"\"\n NER processor has a collation step after running inference\n \"\"\"\n docs = super().bulk_process(docs)\n for doc in docs:\n doc.build_ents()\n return docs\n\n def get_known_tags(self, model_idx=0):\n \"\"\"\n Return the tags known by this model\n\n Removes the S-, B-, etc, and does not include O\n Specify model_idx if the processor has more than one model\n \"\"\" \n return self.trainers[model_idx].get_known_tags()\n"
},
{
"path": "stanza/pipeline/pos_processor.py",
"content": "\"\"\"\nProcessor for performing part-of-speech tagging\n\"\"\"\n\nimport torch\n\nfrom stanza.models.common import doc\nfrom stanza.models.common.utils import unsort\nfrom stanza.models.common.vocab import VOCAB_PREFIX, CompositeVocab\nfrom stanza.models.pos.data import Dataset\nfrom stanza.models.pos.trainer import Trainer\nfrom stanza.pipeline._constants import *\nfrom stanza.pipeline.processor import UDProcessor, register_processor\nfrom stanza.utils.get_tqdm import get_tqdm\n\ntqdm = get_tqdm()\n\n@register_processor(name=POS)\nclass POSProcessor(UDProcessor):\n\n # set of processor requirements this processor fulfills\n PROVIDES_DEFAULT = set([POS])\n # set of processor requirements for this processor\n REQUIRES_DEFAULT = set([TOKENIZE])\n\n def _set_up_model(self, config, pipeline, device):\n # get pretrained word vectors\n self._pretrain = pipeline.foundation_cache.load_pretrain(config['pretrain_path']) if 'pretrain_path' in config else None\n args = {'charlm_forward_file': config.get('forward_charlm_path', None),\n 'charlm_backward_file': config.get('backward_charlm_path', None)}\n # set up trainer\n self._trainer = Trainer(pretrain=self.pretrain, model_file=config['model_path'], device=device, args=args, foundation_cache=pipeline.foundation_cache)\n self._tqdm = 'tqdm' in config and config['tqdm']\n\n def __str__(self):\n return \"POSProcessor(%s)\" % self.config['model_path']\n\n def get_known_xpos(self):\n \"\"\"\n Returns the xpos tags known by this model\n \"\"\"\n if isinstance(self.vocab['xpos'], CompositeVocab):\n if len(self.vocab['xpos']) == 1:\n return [k for k in self.vocab['xpos'][0]._unit2id.keys() if k not in VOCAB_PREFIX]\n else:\n return {k: v.keys() - VOCAB_PREFIX for k, v in self.vocab['xpos']._unit2id.items()}\n return [k for k in self.vocab['xpos']._unit2id.keys() if k not in VOCAB_PREFIX]\n\n def is_composite_xpos(self):\n \"\"\"\n Returns if the xpos tags are part of a composite vocab\n \"\"\"\n return isinstance(self.vocab['xpos'], CompositeVocab)\n\n def get_known_upos(self):\n \"\"\"\n Returns the upos tags known by this model\n \"\"\"\n keys = [k for k in self.vocab['upos']._unit2id.keys() if k not in VOCAB_PREFIX]\n return keys\n\n def get_known_feats(self):\n \"\"\"\n Returns the features known by this model\n \"\"\"\n values = {k: v.keys() - VOCAB_PREFIX for k, v in self.vocab['feats']._unit2id.items()}\n return values\n\n def process(self, document):\n # currently, POS models are saved w/o the batch_maximum_tokens flag\n maximum_tokens = self.config.get('batch_maximum_tokens', 5000)\n\n dataset = Dataset(\n document, self.config, self.pretrain, vocab=self.vocab, evaluation=True,\n sort_during_eval=True)\n batch = iter(dataset.to_length_limited_loader(batch_size=self.config['batch_size'], maximum_tokens=maximum_tokens))\n preds = []\n\n idx = []\n with torch.no_grad():\n if self._tqdm:\n batch = tqdm(batch)\n for i, b in enumerate(batch):\n idx.extend(b[-1])\n preds += self.trainer.predict(b)\n\n preds = unsort(preds, idx)\n dataset.doc.set([doc.UPOS, doc.XPOS, doc.FEATS], [y for x in preds for y in x])\n return dataset.doc\n"
},
{
"path": "stanza/pipeline/processor.py",
"content": "\"\"\"\nBase classes for processors\n\"\"\"\n\nfrom abc import ABC, abstractmethod\n\nfrom stanza.models.common.doc import Document\nfrom stanza.pipeline.registry import NAME_TO_PROCESSOR_CLASS, PIPELINE_NAMES, PROCESSOR_VARIANTS\n\nclass ProcessorRequirementsException(Exception):\n \"\"\" Exception indicating a processor's requirements will not be met \"\"\"\n\n def __init__(self, processors_list, err_processor, provided_reqs):\n self._err_processor = err_processor\n # mark the broken processor as inactive, drop resources\n self.err_processor.mark_inactive()\n self._processors_list = processors_list\n self._provided_reqs = provided_reqs\n self.build_message()\n\n @property\n def err_processor(self):\n \"\"\" The processor that raised the exception \"\"\"\n return self._err_processor\n\n @property\n def processor_type(self):\n return type(self.err_processor).__name__\n\n @property\n def processors_list(self):\n return self._processors_list\n\n @property\n def provided_reqs(self):\n return self._provided_reqs\n\n def build_message(self):\n self.message = (f\"---\\nPipeline Requirements Error!\\n\"\n f\"\\tProcessor: {self.processor_type}\\n\"\n f\"\\tPipeline processors list: {','.join(self.processors_list)}\\n\"\n f\"\\tProcessor Requirements: {self.err_processor.requires}\\n\"\n f\"\\t\\t- fulfilled: {self.err_processor.requires.intersection(self.provided_reqs)}\\n\"\n f\"\\t\\t- missing: {self.err_processor.requires - self.provided_reqs}\\n\"\n f\"\\nThe processors list provided for this pipeline is invalid. Please make sure all \"\n f\"prerequisites are met for every processor.\\n\\n\")\n\n def __str__(self):\n return self.message\n\n\nclass Processor(ABC):\n \"\"\" Base class for all processors \"\"\"\n\n def __init__(self, config, pipeline, device):\n # overall config for the processor\n self._config = config\n # pipeline building this processor (presently processors are only meant to exist in one pipeline)\n self._pipeline = pipeline\n self._set_up_variants(config, device)\n # run set up process\n # set up what annotations are required based on config\n if not self._set_up_variant_requires():\n self._set_up_requires()\n # set up what annotations are provided based on config\n self._set_up_provides()\n # given pipeline constructing this processor, check if requirements are met, throw exception if not\n self._check_requirements()\n\n if hasattr(self, '_variant') and self._variant.OVERRIDE:\n self.process = self._variant.process\n\n def __str__(self):\n \"\"\"\n Simple description of the processor: name(model)\n \"\"\"\n name = self.__class__.__name__\n model = None\n if self._config is not None:\n model = self._config.get('model_path')\n if model is None:\n return name\n else:\n return \"{}({})\".format(name, model)\n\n\n @abstractmethod\n def process(self, doc):\n \"\"\" Process a Document. This is the main method of a processor. \"\"\"\n pass\n\n def bulk_process(self, docs):\n \"\"\" Process a list of Documents. This should be replaced with a more efficient implementation if possible. \"\"\"\n\n if hasattr(self, '_variant'):\n return self._variant.bulk_process(docs)\n\n return [self.process(doc) for doc in docs]\n\n def _set_up_provides(self):\n \"\"\" Set up what processor requirements this processor fulfills. Default is to use a class defined list. \"\"\"\n self._provides = self.__class__.PROVIDES_DEFAULT\n\n def _set_up_requires(self):\n \"\"\" Set up requirements for this processor. Default is to use a class defined list. \"\"\"\n self._requires = self.__class__.REQUIRES_DEFAULT\n\n def _set_up_variant_requires(self):\n \"\"\"\n If this has a variant with its own requirements, use those instead\n\n Returns True iff the _requires is set from the _variant\n \"\"\"\n if not hasattr(self, '_variant'):\n return False\n if hasattr(self._variant, '_set_up_requires'):\n self._variant._set_up_requires()\n self._requires = self._variant._requires\n return True\n if hasattr(self._variant.__class__, 'REQUIRES_DEFAULT'):\n self._requires = self._variant.__class__.REQUIRES_DEFAULT\n return True\n return False\n\n def _set_up_variants(self, config, device):\n processor_name = list(self.__class__.PROVIDES_DEFAULT)[0]\n if any(config.get(f'with_{variant}', False) for variant in PROCESSOR_VARIANTS[processor_name]):\n self._trainer = None\n variant_name = [variant for variant in PROCESSOR_VARIANTS[processor_name] if config.get(f'with_{variant}', False)][0]\n self._variant = PROCESSOR_VARIANTS[processor_name][variant_name](config)\n\n @property\n def config(self):\n \"\"\" Configurations for the processor \"\"\"\n return self._config\n\n @property\n def pipeline(self):\n \"\"\" The pipeline that this processor belongs to \"\"\"\n return self._pipeline\n\n @property\n def provides(self):\n return self._provides\n\n @property\n def requires(self):\n return self._requires\n\n def _check_requirements(self):\n \"\"\" Given a list of fulfilled requirements, check if all of this processor's requirements are met or not. \"\"\"\n if not self.config.get(\"check_requirements\", True):\n return\n provided_reqs = set.union(*[processor.provides for processor in self.pipeline.loaded_processors]+[set([])])\n if self.requires - provided_reqs:\n load_names = [item[0] for item in self.pipeline.load_list]\n raise ProcessorRequirementsException(load_names, self, provided_reqs)\n\n\nclass ProcessorVariant(ABC):\n \"\"\" Base class for all processor variants \"\"\"\n\n OVERRIDE = False # Set to true to override all the processing from the processor\n\n @abstractmethod\n def process(self, doc):\n \"\"\"\n Process a document that is potentially preprocessed by the processor.\n This is the main method of a processor variant.\n\n If `OVERRIDE` is set to True, all preprocessing by the processor would be bypassed, and the processor variant\n would serve as a drop-in replacement of the entire processor, and has to be able to interpret all the configs\n that are typically handled by the processor it replaces.\n \"\"\"\n pass\n\n def bulk_process(self, docs):\n \"\"\" Process a list of Documents. This should be replaced with a more efficient implementation if possible. \"\"\"\n\n return [self.process(doc) for doc in docs]\n\nclass UDProcessor(Processor):\n \"\"\" Base class for the neural UD Processors (tokenize,mwt,pos,lemma,depparse,sentiment,constituency) \"\"\"\n\n def __init__(self, config, pipeline, device):\n super().__init__(config, pipeline, device)\n\n # UD model resources, set up is processor specific\n self._pretrain = None\n self._trainer = None\n self._vocab = None\n if not hasattr(self, '_variant'):\n self._set_up_model(config, pipeline, device)\n\n # build the final config for the processor\n self._set_up_final_config(config)\n\n @abstractmethod\n def _set_up_model(self, config, pipeline, device):\n pass\n\n def _set_up_final_config(self, config):\n \"\"\" Finalize the configurations for this processor, based off of values from a UD model. \"\"\"\n # set configurations from loaded model\n if self._trainer is not None:\n loaded_args, self._vocab = self._trainer.args, self._trainer.vocab\n # filter out unneeded args from model\n loaded_args = {k: v for k, v in loaded_args.items() if not UDProcessor.filter_out_option(k)}\n else:\n loaded_args = {}\n loaded_args.update(config)\n self._config = loaded_args\n\n def mark_inactive(self):\n \"\"\" Drop memory intensive resources if keeping this processor around for reasons other than running it. \"\"\"\n self._trainer = None\n self._vocab = None\n\n @property\n def pretrain(self):\n return self._pretrain\n\n @property\n def trainer(self):\n return self._trainer\n\n @property\n def vocab(self):\n return self._vocab\n\n @staticmethod\n def filter_out_option(option):\n \"\"\" Filter out non-processor configurations \"\"\"\n options_to_filter = ['device', 'cpu', 'cuda', 'dev_conll_gold', 'epochs', 'lang', 'mode', 'save_name', 'shorthand']\n if option.endswith('_file') or option.endswith('_dir'):\n return True\n elif option in options_to_filter:\n return True\n else:\n return False\n\n def bulk_process(self, docs):\n \"\"\"\n Most processors operate on the sentence level, where each sentence is processed independently and processors can benefit\n a lot from the ability to combine sentences from multiple documents for faster batched processing. This is a transparent\n implementation that allows these processors to batch process a list of Documents as if they were from a single Document.\n \"\"\"\n\n if hasattr(self, '_variant'):\n return self._variant.bulk_process(docs)\n\n combined_sents = [sent for doc in docs for sent in doc.sentences]\n combined_doc = Document([])\n combined_doc.sentences = combined_sents\n combined_doc.num_tokens = sum(doc.num_tokens for doc in docs)\n combined_doc.num_words = sum(doc.num_words for doc in docs)\n\n self.process(combined_doc) # annotations are attached to sentence objects\n\n return docs\n\nclass ProcessorRegisterException(Exception):\n \"\"\" Exception indicating processor or processor registration failure \"\"\"\n\n def __init__(self, processor_class, expected_parent):\n self._processor_class = processor_class\n self._expected_parent = expected_parent\n self.build_message()\n\n def build_message(self):\n self.message = f\"Failed to register '{self._processor_class}'. It must be a subclass of '{self._expected_parent}'.\"\n\n def __str__(self):\n return self.message\n\ndef register_processor(name):\n def wrapper(Cls):\n if not issubclass(Cls, Processor):\n raise ProcessorRegisterException(Cls, Processor)\n\n NAME_TO_PROCESSOR_CLASS[name] = Cls\n PIPELINE_NAMES.append(name)\n return Cls\n return wrapper\n\ndef register_processor_variant(name, variant):\n def wrapper(Cls):\n if not issubclass(Cls, ProcessorVariant):\n raise ProcessorRegisterException(Cls, ProcessorVariant)\n\n PROCESSOR_VARIANTS[name][variant] = Cls\n return Cls\n return wrapper\n"
},
{
"path": "stanza/pipeline/registry.py",
"content": "from collections import defaultdict\n\n# these two get filled by register_processor\nNAME_TO_PROCESSOR_CLASS = dict()\nPIPELINE_NAMES = []\n\n# this gets filled by register_processor_variant\nPROCESSOR_VARIANTS = defaultdict(dict)\n"
},
{
"path": "stanza/pipeline/sentiment_processor.py",
"content": "\"\"\"Processor that attaches a sentiment score to a sentence\n\nThe model used is a generally a model trained on the Stanford\nSentiment Treebank or some similar dataset. When run, this processor\nattaches a score in the form of a string to each sentence in the\ndocument.\n\nTODO: a possible way to generalize this would be to make it a\nClassifierProcessor and have \"sentiment\" be an option.\n\"\"\"\n\nimport dataclasses\nimport torch\n\nfrom types import SimpleNamespace\n\nfrom stanza.models.classifiers.trainer import Trainer\n\nfrom stanza.pipeline._constants import *\nfrom stanza.pipeline.processor import UDProcessor, register_processor\n\n@register_processor(SENTIMENT)\nclass SentimentProcessor(UDProcessor):\n # set of processor requirements this processor fulfills\n PROVIDES_DEFAULT = set([SENTIMENT])\n # set of processor requirements for this processor\n # TODO: a constituency based model needs CONSTITUENCY as well\n # issue: by the time we load the model in Processor.__init__,\n # the requirements are already prepared\n REQUIRES_DEFAULT = set([TOKENIZE])\n\n # default batch size, measured in words per batch\n DEFAULT_BATCH_SIZE = 5000\n\n def _set_up_model(self, config, pipeline, device):\n # get pretrained word vectors\n pretrain_path = config.get('pretrain_path', None)\n forward_charlm_path = config.get('forward_charlm_path', None)\n backward_charlm_path = config.get('backward_charlm_path', None)\n # elmo does not have a convenient way to download intermediate\n # models the way stanza downloads charlms & pretrains or\n # transformers downloads bert etc\n # however, elmo in general is not as good as using a\n # transformer, so it is unlikely we will ever fix this\n args = SimpleNamespace(device = device,\n charlm_forward_file = forward_charlm_path,\n charlm_backward_file = backward_charlm_path,\n wordvec_pretrain_file = pretrain_path,\n elmo_model = None,\n use_elmo = False,\n save_dir = None)\n filename = config['model_path']\n if filename is None:\n raise FileNotFoundError(\"No model specified for the sentiment processor. Perhaps it is not supported for the language. {}\".format(config))\n # set up model\n trainer = Trainer.load(filename=filename,\n args=args,\n foundation_cache=pipeline.foundation_cache)\n self._trainer = trainer\n self._model = trainer.model\n self._model_type = self._model.config.model_type\n # batch size counted as words\n self._batch_size = config.get('batch_size', SentimentProcessor.DEFAULT_BATCH_SIZE)\n\n def _set_up_final_config(self, config):\n loaded_args = dataclasses.asdict(self._model.config)\n loaded_args = {k: v for k, v in loaded_args.items() if not UDProcessor.filter_out_option(k)}\n loaded_args.update(config)\n self._config = loaded_args\n\n\n def process(self, document):\n sentences = self._model.extract_sentences(document)\n with torch.no_grad():\n labels = self._model.label_sentences(sentences, batch_size=self._batch_size)\n # TODO: allow a classifier processor for any attribute, not just sentiment\n document.set(SENTIMENT, labels, to_sentence=True)\n return document\n"
},
{
"path": "stanza/pipeline/tokenize_processor.py",
"content": "\"\"\"\nProcessor for performing tokenization\n\"\"\"\n\nimport copy\nimport io\nimport logging\n\nimport torch\n\nfrom stanza.models.tokenization.data import TokenizationDataset\nfrom stanza.models.tokenization.trainer import Trainer\nfrom stanza.models.tokenization.utils import output_predictions\nfrom stanza.pipeline._constants import *\nfrom stanza.pipeline.processor import UDProcessor, register_processor\nfrom stanza.pipeline.registry import PROCESSOR_VARIANTS\nfrom stanza.models.common import doc\n\n# these imports trigger the \"register_variant\" decorations\nfrom stanza.pipeline.external.jieba import JiebaTokenizer\nfrom stanza.pipeline.external.spacy import SpacyTokenizer\nfrom stanza.pipeline.external.sudachipy import SudachiPyTokenizer\nfrom stanza.pipeline.external.pythainlp import PyThaiNLPTokenizer\n\nlogger = logging.getLogger('stanza')\n\nTOKEN_TOO_LONG_REPLACEMENT = \"
\"\n\n# class for running the tokenizer\n@register_processor(name=TOKENIZE)\nclass TokenizeProcessor(UDProcessor):\n\n # set of processor requirements this processor fulfills\n PROVIDES_DEFAULT = set([TOKENIZE])\n # set of processor requirements for this processor\n REQUIRES_DEFAULT = set([])\n # default max sequence length\n MAX_SEQ_LENGTH_DEFAULT = 1000\n\n def _set_up_model(self, config, pipeline, device):\n # set up trainer\n if config.get('pretokenized'):\n self._trainer = None\n else:\n args = {'charlm_forward_file': config.get('forward_charlm_path', None)}\n self._trainer = Trainer(args=args, model_file=config['model_path'], device=device, foundation_cache=pipeline.foundation_cache)\n\n # get and typecheck the postprocessor\n postprocessor = config.get('postprocessor')\n if postprocessor and callable(postprocessor):\n self._postprocessor = postprocessor\n elif not postprocessor:\n self._postprocessor = None\n else:\n raise ValueError(\"Tokenizer received 'postprocessor' option of unrecognized type; postprocessor must be callable. Got %s\" % postprocessor)\n\n def process_pre_tokenized_text(self, input_src):\n \"\"\"\n Pretokenized text can be provided in 2 manners:\n\n 1.) str, tokenized by whitespace, sentence split by newline\n 2.) list of token lists, each token list represents a sentence\n\n generate dictionary data structure\n \"\"\"\n\n document = []\n if isinstance(input_src, str):\n sentences = [sent.strip().split() for sent in input_src.strip().split('\\n') if len(sent.strip()) > 0]\n elif isinstance(input_src, list):\n sentences = input_src\n idx = 0\n for sentence in sentences:\n sent = []\n for token_id, token in enumerate(sentence):\n sent.append({doc.ID: (token_id + 1, ), doc.TEXT: token, doc.MISC: f'start_char={idx}|end_char={idx + len(token)}'})\n idx += len(token) + 1\n document.append(sent)\n raw_text = ' '.join([' '.join(sentence) for sentence in sentences])\n return raw_text, document\n\n def process(self, document):\n if not (isinstance(document, str) or isinstance(document, doc.Document) or (self.config.get('pretokenized') or self.config.get('no_ssplit', False))):\n raise ValueError(\"If neither 'pretokenized' or 'no_ssplit' option is enabled, the input to the TokenizerProcessor must be a string or a Document object. Got %s\" % str(type(document)))\n\n if isinstance(document, doc.Document):\n if self.config.get('pretokenized'):\n return document\n document = document.text\n\n if self.config.get('pretokenized'):\n raw_text, document = self.process_pre_tokenized_text(document)\n return doc.Document(document, raw_text)\n\n if hasattr(self, '_variant'):\n return self._variant.process(document)\n\n raw_text = '\\n\\n'.join(document) if isinstance(document, list) else document\n\n max_seq_len = self.config.get('max_seqlen', TokenizeProcessor.MAX_SEQ_LENGTH_DEFAULT)\n\n # set up batches\n batches = TokenizationDataset(self.config, input_text=raw_text, vocab=self.vocab, evaluation=True, dictionary=self.trainer.dictionary)\n # get dict data\n with torch.no_grad():\n _, _, _, document = output_predictions(None, self.trainer, batches, self.vocab, None,\n max_seq_len,\n orig_text=raw_text,\n no_ssplit=self.config.get('no_ssplit', False),\n num_workers = self.config.get('num_workers', 0),\n postprocessor = self._postprocessor)\n\n # replace excessively long tokens with to avoid downstream GPU memory issues in POS\n for sentence in document:\n for token in sentence:\n if len(token['text']) > max_seq_len:\n token['text'] = TOKEN_TOO_LONG_REPLACEMENT\n\n return doc.Document(document, raw_text)\n\n def bulk_process(self, docs):\n \"\"\"\n The tokenizer cannot use UDProcessor's sentence-level cross-document batching interface, and requires special handling.\n Essentially, this method concatenates the text of multiple documents with \"\\n\\n\", tokenizes it with the neural tokenizer,\n then splits the result into the original Documents and recovers the original character offsets.\n \"\"\"\n if hasattr(self, '_variant'):\n return self._variant.bulk_process(docs)\n\n if self.config.get('pretokenized'):\n res = []\n for document in docs:\n if len(document.sentences) > 0:\n # perhaps this is a document already tokenized,\n # being sent back in for more analysis / reparsing / etc?\n # in that case, no need to try to tokenize it\n # based on whitespace tokenizing the document text\n # which, interestingly, may not even exist depending on\n # how the document was created)\n # by making a whole deepcopy, the original Document is unchanged\n res.append(copy.deepcopy(document))\n else:\n raw_text, document = self.process_pre_tokenized_text(document.text)\n res.append(doc.Document(document, raw_text))\n return res\n\n combined_text = '\\n\\n'.join([thisdoc.text for thisdoc in docs])\n processed_combined = self.process(doc.Document([], text=combined_text))\n\n # postprocess sentences and tokens to reset back pointers and char offsets\n charoffset = 0\n sentst = senten = 0\n for thisdoc in docs:\n while senten < len(processed_combined.sentences) and processed_combined.sentences[senten].tokens[-1].end_char - charoffset <= len(thisdoc.text):\n senten += 1\n\n sentences = processed_combined.sentences[sentst:senten]\n thisdoc.sentences = sentences\n for sent in sentences:\n # fix doc back pointers for sentences\n sent._doc = thisdoc\n\n # fix char offsets for tokens and words\n for token in sent.tokens:\n token._start_char -= charoffset\n token._end_char -= charoffset\n if token.words: # not-yet-processed MWT can leave empty tokens\n for word in token.words:\n word._start_char -= charoffset\n word._end_char -= charoffset\n\n # Here we need to fix up the SpacesAfter for the very last token\n # and the SpacesBefore for the first token of the next doc\n # After all, we had connected the text with \\n\\n\n # Need to be careful about this - in a case such as\n # \" -text one- \"\n # \" -text two- \"\n # We want the SpacesBefore for the second document to reflect\n # the extra space at the start of its text\n # and the SpacesAfter for the first document to reflect\n # the whitespace after its text\n if len(sentences) > 0:\n last_token = sentences[-1].tokens[-1]\n last_whitespace = thisdoc.text[last_token.end_char:]\n last_token.spaces_after = last_whitespace\n\n first_token = sentences[0].tokens[0]\n first_whitespace = thisdoc.text[:first_token.start_char]\n first_token.spaces_before = first_whitespace\n\n thisdoc.num_tokens = sum(len(sent.tokens) for sent in sentences)\n thisdoc.num_words = sum(len(sent.words) for sent in sentences)\n sentst = senten\n\n charoffset += len(thisdoc.text) + 2\n\n return docs\n"
},
{
"path": "stanza/protobuf/CoreNLP_pb2.py",
"content": "# -*- coding: utf-8 -*-\n# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: CoreNLP.proto\n# Protobuf Python Version: 4.25.5\n\"\"\"Generated protocol buffer code.\"\"\"\nfrom google.protobuf import descriptor as _descriptor\nfrom google.protobuf import descriptor_pool as _descriptor_pool\nfrom google.protobuf import symbol_database as _symbol_database\nfrom google.protobuf.internal import builder as _builder\n# @@protoc_insertion_point(imports)\n\n_sym_db = _symbol_database.Default()\n\n\n\n\nDESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\\n\\rCoreNLP.proto\\x12\\x19\\x65\\x64u.stanford.nlp.pipeline\\\"\\xe1\\x05\\n\\x08\\x44ocument\\x12\\x0c\\n\\x04text\\x18\\x01 \\x02(\\t\\x12\\x35\\n\\x08sentence\\x18\\x02 \\x03(\\x0b\\x32#.edu.stanford.nlp.pipeline.Sentence\\x12\\x39\\n\\ncorefChain\\x18\\x03 \\x03(\\x0b\\x32%.edu.stanford.nlp.pipeline.CorefChain\\x12\\r\\n\\x05\\x64ocID\\x18\\x04 \\x01(\\t\\x12\\x0f\\n\\x07\\x64ocDate\\x18\\x07 \\x01(\\t\\x12\\x10\\n\\x08\\x63\\x61lendar\\x18\\x08 \\x01(\\x04\\x12;\\n\\x11sentencelessToken\\x18\\x05 \\x03(\\x0b\\x32 .edu.stanford.nlp.pipeline.Token\\x12\\x33\\n\\tcharacter\\x18\\n \\x03(\\x0b\\x32 .edu.stanford.nlp.pipeline.Token\\x12/\\n\\x05quote\\x18\\x06 \\x03(\\x0b\\x32 .edu.stanford.nlp.pipeline.Quote\\x12\\x37\\n\\x08mentions\\x18\\t \\x03(\\x0b\\x32%.edu.stanford.nlp.pipeline.NERMention\\x12#\\n\\x1bhasEntityMentionsAnnotation\\x18\\r \\x01(\\x08\\x12\\x0e\\n\\x06xmlDoc\\x18\\x0b \\x01(\\x08\\x12\\x34\\n\\x08sections\\x18\\x0c \\x03(\\x0b\\x32\\\".edu.stanford.nlp.pipeline.Section\\x12<\\n\\x10mentionsForCoref\\x18\\x0e \\x03(\\x0b\\x32\\\".edu.stanford.nlp.pipeline.Mention\\x12!\\n\\x19hasCorefMentionAnnotation\\x18\\x0f \\x01(\\x08\\x12\\x1a\\n\\x12hasCorefAnnotation\\x18\\x10 \\x01(\\x08\\x12+\\n#corefMentionToEntityMentionMappings\\x18\\x11 \\x03(\\x05\\x12+\\n#entityMentionToCorefMentionMappings\\x18\\x12 \\x03(\\x05*\\x05\\x08\\x64\\x10\\x80\\x02\\\"\\xf3\\x0f\\n\\x08Sentence\\x12/\\n\\x05token\\x18\\x01 \\x03(\\x0b\\x32 .edu.stanford.nlp.pipeline.Token\\x12\\x18\\n\\x10tokenOffsetBegin\\x18\\x02 \\x02(\\r\\x12\\x16\\n\\x0etokenOffsetEnd\\x18\\x03 \\x02(\\r\\x12\\x15\\n\\rsentenceIndex\\x18\\x04 \\x01(\\r\\x12\\x1c\\n\\x14\\x63haracterOffsetBegin\\x18\\x05 \\x01(\\r\\x12\\x1a\\n\\x12\\x63haracterOffsetEnd\\x18\\x06 \\x01(\\r\\x12\\x37\\n\\tparseTree\\x18\\x07 \\x01(\\x0b\\x32$.edu.stanford.nlp.pipeline.ParseTree\\x12@\\n\\x12\\x62inarizedParseTree\\x18\\x1f \\x01(\\x0b\\x32$.edu.stanford.nlp.pipeline.ParseTree\\x12@\\n\\x12\\x61nnotatedParseTree\\x18 \\x01(\\x0b\\x32$.edu.stanford.nlp.pipeline.ParseTree\\x12\\x11\\n\\tsentiment\\x18! \\x01(\\t\\x12=\\n\\x0fkBestParseTrees\\x18\\\" \\x03(\\x0b\\x32$.edu.stanford.nlp.pipeline.ParseTree\\x12\\x45\\n\\x11\\x62\\x61sicDependencies\\x18\\x08 \\x01(\\x0b\\x32*.edu.stanford.nlp.pipeline.DependencyGraph\\x12I\\n\\x15\\x63ollapsedDependencies\\x18\\t \\x01(\\x0b\\x32*.edu.stanford.nlp.pipeline.DependencyGraph\\x12T\\n collapsedCCProcessedDependencies\\x18\\n \\x01(\\x0b\\x32*.edu.stanford.nlp.pipeline.DependencyGraph\\x12K\\n\\x17\\x61lternativeDependencies\\x18\\r \\x01(\\x0b\\x32*.edu.stanford.nlp.pipeline.DependencyGraph\\x12?\\n\\x0copenieTriple\\x18\\x0e \\x03(\\x0b\\x32).edu.stanford.nlp.pipeline.RelationTriple\\x12<\\n\\tkbpTriple\\x18\\x10 \\x03(\\x0b\\x32).edu.stanford.nlp.pipeline.RelationTriple\\x12\\x45\\n\\x10\\x65ntailedSentence\\x18\\x0f \\x03(\\x0b\\x32+.edu.stanford.nlp.pipeline.SentenceFragment\\x12\\x43\\n\\x0e\\x65ntailedClause\\x18# \\x03(\\x0b\\x32+.edu.stanford.nlp.pipeline.SentenceFragment\\x12H\\n\\x14\\x65nhancedDependencies\\x18\\x11 \\x01(\\x0b\\x32*.edu.stanford.nlp.pipeline.DependencyGraph\\x12P\\n\\x1c\\x65nhancedPlusPlusDependencies\\x18\\x12 \\x01(\\x0b\\x32*.edu.stanford.nlp.pipeline.DependencyGraph\\x12\\x33\\n\\tcharacter\\x18\\x13 \\x03(\\x0b\\x32 .edu.stanford.nlp.pipeline.Token\\x12\\x11\\n\\tparagraph\\x18\\x0b \\x01(\\r\\x12\\x0c\\n\\x04text\\x18\\x0c \\x01(\\t\\x12\\x12\\n\\nlineNumber\\x18\\x14 \\x01(\\r\\x12\\x1e\\n\\x16hasRelationAnnotations\\x18\\x33 \\x01(\\x08\\x12\\x31\\n\\x06\\x65ntity\\x18\\x34 \\x03(\\x0b\\x32!.edu.stanford.nlp.pipeline.Entity\\x12\\x35\\n\\x08relation\\x18\\x35 \\x03(\\x0b\\x32#.edu.stanford.nlp.pipeline.Relation\\x12$\\n\\x1chasNumerizedTokensAnnotation\\x18\\x36 \\x01(\\x08\\x12\\x37\\n\\x08mentions\\x18\\x37 \\x03(\\x0b\\x32%.edu.stanford.nlp.pipeline.NERMention\\x12<\\n\\x10mentionsForCoref\\x18\\x38 \\x03(\\x0b\\x32\\\".edu.stanford.nlp.pipeline.Mention\\x12\\\"\\n\\x1ahasCorefMentionsAnnotation\\x18\\x39 \\x01(\\x08\\x12\\x12\\n\\nsentenceID\\x18: \\x01(\\t\\x12\\x13\\n\\x0bsectionDate\\x18; \\x01(\\t\\x12\\x14\\n\\x0csectionIndex\\x18< \\x01(\\r\\x12\\x13\\n\\x0bsectionName\\x18= \\x01(\\t\\x12\\x15\\n\\rsectionAuthor\\x18> \\x01(\\t\\x12\\r\\n\\x05\\x64ocID\\x18? \\x01(\\t\\x12\\x15\\n\\rsectionQuoted\\x18@ \\x01(\\x08\\x12#\\n\\x1bhasEntityMentionsAnnotation\\x18\\x41 \\x01(\\x08\\x12\\x1f\\n\\x17hasKBPTriplesAnnotation\\x18\\x44 \\x01(\\x08\\x12\\\"\\n\\x1ahasOpenieTriplesAnnotation\\x18\\x45 \\x01(\\x08\\x12\\x14\\n\\x0c\\x63hapterIndex\\x18\\x42 \\x01(\\r\\x12\\x16\\n\\x0eparagraphIndex\\x18\\x43 \\x01(\\r\\x12=\\n\\x10\\x65nhancedSentence\\x18\\x46 \\x01(\\x0b\\x32#.edu.stanford.nlp.pipeline.Sentence\\x12\\x0f\\n\\x07speaker\\x18G \\x01(\\t\\x12\\x13\\n\\x0bspeakerType\\x18H \\x01(\\t*\\x05\\x08\\x64\\x10\\x80\\x02\\\"\\xf6\\x0c\\n\\x05Token\\x12\\x0c\\n\\x04word\\x18\\x01 \\x01(\\t\\x12\\x0b\\n\\x03pos\\x18\\x02 \\x01(\\t\\x12\\r\\n\\x05value\\x18\\x03 \\x01(\\t\\x12\\x10\\n\\x08\\x63\\x61tegory\\x18\\x04 \\x01(\\t\\x12\\x0e\\n\\x06\\x62\\x65\\x66ore\\x18\\x05 \\x01(\\t\\x12\\r\\n\\x05\\x61\\x66ter\\x18\\x06 \\x01(\\t\\x12\\x14\\n\\x0coriginalText\\x18\\x07 \\x01(\\t\\x12\\x0b\\n\\x03ner\\x18\\x08 \\x01(\\t\\x12\\x11\\n\\tcoarseNER\\x18> \\x01(\\t\\x12\\x16\\n\\x0e\\x66ineGrainedNER\\x18? \\x01(\\t\\x12\\x15\\n\\rnerLabelProbs\\x18\\x42 \\x03(\\t\\x12\\x15\\n\\rnormalizedNER\\x18\\t \\x01(\\t\\x12\\r\\n\\x05lemma\\x18\\n \\x01(\\t\\x12\\x11\\n\\tbeginChar\\x18\\x0b \\x01(\\r\\x12\\x0f\\n\\x07\\x65ndChar\\x18\\x0c \\x01(\\r\\x12\\x11\\n\\tutterance\\x18\\r \\x01(\\r\\x12\\x0f\\n\\x07speaker\\x18\\x0e \\x01(\\t\\x12\\x13\\n\\x0bspeakerType\\x18M \\x01(\\t\\x12\\x12\\n\\nbeginIndex\\x18\\x0f \\x01(\\r\\x12\\x10\\n\\x08\\x65ndIndex\\x18\\x10 \\x01(\\r\\x12\\x17\\n\\x0ftokenBeginIndex\\x18\\x11 \\x01(\\r\\x12\\x15\\n\\rtokenEndIndex\\x18\\x12 \\x01(\\r\\x12\\x34\\n\\ntimexValue\\x18\\x13 \\x01(\\x0b\\x32 .edu.stanford.nlp.pipeline.Timex\\x12\\x15\\n\\rhasXmlContext\\x18\\x15 \\x01(\\x08\\x12\\x12\\n\\nxmlContext\\x18\\x16 \\x03(\\t\\x12\\x16\\n\\x0e\\x63orefClusterID\\x18\\x17 \\x01(\\r\\x12\\x0e\\n\\x06\\x61nswer\\x18\\x18 \\x01(\\t\\x12\\x15\\n\\rheadWordIndex\\x18\\x1a \\x01(\\r\\x12\\x35\\n\\x08operator\\x18\\x1b \\x01(\\x0b\\x32#.edu.stanford.nlp.pipeline.Operator\\x12\\x35\\n\\x08polarity\\x18\\x1c \\x01(\\x0b\\x32#.edu.stanford.nlp.pipeline.Polarity\\x12\\x14\\n\\x0cpolarity_dir\\x18\\' \\x01(\\t\\x12-\\n\\x04span\\x18\\x1d \\x01(\\x0b\\x32\\x1f.edu.stanford.nlp.pipeline.Span\\x12\\x11\\n\\tsentiment\\x18\\x1e \\x01(\\t\\x12\\x16\\n\\x0equotationIndex\\x18\\x1f \\x01(\\x05\\x12\\x42\\n\\x0e\\x63onllUFeatures\\x18 \\x01(\\x0b\\x32*.edu.stanford.nlp.pipeline.MapStringString\\x12\\x11\\n\\tcoarseTag\\x18! \\x01(\\t\\x12\\x38\\n\\x0f\\x63onllUTokenSpan\\x18\\\" \\x01(\\x0b\\x32\\x1f.edu.stanford.nlp.pipeline.Span\\x12\\x12\\n\\nconllUMisc\\x18# \\x01(\\t\\x12G\\n\\x13\\x63onllUSecondaryDeps\\x18$ \\x01(\\x0b\\x32*.edu.stanford.nlp.pipeline.MapStringString\\x12\\x17\\n\\x0fwikipediaEntity\\x18% \\x01(\\t\\x12\\x11\\n\\tisNewline\\x18& \\x01(\\x08\\x12\\x0e\\n\\x06gender\\x18\\x33 \\x01(\\t\\x12\\x10\\n\\x08trueCase\\x18\\x34 \\x01(\\t\\x12\\x14\\n\\x0ctrueCaseText\\x18\\x35 \\x01(\\t\\x12\\x13\\n\\x0b\\x63hineseChar\\x18\\x36 \\x01(\\t\\x12\\x12\\n\\nchineseSeg\\x18\\x37 \\x01(\\t\\x12\\x16\\n\\x0e\\x63hineseXMLChar\\x18< \\x01(\\t\\x12\\x11\\n\\tarabicSeg\\x18L \\x01(\\t\\x12\\x13\\n\\x0bsectionName\\x18\\x38 \\x01(\\t\\x12\\x15\\n\\rsectionAuthor\\x18\\x39 \\x01(\\t\\x12\\x13\\n\\x0bsectionDate\\x18: \\x01(\\t\\x12\\x17\\n\\x0fsectionEndLabel\\x18; \\x01(\\t\\x12\\x0e\\n\\x06parent\\x18= \\x01(\\t\\x12\\x19\\n\\x11\\x63orefMentionIndex\\x18@ \\x03(\\r\\x12\\x1a\\n\\x12\\x65ntityMentionIndex\\x18\\x41 \\x01(\\r\\x12\\r\\n\\x05isMWT\\x18\\x43 \\x01(\\x08\\x12\\x12\\n\\nisFirstMWT\\x18\\x44 \\x01(\\x08\\x12\\x0f\\n\\x07mwtText\\x18\\x45 \\x01(\\t\\x12\\x0f\\n\\x07mwtMisc\\x18N \\x01(\\t\\x12\\x14\\n\\x0cnumericValue\\x18\\x46 \\x01(\\x04\\x12\\x13\\n\\x0bnumericType\\x18G \\x01(\\t\\x12\\x1d\\n\\x15numericCompositeValue\\x18H \\x01(\\x04\\x12\\x1c\\n\\x14numericCompositeType\\x18I \\x01(\\t\\x12\\x1c\\n\\x14\\x63odepointOffsetBegin\\x18J \\x01(\\r\\x12\\x1a\\n\\x12\\x63odepointOffsetEnd\\x18K \\x01(\\r\\x12\\r\\n\\x05index\\x18O \\x01(\\r\\x12\\x12\\n\\nemptyIndex\\x18P \\x01(\\r*\\x05\\x08\\x64\\x10\\x80\\x02\\\"\\xe4\\x03\\n\\x05Quote\\x12\\x0c\\n\\x04text\\x18\\x01 \\x01(\\t\\x12\\r\\n\\x05\\x62\\x65gin\\x18\\x02 \\x01(\\r\\x12\\x0b\\n\\x03\\x65nd\\x18\\x03 \\x01(\\r\\x12\\x15\\n\\rsentenceBegin\\x18\\x05 \\x01(\\r\\x12\\x13\\n\\x0bsentenceEnd\\x18\\x06 \\x01(\\r\\x12\\x12\\n\\ntokenBegin\\x18\\x07 \\x01(\\r\\x12\\x10\\n\\x08tokenEnd\\x18\\x08 \\x01(\\r\\x12\\r\\n\\x05\\x64ocid\\x18\\t \\x01(\\t\\x12\\r\\n\\x05index\\x18\\n \\x01(\\r\\x12\\x0e\\n\\x06\\x61uthor\\x18\\x0b \\x01(\\t\\x12\\x0f\\n\\x07mention\\x18\\x0c \\x01(\\t\\x12\\x14\\n\\x0cmentionBegin\\x18\\r \\x01(\\r\\x12\\x12\\n\\nmentionEnd\\x18\\x0e \\x01(\\r\\x12\\x13\\n\\x0bmentionType\\x18\\x0f \\x01(\\t\\x12\\x14\\n\\x0cmentionSieve\\x18\\x10 \\x01(\\t\\x12\\x0f\\n\\x07speaker\\x18\\x11 \\x01(\\t\\x12\\x14\\n\\x0cspeakerSieve\\x18\\x12 \\x01(\\t\\x12\\x18\\n\\x10\\x63\\x61nonicalMention\\x18\\x13 \\x01(\\t\\x12\\x1d\\n\\x15\\x63\\x61nonicalMentionBegin\\x18\\x14 \\x01(\\r\\x12\\x1b\\n\\x13\\x63\\x61nonicalMentionEnd\\x18\\x15 \\x01(\\r\\x12N\\n\\x1a\\x61ttributionDependencyGraph\\x18\\x16 \\x01(\\x0b\\x32*.edu.stanford.nlp.pipeline.DependencyGraph\\\"\\xc7\\x01\\n\\tParseTree\\x12\\x33\\n\\x05\\x63hild\\x18\\x01 \\x03(\\x0b\\x32$.edu.stanford.nlp.pipeline.ParseTree\\x12\\r\\n\\x05value\\x18\\x02 \\x01(\\t\\x12\\x17\\n\\x0fyieldBeginIndex\\x18\\x03 \\x01(\\r\\x12\\x15\\n\\ryieldEndIndex\\x18\\x04 \\x01(\\r\\x12\\r\\n\\x05score\\x18\\x05 \\x01(\\x01\\x12\\x37\\n\\tsentiment\\x18\\x06 \\x01(\\x0e\\x32$.edu.stanford.nlp.pipeline.Sentiment\\\"\\x9b\\x04\\n\\x0f\\x44\\x65pendencyGraph\\x12=\\n\\x04node\\x18\\x01 \\x03(\\x0b\\x32/.edu.stanford.nlp.pipeline.DependencyGraph.Node\\x12=\\n\\x04\\x65\\x64ge\\x18\\x02 \\x03(\\x0b\\x32/.edu.stanford.nlp.pipeline.DependencyGraph.Edge\\x12\\x10\\n\\x04root\\x18\\x03 \\x03(\\rB\\x02\\x10\\x01\\x12/\\n\\x05token\\x18\\x04 \\x03(\\x0b\\x32 .edu.stanford.nlp.pipeline.Token\\x12\\x14\\n\\x08rootNode\\x18\\x05 \\x03(\\rB\\x02\\x10\\x01\\x1aX\\n\\x04Node\\x12\\x15\\n\\rsentenceIndex\\x18\\x01 \\x02(\\r\\x12\\r\\n\\x05index\\x18\\x02 \\x02(\\r\\x12\\x16\\n\\x0e\\x63opyAnnotation\\x18\\x03 \\x01(\\r\\x12\\x12\\n\\nemptyIndex\\x18\\x04 \\x01(\\r\\x1a\\xd6\\x01\\n\\x04\\x45\\x64ge\\x12\\x0e\\n\\x06source\\x18\\x01 \\x02(\\r\\x12\\x0e\\n\\x06target\\x18\\x02 \\x02(\\r\\x12\\x0b\\n\\x03\\x64\\x65p\\x18\\x03 \\x01(\\t\\x12\\x0f\\n\\x07isExtra\\x18\\x04 \\x01(\\x08\\x12\\x12\\n\\nsourceCopy\\x18\\x05 \\x01(\\r\\x12\\x12\\n\\ntargetCopy\\x18\\x06 \\x01(\\r\\x12\\x13\\n\\x0bsourceEmpty\\x18\\x08 \\x01(\\r\\x12\\x13\\n\\x0btargetEmpty\\x18\\t \\x01(\\r\\x12>\\n\\x08language\\x18\\x07 \\x01(\\x0e\\x32#.edu.stanford.nlp.pipeline.Language:\\x07Unknown\\\"\\xc6\\x02\\n\\nCorefChain\\x12\\x0f\\n\\x07\\x63hainID\\x18\\x01 \\x02(\\x05\\x12\\x43\\n\\x07mention\\x18\\x02 \\x03(\\x0b\\x32\\x32.edu.stanford.nlp.pipeline.CorefChain.CorefMention\\x12\\x16\\n\\x0erepresentative\\x18\\x03 \\x02(\\r\\x1a\\xc9\\x01\\n\\x0c\\x43orefMention\\x12\\x11\\n\\tmentionID\\x18\\x01 \\x01(\\x05\\x12\\x13\\n\\x0bmentionType\\x18\\x02 \\x01(\\t\\x12\\x0e\\n\\x06number\\x18\\x03 \\x01(\\t\\x12\\x0e\\n\\x06gender\\x18\\x04 \\x01(\\t\\x12\\x0f\\n\\x07\\x61nimacy\\x18\\x05 \\x01(\\t\\x12\\x12\\n\\nbeginIndex\\x18\\x06 \\x01(\\r\\x12\\x10\\n\\x08\\x65ndIndex\\x18\\x07 \\x01(\\r\\x12\\x11\\n\\theadIndex\\x18\\t \\x01(\\r\\x12\\x15\\n\\rsentenceIndex\\x18\\n \\x01(\\r\\x12\\x10\\n\\x08position\\x18\\x0b \\x01(\\r\\\"\\xef\\x08\\n\\x07Mention\\x12\\x11\\n\\tmentionID\\x18\\x01 \\x01(\\x05\\x12\\x13\\n\\x0bmentionType\\x18\\x02 \\x01(\\t\\x12\\x0e\\n\\x06number\\x18\\x03 \\x01(\\t\\x12\\x0e\\n\\x06gender\\x18\\x04 \\x01(\\t\\x12\\x0f\\n\\x07\\x61nimacy\\x18\\x05 \\x01(\\t\\x12\\x0e\\n\\x06person\\x18\\x06 \\x01(\\t\\x12\\x12\\n\\nstartIndex\\x18\\x07 \\x01(\\r\\x12\\x10\\n\\x08\\x65ndIndex\\x18\\t \\x01(\\r\\x12\\x11\\n\\theadIndex\\x18\\n \\x01(\\x05\\x12\\x12\\n\\nheadString\\x18\\x0b \\x01(\\t\\x12\\x11\\n\\tnerString\\x18\\x0c \\x01(\\t\\x12\\x13\\n\\x0boriginalRef\\x18\\r \\x01(\\x05\\x12\\x1a\\n\\x12goldCorefClusterID\\x18\\x0e \\x01(\\x05\\x12\\x16\\n\\x0e\\x63orefClusterID\\x18\\x0f \\x01(\\x05\\x12\\x12\\n\\nmentionNum\\x18\\x10 \\x01(\\x05\\x12\\x0f\\n\\x07sentNum\\x18\\x11 \\x01(\\x05\\x12\\r\\n\\x05utter\\x18\\x12 \\x01(\\x05\\x12\\x11\\n\\tparagraph\\x18\\x13 \\x01(\\x05\\x12\\x11\\n\\tisSubject\\x18\\x14 \\x01(\\x08\\x12\\x16\\n\\x0eisDirectObject\\x18\\x15 \\x01(\\x08\\x12\\x18\\n\\x10isIndirectObject\\x18\\x16 \\x01(\\x08\\x12\\x1b\\n\\x13isPrepositionObject\\x18\\x17 \\x01(\\x08\\x12\\x0f\\n\\x07hasTwin\\x18\\x18 \\x01(\\x08\\x12\\x0f\\n\\x07generic\\x18\\x19 \\x01(\\x08\\x12\\x13\\n\\x0bisSingleton\\x18\\x1a \\x01(\\x08\\x12\\x1a\\n\\x12hasBasicDependency\\x18\\x1b \\x01(\\x08\\x12\\x1d\\n\\x15hasEnhancedDependency\\x18\\x1c \\x01(\\x08\\x12\\x1b\\n\\x13hasContextParseTree\\x18\\x1d \\x01(\\x08\\x12?\\n\\x0fheadIndexedWord\\x18\\x1e \\x01(\\x0b\\x32&.edu.stanford.nlp.pipeline.IndexedWord\\x12=\\n\\rdependingVerb\\x18\\x1f \\x01(\\x0b\\x32&.edu.stanford.nlp.pipeline.IndexedWord\\x12\\x38\\n\\x08headWord\\x18 \\x01(\\x0b\\x32&.edu.stanford.nlp.pipeline.IndexedWord\\x12;\\n\\x0bspeakerInfo\\x18! \\x01(\\x0b\\x32&.edu.stanford.nlp.pipeline.SpeakerInfo\\x12=\\n\\rsentenceWords\\x18\\x32 \\x03(\\x0b\\x32&.edu.stanford.nlp.pipeline.IndexedWord\\x12<\\n\\x0coriginalSpan\\x18\\x33 \\x03(\\x0b\\x32&.edu.stanford.nlp.pipeline.IndexedWord\\x12\\x12\\n\\ndependents\\x18\\x34 \\x03(\\t\\x12\\x19\\n\\x11preprocessedTerms\\x18\\x35 \\x03(\\t\\x12\\x13\\n\\x0b\\x61ppositions\\x18\\x36 \\x03(\\x05\\x12\\x1c\\n\\x14predicateNominatives\\x18\\x37 \\x03(\\x05\\x12\\x18\\n\\x10relativePronouns\\x18\\x38 \\x03(\\x05\\x12\\x13\\n\\x0blistMembers\\x18\\x39 \\x03(\\x05\\x12\\x15\\n\\rbelongToLists\\x18: \\x03(\\x05\\\"X\\n\\x0bIndexedWord\\x12\\x13\\n\\x0bsentenceNum\\x18\\x01 \\x01(\\x05\\x12\\x12\\n\\ntokenIndex\\x18\\x02 \\x01(\\x05\\x12\\r\\n\\x05\\x64ocID\\x18\\x03 \\x01(\\x05\\x12\\x11\\n\\tcopyCount\\x18\\x04 \\x01(\\r\\\"4\\n\\x0bSpeakerInfo\\x12\\x13\\n\\x0bspeakerName\\x18\\x01 \\x01(\\t\\x12\\x10\\n\\x08mentions\\x18\\x02 \\x03(\\x05\\\"\\\"\\n\\x04Span\\x12\\r\\n\\x05\\x62\\x65gin\\x18\\x01 \\x02(\\r\\x12\\x0b\\n\\x03\\x65nd\\x18\\x02 \\x02(\\r\\\"w\\n\\x05Timex\\x12\\r\\n\\x05value\\x18\\x01 \\x01(\\t\\x12\\x10\\n\\x08\\x61ltValue\\x18\\x02 \\x01(\\t\\x12\\x0c\\n\\x04text\\x18\\x03 \\x01(\\t\\x12\\x0c\\n\\x04type\\x18\\x04 \\x01(\\t\\x12\\x0b\\n\\x03tid\\x18\\x05 \\x01(\\t\\x12\\x12\\n\\nbeginPoint\\x18\\x06 \\x01(\\r\\x12\\x10\\n\\x08\\x65ndPoint\\x18\\x07 \\x01(\\r\\\"\\xdb\\x01\\n\\x06\\x45ntity\\x12\\x11\\n\\theadStart\\x18\\x06 \\x01(\\r\\x12\\x0f\\n\\x07headEnd\\x18\\x07 \\x01(\\r\\x12\\x13\\n\\x0bmentionType\\x18\\x08 \\x01(\\t\\x12\\x16\\n\\x0enormalizedName\\x18\\t \\x01(\\t\\x12\\x16\\n\\x0eheadTokenIndex\\x18\\n \\x01(\\r\\x12\\x0f\\n\\x07\\x63orefID\\x18\\x0b \\x01(\\t\\x12\\x10\\n\\x08objectID\\x18\\x01 \\x01(\\t\\x12\\x13\\n\\x0b\\x65xtentStart\\x18\\x02 \\x01(\\r\\x12\\x11\\n\\textentEnd\\x18\\x03 \\x01(\\r\\x12\\x0c\\n\\x04type\\x18\\x04 \\x01(\\t\\x12\\x0f\\n\\x07subtype\\x18\\x05 \\x01(\\t\\\"\\xb7\\x01\\n\\x08Relation\\x12\\x0f\\n\\x07\\x61rgName\\x18\\x06 \\x03(\\t\\x12.\\n\\x03\\x61rg\\x18\\x07 \\x03(\\x0b\\x32!.edu.stanford.nlp.pipeline.Entity\\x12\\x11\\n\\tsignature\\x18\\x08 \\x01(\\t\\x12\\x10\\n\\x08objectID\\x18\\x01 \\x01(\\t\\x12\\x13\\n\\x0b\\x65xtentStart\\x18\\x02 \\x01(\\r\\x12\\x11\\n\\textentEnd\\x18\\x03 \\x01(\\r\\x12\\x0c\\n\\x04type\\x18\\x04 \\x01(\\t\\x12\\x0f\\n\\x07subtype\\x18\\x05 \\x01(\\t\\\"\\xb2\\x01\\n\\x08Operator\\x12\\x0c\\n\\x04name\\x18\\x01 \\x02(\\t\\x12\\x1b\\n\\x13quantifierSpanBegin\\x18\\x02 \\x02(\\x05\\x12\\x19\\n\\x11quantifierSpanEnd\\x18\\x03 \\x02(\\x05\\x12\\x18\\n\\x10subjectSpanBegin\\x18\\x04 \\x02(\\x05\\x12\\x16\\n\\x0esubjectSpanEnd\\x18\\x05 \\x02(\\x05\\x12\\x17\\n\\x0fobjectSpanBegin\\x18\\x06 \\x02(\\x05\\x12\\x15\\n\\robjectSpanEnd\\x18\\x07 \\x02(\\x05\\\"\\xa9\\x04\\n\\x08Polarity\\x12K\\n\\x12projectEquivalence\\x18\\x01 \\x02(\\x0e\\x32/.edu.stanford.nlp.pipeline.NaturalLogicRelation\\x12Q\\n\\x18projectForwardEntailment\\x18\\x02 \\x02(\\x0e\\x32/.edu.stanford.nlp.pipeline.NaturalLogicRelation\\x12Q\\n\\x18projectReverseEntailment\\x18\\x03 \\x02(\\x0e\\x32/.edu.stanford.nlp.pipeline.NaturalLogicRelation\\x12H\\n\\x0fprojectNegation\\x18\\x04 \\x02(\\x0e\\x32/.edu.stanford.nlp.pipeline.NaturalLogicRelation\\x12K\\n\\x12projectAlternation\\x18\\x05 \\x02(\\x0e\\x32/.edu.stanford.nlp.pipeline.NaturalLogicRelation\\x12\\x45\\n\\x0cprojectCover\\x18\\x06 \\x02(\\x0e\\x32/.edu.stanford.nlp.pipeline.NaturalLogicRelation\\x12L\\n\\x13projectIndependence\\x18\\x07 \\x02(\\x0e\\x32/.edu.stanford.nlp.pipeline.NaturalLogicRelation\\\"\\xdd\\x02\\n\\nNERMention\\x12\\x15\\n\\rsentenceIndex\\x18\\x01 \\x01(\\r\\x12%\\n\\x1dtokenStartInSentenceInclusive\\x18\\x02 \\x02(\\r\\x12#\\n\\x1btokenEndInSentenceExclusive\\x18\\x03 \\x02(\\r\\x12\\x0b\\n\\x03ner\\x18\\x04 \\x02(\\t\\x12\\x15\\n\\rnormalizedNER\\x18\\x05 \\x01(\\t\\x12\\x12\\n\\nentityType\\x18\\x06 \\x01(\\t\\x12/\\n\\x05timex\\x18\\x07 \\x01(\\x0b\\x32 .edu.stanford.nlp.pipeline.Timex\\x12\\x17\\n\\x0fwikipediaEntity\\x18\\x08 \\x01(\\t\\x12\\x0e\\n\\x06gender\\x18\\t \\x01(\\t\\x12\\x1a\\n\\x12\\x65ntityMentionIndex\\x18\\n \\x01(\\r\\x12#\\n\\x1b\\x63\\x61nonicalEntityMentionIndex\\x18\\x0b \\x01(\\r\\x12\\x19\\n\\x11\\x65ntityMentionText\\x18\\x0c \\x01(\\t\\\"Y\\n\\x10SentenceFragment\\x12\\x12\\n\\ntokenIndex\\x18\\x01 \\x03(\\r\\x12\\x0c\\n\\x04root\\x18\\x02 \\x01(\\r\\x12\\x14\\n\\x0c\\x61ssumedTruth\\x18\\x03 \\x01(\\x08\\x12\\r\\n\\x05score\\x18\\x04 \\x01(\\x01\\\":\\n\\rTokenLocation\\x12\\x15\\n\\rsentenceIndex\\x18\\x01 \\x01(\\r\\x12\\x12\\n\\ntokenIndex\\x18\\x02 \\x01(\\r\\\"\\x9a\\x03\\n\\x0eRelationTriple\\x12\\x0f\\n\\x07subject\\x18\\x01 \\x01(\\t\\x12\\x10\\n\\x08relation\\x18\\x02 \\x01(\\t\\x12\\x0e\\n\\x06object\\x18\\x03 \\x01(\\t\\x12\\x12\\n\\nconfidence\\x18\\x04 \\x01(\\x01\\x12?\\n\\rsubjectTokens\\x18\\r \\x03(\\x0b\\x32(.edu.stanford.nlp.pipeline.TokenLocation\\x12@\\n\\x0erelationTokens\\x18\\x0e \\x03(\\x0b\\x32(.edu.stanford.nlp.pipeline.TokenLocation\\x12>\\n\\x0cobjectTokens\\x18\\x0f \\x03(\\x0b\\x32(.edu.stanford.nlp.pipeline.TokenLocation\\x12\\x38\\n\\x04tree\\x18\\x08 \\x01(\\x0b\\x32*.edu.stanford.nlp.pipeline.DependencyGraph\\x12\\x0e\\n\\x06istmod\\x18\\t \\x01(\\x08\\x12\\x10\\n\\x08prefixBe\\x18\\n \\x01(\\x08\\x12\\x10\\n\\x08suffixBe\\x18\\x0b \\x01(\\x08\\x12\\x10\\n\\x08suffixOf\\x18\\x0c \\x01(\\x08\\\"-\\n\\x0fMapStringString\\x12\\x0b\\n\\x03key\\x18\\x01 \\x03(\\t\\x12\\r\\n\\x05value\\x18\\x02 \\x03(\\t\\\"*\\n\\x0cMapIntString\\x12\\x0b\\n\\x03key\\x18\\x01 \\x03(\\r\\x12\\r\\n\\x05value\\x18\\x02 \\x03(\\t\\\"\\xfc\\x01\\n\\x07Section\\x12\\x11\\n\\tcharBegin\\x18\\x01 \\x02(\\r\\x12\\x0f\\n\\x07\\x63harEnd\\x18\\x02 \\x02(\\r\\x12\\x0e\\n\\x06\\x61uthor\\x18\\x03 \\x01(\\t\\x12\\x17\\n\\x0fsentenceIndexes\\x18\\x04 \\x03(\\r\\x12\\x10\\n\\x08\\x64\\x61tetime\\x18\\x05 \\x01(\\t\\x12\\x30\\n\\x06quotes\\x18\\x06 \\x03(\\x0b\\x32 .edu.stanford.nlp.pipeline.Quote\\x12\\x17\\n\\x0f\\x61uthorCharBegin\\x18\\x07 \\x01(\\r\\x12\\x15\\n\\rauthorCharEnd\\x18\\x08 \\x01(\\r\\x12\\x30\\n\\x06xmlTag\\x18\\t \\x02(\\x0b\\x32 .edu.stanford.nlp.pipeline.Token\\\"\\xe4\\x01\\n\\x0eSemgrexRequest\\x12\\x0f\\n\\x07semgrex\\x18\\x01 \\x03(\\t\\x12\\x45\\n\\x05query\\x18\\x02 \\x03(\\x0b\\x32\\x36.edu.stanford.nlp.pipeline.SemgrexRequest.Dependencies\\x1az\\n\\x0c\\x44\\x65pendencies\\x12/\\n\\x05token\\x18\\x01 \\x03(\\x0b\\x32 .edu.stanford.nlp.pipeline.Token\\x12\\x39\\n\\x05graph\\x18\\x02 \\x02(\\x0b\\x32*.edu.stanford.nlp.pipeline.DependencyGraph\\\"\\xfb\\x06\\n\\x0fSemgrexResponse\\x12\\x46\\n\\x06result\\x18\\x01 \\x03(\\x0b\\x32\\x36.edu.stanford.nlp.pipeline.SemgrexResponse.GraphResult\\x1a-\\n\\tNamedNode\\x12\\x0c\\n\\x04name\\x18\\x01 \\x02(\\t\\x12\\x12\\n\\nmatchIndex\\x18\\x02 \\x02(\\x05\\x1a+\\n\\rNamedRelation\\x12\\x0c\\n\\x04name\\x18\\x01 \\x02(\\t\\x12\\x0c\\n\\x04reln\\x18\\x02 \\x02(\\t\\x1a\\x80\\x01\\n\\tNamedEdge\\x12\\x0c\\n\\x04name\\x18\\x01 \\x02(\\t\\x12\\x0e\\n\\x06source\\x18\\x02 \\x02(\\x05\\x12\\x0e\\n\\x06target\\x18\\x03 \\x02(\\x05\\x12\\x0c\\n\\x04reln\\x18\\x04 \\x01(\\t\\x12\\x0f\\n\\x07isExtra\\x18\\x05 \\x01(\\x08\\x12\\x12\\n\\nsourceCopy\\x18\\x06 \\x01(\\r\\x12\\x12\\n\\ntargetCopy\\x18\\x07 \\x01(\\r\\x1a-\\n\\x0eVariableString\\x12\\x0c\\n\\x04name\\x18\\x01 \\x02(\\t\\x12\\r\\n\\x05value\\x18\\x02 \\x02(\\t\\x1a\\xe6\\x02\\n\\x05Match\\x12\\x12\\n\\nmatchIndex\\x18\\x01 \\x02(\\x05\\x12\\x42\\n\\x04node\\x18\\x02 \\x03(\\x0b\\x32\\x34.edu.stanford.nlp.pipeline.SemgrexResponse.NamedNode\\x12\\x46\\n\\x04reln\\x18\\x03 \\x03(\\x0b\\x32\\x38.edu.stanford.nlp.pipeline.SemgrexResponse.NamedRelation\\x12\\x42\\n\\x04\\x65\\x64ge\\x18\\x06 \\x03(\\x0b\\x32\\x34.edu.stanford.nlp.pipeline.SemgrexResponse.NamedEdge\\x12L\\n\\tvarstring\\x18\\x07 \\x03(\\x0b\\x32\\x39.edu.stanford.nlp.pipeline.SemgrexResponse.VariableString\\x12\\x15\\n\\rsentenceIndex\\x18\\x04 \\x01(\\x05\\x12\\x14\\n\\x0csemgrexIndex\\x18\\x05 \\x01(\\x05\\x1aP\\n\\rSemgrexResult\\x12?\\n\\x05match\\x18\\x01 \\x03(\\x0b\\x32\\x30.edu.stanford.nlp.pipeline.SemgrexResponse.Match\\x1aW\\n\\x0bGraphResult\\x12H\\n\\x06result\\x18\\x01 \\x03(\\x0b\\x32\\x38.edu.stanford.nlp.pipeline.SemgrexResponse.SemgrexResult\\\"\\xf0\\x01\\n\\x0fSsurgeonRequest\\x12\\x45\\n\\x08ssurgeon\\x18\\x01 \\x03(\\x0b\\x32\\x33.edu.stanford.nlp.pipeline.SsurgeonRequest.Ssurgeon\\x12\\x39\\n\\x05graph\\x18\\x02 \\x03(\\x0b\\x32*.edu.stanford.nlp.pipeline.DependencyGraph\\x1a[\\n\\x08Ssurgeon\\x12\\x0f\\n\\x07semgrex\\x18\\x01 \\x01(\\t\\x12\\x11\\n\\toperation\\x18\\x02 \\x03(\\t\\x12\\n\\n\\x02id\\x18\\x03 \\x01(\\t\\x12\\r\\n\\x05notes\\x18\\x04 \\x01(\\t\\x12\\x10\\n\\x08language\\x18\\x05 \\x01(\\t\\\"\\xbc\\x01\\n\\x10SsurgeonResponse\\x12J\\n\\x06result\\x18\\x01 \\x03(\\x0b\\x32:.edu.stanford.nlp.pipeline.SsurgeonResponse.SsurgeonResult\\x1a\\\\\\n\\x0eSsurgeonResult\\x12\\x39\\n\\x05graph\\x18\\x01 \\x01(\\x0b\\x32*.edu.stanford.nlp.pipeline.DependencyGraph\\x12\\x0f\\n\\x07\\x63hanged\\x18\\x02 \\x01(\\x08\\\"W\\n\\x12TokensRegexRequest\\x12\\x30\\n\\x03\\x64oc\\x18\\x01 \\x02(\\x0b\\x32#.edu.stanford.nlp.pipeline.Document\\x12\\x0f\\n\\x07pattern\\x18\\x02 \\x03(\\t\\\"\\xa7\\x03\\n\\x13TokensRegexResponse\\x12J\\n\\x05match\\x18\\x01 \\x03(\\x0b\\x32;.edu.stanford.nlp.pipeline.TokensRegexResponse.PatternMatch\\x1a\\x39\\n\\rMatchLocation\\x12\\x0c\\n\\x04text\\x18\\x01 \\x01(\\t\\x12\\r\\n\\x05\\x62\\x65gin\\x18\\x02 \\x01(\\x05\\x12\\x0b\\n\\x03\\x65nd\\x18\\x03 \\x01(\\x05\\x1a\\xb3\\x01\\n\\x05Match\\x12\\x10\\n\\x08sentence\\x18\\x01 \\x02(\\x05\\x12K\\n\\x05match\\x18\\x02 \\x02(\\x0b\\x32<.edu.stanford.nlp.pipeline.TokensRegexResponse.MatchLocation\\x12K\\n\\x05group\\x18\\x03 \\x03(\\x0b\\x32<.edu.stanford.nlp.pipeline.TokensRegexResponse.MatchLocation\\x1aS\\n\\x0cPatternMatch\\x12\\x43\\n\\x05match\\x18\\x01 \\x03(\\x0b\\x32\\x34.edu.stanford.nlp.pipeline.TokensRegexResponse.Match\\\"\\xae\\x01\\n\\x19\\x44\\x65pendencyEnhancerRequest\\x12\\x35\\n\\x08\\x64ocument\\x18\\x01 \\x02(\\x0b\\x32#.edu.stanford.nlp.pipeline.Document\\x12\\x37\\n\\x08language\\x18\\x02 \\x01(\\x0e\\x32#.edu.stanford.nlp.pipeline.LanguageH\\x00\\x12\\x1a\\n\\x10relativePronouns\\x18\\x03 \\x01(\\tH\\x00\\x42\\x05\\n\\x03ref\\\"\\xb4\\x01\\n\\x12\\x46lattenedParseTree\\x12\\x41\\n\\x05nodes\\x18\\x01 \\x03(\\x0b\\x32\\x32.edu.stanford.nlp.pipeline.FlattenedParseTree.Node\\x1a[\\n\\x04Node\\x12\\x12\\n\\x08openNode\\x18\\x01 \\x01(\\x08H\\x00\\x12\\x13\\n\\tcloseNode\\x18\\x02 \\x01(\\x08H\\x00\\x12\\x0f\\n\\x05value\\x18\\x03 \\x01(\\tH\\x00\\x12\\r\\n\\x05score\\x18\\x04 \\x01(\\x01\\x42\\n\\n\\x08\\x63ontents\\\"\\xf6\\x01\\n\\x15\\x45valuateParserRequest\\x12N\\n\\x08treebank\\x18\\x01 \\x03(\\x0b\\x32<.edu.stanford.nlp.pipeline.EvaluateParserRequest.ParseResult\\x1a\\x8c\\x01\\n\\x0bParseResult\\x12;\\n\\x04gold\\x18\\x01 \\x02(\\x0b\\x32-.edu.stanford.nlp.pipeline.FlattenedParseTree\\x12@\\n\\tpredicted\\x18\\x02 \\x03(\\x0b\\x32-.edu.stanford.nlp.pipeline.FlattenedParseTree\\\"E\\n\\x16\\x45valuateParserResponse\\x12\\n\\n\\x02\\x66\\x31\\x18\\x01 \\x02(\\x01\\x12\\x0f\\n\\x07kbestF1\\x18\\x02 \\x01(\\x01\\x12\\x0e\\n\\x06treeF1\\x18\\x03 \\x03(\\x01\\\"\\xc8\\x01\\n\\x0fTsurgeonRequest\\x12H\\n\\noperations\\x18\\x01 \\x03(\\x0b\\x32\\x34.edu.stanford.nlp.pipeline.TsurgeonRequest.Operation\\x12<\\n\\x05trees\\x18\\x02 \\x03(\\x0b\\x32-.edu.stanford.nlp.pipeline.FlattenedParseTree\\x1a-\\n\\tOperation\\x12\\x0e\\n\\x06tregex\\x18\\x01 \\x02(\\t\\x12\\x10\\n\\x08tsurgeon\\x18\\x02 \\x03(\\t\\\"P\\n\\x10TsurgeonResponse\\x12<\\n\\x05trees\\x18\\x01 \\x03(\\x0b\\x32-.edu.stanford.nlp.pipeline.FlattenedParseTree\\\"\\x85\\x01\\n\\x11MorphologyRequest\\x12\\x46\\n\\x05words\\x18\\x01 \\x03(\\x0b\\x32\\x37.edu.stanford.nlp.pipeline.MorphologyRequest.TaggedWord\\x1a(\\n\\nTaggedWord\\x12\\x0c\\n\\x04word\\x18\\x01 \\x02(\\t\\x12\\x0c\\n\\x04xpos\\x18\\x02 \\x01(\\t\\\"\\x9a\\x01\\n\\x12MorphologyResponse\\x12I\\n\\x05words\\x18\\x01 \\x03(\\x0b\\x32:.edu.stanford.nlp.pipeline.MorphologyResponse.WordTagLemma\\x1a\\x39\\n\\x0cWordTagLemma\\x12\\x0c\\n\\x04word\\x18\\x01 \\x02(\\t\\x12\\x0c\\n\\x04xpos\\x18\\x02 \\x01(\\t\\x12\\r\\n\\x05lemma\\x18\\x03 \\x02(\\t\\\"Z\\n\\x1a\\x44\\x65pendencyConverterRequest\\x12<\\n\\x05trees\\x18\\x01 \\x03(\\x0b\\x32-.edu.stanford.nlp.pipeline.FlattenedParseTree\\\"\\x90\\x02\\n\\x1b\\x44\\x65pendencyConverterResponse\\x12`\\n\\x0b\\x63onversions\\x18\\x01 \\x03(\\x0b\\x32K.edu.stanford.nlp.pipeline.DependencyConverterResponse.DependencyConversion\\x1a\\x8e\\x01\\n\\x14\\x44\\x65pendencyConversion\\x12\\x39\\n\\x05graph\\x18\\x01 \\x02(\\x0b\\x32*.edu.stanford.nlp.pipeline.DependencyGraph\\x12;\\n\\x04tree\\x18\\x02 \\x01(\\x0b\\x32-.edu.stanford.nlp.pipeline.FlattenedParseTree*\\xa3\\x01\\n\\x08Language\\x12\\x0b\\n\\x07Unknown\\x10\\x00\\x12\\x07\\n\\x03\\x41ny\\x10\\x01\\x12\\n\\n\\x06\\x41rabic\\x10\\x02\\x12\\x0b\\n\\x07\\x43hinese\\x10\\x03\\x12\\x0b\\n\\x07\\x45nglish\\x10\\x04\\x12\\n\\n\\x06German\\x10\\x05\\x12\\n\\n\\x06\\x46rench\\x10\\x06\\x12\\n\\n\\x06Hebrew\\x10\\x07\\x12\\x0b\\n\\x07Spanish\\x10\\x08\\x12\\x14\\n\\x10UniversalEnglish\\x10\\t\\x12\\x14\\n\\x10UniversalChinese\\x10\\n*h\\n\\tSentiment\\x12\\x13\\n\\x0fSTRONG_NEGATIVE\\x10\\x00\\x12\\x11\\n\\rWEAK_NEGATIVE\\x10\\x01\\x12\\x0b\\n\\x07NEUTRAL\\x10\\x02\\x12\\x11\\n\\rWEAK_POSITIVE\\x10\\x03\\x12\\x13\\n\\x0fSTRONG_POSITIVE\\x10\\x04*\\x93\\x01\\n\\x14NaturalLogicRelation\\x12\\x0f\\n\\x0b\\x45QUIVALENCE\\x10\\x00\\x12\\x16\\n\\x12\\x46ORWARD_ENTAILMENT\\x10\\x01\\x12\\x16\\n\\x12REVERSE_ENTAILMENT\\x10\\x02\\x12\\x0c\\n\\x08NEGATION\\x10\\x03\\x12\\x0f\\n\\x0b\\x41LTERNATION\\x10\\x04\\x12\\t\\n\\x05\\x43OVER\\x10\\x05\\x12\\x10\\n\\x0cINDEPENDENCE\\x10\\x06\\x42*\\n\\x19\\x65\\x64u.stanford.nlp.pipelineB\\rCoreNLPProtos')\n\n_globals = globals()\n_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)\n_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'CoreNLP_pb2', _globals)\nif _descriptor._USE_C_DESCRIPTORS == False:\n _globals['DESCRIPTOR']._options = None\n _globals['DESCRIPTOR']._serialized_options = b'\\n\\031edu.stanford.nlp.pipelineB\\rCoreNLPProtos'\n _globals['_DEPENDENCYGRAPH'].fields_by_name['root']._options = None\n _globals['_DEPENDENCYGRAPH'].fields_by_name['root']._serialized_options = b'\\020\\001'\n _globals['_DEPENDENCYGRAPH'].fields_by_name['rootNode']._options = None\n _globals['_DEPENDENCYGRAPH'].fields_by_name['rootNode']._serialized_options = b'\\020\\001'\n _globals['_LANGUAGE']._serialized_start=13585\n _globals['_LANGUAGE']._serialized_end=13748\n _globals['_SENTIMENT']._serialized_start=13750\n _globals['_SENTIMENT']._serialized_end=13854\n _globals['_NATURALLOGICRELATION']._serialized_start=13857\n _globals['_NATURALLOGICRELATION']._serialized_end=14004\n _globals['_DOCUMENT']._serialized_start=45\n _globals['_DOCUMENT']._serialized_end=782\n _globals['_SENTENCE']._serialized_start=785\n _globals['_SENTENCE']._serialized_end=2820\n _globals['_TOKEN']._serialized_start=2823\n _globals['_TOKEN']._serialized_end=4477\n _globals['_QUOTE']._serialized_start=4480\n _globals['_QUOTE']._serialized_end=4964\n _globals['_PARSETREE']._serialized_start=4967\n _globals['_PARSETREE']._serialized_end=5166\n _globals['_DEPENDENCYGRAPH']._serialized_start=5169\n _globals['_DEPENDENCYGRAPH']._serialized_end=5708\n _globals['_DEPENDENCYGRAPH_NODE']._serialized_start=5403\n _globals['_DEPENDENCYGRAPH_NODE']._serialized_end=5491\n _globals['_DEPENDENCYGRAPH_EDGE']._serialized_start=5494\n _globals['_DEPENDENCYGRAPH_EDGE']._serialized_end=5708\n _globals['_COREFCHAIN']._serialized_start=5711\n _globals['_COREFCHAIN']._serialized_end=6037\n _globals['_COREFCHAIN_COREFMENTION']._serialized_start=5836\n _globals['_COREFCHAIN_COREFMENTION']._serialized_end=6037\n _globals['_MENTION']._serialized_start=6040\n _globals['_MENTION']._serialized_end=7175\n _globals['_INDEXEDWORD']._serialized_start=7177\n _globals['_INDEXEDWORD']._serialized_end=7265\n _globals['_SPEAKERINFO']._serialized_start=7267\n _globals['_SPEAKERINFO']._serialized_end=7319\n _globals['_SPAN']._serialized_start=7321\n _globals['_SPAN']._serialized_end=7355\n _globals['_TIMEX']._serialized_start=7357\n _globals['_TIMEX']._serialized_end=7476\n _globals['_ENTITY']._serialized_start=7479\n _globals['_ENTITY']._serialized_end=7698\n _globals['_RELATION']._serialized_start=7701\n _globals['_RELATION']._serialized_end=7884\n _globals['_OPERATOR']._serialized_start=7887\n _globals['_OPERATOR']._serialized_end=8065\n _globals['_POLARITY']._serialized_start=8068\n _globals['_POLARITY']._serialized_end=8621\n _globals['_NERMENTION']._serialized_start=8624\n _globals['_NERMENTION']._serialized_end=8973\n _globals['_SENTENCEFRAGMENT']._serialized_start=8975\n _globals['_SENTENCEFRAGMENT']._serialized_end=9064\n _globals['_TOKENLOCATION']._serialized_start=9066\n _globals['_TOKENLOCATION']._serialized_end=9124\n _globals['_RELATIONTRIPLE']._serialized_start=9127\n _globals['_RELATIONTRIPLE']._serialized_end=9537\n _globals['_MAPSTRINGSTRING']._serialized_start=9539\n _globals['_MAPSTRINGSTRING']._serialized_end=9584\n _globals['_MAPINTSTRING']._serialized_start=9586\n _globals['_MAPINTSTRING']._serialized_end=9628\n _globals['_SECTION']._serialized_start=9631\n _globals['_SECTION']._serialized_end=9883\n _globals['_SEMGREXREQUEST']._serialized_start=9886\n _globals['_SEMGREXREQUEST']._serialized_end=10114\n _globals['_SEMGREXREQUEST_DEPENDENCIES']._serialized_start=9992\n _globals['_SEMGREXREQUEST_DEPENDENCIES']._serialized_end=10114\n _globals['_SEMGREXRESPONSE']._serialized_start=10117\n _globals['_SEMGREXRESPONSE']._serialized_end=11008\n _globals['_SEMGREXRESPONSE_NAMEDNODE']._serialized_start=10208\n _globals['_SEMGREXRESPONSE_NAMEDNODE']._serialized_end=10253\n _globals['_SEMGREXRESPONSE_NAMEDRELATION']._serialized_start=10255\n _globals['_SEMGREXRESPONSE_NAMEDRELATION']._serialized_end=10298\n _globals['_SEMGREXRESPONSE_NAMEDEDGE']._serialized_start=10301\n _globals['_SEMGREXRESPONSE_NAMEDEDGE']._serialized_end=10429\n _globals['_SEMGREXRESPONSE_VARIABLESTRING']._serialized_start=10431\n _globals['_SEMGREXRESPONSE_VARIABLESTRING']._serialized_end=10476\n _globals['_SEMGREXRESPONSE_MATCH']._serialized_start=10479\n _globals['_SEMGREXRESPONSE_MATCH']._serialized_end=10837\n _globals['_SEMGREXRESPONSE_SEMGREXRESULT']._serialized_start=10839\n _globals['_SEMGREXRESPONSE_SEMGREXRESULT']._serialized_end=10919\n _globals['_SEMGREXRESPONSE_GRAPHRESULT']._serialized_start=10921\n _globals['_SEMGREXRESPONSE_GRAPHRESULT']._serialized_end=11008\n _globals['_SSURGEONREQUEST']._serialized_start=11011\n _globals['_SSURGEONREQUEST']._serialized_end=11251\n _globals['_SSURGEONREQUEST_SSURGEON']._serialized_start=11160\n _globals['_SSURGEONREQUEST_SSURGEON']._serialized_end=11251\n _globals['_SSURGEONRESPONSE']._serialized_start=11254\n _globals['_SSURGEONRESPONSE']._serialized_end=11442\n _globals['_SSURGEONRESPONSE_SSURGEONRESULT']._serialized_start=11350\n _globals['_SSURGEONRESPONSE_SSURGEONRESULT']._serialized_end=11442\n _globals['_TOKENSREGEXREQUEST']._serialized_start=11444\n _globals['_TOKENSREGEXREQUEST']._serialized_end=11531\n _globals['_TOKENSREGEXRESPONSE']._serialized_start=11534\n _globals['_TOKENSREGEXRESPONSE']._serialized_end=11957\n _globals['_TOKENSREGEXRESPONSE_MATCHLOCATION']._serialized_start=11633\n _globals['_TOKENSREGEXRESPONSE_MATCHLOCATION']._serialized_end=11690\n _globals['_TOKENSREGEXRESPONSE_MATCH']._serialized_start=11693\n _globals['_TOKENSREGEXRESPONSE_MATCH']._serialized_end=11872\n _globals['_TOKENSREGEXRESPONSE_PATTERNMATCH']._serialized_start=11874\n _globals['_TOKENSREGEXRESPONSE_PATTERNMATCH']._serialized_end=11957\n _globals['_DEPENDENCYENHANCERREQUEST']._serialized_start=11960\n _globals['_DEPENDENCYENHANCERREQUEST']._serialized_end=12134\n _globals['_FLATTENEDPARSETREE']._serialized_start=12137\n _globals['_FLATTENEDPARSETREE']._serialized_end=12317\n _globals['_FLATTENEDPARSETREE_NODE']._serialized_start=12226\n _globals['_FLATTENEDPARSETREE_NODE']._serialized_end=12317\n _globals['_EVALUATEPARSERREQUEST']._serialized_start=12320\n _globals['_EVALUATEPARSERREQUEST']._serialized_end=12566\n _globals['_EVALUATEPARSERREQUEST_PARSERESULT']._serialized_start=12426\n _globals['_EVALUATEPARSERREQUEST_PARSERESULT']._serialized_end=12566\n _globals['_EVALUATEPARSERRESPONSE']._serialized_start=12568\n _globals['_EVALUATEPARSERRESPONSE']._serialized_end=12637\n _globals['_TSURGEONREQUEST']._serialized_start=12640\n _globals['_TSURGEONREQUEST']._serialized_end=12840\n _globals['_TSURGEONREQUEST_OPERATION']._serialized_start=12795\n _globals['_TSURGEONREQUEST_OPERATION']._serialized_end=12840\n _globals['_TSURGEONRESPONSE']._serialized_start=12842\n _globals['_TSURGEONRESPONSE']._serialized_end=12922\n _globals['_MORPHOLOGYREQUEST']._serialized_start=12925\n _globals['_MORPHOLOGYREQUEST']._serialized_end=13058\n _globals['_MORPHOLOGYREQUEST_TAGGEDWORD']._serialized_start=13018\n _globals['_MORPHOLOGYREQUEST_TAGGEDWORD']._serialized_end=13058\n _globals['_MORPHOLOGYRESPONSE']._serialized_start=13061\n _globals['_MORPHOLOGYRESPONSE']._serialized_end=13215\n _globals['_MORPHOLOGYRESPONSE_WORDTAGLEMMA']._serialized_start=13158\n _globals['_MORPHOLOGYRESPONSE_WORDTAGLEMMA']._serialized_end=13215\n _globals['_DEPENDENCYCONVERTERREQUEST']._serialized_start=13217\n _globals['_DEPENDENCYCONVERTERREQUEST']._serialized_end=13307\n _globals['_DEPENDENCYCONVERTERRESPONSE']._serialized_start=13310\n _globals['_DEPENDENCYCONVERTERRESPONSE']._serialized_end=13582\n _globals['_DEPENDENCYCONVERTERRESPONSE_DEPENDENCYCONVERSION']._serialized_start=13440\n _globals['_DEPENDENCYCONVERTERRESPONSE_DEPENDENCYCONVERSION']._serialized_end=13582\n# @@protoc_insertion_point(module_scope)\n"
},
{
"path": "stanza/protobuf/__init__.py",
"content": "from __future__ import absolute_import\n\nfrom io import BytesIO\nimport warnings\n\nfrom google.protobuf.internal.encoder import _EncodeVarint\nfrom google.protobuf.internal.decoder import _DecodeVarint\nfrom google.protobuf.message import DecodeError\nfrom .CoreNLP_pb2 import *\n\ndef parseFromDelimitedString(obj, buf, offset=0):\n \"\"\"\n Stanford CoreNLP uses the Java \"writeDelimitedTo\" function, which\n writes the size (and offset) of the buffer before writing the object.\n This function handles parsing this message starting from offset 0.\n\n @returns how many bytes of @buf were consumed.\n \"\"\"\n size, pos = _DecodeVarint(buf, offset)\n try:\n obj.ParseFromString(buf[offset+pos:offset+pos+size])\n except DecodeError as e:\n warnings.warn(\"Failed to decode a serialized output from CoreNLP server. An incomplete or empty object will be returned.\", \\\n RuntimeWarning)\n return pos+size\n\ndef writeToDelimitedString(obj, stream=None):\n \"\"\"\n Stanford CoreNLP uses the Java \"writeDelimitedTo\" function, which\n writes the size (and offset) of the buffer before writing the object.\n This function handles parsing this message starting from offset 0.\n\n @returns how many bytes of @buf were consumed.\n \"\"\"\n if stream is None:\n stream = BytesIO()\n\n _EncodeVarint(stream.write, obj.ByteSize(), True)\n stream.write(obj.SerializeToString())\n return stream\n\ndef to_text(sentence):\n \"\"\"\n Helper routine that converts a Sentence protobuf to a string from\n its tokens.\n \"\"\"\n text = \"\"\n for i, tok in enumerate(sentence.token):\n if i != 0:\n text += tok.before\n text += tok.word\n return text\n"
},
{
"path": "stanza/resources/__init__.py",
"content": ""
},
{
"path": "stanza/resources/common.py",
"content": "\"\"\"\nCommon utilities for Stanza resources.\n\"\"\"\n\nfrom collections import defaultdict, namedtuple\nimport errno\nimport hashlib\nimport json\nimport logging\nimport os\nfrom pathlib import Path\nimport requests\nimport shutil\nimport tempfile\nimport zipfile\n\nfrom platformdirs import user_cache_dir\nfrom tqdm.auto import tqdm\n\nfrom stanza.utils.helper_func import make_table\nfrom stanza.pipeline._constants import TOKENIZE, MWT, POS, LEMMA, DEPPARSE, NER, SENTIMENT\nfrom stanza.pipeline.registry import PIPELINE_NAMES, PROCESSOR_VARIANTS\nfrom stanza.resources.default_packages import PACKAGES\nfrom stanza._version import __resources_version__\n\nlogger = logging.getLogger('stanza')\n\n# set home dir for default\nUSER_CACHE_DIR = user_cache_dir('stanza', 'StanfordNLP', __resources_version__)\nSTANFORDNLP_RESOURCES_URL = 'https://nlp.stanford.edu/software/stanza/stanza-resources/'\nSTANZA_RESOURCES_GITHUB = 'https://raw.githubusercontent.com/stanfordnlp/stanza-resources/'\nDEFAULT_RESOURCES_URL = os.getenv('STANZA_RESOURCES_URL', STANZA_RESOURCES_GITHUB + 'main')\nDEFAULT_RESOURCES_VERSION = os.getenv(\n 'STANZA_RESOURCES_VERSION',\n __resources_version__\n)\nDEFAULT_MODEL_URL = os.getenv('STANZA_MODEL_URL', 'default')\nDEFAULT_MODEL_DIR = os.getenv(\n 'STANZA_RESOURCES_DIR',\n os.path.join(USER_CACHE_DIR, 'resources')\n)\n\nPRETRAIN_NAMES = (\"pretrain\", \"forward_charlm\", \"backward_charlm\")\n\nclass ResourcesFileNotFoundError(FileNotFoundError):\n def __init__(self, resources_filepath):\n super().__init__(f\"Resources file not found at: {resources_filepath} Try to download the model again.\")\n self.resources_filepath = resources_filepath\n\nclass UnknownLanguageError(ValueError):\n def __init__(self, unknown):\n super().__init__(f\"Unknown language requested: {unknown}\")\n self.unknown_language = unknown\n\nclass UnknownProcessorError(ValueError):\n def __init__(self, unknown):\n super().__init__(f\"Unknown processor type requested: {unknown}\")\n self.unknown_processor = unknown\n\nModelSpecification = namedtuple('ModelSpecification', ['processor', 'package', 'dependencies'])\n\ndef ensure_dir(path):\n \"\"\"\n Create dir in case it does not exist.\n \"\"\"\n Path(path).mkdir(parents=True, exist_ok=True)\n\ndef get_md5(path):\n \"\"\"\n Get the MD5 value of a path.\n \"\"\"\n try:\n with open(path, 'rb') as fin:\n data = fin.read()\n except OSError as e:\n if not e.filename:\n e.filename = path\n raise\n return hashlib.md5(data).hexdigest()\n\ndef unzip(path, filename):\n \"\"\"\n Fully unzip a file `filename` that's in a directory `dir`.\n \"\"\"\n logger.debug(f'Unzip: {path}/{filename}...')\n with zipfile.ZipFile(os.path.join(path, filename)) as f:\n f.extractall(path)\n\ndef get_root_from_zipfile(filename):\n \"\"\"\n Get the root directory from a archived zip file.\n \"\"\"\n zf = zipfile.ZipFile(filename, \"r\")\n assert len(zf.filelist) > 0, \\\n f\"Zip file at f{filename} seems to be corrupted. Please check it.\"\n return os.path.dirname(zf.filelist[0].filename)\n\ndef file_exists(path, md5):\n \"\"\"\n Check if the file at `path` exists and match the provided md5 value.\n \"\"\"\n return os.path.exists(path) and get_md5(path) == md5\n\ndef assert_file_exists(path, md5=None, alternate_md5=None):\n if not os.path.exists(path):\n raise FileNotFoundError(errno.ENOENT, \"Cannot find expected file\", path)\n if md5:\n file_md5 = get_md5(path)\n if file_md5 != md5:\n if file_md5 == alternate_md5:\n logger.debug(\"Found a possibly older version of file %s, md5 %s instead of %s\", path, alternate_md5, md5)\n else:\n raise ValueError(\"md5 for %s is %s, expected %s\" % (path, file_md5, md5))\n\ndef download_file(url, path, proxies, raise_for_status=False):\n \"\"\"\n Download a URL into a file as specified by `path`.\n \"\"\"\n verbose = logger.level in [0, 10, 20]\n r = requests.get(url, stream=True, proxies=proxies)\n if raise_for_status:\n r.raise_for_status()\n with open(path, 'wb') as f:\n file_size = r.headers.get('content-length', None)\n if file_size:\n file_size = int(file_size)\n default_chunk_size = 131072\n desc = 'Downloading ' + url\n with tqdm(total=file_size, unit='B', unit_scale=True, \\\n disable=not verbose, desc=desc) as pbar:\n for chunk in r.iter_content(chunk_size=default_chunk_size):\n if chunk:\n f.write(chunk)\n f.flush()\n pbar.update(len(chunk))\n return r.status_code\n\ndef request_file(url, path, proxies=None, md5=None, raise_for_status=False, log_info=True, alternate_md5=None):\n \"\"\"\n A complete wrapper over download_file() that also make sure the directory of\n `path` exists, and that a file matching the md5 value does not exist.\n\n alternate_md5 allows for an alternate md5 that is acceptable (such as if an older version of a file is okay)\n \"\"\"\n basedir = Path(path).parent\n ensure_dir(basedir)\n if file_exists(path, md5):\n if log_info:\n logger.info(f'File exists: {path}')\n else:\n logger.debug(f'File exists: {path}')\n return\n # We write data first to a temporary directory,\n # then use os.replace() so that multiple processes\n # running at the same time don't clobber each other\n # with partially downloaded files\n # This was especially common with resources.json\n with tempfile.TemporaryDirectory(dir=basedir) as temp:\n temppath = os.path.join(temp, os.path.split(path)[-1])\n download_file(url, temppath, proxies, raise_for_status)\n os.replace(temppath, path)\n assert_file_exists(path, md5, alternate_md5)\n if log_info:\n logger.info(f'Downloaded file to {path}')\n else:\n logger.debug(f'Downloaded file to {path}')\n\ndef sort_processors(processor_list):\n sorted_list = []\n for processor in PIPELINE_NAMES:\n for item in processor_list:\n if item[0] == processor:\n sorted_list.append(item)\n # going just by processors in PIPELINE_NAMES, this drops any names\n # which are not an official processor but might still be useful\n # check the list and append them to the end\n # this is especially useful when downloading pretrain or charlm models\n for processor in processor_list:\n for item in sorted_list:\n if processor[0] == item[0]:\n break\n else:\n sorted_list.append(item)\n return sorted_list\n\ndef add_mwt(processors, resources, lang):\n \"\"\"Add mwt if tokenize is passed without mwt.\n\n If tokenize is in the list, but mwt is not, and there is a corresponding\n tokenize and mwt pair in the resources file, mwt is added so no missing\n mwt errors are raised.\n \"\"\"\n value = processors[TOKENIZE]\n if value in resources[lang][PACKAGES] and MWT in resources[lang][PACKAGES][value]:\n logger.warning(\"Language %s package %s expects mwt, which has been added\", lang, value)\n processors[MWT] = value\n elif (value in resources[lang][TOKENIZE] and MWT in resources[lang] and value in resources[lang][MWT]):\n logger.warning(\"Language %s package %s expects mwt, which has been added\", lang, value)\n processors[MWT] = value\n\ndef maintain_processor_list(resources, lang, package, processors, allow_pretrain=False, maybe_add_mwt=True):\n \"\"\"\n Given a parsed resources file, language, and possible package\n and/or processors, expands the package to the list of processors\n\n Returns a list of processors\n Each item in the list of processors is a pair:\n name, then a list of ModelSpecification\n so, for example:\n [['pos', [ModelSpecification(processor='pos', package='gsd', dependencies=None)]],\n ['depparse', [ModelSpecification(processor='depparse', package='gsd', dependencies=None)]]]\n \"\"\"\n processor_list = defaultdict(list)\n # resolve processor models\n if processors:\n logger.debug(f'Processing parameter \"processors\"...')\n if maybe_add_mwt and TOKENIZE in processors and MWT not in processors:\n add_mwt(processors, resources, lang)\n for key, plist in processors.items():\n if not isinstance(key, str):\n raise ValueError(\"Processor names must be strings\")\n if not isinstance(plist, (tuple, list, str)):\n raise ValueError(\"Processor values must be strings\")\n if isinstance(plist, str):\n plist = [plist]\n if key not in PIPELINE_NAMES:\n if not allow_pretrain or key not in PRETRAIN_NAMES:\n raise UnknownProcessorError(key)\n for value in plist:\n # check if keys and values can be found\n if key in resources[lang] and value in resources[lang][key]:\n logger.debug(f'Found {key}: {value}.')\n processor_list[key].append(value)\n # allow values to be default in some cases\n elif value in resources[lang][PACKAGES] and key in resources[lang][PACKAGES][value]:\n logger.debug(\n f'Found {key}: {resources[lang][PACKAGES][value][key]}.'\n )\n processor_list[key].append(resources[lang][PACKAGES][value][key])\n # optional defaults will be activated if specifically turned on\n elif value in resources[lang][PACKAGES] and 'optional' in resources[lang][PACKAGES][value] and key in resources[lang][PACKAGES][value]['optional']:\n logger.debug(\n f\"Found {key}: {resources[lang][PACKAGES][value]['optional'][key]}.\"\n )\n processor_list[key].append(resources[lang][PACKAGES][value]['optional'][key])\n # allow processors to be set to variants that we didn't implement\n elif value in PROCESSOR_VARIANTS[key]:\n logger.debug(\n f'Found {key}: {value}. '\n f'Using external {value} variant for the {key} processor.'\n )\n processor_list[key].append(value)\n # allow lemma to be set to \"identity\"\n elif key == LEMMA and value == 'identity':\n logger.debug(\n f'Found {key}: {value}. Using identity lemmatizer.'\n )\n processor_list[key].append(value)\n # not a processor in the officially supported processor list\n elif key not in resources[lang]:\n logger.debug(\n f'{key}: {value} is not officially supported by Stanza, '\n f'loading it anyway.'\n )\n processor_list[key].append(value)\n # cannot find the package for a processor and warn user\n else:\n logger.warning(\n f'Can not find {key}: {value} from official model list. '\n f'Ignoring it.'\n )\n # resolve package\n if package:\n logger.debug(f'Processing parameter \"package\"...')\n if PACKAGES in resources[lang] and package in resources[lang][PACKAGES]:\n for key, value in resources[lang][PACKAGES][package].items():\n if key != 'optional' and key not in processor_list:\n logger.debug(f'Found {key}: {value}.')\n processor_list[key].append(value)\n else:\n flag = False\n for key in PIPELINE_NAMES:\n if key not in resources[lang]: continue\n if package in resources[lang][key]:\n flag = True\n if key not in processor_list:\n logger.debug(f'Found {key}: {package}.')\n processor_list[key].append(package)\n else:\n logger.debug(\n f'{key}: {package} is overwritten by '\n f'{key}: {processors[key]}.'\n )\n if not flag: logger.warning((f'Can not find package: {package}.'))\n processor_list = [[key, [ModelSpecification(processor=key, package=value, dependencies=None) for value in plist]] for key, plist in processor_list.items()]\n processor_list = sort_processors(processor_list)\n return processor_list\n\ndef add_dependencies(resources, lang, processor_list):\n \"\"\"\n Expand the processor_list as given in maintain_processor_list to have the dependencies\n\n Still a list of model types to ModelSpecifications\n the dependencies are tuples: name and package\n for example:\n [['pos', (ModelSpecification(processor='pos', package='gsd', dependencies=(('pretrain', 'gsd'),)),)],\n ['depparse', (ModelSpecification(processor='depparse', package='gsd', dependencies=(('pretrain', 'gsd'),)),)]]\n \"\"\"\n lang_resources = resources[lang]\n for item in processor_list:\n processor, model_specs = item\n new_model_specs = []\n for model_spec in model_specs:\n # skip dependency checking for external variants of processors and identity lemmatizer\n if not any([\n model_spec.package in PROCESSOR_VARIANTS[processor],\n processor == LEMMA and model_spec.package == 'identity'\n ]):\n dependencies = lang_resources.get(processor, {}).get(model_spec.package, {}).get('dependencies', [])\n dependencies = [(dependency['model'], dependency['package']) for dependency in dependencies]\n model_spec = model_spec._replace(dependencies=tuple(dependencies))\n logger.debug(\"Found dependencies %s for processor %s model %s\", dependencies, processor, model_spec.package)\n new_model_specs.append(model_spec)\n item[1] = tuple(new_model_specs)\n return processor_list\n\ndef flatten_processor_list(processor_list):\n \"\"\"\n The flattened processor list is just a list of types & packages\n\n For example:\n [['pos', 'gsd'], ['depparse', 'gsd'], ['pretrain', 'gsd']]\n \"\"\"\n flattened_processor_list = []\n dependencies_list = []\n for item in processor_list:\n processor, model_specs = item\n for model_spec in model_specs:\n package = model_spec.package\n dependencies = model_spec.dependencies\n flattened_processor_list.append([processor, package])\n if dependencies:\n dependencies_list += [tuple(dependency) for dependency in dependencies]\n dependencies_list = [list(item) for item in set(dependencies_list)]\n for processor, package in dependencies_list:\n logger.debug(f'Find dependency {processor}: {package}.')\n flattened_processor_list += dependencies_list\n return flattened_processor_list\n\ndef set_logging_level(logging_level, verbose):\n # Check verbose for easy logging control\n if verbose == False:\n logging_level = 'ERROR'\n elif verbose == True:\n logging_level = 'INFO'\n\n if logging_level is None:\n # default logging level of INFO is set in stanza.__init__\n # but the user may have set it via the logging API\n # it should NOT be 0, but let's check to be sure...\n if logger.level == 0:\n logger.setLevel('INFO')\n return logger.level\n\n # Set logging level\n logging_level = logging_level.upper()\n all_levels = ['DEBUG', 'INFO', 'WARNING', 'WARN', 'ERROR', 'CRITICAL', 'FATAL']\n if logging_level not in all_levels:\n raise ValueError(\n f\"Unrecognized logging level for pipeline: \"\n f\"{logging_level}. Must be one of {', '.join(all_levels)}.\"\n )\n logger.setLevel(logging_level)\n return logger.level\n\ndef process_pipeline_parameters(lang, model_dir, package, processors):\n # Check parameter types and convert values to lower case\n if isinstance(lang, str):\n lang = lang.strip().lower()\n elif lang is not None:\n raise TypeError(\n f\"The parameter 'lang' should be str, \"\n f\"but got {type(lang).__name__} instead.\"\n )\n\n if isinstance(model_dir, str):\n model_dir = model_dir.strip()\n elif model_dir is not None:\n raise TypeError(\n f\"The parameter 'model_dir' should be str, \"\n f\"but got {type(model_dir).__name__} instead.\"\n )\n\n if isinstance(processors, (str, list, tuple)):\n # Special case: processors is str, compatible with older version\n # also allow for setting alternate packages for these processors\n # via the package argument\n if package is None:\n # each processor will be 'default' for this language\n package = defaultdict(lambda: 'default')\n elif isinstance(package, str):\n # same, but now the named package will be the default instead\n default = package\n package = defaultdict(lambda: default)\n elif isinstance(package, dict):\n # the dictionary of packages will be used to build the processors dict\n # any processor not specified in package will be 'default'\n package = defaultdict(lambda: 'default', package)\n else:\n raise TypeError(\n f\"The parameter 'package' should be None, str, or dict, \"\n f\"but got {type(package).__name__} instead.\"\n )\n if isinstance(processors, str):\n processors = [x.strip().lower() for x in processors.split(\",\")]\n processors = {\n processor: package[processor] for processor in processors\n }\n package = None\n elif isinstance(processors, dict):\n processors = {\n k.strip().lower(): ([v_i.strip().lower() for v_i in v] if isinstance(v, (tuple, list)) else v.strip().lower())\n for k, v in processors.items()\n }\n elif processors is not None:\n raise TypeError(\n f\"The parameter 'processors' should be dict or str, \"\n f\"but got {type(processors).__name__} instead.\"\n )\n\n if isinstance(package, str):\n package = package.strip().lower()\n elif package is not None:\n raise TypeError(\n f\"The parameter 'package' should be str, or a dict if 'processors' is a str, \"\n f\"but got {type(package).__name__} instead.\"\n )\n\n return lang, model_dir, package, processors\n\ndef download_resources_json(model_dir=DEFAULT_MODEL_DIR,\n resources_url=DEFAULT_RESOURCES_URL,\n resources_branch=None,\n resources_version=DEFAULT_RESOURCES_VERSION,\n resources_filepath=None,\n proxies=None):\n \"\"\"\n Downloads resources.json to obtain latest packages.\n \"\"\"\n if resources_url == DEFAULT_RESOURCES_URL and resources_branch is not None:\n resources_url = STANZA_RESOURCES_GITHUB + resources_branch\n # handle short name for resources urls; otherwise treat it as url\n if resources_url.lower() in ('stanford', 'stanfordnlp'):\n resources_url = STANFORDNLP_RESOURCES_URL\n resources_url = f'{resources_url}/resources_{resources_version}.json'\n logger.debug('Downloading resource file from %s', resources_url)\n if resources_filepath is None:\n resources_filepath = os.path.join(model_dir, 'resources.json')\n # make request\n request_file(\n resources_url,\n resources_filepath,\n proxies,\n raise_for_status=True\n )\n\n\ndef load_resources_json(model_dir=DEFAULT_MODEL_DIR, resources_filepath=None):\n \"\"\"\n Unpack the resources json file from the given model_dir\n \"\"\"\n if resources_filepath is None:\n resources_filepath = os.path.join(model_dir, 'resources.json')\n if not os.path.exists(resources_filepath):\n raise ResourcesFileNotFoundError(resources_filepath)\n with open(resources_filepath, encoding=\"utf-8\") as fin:\n resources = json.load(fin)\n return resources\n\ndef get_language_resources(resources, lang):\n \"\"\"\n Get the resources for a lang from an already loaded resources json, following 'alias' if needed\n \"\"\"\n if lang not in resources:\n return None\n\n lang_resources = resources[lang]\n while 'alias' in lang_resources:\n lang = lang_resources['alias']\n lang_resources = resources[lang]\n\n return lang_resources\n\ndef list_available_languages(model_dir=DEFAULT_MODEL_DIR,\n resources_url=DEFAULT_RESOURCES_URL,\n resources_branch=None,\n resources_version=DEFAULT_RESOURCES_VERSION,\n proxies=None):\n \"\"\"\n List the non-alias languages in the resources file\n \"\"\"\n download_resources_json(model_dir, resources_url, resources_branch, resources_version, resources_filepath=None, proxies=proxies)\n resources = load_resources_json(model_dir)\n # isinstance(str) is because of fields such as \"url\"\n # 'alias' is because we want to skip German, alias of de, for example\n languages = [lang for lang in resources\n if not isinstance(resources[lang], str) and 'alias' not in resources[lang]]\n languages = sorted(languages)\n return languages\n\ndef expand_model_url(resources, model_url):\n \"\"\"\n Returns the url in the resources dict if model_url is default, or returns the model_url\n \"\"\"\n return resources['url'] if model_url.lower() == 'default' else model_url\n\ndef download_models(download_list,\n resources,\n lang,\n model_dir=DEFAULT_MODEL_DIR,\n resources_version=DEFAULT_RESOURCES_VERSION,\n model_url=DEFAULT_MODEL_URL,\n proxies=None,\n log_info=True):\n lang_name = resources.get(lang, {}).get('lang_name', lang)\n download_table = make_table(['Processor', 'Package'], download_list)\n if log_info:\n log_msg = logger.info\n else:\n log_msg = logger.debug\n log_msg(\n f'Downloading these customized packages for language: '\n f'{lang} ({lang_name})...\\n{download_table}'\n )\n\n url = expand_model_url(resources, model_url)\n\n # Download packages\n for key, value in download_list:\n try:\n request_file(\n url.format(resources_version=resources_version, lang=lang, filename=f\"{key}/{value}.pt\"),\n os.path.join(model_dir, lang, key, f'{value}.pt'),\n proxies,\n md5=resources[lang][key][value]['md5'],\n log_info=log_info,\n alternate_md5=resources[lang][key][value].get('alternate_md5', None)\n )\n except KeyError as e:\n raise ValueError(\n f'Cannot find the following processor and model name combination: '\n f'{key}, {value}. Please check if you have provided the correct model name.'\n ) from e\n\n# main download function\ndef download(\n lang='en',\n model_dir=DEFAULT_MODEL_DIR,\n package='default',\n processors={},\n logging_level=None,\n verbose=None,\n resources_url=DEFAULT_RESOURCES_URL,\n resources_branch=None,\n resources_version=DEFAULT_RESOURCES_VERSION,\n model_url=DEFAULT_MODEL_URL,\n proxies=None,\n download_json=True\n ):\n # set global logging level\n set_logging_level(logging_level, verbose)\n # process different pipeline parameters\n lang, model_dir, package, processors = process_pipeline_parameters(\n lang, model_dir, package, processors\n )\n\n if download_json or not os.path.exists(os.path.join(model_dir, 'resources.json')):\n if not download_json:\n logger.warning(\"Asked to skip downloading resources.json, but the file does not exist. Downloading anyway\")\n download_resources_json(model_dir, resources_url, resources_branch, resources_version, resources_filepath=None, proxies=proxies)\n\n resources = load_resources_json(model_dir)\n if lang not in resources:\n raise UnknownLanguageError(lang)\n if 'alias' in resources[lang]:\n logger.info(f'\"{lang}\" is an alias for \"{resources[lang][\"alias\"]}\"')\n lang = resources[lang]['alias']\n lang_name = resources.get(lang, {}).get('lang_name', lang)\n url = expand_model_url(resources, model_url)\n\n # Default: download zipfile and unzip\n if package == 'default' and (processors is None or len(processors) == 0):\n logger.info(\n f'Downloading default packages for language: {lang} ({lang_name}) ...'\n )\n # want the URL to become, for example:\n # https://huggingface.co/stanfordnlp/stanza-af/resolve/v1.3.0/models/default.zip\n # so we hopefully start from\n # https://huggingface.co/stanfordnlp/stanza-{lang}/resolve/v{resources_version}/models/{filename}\n request_file(\n url.format(resources_version=resources_version, lang=lang, filename=\"default.zip\"),\n os.path.join(model_dir, lang, f'default.zip'),\n proxies,\n md5=resources[lang]['default_md5'],\n )\n unzip(os.path.join(model_dir, lang), 'default.zip')\n download_list = [['zip', 'default.zip']]\n # Customize: maintain download list\n else:\n download_list = maintain_processor_list(resources, lang, package, processors, allow_pretrain=True)\n download_list = add_dependencies(resources, lang, download_list)\n download_list = flatten_processor_list(download_list)\n download_models(download_list=download_list,\n resources=resources,\n lang=lang,\n model_dir=model_dir,\n resources_version=resources_version,\n model_url=model_url,\n proxies=proxies,\n log_info=True)\n logger.info(f'Finished downloading models and saved to {model_dir}')\n return download_list\n"
},
{
"path": "stanza/resources/default_packages.py",
"content": "\"\"\"\nConstants for default packages, default pretrains, charlms, etc\n\nSeparated from prepare_resources.py so that other modules can use the\nsame lists / maps without importing the resources script and possibly\ncausing a circular import\n\"\"\"\n\nimport copy\n\n# all languages will have a map which represents the available packages\nPACKAGES = \"packages\"\n\n# default treebank for languages\ndefault_treebanks = {\n \"ab\": \"abnc\",\n \"af\": \"afribooms\",\n # currently not publicly released! sent to us from the group developing this resource\n \"ang\": \"nerthus\",\n \"ar\": \"padt\",\n \"be\": \"hse\",\n \"bg\": \"btb\",\n \"bxr\": \"bdt\",\n \"ca\": \"ancora\",\n \"cop\": \"scriptorium\",\n \"cs\": \"pdt\",\n \"cu\": \"proiel\",\n \"cy\": \"ccg\",\n \"da\": \"ddt\",\n \"de\": \"combined\",\n \"el\": \"gdt\",\n \"en\": \"combined\",\n \"es\": \"combined\",\n \"et\": \"edt\",\n \"eu\": \"bdt\",\n \"fa\": \"perdt\",\n \"fi\": \"tdt\",\n \"fo\": \"farpahc\",\n \"fr\": \"combined\",\n \"fro\": \"profiterole\",\n \"ga\": \"idt\",\n \"gd\": \"arcosg\",\n \"gl\": \"ctg\",\n \"got\": \"proiel\",\n \"grc\": \"perseus\",\n \"gv\": \"cadhan\",\n \"hbo\": \"ptnk\",\n \"he\": \"combined\",\n \"hi\": \"hdtb\",\n \"hr\": \"set\",\n \"hsb\": \"ufal\",\n \"hu\": \"szeged\",\n \"hy\": \"armtdp\",\n \"hyw\": \"armtdp\",\n \"id\": \"gsd\",\n \"is\": \"icepahc\",\n \"it\": \"combined\",\n \"ja\": \"combined\",\n \"ka\": \"glc\",\n \"kk\": \"ktb\",\n \"kmr\": \"mg\",\n \"ko\": \"kaist\",\n \"kpv\": \"lattice\",\n \"ky\": \"ktmu\",\n \"la\": \"ittb\",\n \"lij\": \"glt\",\n \"lt\": \"alksnis\",\n \"lv\": \"lvtb\",\n \"lzh\": \"kyoto\",\n \"mr\": \"ufal\",\n \"mt\": \"mudt\",\n \"my\": \"ucsy\",\n \"myv\": \"jr\",\n \"nb\": \"bokmaal\",\n \"nds\": \"lsdc\",\n \"nl\": \"alpino\",\n \"nn\": \"nynorsk\",\n \"olo\": \"kkpp\",\n \"orv\": \"torot\",\n \"ota\": \"boun\",\n \"pcm\": \"nsc\",\n \"pl\": \"pdb\",\n \"pt\": \"bosque\",\n \"qaf\": \"arabizi\",\n \"qpm\": \"philotis\",\n \"qtd\": \"sagt\",\n \"ro\": \"rrt\",\n \"ru\": \"syntagrus\",\n \"sa\": \"vedic\",\n \"sd\": \"isra\",\n \"sk\": \"snk\",\n \"sl\": \"ssj\",\n \"sme\": \"giella\",\n \"sq\": \"combined\",\n \"sr\": \"set\",\n \"sv\": \"talbanken\",\n \"swl\": \"sslc\",\n \"ta\": \"ttb\",\n \"te\": \"mtg\",\n \"th\": \"tud\",\n \"tr\": \"imst\",\n \"ug\": \"udt\",\n \"uk\": \"iu\",\n \"ur\": \"udtb\",\n \"vi\": \"vtb\",\n \"wo\": \"wtb\",\n \"xcl\": \"caval\",\n \"zh-hans\": \"gsdsimp\",\n \"zh-hant\": \"gsd\",\n \"multilingual\": \"ud\"\n}\n\nno_pretrain_languages = set([\n \"cop\",\n \"olo\",\n \"orv\",\n \"pcm\",\n \"qaf\", # the QAF treebank is code switched and Romanized, so not easy to reuse existing resources\n \"qpm\", # have talked about deriving this from a language neighborinig to Pomak, but that hasn't happened yet\n \"qtd\",\n \"swl\",\n\n \"multilingual\", # special case so that all languages with a default treebank are represented somewhere\n])\n\n\n# in some cases, we give the pretrain a name other than the original\n# name for the UD dataset\n# we will eventually do this for all of the pretrains\nspecific_default_pretrains = {\n \"ab\": \"fasttextwiki\",\n \"af\": \"fasttextwiki\",\n \"ang\": \"nerthus\",\n \"ar\": \"conll17\",\n \"be\": \"fasttextwiki\",\n \"bg\": \"conll17\",\n \"bxr\": \"fasttextwiki\",\n \"ca\": \"conll17\",\n \"cs\": \"conll17\",\n \"cu\": \"conll17\",\n \"cy\": \"fasttext157\",\n \"da\": \"conll17\",\n \"de\": \"conll17\",\n \"el\": \"conll17\",\n \"en\": \"conll17\",\n \"es\": \"conll17\",\n \"et\": \"conll17\",\n \"eu\": \"conll17\",\n \"fa\": \"conll17\",\n \"fi\": \"conll17\",\n \"fo\": \"fasttextwiki\",\n \"fr\": \"conll17\",\n \"fro\": \"conll17\",\n \"ga\": \"conll17\",\n \"gd\": \"fasttextwiki\",\n \"gl\": \"conll17\",\n \"got\": \"fasttextwiki\",\n \"grc\": \"conll17\",\n \"gv\": \"fasttext157\",\n \"hbo\": \"utah\",\n \"he\": \"conll17\",\n \"hi\": \"conll17\",\n \"hr\": \"conll17\",\n \"hsb\": \"fasttextwiki\",\n \"hu\": \"conll17\",\n \"hy\": \"isprasglove\",\n \"hyw\": \"isprasglove\",\n \"id\": \"conll17\",\n \"is\": \"fasttext157\",\n \"it\": \"conll17\",\n \"ja\": \"conll17\",\n \"ka\": \"fasttext157\",\n \"kk\": \"fasttext157\",\n \"kmr\": \"fasttextwiki\",\n \"ko\": \"conll17\",\n \"kpv\": \"fasttextwiki\",\n \"ky\": \"fasttext157\",\n \"la\": \"conll17\",\n \"lij\": \"fasttextwiki\",\n \"lt\": \"fasttextwiki\",\n \"lv\": \"conll17\",\n \"lzh\": \"fasttextwiki\",\n \"mr\": \"fasttextwiki\",\n \"mt\": \"fasttextwiki\",\n \"my\": \"ucsy\",\n \"myv\": \"mokha\",\n \"nb\": \"conll17\",\n \"nds\": \"fasttext157\",\n \"nl\": \"conll17\",\n \"nn\": \"conll17\",\n \"or\": \"fasttext157\",\n \"ota\": \"conll17\",\n \"pl\": \"conll17\",\n \"pt\": \"conll17\",\n \"ro\": \"conll17\",\n \"ru\": \"conll17\",\n \"sa\": \"fasttext157\",\n \"sd\": \"isra\",\n \"sk\": \"conll17\",\n \"sl\": \"conll17\",\n \"sme\": \"fasttextwiki\",\n \"sq\": \"fasttext157\",\n \"sr\": \"fasttextwiki\",\n \"sv\": \"conll17\",\n \"ta\": \"fasttextwiki\",\n \"te\": \"fasttextwiki\",\n \"th\": \"fasttext157\",\n \"tr\": \"conll17\",\n \"ug\": \"conll17\",\n \"uk\": \"conll17\",\n \"ur\": \"conll17\",\n \"vi\": \"conll17\",\n \"wo\": \"fasttextwiki\",\n \"xcl\": \"caval\",\n \"zh-hans\": \"fasttext157\",\n \"zh-hant\": \"conll17\",\n}\n\n\ndef build_default_pretrains(default_treebanks):\n default_pretrains = dict(default_treebanks)\n for lang in no_pretrain_languages:\n default_pretrains.pop(lang, None)\n for lang in specific_default_pretrains.keys():\n default_pretrains[lang] = specific_default_pretrains[lang]\n return default_pretrains\n\ndefault_pretrains = build_default_pretrains(default_treebanks)\n\npos_pretrains = {\n \"en\": {\n \"craft\": \"biomed\",\n \"genia\": \"biomed\",\n \"mimic\": \"mimic\",\n },\n}\n\ndepparse_pretrains = pos_pretrains\n\nner_pretrains = {\n \"ar\": {\n \"aqmar\": \"fasttextwiki\",\n },\n \"de\": {\n \"conll03\": \"fasttextwiki\",\n # the bert version of germeval uses the smaller vector file\n \"germeval2014\": \"fasttextwiki\",\n },\n \"en\": {\n \"anatem\": \"biomed\",\n \"bc4chemd\": \"biomed\",\n \"bc5cdr\": \"biomed\",\n \"bionlp13cg\": \"biomed\",\n \"jnlpba\": \"biomed\",\n \"linnaeus\": \"biomed\",\n \"ncbi_disease\": \"biomed\",\n \"s800\": \"biomed\",\n\n \"ontonotes\": \"fasttextcrawl\",\n # the stanza-train sample NER model should use the default NER pretrain\n # for English, that is the same as ontonotes\n \"sample\": \"fasttextcrawl\",\n\n \"conll03\": \"glove\",\n\n \"i2b2\": \"mimic\",\n \"radiology\": \"mimic\",\n },\n \"es\": {\n \"ancora\": \"fasttextwiki\",\n \"conll02\": \"fasttextwiki\",\n },\n \"nl\": {\n \"conll02\": \"fasttextwiki\",\n \"wikiner\": \"fasttextwiki\",\n },\n \"ru\": {\n \"wikiner\": \"fasttextwiki\",\n },\n \"th\": {\n \"lst20\": \"fasttext157\",\n },\n}\n\n\n# default charlms for languages\ndefault_charlms = {\n \"af\": \"oscar\",\n \"ang\": \"nerthus1024\",\n \"ar\": \"ccwiki\",\n \"bg\": \"conll17\",\n \"da\": \"oscar\",\n \"de\": \"newswiki\",\n \"en\": \"1billion\",\n \"es\": \"newswiki\",\n \"fa\": \"conll17\",\n \"fi\": \"conll17\",\n \"fr\": \"newswiki\",\n \"he\": \"oscar\",\n \"hi\": \"oscar\",\n \"id\": \"oscar2023\",\n \"it\": \"conll17\",\n \"ja\": \"conll17\",\n \"kk\": \"oscar\",\n \"mr\": \"l3cube\",\n \"my\": \"oscar\",\n \"nb\": \"conll17\",\n \"nl\": \"ccwiki\",\n \"pl\": \"oscar\",\n \"pt\": \"oscar2023\",\n \"ru\": \"newswiki\",\n \"sd\": \"isra\",\n \"sv\": \"conll17\",\n \"te\": \"oscar2022\",\n \"th\": \"oscar\",\n \"tr\": \"conll17\",\n \"uk\": \"conll17\",\n \"vi\": \"conll17\",\n \"zh-hans\": \"gigaword\"\n}\n\npos_charlms = {\n \"en\": {\n # none of the English charlms help with craft or genia\n \"craft\": None,\n \"genia\": None,\n \"mimic\": \"mimic\",\n },\n \"tr\": { # no idea why, but this particular one goes down in dev score\n \"boun\": None,\n },\n}\n\ndepparse_charlms = copy.deepcopy(pos_charlms)\n\nlemma_charlms = copy.deepcopy(pos_charlms)\n\ntokenizer_charlms = copy.deepcopy(pos_charlms)\n\nner_charlms = {\n \"en\": {\n \"conll03\": \"1billion\",\n \"ontonotes\": \"1billion\",\n \"anatem\": \"pubmed\",\n \"bc4chemd\": \"pubmed\",\n \"bc5cdr\": \"pubmed\",\n \"bionlp13cg\": \"pubmed\",\n \"i2b2\": \"mimic\",\n \"jnlpba\": \"pubmed\",\n \"linnaeus\": \"pubmed\",\n \"ncbi_disease\": \"pubmed\",\n \"radiology\": \"mimic\",\n \"s800\": \"pubmed\",\n },\n \"hu\": {\n \"combined\": None,\n },\n \"nn\": {\n \"norne\": None,\n },\n}\n\n# default ner for languages\ndefault_ners = {\n \"af\": \"nchlt\",\n \"ang\": \"oedt_charlm\",\n \"ar\": \"aqmar_charlm\",\n \"bg\": \"bsnlp19\",\n \"da\": \"ddt\",\n \"de\": \"germeval2014\",\n \"en\": \"ontonotes-ww-multi_charlm\",\n \"es\": \"conll02\",\n \"fa\": \"arman\",\n \"fi\": \"turku\",\n \"fr\": \"wikinergold_charlm\",\n \"he\": \"iahlt_charlm\",\n \"hi\": \"ilner_charlm\",\n \"hu\": \"combined\",\n \"hy\": \"armtdp\",\n \"it\": \"fbk\",\n \"ja\": \"gsd\",\n \"kk\": \"kazNERD\",\n \"mr\": \"l3cube\",\n \"my\": \"ucsy\",\n \"nb\": \"norne\",\n \"nl\": \"conll02\",\n \"nn\": \"norne\",\n \"pl\": \"nkjp\",\n \"ru\": \"wikiner\",\n \"sd\": \"siner\",\n \"sv\": \"suc3shuffle\",\n \"te\": \"ilner_charlm\",\n \"th\": \"lst20\",\n \"tr\": \"starlang\",\n \"uk\": \"languk\",\n \"ur\": \"ilner_nocharlm\",\n \"vi\": \"vlsp\",\n \"zh-hans\": \"ontonotes\",\n}\n\n# a few languages have sentiment classifier models\ndefault_sentiment = {\n \"en\": \"sstplus_charlm\",\n \"de\": \"sb10k_charlm\",\n \"es\": \"tass2020_charlm\",\n \"mr\": \"l3cube_charlm\",\n \"vi\": \"vsfc_charlm\",\n \"zh-hans\": \"ren_charlm\",\n}\n\n# also, a few languages (very few, currently) have constituency parser models\ndefault_constituency = {\n \"da\": \"arboretum_charlm\",\n \"de\": \"spmrl_charlm\",\n \"en\": \"ptb3-revised_charlm\",\n \"es\": \"combined_charlm\",\n \"id\": \"icon_charlm\",\n \"it\": \"vit_charlm\",\n \"ja\": \"alt_charlm\",\n \"pt\": \"cintil_charlm\",\n #\"tr\": \"starlang_charlm\",\n \"vi\": \"vlsp22_charlm\",\n \"zh-hans\": \"ctb-51_charlm\",\n}\n\noptional_constituency = {\n \"tr\": \"starlang_charlm\",\n}\n\n# an alternate tokenizer for languages which aren't trained from a base UD source\ndefault_tokenizer = {\n \"my\": \"alt\",\n}\n\n# ideally we would have a less expensive model as the base model\n#default_coref = {\n# \"en\": \"ontonotes_roberta-large_finetuned\",\n#}\n\noptional_coref = {\n \"ca\": \"udcoref_xlm-roberta-lora\",\n \"cs\": \"udcoref_xlm-roberta-lora\",\n \"de\": \"udcoref_xlm-roberta-lora\",\n \"en\": \"udcoref_xlm-roberta-lora\",\n \"es\": \"udcoref_xlm-roberta-lora\",\n \"fr\": \"udcoref_xlm-roberta-lora\",\n \"he\": \"iahlt_xlm-roberta-lora\",\n \"hi\": \"deeph_muril-large-cased-lora\",\n # UD Coref has both nb and nn datasets for Norwegian\n \"nb\": \"udcoref_xlm-roberta-lora\",\n \"nn\": \"udcoref_xlm-roberta-lora\",\n \"pl\": \"udcoref_xlm-roberta-lora\",\n \"ru\": \"udcoref_xlm-roberta-lora\",\n \"ta\": \"kbc_muril-large-cased-lora\",\n}\n\n\"\"\"\ndefault transformers to use for various languages\n\nwe try to document why we choose a particular model in each case\n\"\"\"\nTRANSFORMERS = {\n # We tested three candidate AR models on POS, Depparse, and NER\n #\n # POS: padt dev set scores, AllTags\n # depparse: padt dev set scores, LAS\n # NER: dev scores on a random split of AQMAR, entity scores\n #\n # pos depparse ner\n # none (pt & charlm only) 94.08 83.49 84.19\n # asafaya/bert-base-arabic 95.10 84.96 85.98\n # aubmindlab/bert-base-arabertv2 95.33 85.28 84.93\n # aubmindlab/araelectra-base-discriminator 95.66 85.83 86.10\n \"ar\": \"aubmindlab/araelectra-base-discriminator\",\n\n # https://huggingface.co/Maltehb/danish-bert-botxo\n # contrary to normal expectations, this hurts F1\n # on a dev split by about 1 F1\n # \"da\": \"Maltehb/danish-bert-botxo\",\n #\n # the multilingual bert is a marginal improvement for conparse\n #\n # December 2022 update:\n # there are quite a few Danish transformers available on HuggingFace\n # here are the results of training a constituency parser with adadelta/adamw\n # on each of them:\n #\n # no bert 0.8245 0.8230\n # alexanderfalk/danbert-small-cased 0.8236 0.8286\n # Geotrend/distilbert-base-da-cased 0.8268 0.8306\n # sarnikowski/convbert-small-da-cased 0.8322 0.8341\n # bert-base-multilingual-cased 0.8341 0.8342\n # vesteinn/ScandiBERT-no-faroese 0.8373 0.8408\n # Maltehb/danish-bert-botxo 0.8383 0.8408\n # vesteinn/ScandiBERT 0.8421 0.8475\n #\n # Also, two models have token windows too short for use with the\n # Danish dataset:\n # jonfd/electra-small-nordic\n # Maltehb/aelaectra-danish-electra-small-cased\n #\n \"da\": \"vesteinn/ScandiBERT\",\n\n # As of April 2022, the bert models available have a weird\n # tokenizer issue where soft hyphen causes it to crash.\n # We attempt to compensate for that in the dev branch\n #\n # NER scores\n # model dev text\n # xlm-roberta-large 86.56 85.23\n # bert-base-german-cased 87.59 86.95\n # dbmdz/bert-base-german-cased 88.27 87.47\n # german-nlp-group/electra-base-german-uncased 88.60 87.09\n #\n # constituency scores w/ peft, March 2024 model, in-order\n # model dev test\n # xlm-roberta-base 95.17 93.34\n # xlm-roberta-large 95.86 94.46 (!!!)\n # bert-base 95.24 93.24\n # dbmdz/bert 95.32 93.33\n # german/electra 95.72 94.05\n #\n # POS scores\n # model dev test\n # None 88.65 87.28\n # xlm-roberta-large 89.21 88.11\n # bert-base 89.52 88.42\n # dbmdz/bert 89.67 88.54\n # german/electra 89.98 88.66\n #\n # depparse scores, LAS\n # model dev test\n # None 87.76 84.37\n # xlm-roberta-large 89.00 85.79\n # bert-base 88.72 85.40\n # dbmdz/bert 88.70 85.14\n # german/electra 89.21 86.06\n \"de\": \"german-nlp-group/electra-base-german-uncased\",\n\n # experiments on various forms of roberta & electra\n # https://huggingface.co/roberta-base\n # https://huggingface.co/roberta-large\n # https://huggingface.co/google/electra-small-discriminator\n # https://huggingface.co/google/electra-base-discriminator\n # https://huggingface.co/google/electra-large-discriminator\n #\n # experiments using the different models for POS tagging,\n # dev set, including WV and charlm, AllTags score:\n # roberta-base: 95.67\n # roberta-large: 95.98\n # electra-small: 95.31\n # electra-base: 95.90\n # electra-large: 96.01\n #\n # depparse scores, dev set, no finetuning, with WV and charlm\n # UAS LAS CLAS MLAS BLEX\n # roberta-base: 93.16 91.20 89.87 89.38 89.87\n # roberta-large: 93.47 91.56 90.13 89.71 90.13\n # electra-small: 92.17 90.02 88.25 87.66 88.25\n # electra-base: 93.42 91.44 90.10 89.67 90.10\n # electra-large: 94.07 92.17 90.99 90.53 90.99\n #\n # conparse scores, dev & test set, with WV and charlm\n # roberta_base: 96.05 95.60\n # roberta_large: 95.95 95.60\n # electra-small: 95.33 95.04\n # electra-base: 96.09 95.98\n # electra-large: 96.25 96.14\n #\n # conparse scores w/ finetune, dev & test set, with WV and charlm\n # roberta_base: 96.07 95.81\n # roberta_large: 96.37 96.41 (!!!)\n # electra-small: 95.62 95.36\n # electra-base: 96.21 95.94\n # electra-large: 96.40 96.32\n #\n \"en\": \"google/electra-large-discriminator\",\n\n # TODO need to test, possibly compare with others\n \"es\": \"bertin-project/bertin-roberta-base-spanish\",\n\n # NER scores for a couple Persian options:\n # none:\n # dev: 2022-04-23 01:44:53 INFO: fa_arman 79.46\n # test: 2022-04-23 01:45:03 INFO: fa_arman 80.06\n #\n # HooshvareLab/bert-fa-zwnj-base\n # dev: 2022-04-23 02:43:44 INFO: fa_arman 80.87\n # test: 2022-04-23 02:44:07 INFO: fa_arman 80.81\n #\n # HooshvareLab/roberta-fa-zwnj-base\n # dev: 2022-04-23 16:23:25 INFO: fa_arman 81.23\n # test: 2022-04-23 16:23:48 INFO: fa_arman 81.11\n #\n # HooshvareLab/bert-base-parsbert-uncased\n # dev: 2022-04-26 10:42:09 INFO: fa_arman 82.49\n # test: 2022-04-26 10:42:31 INFO: fa_arman 83.16\n \"fa\": 'HooshvareLab/bert-base-parsbert-uncased',\n\n # NER scores for a couple options:\n # none:\n # dev: 2022-03-04 INFO: fi_turku 83.45\n # test: 2022-03-04 INFO: fi_turku 86.25\n #\n # bert-base-multilingual-cased\n # dev: 2022-03-04 INFO: fi_turku 85.23\n # test: 2022-03-04 INFO: fi_turku 89.00\n #\n # TurkuNLP/bert-base-finnish-cased-v1:\n # dev: 2022-03-04 INFO: fi_turku 88.41\n # test: 2022-03-04 INFO: fi_turku 91.36\n \"fi\": \"TurkuNLP/bert-base-finnish-cased-v1\",\n\n # POS dev set tagging results for French:\n # No bert:\n # 98.60 100.00 98.55 98.04\n # dbmdz/electra-base-french-europeana-cased-discriminator\n # 98.70 100.00 98.69 98.24\n # benjamin/roberta-base-wechsel-french\n # 98.71 100.00 98.75 98.26\n # camembert/camembert-large\n # 98.75 100.00 98.75 98.30\n # camembert-base\n # 98.78 100.00 98.77 98.33\n #\n # GSD depparse dev set results for French:\n # No bert:\n # 95.83 94.52 91.34 91.10 91.34\n # camembert/camembert-large\n # 96.80 95.71 93.37 93.13 93.37\n # TODO: the rest of the chart\n \"fr\": \"camembert/camembert-large\",\n\n # Ancient Greek has a surprising number of transformers, considering\n # Model POS Depparse LAS\n # None 0.8812 0.7684\n # Microbert M 0.8883 0.7706\n # Microbert MX 0.8910 0.7755\n # Microbert MXP 0.8916 0.7742\n # Pranaydeeps Bert 0.9139 0.7987\n \"grc\": \"pranaydeeps/Ancient-Greek-BERT\",\n\n # a couple possibilities to experiment with for Hebrew\n # dev scores for POS and depparse\n # https://huggingface.co/imvladikon/alephbertgimmel-base-512\n # UPOS XPOS UFeats AllTags\n # 97.25 97.25 92.84 91.81\n # UAS LAS CLAS MLAS BLEX\n # 94.42 92.47 89.49 88.82 89.49\n #\n # https://huggingface.co/onlplab/alephbert-base\n # UPOS XPOS UFeats AllTags\n # 97.37 97.37 92.50 91.55\n # UAS LAS CLAS MLAS BLEX\n # 94.06 92.12 88.80 88.13 88.80\n #\n # https://huggingface.co/avichr/heBERT\n # UPOS XPOS UFeats AllTags\n # 97.09 97.09 92.36 91.28\n # UAS LAS CLAS MLAS BLEX\n # 94.29 92.30 88.99 88.38 88.99\n \"he\": \"imvladikon/alephbertgimmel-base-512\",\n\n # can also experiment with xlm-roberta\n # on a coref dataset from IITH, span F1:\n # dev test\n # xlm-roberta-large 0.63635 0.66579\n # muril-large 0.65369 0.68290\n \"hi\": \"google/muril-large-cased\",\n\n # https://huggingface.co/xlm-roberta-base\n # Scores by entity for armtdp NER on 18 labels:\n # no bert : 86.68\n # xlm-roberta-base : 89.31\n \"hy\": \"xlm-roberta-base\",\n\n # Indonesian POS experiments: dev set of GSD\n # python3 stanza/utils/training/run_pos.py id_gsd --no_bert\n # python3 stanza/utils/training/run_pos.py id_gsd --bert_model ...\n # also ran on the ICON constituency dataset\n # model POS CON\n # no_bert 89.95 84.74\n # flax-community/indonesian-roberta-large 89.78 (!) xxx\n # flax-community/indonesian-roberta-base 90.14 xxx\n # indobenchmark/indobert-base-p2 90.09\n # indobenchmark/indobert-base-p1 90.14\n # indobenchmark/indobert-large-p1 90.19\n # indolem/indobert-base-uncased 90.21 88.60\n # cahya/bert-base-indonesian-1.5G 90.32 88.15\n # cahya/roberta-base-indonesian-1.5G 90.40 87.27\n \"id\": \"indolem/indobert-base-uncased\",\n\n # from https://github.com/idb-ita/GilBERTo\n # annoyingly, it doesn't handle cased text\n # supposedly there is an argument \"do_lower_case\"\n # but that still leaves a lot of unk tokens\n # \"it\": \"idb-ita/gilberto-uncased-from-camembert\",\n #\n # from https://github.com/musixmatchresearch/umberto\n # on NER, this gets 88.37 dev and 91.02 test\n # another option is dbmdz/bert-base-italian-cased,\n # which gets 87.27 dev and 90.32 test\n #\n # in-order constituency parser on the VIT dev set:\n # dbmdz/bert-base-italian-cased 0.8079\n # dbmdz/bert-base-italian-xxl-cased: 0.8195\n # Musixmatch/umberto-commoncrawl-cased-v1: 0.8256\n # dbmdz/electra-base-italian-xxl-cased-discriminator: 0.8314\n #\n # FBK NER dev set:\n # dbmdz/bert-base-italian-cased: 87.76\n # Musixmatch/umberto-commoncrawl-cased-v1: 88.62\n # dbmdz/bert-base-italian-xxl-cased: 88.84\n # dbmdz/electra-base-italian-xxl-cased-discriminator: 89.91\n #\n # combined UD POS dev set: UPOS XPOS UFeats AllTags\n # dbmdz/bert-base-italian-cased: 98.62 98.53 98.06 97.49\n # dbmdz/bert-base-italian-xxl-cased: 98.61 98.54 98.07 97.58\n # dbmdz/electra-base-italian-xxl-cased-discriminator: 98.64 98.54 98.14 97.61\n # Musixmatch/umberto-commoncrawl-cased-v1: 98.56 98.45 98.13 97.62\n \"it\": \"dbmdz/electra-base-italian-xxl-cased-discriminator\",\n\n # for Japanese\n # there are others that would also work,\n # but they require different tokenizers instead of being\n # plug & play\n #\n # Constitutency scores on ALT (in-order)\n # no bert: 90.68 dev, 91.40 test\n # rinna: 91.54 dev, 91.89 test\n \"ja\": \"rinna/japanese-roberta-base\",\n\n # could also try:\n # l3cube-pune/marathi-bert-v2\n # or\n # https://huggingface.co/l3cube-pune/hindi-marathi-dev-roberta\n # l3cube-pune/hindi-marathi-dev-roberta\n #\n # depparse ufal dev scores:\n # no transformer 74.89 63.70 57.43 53.01 57.43\n # l3cube-pune/marathi-roberta 76.48 66.21 61.20 57.60 61.20\n \"mr\": \"l3cube-pune/marathi-roberta\",\n\n \"or\": \"google/muril-large-cased\",\n\n # https://huggingface.co/allegro/herbert-base-cased\n # Scores by entity on the NKJP NER task:\n # no bert (dev/test): 88.64/88.75\n # herbert-base-cased (dev/test): 91.48/91.02,\n # herbert-large-cased (dev/test): 92.25/91.62\n # sdadas/polish-roberta-large-v2 (dev/test): 92.66/91.22\n \"pl\": \"allegro/herbert-base-cased\",\n\n # experiments on the cintil conparse dataset\n # ran a variety of transformer settings\n # found the following dev set scores after 400 iterations:\n # Geotrend/distilbert-base-pt-cased : not plug & play\n # no bert: 0.9082\n # xlm-roberta-base: 0.9109\n # xlm-roberta-large: 0.9254\n # adalbertojunior/distilbert-portuguese-cased: 0.9300\n # neuralmind/bert-base-portuguese-cased: 0.9307\n # neuralmind/bert-large-portuguese-cased: 0.9343\n \"pt\": \"neuralmind/bert-large-portuguese-cased\",\n\n # hope is actually to build our own using a large text collection\n \"sd\": \"google/muril-large-cased\",\n\n # Tamil options: quite a few, need to run a bunch of experiments\n # dev pos dev depparse las\n # no transformer 82.82 69.12\n # ai4bharat/indic-bert 82.98 70.47\n # lgessler/microbert-tamil-mxp 83.21 69.28\n # monsoon-nlp/tamillion 83.37 69.28\n # l3cube-pune/tamil-bert 85.27 72.53\n # d42kw01f/Tamil-RoBERTa 85.59 70.55\n # google/muril-base-cased 85.67 72.68\n # google/muril-large-cased 86.30 72.45\n #\n # should also consider xlm-roberta-large\n # updated on UD 2.16 data: dev pos ner\n # google/muril-large-cased 86.86 65.08\n # xlm-roberta-large 66.28\n \"ta\": \"google/muril-large-cased\",\n\n \"te\": \"google/muril-large-cased\",\n\n # https://huggingface.co/airesearch/wangchanberta-base-att-spm-uncased\n # this is clearly better than no transformer on a couple datasets:\n #\n # TUD dev upos TUD dev depparse LAS\n # no transformer 91.26 73.57\n # wangchanberta 92.21 76.65\n \"th\": \"airesearch/wangchanberta-base-att-spm-uncased\",\n\n # https://huggingface.co/dbmdz/bert-base-turkish-128k-cased\n # helps the Turkish model quite a bit\n \"tr\": \"dbmdz/bert-base-turkish-128k-cased\",\n\n \"ur\": \"google/muril-large-cased\",\n\n # from https://github.com/VinAIResearch/PhoBERT\n # \"vi\": \"vinai/phobert-base\",\n # using 6 or 7 layers of phobert-large is slightly\n # more effective for constituency parsing than\n # using 4 layers of phobert-base\n # ... going beyond 4 layers of phobert-base\n # does not help the scores\n \"vi\": \"vinai/phobert-large\",\n\n # https://github.com/ymcui/Chinese-BERT-wwm\n # there's also hfl/chinese-roberta-wwm-ext-large\n # or hfl/chinese-electra-base-discriminator\n # or hfl/chinese-electra-180g-large-discriminator,\n # which works better than the below roberta on constituency\n # \"zh-hans\": \"hfl/chinese-roberta-wwm-ext\",\n # conparse dev scores (averaged over 5):\n # google bert: 0.9422\n # hfl bert: 0.9469\n # hfl roberta: 0.9459\n # hfl electra: 0.9515\n # hfl macbert: 0.9530\n # There is also a ShannonAI model, but our current codebase is\n # somehow not compatible\n # further comparing HFL:\n # POS dev Depparse dev LAS NER dev\n # HFL Electra 96.90 85.66 77.90\n # HFL Macbert 96.53 84.72 78.46\n # \"zh-hans\": \"hfl/chinese-macbert-large\",\n \"zh-hans\": \"hfl/chinese-electra-180g-large-discriminator\",\n}\n\nTRANSFORMER_LAYERS = {\n # not clear what the best number is without more experiments,\n # but more than 4 is working better than just 4\n \"vi\": 7,\n}\n\nTRANSFORMER_NICKNAMES = {\n # ar\n \"asafaya/bert-base-arabic\": \"asafaya-bert\",\n \"aubmindlab/araelectra-base-discriminator\": \"aubmind-electra\",\n \"aubmindlab/bert-base-arabertv2\": \"aubmind-bert\",\n\n # da\n \"vesteinn/ScandiBERT\": \"scandibert\",\n\n # de\n \"bert-base-german-cased\": \"bert-base-german-cased\",\n \"dbmdz/bert-base-german-cased\": \"dbmdz-bert-german-cased\",\n \"german-nlp-group/electra-base-german-uncased\": \"german-nlp-electra\",\n\n # en\n \"bert-base-multilingual-cased\": \"mbert\",\n \"xlm-roberta-large\": \"xlm-roberta-large\",\n \"google/electra-large-discriminator\": \"electra-large\",\n \"microsoft/deberta-v3-large\": \"deberta-v3-large\",\n \"princeton-nlp/Sheared-LLaMA-1.3B\": \"sheared-llama-1b3\",\n\n # es\n \"bertin-project/bertin-roberta-base-spanish\": \"bertin-roberta\",\n\n # fa\n \"HooshvareLab/bert-base-parsbert-uncased\": \"parsbert\",\n\n # fi\n \"TurkuNLP/bert-base-finnish-cased-v1\": \"bert\",\n\n # fr\n \"benjamin/roberta-base-wechsel-french\": \"wechsel-roberta\",\n \"camembert-base\": \"camembert-base\",\n \"camembert/camembert-large\": \"camembert-large\",\n \"dbmdz/electra-base-french-europeana-cased-discriminator\": \"dbmdz-electra\",\n\n # grc\n \"pranaydeeps/Ancient-Greek-BERT\": \"grc-pranaydeeps\",\n \"lgessler/microbert-ancient-greek-m\": \"grc-microbert-m\",\n \"lgessler/microbert-ancient-greek-mx\": \"grc-microbert-mx\",\n \"lgessler/microbert-ancient-greek-mxp\": \"grc-microbert-mxp\",\n \"altsoph/bert-base-ancientgreek-uncased\": \"grc-altsoph\",\n\n # he\n \"HeNLP/HeRo\": \"hero-roberta\",\n \"imvladikon/alephbertgimmel-base-512\": \"alephbertgimmel\",\n \"onlplab/alephbert-base\": \"alephbert\",\n\n # hy\n \"xlm-roberta-base\": \"xlm-roberta-base\",\n\n # id\n \"indolem/indobert-base-uncased\": \"indobert\",\n \"indobenchmark/indobert-large-p1\": \"indobenchmark-large-p1\",\n \"indobenchmark/indobert-base-p1\": \"indobenchmark-base-p1\",\n \"indobenchmark/indobert-lite-large-p1\": \"indobenchmark-lite-large-p1\",\n \"indobenchmark/indobert-lite-base-p1\": \"indobenchmark-lite-base-p1\",\n \"indobenchmark/indobert-large-p2\": \"indobenchmark-large-p2\",\n \"indobenchmark/indobert-base-p2\": \"indobenchmark-base-p2\",\n \"indobenchmark/indobert-lite-large-p2\": \"indobenchmark-lite-large-p2\",\n \"indobenchmark/indobert-lite-base-p2\": \"indobenchmark-lite-base-p2\",\n\n # it\n \"dbmdz/electra-base-italian-xxl-cased-discriminator\": \"electra\",\n\n # ja\n \"rinna/japanese-roberta-base\": \"rinna-roberta\",\n\n # mr\n \"l3cube-pune/marathi-roberta\": \"l3cube-marathi-roberta\",\n\n # pl\n \"allegro/herbert-base-cased\": \"herbert\",\n\n # pt\n \"neuralmind/bert-large-portuguese-cased\": \"bertimbau\",\n\n # ta: tamil\n \"monsoon-nlp/tamillion\": \"tamillion\",\n \"lgessler/microbert-tamil-m\": \"ta-microbert-m\",\n \"lgessler/microbert-tamil-mxp\": \"ta-microbert-mxp\",\n \"l3cube-pune/tamil-bert\": \"l3cube-tamil-bert\",\n \"d42kw01f/Tamil-RoBERTa\": \"ta-d42kw01f-roberta\",\n\n # th\n \"airesearch/wangchanberta-base-att-spm-uncased\": \"wangchanberta\",\n\n # tr\n \"dbmdz/bert-base-turkish-128k-cased\": \"bert\",\n\n # vi\n \"vinai/phobert-base\": \"phobert-base\",\n \"vinai/phobert-large\": \"phobert-large\",\n\n # zh\n \"google-bert/bert-base-chinese\": \"google-bert-chinese\",\n \"hfl/chinese-bert-wwm\": \"hfl-bert-chinese\",\n \"hfl/chinese-macbert-large\": \"hfl-macbert-chinese\",\n \"hfl/chinese-roberta-wwm-ext\": \"hfl-roberta-chinese\",\n \"hfl/chinese-electra-180g-large-discriminator\": \"electra-large\",\n \"ShannonAI/ChineseBERT-base\": \"shannonai-chinese-bert\",\n\n # multi-lingual Indic\n \"ai4bharat/indic-bert\": \"indic-bert\",\n \"google/muril-base-cased\": \"muril-base-cased\",\n \"google/muril-large-cased\": \"muril-large-cased\",\n\n # multi-lingual\n \"FacebookAI/xlm-roberta-large\": \"xlm-roberta-large\",\n}\n\ndef known_nicknames():\n \"\"\"\n Return a list of all the transformer nicknames\n\n We return a list so that we can sort them in decreasing key length\n \"\"\"\n nicknames = list(value for key, value in TRANSFORMER_NICKNAMES.items())\n\n # previously unspecific transformers get \"transformer\" as the nickname\n nicknames.append(\"transformer\")\n\n nicknames = sorted(nicknames, key=lambda x: -len(x))\n\n return nicknames\n"
},
{
"path": "stanza/resources/installation.py",
"content": "\"\"\"\nFunctions for setting up the environments.\n\"\"\"\n\nimport os\nimport logging\nimport zipfile\nimport shutil\n\nfrom stanza.resources.common import USER_CACHE_DIR, request_file, unzip, \\\n get_root_from_zipfile, set_logging_level\n\nlogger = logging.getLogger('stanza')\n\nDEFAULT_CORENLP_MODEL_URL = os.getenv(\n 'CORENLP_MODEL_URL',\n 'https://huggingface.co/stanfordnlp/corenlp-{model}/resolve/{tag}/stanford-corenlp-models-{model}.jar'\n)\nBACKUP_CORENLP_MODEL_URL = \"http://nlp.stanford.edu/software/stanford-corenlp-{version}-models-{model}.jar\"\n\nDEFAULT_CORENLP_URL = os.getenv(\n 'CORENLP_MODEL_URL',\n 'https://huggingface.co/stanfordnlp/CoreNLP/resolve/{tag}/stanford-corenlp-latest.zip'\n)\n\nDEFAULT_CORENLP_DIR = os.getenv(\n 'CORENLP_HOME',\n os.path.join(USER_CACHE_DIR, 'corenlp')\n)\n\nAVAILABLE_MODELS = set(['arabic', 'chinese', 'english-extra', 'english-kbp', 'french', 'german', 'hungarian', 'italian', 'spanish'])\n\n\ndef download_corenlp_models(model, version, dir=DEFAULT_CORENLP_DIR, url=DEFAULT_CORENLP_MODEL_URL, logging_level='INFO', proxies=None, force=True):\n \"\"\"\n A automatic way to download the CoreNLP models.\n\n Args:\n model: the name of the model, can be one of 'arabic', 'chinese', 'english',\n 'english-kbp', 'french', 'german', 'hungarian', 'italian', 'spanish'\n version: the version of the model\n dir: the directory to download CoreNLP model into; alternatively can be\n set up with environment variable $CORENLP_HOME\n url: The link to download CoreNLP models.\n It will need {model} and either {version} or {tag} to properly format the URL\n logging_level: logging level to use during installation\n force: Download model anyway, no matter model file exists or not\n \"\"\"\n dir = os.path.expanduser(dir)\n if not model or not version:\n raise ValueError(\n \"Both model and model version should be specified.\"\n )\n logger.info(f\"Downloading {model} models (version {version}) into directory {dir}\")\n model = model.strip().lower()\n if model not in AVAILABLE_MODELS:\n raise KeyError(\n f'{model} is currently not supported. '\n f'Must be one of: {list(AVAILABLE_MODELS)}.'\n )\n # for example:\n # https://huggingface.co/stanfordnlp/CoreNLP/resolve/v4.2.2/stanford-corenlp-models-french.jar\n tag = version if version == 'main' else 'v' + version\n download_url = url.format(tag=tag, model=model, version=version)\n model_path = os.path.join(dir, f'stanford-corenlp-{version}-models-{model}.jar')\n\n if os.path.exists(model_path) and not force:\n logger.warn(\n f\"Model file {model_path} already exists. \"\n f\"Please download this model to a new directory.\")\n return\n\n try:\n request_file(\n download_url,\n model_path,\n proxies\n )\n except (KeyboardInterrupt, SystemExit):\n raise\n except Exception as e:\n raise RuntimeError(\n \"Downloading CoreNLP model file failed. \"\n \"Please try manual downloading at: https://stanfordnlp.github.io/CoreNLP/.\"\n ) from e\n\n\ndef install_corenlp(dir=DEFAULT_CORENLP_DIR, url=DEFAULT_CORENLP_URL, logging_level=None, proxies=None, version=\"main\"):\n \"\"\"\n A fully automatic way to install and setting up the CoreNLP library \n to use the client functionality.\n\n Args:\n dir: the directory to download CoreNLP model into; alternatively can be\n set up with environment variable $CORENLP_HOME\n url: The link to download CoreNLP models\n Needs a {version} or {tag} parameter to specify the version\n logging_level: logging level to use during installation\n \"\"\"\n dir = os.path.expanduser(dir)\n set_logging_level(logging_level=logging_level, verbose=None)\n if os.path.exists(dir) and len(os.listdir(dir)) > 0:\n logger.warn(\n f\"Directory {dir} already exists. \"\n f\"Please install CoreNLP to a new directory.\")\n return\n\n logger.info(f\"Installing CoreNLP package into {dir}\")\n # First download the URL package\n logger.debug(f\"Download to destination file: {os.path.join(dir, 'corenlp.zip')}\")\n tag = version if version == 'main' else 'v' + version\n url = url.format(version=version, tag=tag)\n try:\n request_file(url, os.path.join(dir, 'corenlp.zip'), proxies)\n\n except (KeyboardInterrupt, SystemExit):\n raise\n except Exception as e:\n raise RuntimeError(\n \"Downloading CoreNLP zip file failed. \"\n \"Please try manual installation: https://stanfordnlp.github.io/CoreNLP/.\"\n ) from e\n\n # Unzip corenlp into dir\n logger.debug(\"Unzipping downloaded zip file...\")\n unzip(dir, 'corenlp.zip')\n\n # By default CoreNLP will be unzipped into a version-dependent folder, \n # e.g., stanford-corenlp-4.0.0. We need some hack around that and move\n # files back into our designated folder\n logger.debug(f\"Moving files into the designated folder at: {dir}\")\n corenlp_dirname = get_root_from_zipfile(os.path.join(dir, 'corenlp.zip'))\n corenlp_dirname = os.path.join(dir, corenlp_dirname)\n for f in os.listdir(corenlp_dirname):\n shutil.move(os.path.join(corenlp_dirname, f), dir)\n\n # Remove original zip and folder\n logger.debug(\"Removing downloaded zip file...\")\n os.remove(os.path.join(dir, 'corenlp.zip'))\n shutil.rmtree(corenlp_dirname)\n\n # Warn user to set up env\n if dir != DEFAULT_CORENLP_DIR:\n logger.warning(\n f\"For customized installation location, please set the `CORENLP_HOME` \"\n f\"environment variable to the location of the installation. \"\n f\"In Unix, this is done with `export CORENLP_HOME={dir}`.\")\n\n"
},
{
"path": "stanza/resources/prepare_resources.py",
"content": "\"\"\"\nConverts a directory of models organized by type into a directory organized by language.\n\nAlso produces the resources.json file.\n\nFor example, on the cluster, you can do this:\n\npython3 -m stanza.resources.prepare_resources --input_dir /u/nlp/software/stanza/models/current-models-1.5.0 --output_dir /u/nlp/software/stanza/models/1.5.0 > resources.out 2>&1\nnlprun -a stanza-1.2 -q john \"python3 -m stanza.resources.prepare_resources --input_dir /u/nlp/software/stanza/models/current-models-1.5.0 --output_dir /u/nlp/software/stanza/models/1.5.0\" -o resources.out\n\"\"\"\n\nimport argparse\nfrom collections import defaultdict\nimport json\nimport os\nfrom pathlib import Path\nimport hashlib\nimport shutil\nimport zipfile\n\nfrom stanza import __resources_version__\nfrom stanza.models.common.constant import lcode2lang, two_to_three_letters, three_to_two_letters, extra_lang_to_lcodes\nfrom stanza.resources.default_packages import PACKAGES, TRANSFORMERS, TRANSFORMER_NICKNAMES\nfrom stanza.resources.default_packages import *\nfrom stanza.utils.datasets.prepare_lemma_classifier import DATASET_MAPPING as LEMMA_CLASSIFIER_DATASETS\nfrom stanza.utils.get_tqdm import get_tqdm\n\ntqdm = get_tqdm()\n\ndef parse_args():\n parser = argparse.ArgumentParser()\n parser.add_argument('--input_dir', type=str, default=\"/u/nlp/software/stanza/models/current-models-%s\" % __resources_version__, help='Input dir for various models. Defaults to the recommended home on the nlp cluster')\n parser.add_argument('--output_dir', type=str, default=\"/u/nlp/software/stanza/models/%s\" % __resources_version__, help='Output dir for various models.')\n parser.add_argument('--packages_only', action='store_true', default=False, help='Only build the package maps instead of rebuilding everything')\n parser.add_argument('--lang', type=str, default=None, help='Only process this language or a comma-separated list of languages. If left blank, will prepare all languages. To use this argument, a previous prepared resources with all of the languages is necessary.')\n args = parser.parse_args()\n args.input_dir = os.path.abspath(args.input_dir)\n args.output_dir = os.path.abspath(args.output_dir)\n if args.lang is not None:\n args.lang = \",\".join(args.lang.strip().split())\n return args\n\n\nallowed_empty_languages = [\n # only tokenize and NER for Myanmar right now (soon...)\n \"my\",\n # currently only an NER, not even a tokenizer, for Oriya\n \"or\",\n]\n\n# map processor name to file ending\n# the order of this dict determines the order in which default.zip files are built\n# changing it will necessitate rebuilding all of the default.zip files\n# not a disaster, but it would involve a bunch of uploading\nprocessor_to_ending = {\n \"tokenize\": \"tokenizer\",\n \"mwt\": \"mwt_expander\",\n \"lemma\": \"lemmatizer\",\n \"pos\": \"tagger\",\n \"depparse\": \"parser\",\n \"pretrain\": \"pretrain\",\n \"ner\": \"nertagger\",\n \"forward_charlm\": \"forward_charlm\",\n \"backward_charlm\": \"backward_charlm\",\n \"sentiment\": \"sentiment\",\n \"constituency\": \"constituency\",\n \"coref\": \"coref\",\n \"langid\": \"langid\",\n}\nending_to_processor = {j: i for i, j in processor_to_ending.items()}\nPROCESSORS = list(processor_to_ending.keys())\n\ndef ensure_dir(dir):\n Path(dir).mkdir(parents=True, exist_ok=True)\n\n\ndef copy_file(src, dst):\n ensure_dir(Path(dst).parent)\n shutil.copy2(src, dst)\n\n\ndef get_md5(path):\n data = open(path, 'rb').read()\n return hashlib.md5(data).hexdigest()\n\n\ndef split_model_name(model):\n \"\"\"\n Split model names by _\n\n Takes into account packages with _ and processor types with _\n \"\"\"\n model = model[:-3].replace('.', '_')\n # sort by key length so that nertagger is checked before tagger, for example\n for processor in sorted(ending_to_processor.keys(), key=lambda x: -len(x)):\n if model.endswith(processor):\n model = model[:-(len(processor)+1)]\n processor = ending_to_processor[processor]\n break\n else:\n raise AssertionError(f\"Could not find a processor type in {model}\")\n lang, package = model.split('_', 1)\n return lang, package, processor\n\ndef split_package(package, default_use_charlm=True):\n if package.endswith(\"_finetuned\"):\n package = package[:-10]\n\n if package.endswith(\"_nopretrain\"):\n package = package[:-11]\n return package, False, False\n if package.endswith(\"_nocharlm\"):\n package = package[:-9]\n return package, True, False\n if package.endswith(\"_charlm\"):\n package = package[:-7]\n return package, True, True\n underscore = package.rfind(\"_\")\n if underscore >= 0:\n # +1 to skip the underscore\n nickname = package[underscore+1:]\n if nickname in known_nicknames():\n return package[:underscore], True, True\n\n # guess it was a model which wasn't built with the new naming convention of putting the pretrain type at the end\n # assume WV and charlm... if the language / package doesn't allow for one, that should be caught later\n return package, True, default_use_charlm\n\ndef get_pretrain_package(lang, package, model_pretrains, default_pretrains):\n package, uses_pretrain, _ = split_package(package)\n\n if not uses_pretrain or lang in no_pretrain_languages:\n return None\n elif model_pretrains is not None and lang in model_pretrains and package in model_pretrains[lang]:\n return model_pretrains[lang][package]\n elif lang in default_pretrains:\n return default_pretrains[lang]\n\n raise RuntimeError(\"pretrain not specified for lang %s package %s\" % (lang, package))\n\ndef get_charlm_package(lang, package, model_charlms, default_charlms, default_use_charlm=True):\n package, _, uses_charlm = split_package(package, default_use_charlm)\n\n if not uses_charlm:\n return None\n\n if model_charlms is not None and lang in model_charlms and package in model_charlms[lang]:\n return model_charlms[lang][package]\n else:\n return default_charlms.get(lang, None)\n\ndef get_con_dependencies(lang, package):\n # so far, this invariant is true:\n # constituency models use the default pretrain and charlm for the language\n # sometimes there is no charlm for a language that has constituency, though\n pretrain_package = get_pretrain_package(lang, package, None, default_pretrains)\n dependencies = [{'model': 'pretrain', 'package': pretrain_package}]\n\n charlm_package = default_charlms.get(lang, None)\n if charlm_package is not None:\n dependencies.append({'model': 'forward_charlm', 'package': charlm_package})\n dependencies.append({'model': 'backward_charlm', 'package': charlm_package})\n\n return dependencies\n\ndef get_pos_charlm_package(lang, package):\n return get_charlm_package(lang, package, pos_charlms, default_charlms)\n\ndef get_pos_dependencies(lang, package):\n dependencies = []\n\n pretrain_package = get_pretrain_package(lang, package, pos_pretrains, default_pretrains)\n if pretrain_package is not None:\n dependencies.append({'model': 'pretrain', 'package': pretrain_package})\n\n charlm_package = get_pos_charlm_package(lang, package)\n if charlm_package is not None:\n dependencies.append({'model': 'forward_charlm', 'package': charlm_package})\n dependencies.append({'model': 'backward_charlm', 'package': charlm_package})\n\n return dependencies\n\ndef get_lemma_pretrain_package(lang, package):\n package, uses_pretrain, uses_charlm = split_package(package)\n if not uses_pretrain:\n return None\n if not uses_charlm:\n # currently the contextual lemma classifier is only active\n # for the charlm lemmatizers\n return None\n if \"%s_%s\" % (lang, package) not in LEMMA_CLASSIFIER_DATASETS:\n return None\n return get_pretrain_package(lang, package, {}, default_pretrains)\n\ndef get_lemma_charlm_package(lang, package):\n return get_charlm_package(lang, package, lemma_charlms, default_charlms)\n\ndef get_lemma_dependencies(lang, package):\n dependencies = []\n\n pretrain_package = get_lemma_pretrain_package(lang, package)\n if pretrain_package is not None:\n dependencies.append({'model': 'pretrain', 'package': pretrain_package})\n\n charlm_package = get_lemma_charlm_package(lang, package)\n if charlm_package is not None:\n dependencies.append({'model': 'forward_charlm', 'package': charlm_package})\n dependencies.append({'model': 'backward_charlm', 'package': charlm_package})\n\n return dependencies\n\n\ndef get_tokenizer_charlm_package(lang, package):\n return get_charlm_package(lang, package, tokenizer_charlms, default_charlms, default_use_charlm=False)\n\ndef get_tokenizer_dependencies(lang, package):\n dependencies = []\n charlm_package = get_tokenizer_charlm_package(lang, package)\n if charlm_package is not None:\n dependencies.append({'model': 'forward_charlm', 'package': charlm_package})\n return dependencies\n\ndef get_depparse_charlm_package(lang, package):\n return get_charlm_package(lang, package, depparse_charlms, default_charlms)\n\ndef get_depparse_dependencies(lang, package):\n dependencies = []\n\n pretrain_package = get_pretrain_package(lang, package, depparse_pretrains, default_pretrains)\n if pretrain_package is not None:\n dependencies.append({'model': 'pretrain', 'package': pretrain_package})\n\n charlm_package = get_depparse_charlm_package(lang, package)\n if charlm_package is not None:\n dependencies.append({'model': 'forward_charlm', 'package': charlm_package})\n dependencies.append({'model': 'backward_charlm', 'package': charlm_package})\n\n return dependencies\n\ndef get_ner_charlm_package(lang, package):\n return get_charlm_package(lang, package, ner_charlms, default_charlms)\n\ndef get_ner_pretrain_package(lang, package):\n return get_pretrain_package(lang, package, ner_pretrains, default_pretrains)\n\ndef get_ner_dependencies(lang, package):\n dependencies = []\n\n pretrain_package = get_ner_pretrain_package(lang, package)\n if pretrain_package is not None:\n dependencies.append({'model': 'pretrain', 'package': pretrain_package})\n\n charlm_package = get_ner_charlm_package(lang, package)\n if charlm_package is not None:\n dependencies.append({'model': 'forward_charlm', 'package': charlm_package})\n dependencies.append({'model': 'backward_charlm', 'package': charlm_package})\n\n return dependencies\n\ndef get_sentiment_dependencies(lang, package):\n \"\"\"\n Return a list of dependencies for the sentiment model\n\n Generally this will be pretrain, forward & backward charlm\n So far, this invariant is true:\n sentiment models use the default pretrain for the language\n also, they all use the default charlm for a language\n \"\"\"\n pretrain_package = get_pretrain_package(lang, package, None, default_pretrains)\n dependencies = [{'model': 'pretrain', 'package': pretrain_package}]\n\n charlm_package = default_charlms.get(lang, None)\n if charlm_package is not None:\n dependencies.append({'model': 'forward_charlm', 'package': charlm_package})\n dependencies.append({'model': 'backward_charlm', 'package': charlm_package})\n\n return dependencies\n\ndef get_dependencies(processor, lang, package):\n \"\"\"\n Get the dependencies for a particular lang/package based on the package name\n\n The package can include descriptors such as _nopretrain, _nocharlm, _charlm\n which inform whether or not this particular model uses charlm or pretrain\n \"\"\"\n if processor == 'depparse':\n return get_depparse_dependencies(lang, package)\n elif processor == 'lemma':\n return get_lemma_dependencies(lang, package)\n elif processor == 'pos':\n return get_pos_dependencies(lang, package)\n elif processor == 'ner':\n return get_ner_dependencies(lang, package)\n elif processor == 'sentiment':\n return get_sentiment_dependencies(lang, package)\n elif processor == 'constituency':\n return get_con_dependencies(lang, package)\n elif processor == 'tokenize':\n return get_tokenizer_dependencies(lang, package)\n return {}\n\ndef process_dirs(args):\n dirs = sorted(os.listdir(args.input_dir))\n resources = {}\n if args.lang:\n resources = json.load(open(os.path.join(args.output_dir, 'resources.json')))\n # this one language gets overridden\n # if this is not done, and we reuse the old resources,\n # any models which were deleted will still be in the resources\n for lang in args.lang.split(\",\"):\n resources[lang] = {}\n\n for model_dir in dirs:\n print(f\"Processing models in {model_dir}\")\n models = sorted(os.listdir(os.path.join(args.input_dir, model_dir)))\n for model in tqdm(models):\n if not model.endswith('.pt'): continue\n # get processor\n lang, package, processor = split_model_name(model)\n if args.lang and lang not in args.lang.split(\",\"):\n continue\n\n # copy file\n input_path = os.path.join(args.input_dir, model_dir, model)\n output_path = os.path.join(args.output_dir, lang, \"models\", processor, package + '.pt')\n copy_file(input_path, output_path)\n # maintain md5\n md5 = get_md5(output_path)\n # maintain dependencies\n dependencies = get_dependencies(processor, lang, package)\n # maintain resources\n if lang not in resources: resources[lang] = {}\n if processor not in resources[lang]: resources[lang][processor] = {}\n if dependencies:\n resources[lang][processor][package] = {'md5': md5, 'dependencies': dependencies}\n else:\n resources[lang][processor][package] = {'md5': md5}\n print(\"Processed initial model directories. Writing preliminary resources.json\")\n json.dump(resources, open(os.path.join(args.output_dir, 'resources.json'), 'w'), indent=2)\n\ndef get_default_pos_package(lang, ud_package):\n charlm_package = get_pos_charlm_package(lang, ud_package)\n if charlm_package is not None:\n return ud_package + \"_charlm\"\n if lang in no_pretrain_languages:\n return ud_package + \"_nopretrain\"\n return ud_package + \"_nocharlm\"\n\ndef get_default_depparse_package(lang, ud_package):\n charlm_package = get_depparse_charlm_package(lang, ud_package)\n if charlm_package is not None:\n return ud_package + \"_charlm\"\n if lang in no_pretrain_languages:\n return ud_package + \"_nopretrain\"\n return ud_package + \"_nocharlm\"\n\ndef process_default_zips(args):\n resources = json.load(open(os.path.join(args.output_dir, 'resources.json')))\n for lang in resources:\n # check url, alias, and lang_name in case we are rerunning this step on an already built resources.json\n if lang == 'url':\n continue\n if 'alias' in resources[lang]:\n continue\n if all(k in (\"backward_charlm\", \"forward_charlm\", \"pretrain\", \"lang_name\") for k in resources[lang].keys()):\n continue\n if lang in allowed_empty_languages and lang not in default_treebanks:\n continue\n if lang not in default_treebanks:\n raise AssertionError(f'{lang} not in default treebanks!!!')\n\n if args.lang and lang not in args.lang.split(\",\"):\n continue\n\n print(f'Preparing default models for language {lang}')\n\n models_needed = defaultdict(set)\n\n packages = resources[lang][PACKAGES][\"default\"]\n for processor, package in packages.items():\n if processor == 'lemma' and package == 'identity':\n continue\n if processor == 'optional':\n continue\n models_needed[processor].add(package)\n dependencies = get_dependencies(processor, lang, package)\n for dependency in dependencies:\n models_needed[dependency['model']].add(dependency['package'])\n\n model_files = []\n for processor in PROCESSORS:\n if processor in models_needed:\n for package in sorted(models_needed[processor]):\n filename = os.path.join(args.output_dir, lang, \"models\", processor, package + '.pt')\n if os.path.exists(filename):\n print(\" Model {} package {}: file {}\".format(processor, package, filename))\n model_files.append((filename, processor, package))\n else:\n raise FileNotFoundError(f\"Processor {processor} package {package} needed for {lang} but cannot be found at {filename}\")\n\n with zipfile.ZipFile(os.path.join(args.output_dir, lang, 'models', 'default.zip'), 'w', zipfile.ZIP_DEFLATED) as zipf:\n for filename, processor, package in model_files:\n zipf.write(filename=filename, arcname=os.path.join(processor, package + '.pt'))\n\n default_md5 = get_md5(os.path.join(args.output_dir, lang, 'models', 'default.zip'))\n resources[lang]['default_md5'] = default_md5\n\n print(\"Processed default model zips. Writing resources.json\")\n json.dump(resources, open(os.path.join(args.output_dir, 'resources.json'), 'w'), indent=2)\n\ndef get_default_processors(resources, lang):\n \"\"\"\n Build a default package for this language\n\n Will add each of pos, lemma, depparse, etc if those are available\n Uses the existing models scraped from the language directories into resources.json, as relevant\n \"\"\"\n if lang == \"multilingual\":\n return {\"langid\": \"ud\"}\n\n default_package = default_treebanks[lang]\n default_processors = {}\n if lang in default_tokenizer:\n default_processors['tokenize'] = default_tokenizer[lang]\n else:\n tokenize_package = default_package\n if tokenize_package not in resources[lang]['tokenize']:\n tokenize_package = tokenize_package + \"_nocharlm\"\n if tokenize_package not in resources[lang]['tokenize']:\n raise AssertionError(\"Can't find a tokenizer package for %s! Tried %s and %s\" % (lang, default_package, tokenize_package))\n default_processors['tokenize'] = tokenize_package\n\n if 'mwt' in resources[lang] and default_package in resources[lang]['mwt']:\n # if this doesn't happen, we just skip MWT\n default_processors['mwt'] = default_package\n\n if 'lemma' in resources[lang]:\n expected_lemma = default_package + \"_nocharlm\"\n if expected_lemma in resources[lang]['lemma']:\n default_processors['lemma'] = expected_lemma\n else:\n expected_lemma = default_package + \"_charlm\"\n if expected_lemma in resources[lang]['lemma']:\n default_processors['lemma'] = expected_lemma\n print(\"WARNING: nocharlm lemmatizer for %s model does not exist, but %s does\" % (default_package, expected_lemma))\n elif lang not in allowed_empty_languages:\n default_processors['lemma'] = 'identity'\n\n if 'pos' in resources[lang]:\n default_processors['pos'] = get_default_pos_package(lang, default_package)\n if default_processors['pos'] not in resources[lang]['pos']:\n raise AssertionError(\"Expected POS model not in resources: %s\" % default_processors['pos'])\n elif lang not in allowed_empty_languages:\n raise AssertionError(\"Expected to find POS models for language %s\" % lang)\n\n if 'depparse' in resources[lang]:\n default_processors['depparse'] = get_default_depparse_package(lang, default_package)\n if default_processors['depparse'] not in resources[lang]['depparse']:\n raise AssertionError(\"Expected depparse model not in resources: %s\" % default_processors['depparse'])\n elif lang not in allowed_empty_languages:\n raise AssertionError(\"Expected to find depparse models for language %s\" % lang)\n\n if lang in default_ners:\n default_processors['ner'] = default_ners[lang]\n\n if lang in default_sentiment:\n default_processors['sentiment'] = default_sentiment[lang]\n\n if lang in default_constituency:\n default_processors['constituency'] = default_constituency[lang]\n\n optional = get_default_optional_processors(resources, lang)\n if optional:\n default_processors['optional'] = optional\n\n return default_processors\n\ndef get_default_optional_processors(resources, lang):\n optional_processors = {}\n if lang in optional_constituency:\n optional_processors['constituency'] = optional_constituency[lang]\n\n if lang in optional_coref:\n optional_processors['coref'] = optional_coref[lang]\n\n return optional_processors\n\ndef update_processor_add_transformer(resources, lang, current_processors, processor, transformer):\n if processor not in current_processors:\n return\n\n new_model = current_processors[processor].replace('_charlm', \"_\" + transformer).replace('_nocharlm', \"_\" + transformer)\n if new_model in resources[lang][processor]:\n current_processors[processor] = new_model\n else:\n print(\"WARNING: wanted to use %s for %s accurate %s, but that model does not exist\" % (new_model, lang, processor))\n\ndef get_default_accurate(resources, lang):\n \"\"\"\n A package that, if available, uses charlm and transformer models for each processor\n \"\"\"\n default_processors = get_default_processors(resources, lang)\n\n tokenizer_model = default_processors['tokenize']\n if tokenizer_model.endswith('_nocharlm'):\n tokenizer_model = tokenizer_model.replace('_nocharlm', '_charlm')\n elif 'charlm' not in tokenizer_model:\n tokenizer_model = tokenizer_model + '_charlm'\n if tokenizer_model.endswith('_charlm') and tokenizer_model in resources[lang]['tokenize']:\n default_processors['tokenize'] = tokenizer_model\n print(\"TOKENIZE found a charlm version %s for %s default_accurate\" % (tokenizer_model, lang))\n\n if 'lemma' in default_processors and default_processors['lemma'] != 'identity':\n lemma_model = default_processors['lemma']\n lemma_model = lemma_model.replace('_nocharlm', '_charlm')\n charlm_package = get_lemma_charlm_package(lang, lemma_model)\n if charlm_package is not None:\n if lemma_model in resources[lang]['lemma']:\n default_processors['lemma'] = lemma_model\n else:\n print(\"WARNING: wanted to use %s for %s default_accurate lemma, but that model does not exist\" % (lemma_model, lang))\n\n transformer = TRANSFORMER_NICKNAMES.get(TRANSFORMERS.get(lang, None), None)\n if transformer is not None:\n for processor in ('pos', 'depparse', 'constituency', 'sentiment'):\n update_processor_add_transformer(resources, lang, default_processors, processor, transformer)\n if 'ner' in default_processors and (default_processors['ner'].endswith(\"_charlm\") or default_processors['ner'].endswith(\"_nocharlm\")):\n update_processor_add_transformer(resources, lang, default_processors, \"ner\", transformer)\n\n optional = get_optional_accurate(resources, lang)\n if optional:\n default_processors['optional'] = optional\n\n return default_processors\n\ndef get_optional_accurate(resources, lang):\n optional_processors = get_default_optional_processors(resources, lang)\n\n transformer = TRANSFORMER_NICKNAMES.get(TRANSFORMERS.get(lang, None), None)\n if transformer is not None:\n for processor in ('pos', 'depparse', 'constituency', 'sentiment'):\n update_processor_add_transformer(resources, lang, optional_processors, processor, transformer)\n\n if lang in optional_coref:\n optional_processors['coref'] = optional_coref[lang]\n\n return optional_processors\n\n\ndef get_default_fast(resources, lang):\n \"\"\"\n Build a packages entry which only has the nocharlm models\n\n Will make it easy for people to use the lower tier of models\n\n We do this by building the same default package as normal,\n then switching everything out for the lower tier model when possible.\n We also remove constituency, as it is super slow.\n Note that in the case of a language which doesn't have a charlm,\n that means we wind up building the same for default and default_nocharlm\n \"\"\"\n default_processors = get_default_processors(resources, lang)\n\n # this is a slow model and we don't have non-charlm versions of it yet\n if 'constituency' in default_processors:\n default_processors.pop('constituency')\n\n for processor, model in default_processors.items():\n if \"_charlm\" in model:\n nocharlm = model.replace(\"_charlm\", \"_nocharlm\")\n if nocharlm not in resources[lang][processor]:\n print(\"WARNING: wanted to use %s for %s default_fast processor %s, but that model does not exist\" % (nocharlm, lang, processor))\n else:\n default_processors[processor] = nocharlm\n\n return default_processors\n\ndef process_packages(args):\n \"\"\"\n Build a package for a language's default processors and all of the treebanks specifically used for that language\n \"\"\"\n resources = json.load(open(os.path.join(args.output_dir, 'resources.json')))\n\n for lang in resources:\n # check url, alias, and lang_name in case we are rerunning this step on an already built resources.json\n if lang == 'url':\n continue\n if 'alias' in resources[lang]:\n continue\n if all(k in (\"backward_charlm\", \"forward_charlm\", \"pretrain\", \"lang_name\") for k in resources[lang].keys()):\n continue\n if lang in allowed_empty_languages and lang not in default_treebanks:\n continue\n if lang not in default_treebanks:\n raise AssertionError(f'{lang} not in default treebanks!!!')\n\n if args.lang and lang not in args.lang.split(\",\"):\n continue\n\n default_processors = get_default_processors(resources, lang)\n\n # TODO: eventually we can remove default_processors\n # For now, we want to keep this so that v1.5.1 is compatible\n # with the next iteration of resources files\n resources[lang]['default_processors'] = default_processors\n resources[lang][PACKAGES] = {}\n resources[lang][PACKAGES]['default'] = default_processors\n\n if lang not in no_pretrain_languages and lang != \"multilingual\":\n default_fast = get_default_fast(resources, lang)\n resources[lang][PACKAGES]['default_fast'] = default_fast\n\n default_accurate = get_default_accurate(resources, lang)\n resources[lang][PACKAGES]['default_accurate'] = default_accurate\n\n # Now we loop over each of the tokenizers for this language\n # ... we use this as a proxy for the available UD treebanks\n # This loop also catches things such as \"craft\" which are\n # included treebanks that aren't UD\n # We then create a package in the packages dict for each of those treebanks\n if 'tokenize' in resources[lang]:\n for package in resources[lang]['tokenize']:\n package, _, _ = split_package(package)\n if package in resources[lang][PACKAGES]:\n # can happen in the case of a _nocharlm and _charlm version of the tokenizer\n continue\n\n processors = {}\n # TODO: when we rebuild all the models, make all the tokenizers say _nocharlm\n if package in resources[lang]['tokenize']:\n processors[\"tokenize\"] = package\n elif package + \"_nocharlm\" in resources[lang]['tokenize']:\n processors[\"tokenize\"] = package + \"_nocharlm\"\n else:\n raise AssertionError(\"Should have found a tokenizer for lang %s package %s\" % (lang, package))\n\n if \"mwt\" in resources[lang] and package in resources[lang][\"mwt\"]:\n processors[\"mwt\"] = package\n\n if \"pos\" in resources[lang]:\n if package + \"_charlm\" in resources[lang][\"pos\"]:\n processors[\"pos\"] = package + \"_charlm\"\n elif package + \"_nocharlm\" in resources[lang][\"pos\"]:\n processors[\"pos\"] = package + \"_nocharlm\"\n\n if \"lemma\" in resources[lang] and \"pos\" in processors:\n lemma_package = package + \"_nocharlm\"\n if lemma_package in resources[lang][\"lemma\"]:\n processors[\"lemma\"] = lemma_package\n else:\n lemma_package = package + \"_charlm\"\n if lemma_package in resources[lang]['lemma']:\n processors['lemma'] = lemma_package\n print(\"WARNING: nocharlm lemmatizer for %s model does not exist, but %s does\" % (package, lemma_package))\n\n if \"depparse\" in resources[lang] and \"pos\" in processors:\n depparse_package = None\n if package + \"_charlm\" in resources[lang][\"depparse\"]:\n depparse_package = package + \"_charlm\"\n elif package + \"_nocharlm\" in resources[lang][\"depparse\"]:\n depparse_package = package + \"_nocharlm\"\n # we want to set the lemma first if it's identity\n # THEN set the depparse\n if depparse_package is not None:\n if \"lemma\" not in processors:\n processors[\"lemma\"] = \"identity\"\n processors[\"depparse\"] = depparse_package\n\n resources[lang][PACKAGES][package] = processors\n\n print(\"Processed packages. Writing resources.json\")\n json.dump(resources, open(os.path.join(args.output_dir, 'resources.json'), 'w'), indent=2)\n\ndef process_lcode(args):\n resources = json.load(open(os.path.join(args.output_dir, 'resources.json')))\n resources_new = {}\n resources_new[\"multilingual\"] = resources[\"multilingual\"]\n for lang in resources:\n if lang == 'multilingual':\n continue\n if 'alias' in resources[lang]:\n continue\n if lang not in lcode2lang:\n print(lang + ' not found in lcode2lang!')\n continue\n lang_name = lcode2lang[lang]\n resources[lang]['lang_name'] = lang_name\n resources_new[lang.lower()] = resources[lang.lower()]\n resources_new[lang_name.lower()] = {'alias': lang.lower()}\n if lang.lower() in two_to_three_letters:\n resources_new[two_to_three_letters[lang.lower()]] = {'alias': lang.lower()}\n elif lang.lower() in three_to_two_letters:\n resources_new[three_to_two_letters[lang.lower()]] = {'alias': lang.lower()}\n if lang.lower() in extra_lang_to_lcodes:\n alternative = extra_lang_to_lcodes[lang.lower()].lower()\n if alternative not in resources_new:\n resources_new[alternative] = {'alias': lang.lower()}\n print(\"Processed lcode aliases. Writing resources.json\")\n json.dump(resources_new, open(os.path.join(args.output_dir, 'resources.json'), 'w'), indent=2)\n\n\ndef process_misc(args):\n resources = json.load(open(os.path.join(args.output_dir, 'resources.json')))\n resources['no'] = {'alias': 'nb'}\n resources['zh'] = {'alias': 'zh-hans'}\n # This is intended to be unformatted. expand_model_url in common.py will fill in the raw string\n # with the appropriate values in order to find the needed model file on huggingface\n resources['url'] = 'https://huggingface.co/stanfordnlp/stanza-{lang}/resolve/v{resources_version}/models/{filename}'\n print(\"Finalized misc attributes. Writing resources.json\")\n json.dump(resources, open(os.path.join(args.output_dir, 'resources.json'), 'w'), indent=2)\n\n\ndef main():\n args = parse_args()\n print(\"Converting models from %s to %s\" % (args.input_dir, args.output_dir))\n if not args.packages_only:\n process_dirs(args)\n process_packages(args)\n if not args.packages_only:\n process_default_zips(args)\n process_lcode(args)\n process_misc(args)\n\n\nif __name__ == '__main__':\n main()\n\n"
},
{
"path": "stanza/resources/print_charlm_depparse.py",
"content": "\"\"\"\nA small utility script to output which depparse models use charlm\n\n(It should skip en_genia, en_craft, but currently doesn't)\n\nNot frequently useful, but seems like the kind of thing that might get used a couple times\n\"\"\"\n\nfrom stanza.resources.common import load_resources_json\nfrom stanza.resources.default_packages import default_charlms, depparse_charlms\n\ndef list_depparse():\n charlm_langs = list(default_charlms.keys())\n resources = load_resources_json()\n\n models = [\"%s_%s\" % (lang, model) for lang in charlm_langs for model in resources[lang].get(\"depparse\", {})\n if lang not in depparse_charlms or model not in depparse_charlms[lang] or depparse_charlms[lang][model] is not None]\n return models\n\nif __name__ == \"__main__\":\n models = list_depparse()\n print(\" \".join(models))\n"
},
{
"path": "stanza/server/__init__.py",
"content": "from stanza.protobuf import to_text\nfrom stanza.protobuf import Document, Sentence, Token, IndexedWord, Span\nfrom stanza.protobuf import ParseTree, DependencyGraph, CorefChain\nfrom stanza.protobuf import Mention, NERMention, Entity, Relation, RelationTriple, Timex\nfrom stanza.protobuf import Quote, SpeakerInfo\nfrom stanza.protobuf import Operator, Polarity\nfrom stanza.protobuf import SentenceFragment, TokenLocation\nfrom stanza.protobuf import MapStringString, MapIntString\nfrom .client import CoreNLPClient, AnnotationException, TimeoutException, PermanentlyFailedException, StartServer\nfrom .annotator import Annotator\n"
},
{
"path": "stanza/server/annotator.py",
"content": "\"\"\"\nDefines a base class that can be used to annotate.\n\"\"\"\nimport io\nfrom multiprocessing import Process\nfrom http.server import BaseHTTPRequestHandler, HTTPServer\nfrom http import client as HTTPStatus\n\nfrom stanza.protobuf import Document, parseFromDelimitedString, writeToDelimitedString\n\nclass Annotator(Process):\n \"\"\"\n This annotator base class hosts a lightweight server that accepts\n annotation requests from CoreNLP.\n Each annotator simply defines 3 functions: requires, provides and annotate.\n\n This class takes care of defining appropriate endpoints to interface\n with CoreNLP.\n \"\"\"\n @property\n def name(self):\n \"\"\"\n Name of the annotator (used by CoreNLP)\n \"\"\"\n raise NotImplementedError()\n\n @property\n def requires(self):\n \"\"\"\n Requires has to specify all the annotations required before we\n are called.\n \"\"\"\n raise NotImplementedError()\n\n @property\n def provides(self):\n \"\"\"\n The set of annotations guaranteed to be provided when we are done.\n NOTE: that these annotations are either fully qualified Java\n class names or refer to nested classes of\n edu.stanford.nlp.ling.CoreAnnotations (as is the case below).\n \"\"\"\n raise NotImplementedError()\n\n def annotate(self, ann):\n \"\"\"\n @ann: is a protobuf annotation object.\n Actually populate @ann with tokens.\n \"\"\"\n raise NotImplementedError()\n\n @property\n def properties(self):\n \"\"\"\n Defines a Java property to define this annotator to CoreNLP.\n \"\"\"\n return {\n \"customAnnotatorClass.{}\".format(self.name): \"edu.stanford.nlp.pipeline.GenericWebServiceAnnotator\",\n \"generic.endpoint\": \"http://{}:{}\".format(self.host, self.port),\n \"generic.requires\": \",\".join(self.requires),\n \"generic.provides\": \",\".join(self.provides),\n }\n\n class _Handler(BaseHTTPRequestHandler):\n annotator = None\n\n def __init__(self, request, client_address, server):\n BaseHTTPRequestHandler.__init__(self, request, client_address, server)\n\n def do_GET(self):\n \"\"\"\n Handle a ping request\n \"\"\"\n if not self.path.endswith(\"/\"): self.path += \"/\"\n if self.path == \"/ping/\":\n msg = \"pong\".encode(\"UTF-8\")\n\n self.send_response(HTTPStatus.OK)\n self.send_header(\"Content-Type\", \"text/application\")\n self.send_header(\"Content-Length\", len(msg))\n self.end_headers()\n self.wfile.write(msg)\n else:\n self.send_response(HTTPStatus.BAD_REQUEST)\n self.end_headers()\n\n def do_POST(self):\n \"\"\"\n Handle an annotate request\n \"\"\"\n if not self.path.endswith(\"/\"): self.path += \"/\"\n if self.path == \"/annotate/\":\n # Read message\n length = int(self.headers.get('content-length'))\n msg = self.rfile.read(length)\n\n # Do the annotation\n doc = Document()\n parseFromDelimitedString(doc, msg)\n self.annotator.annotate(doc)\n\n with io.BytesIO() as stream:\n writeToDelimitedString(doc, stream)\n msg = stream.getvalue()\n\n # write message\n self.send_response(HTTPStatus.OK)\n self.send_header(\"Content-Type\", \"application/x-protobuf\")\n self.send_header(\"Content-Length\", len(msg))\n self.end_headers()\n self.wfile.write(msg)\n\n else:\n self.send_response(HTTPStatus.BAD_REQUEST)\n self.end_headers()\n\n def __init__(self, host=\"\", port=8432):\n \"\"\"\n Launches a server endpoint to communicate with CoreNLP\n \"\"\"\n Process.__init__(self)\n self.host, self.port = host, port\n self._Handler.annotator = self\n\n def run(self):\n \"\"\"\n Runs the server using Python's simple HTTPServer.\n TODO: make this multithreaded.\n \"\"\"\n httpd = HTTPServer((self.host, self.port), self._Handler)\n sa = httpd.socket.getsockname()\n serve_message = \"Serving HTTP on {host} port {port} (http://{host}:{port}/) ...\"\n print(serve_message.format(host=sa[0], port=sa[1]))\n try:\n httpd.serve_forever()\n except KeyboardInterrupt:\n print(\"\\nKeyboard interrupt received, exiting.\")\n httpd.shutdown()\n"
},
{
"path": "stanza/server/client.py",
"content": "\"\"\"\nClient for accessing Stanford CoreNLP in Python\n\"\"\"\n\nimport atexit\nimport contextlib\nimport enum\nimport io\nimport os\nimport re\nimport requests\nimport logging\nimport json\nimport shlex\nimport socket\nimport subprocess\nimport time\nimport sys\nimport uuid\n\nfrom datetime import datetime\nfrom pathlib import Path\nfrom urllib.parse import urlparse\n\nfrom stanza.protobuf import Document, parseFromDelimitedString, writeToDelimitedString, to_text\n__author__ = 'arunchaganty, kelvinguu, vzhong, wmonroe4'\n\nlogger = logging.getLogger('stanza')\n\n# pattern tmp props file should follow\nSERVER_PROPS_TMP_FILE_PATTERN = re.compile('corenlp_server-(.*).props')\n\n# Check if str is CoreNLP supported language\nCORENLP_LANGS = ['ar', 'arabic', 'chinese', 'zh', 'english', 'en', 'french', 'fr', 'de', 'german', 'hu', 'hungarian',\n 'it', 'italian', 'es', 'spanish']\n\n# map shorthands to full language names\nLANGUAGE_SHORTHANDS_TO_FULL = {\n \"ar\": \"arabic\",\n \"zh\": \"chinese\",\n \"en\": \"english\",\n \"fr\": \"french\",\n \"de\": \"german\",\n \"hu\": \"hungarian\",\n \"it\": \"italian\",\n \"es\": \"spanish\"\n}\n\n\ndef is_corenlp_lang(props_str):\n \"\"\" Check if a string references a CoreNLP language \"\"\"\n return props_str.lower() in CORENLP_LANGS\n\n\n# Validate CoreNLP properties\nCORENLP_OUTPUT_VALS = [\"conll\", \"conllu\", \"json\", \"serialized\", \"text\", \"xml\", \"inlinexml\"]\n\n\ndef validate_corenlp_props(properties=None, annotators=None, output_format=None):\n \"\"\" Do basic checks to validate CoreNLP properties \"\"\"\n if output_format and output_format.lower() not in CORENLP_OUTPUT_VALS:\n raise ValueError(f\"{output_format} not a valid CoreNLP outputFormat value! Choose from: {CORENLP_OUTPUT_VALS}\")\n if type(properties) == dict:\n if \"outputFormat\" in properties and properties[\"outputFormat\"].lower() not in CORENLP_OUTPUT_VALS:\n raise ValueError(f\"{properties['outputFormat']} not a valid CoreNLP outputFormat value! Choose from: \"\n f\"{CORENLP_OUTPUT_VALS}\")\n\n\nclass AnnotationException(Exception):\n \"\"\" Exception raised when there was an error communicating with the CoreNLP server. \"\"\"\n pass\n\n\nclass TimeoutException(AnnotationException):\n \"\"\" Exception raised when the CoreNLP server timed out. \"\"\"\n pass\n\n\nclass ShouldRetryException(Exception):\n \"\"\" Exception raised if the service should retry the request. \"\"\"\n pass\n\n\nclass PermanentlyFailedException(Exception):\n \"\"\" Exception raised if the service should NOT retry the request. \"\"\"\n pass\n\nclass StartServer(enum.Enum):\n DONT_START = 0\n FORCE_START = 1\n TRY_START = 2\n\n\ndef clean_props_file(props_file):\n # check if there is a temp server props file to remove and remove it\n if props_file:\n if os.path.isfile(props_file) and SERVER_PROPS_TMP_FILE_PATTERN.match(os.path.basename(props_file)):\n os.remove(props_file)\n\n\nclass RobustService(object):\n \"\"\" Service that resuscitates itself if it is not available. \"\"\"\n CHECK_ALIVE_TIMEOUT = 120\n\n def __init__(self, start_cmd, stop_cmd, endpoint, stdout=None,\n stderr=None, be_quiet=False, host=None, port=None, ignore_binding_error=False):\n self.start_cmd = start_cmd and shlex.split(start_cmd)\n self.stop_cmd = stop_cmd and shlex.split(stop_cmd)\n self.endpoint = endpoint\n self.stdout = stdout\n self.stderr = stderr\n\n self.server = None\n self.is_active = False\n self.be_quiet = be_quiet\n self.host = host\n self.port = port\n self.ignore_binding_error = ignore_binding_error\n atexit.register(self.atexit_kill)\n\n def is_alive(self):\n try:\n if not self.ignore_binding_error and self.server is not None and self.server.poll() is not None:\n return False\n return requests.get(self.endpoint + \"/ping\").ok\n except requests.exceptions.ConnectionError as e:\n raise ShouldRetryException(e)\n\n def start(self):\n if self.start_cmd:\n if self.host and self.port:\n with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:\n try:\n sock.bind((self.host, self.port))\n except socket.error as e:\n if self.ignore_binding_error:\n logger.info(f\"Connecting to existing CoreNLP server at {self.host}:{self.port}\")\n self.server = None\n return\n else:\n raise PermanentlyFailedException(\"Error: unable to start the CoreNLP server on port %d \"\n \"(possibly something is already running there)\" % self.port) from e\n if self.be_quiet:\n # Issue #26: subprocess.DEVNULL isn't supported in python 2.7.\n if hasattr(subprocess, 'DEVNULL'):\n stderr = subprocess.DEVNULL\n else:\n stderr = open(os.devnull, 'w')\n stdout = stderr\n else:\n stdout = self.stdout\n stderr = self.stderr\n logger.info(f\"Starting server with command: {' '.join(self.start_cmd)}\")\n try:\n self.server = subprocess.Popen(self.start_cmd,\n stderr=stderr,\n stdout=stdout)\n except FileNotFoundError as e:\n raise FileNotFoundError(\"When trying to run CoreNLP, a FileNotFoundError occurred, which frequently means Java was not installed or was not in the classpath.\") from e\n\n def atexit_kill(self):\n # make some kind of effort to stop the service (such as a\n # CoreNLP server) at the end of the program. not waiting so\n # that the python script exiting isn't delayed\n if self.server and self.server.poll() is None:\n self.server.terminate()\n\n def stop(self):\n if self.server:\n self.server.terminate()\n try:\n self.server.wait(5)\n except subprocess.TimeoutExpired:\n # Resorting to more aggressive measures...\n self.server.kill()\n try:\n self.server.wait(5)\n except subprocess.TimeoutExpired:\n # oh well\n pass\n self.server = None\n if self.stop_cmd:\n subprocess.run(self.stop_cmd, check=True)\n self.is_active = False\n\n def __enter__(self):\n self.start()\n return self\n\n def __exit__(self, _, __, ___):\n self.stop()\n\n def ensure_alive(self):\n # Check if the service is active and alive\n if self.is_active:\n try:\n if self.is_alive():\n return\n else:\n self.stop()\n except ShouldRetryException:\n pass\n\n # If not, try to start up the service.\n if self.server is None:\n self.start()\n\n # Wait for the service to start up.\n start_time = time.time()\n while True:\n try:\n if self.is_alive():\n break\n except ShouldRetryException:\n pass\n\n if time.time() - start_time < self.CHECK_ALIVE_TIMEOUT:\n time.sleep(1)\n else:\n raise PermanentlyFailedException(\"Timed out waiting for service to come alive.\")\n\n # At this point we are guaranteed that the service is alive.\n self.is_active = True\n\n\ndef resolve_classpath(classpath=None):\n \"\"\"\n Returns the classpath to use for corenlp.\n\n Prefers to use the given classpath parameter, if available. If\n not, uses the CORENLP_HOME environment variable. Resolves $CLASSPATH\n (the exact string) in either the classpath parameter or $CORENLP_HOME.\n \"\"\"\n if classpath == '$CLASSPATH' or (classpath is None and os.getenv(\"CORENLP_HOME\", None) == '$CLASSPATH'):\n classpath = os.getenv(\"CLASSPATH\")\n elif classpath is None:\n classpath = os.getenv(\"CORENLP_HOME\", os.path.join(str(Path.home()), 'stanza_corenlp'))\n\n if not os.path.exists(classpath):\n raise FileNotFoundError(\"Please install CoreNLP by running `stanza.install_corenlp()`. If you have installed it, please define \"\n \"$CORENLP_HOME to be location of your CoreNLP distribution or pass in a classpath parameter. \"\n \"$CORENLP_HOME={}\".format(os.getenv(\"CORENLP_HOME\")))\n classpath = os.path.join(classpath, \"*\")\n return classpath\n\n\nclass CoreNLPClient(RobustService):\n \"\"\" A client to the Stanford CoreNLP server. \"\"\"\n\n DEFAULT_ENDPOINT = \"http://localhost:9000\"\n DEFAULT_TIMEOUT = 60000\n DEFAULT_THREADS = 5\n DEFAULT_OUTPUT_FORMAT = \"serialized\"\n DEFAULT_MEMORY = \"5G\"\n DEFAULT_MAX_CHAR_LENGTH = 100000\n\n def __init__(self, start_server=StartServer.FORCE_START,\n endpoint=DEFAULT_ENDPOINT,\n timeout=DEFAULT_TIMEOUT,\n threads=DEFAULT_THREADS,\n annotators=None,\n pretokenized=False,\n output_format=None,\n properties=None,\n stdout=None,\n stderr=None,\n memory=DEFAULT_MEMORY,\n be_quiet=False,\n max_char_length=DEFAULT_MAX_CHAR_LENGTH,\n preload=True,\n classpath=None,\n **kwargs):\n\n # whether or not server should be started by client\n self.start_server = start_server\n self.server_props_path = None\n self.server_start_time = None\n self.server_host = None\n self.server_port = None\n self.server_classpath = None\n # validate properties\n validate_corenlp_props(properties=properties, annotators=annotators, output_format=output_format)\n # set up client defaults\n self.properties = properties\n self.annotators = annotators\n self.pretokenized = pretokenized\n self.output_format = output_format\n self._setup_client_defaults()\n # start the server\n if isinstance(start_server, bool):\n warning_msg = f\"Setting 'start_server' to a boolean value when constructing {self.__class__.__name__} is deprecated and will stop\" + \\\n \" to function in a future version of stanza. Please consider switching to using a value from stanza.server.StartServer.\"\n logger.warning(warning_msg)\n start_server = StartServer.FORCE_START if start_server is True else StartServer.DONT_START\n\n # start the server\n if start_server is StartServer.FORCE_START or start_server is StartServer.TRY_START:\n # record info for server start\n self.server_start_time = datetime.now()\n # set up default properties for server\n self._setup_server_defaults()\n host, port = urlparse(endpoint).netloc.split(\":\")\n port = int(port)\n assert host == \"localhost\", \"If starting a server, endpoint must be localhost\"\n classpath = resolve_classpath(classpath)\n start_cmd = f\"java -Xmx{memory} -cp '{classpath}' edu.stanford.nlp.pipeline.StanfordCoreNLPServer \" \\\n f\"-port {port} -timeout {timeout} -threads {threads} -maxCharLength {max_char_length} \" \\\n f\"-quiet {be_quiet} \"\n\n self.server_classpath = classpath\n self.server_host = host\n self.server_port = port\n\n # set up server defaults\n if self.server_props_path is not None:\n start_cmd += f\" -serverProperties {self.server_props_path}\"\n\n # possibly set pretokenized\n if self.pretokenized:\n start_cmd += f\" -preTokenized\"\n\n # set annotators for server default\n if self.annotators is not None:\n annotators_str = self.annotators if type(annotators) == str else \",\".join(annotators)\n start_cmd += f\" -annotators {annotators_str}\"\n\n # specify what to preload, if anything\n if preload:\n if type(preload) == bool:\n # -preload flag means to preload all default annotators\n start_cmd += \" -preload\"\n elif type(preload) == list:\n # turn list into comma separated list string, only preload these annotators\n start_cmd += f\" -preload {','.join(preload)}\"\n elif type(preload) == str:\n # comma separated list of annotators\n start_cmd += f\" -preload {preload}\"\n\n # set outputFormat for server default\n # if no output format requested by user, set to serialized\n start_cmd += f\" -outputFormat {self.output_format}\"\n\n # additional options for server:\n # - server_id\n # - ssl\n # - status_port\n # - uriContext\n # - strict\n # - key\n # - username\n # - password\n # - blockList\n for kw in ['ssl', 'strict']:\n if kwargs.get(kw) is not None:\n start_cmd += f\" -{kw}\"\n for kw in ['status_port', 'uriContext', 'key', 'username', 'password', 'blockList', 'server_id']:\n if kwargs.get(kw) is not None:\n start_cmd += f\" -{kw} {kwargs.get(kw)}\"\n stop_cmd = None\n else:\n start_cmd = stop_cmd = None\n host = port = None\n\n super(CoreNLPClient, self).__init__(start_cmd, stop_cmd, endpoint,\n stdout, stderr, be_quiet, host=host, port=port, ignore_binding_error=(start_server == StartServer.TRY_START))\n\n self.timeout = timeout\n\n def _setup_client_defaults(self):\n \"\"\"\n Do some processing of annotators and output_format specified for the client.\n If interacting with an externally started server, these will be defaults for annotate() calls.\n :return: None\n \"\"\"\n # normalize annotators to str\n if self.annotators is not None:\n self.annotators = self.annotators if type(self.annotators) == str else \",\".join(self.annotators)\n\n # handle case where no output format is specified\n if self.output_format is None:\n if type(self.properties) == dict and 'outputFormat' in self.properties:\n self.output_format = self.properties['outputFormat']\n else:\n self.output_format = CoreNLPClient.DEFAULT_OUTPUT_FORMAT\n\n def _setup_server_defaults(self):\n \"\"\"\n Set up the default properties for the server.\n\n The properties argument can take on one of 3 value types\n\n 1. File path on system or in CLASSPATH (e.g. /path/to/server.props or StanfordCoreNLP-french.properties\n 2. Name of a Stanford CoreNLP supported language (e.g. french or fr)\n 3. Python dictionary (properties written to tmp file for Java server, erased at end)\n\n In addition, an annotators list and output_format can be specified directly with arguments. These\n will overwrite any settings in the specified properties.\n\n If no properties are specified, the standard Stanford CoreNLP English server will be launched. The outputFormat\n will be set to 'serialized' and use the ProtobufAnnotationSerializer.\n \"\"\"\n\n # ensure properties is str or dict\n if self.properties is None or (not isinstance(self.properties, str) and not isinstance(self.properties, dict)):\n if self.properties is not None:\n logger.warning('properties passed invalid value (not a str or dict), setting properties = {}')\n self.properties = {}\n # check if properties is a string, pass on to server which can handle\n if isinstance(self.properties, str):\n # try to translate to Stanford CoreNLP language name, or assume properties is a path\n if is_corenlp_lang(self.properties):\n if self.properties.lower() in LANGUAGE_SHORTHANDS_TO_FULL:\n self.properties = LANGUAGE_SHORTHANDS_TO_FULL[self.properties]\n logger.info(\n f\"Using CoreNLP default properties for: {self.properties}. Make sure to have \"\n f\"{self.properties} models jar (available for download here: \"\n f\"https://stanfordnlp.github.io/CoreNLP/) in CLASSPATH\")\n else:\n if not os.path.isfile(self.properties):\n logger.warning(f\"{self.properties} does not correspond to a file path. Make sure this file is in \"\n f\"your CLASSPATH.\")\n self.server_props_path = self.properties\n elif isinstance(self.properties, dict):\n # make a copy\n server_start_properties = dict(self.properties)\n if self.annotators is not None:\n server_start_properties['annotators'] = self.annotators\n if self.output_format is not None and isinstance(self.output_format, str):\n server_start_properties['outputFormat'] = self.output_format\n # write desired server start properties to tmp file\n # set up to erase on exit\n tmp_path = write_corenlp_props(server_start_properties)\n logger.info(f\"Writing properties to tmp file: {tmp_path}\")\n atexit.register(clean_props_file, tmp_path)\n self.server_props_path = tmp_path\n\n def _request(self, buf, properties, reset_default=False, **kwargs):\n \"\"\"\n Send a request to the CoreNLP server.\n\n :param (str | bytes) buf: data to be sent with the request\n :param (dict) properties: properties that the server expects\n :return: request result\n \"\"\"\n if self.start_server is not StartServer.DONT_START:\n self.ensure_alive()\n\n try:\n input_format = properties.get(\"inputFormat\", \"text\")\n if input_format == \"text\":\n ctype = \"text/plain; charset=utf-8\"\n elif input_format == \"serialized\":\n ctype = \"application/x-protobuf\"\n else:\n raise ValueError(\"Unrecognized inputFormat \" + input_format)\n # handle auth\n if 'username' in kwargs and 'password' in kwargs:\n kwargs['auth'] = requests.auth.HTTPBasicAuth(kwargs['username'], kwargs['password'])\n kwargs.pop('username')\n kwargs.pop('password')\n r = requests.post(self.endpoint,\n params={'properties': str(properties), 'resetDefault': str(reset_default).lower()},\n data=buf, headers={'content-type': ctype},\n timeout=(self.timeout*2)/1000, **kwargs)\n r.raise_for_status()\n return r\n except requests.exceptions.Timeout as e:\n raise TimeoutException(\"Timeout requesting to CoreNLPServer. Maybe server is unavailable or your document is too long\")\n except requests.exceptions.RequestException as e:\n if e.response is not None and e.response.text is not None:\n raise AnnotationException(e.response.text) from e\n elif e.args:\n raise AnnotationException(e.args[0]) from e\n raise AnnotationException() from e\n\n def annotate(self, text, annotators=None, output_format=None, properties=None, reset_default=None, **kwargs):\n \"\"\"\n Send a request to the CoreNLP server.\n\n :param (str | unicode) text: raw text for the CoreNLPServer to parse\n :param (list | string) annotators: list of annotators to use\n :param (str) output_format: output type from server: serialized, json, text, conll, conllu, or xml\n :param (dict) properties: additional request properties (written on top of defaults)\n :param (bool) reset_default: don't use server defaults\n\n Precedence for settings:\n\n 1. annotators and output_format args\n 2. Values from properties dict\n 3. Client defaults self.annotators and self.output_format (set during client construction)\n 4. Server defaults\n\n Additional request parameters (apart from CoreNLP pipeline properties) such as 'username' and 'password'\n can be specified with the kwargs.\n\n :return: request result\n \"\"\"\n\n # validate request properties\n validate_corenlp_props(properties=properties, annotators=annotators, output_format=output_format)\n # set request properties\n request_properties = {}\n\n # start with client defaults\n if self.annotators is not None:\n request_properties['annotators'] = self.annotators\n if self.output_format is not None:\n request_properties['outputFormat'] = self.output_format\n\n # add values from properties arg\n # handle str case\n if type(properties) == str:\n if is_corenlp_lang(properties):\n properties = {'pipelineLanguage': properties.lower()}\n if reset_default is None:\n reset_default = True\n else:\n raise ValueError(f\"Unrecognized properties keyword {properties}\")\n\n if type(properties) == dict:\n request_properties.update(properties)\n\n # if annotators list is specified, override with that\n # also can use the annotators field the object was created with\n if annotators is not None and (type(annotators) == str or type(annotators) == list):\n request_properties['annotators'] = annotators if type(annotators) == str else \",\".join(annotators)\n\n # if output format is specified, override with that\n if output_format is not None and type(output_format) == str:\n request_properties['outputFormat'] = output_format\n\n # make the request\n # if not explicitly set or the case of pipelineLanguage, reset_default should be None\n if reset_default is None:\n reset_default = False\n r = self._request(text.encode('utf-8'), request_properties, reset_default, **kwargs)\n if request_properties[\"outputFormat\"] == \"json\":\n return r.json()\n elif request_properties[\"outputFormat\"] == \"serialized\":\n doc = Document()\n parseFromDelimitedString(doc, r.content)\n return doc\n elif request_properties[\"outputFormat\"] in [\"text\", \"conllu\", \"conll\", \"xml\"]:\n return r.text\n else:\n return r\n\n def update(self, doc, annotators=None, properties=None):\n if properties is None:\n properties = {}\n properties.update({\n 'inputFormat': 'serialized',\n 'outputFormat': 'serialized',\n 'serializer': 'edu.stanford.nlp.pipeline.ProtobufAnnotationSerializer'\n })\n if annotators:\n properties['annotators'] = annotators if type(annotators) == str else \",\".join(annotators)\n with io.BytesIO() as stream:\n writeToDelimitedString(doc, stream)\n msg = stream.getvalue()\n\n r = self._request(msg, properties)\n doc = Document()\n parseFromDelimitedString(doc, r.content)\n return doc\n\n def tokensregex(self, text, pattern, filter=False, to_words=False, annotators=None, properties=None):\n # this is required for some reason\n matches = self.__regex('/tokensregex', text, pattern, filter, annotators, properties)\n if to_words:\n matches = regex_matches_to_indexed_words(matches)\n return matches\n\n def semgrex(self, text, pattern, filter=False, to_words=False, annotators=None, properties=None):\n matches = self.__regex('/semgrex', text, pattern, filter, annotators, properties)\n if to_words:\n matches = regex_matches_to_indexed_words(matches)\n return matches\n\n def fill_tree_proto(self, tree, proto_tree):\n if tree.label:\n proto_tree.value = tree.label\n for child in tree.children:\n proto_child = proto_tree.child.add()\n self.fill_tree_proto(child, proto_child)\n\n def tregex(self, text=None, pattern=None, filter=False, annotators=None, properties=None, trees=None):\n # parse is not included by default in some of the pipelines,\n # so we may need to manually override the annotators\n # to include parse in order for tregex to do anything\n if annotators is None and self.annotators is not None:\n assert isinstance(self.annotators, str)\n pieces = self.annotators.split(\",\")\n if \"parse\" not in pieces:\n annotators = self.annotators + \",parse\"\n else:\n annotators = \"tokenize,ssplit,pos,parse\"\n if pattern is None:\n raise ValueError(\"Cannot have None as a pattern for tregex\")\n\n # TODO: we could also allow for passing in a complete document,\n # along with the original text, so that the spans returns are more accurate\n if trees is not None:\n if properties is None:\n properties = {}\n properties['inputFormat'] = 'serialized'\n if 'serializer' not in properties:\n properties['serializer'] = 'edu.stanford.nlp.pipeline.ProtobufAnnotationSerializer'\n doc = Document()\n full_text = []\n for tree_idx, tree in enumerate(trees):\n sentence = doc.sentence.add()\n sentence.sentenceIndex = tree_idx\n sentence.tokenOffsetBegin = len(full_text)\n leaves = tree.leaf_labels()\n full_text.extend(leaves)\n sentence.tokenOffsetEnd = len(full_text)\n self.fill_tree_proto(tree, sentence.parseTree)\n for word in leaves:\n token = sentence.token.add()\n # the other side uses both value and word, weirdly enough\n token.value = word\n token.word = word\n # without the actual tokenization, at least we can\n # stop the words from running together\n token.after = \" \"\n doc.text = \" \".join(full_text)\n with io.BytesIO() as stream:\n writeToDelimitedString(doc, stream)\n text = stream.getvalue()\n\n return self.__regex('/tregex', text, pattern, filter, annotators, properties)\n\n def __regex(self, path, text, pattern, filter, annotators=None, properties=None):\n \"\"\"\n Send a regex-related request to the CoreNLP server.\n\n :param (str | unicode) path: the path for the regex endpoint\n :param text: raw text for the CoreNLPServer to apply the regex\n :param (str | unicode) pattern: regex pattern\n :param (bool) filter: option to filter sentences that contain matches, if false returns matches\n :param properties: option to filter sentences that contain matches, if false returns matches\n :return: request result\n \"\"\"\n if self.start_server is not StartServer.DONT_START:\n self.ensure_alive()\n if properties is None:\n properties = {}\n properties.update({\n 'inputFormat': 'text',\n 'serializer': 'edu.stanford.nlp.pipeline.ProtobufAnnotationSerializer'\n })\n if annotators:\n properties['annotators'] = \",\".join(annotators) if isinstance(annotators, list) else annotators\n\n # force output for regex requests to be json\n properties['outputFormat'] = 'json'\n # if the server is trying to send back character offsets, it\n # should send back codepoints counts as well in case the text\n # has extra wide characters\n properties['tokenize.codepoint'] = 'true'\n\n try:\n # Error occurs unless put properties in params\n input_format = properties.get(\"inputFormat\", \"text\")\n if input_format == \"text\":\n ctype = \"text/plain; charset=utf-8\"\n elif input_format == \"serialized\":\n ctype = \"application/x-protobuf\"\n else:\n raise ValueError(\"Unrecognized inputFormat \" + input_format)\n # change request method from `get` to `post` as required by CoreNLP\n r = requests.post(\n self.endpoint + path, params={\n 'pattern': pattern,\n 'filter': filter,\n 'properties': str(properties)\n },\n data=text.encode('utf-8') if isinstance(text, str) else text,\n headers={'content-type': ctype},\n timeout=(self.timeout*2)/1000,\n )\n r.raise_for_status()\n if r.encoding is None:\n r.encoding = \"utf-8\"\n return json.loads(r.text)\n except requests.HTTPError as e:\n if r.text.startswith(\"Timeout\"):\n raise TimeoutException(r.text)\n else:\n raise AnnotationException(r.text)\n except json.JSONDecodeError:\n raise AnnotationException(r.text)\n\n\n def scenegraph(self, text, properties=None):\n \"\"\"\n Send a request to the server which processes the text using SceneGraph\n\n This will require a new CoreNLP release, 4.5.5 or later\n \"\"\"\n # since we're using requests ourself,\n # check if the server has started or not\n if self.start_server is not StartServer.DONT_START:\n self.ensure_alive()\n\n if properties is None:\n properties = {}\n # the only thing the scenegraph knows how to use is text\n properties['inputFormat'] = 'text'\n ctype = \"text/plain; charset=utf-8\"\n # the json output format is much more useful\n properties['outputFormat'] = 'json'\n try:\n r = requests.post(\n self.endpoint + \"/scenegraph\",\n params={\n 'properties': str(properties)\n },\n data=text.encode('utf-8') if isinstance(text, str) else text,\n headers={'content-type': ctype},\n timeout=(self.timeout*2)/1000,\n )\n r.raise_for_status()\n if r.encoding is None:\n r.encoding = \"utf-8\"\n return json.loads(r.text)\n except requests.HTTPError as e:\n if r.text.startswith(\"Timeout\"):\n raise TimeoutException(r.text)\n else:\n raise AnnotationException(r.text)\n except json.JSONDecodeError:\n raise AnnotationException(r.text)\n\n\ndef read_corenlp_props(props_path):\n \"\"\" Read a Stanford CoreNLP properties file into a dict \"\"\"\n props_dict = {}\n with open(props_path) as props_file:\n entry_lines = [entry_line for entry_line in props_file.read().split('\\n')\n if entry_line.strip() and not entry_line.startswith('#')]\n for entry_line in entry_lines:\n k = entry_line.split('=')[0]\n k_len = len(k+\"=\")\n v = entry_line[k_len:]\n props_dict[k.strip()] = v\n return props_dict\n\n\ndef write_corenlp_props(props_dict, file_path=None):\n \"\"\" Write a Stanford CoreNLP properties dict to a file \"\"\"\n if file_path is None:\n file_path = f\"corenlp_server-{uuid.uuid4().hex[:16]}.props\"\n # confirm tmp file path matches pattern\n assert SERVER_PROPS_TMP_FILE_PATTERN.match(file_path)\n with open(file_path, 'w') as props_file:\n for k, v in props_dict.items():\n if isinstance(v, list):\n writeable_v = \",\".join(v)\n else:\n writeable_v = v\n props_file.write(f'{k} = {writeable_v}\\n\\n')\n return file_path\n\n\ndef regex_matches_to_indexed_words(matches):\n \"\"\"\n Transforms tokensregex and semgrex matches to indexed words.\n :param matches: unprocessed regex matches\n :return: flat array of indexed words\n \"\"\"\n words = [dict(v, **dict([('sentence', i)]))\n for i, s in enumerate(matches['sentences'])\n for k, v in s.items() if k != 'length']\n return words\n\n\n__all__ = [\"CoreNLPClient\", \"AnnotationException\", \"TimeoutException\", \"to_text\"]\n"
},
{
"path": "stanza/server/dependency_converter.py",
"content": "\"\"\"\nA converter from constituency trees to dependency trees using CoreNLP's UniversalEnglish converter.\n\nONLY works on English.\n\"\"\"\n\nimport stanza\nfrom stanza.protobuf import DependencyConverterRequest, DependencyConverterResponse\nfrom stanza.server.java_protobuf_requests import send_request, build_tree, JavaProtobufContext\n\nCONVERTER_JAVA = \"edu.stanford.nlp.trees.ProcessDependencyConverterRequest\"\n\ndef send_converter_request(request, classpath=None):\n return send_request(request, DependencyConverterResponse, CONVERTER_JAVA, classpath=classpath)\n\ndef build_request(doc):\n \"\"\"\n Request format is simple: one tree per sentence in the document\n \"\"\"\n request = DependencyConverterRequest()\n for sentence in doc.sentences:\n request.trees.append(build_tree(sentence.constituency, None))\n return request\n\ndef process_doc(doc, classpath=None):\n \"\"\"\n Convert the constituency trees in the document,\n then attach the resulting dependencies to the sentences\n \"\"\"\n request = build_request(doc)\n response = send_converter_request(request, classpath=classpath)\n attach_dependencies(doc, response)\n\ndef attach_dependencies(doc, response):\n if len(doc.sentences) != len(response.conversions):\n raise ValueError(\"Sent %d sentences but got back %d conversions\" % (len(doc.sentences), len(response.conversions)))\n for sent_idx, (sentence, conversion) in enumerate(zip(doc.sentences, response.conversions)):\n graph = conversion.graph\n\n # The deterministic conversion should have an equal number of words and one fewer edge\n # ... the root is represented by a word with no parent\n if len(sentence.words) != len(graph.node):\n raise ValueError(\"Sentence %d of the conversion should have %d words but got back %d nodes in the graph\" % (sent_idx, len(sentence.words), len(graph.node))) \n if len(sentence.words) != len(graph.edge) + 1:\n raise ValueError(\"Sentence %d of the conversion should have %d edges (one per word, plus the root) but got back %d edges in the graph\" % (sent_idx, len(sentence.words) - 1, len(graph.edge)))\n\n expected_nodes = set(range(1, len(sentence.words) + 1))\n targets = set()\n for edge in graph.edge:\n if edge.target in targets:\n raise ValueError(\"Found two parents of %d in sentence %d\" % (edge.target, sent_idx))\n targets.add(edge.target)\n # -1 since the words are 0 indexed in the sentence,\n # but we count dependencies from 1\n sentence.words[edge.target-1].head = edge.source\n sentence.words[edge.target-1].deprel = edge.dep\n roots = expected_nodes - targets\n assert len(roots) == 1\n for root in roots:\n sentence.words[root-1].head = 0\n sentence.words[root-1].deprel = \"root\"\n sentence.build_dependencies()\n\n\nclass DependencyConverter(JavaProtobufContext):\n \"\"\"\n Context window for the dependency converter\n\n This is a context window which keeps a process open. Should allow\n for multiple requests without launching new java processes each time.\n \"\"\"\n def __init__(self, classpath=None):\n super(DependencyConverter, self).__init__(classpath, DependencyConverterResponse, CONVERTER_JAVA)\n\n def process(self, doc):\n \"\"\"\n Converts a constituency tree to dependency trees for each of the sentences in the document\n \"\"\"\n request = build_request(doc)\n response = self.process_request(request)\n attach_dependencies(doc, response)\n return doc\n\ndef main():\n nlp = stanza.Pipeline('en',\n processors='tokenize,pos,constituency')\n\n doc = nlp('I like blue antennae.')\n print(\"{:C}\".format(doc))\n process_doc(doc, classpath=\"$CLASSPATH\")\n print(\"{:C}\".format(doc))\n\n doc = nlp('And I cannot lie.')\n print(\"{:C}\".format(doc))\n with DependencyConverter(classpath=\"$CLASSPATH\") as converter:\n converter.process(doc)\n print(\"{:C}\".format(doc))\n\n\nif __name__ == '__main__':\n main()\n"
},
{
"path": "stanza/server/java_protobuf_requests.py",
"content": "from collections import deque\nimport subprocess\n\nfrom stanza.models.common.utils import misc_to_space_after\nfrom stanza.models.constituency.parse_tree import Tree\nfrom stanza.protobuf import DependencyGraph, FlattenedParseTree\nfrom stanza.server.client import resolve_classpath\n\ndef send_request(request, response_type, java_main, classpath=None):\n \"\"\"\n Use subprocess to run a Java protobuf processor on the given request\n\n Returns the protobuf response\n \"\"\"\n classpath = resolve_classpath(classpath)\n if classpath is None:\n raise ValueError(\"Classpath is None, Perhaps you need to set the $CLASSPATH or $CORENLP_HOME environment variable to point to a CoreNLP install.\")\n pipe = subprocess.run([\"java\", \"-cp\", classpath, java_main],\n input=request.SerializeToString(),\n stdout=subprocess.PIPE,\n check=True)\n response = response_type()\n response.ParseFromString(pipe.stdout)\n return response\n\ndef add_tree_nodes(proto_tree, tree, score):\n # add an open node\n node = proto_tree.nodes.add()\n node.openNode = True\n if score is not None:\n node.score = score\n\n # add the content of this node\n node = proto_tree.nodes.add()\n node.value = tree.label\n\n # add all children...\n # leaves get just one node\n # branches are called recursively\n for child in tree.children:\n if child.is_leaf():\n node = proto_tree.nodes.add()\n node.value = child.label\n else:\n add_tree_nodes(proto_tree, child, None)\n\n node = proto_tree.nodes.add()\n node.closeNode = True\n\ndef build_tree(tree, score):\n \"\"\"\n Builds a FlattenedParseTree from CoreNLP.proto\n\n Populates the value field from tree.label and iterates through the\n children via tree.children. Should work on any tree structure\n which follows that layout\n\n The score will be added to the top node (if it is not None)\n\n Operates by recursively calling add_tree_nodes\n \"\"\"\n proto_tree = FlattenedParseTree()\n add_tree_nodes(proto_tree, tree, score)\n return proto_tree\n\ndef from_tree(proto_tree):\n \"\"\"\n Convert a FlattenedParseTree back into a Tree\n\n returns Tree, score\n (score might be None if it is missing)\n \"\"\"\n score = None\n stack = deque()\n for node in proto_tree.nodes:\n if node.HasField(\"score\") and score is None:\n score = node.score\n\n if node.openNode:\n if len(stack) > 0 and isinstance(stack[-1], FlattenedParseTree.Node) and stack[-1].openNode:\n raise ValueError(\"Got a proto with no label on a node: {}\".format(proto_tree))\n stack.append(node)\n continue\n if not node.closeNode:\n child = Tree(label=node.value)\n # TODO: do something with the score\n stack.append(child)\n continue\n\n # must be a close operation...\n if len(stack) <= 1:\n raise ValueError(\"Got a proto with too many close operations: {}\".format(proto_tree))\n # on a close operation, pop until we hit the open\n # then turn everything in that span into a new node\n children = []\n nextNode = stack.pop()\n while not isinstance(nextNode, FlattenedParseTree.Node):\n children.append(nextNode)\n nextNode = stack.pop()\n if len(children) == 0:\n raise ValueError(\"Got a proto with an open immediately followed by a close: {}\".format(proto_tree))\n children.reverse()\n label = children[0]\n children = children[1:]\n subtree = Tree(label=label.label, children=children)\n stack.append(subtree)\n\n if len(stack) > 1:\n raise ValueError(\"Got a proto which does not close all of the nodes: {}\".format(proto_tree))\n tree = stack.pop()\n if not isinstance(tree, Tree):\n raise ValueError(\"Got a proto which was just one Open operation: {}\".format(proto_tree))\n return tree, score\n\ndef add_token(token_list, word, token):\n \"\"\"\n Add a token to a proto request.\n\n CoreNLP tokens have components of both word and token from stanza.\n\n We pass along \"after\" but not \"before\"\n \"\"\"\n if token is None and isinstance(word.id, int):\n raise AssertionError(\"Only expected word w/o token for 'extra' words\")\n\n query_token = token_list.add()\n query_token.word = word.text\n query_token.value = word.text\n if word.lemma is not None:\n query_token.lemma = word.lemma\n if word.xpos is not None:\n query_token.pos = word.xpos\n if word.upos is not None:\n query_token.coarseTag = word.upos\n if word.feats and word.feats != \"_\":\n for feature in word.feats.split(\"|\"):\n key, value = feature.split(\"=\", maxsplit=1)\n query_token.conllUFeatures.key.append(key)\n query_token.conllUFeatures.value.append(value)\n if token is not None:\n if token.ner is not None:\n query_token.ner = token.ner\n if token is not None and len(token.id) > 1:\n query_token.mwtText = token.text\n query_token.isMWT = True\n query_token.isFirstMWT = token.id[0] == word.id\n if token.id[-1] != word.id:\n # if we are not the last word of an MWT token\n # we are absolutely not followed by space\n pass\n else:\n query_token.after = token.spaces_after\n\n query_token.index = word.id\n else:\n # presumably empty words won't really be written this way,\n # but we can still keep track of it\n query_token.after = misc_to_space_after(word.misc)\n\n query_token.index = word.id[0]\n query_token.emptyIndex = word.id[1]\n\n if word.misc and word.misc != \"_\":\n query_token.conllUMisc = word.misc\n if token is not None and token.misc and token.misc != \"_\":\n query_token.mwtMisc = token.misc\n\ndef add_sentence(request_sentences, sentence, num_tokens):\n \"\"\"\n Add the tokens for this stanza sentence to a list of protobuf sentences\n \"\"\"\n request_sentence = request_sentences.add()\n request_sentence.tokenOffsetBegin = num_tokens\n request_sentence.tokenOffsetEnd = num_tokens + sum(len(token.words) for token in sentence.tokens)\n for token in sentence.tokens:\n for word in token.words:\n add_token(request_sentence.token, word, token)\n return request_sentence\n\ndef add_word_to_graph(graph, word, sent_idx):\n \"\"\"\n Add a node and possibly an edge for a word in a basic dependency graph.\n \"\"\"\n node = graph.node.add()\n node.sentenceIndex = sent_idx+1\n if isinstance(word.id, int):\n node.index = word.id\n else:\n node.index = word.id[0]\n node.emptyIndex = word.id[1]\n\n if word.head != 0 and word.head is not None:\n edge = graph.edge.add()\n edge.source = word.head\n if isinstance(word.id, int):\n edge.target = word.id\n else:\n edge.target = word.id[0]\n edge.targetEmpty = word.id[1]\n if word.deprel is not None:\n edge.dep = word.deprel\n else:\n # the receiving side doesn't like null as a dependency\n edge.dep = \"_\"\n\ndef convert_networkx_graph(graph_proto, sentence, sent_idx):\n \"\"\"\n Turns a networkx graph into a DependencyGraph from the proto file\n \"\"\"\n for token in sentence.tokens:\n for word in token.words:\n add_token(graph_proto.token, word, token)\n for word in sentence.empty_words:\n add_token(graph_proto.token, word, None)\n\n dependencies = sentence._enhanced_dependencies\n for target in dependencies:\n if target == 0:\n # don't need to send the explicit root\n continue\n for source in dependencies.predecessors(target):\n if source == 0:\n # unlike with basic, we need to send over the roots,\n # as the enhanced can have loops\n graph_proto.rootNode.append(len(graph_proto.node))\n continue\n for deprel in dependencies.get_edge_data(source, target):\n edge = graph_proto.edge.add()\n if isinstance(source, int):\n edge.source = source\n else:\n edge.source = source[0]\n if source[1] != 0:\n edge.sourceEmpty = source[1]\n if isinstance(target, int):\n edge.target = target\n else:\n edge.target = target[0]\n if target[1] != 0:\n edge.targetEmpty = target[1]\n edge.dep = deprel\n node = graph_proto.node.add()\n node.sentenceIndex = sent_idx + 1\n # the nodes in the networkx graph are indexed from 1, not counting the root\n if isinstance(target, int):\n node.index = target\n else:\n node.index = target[0]\n if target[1] != 0:\n node.emptyIndex = target[1]\n return graph_proto\n\ndef features_to_string(features):\n if not features:\n return None\n if len(features.key) == 0:\n return None\n return \"|\".join(\"%s=%s\" % (key, value) for key, value in zip(features.key, features.value))\n\ndef misc_space_pieces(misc):\n \"\"\"\n Return only the space-related misc pieces\n \"\"\"\n if misc is None or misc == \"\" or misc == \"_\":\n return misc\n pieces = misc.split(\"|\")\n pieces = [x for x in pieces if x.split(\"=\", maxsplit=1)[0] in (\"SpaceAfter\", \"SpacesAfter\", \"SpacesBefore\")]\n if len(pieces) > 0:\n return \"|\".join(pieces)\n return None\n\ndef remove_space_misc(misc):\n \"\"\"\n Remove any pieces from misc which are space-related\n \"\"\"\n if misc is None or misc == \"\" or misc == \"_\":\n return misc\n pieces = misc.split(\"|\")\n pieces = [x for x in pieces if x.split(\"=\", maxsplit=1)[0] not in (\"SpaceAfter\", \"SpacesAfter\", \"SpacesBefore\")]\n if len(pieces) > 0:\n return \"|\".join(pieces)\n return None\n\ndef substitute_space_misc(misc, space_misc):\n space_misc_pieces = space_misc.split(\"|\") if space_misc else []\n space_misc_after = None\n space_misc_before = None\n for piece in space_misc_pieces:\n if piece.startswith(\"SpaceBefore\"):\n space_misc_before = piece\n elif piece.startswith(\"SpaceAfter\") or piece.startswith(\"SpacesAfter\"):\n space_misc_after = piece\n else:\n raise AssertionError(\"An unknown piece wound up in the misc space fields: %s\" % piece)\n\n pieces = misc.split(\"|\")\n new_pieces = []\n for piece in pieces:\n if piece.startswith(\"SpaceBefore\"):\n if space_misc_before:\n new_pieces.append(space_misc_before)\n space_misc_before = None\n elif piece.startswith(\"SpaceAfter\") or piece.startswith(\"SpacesAfter\"):\n if space_misc_after:\n new_pieces.append(space_misc_after)\n space_misc_after = None\n else:\n new_pieces.append(piece)\n if space_misc_after:\n new_pieces.append(space_misc_after)\n if space_misc_before:\n new_pieces.append(space_misc_before)\n if len(new_pieces) == 0:\n return None\n return \"|\".join(new_pieces)\n\nclass JavaProtobufContext(object):\n \"\"\"\n A generic context for sending requests to a java program using protobufs in a subprocess\n \"\"\"\n def __init__(self, classpath, build_response, java_main, extra_args=None):\n self.classpath = resolve_classpath(classpath)\n self.build_response = build_response\n self.java_main = java_main\n\n if extra_args is None:\n extra_args = []\n self.extra_args = extra_args\n self.pipe = None\n\n def open_pipe(self):\n self.pipe = subprocess.Popen([\"java\", \"-cp\", self.classpath, self.java_main, \"-multiple\"] + self.extra_args,\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE)\n\n def close_pipe(self):\n if self.pipe.poll() is None:\n self.pipe.stdin.write((0).to_bytes(4, 'big'))\n self.pipe.stdin.flush()\n self.pipe = None\n\n def __enter__(self):\n self.open_pipe()\n return self\n\n def __exit__(self, type, value, traceback):\n self.close_pipe()\n\n def process_request(self, request):\n if self.pipe is None:\n raise RuntimeError(\"Pipe to java process is not open or was closed\")\n\n text = request.SerializeToString()\n self.pipe.stdin.write(len(text).to_bytes(4, 'big'))\n self.pipe.stdin.write(text)\n self.pipe.stdin.flush()\n response_length = self.pipe.stdout.read(4)\n if len(response_length) < 4:\n raise BrokenPipeError(\"Could not communicate with java process!\")\n response_length = int.from_bytes(response_length, \"big\")\n response_text = self.pipe.stdout.read(response_length)\n response = self.build_response()\n response.ParseFromString(response_text)\n return response\n\n"
},
{
"path": "stanza/server/main.py",
"content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\"\"\"\nSimple shell program to pipe in \n\"\"\"\n\nimport corenlp\n\nimport json\nimport re\nimport csv\nimport sys\nfrom collections import namedtuple, OrderedDict\n\nFLOAT_RE = re.compile(r\"\\d*\\.\\d+\")\nINT_RE = re.compile(r\"\\d+\")\n\ndef dictstr(arg):\n \"\"\"\n Parse a key=value string as a tuple (key, value) that can be provided as an argument to dict()\n \"\"\"\n key, value = arg.split(\"=\")\n\n if value.lower() == \"true\" or value.lower() == \"false\":\n value = bool(value)\n elif INT_RE.match(value):\n value = int(value)\n elif FLOAT_RE.match(value):\n value = float(value)\n return (key, value)\n\n\ndef do_annotate(args):\n args.props = dict(args.props) if args.props else {}\n if args.sentence_mode:\n args.props[\"ssplit.isOneSentence\"] = True\n\n with corenlp.CoreNLPClient(annotators=args.annotators, properties=args.props, be_quiet=not args.verbose_server) as client:\n for line in args.input:\n if line.startswith(\"#\"): continue\n\n ann = client.annotate(line.strip(), output_format=args.format)\n\n if args.format == \"json\":\n if args.sentence_mode:\n ann = ann[\"sentences\"][0]\n\n args.output.write(json.dumps(ann))\n args.output.write(\"\\n\")\n\ndef main():\n import argparse\n parser = argparse.ArgumentParser(description='Annotate data')\n parser.add_argument('-i', '--input', type=argparse.FileType('r'), default=sys.stdin, help=\"Input file to process; each line contains one document (default: stdin)\")\n parser.add_argument('-o', '--output', type=argparse.FileType('w'), default=sys.stdout, help=\"File to write annotations to (default: stdout)\")\n parser.add_argument('-f', '--format', choices=[\"json\",], default=\"json\", help=\"Output format\")\n parser.add_argument('-a', '--annotators', nargs=\"+\", type=str, default=[\"tokenize ssplit lemma pos\"], help=\"A list of annotators\")\n parser.add_argument('-s', '--sentence-mode', action=\"store_true\",help=\"Assume each line of input is a sentence.\")\n parser.add_argument('-v', '--verbose-server', action=\"store_true\",help=\"Server is made verbose\")\n parser.add_argument('-m', '--memory', type=str, default=\"4G\", help=\"Memory to use for the server\")\n parser.add_argument('-p', '--props', nargs=\"+\", type=dictstr, help=\"Properties as a list of key=value pairs\")\n parser.set_defaults(func=do_annotate)\n\n ARGS = parser.parse_args()\n if ARGS.func is None:\n parser.print_help()\n sys.exit(1)\n else:\n ARGS.func(ARGS)\n\nif __name__ == \"__main__\":\n main()\n"
},
{
"path": "stanza/server/morphology.py",
"content": "\"\"\"\nDirect pipe connection to the Java CoreNLP Morphology class\n\nOnly effective for English. Must be supplied with PTB scheme xpos, not upos\n\"\"\"\n\n\nfrom stanza.protobuf import MorphologyRequest, MorphologyResponse\nfrom stanza.server.java_protobuf_requests import send_request, JavaProtobufContext\n\n\nMORPHOLOGY_JAVA = \"edu.stanford.nlp.process.ProcessMorphologyRequest\"\n\ndef send_morphology_request(request):\n return send_request(request, MorphologyResponse, MORPHOLOGY_JAVA)\n\ndef build_request(words, xpos_tags):\n \"\"\"\n Turn a list of words and a list of tags into a request\n\n tags must be xpos, not upos\n \"\"\"\n request = MorphologyRequest()\n for word, tag in zip(words, xpos_tags):\n tagged_word = request.words.add()\n tagged_word.word = word\n tagged_word.xpos = tag\n return request\n\n\ndef process_text(words, xpos_tags):\n \"\"\"\n Get the lemmata for each word/tag pair\n\n Currently the return is a MorphologyResponse from CoreNLP.proto\n\n tags must be xpos, not upos\n \"\"\"\n request = build_request(words, xpos_tags)\n\n return send_morphology_request(request)\n\n\n\nclass Morphology(JavaProtobufContext):\n \"\"\"\n Morphology context window\n\n This is a context window which keeps a process open. Should allow\n for multiple requests without launching new java processes each time.\n\n (much faster than calling process_text over and over)\n \"\"\"\n def __init__(self, classpath=None):\n super(Morphology, self).__init__(classpath, MorphologyResponse, MORPHOLOGY_JAVA)\n\n def process(self, words, xpos_tags):\n \"\"\"\n Get the lemmata for each word/tag pair\n \"\"\"\n request = build_request(words, xpos_tags)\n return self.process_request(request)\n\n\ndef main():\n # TODO: turn this into a unit test, once a new CoreNLP is released\n words = [\"Jennifer\", \"has\", \"the\", \"prettiest\", \"antennae\"]\n tags = [\"NNP\", \"VBZ\", \"DT\", \"JJS\", \"NNS\"]\n expected = [\"Jennifer\", \"have\", \"the\", \"pretty\", \"antenna\"]\n result = process_text(words, tags)\n lemma = [x.lemma for x in result.words]\n print(lemma)\n assert lemma == expected\n\n with Morphology() as morph:\n result = morph.process(words, tags)\n lemma = [x.lemma for x in result.words]\n assert lemma == expected\n\nif __name__ == '__main__':\n main()\n"
},
{
"path": "stanza/server/parser_eval.py",
"content": "\"\"\"\nThis class runs a Java process to evaluate a treebank prediction using CoreNLP\n\"\"\"\n\nfrom collections import namedtuple\nimport sys\n\nimport stanza\nfrom stanza.protobuf import EvaluateParserRequest, EvaluateParserResponse\nfrom stanza.server.java_protobuf_requests import send_request, build_tree, JavaProtobufContext\nfrom stanza.models.constituency.tree_reader import read_treebank\n\nEVALUATE_JAVA = \"edu.stanford.nlp.parser.metrics.EvaluateExternalParser\"\n\nParseResult = namedtuple(\"ParseResult\", ['gold', 'predictions', 'state', 'constituents'])\nScoredTree = namedtuple(\"ScoredTree\", ['tree', 'score'])\n\ndef build_request(treebank):\n \"\"\"\n treebank should be a list of pairs: [gold, predictions]\n each predictions is a list of tuples (prediction, score, state)\n state is ignored and can be None\n Note that for now, only one tree is measured, but this may be extensible in the future\n Trees should be in the form of a Tree from parse_tree.py\n \"\"\"\n request = EvaluateParserRequest()\n for raw_result in treebank:\n gold = raw_result.gold\n predictions = raw_result.predictions\n parse_result = request.treebank.add()\n parse_result.gold.CopyFrom(build_tree(gold, None))\n for pred in predictions:\n if isinstance(pred, tuple):\n prediction, score = pred\n else:\n prediction = pred\n score = None\n try:\n parse_result.predicted.append(build_tree(prediction, score))\n except Exception as e:\n raise RuntimeError(\"Unable to build parser request from tree {}\".format(pred)) from e\n\n return request\n\ndef collate(gold_treebank, predictions_treebank):\n \"\"\"\n Turns a list of gold and prediction into a evaluation object\n \"\"\"\n treebank = []\n for gold, prediction in zip(gold_treebank, predictions_treebank):\n result = ParseResult(gold, [prediction], None, None)\n treebank.append(result)\n return treebank\n\n\nclass EvaluateParser(JavaProtobufContext):\n \"\"\"\n Parser evaluation context window\n\n This is a context window which keeps a process open. Should allow\n for multiple requests without launching new java processes each time.\n \"\"\"\n def __init__(self, classpath=None, kbest=None, silent=False):\n if kbest is not None:\n extra_args = [\"-evalPCFGkBest\", \"{}\".format(kbest), \"-evals\", \"pcfgTopK\"]\n else:\n extra_args = []\n\n if silent:\n extra_args.extend([\"-evals\", \"summary=False\"])\n\n super(EvaluateParser, self).__init__(classpath, EvaluateParserResponse, EVALUATE_JAVA, extra_args=extra_args)\n\n def process(self, treebank):\n request = build_request(treebank)\n return self.process_request(request)\n\n\ndef main():\n gold = read_treebank(sys.argv[1])\n predictions = read_treebank(sys.argv[2])\n treebank = collate(gold, predictions)\n\n with EvaluateParser() as ep:\n ep.process(treebank)\n\n\nif __name__ == '__main__':\n main()\n"
},
{
"path": "stanza/server/semgrex.py",
"content": "\"\"\"Invokes the Java semgrex on a document\n\nThe server client has a method \"semgrex\" which sends text to Java\nCoreNLP for processing with a semgrex (SEMantic GRaph regEX) query:\n\nhttps://nlp.stanford.edu/nlp/javadoc/javanlp/edu/stanford/nlp/semgraph/semgrex/SemgrexPattern.html\n\nHowever, this operates on text using the CoreNLP tools, which means\nthe dependency graphs may not align with stanza's depparse module, and\nthis also limits the languages for which it can be used. This module\nallows for running semgrex commands on the graphs produced by\ndepparse.\n\nTo use, first process text into a doc using stanza.Pipeline\n\nNext, pass the processed doc and a list of semgrex patterns to\nprocess_doc in this module. It will run the java semgrex module as a\nsubprocess and return the result in the form of a SemgrexResponse,\nwhose description is in the proto file included with stanza.\n\nA minimal example is the main method of this module.\n\nNote that launching the subprocess is potentially quite expensive\nrelative to the search if used many times on small documents. Ideally\nlarger texts would be processed, and all of the desired semgrex\npatterns would be run at once. The worst thing to do would be to call\nthis multiple times on a large document, one invocation per semgrex\npattern, as that would serialize the document each time.\nIncluded here is a context manager which allows for keeping the same\njava process open for multiple requests. This saves on the subprocess\nlaunching time. It is still important not to wastefully serialize the\nsame document over and over, though.\n\"\"\"\n\nimport argparse\nfrom collections import namedtuple\nimport copy\nimport os\nimport re\n\nimport stanza\nfrom stanza.protobuf import SemgrexRequest, SemgrexResponse\nfrom stanza.server.java_protobuf_requests import send_request, add_token, add_word_to_graph, JavaProtobufContext, convert_networkx_graph\nfrom stanza.utils.conll import CoNLL\n\nSEMGREX_JAVA = \"edu.stanford.nlp.semgraph.semgrex.ProcessSemgrexRequest\"\n\nSemgrexQuery = namedtuple(\"SemgrexQuery\", \"pattern comments\")\n\ndef send_semgrex_request(request):\n return send_request(request, SemgrexResponse, SEMGREX_JAVA)\n\ndef build_request(doc, semgrex_patterns, enhanced=False):\n request = SemgrexRequest()\n if isinstance(semgrex_patterns, str):\n semgrex_patterns = [semgrex_patterns]\n semgrex_patterns = [x if isinstance(x, SemgrexQuery) else SemgrexQuery(x, []) for x in semgrex_patterns]\n for semgrex in semgrex_patterns:\n request.semgrex.append(semgrex.pattern)\n\n for sent_idx, sentence in enumerate(doc.sentences):\n query = request.query.add()\n if enhanced:\n # tokens will be added on to the graph object\n convert_networkx_graph(query.graph, sentence, sent_idx)\n else:\n word_idx = 0\n for token in sentence.tokens:\n for word in token.words:\n add_token(query.token, word, token)\n add_word_to_graph(query.graph, word, sent_idx)\n\n word_idx = word_idx + 1\n\n return request\n\ndef process_doc(doc, *semgrex_patterns, enhanced=False):\n \"\"\"\n Returns the result of processing the given semgrex expression on the stanza doc.\n\n Currently the return is a SemgrexResponse from CoreNLP.proto\n \"\"\"\n request = build_request(doc, semgrex_patterns, enhanced=enhanced)\n\n return send_semgrex_request(request)\n\nclass Semgrex(JavaProtobufContext):\n \"\"\"\n Semgrex context window\n\n This is a context window which keeps a process open. Should allow\n for multiple requests without launching new java processes each time.\n \"\"\"\n def __init__(self, classpath=None):\n super(Semgrex, self).__init__(classpath, SemgrexResponse, SEMGREX_JAVA)\n\n def process(self, doc, *semgrex_patterns):\n \"\"\"\n Apply each of the semgrex patterns to each of the dependency trees in doc\n \"\"\"\n request = build_request(doc, semgrex_patterns)\n return self.process_request(request)\n\ndef annotate_doc(doc, semgrex_result, semgrex_patterns, matches_only, exclude_matches):\n \"\"\"\n Put comments on the sentences which describe the matching semgrex patterns\n \"\"\"\n doc = copy.deepcopy(doc)\n if isinstance(semgrex_patterns, str):\n semgrex_patterns = [semgrex_patterns]\n semgrex_patterns = [x if isinstance(x, SemgrexQuery) else SemgrexQuery(x, []) for x in semgrex_patterns]\n matched_ids = set()\n for sentence_result in semgrex_result.result:\n for pattern_result in sentence_result.result:\n for match in pattern_result.match:\n matched_ids.add(match.sentenceIndex)\n\n pattern_texts = [semgrex_pattern.pattern.replace(\"\\n\", \" \") for semgrex_pattern in semgrex_patterns]\n\n matching_sentences = []\n for sentence_result in semgrex_result.result:\n sentence_matched = False\n matched_semgrex_ids = set()\n for pattern_result in sentence_result.result:\n if len(pattern_result.match) == 0:\n continue\n\n highlight_tokens = []\n highlight_edges = []\n for match in pattern_result.match:\n sentence_matched = True\n sentence = doc.sentences[match.sentenceIndex]\n semgrex_pattern = semgrex_patterns[match.semgrexIndex]\n pattern_text = pattern_texts[match.semgrexIndex]\n matched_semgrex_ids.add(match.semgrexIndex)\n\n match_word = \"%d:%s\" % (match.matchIndex, sentence.words[match.matchIndex-1].text)\n if len(match.node) == 0:\n node_matches = \"\"\n else:\n node_matches = [\"%s=%d:%s\" % (node.name, node.matchIndex, sentence.words[node.matchIndex-1].text)\n for node in match.node]\n node_matches = \" \" + \" \".join(node_matches)\n if len(match.varstring) == 0:\n var_values = \"\"\n else:\n var_values = [\"%s=%s\" % (v.name, v.value) for v in match.varstring]\n var_values = \" \" + \" \".join(var_values)\n sentence.add_comment(\"# semgrex pattern |%s| matched at %s%s%s\" % (pattern_text, match_word, node_matches, var_values))\n for comment in semgrex_pattern.comments:\n sentence.add_comment(\"# semgrex comment: %s\" % comment)\n highlight_tokens.append(match.matchIndex)\n for edge in match.edge:\n highlight_edges.append(edge.target)\n if len(highlight_tokens) > 0:\n sentence.add_comment(\"# highlight tokens = %s\" % (\" \".join(\"%d\" % x for x in highlight_tokens)))\n if len(highlight_edges) > 0:\n sentence.add_comment(\"# highlight deprels = %s\" % (\" \".join(\"%d\" % x for x in highlight_edges)))\n\n if sentence_matched and not matches_only:\n for semgrex_idx, pattern_text in enumerate(pattern_texts):\n if semgrex_idx not in matched_semgrex_ids:\n sentence.add_comment(\"# semgrex pattern |%s| did not match!\" % pattern_text)\n\n if sentence_matched:\n matching_sentences.append(sentence)\n\n nonmatching_sentences = [sentence for sentence_idx, sentence in enumerate(doc.sentences) if sentence_idx not in matched_ids]\n for sentence in nonmatching_sentences:\n for semgrex_idx, pattern_text in enumerate(pattern_texts):\n sentence.add_comment(\"# semgrex pattern |%s| did not match!\" % pattern_text)\n\n if matches_only:\n doc.sentences = matching_sentences\n elif exclude_matches:\n doc.sentences = nonmatching_sentences\n return doc\n\n\ndef main():\n \"\"\"\n Runs a toy example, or can run a given semgrex expression on the given input file.\n\n For example:\n python3 -m stanza.server.semgrex --input_file demo/semgrex_sample.conllu\n\n --matches_only to only print sentences that match the semgrex pattern\n --no_print_input to not print the input\n \"\"\"\n parser = argparse.ArgumentParser()\n parser.add_argument('--input', type=str, default=None, help='Process this file or directory')\n parser.add_argument('--input_filter', type=str, default=None, help='Only process files that match this regex')\n parser.add_argument('semgrex', type=str, nargs=\"*\", default=[\"{}=source >obj=zzz {}=target\"], help=\"Semgrex to apply to the text. The default looks for sentences with objects\")\n parser.add_argument('--semgrex_file', type=str, default=None, help=\"File to read semgrex patterns from - relevant in case the pattern you want to use doesn't work well on the command line, for example\")\n parser.add_argument('--print_input', dest='print_input', action='store_true', default=False, help=\"Print the input alongside the output - gets kind of noisy\")\n parser.add_argument('--no_print_input', dest='print_input', action='store_false', help=\"Don't print the input alongside the output - gets kind of noisy\")\n\n parser.add_argument('--matches_only', action='store_true', default=True, help=\"Only print the matching sentences\")\n parser.add_argument('--no_matches_only', dest='matches_only', action='store_false', help=\"Only print the matching sentences\")\n parser.add_argument('--exclude_matches', action='store_true', default=False, help=\"Only print the NON-matching sentences\")\n\n parser.add_argument('--enhanced', action='store_true', default=False, help='Use the enhanced dependencies instead of the basic')\n parser.add_argument('--no_combined_doc', dest='combined_doc', action='store_false', default=True, help='By default, combine all the input docs into one big document. Allows for easier secondary processing like sorting')\n args = parser.parse_args()\n\n if args.semgrex_file:\n with open(args.semgrex_file) as fin:\n args.semgrex = [x.strip() for x in fin.readlines()]\n\n semgrex_patterns = []\n current_comments = []\n for line in args.semgrex:\n if not line:\n current_comments = []\n elif line.startswith(\"#\"):\n current_comments.append(line[1:].strip())\n else:\n semgrex_patterns.append(SemgrexQuery(line, current_comments))\n current_comments = []\n args.semgrex = semgrex_patterns\n\n if args.input:\n if os.path.isfile(args.input):\n docs = [CoNLL.conll2doc(input_file=args.input, ignore_gapping=False)]\n else:\n filenames = sorted(os.listdir(args.input))\n if args.input_filter:\n input_filter = re.compile(args.input_filter)\n filenames = [x for x in filenames if input_filter.match(x)]\n filenames = [os.path.join(args.input, filename) for filename in filenames]\n filenames = [filename for filename in filenames if os.path.isfile(filename)]\n docs = [CoNLL.conll2doc(input_file=filename, ignore_gapping=False) for filename in filenames]\n else:\n nlp = stanza.Pipeline('en', processors='tokenize,pos,lemma,depparse')\n docs = [nlp('Uro ruined modern. Fortunately, Wotc banned him.')]\n\n if args.combined_doc:\n sentences = [sent for doc in docs for sent in doc.sentences]\n docs = [docs[0]]\n docs[0].sentences = sentences\n\n for doc in docs:\n if args.print_input:\n print(\"{:C}\".format(doc))\n print()\n print(\"-\" * 75)\n print()\n semgrex_result = process_doc(doc, *args.semgrex, enhanced=args.enhanced)\n doc = annotate_doc(doc, semgrex_result, args.semgrex, args.matches_only, args.exclude_matches)\n if len(doc.sentences) > 0:\n print(\"{:C}\\n\".format(doc))\n\nif __name__ == '__main__':\n main()\n"
},
{
"path": "stanza/server/ssurgeon.py",
"content": "\"\"\"Invokes the Java ssurgeon on a document\n\n\"ssurgeon\" sends text to Java CoreNLP for processing with a ssurgeon\n(Semantic graph SURGEON) query\n\nThe main program in this file gives a very short intro to how to use it.\n\"\"\"\n\n\nimport argparse\nfrom collections import namedtuple\nimport copy\nimport os\nimport re\nimport sys\n\nfrom stanza.models.common.utils import misc_to_space_after, space_after_to_misc\nfrom stanza.protobuf import SsurgeonRequest, SsurgeonResponse\nfrom stanza.server import java_protobuf_requests\nfrom stanza.utils.conll import CoNLL\n\nfrom stanza.models.common.doc import ID, TEXT, LEMMA, UPOS, XPOS, FEATS, HEAD, DEPREL, DEPS, MISC, START_CHAR, END_CHAR, NER, Word, Token, Sentence\n\nSSURGEON_JAVA = \"edu.stanford.nlp.semgraph.semgrex.ssurgeon.ProcessSsurgeonRequest\"\n\nSsurgeonEdit = namedtuple(\"SsurgeonEdit\",\n \"semgrex_pattern ssurgeon_edits ssurgeon_id notes language\",\n defaults=[None, None, \"UniversalEnglish\"])\n\ndef parse_ssurgeon_edits(ssurgeon_text):\n ssurgeon_text = ssurgeon_text.strip()\n ssurgeon_blocks = re.split(\"\\n\\n+\", ssurgeon_text)\n ssurgeon_edits = []\n for idx, block in enumerate(ssurgeon_blocks):\n lines = block.split(\"\\n\")\n comments = [line[1:].strip() for line in lines if line.startswith(\"#\")]\n notes = \" \".join(comments)\n lines = [x.strip() for x in lines if x.strip() and not x.startswith(\"#\")]\n if len(lines) == 0:\n # was a block of entirely comments\n continue\n semgrex = lines[0]\n ssurgeon = lines[1:]\n ssurgeon_edits.append(SsurgeonEdit(semgrex, ssurgeon, \"%d\" % (idx + 1), notes))\n return ssurgeon_edits\n\ndef read_ssurgeon_edits(edit_file):\n with open(edit_file, encoding=\"utf-8\") as fin:\n return parse_ssurgeon_edits(fin.read())\n\ndef send_ssurgeon_request(request):\n return java_protobuf_requests.send_request(request, SsurgeonResponse, SSURGEON_JAVA)\n\ndef build_request(doc, ssurgeon_edits):\n request = SsurgeonRequest()\n\n for ssurgeon in ssurgeon_edits:\n ssurgeon_proto = request.ssurgeon.add()\n ssurgeon_proto.semgrex = ssurgeon.semgrex_pattern\n for operation in ssurgeon.ssurgeon_edits:\n ssurgeon_proto.operation.append(operation)\n if ssurgeon.ssurgeon_id is not None:\n ssurgeon_proto.id = ssurgeon.ssurgeon_id\n if ssurgeon.notes is not None:\n ssurgeon_proto.notes = ssurgeon.notes\n if ssurgeon.language is not None:\n ssurgeon_proto.language = ssurgeon.language\n\n try:\n for sent_idx, sentence in enumerate(doc.sentences):\n graph = request.graph.add()\n word_idx = 0\n for token in sentence.tokens:\n for word in token.words:\n java_protobuf_requests.add_token(graph.token, word, token)\n java_protobuf_requests.add_word_to_graph(graph, word, sent_idx)\n\n word_idx = word_idx + 1\n except Exception as e:\n raise RuntimeError(\"Failed to process sentence {}:\\n{:C}\".format(sent_idx, sentence)) from e\n\n return request\n\ndef build_request_one_operation(doc, semgrex_pattern, ssurgeon_edits, ssurgeon_id=None, notes=None):\n ssurgeon_edit = SsurgeonEdit(semgrex_pattern, ssurgeon_edits, ssurgeon_id, notes)\n return build_request(doc, [ssurgeon_edit])\n\ndef process_doc(doc, ssurgeon_edits):\n \"\"\"\n Returns the result of processing the given semgrex expression and ssurgeon edits on the stanza doc.\n\n Currently the return is a SsurgeonResponse from CoreNLP.proto\n \"\"\"\n request = build_request(doc, ssurgeon_edits)\n\n return send_ssurgeon_request(request)\n\ndef process_doc_one_operation(doc, semgrex_pattern, ssurgeon_edits, ssurgeon_id=None, notes=None):\n request = build_request_one_operation(doc, semgrex_pattern, ssurgeon_edits, ssurgeon_id, notes)\n\n return send_ssurgeon_request(request)\n\ndef build_word_entry(word_index, graph_word):\n word_entry = {\n ID: word_index,\n TEXT: graph_word.word if graph_word.word else None,\n LEMMA: graph_word.lemma if graph_word.lemma else None,\n UPOS: graph_word.coarseTag if graph_word.coarseTag else None,\n XPOS: graph_word.pos if graph_word.pos else None,\n FEATS: java_protobuf_requests.features_to_string(graph_word.conllUFeatures),\n DEPS: None,\n NER: graph_word.ner if graph_word.ner else None,\n MISC: None,\n START_CHAR: None, # TODO: fix this? one problem is the text positions\n END_CHAR: None, # might change across all of the sentences\n # presumably python will complain if this conflicts\n # with one of the constants above\n \"is_mwt\": graph_word.isMWT,\n \"is_first_mwt\": graph_word.isFirstMWT,\n \"mwt_text\": graph_word.mwtText,\n \"mwt_misc\": graph_word.mwtMisc,\n }\n # TODO: do \"before\" as well\n word_entry[MISC] = space_after_to_misc(graph_word.after)\n if graph_word.conllUMisc:\n word_entry[MISC] = java_protobuf_requests.substitute_space_misc(graph_word.conllUMisc, word_entry[MISC])\n return word_entry\n\ndef convert_response_to_doc(doc, semgrex_response, add_missing_text):\n doc = copy.deepcopy(doc)\n try:\n for sent_idx, (sentence, ssurgeon_result) in enumerate(zip(doc.sentences, semgrex_response.result)):\n # EditNode is currently bugged... :/\n # TODO: change this after next CoreNLP release (after 4.5.3)\n #if not ssurgeon_result.changed:\n # continue\n\n ssurgeon_graph = ssurgeon_result.graph\n tokens = []\n token_id_to_idx = {}\n for graph_node, graph_word in zip(ssurgeon_graph.node, ssurgeon_graph.token):\n word_entry = build_word_entry(graph_node.index, graph_word)\n token_id_to_idx[graph_node.index] = len(tokens)\n tokens.append(word_entry)\n for root in ssurgeon_graph.root:\n tokens[token_id_to_idx[root]][HEAD] = 0\n tokens[token_id_to_idx[root]][DEPREL] = \"root\"\n for edge in ssurgeon_graph.edge:\n # can't do anything about the extra dependencies for now\n # TODO: put them all in .deps\n if edge.isExtra:\n continue\n tokens[token_id_to_idx[edge.target]][HEAD] = edge.source\n tokens[token_id_to_idx[edge.target]][DEPREL] = edge.dep\n\n tokens.sort(key=lambda x: x[ID])\n\n # for any MWT, produce a token_entry which represents the word range\n mwt_tokens = []\n for word_start_idx, word in enumerate(tokens):\n if not word[\"is_first_mwt\"]:\n if word[\"is_mwt\"]:\n word[MISC] = java_protobuf_requests.remove_space_misc(word[MISC])\n mwt_tokens.append(word)\n continue\n word_end_idx = word_start_idx + 1\n while word_end_idx < len(tokens) and tokens[word_end_idx][\"is_mwt\"] and not tokens[word_end_idx][\"is_first_mwt\"]:\n word_end_idx += 1\n mwt_token_entry = {\n # the tokens don't fencepost the way lists do\n ID: (tokens[word_start_idx][ID], tokens[word_end_idx-1][ID]),\n TEXT: word[\"mwt_text\"],\n NER: word[NER],\n # use the SpaceAfter=No (or not) from the last word in the token\n MISC: None,\n }\n mwt_token_entry[MISC] = java_protobuf_requests.misc_space_pieces(tokens[word_end_idx-1][MISC])\n if tokens[word_end_idx-1][\"mwt_misc\"]:\n mwt_token_entry[MISC] = java_protobuf_requests.substitute_space_misc(tokens[word_end_idx-1][\"mwt_misc\"], mwt_token_entry[MISC])\n word[MISC] = java_protobuf_requests.remove_space_misc(word[MISC])\n mwt_tokens.append(mwt_token_entry)\n mwt_tokens.append(word)\n\n old_comments = list(sentence.comments)\n sentence = Sentence(mwt_tokens, doc)\n\n token_text = []\n for token_idx, token in enumerate(sentence.tokens):\n token_text.append(token.text)\n if token_idx == len(sentence.tokens) - 1:\n break\n token_text.append(token.spaces_after)\n\n sentence_text = \"\".join(token_text)\n\n found_text = False\n for comment in old_comments:\n if comment.startswith(\"# text \") or comment.startswith(\"#text \") or comment.startswith(\"# text=\") or comment.startswith(\"#text=\"):\n sentence.add_comment(\"# text = \" + sentence_text)\n found_text = True\n else:\n sentence.add_comment(comment)\n if not found_text and add_missing_text:\n sentence.add_comment(\"# text = \" + sentence_text)\n\n doc.sentences[sent_idx] = sentence\n\n sentence.rebuild_dependencies()\n except Exception as e:\n raise RuntimeError(\"Ssurgeon could not process sentence {}\\nSsurgeon result:\\n{}\\nOriginal sentence:\\n{:C}\".format(sent_idx, ssurgeon_result, sentence)) from e\n return doc\n\nclass Ssurgeon(java_protobuf_requests.JavaProtobufContext):\n \"\"\"\n Ssurgeon context window\n\n This is a context window which keeps a process open. Should allow\n for multiple requests without launching new java processes each time.\n \"\"\"\n def __init__(self, classpath=None):\n super(Ssurgeon, self).__init__(classpath, SsurgeonResponse, SSURGEON_JAVA)\n\n def process(self, doc, ssurgeon_edits):\n \"\"\"\n Apply each of the ssurgeon patterns to each of the dependency trees in doc\n \"\"\"\n request = build_request(doc, ssurgeon_edits)\n return self.process_request(request)\n\n def process_one_operation(self, doc, semgrex_pattern, ssurgeon_edits, ssurgeon_id=None, notes=None):\n \"\"\"\n Convenience method - build one operation, then apply it\n \"\"\"\n request = build_request_one_operation(doc, semgrex_pattern, ssurgeon_edits, ssurgeon_id, notes)\n return self.process_request(request)\n\nSAMPLE_DOC = \"\"\"\n# sent_id = 271\n# text = Hers is easy to clean.\n# previous = What did the dealer like about Alex's car?\n# comment = extraction/raising via \"tough extraction\" and clausal subject\n1\tHers\thers\tPRON\tPRP\tGender=Fem|Number=Sing|Person=3|Poss=Yes|PronType=Prs\t3\tnsubj\t_\t_\n2\tis\tbe\tAUX\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t3\tcop\t_\t_\n3\teasy\teasy\tADJ\tJJ\tDegree=Pos\t0\troot\t_\t_\n4\tto\tto\tPART\tTO\t_\t5\tmark\t_\t_\n5\tclean\tclean\tVERB\tVB\tVerbForm=Inf\t3\tcsubj\t_\tSpaceAfter=No\n6\t.\t.\tPUNCT\t.\t_\t5\tpunct\t_\t_\n\"\"\"\n\ndef main():\n # for Windows, so that we aren't randomly printing garbage (or just failing to print)\n try:\n sys.stdout.reconfigure(encoding='utf-8')\n except AttributeError:\n # TODO: deprecate 3.6 support after the next release\n pass\n\n # The default semgrex detects sentences in the UD_English-Pronouns dataset which have both nsubj and csubj on the same word.\n # The default ssurgeon transforms the unwanted csubj to advcl\n # See https://github.com/UniversalDependencies/docs/issues/923\n parser = argparse.ArgumentParser()\n parser.add_argument('--input', type=str, default=None, help=\"Input file / directory to process (otherwise will process a sample text)\")\n parser.add_argument('--output', type=str, default=None, help=\"Output location (otherwise will write back to the input directory)\")\n parser.add_argument('--stdout', action='store_true', default=False, help='Output to stdout')\n parser.add_argument('--input_filter', type=str, default=\".*[.]conllu\", help=\"If processing a directory, only process files from --input that match this filter - regex, not shell filter. Default: %(default)s\")\n parser.add_argument('--no_input_filter', action='store_const', const=None, dest=\"input_filter\", help=\"Remove the default input filename filter\")\n parser.add_argument('--edit_file', type=str, default=None, help=\"File to get semgrex and ssurgeon rules from\")\n parser.add_argument('--semgrex', type=str, default=\"{}=source >nsubj {} >csubj=bad {}\", help=\"Semgrex to apply to the text. A default detects words which have both an nsubj and a csubj. Default: %(default)s\")\n parser.add_argument('ssurgeon', type=str, default=[\"relabelNamedEdge -edge bad -reln advcl\"], nargs=\"*\", help=\"Ssurgeon edits to apply based on the Semgrex. Can have multiple edits in a row. A default exists to transform csubj into advcl. Default: %(default)s\")\n parser.add_argument('--print_input', dest='print_input', action='store_true', default=False, help=\"Print the input alongside the output - gets kind of noisy. Default: %(default)s\")\n parser.add_argument('--no_print_input', dest='print_input', action='store_false', help=\"Don't print the input alongside the output - gets kind of noisy\")\n parser.add_argument('--no_add_missing_text', dest='add_missing_text', action='store_false', help=\"By default, the tool will add a #text comment if one does not exist. This leaves that blank\")\n args = parser.parse_args()\n\n if args.edit_file:\n ssurgeon_edits = read_ssurgeon_edits(args.edit_file)\n else:\n ssurgeon_edits = [SsurgeonEdit(args.semgrex, args.ssurgeon)]\n\n if args.input:\n if os.path.isfile(args.input):\n docs = [CoNLL.conll2doc(input_file=args.input)]\n if args.output is None:\n outputs = [args.input]\n else:\n # TODO: could check if --output is a directory\n outputs = [args.output]\n input_output = zip(docs, outputs)\n else:\n if not args.output:\n args.output = args.input\n if not os.path.exists(args.output):\n os.makedirs(args.output)\n def read_docs():\n for doc_filename in os.listdir(args.input):\n if args.input_filter:\n if not re.match(args.input_filter, doc_filename):\n continue\n doc_path = os.path.join(args.input, doc_filename)\n if not os.path.isfile(doc_path):\n continue\n output_path = os.path.join(args.output, doc_filename)\n print(\"Processing %s to %s\" % (doc_path, output_path))\n yield CoNLL.conll2doc(input_file=doc_path), output_path\n input_output = read_docs()\n else:\n docs = [CoNLL.conll2doc(input_str=SAMPLE_DOC)]\n outputs = [None]\n input_output = zip(docs, outputs)\n args.stdout = True\n\n for doc, output in input_output:\n if args.print_input:\n print(\"{:C}\".format(doc))\n ssurgeon_request = build_request(doc, ssurgeon_edits)\n ssurgeon_response = send_ssurgeon_request(ssurgeon_request)\n updated_doc = convert_response_to_doc(doc, ssurgeon_response, args.add_missing_text)\n if output is not None:\n with open(output, \"w\", encoding=\"utf-8\") as fout:\n fout.write(\"{:C}\\n\\n\".format(updated_doc))\n if args.stdout:\n print(\"{:C}\\n\".format(updated_doc))\n\nif __name__ == '__main__':\n main()\n"
},
{
"path": "stanza/server/tokensregex.py",
"content": "\"\"\"Invokes the Java tokensregex on a document\n\nThis operates tokensregex on docs processed with stanza models.\n\nhttps://nlp.stanford.edu/software/tokensregex.html\n\nA minimal example is the main method of this module.\n\"\"\"\n\nimport stanza\n\nfrom stanza.protobuf import TokensRegexRequest, TokensRegexResponse\nfrom stanza.server.java_protobuf_requests import send_request, add_sentence\n\ndef send_tokensregex_request(request):\n return send_request(request, TokensRegexResponse,\n \"edu.stanford.nlp.ling.tokensregex.ProcessTokensRegexRequest\")\n\ndef process_doc(doc, *patterns):\n request = TokensRegexRequest()\n for pattern in patterns:\n request.pattern.append(pattern)\n\n request_doc = request.doc\n request_doc.text = doc.text\n num_tokens = 0\n for sentence in doc.sentences:\n add_sentence(request_doc.sentence, sentence, num_tokens)\n num_tokens = num_tokens + sum(len(token.words) for token in sentence.tokens)\n\n return send_tokensregex_request(request)\n\ndef main():\n #nlp = stanza.Pipeline('en',\n # processors='tokenize,pos,lemma,ner')\n nlp = stanza.Pipeline('en',\n processors='tokenize')\n\n doc = nlp('Uro ruined modern. Fortunately, Wotc banned him')\n print(process_doc(doc, \"him\", \"ruined\"))\n\n\nif __name__ == '__main__':\n main()\n"
},
{
"path": "stanza/server/tsurgeon.py",
"content": "\"\"\"Invokes the Java tsurgeon on a list of trees\n\nIncluded with CoreNLP is a mechanism for modifying trees based on\nexisting patterns within a tree. The patterns are found using tregex:\n\nhttps://nlp.stanford.edu/nlp/javadoc/javanlp/edu/stanford/nlp/trees/tregex/TregexPattern.html\n\nThe modifications are then performed using tsurgeon:\n\nhttps://nlp.stanford.edu/nlp/javadoc/javanlp/edu/stanford/nlp/trees/tregex/tsurgeon/Tsurgeon.html\n\nThis module accepts Tree objects as produced by the conparser and\nreturns the modified trees that result from one or more tsurgeon\noperations.\n\"\"\"\n\nfrom stanza.models.constituency import tree_reader\nfrom stanza.models.constituency.parse_tree import Tree\nfrom stanza.protobuf import TsurgeonRequest, TsurgeonResponse\nfrom stanza.server.java_protobuf_requests import send_request, build_tree, from_tree, JavaProtobufContext\n\nTSURGEON_JAVA = \"edu.stanford.nlp.trees.tregex.tsurgeon.ProcessTsurgeonRequest\"\n\ndef send_tsurgeon_request(request):\n return send_request(request, TsurgeonResponse, TSURGEON_JAVA)\n\n\ndef build_request(trees, operations):\n \"\"\"\n Build the TsurgeonRequest object\n\n trees: a list of trees\n operations: a list of (tregex, tsurgeon, tsurgeon, ...)\n \"\"\"\n if isinstance(trees, Tree):\n trees = (trees,)\n\n request = TsurgeonRequest()\n for tree in trees:\n request.trees.append(build_tree(tree, 0.0))\n if all(isinstance(x, str) for x in operations):\n operations = (operations,)\n for operation in operations:\n if len(operation) == 1:\n raise ValueError(\"Expected [tregex, tsurgeon, ...] but just got a tregex\")\n operation_request = request.operations.add()\n operation_request.tregex = operation[0]\n for tsurgeon in operation[1:]:\n operation_request.tsurgeon.append(tsurgeon)\n return request\n\n\ndef process_trees(trees, *operations):\n \"\"\"\n Returns the result of processing the given tsurgeon operations on the given trees\n\n Returns a list of modified trees, eg, the result is already processed\n \"\"\"\n request = build_request(trees, operations)\n result = send_tsurgeon_request(request)\n\n return [from_tree(t)[0] for t in result.trees]\n\n\nclass Tsurgeon(JavaProtobufContext):\n \"\"\"\n Tsurgeon context window\n\n This is a context window which keeps a process open. Should allow\n for multiple requests without launching new java processes each time.\n \"\"\"\n def __init__(self, classpath=None):\n super(Tsurgeon, self).__init__(classpath, TsurgeonResponse, TSURGEON_JAVA)\n\n def process(self, trees, *operations):\n request = build_request(trees, operations)\n result = self.process_request(request)\n return [from_tree(t)[0] for t in result.trees]\n\n\ndef main():\n \"\"\"\n A small demonstration of a tsurgeon operation\n \"\"\"\n text=\"( (SBARQ (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (IN in) (NP (DT this) (NN seat))))) (. ?)))\"\n trees = tree_reader.read_trees(text)\n\n tregex = \"WP=wp\"\n tsurgeon = \"relabel wp WWWPPP\"\n\n result = process_trees(trees, (tregex, tsurgeon))\n print(result)\n\nif __name__ == '__main__':\n main()\n"
},
{
"path": "stanza/server/ud_enhancer.py",
"content": "\n\nimport stanza\nfrom stanza.protobuf import DependencyEnhancerRequest, Document, Language\nfrom stanza.server.java_protobuf_requests import send_request, add_sentence, JavaProtobufContext\n\nENHANCER_JAVA = \"edu.stanford.nlp.trees.ud.ProcessUniversalEnhancerRequest\"\n\ndef build_enhancer_request(doc, language, pronouns_pattern):\n if bool(language) == bool(pronouns_pattern):\n raise ValueError(\"Should set exactly one of language and pronouns_pattern\")\n\n request = DependencyEnhancerRequest()\n if pronouns_pattern:\n request.setRelativePronouns(pronouns_pattern)\n elif language.lower() in (\"en\", \"english\"):\n request.language = Language.UniversalEnglish\n elif language.lower() in (\"zh\", \"zh-hans\", \"chinese\"):\n request.language = Language.UniversalChinese\n else:\n raise ValueError(\"Sorry, but language \" + language + \" is not supported yet. Either set a pronouns pattern or file an issue at https://stanfordnlp.github.io/stanza suggesting a mechanism for converting this language\")\n\n request_doc = request.document\n request_doc.text = doc.text\n num_tokens = 0\n for sent_idx, sentence in enumerate(doc.sentences):\n request_sentence = add_sentence(request_doc.sentence, sentence, num_tokens)\n num_tokens = num_tokens + sum(len(token.words) for token in sentence.tokens)\n\n graph = request_sentence.basicDependencies\n nodes = []\n word_index = 0\n for token in sentence.tokens:\n for word in token.words:\n # TODO: refactor with the bit in java_protobuf_requests\n word_index = word_index + 1\n node = graph.node.add()\n node.sentenceIndex = sent_idx\n node.index = word_index\n\n if word.head != 0:\n edge = graph.edge.add()\n edge.source = word.head\n edge.target = word_index\n edge.dep = word.deprel\n\n return request\n\ndef process_doc(doc, language=None, pronouns_pattern=None):\n request = build_enhancer_request(doc, language, pronouns_pattern)\n return send_request(request, Document, ENHANCER_JAVA)\n\nclass UniversalEnhancer(JavaProtobufContext):\n \"\"\"\n UniversalEnhancer context window\n\n This is a context window which keeps a process open. Should allow\n for multiple requests without launching new java processes each time.\n \"\"\"\n def __init__(self, language=None, pronouns_pattern=None, classpath=None):\n super(UniversalEnhancer, self).__init__(classpath, Document, ENHANCER_JAVA)\n if bool(language) == bool(pronouns_pattern):\n raise ValueError(\"Should set exactly one of language and pronouns_pattern\")\n self.language = language\n self.pronouns_pattern = pronouns_pattern\n\n def process(self, doc):\n request = build_enhancer_request(doc, self.language, self.pronouns_pattern)\n return self.process_request(request)\n\ndef main():\n nlp = stanza.Pipeline('en',\n processors='tokenize,pos,lemma,depparse')\n\n with UniversalEnhancer(language=\"en\") as enhancer:\n doc = nlp(\"This is the car that I bought\")\n result = enhancer.process(doc)\n print(result.sentence[0].enhancedDependencies)\n\nif __name__ == '__main__':\n main()\n"
},
{
"path": "stanza/tests/__init__.py",
"content": "\"\"\"\nUtilities for testing\n\"\"\"\n\nimport os\nimport re\n\nfrom platformdirs import user_cache_dir\n\nfrom stanza import __resources_version__\n\n# Environment Variables\n# set this to specify working directory of tests\nTEST_HOME_VAR = 'STANZA_TEST_HOME'\n\n# Global Variables\nTEST_DIR_BASE_NAME = 'stanza_test'\n\nTEST_WORKING_DIR = os.getenv(TEST_HOME_VAR, None)\nif not TEST_WORKING_DIR:\n TEST_WORKING_DIR = user_cache_dir(TEST_DIR_BASE_NAME, 'StanfordNLP', __resources_version__)\n\nTEST_MODELS_DIR = f'{TEST_WORKING_DIR}/models'\nTEST_CORENLP_DIR = f'{TEST_WORKING_DIR}/corenlp_dir'\n\n# server resources\nSERVER_TEST_PROPS = f'{TEST_WORKING_DIR}/scripts/external_server.properties'\n\n# language resources\nLANGUAGE_RESOURCES = {}\n\nTOKENIZE_MODEL = 'tokenizer.pt'\nMWT_MODEL = 'mwt_expander.pt'\nPOS_MODEL = 'tagger.pt'\nPOS_PRETRAIN = 'pretrain.pt'\nLEMMA_MODEL = 'lemmatizer.pt'\nDEPPARSE_MODEL = 'parser.pt'\nDEPPARSE_PRETRAIN = 'pretrain.pt'\n\nMODEL_FILES = [TOKENIZE_MODEL, MWT_MODEL, POS_MODEL, POS_PRETRAIN, LEMMA_MODEL, DEPPARSE_MODEL, DEPPARSE_PRETRAIN]\n\n# English resources\nEN_KEY = 'en'\nEN_SHORTHAND = 'en_ewt'\n# models\nEN_MODELS_DIR = f'{TEST_WORKING_DIR}/models/{EN_SHORTHAND}_models'\nEN_MODEL_FILES = [f'{EN_MODELS_DIR}/{EN_SHORTHAND}_{model_fname}' for model_fname in MODEL_FILES]\n\n# French resources\nFR_KEY = 'fr'\nFR_SHORTHAND = 'fr_gsd'\n# regression file paths\nFR_TEST_IN = f'{TEST_WORKING_DIR}/in/fr_gsd.test.txt'\nFR_TEST_OUT = f'{TEST_WORKING_DIR}/out/fr_gsd.test.txt.out'\nFR_TEST_GOLD_OUT = f'{TEST_WORKING_DIR}/out/fr_gsd.test.txt.out.gold'\n# models\nFR_MODELS_DIR = f'{TEST_WORKING_DIR}/models/{FR_SHORTHAND}_models'\nFR_MODEL_FILES = [f'{FR_MODELS_DIR}/{FR_SHORTHAND}_{model_fname}' for model_fname in MODEL_FILES]\n\n# Other language resources\nAR_SHORTHAND = 'ar_padt'\nDE_SHORTHAND = 'de_gsd'\nKK_SHORTHAND = 'kk_ktb'\nKO_SHORTHAND = 'ko_gsd'\n\n\n# utils for clean up\n# only allow removal of dirs/files in this approved list\nREMOVABLE_PATHS = ['en_ewt_models', 'en_ewt_tokenizer.pt', 'en_ewt_mwt_expander.pt', 'en_ewt_tagger.pt',\n 'en_ewt.pretrain.pt', 'en_ewt_lemmatizer.pt', 'en_ewt_parser.pt', 'fr_gsd_models',\n 'fr_gsd_tokenizer.pt', 'fr_gsd_mwt_expander.pt', 'fr_gsd_tagger.pt', 'fr_gsd.pretrain.pt',\n 'fr_gsd_lemmatizer.pt', 'fr_gsd_parser.pt', 'ar_padt_models', 'ar_padt_tokenizer.pt',\n 'ar_padt_mwt_expander.pt', 'ar_padt_tagger.pt', 'ar_padt.pretrain.pt', 'ar_padt_lemmatizer.pt',\n 'ar_padt_parser.pt', 'de_gsd_models', 'de_gsd_tokenizer.pt', 'de_gsd_mwt_expander.pt',\n 'de_gsd_tagger.pt', 'de_gsd.pretrain.pt', 'de_gsd_lemmatizer.pt', 'de_gsd_parser.pt',\n 'kk_ktb_models', 'kk_ktb_tokenizer.pt', 'kk_ktb_mwt_expander.pt', 'kk_ktb_tagger.pt',\n 'kk_ktb.pretrain.pt', 'kk_ktb_lemmatizer.pt', 'kk_ktb_parser.pt', 'ko_gsd_models',\n 'ko_gsd_tokenizer.pt', 'ko_gsd_mwt_expander.pt', 'ko_gsd_tagger.pt', 'ko_gsd.pretrain.pt',\n 'ko_gsd_lemmatizer.pt', 'ko_gsd_parser.pt']\n\n\ndef safe_rm(path_to_rm):\n \"\"\"\n Safely remove a directory of files or a file\n 1.) check path exists, files are files, dirs are dirs\n 2.) only remove things on approved list REMOVABLE_PATHS\n 3.) assert no longer exists\n \"\"\"\n # just return if path doesn't exist\n if not os.path.exists(path_to_rm):\n return\n # handle directory\n if os.path.isdir(path_to_rm):\n files_to_rm = [f'{path_to_rm}/{fname}' for fname in os.listdir(path_to_rm)]\n dir_to_rm = path_to_rm\n else:\n files_to_rm = [path_to_rm]\n dir_to_rm = None\n # clear out files\n for file_to_rm in files_to_rm:\n if os.path.isfile(file_to_rm) and os.path.basename(file_to_rm) in REMOVABLE_PATHS:\n os.remove(file_to_rm)\n assert not os.path.exists(file_to_rm), f'Error removing: {file_to_rm}'\n # clear out directory\n if dir_to_rm is not None and os.path.isdir(dir_to_rm):\n os.rmdir(dir_to_rm)\n assert not os.path.exists(dir_to_rm), f'Error removing: {dir_to_rm}'\n\ndef compare_ignoring_whitespace(predicted, expected):\n predicted = re.sub('[ \\t]+', ' ', predicted.strip())\n predicted = re.sub('\\r\\n', '\\n', predicted)\n expected = re.sub('[ \\t]+', ' ', expected.strip())\n expected = re.sub('\\r\\n', '\\n', expected)\n assert predicted == expected\n\n"
},
{
"path": "stanza/tests/classifiers/__init__.py",
"content": ""
},
{
"path": "stanza/tests/classifiers/test_classifier.py",
"content": "import glob\nimport os\n\nimport pytest\n\nimport numpy as np\nimport torch\n\nimport stanza\nimport stanza.models.classifier as classifier\nimport stanza.models.classifiers.data as data\nfrom stanza.models.classifiers.trainer import Trainer\nfrom stanza.models.common import pretrain\nfrom stanza.models.common import utils\n\nfrom stanza.tests import TEST_MODELS_DIR\nfrom stanza.tests.classifiers.test_data import train_file, dev_file, test_file, DATASET, SENTENCES\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\nEMB_DIM = 5\n\n@pytest.fixture(scope=\"module\")\ndef fake_embeddings(tmp_path_factory):\n \"\"\"\n will return a path to a fake embeddings file with the words in SENTENCES\n \"\"\"\n # could set np random seed here\n words = sorted(set([x.lower() for y in SENTENCES for x in y]))\n words = words[:-1]\n embedding_dir = tmp_path_factory.mktemp(\"data\")\n embedding_txt = embedding_dir / \"embedding.txt\"\n embedding_pt = embedding_dir / \"embedding.pt\"\n embedding = np.random.random((len(words), EMB_DIM))\n\n with open(embedding_txt, \"w\", encoding=\"utf-8\") as fout:\n for word, emb in zip(words, embedding):\n fout.write(word)\n fout.write(\"\\t\")\n fout.write(\"\\t\".join(str(x) for x in emb))\n fout.write(\"\\n\")\n\n pt = pretrain.Pretrain(str(embedding_pt), str(embedding_txt))\n pt.load()\n assert os.path.exists(embedding_pt)\n return embedding_pt\n\nclass TestClassifier:\n def build_model(self, tmp_path, fake_embeddings, train_file, dev_file, extra_args=None, checkpoint_file=None):\n \"\"\"\n Build a model to be used by one of the later tests\n \"\"\"\n save_dir = str(tmp_path / \"classifier\")\n save_name = \"model.pt\"\n args = [\"--save_dir\", save_dir,\n \"--save_name\", save_name,\n \"--wordvec_pretrain_file\", str(fake_embeddings),\n \"--filter_channels\", \"20\",\n \"--fc_shapes\", \"20,10\",\n \"--train_file\", str(train_file),\n \"--dev_file\", str(dev_file),\n \"--max_epochs\", \"2\",\n \"--batch_size\", \"60\"]\n if extra_args is not None:\n args = args + extra_args\n args = classifier.parse_args(args)\n train_set = data.read_dataset(args.train_file, args.wordvec_type, args.min_train_len)\n if checkpoint_file:\n trainer = Trainer.load(checkpoint_file, args, load_optimizer=True)\n else:\n trainer = Trainer.build_new_model(args, train_set)\n return trainer, train_set, args\n\n def run_training(self, tmp_path, fake_embeddings, train_file, dev_file, extra_args=None, checkpoint_file=None):\n \"\"\"\n Iterate a couple times over a model\n \"\"\"\n trainer, train_set, args = self.build_model(tmp_path, fake_embeddings, train_file, dev_file, extra_args, checkpoint_file)\n dev_set = data.read_dataset(args.dev_file, args.wordvec_type, args.min_train_len)\n labels = data.dataset_labels(train_set)\n\n save_filename = os.path.join(args.save_dir, args.save_name)\n if checkpoint_file is None:\n checkpoint_file = utils.checkpoint_name(args.save_dir, save_filename, args.checkpoint_save_name)\n classifier.train_model(trainer, save_filename, checkpoint_file, args, train_set, dev_set, labels)\n return trainer, save_filename, checkpoint_file\n\n def test_build_model(self, tmp_path, fake_embeddings, train_file, dev_file):\n \"\"\"\n Test that building a basic model works\n \"\"\"\n self.build_model(tmp_path, fake_embeddings, train_file, dev_file, extra_args=[\"--bilstm_hidden_dim\", \"20\"])\n\n def test_save_load(self, tmp_path, fake_embeddings, train_file, dev_file):\n \"\"\"\n Test that a basic model can save & load\n \"\"\"\n trainer, _, args = self.build_model(tmp_path, fake_embeddings, train_file, dev_file, extra_args=[\"--bilstm_hidden_dim\", \"20\"])\n\n save_filename = os.path.join(args.save_dir, args.save_name)\n trainer.save(save_filename)\n\n args.load_name = args.save_name\n trainer = Trainer.load(args.load_name, args)\n args.load_name = save_filename\n trainer = Trainer.load(args.load_name, args)\n\n def test_train_basic(self, tmp_path, fake_embeddings, train_file, dev_file):\n self.run_training(tmp_path, fake_embeddings, train_file, dev_file, extra_args=[\"--bilstm_hidden_dim\", \"20\"])\n\n def test_train_bilstm(self, tmp_path, fake_embeddings, train_file, dev_file):\n \"\"\"\n Test w/ and w/o bilstm variations of the classifier\n \"\"\"\n args = [\"--bilstm\", \"--bilstm_hidden_dim\", \"20\"]\n self.run_training(tmp_path, fake_embeddings, train_file, dev_file, args)\n\n args = [\"--no_bilstm\"]\n self.run_training(tmp_path, fake_embeddings, train_file, dev_file, args)\n\n def test_train_maxpool_width(self, tmp_path, fake_embeddings, train_file, dev_file):\n \"\"\"\n Test various maxpool widths\n\n Also sets --filter_channels to a multiple of 2 but not of 3 for\n the test to make sure the math is done correctly on a non-divisible width\n \"\"\"\n args = [\"--maxpool_width\", \"1\", \"--filter_channels\", \"20\", \"--bilstm_hidden_dim\", \"20\"]\n self.run_training(tmp_path, fake_embeddings, train_file, dev_file, args)\n\n args = [\"--maxpool_width\", \"2\", \"--filter_channels\", \"20\", \"--bilstm_hidden_dim\", \"20\"]\n self.run_training(tmp_path, fake_embeddings, train_file, dev_file, args)\n\n args = [\"--maxpool_width\", \"3\", \"--filter_channels\", \"20\", \"--bilstm_hidden_dim\", \"20\"]\n self.run_training(tmp_path, fake_embeddings, train_file, dev_file, args)\n\n def test_train_conv_2d(self, tmp_path, fake_embeddings, train_file, dev_file):\n args = [\"--filter_sizes\", \"(3,4,5)\", \"--filter_channels\", \"20\", \"--bilstm_hidden_dim\", \"20\"]\n self.run_training(tmp_path, fake_embeddings, train_file, dev_file, args)\n\n args = [\"--filter_sizes\", \"((3,2),)\", \"--filter_channels\", \"20\", \"--bilstm_hidden_dim\", \"20\"]\n self.run_training(tmp_path, fake_embeddings, train_file, dev_file, args)\n\n args = [\"--filter_sizes\", \"((3,2),3)\", \"--filter_channels\", \"20\", \"--bilstm_hidden_dim\", \"20\"]\n self.run_training(tmp_path, fake_embeddings, train_file, dev_file, args)\n\n def test_train_filter_channels(self, tmp_path, fake_embeddings, train_file, dev_file):\n args = [\"--filter_sizes\", \"((3,2),3)\", \"--filter_channels\", \"20\", \"--no_bilstm\"]\n trainer, _, _ = self.run_training(tmp_path, fake_embeddings, train_file, dev_file, args)\n assert trainer.model.fc_input_size == 40\n\n args = [\"--filter_sizes\", \"((3,2),3)\", \"--filter_channels\", \"15,20\", \"--no_bilstm\"]\n trainer, _, _ = self.run_training(tmp_path, fake_embeddings, train_file, dev_file, args)\n # 50 = 2x15 for the 2d conv (over 5 dim embeddings) + 20\n assert trainer.model.fc_input_size == 50\n\n def test_train_bert(self, tmp_path, fake_embeddings, train_file, dev_file):\n \"\"\"\n Test on a tiny Bert WITHOUT finetuning, which hopefully does not take up too much disk space or memory\n \"\"\"\n bert_model = \"hf-internal-testing/tiny-bert\"\n\n trainer, save_filename, _ = self.run_training(tmp_path, fake_embeddings, train_file, dev_file, extra_args=[\"--bilstm_hidden_dim\", \"20\", \"--bert_model\", bert_model])\n assert os.path.exists(save_filename)\n saved_model = torch.load(save_filename, lambda storage, loc: storage, weights_only=True)\n # check that the bert model wasn't saved as part of the classifier\n assert not saved_model['params']['config']['force_bert_saved']\n assert not any(x.startswith(\"bert_model\") for x in saved_model['params']['model'].keys())\n\n def test_finetune_bert(self, tmp_path, fake_embeddings, train_file, dev_file):\n \"\"\"\n Test on a tiny Bert WITH finetuning, which hopefully does not take up too much disk space or memory\n \"\"\"\n bert_model = \"hf-internal-testing/tiny-bert\"\n\n trainer, save_filename, _ = self.run_training(tmp_path, fake_embeddings, train_file, dev_file, extra_args=[\"--bilstm_hidden_dim\", \"20\", \"--bert_model\", bert_model, \"--bert_finetune\"])\n assert os.path.exists(save_filename)\n saved_model = torch.load(save_filename, lambda storage, loc: storage, weights_only=True)\n # after finetuning the bert model, make sure that the save file DOES contain parts of the transformer\n assert saved_model['params']['config']['force_bert_saved']\n assert any(x.startswith(\"bert_model\") for x in saved_model['params']['model'].keys())\n\n def test_finetune_bert_layers(self, tmp_path, fake_embeddings, train_file, dev_file):\n \"\"\"Test on a tiny Bert WITH finetuning, which hopefully does not take up too much disk space or memory, using 2 layers\n\n As an added bonus (or eager test), load the finished model and continue\n training from there. Then check that the initial model and\n the middle model are different, then that the middle model and\n final model are different\n\n \"\"\"\n bert_model = \"hf-internal-testing/tiny-bert\"\n\n trainer, save_filename, checkpoint_file = self.run_training(tmp_path, fake_embeddings, train_file, dev_file, extra_args=[\"--bilstm_hidden_dim\", \"20\", \"--bert_model\", bert_model, \"--bert_finetune\", \"--bert_hidden_layers\", \"2\", \"--save_intermediate_models\"])\n assert os.path.exists(save_filename)\n\n save_path = os.path.split(save_filename)[0]\n\n initial_model = glob.glob(os.path.join(save_path, \"*E0000*\"))\n assert len(initial_model) == 1\n initial_model = initial_model[0]\n initial_model = torch.load(initial_model, lambda storage, loc: storage, weights_only=True)\n\n second_model_file = glob.glob(os.path.join(save_path, \"*E0002*\"))\n assert len(second_model_file) == 1\n second_model_file = second_model_file[0]\n second_model = torch.load(second_model_file, lambda storage, loc: storage, weights_only=True)\n\n for layer_idx in range(2):\n bert_names = [x for x in second_model['params']['model'].keys() if x.startswith(\"bert_model\") and \"layer.%d.\" % layer_idx in x]\n assert len(bert_names) > 0\n assert all(x in initial_model['params']['model'] and x in second_model['params']['model'] for x in bert_names)\n assert not all(torch.allclose(initial_model['params']['model'].get(x), second_model['params']['model'].get(x)) for x in bert_names)\n\n # put some random marker in the file to look for later,\n # check the continued training didn't clobber the expected file\n assert \"asdf\" not in second_model\n second_model[\"asdf\"] = 1234\n torch.save(second_model, second_model_file)\n\n trainer, save_filename, checkpoint_file = self.run_training(tmp_path, fake_embeddings, train_file, dev_file, extra_args=[\"--bilstm_hidden_dim\", \"20\", \"--bert_model\", bert_model, \"--bert_finetune\", \"--bert_hidden_layers\", \"2\", \"--save_intermediate_models\", \"--max_epochs\", \"5\"], checkpoint_file=checkpoint_file)\n\n second_model_file_redo = glob.glob(os.path.join(save_path, \"*E0002*\"))\n assert len(second_model_file_redo) == 1\n assert second_model_file == second_model_file_redo[0]\n second_model = torch.load(second_model_file, lambda storage, loc: storage, weights_only=True)\n assert \"asdf\" in second_model\n\n fifth_model_file = glob.glob(os.path.join(save_path, \"*E0005*\"))\n assert len(fifth_model_file) == 1\n\n final_model = torch.load(fifth_model_file[0], lambda storage, loc: storage, weights_only=True)\n for layer_idx in range(2):\n bert_names = [x for x in final_model['params']['model'].keys() if x.startswith(\"bert_model\") and \"layer.%d.\" % layer_idx in x]\n assert len(bert_names) > 0\n assert all(x in final_model['params']['model'] and x in second_model['params']['model'] for x in bert_names)\n assert not all(torch.allclose(final_model['params']['model'].get(x), second_model['params']['model'].get(x)) for x in bert_names)\n\n def test_finetune_peft(self, tmp_path, fake_embeddings, train_file, dev_file):\n \"\"\"\n Test on a tiny Bert with PEFT finetuning\n \"\"\"\n bert_model = \"hf-internal-testing/tiny-bert\"\n\n trainer, save_filename, _ = self.run_training(tmp_path, fake_embeddings, train_file, dev_file, extra_args=[\"--bilstm_hidden_dim\", \"20\", \"--bert_model\", bert_model, \"--bert_finetune\", \"--use_peft\", \"--lora_modules_to_save\", \"pooler\"])\n assert os.path.exists(save_filename)\n saved_model = torch.load(save_filename, lambda storage, loc: storage, weights_only=True)\n # after finetuning the bert model, make sure that the save file DOES contain parts of the transformer, but only in peft form\n assert saved_model['params']['config']['bert_model'] == bert_model\n assert saved_model['params']['config']['force_bert_saved']\n assert saved_model['params']['config']['use_peft']\n\n assert not saved_model['params']['config']['has_charlm_forward']\n assert not saved_model['params']['config']['has_charlm_backward']\n\n assert len(saved_model['params']['bert_lora']) > 0\n assert any(x.find(\".pooler.\") >= 0 for x in saved_model['params']['bert_lora'])\n assert any(x.find(\".encoder.\") >= 0 for x in saved_model['params']['bert_lora'])\n assert not any(x.startswith(\"bert_model\") for x in saved_model['params']['model'].keys())\n\n # The Pipeline should load and run a PEFT trained model,\n # although obviously we don't expect the results to do\n # anything correct\n pipeline = stanza.Pipeline(\"en\", download_method=None, model_dir=TEST_MODELS_DIR, processors=\"tokenize,sentiment\", sentiment_model_path=save_filename, sentiment_pretrain_path=str(fake_embeddings))\n doc = pipeline(\"This is a test\")\n\n def test_finetune_peft_restart(self, tmp_path, fake_embeddings, train_file, dev_file):\n \"\"\"\n Test that if we restart training on a peft model, the peft weights change\n \"\"\"\n bert_model = \"hf-internal-testing/tiny-bert\"\n\n trainer, save_file, checkpoint_file = self.run_training(tmp_path, fake_embeddings, train_file, dev_file, extra_args=[\"--bilstm_hidden_dim\", \"20\", \"--bert_model\", bert_model, \"--bert_finetune\", \"--use_peft\", \"--lora_modules_to_save\", \"pooler\", \"--save_intermediate_models\"])\n\n assert os.path.exists(save_file)\n saved_model = torch.load(save_file, lambda storage, loc: storage, weights_only=True)\n assert any(x.find(\".encoder.\") >= 0 for x in saved_model['params']['bert_lora'])\n\n\n trainer, save_file, checkpoint_file = self.run_training(tmp_path, fake_embeddings, train_file, dev_file, extra_args=[\"--bilstm_hidden_dim\", \"20\", \"--bert_model\", bert_model, \"--bert_finetune\", \"--use_peft\", \"--lora_modules_to_save\", \"pooler\", \"--save_intermediate_models\", \"--max_epochs\", \"5\"], checkpoint_file=checkpoint_file)\n\n save_path = os.path.split(save_file)[0]\n\n initial_model_file = glob.glob(os.path.join(save_path, \"*E0000*\"))\n assert len(initial_model_file) == 1\n initial_model_file = initial_model_file[0]\n initial_model = torch.load(initial_model_file, lambda storage, loc: storage, weights_only=True)\n\n second_model_file = glob.glob(os.path.join(save_path, \"*E0002*\"))\n assert len(second_model_file) == 1\n second_model_file = second_model_file[0]\n second_model = torch.load(second_model_file, lambda storage, loc: storage, weights_only=True)\n\n final_model_file = glob.glob(os.path.join(save_path, \"*E0005*\"))\n assert len(final_model_file) == 1\n final_model_file = final_model_file[0]\n final_model = torch.load(final_model_file, lambda storage, loc: storage, weights_only=True)\n\n # params in initial_model & second_model start with \"base_model.model.\"\n # whereas params in final_model start directly with \"encoder\" or \"pooler\"\n initial_lora = initial_model['params']['bert_lora']\n second_lora = second_model['params']['bert_lora']\n final_lora = final_model['params']['bert_lora']\n for side in (\"_A.\", \"_B.\"):\n for layer in (\".0.\", \".1.\"):\n initial_params = sorted([x for x in initial_lora if x.find(\".encoder.\") > 0 and x.find(side) > 0 and x.find(layer) > 0])\n second_params = sorted([x for x in second_lora if x.find(\".encoder.\") > 0 and x.find(side) > 0 and x.find(layer) > 0])\n final_params = sorted([x for x in final_lora if x.startswith(\"encoder.\") > 0 and x.find(side) > 0 and x.find(layer) > 0])\n assert len(initial_params) > 0\n assert len(initial_params) == len(second_params)\n assert len(initial_params) == len(final_params)\n for x, y in zip(second_params, final_params):\n assert x.endswith(y)\n if side != \"_A.\": # the A tensors don't move very much, if at all\n assert not torch.allclose(initial_lora.get(x), second_lora.get(x))\n assert not torch.allclose(second_lora.get(x), final_lora.get(y))\n\n"
},
{
"path": "stanza/tests/classifiers/test_constituency_classifier.py",
"content": "import os\n\nimport pytest\n\nimport stanza\nimport stanza.models.classifier as classifier\nimport stanza.models.classifiers.data as data\nfrom stanza.models.classifiers.trainer import Trainer\nfrom stanza.tests import TEST_MODELS_DIR\nfrom stanza.tests.classifiers.test_classifier import fake_embeddings\nfrom stanza.tests.classifiers.test_data import train_file_with_trees, dev_file_with_trees\nfrom stanza.models.common import utils\nfrom stanza.tests.constituency.test_trainer import build_trainer, TREEBANK\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\nclass TestConstituencyClassifier:\n @pytest.fixture(scope=\"class\")\n def constituency_model(self, fake_embeddings, tmp_path_factory):\n args = ['--pattn_num_layers', '0', '--lattn_d_proj', '0', '--hidden_size', '20', '--delta_embedding_dim', '10']\n trainer = build_trainer(str(fake_embeddings), *args, treebank=TREEBANK)\n\n trainer_pt = str(tmp_path_factory.mktemp(\"constituency\") / \"constituency.pt\")\n trainer.save(trainer_pt, save_optimizer=False)\n return trainer_pt\n\n def build_model(self, tmp_path, constituency_model, fake_embeddings, train_file_with_trees, dev_file_with_trees, extra_args=None):\n \"\"\"\n Build a Constituency Classifier model to be used by one of the later tests\n \"\"\"\n save_dir = str(tmp_path / \"classifier\")\n save_name = \"model.pt\"\n args = [\"--save_dir\", save_dir,\n \"--save_name\", save_name,\n \"--model_type\", \"constituency\",\n \"--constituency_model\", constituency_model,\n \"--wordvec_pretrain_file\", str(fake_embeddings),\n \"--fc_shapes\", \"20,10\",\n \"--train_file\", str(train_file_with_trees),\n \"--dev_file\", str(dev_file_with_trees),\n \"--max_epochs\", \"2\",\n \"--batch_size\", \"60\"]\n if extra_args is not None:\n args = args + extra_args\n args = classifier.parse_args(args)\n train_set = data.read_dataset(args.train_file, args.wordvec_type, args.min_train_len)\n trainer = Trainer.build_new_model(args, train_set)\n return trainer, train_set, args\n\n def run_training(self, tmp_path, constituency_model, fake_embeddings, train_file_with_trees, dev_file_with_trees, extra_args=None):\n \"\"\"\n Iterate a couple times over a model\n \"\"\"\n trainer, train_set, args = self.build_model(tmp_path, constituency_model, fake_embeddings, train_file_with_trees, dev_file_with_trees, extra_args)\n dev_set = data.read_dataset(args.dev_file, args.wordvec_type, args.min_train_len)\n labels = data.dataset_labels(train_set)\n\n save_filename = os.path.join(args.save_dir, args.save_name)\n checkpoint_file = utils.checkpoint_name(args.save_dir, save_filename, args.checkpoint_save_name)\n classifier.train_model(trainer, save_filename, checkpoint_file, args, train_set, dev_set, labels)\n return trainer, train_set, args\n\n def test_build_model(self, tmp_path, constituency_model, fake_embeddings, train_file_with_trees, dev_file_with_trees):\n \"\"\"\n Test that building a basic constituency-based model works\n \"\"\"\n self.build_model(tmp_path, constituency_model, fake_embeddings, train_file_with_trees, dev_file_with_trees)\n\n def test_save_load(self, tmp_path, constituency_model, fake_embeddings, train_file_with_trees, dev_file_with_trees):\n \"\"\"\n Test that a constituency model can save & load\n \"\"\"\n trainer, _, args = self.build_model(tmp_path, constituency_model, fake_embeddings, train_file_with_trees, dev_file_with_trees)\n\n save_filename = os.path.join(args.save_dir, args.save_name)\n trainer.save(save_filename)\n\n args.load_name = args.save_name\n trainer = Trainer.load(args.load_name, args)\n\n def test_train_basic(self, tmp_path, constituency_model, fake_embeddings, train_file_with_trees, dev_file_with_trees):\n self.run_training(tmp_path, constituency_model, fake_embeddings, train_file_with_trees, dev_file_with_trees)\n\n def test_train_pipeline(self, tmp_path, constituency_model, fake_embeddings, train_file_with_trees, dev_file_with_trees):\n \"\"\"\n Test that writing out a temp model, then loading it in the pipeline is a thing that works\n \"\"\"\n trainer, _, args = self.run_training(tmp_path, constituency_model, fake_embeddings, train_file_with_trees, dev_file_with_trees)\n save_filename = os.path.join(args.save_dir, args.save_name)\n assert os.path.exists(save_filename)\n assert os.path.exists(args.constituency_model)\n\n pipeline_args = {\"lang\": \"en\",\n \"download_method\": None,\n \"model_dir\": TEST_MODELS_DIR,\n \"processors\": \"tokenize,pos,constituency,sentiment\",\n \"tokenize_pretokenized\": True,\n \"constituency_model_path\": args.constituency_model,\n \"constituency_pretrain_path\": args.wordvec_pretrain_file,\n \"constituency_backward_charlm_path\": None,\n \"constituency_forward_charlm_path\": None,\n \"sentiment_model_path\": save_filename,\n \"sentiment_pretrain_path\": args.wordvec_pretrain_file,\n \"sentiment_backward_charlm_path\": None,\n \"sentiment_forward_charlm_path\": None}\n pipeline = stanza.Pipeline(**pipeline_args)\n doc = pipeline(\"This is a test\")\n # since the model is random, we have no expectations for what the result actually is\n assert doc.sentences[0].sentiment is not None\n\n\n def test_train_all_words(self, tmp_path, constituency_model, fake_embeddings, train_file_with_trees, dev_file_with_trees):\n self.run_training(tmp_path, constituency_model, fake_embeddings, train_file_with_trees, dev_file_with_trees, ['--constituency_all_words'])\n\n self.run_training(tmp_path, constituency_model, fake_embeddings, train_file_with_trees, dev_file_with_trees, ['--no_constituency_all_words'])\n\n def test_train_top_layer(self, tmp_path, constituency_model, fake_embeddings, train_file_with_trees, dev_file_with_trees):\n self.run_training(tmp_path, constituency_model, fake_embeddings, train_file_with_trees, dev_file_with_trees, ['--constituency_top_layer'])\n\n self.run_training(tmp_path, constituency_model, fake_embeddings, train_file_with_trees, dev_file_with_trees, ['--no_constituency_top_layer'])\n\n def test_train_attn(self, tmp_path, constituency_model, fake_embeddings, train_file_with_trees, dev_file_with_trees):\n self.run_training(tmp_path, constituency_model, fake_embeddings, train_file_with_trees, dev_file_with_trees, ['--constituency_node_attn', '--no_constituency_all_words'])\n\n self.run_training(tmp_path, constituency_model, fake_embeddings, train_file_with_trees, dev_file_with_trees, ['--constituency_node_attn', '--constituency_all_words'])\n\n self.run_training(tmp_path, constituency_model, fake_embeddings, train_file_with_trees, dev_file_with_trees, ['--no_constituency_node_attn'])\n\n"
},
{
"path": "stanza/tests/classifiers/test_data.py",
"content": "import json\nimport pytest\n\nimport stanza.models.classifiers.data as data\nfrom stanza.models.classifiers.utils import WVType\nfrom stanza.models.common.vocab import PAD, UNK\nfrom stanza.models.constituency.parse_tree import Tree\n\nSENTENCES = [\n [\"I\", \"hate\", \"the\", \"Opal\", \"banning\"],\n [\"Tell\", \"my\", \"wife\", \"hello\"], # obviously this is the neutral result\n [\"I\", \"like\", \"Sh'reyan\", \"'s\", \"antennae\"],\n]\n\nDATASET = [\n {\"sentiment\": \"0\", \"text\": SENTENCES[0]},\n {\"sentiment\": \"1\", \"text\": SENTENCES[1]},\n {\"sentiment\": \"2\", \"text\": SENTENCES[2]},\n]\n\nTREES = [\n \"(ROOT (S (NP (PRP I)) (VP (VBP hate) (NP (DT the) (NN Opal) (NN banning)))))\",\n \"(ROOT (S (VP (VB Tell) (NP (PRP$ my) (NN wife)) (NP (UH hello)))))\",\n \"(ROOT (S (NP (PRP I)) (VP (VBP like) (NP (NP (NNP Sh'reyan) (POS 's)) (NNS antennae)))))\",\n]\n\nDATASET_WITH_TREES = [\n {\"sentiment\": \"0\", \"text\": SENTENCES[0], \"constituency\": TREES[0]},\n {\"sentiment\": \"1\", \"text\": SENTENCES[1], \"constituency\": TREES[1]},\n {\"sentiment\": \"2\", \"text\": SENTENCES[2], \"constituency\": TREES[2]},\n]\n\n@pytest.fixture(scope=\"module\")\ndef train_file(tmp_path_factory):\n train_set = DATASET * 20\n train_filename = tmp_path_factory.mktemp(\"data\") / \"train.json\"\n with open(train_filename, \"w\", encoding=\"utf-8\") as fout:\n json.dump(train_set, fout, ensure_ascii=False)\n return train_filename\n\n@pytest.fixture(scope=\"module\")\ndef dev_file(tmp_path_factory):\n dev_set = DATASET * 2\n dev_filename = tmp_path_factory.mktemp(\"data\") / \"dev.json\"\n with open(dev_filename, \"w\", encoding=\"utf-8\") as fout:\n json.dump(dev_set, fout, ensure_ascii=False)\n return dev_filename\n\n@pytest.fixture(scope=\"module\")\ndef test_file(tmp_path_factory):\n test_set = DATASET\n test_filename = tmp_path_factory.mktemp(\"data\") / \"test.json\"\n with open(test_filename, \"w\", encoding=\"utf-8\") as fout:\n json.dump(test_set, fout, ensure_ascii=False)\n return test_filename\n\n@pytest.fixture(scope=\"module\")\ndef train_file_with_trees(tmp_path_factory):\n train_set = DATASET_WITH_TREES * 20\n train_filename = tmp_path_factory.mktemp(\"data\") / \"train_trees.json\"\n with open(train_filename, \"w\", encoding=\"utf-8\") as fout:\n json.dump(train_set, fout, ensure_ascii=False)\n return train_filename\n\n@pytest.fixture(scope=\"module\")\ndef dev_file_with_trees(tmp_path_factory):\n dev_set = DATASET_WITH_TREES * 2\n dev_filename = tmp_path_factory.mktemp(\"data\") / \"dev_trees.json\"\n with open(dev_filename, \"w\", encoding=\"utf-8\") as fout:\n json.dump(dev_set, fout, ensure_ascii=False)\n return dev_filename\n\nclass TestClassifierData:\n def test_read_data(self, train_file):\n \"\"\"\n Test reading of the json format\n \"\"\"\n train_set = data.read_dataset(str(train_file), WVType.OTHER, 1)\n assert len(train_set) == 60\n\n def test_read_data_with_trees(self, train_file, train_file_with_trees):\n \"\"\"\n Test reading of the json format\n \"\"\"\n train_trees_set = data.read_dataset(str(train_file_with_trees), WVType.OTHER, 1)\n assert len(train_trees_set) == 60\n for idx, x in enumerate(train_trees_set):\n assert isinstance(x.constituency, Tree)\n assert str(x.constituency) == TREES[idx % len(TREES)]\n\n train_set = data.read_dataset(str(train_file), WVType.OTHER, 1)\n\n def test_dataset_vocab(self, train_file):\n \"\"\"\n Converting a dataset to vocab should have a specific set of words along with PAD and UNK\n \"\"\"\n train_set = data.read_dataset(str(train_file), WVType.OTHER, 1)\n vocab = data.dataset_vocab(train_set)\n expected = set([PAD, UNK] + [x.lower() for y in SENTENCES for x in y])\n assert set(vocab) == expected\n\n def test_dataset_labels(self, train_file):\n \"\"\"\n Test the extraction of labels from a dataset\n \"\"\"\n train_set = data.read_dataset(str(train_file), WVType.OTHER, 1)\n labels = data.dataset_labels(train_set)\n assert labels == [\"0\", \"1\", \"2\"]\n\n def test_sort_by_length(self, train_file):\n \"\"\"\n There are two unique lengths in the toy dataset\n \"\"\"\n train_set = data.read_dataset(str(train_file), WVType.OTHER, 1)\n sorted_dataset = data.sort_dataset_by_len(train_set)\n assert list(sorted_dataset.keys()) == [4, 5]\n assert len(sorted_dataset[4]) == len(train_set) // 3\n assert len(sorted_dataset[5]) == 2 * len(train_set) // 3\n\n def test_check_labels(self, train_file):\n \"\"\"\n Check that an exception is thrown for an unknown label\n \"\"\"\n train_set = data.read_dataset(str(train_file), WVType.OTHER, 1)\n labels = sorted(set([x[\"sentiment\"] for x in DATASET]))\n assert len(labels) > 1\n data.check_labels(labels, train_set)\n with pytest.raises(RuntimeError):\n data.check_labels(labels[:1], train_set)\n\n"
},
{
"path": "stanza/tests/classifiers/test_process_utils.py",
"content": "\"\"\"\nA few tests of the utils module for the sentiment datasets\n\"\"\"\n\nimport os\nimport pytest\n\nimport stanza\n\nfrom stanza.models.classifiers import data\nfrom stanza.models.classifiers.data import SentimentDatum\nfrom stanza.models.classifiers.utils import WVType\nfrom stanza.utils.datasets.sentiment import process_utils\n\nfrom stanza.tests import TEST_MODELS_DIR\nfrom stanza.tests.classifiers.test_data import train_file, dev_file, test_file\n\n\ndef test_write_list(tmp_path, train_file):\n \"\"\"\n Test that writing a single list of items to an output file works\n \"\"\"\n train_set = data.read_dataset(train_file, WVType.OTHER, 1)\n\n dataset_file = tmp_path / \"foo.json\"\n process_utils.write_list(dataset_file, train_set)\n\n train_copy = data.read_dataset(dataset_file, WVType.OTHER, 1)\n assert train_copy == train_set\n\ndef test_write_dataset(tmp_path, train_file, dev_file, test_file):\n \"\"\"\n Test that writing all three parts of a dataset works\n \"\"\"\n dataset = [data.read_dataset(filename, WVType.OTHER, 1) for filename in (train_file, dev_file, test_file)]\n process_utils.write_dataset(dataset, tmp_path, \"en_test\")\n\n expected_files = ['en_test.train.json', 'en_test.dev.json', 'en_test.test.json']\n dataset_files = os.listdir(tmp_path)\n assert sorted(dataset_files) == sorted(expected_files)\n\n for filename, expected in zip(expected_files, dataset):\n written = data.read_dataset(tmp_path / filename, WVType.OTHER, 1)\n assert written == expected\n\ndef test_read_snippets(tmp_path):\n \"\"\"\n Test the basic operation of the read_snippets function\n \"\"\"\n filename = tmp_path / \"foo.csv\"\n with open(filename, \"w\", encoding=\"utf-8\") as fout:\n fout.write(\"FOO\\tThis is a test\\thappy\\n\")\n fout.write(\"FOO\\tThis is a second sentence\\tsad\\n\")\n\n nlp = stanza.Pipeline(\"en\", dir=TEST_MODELS_DIR, processors=\"tokenize\", download_method=None)\n\n mapping = {\"happy\": 0, \"sad\": 1}\n\n snippets = process_utils.read_snippets(filename, 2, 1, \"en\", mapping, nlp=nlp)\n assert len(snippets) == 2\n assert snippets == [SentimentDatum(sentiment=0, text=['This', 'is', 'a', 'test']),\n SentimentDatum(sentiment=1, text=['This', 'is', 'a', 'second', 'sentence'])]\n\ndef test_read_snippets_two_columns(tmp_path):\n \"\"\"\n Test what happens when multiple columns are combined for the sentiment value\n \"\"\"\n filename = tmp_path / \"foo.csv\"\n with open(filename, \"w\", encoding=\"utf-8\") as fout:\n fout.write(\"FOO\\tThis is a test\\thappy\\tfoo\\n\")\n fout.write(\"FOO\\tThis is a second sentence\\tsad\\tbar\\n\")\n fout.write(\"FOO\\tThis is a third sentence\\tsad\\tfoo\\n\")\n\n nlp = stanza.Pipeline(\"en\", dir=TEST_MODELS_DIR, processors=\"tokenize\", download_method=None)\n\n mapping = {(\"happy\", \"foo\"): 0, (\"sad\", \"bar\"): 1, (\"sad\", \"foo\"): 2}\n\n snippets = process_utils.read_snippets(filename, (2,3), 1, \"en\", mapping, nlp=nlp)\n assert len(snippets) == 3\n assert snippets == [SentimentDatum(sentiment=0, text=['This', 'is', 'a', 'test']),\n SentimentDatum(sentiment=1, text=['This', 'is', 'a', 'second', 'sentence']),\n SentimentDatum(sentiment=2, text=['This', 'is', 'a', 'third', 'sentence'])]\n\n"
},
{
"path": "stanza/tests/common/__init__.py",
"content": ""
},
{
"path": "stanza/tests/common/test_bert_embedding.py",
"content": "import pytest\nimport torch\n\nfrom stanza.models.common.bert_embedding import load_bert, extract_bert_embeddings\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\nBERT_MODEL = \"hf-internal-testing/tiny-bert\"\n\n@pytest.fixture(scope=\"module\")\ndef tiny_bert():\n m, t = load_bert(BERT_MODEL)\n return m, t\n\ndef test_load_bert(tiny_bert):\n \"\"\"\n Empty method that just tests loading the bert\n \"\"\"\n m, t = tiny_bert\n\ndef test_run_bert(tiny_bert):\n m, t = tiny_bert\n device = next(m.parameters()).device\n extract_bert_embeddings(BERT_MODEL, t, m, [[\"This\", \"is\", \"a\", \"test\"]], device, True)\n\ndef test_run_bert_empty_word(tiny_bert):\n m, t = tiny_bert\n device = next(m.parameters()).device\n foo = extract_bert_embeddings(BERT_MODEL, t, m, [[\"This\", \"is\", \"-\", \"a\", \"test\"]], device, True)\n bar = extract_bert_embeddings(BERT_MODEL, t, m, [[\"This\", \"is\", \"\", \"a\", \"test\"]], device, True)\n\n assert len(foo) == 1\n assert torch.allclose(foo[0], bar[0])\n"
},
{
"path": "stanza/tests/common/test_char_model.py",
"content": "\"\"\"\nCurrently tests a few configurations of files for creating a charlm vocab\n\nAlso has a skeleton test of loading & saving a charlm\n\"\"\"\n\nfrom collections import Counter\nimport glob\nimport lzma\nimport os\nimport tempfile\n\nimport pytest\n\nfrom stanza.models import charlm\nfrom stanza.models.common import char_model\nfrom stanza.tests import TEST_MODELS_DIR\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\nfake_text_1 = \"\"\"\nUnban mox opal!\nI hate watching Peppa Pig\n\"\"\"\n\nfake_text_2 = \"\"\"\nThis is plastic cheese\n\"\"\"\n\nclass TestCharModel:\n def test_single_file_vocab(self):\n with tempfile.TemporaryDirectory() as tempdir:\n sample_file = os.path.join(tempdir, \"text.txt\")\n with open(sample_file, \"w\", encoding=\"utf-8\") as fout:\n fout.write(fake_text_1)\n vocab = char_model.build_charlm_vocab(sample_file)\n\n for i in fake_text_1:\n assert i in vocab\n assert \"Q\" not in vocab\n\n def test_single_file_xz_vocab(self):\n with tempfile.TemporaryDirectory() as tempdir:\n sample_file = os.path.join(tempdir, \"text.txt.xz\")\n with lzma.open(sample_file, \"wt\", encoding=\"utf-8\") as fout:\n fout.write(fake_text_1)\n vocab = char_model.build_charlm_vocab(sample_file)\n\n for i in fake_text_1:\n assert i in vocab\n assert \"Q\" not in vocab\n\n def test_single_file_dir_vocab(self):\n with tempfile.TemporaryDirectory() as tempdir:\n sample_file = os.path.join(tempdir, \"text.txt\")\n with open(sample_file, \"w\", encoding=\"utf-8\") as fout:\n fout.write(fake_text_1)\n vocab = char_model.build_charlm_vocab(tempdir)\n\n for i in fake_text_1:\n assert i in vocab\n assert \"Q\" not in vocab\n\n def test_multiple_files_vocab(self):\n with tempfile.TemporaryDirectory() as tempdir:\n sample_file = os.path.join(tempdir, \"t1.txt\")\n with open(sample_file, \"w\", encoding=\"utf-8\") as fout:\n fout.write(fake_text_1)\n sample_file = os.path.join(tempdir, \"t2.txt.xz\")\n with lzma.open(sample_file, \"wt\", encoding=\"utf-8\") as fout:\n fout.write(fake_text_2)\n vocab = char_model.build_charlm_vocab(tempdir)\n\n for i in fake_text_1:\n assert i in vocab\n for i in fake_text_2:\n assert i in vocab\n assert \"Q\" not in vocab\n\n def test_cutoff_vocab(self):\n with tempfile.TemporaryDirectory() as tempdir:\n sample_file = os.path.join(tempdir, \"t1.txt\")\n with open(sample_file, \"w\", encoding=\"utf-8\") as fout:\n fout.write(fake_text_1)\n sample_file = os.path.join(tempdir, \"t2.txt.xz\")\n with lzma.open(sample_file, \"wt\", encoding=\"utf-8\") as fout:\n fout.write(fake_text_2)\n\n vocab = char_model.build_charlm_vocab(tempdir, cutoff=2)\n\n counts = Counter(fake_text_1) + Counter(fake_text_2)\n for letter, count in counts.most_common():\n if count < 2:\n assert letter not in vocab\n else:\n assert letter in vocab\n\n def test_build_model(self):\n \"\"\"\n Test the whole thing on a small dataset for an iteration or two\n \"\"\"\n with tempfile.TemporaryDirectory() as tempdir:\n eval_file = os.path.join(tempdir, \"en_test.dev.txt\")\n with open(eval_file, \"w\", encoding=\"utf-8\") as fout:\n fout.write(fake_text_1)\n train_file = os.path.join(tempdir, \"en_test.train.txt\")\n with open(train_file, \"w\", encoding=\"utf-8\") as fout:\n for i in range(1000):\n fout.write(fake_text_1)\n fout.write(\"\\n\")\n fout.write(fake_text_2)\n fout.write(\"\\n\")\n save_name = 'en_test.forward.pt'\n vocab_save_name = 'en_text.vocab.pt'\n checkpoint_save_name = 'en_text.checkpoint.pt'\n args = ['--train_file', train_file,\n '--eval_file', eval_file,\n '--eval_steps', '0', # eval once per opoch\n '--epochs', '2',\n '--cutoff', '1',\n '--batch_size', '%d' % len(fake_text_1),\n '--shorthand', 'en_test',\n '--save_dir', tempdir,\n '--save_name', save_name,\n '--vocab_save_name', vocab_save_name,\n '--checkpoint_save_name', checkpoint_save_name]\n args = charlm.parse_args(args)\n charlm.train(args)\n\n assert os.path.exists(os.path.join(tempdir, vocab_save_name))\n\n # test that saving & loading of the model worked\n assert os.path.exists(os.path.join(tempdir, save_name))\n model = char_model.CharacterLanguageModel.load(os.path.join(tempdir, save_name))\n\n # test that saving & loading of the checkpoint worked\n assert os.path.exists(os.path.join(tempdir, checkpoint_save_name))\n model = char_model.CharacterLanguageModel.load(os.path.join(tempdir, checkpoint_save_name))\n trainer = char_model.CharacterLanguageModelTrainer.load(args, os.path.join(tempdir, checkpoint_save_name))\n\n assert trainer.global_step > 0\n assert trainer.epoch == 2\n\n # quick test to verify this method works with a trained model\n charlm.get_current_lr(trainer, args)\n\n # test loading a vocab built by the training method...\n vocab = charlm.load_char_vocab(os.path.join(tempdir, vocab_save_name))\n trainer = char_model.CharacterLanguageModelTrainer.from_new_model(args, vocab)\n # ... and test the get_current_lr for an untrained model as well\n # this test is super \"eager\"\n assert charlm.get_current_lr(trainer, args) == args['lr0']\n\n @pytest.fixture(scope=\"class\")\n def english_forward(self):\n # eg, stanza_test/models/en/forward_charlm/1billion.pt\n models_path = os.path.join(TEST_MODELS_DIR, \"en\", \"forward_charlm\", \"*\")\n models = glob.glob(models_path)\n # we expect at least one English model downloaded for the tests\n assert len(models) >= 1\n model_file = models[0]\n return char_model.CharacterLanguageModel.load(model_file)\n\n @pytest.fixture(scope=\"class\")\n def english_backward(self):\n # eg, stanza_test/models/en/forward_charlm/1billion.pt\n models_path = os.path.join(TEST_MODELS_DIR, \"en\", \"backward_charlm\", \"*\")\n models = glob.glob(models_path)\n # we expect at least one English model downloaded for the tests\n assert len(models) >= 1\n model_file = models[0]\n return char_model.CharacterLanguageModel.load(model_file)\n\n def test_load_model(self, english_forward, english_backward):\n \"\"\"\n Check that basic loading functions work\n \"\"\"\n assert english_forward.is_forward_lm\n assert not english_backward.is_forward_lm\n\n def test_save_load_model(self, english_forward, english_backward):\n \"\"\"\n Load, save, and load again\n \"\"\"\n with tempfile.TemporaryDirectory() as tempdir:\n for model in (english_forward, english_backward):\n save_file = os.path.join(tempdir, \"resaved\", \"charlm.pt\")\n model.save(save_file)\n reloaded = char_model.CharacterLanguageModel.load(save_file)\n assert model.is_forward_lm == reloaded.is_forward_lm\n"
},
{
"path": "stanza/tests/common/test_chuliu_edmonds.py",
"content": "\"\"\"\nTest some use cases of the chuliu_edmonds algorithm\n\n(currently just the tarjan implementation)\n\"\"\"\n\nimport numpy as np\nimport pytest\n\nfrom stanza.models.common.chuliu_edmonds import tarjan\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\ndef test_tarjan_basic():\n simple = np.array([0, 4, 4, 4, 0])\n result = tarjan(simple)\n assert result == []\n\n simple = np.array([0, 2, 0, 4, 2, 2])\n result = tarjan(simple)\n assert result == []\n\ndef test_tarjan_cycle():\n cycle_graph = np.array([0, 3, 1, 2])\n result = tarjan(cycle_graph)\n expected = np.array([False, True, True, True])\n assert len(result) == 1\n np.testing.assert_array_equal(result[0], expected)\n\n cycle_graph = np.array([0, 3, 1, 2, 5, 6, 4])\n result = tarjan(cycle_graph)\n assert len(result) == 2\n expected = [np.array([False, True, True, True, False, False, False]),\n np.array([False, False, False, False, True, True, True])]\n for r, e in zip(result, expected):\n np.testing.assert_array_equal(r, e)\n"
},
{
"path": "stanza/tests/common/test_common_data.py",
"content": "import pytest\nimport stanza\n\nfrom stanza.tests import *\nfrom stanza.models.common.data import get_augment_ratio, augment_punct\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\ndef test_augment_ratio():\n data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n should_augment = lambda x: x >= 3\n can_augment = lambda x: x >= 4\n # check that zero is returned if no augmentation is needed\n # which will be the case since 2 are already satisfactory\n assert get_augment_ratio(data, should_augment, can_augment, desired_ratio=0.1) == 0.0\n\n # this should throw an error\n with pytest.raises(AssertionError):\n get_augment_ratio(data, can_augment, should_augment)\n\n # with a desired ratio of 0.4,\n # there are already 2 that don't need augmenting\n # and 7 that are eligible to be augmented\n # so 2/7 will need to be augmented\n assert get_augment_ratio(data, should_augment, can_augment, desired_ratio=0.4) == pytest.approx(2/7)\n\ndef test_augment_punct():\n data = [[\"Simple\", \"test\", \".\"]]\n should_augment = lambda x: x[-1] == \".\"\n can_augment = should_augment\n new_data = augment_punct(data, 1.0, should_augment, can_augment)\n assert new_data == [[\"Simple\", \"test\"]]\n"
},
{
"path": "stanza/tests/common/test_confusion.py",
"content": "\"\"\"\nTest a couple simple confusion matrices and output formats\n\"\"\"\n\nfrom collections import defaultdict\nimport pytest\n\nfrom stanza.utils.confusion import format_confusion, confusion_to_f1, confusion_to_macro_f1, confusion_to_weighted_f1\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\n@pytest.fixture\ndef simple_confusion():\n confusion = defaultdict(lambda: defaultdict(int))\n confusion[\"B-ORG\"][\"B-ORG\"] = 1\n confusion[\"B-ORG\"][\"B-PER\"] = 1\n confusion[\"E-ORG\"][\"E-ORG\"] = 1\n confusion[\"E-ORG\"][\"E-PER\"] = 1\n confusion[\"O\"][\"O\"] = 4\n return confusion\n\n@pytest.fixture\ndef short_confusion():\n \"\"\"\n Same thing, but with a short name. This should not be sorted by entity type\n \"\"\"\n confusion = defaultdict(lambda: defaultdict(int))\n confusion[\"A\"][\"B-ORG\"] = 1\n confusion[\"B-ORG\"][\"B-PER\"] = 1\n confusion[\"E-ORG\"][\"E-ORG\"] = 1\n confusion[\"E-ORG\"][\"E-PER\"] = 1\n confusion[\"O\"][\"O\"] = 4\n return confusion\n\nEXPECTED_SIMPLE_OUTPUT = \"\"\"\n t\\\\p O B-ORG E-ORG B-PER E-PER\n O 4 0 0 0 0\n B-ORG 0 1 0 1 0\n E-ORG 0 0 1 0 1\n B-PER 0 0 0 0 0\n E-PER 0 0 0 0 0\n\"\"\"[1:-1] # don't want to strip\n\nEXPECTED_SHORT_OUTPUT = \"\"\"\n t\\\\p O A B-ORG B-PER E-ORG E-PER\n O 4 0 0 0 0 0\n A 0 0 1 0 0 0\n B-ORG 0 0 0 1 0 0\n B-PER 0 0 0 0 0 0\n E-ORG 0 0 0 0 1 1\n E-PER 0 0 0 0 0 0\n\"\"\"[1:-1]\n\nEXPECTED_HIDE_BLANK_SHORT_OUTPUT = \"\"\"\n t\\\\p O B-ORG E-ORG B-PER E-PER\n O 4 0 0 0 0\n A 0 1 0 0 0\n B-ORG 0 0 0 1 0\n E-ORG 0 0 1 0 1\n\"\"\"[1:-1]\n\ndef test_simple_output(simple_confusion):\n assert EXPECTED_SIMPLE_OUTPUT == format_confusion(simple_confusion)\n\ndef test_short_output(short_confusion):\n assert EXPECTED_SHORT_OUTPUT == format_confusion(short_confusion)\n\ndef test_hide_blank_short_output(short_confusion):\n assert EXPECTED_HIDE_BLANK_SHORT_OUTPUT == format_confusion(short_confusion, hide_blank=True)\n\ndef test_macro_f1(simple_confusion, short_confusion):\n assert confusion_to_macro_f1(simple_confusion) == pytest.approx(0.466666666666)\n assert confusion_to_macro_f1(short_confusion) == pytest.approx(0.277777777777)\n\ndef test_weighted_f1(simple_confusion, short_confusion):\n assert confusion_to_weighted_f1(simple_confusion) == pytest.approx(0.83333333)\n assert confusion_to_weighted_f1(short_confusion) == pytest.approx(0.66666666)\n\n assert confusion_to_weighted_f1(simple_confusion, exclude=[\"O\"]) == pytest.approx(0.66666666)\n assert confusion_to_weighted_f1(short_confusion, exclude=[\"O\"]) == pytest.approx(0.33333333)\n\n"
},
{
"path": "stanza/tests/common/test_constant.py",
"content": "\"\"\"\nTest the conversion to lcodes and splitting of dataset names\n\"\"\"\n\nimport tempfile\n\nimport pytest\n\nimport stanza\nfrom stanza.models.common.constant import treebank_to_short_name, lang_to_langcode, is_right_to_left, two_to_three_letters, langlower2lcode\nfrom stanza.tests import *\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\ndef test_treebank():\n \"\"\"\n Test the entire treebank name conversion\n \"\"\"\n # conversion of a UD_ name\n assert \"hi_hdtb\" == treebank_to_short_name(\"UD_Hindi-HDTB\")\n # conversion of names without UD\n assert \"hi_fire2013\" == treebank_to_short_name(\"Hindi-fire2013\")\n assert \"hi_fire2013\" == treebank_to_short_name(\"Hindi-Fire2013\")\n assert \"hi_fire2013\" == treebank_to_short_name(\"Hindi-FIRE2013\")\n # already short names are generally preserved\n assert \"hi_fire2013\" == treebank_to_short_name(\"hi-fire2013\")\n assert \"hi_fire2013\" == treebank_to_short_name(\"hi_fire2013\")\n # a special case\n assert \"zh-hant_pud\" == treebank_to_short_name(\"UD_Chinese-PUD\")\n # a special case already converted once\n assert \"zh-hant_pud\" == treebank_to_short_name(\"zh-hant_pud\")\n assert \"zh-hant_pud\" == treebank_to_short_name(\"zh-hant-pud\")\n assert \"zh-hans_gsdsimp\" == treebank_to_short_name(\"zh-hans_gsdsimp\")\n\n assert \"wo_masakhane\" == treebank_to_short_name(\"wo_masakhane\")\n assert \"wo_masakhane\" == treebank_to_short_name(\"wol_masakhane\")\n assert \"wo_masakhane\" == treebank_to_short_name(\"Wol_masakhane\")\n assert \"wo_masakhane\" == treebank_to_short_name(\"wolof_masakhane\")\n assert \"wo_masakhane\" == treebank_to_short_name(\"Wolof_masakhane\")\n\ndef test_lang_to_langcode():\n assert \"hi\" == lang_to_langcode(\"Hindi\")\n assert \"hi\" == lang_to_langcode(\"HINDI\")\n assert \"hi\" == lang_to_langcode(\"hindi\")\n assert \"hi\" == lang_to_langcode(\"HI\")\n assert \"hi\" == lang_to_langcode(\"hi\")\n\ndef test_right_to_left():\n assert is_right_to_left(\"ar\")\n assert is_right_to_left(\"Arabic\")\n\n assert not is_right_to_left(\"en\")\n assert not is_right_to_left(\"English\")\n\ndef test_two_to_three():\n assert lang_to_langcode(\"Wolof\") == \"wo\"\n assert lang_to_langcode(\"wol\") == \"wo\"\n\n assert \"wo\" in two_to_three_letters\n assert two_to_three_letters[\"wo\"] == \"wol\"\n\ndef test_langlower():\n assert lang_to_langcode(\"WOLOF\") == \"wo\"\n assert lang_to_langcode(\"nOrWeGiAn\") == \"nb\"\n\n assert \"soj\" == langlower2lcode[\"soi\"]\n assert \"soj\" == langlower2lcode[\"sohi\"]\n"
},
{
"path": "stanza/tests/common/test_data_conversion.py",
"content": "\"\"\"\nBasic tests of the data conversion\n\"\"\"\n\nimport io\nimport pytest\nimport tempfile\nfrom zipfile import ZipFile\n\nimport stanza\nfrom stanza.utils.conll import CoNLL\nfrom stanza.models.common.doc import Document\nfrom stanza.tests import *\n\npytestmark = pytest.mark.pipeline\n\n# data for testing\nCONLL = [[['1', 'Nous', 'il', 'PRON', '_', 'Number=Plur|Person=1|PronType=Prs', '3', 'nsubj', '_', 'start_char=0|end_char=4'],\n ['2', 'avons', 'avoir', 'AUX', '_', 'Mood=Ind|Number=Plur|Person=1|Tense=Pres|VerbForm=Fin', '3', 'aux:tense', '_', 'start_char=5|end_char=10'],\n ['3', 'atteint', 'atteindre', 'VERB', '_', 'Gender=Masc|Number=Sing|Tense=Past|VerbForm=Part', '0', 'root', '_', 'start_char=11|end_char=18'],\n ['4', 'la', 'le', 'DET', '_', 'Definite=Def|Gender=Fem|Number=Sing|PronType=Art', '5', 'det', '_', 'start_char=19|end_char=21'],\n ['5', 'fin', 'fin', 'NOUN', '_', 'Gender=Fem|Number=Sing', '3', 'obj', '_', 'start_char=22|end_char=25'],\n ['6-7', 'du', '_', '_', '_', '_', '_', '_', '_', 'start_char=26|end_char=28'],\n ['6', 'de', 'de', 'ADP', '_', '_', '8', 'case', '_', '_'],\n ['7', 'le', 'le', 'DET', '_', 'Definite=Def|Gender=Masc|Number=Sing|PronType=Art', '8', 'det', '_', '_'],\n ['8', 'sentier', 'sentier', 'NOUN', '_', 'Gender=Masc|Number=Sing', '5', 'nmod', '_', 'start_char=29|end_char=36'],\n ['9', '.', '.', 'PUNCT', '_', '_', '3', 'punct', '_', 'start_char=36|end_char=37']]]\n\n\nDICT = [[{'id': (1,), 'text': 'Nous', 'lemma': 'il', 'upos': 'PRON', 'feats': 'Number=Plur|Person=1|PronType=Prs', 'head': 3, 'deprel': 'nsubj', 'misc': 'start_char=0|end_char=4'},\n {'id': (2,), 'text': 'avons', 'lemma': 'avoir', 'upos': 'AUX', 'feats': 'Mood=Ind|Number=Plur|Person=1|Tense=Pres|VerbForm=Fin', 'head': 3, 'deprel': 'aux:tense', 'misc': 'start_char=5|end_char=10'},\n {'id': (3,), 'text': 'atteint', 'lemma': 'atteindre', 'upos': 'VERB', 'feats': 'Gender=Masc|Number=Sing|Tense=Past|VerbForm=Part', 'head': 0, 'deprel': 'root', 'misc': 'start_char=11|end_char=18'},\n {'id': (4,), 'text': 'la', 'lemma': 'le', 'upos': 'DET', 'feats': 'Definite=Def|Gender=Fem|Number=Sing|PronType=Art', 'head': 5, 'deprel': 'det', 'misc': 'start_char=19|end_char=21'},\n {'id': (5,), 'text': 'fin', 'lemma': 'fin', 'upos': 'NOUN', 'feats': 'Gender=Fem|Number=Sing', 'head': 3, 'deprel': 'obj', 'misc': 'start_char=22|end_char=25'},\n {'id': (6, 7), 'text': 'du', 'misc': 'start_char=26|end_char=28'},\n {'id': (6,), 'text': 'de', 'lemma': 'de', 'upos': 'ADP', 'head': 8, 'deprel': 'case'},\n {'id': (7,), 'text': 'le', 'lemma': 'le', 'upos': 'DET', 'feats': 'Definite=Def|Gender=Masc|Number=Sing|PronType=Art', 'head': 8, 'deprel': 'det'},\n {'id': (8,), 'text': 'sentier', 'lemma': 'sentier', 'upos': 'NOUN', 'feats': 'Gender=Masc|Number=Sing', 'head': 5, 'deprel': 'nmod', 'misc': 'start_char=29|end_char=36'},\n {'id': (9,), 'text': '.', 'lemma': '.', 'upos': 'PUNCT', 'head': 3, 'deprel': 'punct', 'misc': 'start_char=36|end_char=37'}]]\n\ndef test_conll_to_dict():\n dicts, empty = CoNLL.convert_conll(CONLL)\n assert dicts == DICT\n assert len(dicts) == len(empty)\n assert all(len(x) == 0 for x in empty)\n\ndef test_dict_to_conll():\n document = Document(DICT)\n # :c = no comments\n conll = [[sentence.split(\"\\t\") for sentence in doc.split(\"\\n\")] for doc in \"{:c}\".format(document).split(\"\\n\\n\")]\n assert conll == CONLL\n\ndef test_dict_to_doc_and_doc_to_dict():\n \"\"\"\n Test the conversion from raw dict to Document and back\n\n This code path will first turn start_char|end_char into start_char & end_char fields in the Document\n That version to a dict will have separate fields for each of those\n Finally, the conversion from that dict to a list of conll entries should convert that back to misc\n \"\"\"\n document = Document(DICT)\n dicts = document.to_dict()\n document = Document(dicts)\n conll = [[sentence.split(\"\\t\") for sentence in doc.split(\"\\n\")] for doc in \"{:c}\".format(document).split(\"\\n\\n\")]\n assert conll == CONLL\n\n# sample is two sentences long so that the tests check multiple sentences\nRUSSIAN_SAMPLE=\"\"\"\n# sent_id = yandex.reviews-f-8xh5zqnmwak3t6p68y4rhwd4e0-1969-9253\n# genre = review\n# text = Как- то слишком мало цветов получают актёры после спектакля.\n1\tКак\tкак-то\tADV\t_\tDegree=Pos|PronType=Ind\t7\tadvmod\t_\tSpaceAfter=No\n2\t-\t-\tPUNCT\t_\t_\t3\tpunct\t_\t_\n3\tто\tто\tPART\t_\t_\t1\tlist\t_\tdeprel=list:goeswith\n4\tслишком\tслишком\tADV\t_\tDegree=Pos\t5\tadvmod\t_\t_\n5\tмало\tмало\tADV\t_\tDegree=Pos\t6\tadvmod\t_\t_\n6\tцветов\tцветок\tNOUN\t_\tAnimacy=Inan|Case=Gen|Gender=Masc|Number=Plur\t7\tobj\t_\t_\n7\tполучают\tполучать\tVERB\t_\tAspect=Imp|Mood=Ind|Number=Plur|Person=3|Tense=Pres|VerbForm=Fin|Voice=Act\t0\troot\t_\t_\n8\tактёры\tактер\tNOUN\t_\tAnimacy=Anim|Case=Nom|Gender=Masc|Number=Plur\t7\tnsubj\t_\t_\n9\tпосле\tпосле\tADP\t_\t_\t10\tcase\t_\t_\n10\tспектакля\tспектакль\tNOUN\t_\tAnimacy=Inan|Case=Gen|Gender=Masc|Number=Sing\t7\tobl\t_\tSpaceAfter=No\n11\t.\t.\tPUNCT\t_\t_\t7\tpunct\t_\t_\n\n# sent_id = 4\n# genre = social\n# text = В женщине важна верность, а не красота.\n1\tВ\tв\tADP\t_\t_\t2\tcase\t_\t_\n2\tженщине\tженщина\tNOUN\t_\tAnimacy=Anim|Case=Loc|Gender=Fem|Number=Sing\t3\tobl\t_\t_\n3\tважна\tважный\tADJ\t_\tDegree=Pos|Gender=Fem|Number=Sing|Variant=Short\t0\troot\t_\t_\n4\tверность\tверность\tNOUN\t_\tAnimacy=Inan|Case=Nom|Gender=Fem|Number=Sing\t3\tnsubj\t_\tSpaceAfter=No\n5\t,\t,\tPUNCT\t_\t_\t8\tpunct\t_\t_\n6\tа\tа\tCCONJ\t_\t_\t8\tcc\t_\t_\n7\tне\tне\tPART\t_\tPolarity=Neg\t8\tadvmod\t_\t_\n8\tкрасота\tкрасота\tNOUN\t_\tAnimacy=Inan|Case=Nom|Gender=Fem|Number=Sing\t4\tconj\t_\tSpaceAfter=No\n9\t.\t.\tPUNCT\t_\t_\t3\tpunct\t_\t_\n\"\"\".strip()\n\nRUSSIAN_TEXT = [\"Как- то слишком мало цветов получают актёры после спектакля.\", \"В женщине важна верность, а не красота.\"]\nRUSSIAN_IDS = [\"yandex.reviews-f-8xh5zqnmwak3t6p68y4rhwd4e0-1969-9253\", \"4\"]\n\ndef check_russian_doc(doc):\n \"\"\"\n Refactored the test for the Russian doc so we can use it to test various file methods\n \"\"\"\n lines = RUSSIAN_SAMPLE.split(\"\\n\")\n assert len(doc.sentences) == 2\n assert lines[0] == doc.sentences[0].comments[0]\n assert lines[1] == doc.sentences[0].comments[1]\n assert lines[2] == doc.sentences[0].comments[2]\n for sent_idx, (expected_text, expected_id, sentence) in enumerate(zip(RUSSIAN_TEXT, RUSSIAN_IDS, doc.sentences)):\n assert expected_text == sentence.text\n assert expected_id == sentence.sent_id\n assert sent_idx == sentence.index\n assert len(sentence.comments) == 3\n assert not sentence.has_enhanced_dependencies()\n\n sentences = \"{:C}\".format(doc)\n sentences = sentences.split(\"\\n\\n\")\n assert len(sentences) == 2\n\n sentence = sentences[0].split(\"\\n\")\n assert len(sentence) == 14\n assert lines[0] == sentence[0]\n assert lines[1] == sentence[1]\n assert lines[2] == sentence[2]\n\n # assert that the weird deprel=list:goeswith was properly handled\n assert doc.sentences[0].words[2].head == 1\n assert doc.sentences[0].words[2].deprel == \"list:goeswith\"\n\ndef test_write_russian_doc(tmp_path):\n \"\"\"\n Specifically test the write_doc2conll method\n \"\"\"\n filename = tmp_path / \"russian.conll\"\n doc = CoNLL.conll2doc(input_str=RUSSIAN_SAMPLE)\n check_russian_doc(doc)\n CoNLL.write_doc2conll(doc, filename)\n\n with open(filename, encoding=\"utf-8\") as fin:\n text = fin.read()\n\n # the conll docs have to end with \\n\\n\n assert text.endswith(\"\\n\\n\")\n\n # but to compare against the original, strip off the whitespace\n text = text.strip()\n\n # we skip the first sentence because the \"deprel=list:goeswith\" is weird\n # note that the deprel itself is checked in check_russian_doc\n text = text[text.find(\"# sent_id = 4\"):]\n sample = RUSSIAN_SAMPLE[RUSSIAN_SAMPLE.find(\"# sent_id = 4\"):]\n assert text == sample\n\n doc2 = CoNLL.conll2doc(filename)\n check_russian_doc(doc2)\n\n# random sentence from EN_Pronouns\nENGLISH_SAMPLE = \"\"\"\n# newdoc\n# sent_id = 1\n# text = It is hers.\n# previous = Which person owns this?\n# comment = copular subject\n1\tIt\tit\tPRON\tPRP\tNumber=Sing|Person=3|PronType=Prs\t3\tnsubj\t_\t_\n2\tis\tbe\tAUX\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t3\tcop\t_\t_\n3\thers\thers\tPRON\tPRP\tGender=Fem|Number=Sing|Person=3|Poss=Yes|PronType=Prs\t0\troot\t_\tSpaceAfter=No\n4\t.\t.\tPUNCT\t.\t_\t3\tpunct\t_\t_\n\"\"\".strip()\n\ndef test_write_to_io():\n doc = CoNLL.conll2doc(input_str=ENGLISH_SAMPLE)\n output = io.StringIO()\n CoNLL.write_doc2conll(doc, output)\n output_value = output.getvalue()\n assert output_value.endswith(\"\\n\\n\")\n assert output_value.strip() == ENGLISH_SAMPLE\n\ndef test_write_doc2conll_append(tmp_path):\n doc = CoNLL.conll2doc(input_str=ENGLISH_SAMPLE)\n filename = tmp_path / \"english.conll\"\n CoNLL.write_doc2conll(doc, filename)\n CoNLL.write_doc2conll(doc, filename, mode=\"a\")\n\n with open(filename) as fin:\n text = fin.read()\n expected = ENGLISH_SAMPLE + \"\\n\\n\" + ENGLISH_SAMPLE + \"\\n\\n\"\n assert text == expected\n\ndef test_doc_with_comments():\n \"\"\"\n Test that a doc with comments gets converted back with comments\n \"\"\"\n doc = CoNLL.conll2doc(input_str=RUSSIAN_SAMPLE)\n check_russian_doc(doc)\n\ndef test_unusual_misc():\n \"\"\"\n The above RUSSIAN_SAMPLE resulted in a blank misc field in one particular implementation of the conll code\n (the below test would fail)\n \"\"\"\n doc = CoNLL.conll2doc(input_str=RUSSIAN_SAMPLE)\n sentences = \"{:C}\".format(doc).split(\"\\n\\n\")\n assert len(sentences) == 2\n sentence = sentences[0].split(\"\\n\")\n assert len(sentence) == 14\n\n for word in sentence:\n pieces = word.split(\"\\t\")\n assert len(pieces) == 1 or len(pieces) == 10\n if len(pieces) == 10:\n assert all(piece for piece in pieces)\n\ndef test_file():\n \"\"\"\n Test loading a doc from a file\n \"\"\"\n with tempfile.TemporaryDirectory() as tempdir:\n filename = os.path.join(tempdir, \"russian.conll\")\n with open(filename, \"w\", encoding=\"utf-8\") as fout:\n fout.write(RUSSIAN_SAMPLE)\n doc = CoNLL.conll2doc(input_file=filename)\n check_russian_doc(doc)\n\ndef test_zip_file():\n \"\"\"\n Test loading a doc from a zip file\n \"\"\"\n with tempfile.TemporaryDirectory() as tempdir:\n zip_file = os.path.join(tempdir, \"russian.zip\")\n filename = \"russian.conll\"\n with ZipFile(zip_file, \"w\") as zout:\n with zout.open(filename, \"w\") as fout:\n fout.write(RUSSIAN_SAMPLE.encode())\n\n doc = CoNLL.conll2doc(input_file=filename, zip_file=zip_file)\n check_russian_doc(doc)\n\nSIMPLE_NER = \"\"\"\n# text = Teferi's best friend is Karn\n# sent_id = 0\n1\tTeferi\t_\t_\t_\t_\t0\t_\t_\tstart_char=0|end_char=6|ner=S-PERSON\n2\t's\t_\t_\t_\t_\t1\t_\t_\tstart_char=6|end_char=8|ner=O\n3\tbest\t_\t_\t_\t_\t2\t_\t_\tstart_char=9|end_char=13|ner=O\n4\tfriend\t_\t_\t_\t_\t3\t_\t_\tstart_char=14|end_char=20|ner=O\n5\tis\t_\t_\t_\t_\t4\t_\t_\tstart_char=21|end_char=23|ner=O\n6\tKarn\t_\t_\t_\t_\t5\t_\t_\tstart_char=24|end_char=28|ner=S-PERSON\n\"\"\".strip()\n\ndef test_simple_ner_conversion():\n \"\"\"\n Test that tokens get properly created with NER tags\n \"\"\"\n doc = CoNLL.conll2doc(input_str=SIMPLE_NER)\n assert len(doc.sentences) == 1\n sentence = doc.sentences[0]\n assert len(sentence.tokens) == 6\n EXPECTED_NER = [\"S-PERSON\", \"O\", \"O\", \"O\", \"O\", \"S-PERSON\"]\n for token, ner in zip(sentence.tokens, EXPECTED_NER):\n assert token.ner == ner\n # check that the ner, start_char, end_char fields were not put on the token's misc\n # those should all be set as specific fields on the token\n assert not token.misc\n assert len(token.words) == 1\n # they should also not reach the word's misc field\n assert not token.words[0].misc\n\n conll = \"{:C}\".format(doc)\n assert conll == SIMPLE_NER\n\nMWT_NER = \"\"\"\n# text = This makes John's headache worse\n# sent_id = 0\n1\tThis\t_\t_\t_\t_\t0\t_\t_\tstart_char=0|end_char=4|ner=O\n2\tmakes\t_\t_\t_\t_\t1\t_\t_\tstart_char=5|end_char=10|ner=O\n3-4\tJohn's\t_\t_\t_\t_\t_\t_\t_\tstart_char=11|end_char=17|ner=S-PERSON\n3\tJohn\t_\t_\t_\t_\t2\t_\t_\t_\n4\t's\t_\t_\t_\t_\t3\t_\t_\t_\n5\theadache\t_\t_\t_\t_\t4\t_\t_\tstart_char=18|end_char=26|ner=O\n6\tworse\t_\t_\t_\t_\t5\t_\t_\tstart_char=27|end_char=32|ner=O\n\"\"\".strip()\n\ndef test_mwt_ner_conversion():\n \"\"\"\n Test that tokens including MWT get properly created with NER tags\n\n Note that this kind of thing happens with the EWT tokenizer for English, for example\n \"\"\"\n doc = CoNLL.conll2doc(input_str=MWT_NER)\n assert len(doc.sentences) == 1\n sentence = doc.sentences[0]\n assert len(sentence.tokens) == 5\n assert not sentence.has_enhanced_dependencies()\n EXPECTED_NER = [\"O\", \"O\", \"S-PERSON\", \"O\", \"O\"]\n EXPECTED_WORDS = [1, 1, 2, 1, 1]\n for token, ner, expected_words in zip(sentence.tokens, EXPECTED_NER, EXPECTED_WORDS):\n assert token.ner == ner\n # check that the ner, start_char, end_char fields were not put on the token's misc\n # those should all be set as specific fields on the token\n assert not token.misc\n assert len(token.words) == expected_words\n # they should also not reach the word's misc field\n assert not token.words[0].misc\n\n conll = \"{:C}\".format(doc)\n assert conll == MWT_NER\n\nALL_OFFSETS_CONLLU = \"\"\"\n# text = This makes John's headache worse\n# sent_id = 0\n1\tThis\t_\t_\t_\t_\t0\t_\t_\tstart_char=0|end_char=4\n2\tmakes\t_\t_\t_\t_\t1\t_\t_\tstart_char=5|end_char=10\n3-4\tJohn's\t_\t_\t_\t_\t_\t_\t_\tstart_char=11|end_char=17\n3\tJohn\t_\t_\t_\t_\t2\t_\t_\tstart_char=11|end_char=15\n4\t's\t_\t_\t_\t_\t3\t_\t_\tstart_char=15|end_char=17\n5\theadache\t_\t_\t_\t_\t4\t_\t_\tstart_char=18|end_char=26\n6\tworse\t_\t_\t_\t_\t5\t_\t_\tSpaceAfter=No|start_char=27|end_char=32\n\"\"\".strip()\n\nNO_OFFSETS_CONLLU = \"\"\"\n# text = This makes John's headache worse\n# sent_id = 0\n1\tThis\t_\t_\t_\t_\t0\t_\t_\t_\n2\tmakes\t_\t_\t_\t_\t1\t_\t_\t_\n3-4\tJohn's\t_\t_\t_\t_\t_\t_\t_\t_\n3\tJohn\t_\t_\t_\t_\t2\t_\t_\t_\n4\t's\t_\t_\t_\t_\t3\t_\t_\t_\n5\theadache\t_\t_\t_\t_\t4\t_\t_\t_\n6\tworse\t_\t_\t_\t_\t5\t_\t_\tSpaceAfter=No\n\"\"\".strip()\n\nNO_COMMENTS_NO_OFFSETS_CONLLU = \"\"\"\n1\tThis\t_\t_\t_\t_\t0\t_\t_\t_\n2\tmakes\t_\t_\t_\t_\t1\t_\t_\t_\n3-4\tJohn's\t_\t_\t_\t_\t_\t_\t_\t_\n3\tJohn\t_\t_\t_\t_\t2\t_\t_\t_\n4\t's\t_\t_\t_\t_\t3\t_\t_\t_\n5\theadache\t_\t_\t_\t_\t4\t_\t_\t_\n6\tworse\t_\t_\t_\t_\t5\t_\t_\tSpaceAfter=No\n\"\"\".strip()\n\n\ndef test_no_offsets_output():\n doc = CoNLL.conll2doc(input_str=ALL_OFFSETS_CONLLU)\n assert len(doc.sentences) == 1\n sentence = doc.sentences[0]\n assert len(sentence.tokens) == 5\n\n conll = \"{:C}\".format(doc)\n assert conll == ALL_OFFSETS_CONLLU\n\n conll = \"{:C-o}\".format(doc)\n assert conll == NO_OFFSETS_CONLLU\n\n conll = \"{:c-o}\".format(doc)\n assert conll == NO_COMMENTS_NO_OFFSETS_CONLLU\n\n# A random sentence from et_ewt-ud-train.conllu\n# which we use to test the deps conversion for multiple deps\nESTONIAN_DEPS = \"\"\"\n# newpar\n# sent_id = aia_foorum_37\n# text = Sestpeale ei mõistagi neid, kes koduaias sortidega tegelevad.\n1\tSestpeale\tsest_peale\tADV\tD\t_\t3\tadvmod\t3:advmod\t_\n2\tei\tei\tAUX\tV\tPolarity=Neg\t3\taux\t3:aux\t_\n3\tmõistagi\tmõistma\tVERB\tV\tConnegative=Yes|Mood=Ind|Tense=Pres|VerbForm=Fin|Voice=Act\t0\troot\t0:root\t_\n4\tneid\ttema\tPRON\tP\tCase=Par|Number=Plur|Person=3|PronType=Prs\t3\tobj\t3:obj|9:nsubj\tSpaceAfter=No\n5\t,\t,\tPUNCT\tZ\t_\t9\tpunct\t9:punct\t_\n6\tkes\tkes\tPRON\tP\tCase=Nom|Number=Plur|PronType=Int,Rel\t9\tnsubj\t4:ref\t_\n7\tkoduaias\tkodu_aed\tNOUN\tS\tCase=Ine|Number=Sing\t9\tobl\t9:obl\t_\n8\tsortidega\tsort\tNOUN\tS\tCase=Com|Number=Plur\t9\tobl\t9:obl\t_\n9\ttegelevad\ttegelema\tVERB\tV\tMood=Ind|Number=Plur|Person=3|Tense=Pres|VerbForm=Fin|Voice=Act\t4\tacl:relcl\t4:acl\tSpaceAfter=No\n10\t.\t.\tPUNCT\tZ\t_\t3\tpunct\t3:punct\t_\n\"\"\".strip()\n\ndef test_deps_conversion():\n doc = CoNLL.conll2doc(input_str=ESTONIAN_DEPS)\n assert len(doc.sentences) == 1\n sentence = doc.sentences[0]\n assert len(sentence.tokens) == 10\n assert sentence.has_enhanced_dependencies()\n\n word = doc.sentences[0].words[3]\n assert word.deps == \"3:obj|9:nsubj\"\n\n conll = \"{:C}\".format(doc)\n assert conll == ESTONIAN_DEPS\n\nESTONIAN_EMPTY_DEPS = \"\"\"\n# sent_id = ewtb2_000035_15\n# text = Ja paari aasta pärast rôômalt maasikatele ...\n1\tJa\tja\tCCONJ\tJ\t_\t3\tcc\t5.1:cc\t_\n2\tpaari\tpaar\tNUM\tN\tCase=Gen|Number=Sing|NumForm=Word|NumType=Card\t3\tnummod\t3:nummod\t_\n3\taasta\taasta\tNOUN\tS\tCase=Gen|Number=Sing\t0\troot\t5.1:obl\t_\n4\tpärast\tpärast\tADP\tK\tAdpType=Post\t3\tcase\t3:case\t_\n5\trôômalt\trõõmsalt\tADV\tD\tTypo=Yes\t3\tadvmod\t5.1:advmod\tOrphan=Yes|CorrectForm=rõõmsalt\n5.1\tpanna\tpanema\tVERB\tV\tVerbForm=Inf\t_\t_\t0:root\tEmpty=5.1\n6\tmaasikatele\tmaasikas\tNOUN\tS\tCase=All|Number=Plur\t3\tobl\t5.1:obl\tOrphan=Yes\n7\t...\t...\tPUNCT\tZ\t_\t3\tpunct\t5.1:punct\t_\n\"\"\".strip()\n\nESTONIAN_EMPTY_END_DEPS = \"\"\"\n# sent_id = ewtb2_000035_15\n# text = Ja paari aasta pärast rôômalt maasikatele ...\n1\tJa\tja\tCCONJ\tJ\t_\t3\tcc\t5.1:cc\t_\n2\tpaari\tpaar\tNUM\tN\tCase=Gen|Number=Sing|NumForm=Word|NumType=Card\t3\tnummod\t3:nummod\t_\n3\taasta\taasta\tNOUN\tS\tCase=Gen|Number=Sing\t0\troot\t5.1:obl\t_\n4\tpärast\tpärast\tADP\tK\tAdpType=Post\t3\tcase\t3:case\t_\n5\trôômalt\trõõmsalt\tADV\tD\tTypo=Yes\t3\tadvmod\t5.1:advmod\tOrphan=Yes|CorrectForm=rõõmsalt\n5.1\tpanna\tpanema\tVERB\tV\tVerbForm=Inf\t_\t_\t0:root\tEmpty=5.1\n\"\"\".strip()\n\ndef test_empty_deps_conversion():\n \"\"\"\n Check that we can read and then output a sentence with empty dependencies\n \"\"\"\n check_empty_deps_conversion(ESTONIAN_EMPTY_DEPS, 7)\n\ndef test_empty_deps_at_end_conversion():\n \"\"\"\n The empty deps conversion should also work if the empty dep is at the end\n \"\"\"\n check_empty_deps_conversion(ESTONIAN_EMPTY_END_DEPS, 5)\n\ndef check_empty_deps_conversion(input_str, expected_words):\n doc = CoNLL.conll2doc(input_str=input_str, ignore_gapping=False)\n assert len(doc.sentences) == 1\n assert len(doc.sentences[0].tokens) == expected_words\n assert len(doc.sentences[0].words) == expected_words\n assert len(doc.sentences[0].empty_words) == 1\n\n sentence = doc.sentences[0]\n conll = \"{:C}\".format(doc)\n assert conll == input_str\n\n sentence_dict = doc.sentences[0].to_dict()\n assert len(sentence_dict) == expected_words + 1\n # currently this is true for both of the examples we run\n assert sentence_dict[5]['id'] == (5, 1)\n\n # redo the above checks to make sure\n # there are no weird bugs in the accessors\n assert len(doc.sentences) == 1\n assert len(doc.sentences[0].tokens) == expected_words\n assert len(doc.sentences[0].words) == expected_words\n assert len(doc.sentences[0].empty_words) == 1\n\n\nESTONIAN_DOC_ID = \"\"\"\n# doc_id = this_is_a_doc\n# sent_id = ewtb2_000035_15\n# text = Ja paari aasta pärast rôômalt maasikatele ...\n1\tJa\tja\tCCONJ\tJ\t_\t3\tcc\t5.1:cc\t_\n2\tpaari\tpaar\tNUM\tN\tCase=Gen|Number=Sing|NumForm=Word|NumType=Card\t3\tnummod\t3:nummod\t_\n3\taasta\taasta\tNOUN\tS\tCase=Gen|Number=Sing\t0\troot\t5.1:obl\t_\n4\tpärast\tpärast\tADP\tK\tAdpType=Post\t3\tcase\t3:case\t_\n5\trôômalt\trõõmsalt\tADV\tD\tTypo=Yes\t3\tadvmod\t5.1:advmod\tOrphan=Yes|CorrectForm=rõõmsalt\n5.1\tpanna\tpanema\tVERB\tV\tVerbForm=Inf\t_\t_\t0:root\tEmpty=5.1\n6\tmaasikatele\tmaasikas\tNOUN\tS\tCase=All|Number=Plur\t3\tobl\t5.1:obl\tOrphan=Yes\n7\t...\t...\tPUNCT\tZ\t_\t3\tpunct\t5.1:punct\t_\n\"\"\".strip()\n\ndef test_read_doc_id():\n doc = CoNLL.conll2doc(input_str=ESTONIAN_DOC_ID, ignore_gapping=False)\n assert \"{:C}\".format(doc) == ESTONIAN_DOC_ID\n assert doc.sentences[0].doc_id == 'this_is_a_doc'\n\nSIMPLE_DEPENDENCY_INDEX_ERROR = \"\"\"\n# text = Teferi's best friend is Karn\n# sent_id = 0\n# notes = this sentence has a dependency index outside the sentence. it should throw an IndexError\n1\tTeferi\t_\t_\t_\t_\t0\troot\t_\tstart_char=0|end_char=6|ner=S-PERSON\n2\t's\t_\t_\t_\t_\t1\tdep\t_\tstart_char=6|end_char=8|ner=O\n3\tbest\t_\t_\t_\t_\t2\tdep\t_\tstart_char=9|end_char=13|ner=O\n4\tfriend\t_\t_\t_\t_\t3\tdep\t_\tstart_char=14|end_char=20|ner=O\n5\tis\t_\t_\t_\t_\t4\tdep\t_\tstart_char=21|end_char=23|ner=O\n6\tKarn\t_\t_\t_\t_\t8\tdep\t_\tstart_char=24|end_char=28|ner=S-PERSON\n\"\"\".strip()\n\ndef test_read_dependency_errors():\n with pytest.raises(IndexError):\n doc = CoNLL.conll2doc(input_str=SIMPLE_DEPENDENCY_INDEX_ERROR)\n\nMULTIPLE_DOC_IDS = \"\"\"\n# doc_id = doc_1\n# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0020\n# text = His mother was also killed in the attack.\n1\tHis\this\tPRON\tPRP$\tCase=Gen|Gender=Masc|Number=Sing|Person=3|Poss=Yes|PronType=Prs\t2\tnmod:poss\t2:nmod:poss\t_\n2\tmother\tmother\tNOUN\tNN\tNumber=Sing\t5\tnsubj:pass\t5:nsubj:pass\t_\n3\twas\tbe\tAUX\tVBD\tMood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin\t5\taux:pass\t5:aux:pass\t_\n4\talso\talso\tADV\tRB\t_\t5\tadvmod\t5:advmod\t_\n5\tkilled\tkill\tVERB\tVBN\tTense=Past|VerbForm=Part|Voice=Pass\t0\troot\t0:root\t_\n6\tin\tin\tADP\tIN\t_\t8\tcase\t8:case\t_\n7\tthe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t8\tdet\t8:det\t_\n8\tattack\tattack\tNOUN\tNN\tNumber=Sing\t5\tobl\t5:obl:in\tSpaceAfter=No\n9\t.\t.\tPUNCT\t.\t_\t5\tpunct\t5:punct\t_\n\n# doc_id = doc_1\n# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0028\n# text = This item is a small one and easily missed.\n1\tThis\tthis\tDET\tDT\tNumber=Sing|PronType=Dem\t2\tdet\t2:det\t_\n2\titem\titem\tNOUN\tNN\tNumber=Sing\t6\tnsubj\t6:nsubj|9:nsubj:pass\t_\n3\tis\tbe\tAUX\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t6\tcop\t6:cop\t_\n4\ta\ta\tDET\tDT\tDefinite=Ind|PronType=Art\t6\tdet\t6:det\t_\n5\tsmall\tsmall\tADJ\tJJ\tDegree=Pos\t6\tamod\t6:amod\t_\n6\tone\tone\tNOUN\tNN\tNumber=Sing\t0\troot\t0:root\t_\n7\tand\tand\tCCONJ\tCC\t_\t9\tcc\t9:cc\t_\n8\teasily\teasily\tADV\tRB\t_\t9\tadvmod\t9:advmod\t_\n9\tmissed\tmiss\tVERB\tVBN\tTense=Past|VerbForm=Part|Voice=Pass\t6\tconj\t6:conj:and\tSpaceAfter=No\n10\t.\t.\tPUNCT\t.\t_\t6\tpunct\t6:punct\t_\n\n# doc_id = doc_2\n# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0029\n# text = But in my view it is highly significant.\n1\tBut\tbut\tCCONJ\tCC\t_\t8\tcc\t8:cc\t_\n2\tin\tin\tADP\tIN\t_\t4\tcase\t4:case\t_\n3\tmy\tmy\tPRON\tPRP$\tCase=Gen|Number=Sing|Person=1|Poss=Yes|PronType=Prs\t4\tnmod:poss\t4:nmod:poss\t_\n4\tview\tview\tNOUN\tNN\tNumber=Sing\t8\tobl\t8:obl:in\t_\n5\tit\tit\tPRON\tPRP\tCase=Nom|Gender=Neut|Number=Sing|Person=3|PronType=Prs\t8\tnsubj\t8:nsubj\t_\n6\tis\tbe\tAUX\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t8\tcop\t8:cop\t_\n7\thighly\thighly\tADV\tRB\t_\t8\tadvmod\t8:advmod\t_\n8\tsignificant\tsignificant\tADJ\tJJ\tDegree=Pos\t0\troot\t0:root\tSpaceAfter=No\n9\t.\t.\tPUNCT\t.\t_\t8\tpunct\t8:punct\t_\n\n# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0040\n# text = The trial begins again Nov.28.\n1\tThe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t2\tdet\t2:det\t_\n2\ttrial\ttrial\tNOUN\tNN\tNumber=Sing\t3\tnsubj\t3:nsubj\t_\n3\tbegins\tbegin\tVERB\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t0\troot\t0:root\t_\n4\tagain\tagain\tADV\tRB\t_\t3\tadvmod\t3:advmod\t_\n5\tNov.\tNovember\tPROPN\tNNP\tAbbr=Yes|Number=Sing\t3\tobl:tmod\t3:obl:tmod\tSpaceAfter=No\n6\t28\t28\tNUM\tCD\tNumForm=Digit|NumType=Card\t5\tnummod\t5:nummod\tSpaceAfter=No\n7\t.\t.\tPUNCT\t.\t_\t3\tpunct\t3:punct\t_\n\n\"\"\".lstrip()\n\ndef test_read_multiple_doc_ids():\n docs = CoNLL.conll2multi_docs(input_str=MULTIPLE_DOC_IDS)\n assert len(docs) == 2\n assert len(docs[0].sentences) == 2\n assert len(docs[1].sentences) == 2\n\n # remove the first doc_id comment\n text = \"\\n\".join(MULTIPLE_DOC_IDS.split(\"\\n\")[1:])\n docs = CoNLL.conll2multi_docs(input_str=text)\n assert len(docs) == 3\n assert len(docs[0].sentences) == 1\n assert len(docs[1].sentences) == 1\n assert len(docs[2].sentences) == 2\n\nENGLISH_TEST_SENTENCE = \"\"\"\n# text = This is a test\n# sent_id = 0\n1\tThis\tthis\tPRON\tDT\tNumber=Sing|PronType=Dem\t4\tnsubj\t_\tstart_char=0|end_char=4\n2\tis\tbe\tAUX\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t4\tcop\t_\tstart_char=5|end_char=7\n3\ta\ta\tDET\tDT\tDefinite=Ind|PronType=Art\t4\tdet\t_\tstart_char=8|end_char=9\n4\ttest\ttest\tNOUN\tNN\tNumber=Sing\t0\troot\t_\tSpaceAfter=No|start_char=10|end_char=14\n\"\"\".lstrip()\n\ndef test_convert_dict():\n doc = CoNLL.conll2doc(input_str=ENGLISH_TEST_SENTENCE)\n converted = CoNLL.convert_dict(doc.to_dict())\n\n expected = [[['1', 'This', 'this', 'PRON', 'DT', 'Number=Sing|PronType=Dem', '4', 'nsubj', '_', 'start_char=0|end_char=4'],\n ['2', 'is', 'be', 'AUX', 'VBZ', 'Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin', '4', 'cop', '_', 'start_char=5|end_char=7'],\n ['3', 'a', 'a', 'DET', 'DT', 'Definite=Ind|PronType=Art', '4', 'det', '_', 'start_char=8|end_char=9'],\n ['4', 'test', 'test', 'NOUN', 'NN', 'Number=Sing', '0', 'root', '_', 'SpaceAfter=No|start_char=10|end_char=14']]]\n\n assert converted == expected\n\ndef test_line_numbers():\n doc = CoNLL.conll2doc(input_str=ENGLISH_TEST_SENTENCE, keep_line_numbers=True)\n # currently the line numbers are not output in the conllu format\n doc_conllu = \"{:C}\\n\".format(doc)\n assert doc_conllu == ENGLISH_TEST_SENTENCE\n\n # currently the line numbers are not output in the dict format\n converted = CoNLL.convert_dict(doc.to_dict())\n expected = [[['1', 'This', 'this', 'PRON', 'DT', 'Number=Sing|PronType=Dem', '4', 'nsubj', '_', 'start_char=0|end_char=4'],\n ['2', 'is', 'be', 'AUX', 'VBZ', 'Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin', '4', 'cop', '_', 'start_char=5|end_char=7'],\n ['3', 'a', 'a', 'DET', 'DT', 'Definite=Ind|PronType=Art', '4', 'det', '_', 'start_char=8|end_char=9'],\n ['4', 'test', 'test', 'NOUN', 'NN', 'Number=Sing', '0', 'root', '_', 'SpaceAfter=No|start_char=10|end_char=14']]]\n assert converted == expected\n\n for word_idx, word in enumerate(doc.sentences[0].words):\n # the test sentence has two comments in it\n assert word.line_number == word_idx + 2\n\n\nSPEAKER_EXAMPLE = \"\"\"\n# sent_id = GUM_fiction_pag-57\n# speaker = Siri\n# addressee = Pag\n# text = \"Sorry.\"\n1\t\"\t\"\tPUNCT\t``\t_\t2\tpunct\t2:punct\tDiscourse=joint-sequence_m:130->128:1:_|SpaceAfter=No\n2\tSorry\tsorry\tADJ\tJJ\tDegree=Pos\t0\troot\t0:root\tMSeg=Sorr-y|SpaceAfter=No\n3\t.\t.\tPUNCT\t.\t_\t2\tpunct\t2:punct\tSpaceAfter=No\n4\t\"\t\"\tPUNCT\t''\t_\t2\tpunct\t2:punct\t_\n\"\"\".lstrip()\n\ndef test_speaker():\n doc = CoNLL.conll2doc(input_str=SPEAKER_EXAMPLE)\n assert len(doc.sentences) == 1\n assert doc.sentences[0].speaker == 'Siri'\n assert \"# speaker = Siri\" in doc.sentences[0].comments\n\n doc.sentences[0].speaker = \"foo\"\n assert doc.sentences[0].speaker == 'foo'\n assert any(comment.startswith(\"# speaker\") for comment in doc.sentences[0].comments)\n assert \"# speaker = foo\" in doc.sentences[0].comments\n\n doc.sentences[0].speaker = None\n assert not any(comment.startswith(\"# speaker\") for comment in doc.sentences[0].comments)\n assert doc.sentences[0].speaker is None\n\n doc.sentences[0].speaker = \"Siri\"\n assert doc.sentences[0].speaker == 'Siri'\n assert \"# speaker = Siri\" in doc.sentences[0].comments\n"
},
{
"path": "stanza/tests/common/test_data_objects.py",
"content": "\"\"\"\nBasic tests of the stanza data objects, especially the setter/getter routines\n\"\"\"\nimport pytest\n\nimport stanza\nfrom stanza.models.common.doc import Document, Sentence, Word\nfrom stanza.tests import *\n\npytestmark = pytest.mark.pipeline\n\n# data for testing\nEN_DOC = \"This is a test document. Pretty cool!\"\n\nEN_DOC_UPOS_XPOS = (('PRON_DT', 'AUX_VBZ', 'DET_DT', 'NOUN_NN', 'NOUN_NN', 'PUNCT_.'), ('ADV_RB', 'ADJ_JJ', 'PUNCT_.'))\n\nEN_DOC2 = \"Chris Manning wrote a sentence. Then another.\"\n\n@pytest.fixture(scope=\"module\")\ndef nlp_pipeline():\n nlp = stanza.Pipeline(dir=TEST_MODELS_DIR, lang='en')\n return nlp\n\ndef test_readonly(nlp_pipeline):\n Document.add_property('some_property', 123)\n doc = nlp_pipeline(EN_DOC)\n assert doc.some_property == 123\n with pytest.raises(ValueError):\n doc.some_property = 456\n\n\ndef test_getter(nlp_pipeline):\n Word.add_property('upos_xpos', getter=lambda self: f\"{self.upos}_{self.xpos}\")\n\n doc = nlp_pipeline(EN_DOC)\n\n assert EN_DOC_UPOS_XPOS == tuple(tuple(word.upos_xpos for word in sentence.words) for sentence in doc.sentences)\n\ndef test_setter_getter(nlp_pipeline):\n int2str = {0: 'ok', 1: 'good', 2: 'bad'}\n str2int = {'ok': 0, 'good': 1, 'bad': 2}\n def setter(self, value):\n self._classname = str2int[value]\n Sentence.add_property('classname', getter=lambda self: int2str[self._classname] if self._classname is not None else None, setter=setter)\n\n doc = nlp_pipeline(EN_DOC)\n sentence = doc.sentences[0]\n sentence.classname = 'good'\n assert sentence._classname == 1\n\n # don't try this at home\n sentence._classname = 2\n assert sentence.classname == 'bad'\n\ndef test_backpointer(nlp_pipeline):\n doc = nlp_pipeline(EN_DOC2)\n ent = doc.ents[0]\n assert ent.sent is doc.sentences[0]\n assert list(doc.iter_words())[0].sent is doc.sentences[0]\n assert list(doc.iter_tokens())[-1].sent is doc.sentences[-1]\n"
},
{
"path": "stanza/tests/common/test_doc.py",
"content": "import pytest\n\nimport stanza\nfrom stanza.tests import *\nfrom stanza.models.common.doc import Document, ID, TEXT, NER, CONSTITUENCY, SENTIMENT\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\n@pytest.fixture\ndef sentences_dict():\n return [[{ID: 1, TEXT: \"unban\"},\n {ID: 2, TEXT: \"mox\"},\n {ID: 3, TEXT: \"opal\"}],\n [{ID: 4, TEXT: \"ban\"},\n {ID: 5, TEXT: \"Lurrus\"}]]\n\n@pytest.fixture\ndef doc(sentences_dict):\n doc = Document(sentences_dict)\n return doc\n\ndef test_basic_values(doc, sentences_dict):\n \"\"\"\n Test that sentences & token text are properly set when constructing a doc\n \"\"\"\n assert len(doc.sentences) == len(sentences_dict)\n\n for sentence, raw_sentence in zip(doc.sentences, sentences_dict):\n assert sentence.doc == doc\n assert len(sentence.tokens) == len(raw_sentence)\n for token, raw_token in zip(sentence.tokens, raw_sentence):\n assert token.text == raw_token[TEXT]\n\ndef test_set_sentence(doc):\n \"\"\"\n Test setting a field on the sentences themselves\n \"\"\"\n doc.set(fields=\"sentiment\",\n contents=[\"4\", \"0\"],\n to_sentence=True)\n\n assert doc.sentences[0].sentiment == \"4\"\n assert doc.sentences[1].sentiment == \"0\"\n\ndef test_set_tokens(doc):\n \"\"\"\n Test setting values on tokens\n \"\"\"\n ner_contents = [\"O\", \"ARTIFACT\", \"ARTIFACT\", \"O\", \"CAT\"]\n doc.set(fields=NER,\n contents=ner_contents,\n to_token=True)\n\n result = doc.get(NER, from_token=True)\n assert result == ner_contents\n\n\ndef test_constituency_comment(doc):\n \"\"\"\n Test that setting the constituency tree on a doc sets the constituency comment\n \"\"\"\n for sentence in doc.sentences:\n assert len([x for x in sentence.comments if x.startswith(\"# constituency\")]) == 0\n\n # currently nothing is checking that the items are actually trees\n trees = [\"asdf\", \"zzzz\"]\n doc.set(fields=CONSTITUENCY,\n contents=trees,\n to_sentence=True)\n\n for sentence, expected in zip(doc.sentences, trees):\n constituency_comments = [x for x in sentence.comments if x.startswith(\"# constituency\")]\n assert len(constituency_comments) == 1\n assert constituency_comments[0].endswith(expected)\n\n # Test that if we replace the trees with an updated tree, the comment is also replaced\n trees = [\"zzzz\", \"asdf\"]\n doc.set(fields=CONSTITUENCY,\n contents=trees,\n to_sentence=True)\n\n for sentence, expected in zip(doc.sentences, trees):\n constituency_comments = [x for x in sentence.comments if x.startswith(\"# constituency\")]\n assert len(constituency_comments) == 1\n assert constituency_comments[0].endswith(expected)\n\ndef test_sentiment_comment(doc):\n \"\"\"\n Test that setting the sentiment on a doc sets the sentiment comment\n \"\"\"\n for sentence in doc.sentences:\n assert len([x for x in sentence.comments if x.startswith(\"# sentiment\")]) == 0\n\n # currently nothing is checking that the items are actually trees\n sentiments = [\"1\", \"2\"]\n doc.set(fields=SENTIMENT,\n contents=sentiments,\n to_sentence=True)\n\n for sentence, expected in zip(doc.sentences, sentiments):\n sentiment_comments = [x for x in sentence.comments if x.startswith(\"# sentiment\")]\n assert len(sentiment_comments) == 1\n assert sentiment_comments[0].endswith(expected)\n\n # Test that if we replace the trees with an updated tree, the comment is also replaced\n sentiments = [\"3\", \"4\"]\n doc.set(fields=SENTIMENT,\n contents=sentiments,\n to_sentence=True)\n\n for sentence, expected in zip(doc.sentences, sentiments):\n sentiment_comments = [x for x in sentence.comments if x.startswith(\"# sentiment\")]\n assert len(sentiment_comments) == 1\n assert sentiment_comments[0].endswith(expected)\n\ndef test_sent_id_comment(doc):\n \"\"\"\n Test that setting the sent_id on a sentence sets the sentiment comment\n \"\"\"\n for sent_idx, sentence in enumerate(doc.sentences):\n assert len([x for x in sentence.comments if x.startswith(\"# sent_id\")]) == 1\n assert sentence.sent_id == \"%d\" % sent_idx\n doc.sentences[0].sent_id = \"foo\"\n assert doc.sentences[0].sent_id == \"foo\"\n assert len([x for x in doc.sentences[0].comments if x.startswith(\"# sent_id\")]) == 1\n assert \"# sent_id = foo\" in doc.sentences[0].comments\n\n doc.reindex_sentences(10)\n for sent_idx, sentence in enumerate(doc.sentences):\n assert sentence.sent_id == \"%d\" % (sent_idx + 10)\n assert len([x for x in doc.sentences[0].comments if x.startswith(\"# sent_id\")]) == 1\n assert \"# sent_id = %d\" % (sent_idx + 10) in sentence.comments\n\n doc.sentences[0].add_comment(\"# sent_id = bar\")\n assert doc.sentences[0].sent_id == \"bar\"\n assert \"# sent_id = bar\" in doc.sentences[0].comments\n assert len([x for x in doc.sentences[0].comments if x.startswith(\"# sent_id\")]) == 1\n\ndef test_doc_id_comment(doc):\n \"\"\"\n Test that setting the doc_id on a sentence sets the document comment\n \"\"\"\n assert doc.sentences[0].doc_id is None\n assert len([x for x in doc.sentences[0].comments if x.startswith(\"# doc_id\")]) == 0\n\n doc.sentences[0].doc_id = \"foo\"\n assert len([x for x in doc.sentences[0].comments if x.startswith(\"# doc_id\")]) == 1\n assert \"# doc_id = foo\" in doc.sentences[0].comments\n assert doc.sentences[0].doc_id == \"foo\"\n\n doc.sentences[0].add_comment(\"# doc_id = bar\")\n assert len([x for x in doc.sentences[0].comments if x.startswith(\"# doc_id\")]) == 1\n assert doc.sentences[0].doc_id == \"bar\"\n\n@pytest.fixture(scope=\"module\")\ndef pipeline():\n return stanza.Pipeline(dir=TEST_MODELS_DIR)\n\ndef test_serialized(pipeline):\n \"\"\"\n Brief test of the serialized format\n\n Checks that NER entities are correctly set.\n Also checks that constituency & sentiment are set on the sentences.\n \"\"\"\n text = \"John Bauer works at Stanford\"\n doc = pipeline(text)\n assert len(doc.ents) == 2\n serialized = doc.to_serialized()\n doc2 = Document.from_serialized(serialized)\n assert len(doc2.sentences) == 1\n assert len(doc2.ents) == 2\n assert doc.sentences[0].constituency == doc2.sentences[0].constituency\n assert doc.sentences[0].sentiment == doc2.sentences[0].sentiment\n"
},
{
"path": "stanza/tests/common/test_dropout.py",
"content": "import pytest\n\nimport torch\n\nimport stanza\nfrom stanza.models.common.dropout import WordDropout\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\ndef test_word_dropout():\n \"\"\"\n Test that word_dropout is randomly dropping out the entire final dimension of a tensor\n\n Doing 600 small rows should be super fast, but it leaves us with\n something like a 1 in 10^180 chance of the test failing. Not very\n common, in other words\n \"\"\"\n wd = WordDropout(0.5)\n batch = torch.randn(600, 4)\n dropped = wd(batch)\n # the one time any of this happens, it's going to be really confusing\n assert not torch.allclose(batch, dropped)\n num_zeros = 0\n for i in range(batch.shape[0]):\n assert torch.allclose(dropped[i], batch[i]) or torch.sum(dropped[i]) == 0.0\n if torch.sum(dropped[i]) == 0.0:\n num_zeros += 1\n assert num_zeros > 0 and num_zeros < batch.shape[0]\n"
},
{
"path": "stanza/tests/common/test_foundation_cache.py",
"content": "import glob\nimport os\nimport shutil\nimport tempfile\n\nimport pytest\n\nimport stanza\nfrom stanza.models.common.foundation_cache import FoundationCache, load_charlm\nfrom stanza.tests import TEST_MODELS_DIR\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\ndef test_charlm_cache():\n models_path = os.path.join(TEST_MODELS_DIR, \"en\", \"backward_charlm\", \"*\")\n models = glob.glob(models_path)\n # we expect at least one English model downloaded for the tests\n assert len(models) >= 1\n model_file = models[0]\n\n cache = FoundationCache()\n with tempfile.TemporaryDirectory(dir=\".\") as test_dir:\n temp_file = os.path.join(test_dir, \"charlm.pt\")\n shutil.copy2(model_file, temp_file)\n # this will work\n model = load_charlm(temp_file)\n\n # this will save the model\n model = cache.load_charlm(temp_file)\n\n # this should no longer work\n with pytest.raises(FileNotFoundError):\n model = load_charlm(temp_file)\n\n # it should remember the cached version\n model = cache.load_charlm(temp_file)\n"
},
{
"path": "stanza/tests/common/test_pretrain.py",
"content": "import os\nimport tempfile\n\nimport pytest\nimport numpy as np\nimport torch\n\nfrom stanza.models.common import pretrain\nfrom stanza.models.common.vocab import UNK_ID\nfrom stanza.tests import *\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\ndef check_vocab(vocab):\n # 4 base vectors, plus the 3 vectors actually present in the file\n assert len(vocab) == 7\n assert 'unban' in vocab\n assert 'mox' in vocab\n assert 'opal' in vocab\n\ndef check_embedding(emb, unk=False):\n expected = np.array([[ 0., 0., 0., 0.,],\n [ 0., 0., 0., 0.,],\n [ 0., 0., 0., 0.,],\n [ 0., 0., 0., 0.,],\n [ 1., 2., 3., 4.,],\n [ 5., 6., 7., 8.,],\n [ 9., 10., 11., 12.,]])\n if unk:\n expected[UNK_ID] = -1\n np.testing.assert_allclose(emb, expected)\n\ndef check_pretrain(pt):\n check_vocab(pt.vocab)\n check_embedding(pt.emb)\n\ndef test_text_pretrain():\n pt = pretrain.Pretrain(vec_filename=f'{TEST_WORKING_DIR}/in/tiny_emb.txt', save_to_file=False)\n check_pretrain(pt)\n\ndef test_xz_pretrain():\n pt = pretrain.Pretrain(vec_filename=f'{TEST_WORKING_DIR}/in/tiny_emb.xz', save_to_file=False)\n check_pretrain(pt)\n\ndef test_gz_pretrain():\n pt = pretrain.Pretrain(vec_filename=f'{TEST_WORKING_DIR}/in/tiny_emb.gz', save_to_file=False)\n check_pretrain(pt)\n\ndef test_zip_pretrain():\n pt = pretrain.Pretrain(vec_filename=f'{TEST_WORKING_DIR}/in/tiny_emb.zip', save_to_file=False)\n check_pretrain(pt)\n\ndef test_csv_pretrain():\n pt = pretrain.Pretrain(csv_filename=f'{TEST_WORKING_DIR}/in/tiny_emb.csv', save_to_file=False)\n check_pretrain(pt)\n\ndef test_resave_pretrain():\n \"\"\"\n Test saving a pretrain and then loading from the existing file\n \"\"\"\n test_pt_file = tempfile.NamedTemporaryFile(dir=f'{TEST_WORKING_DIR}/out', suffix=\".pt\", delete=False)\n try:\n test_pt_file.close()\n # note that this tests the ability to save a pretrain and the\n # ability to fall back when the existing pretrain isn't working\n pt = pretrain.Pretrain(filename=test_pt_file.name,\n vec_filename=f'{TEST_WORKING_DIR}/in/tiny_emb.xz')\n check_pretrain(pt)\n\n pt2 = pretrain.Pretrain(filename=test_pt_file.name,\n vec_filename=f'unban_mox_opal')\n check_pretrain(pt2)\n\n pt3 = torch.load(test_pt_file.name, weights_only=True)\n check_embedding(pt3['emb'])\n finally:\n os.unlink(test_pt_file.name)\n\nSPACE_PRETRAIN=\"\"\"\n3 4\nunban mox 1 2 3 4\nopal 5 6 7 8\nfoo 9 10 11 12\n\"\"\".strip()\n\ndef test_whitespace():\n \"\"\"\n Test reading a pretrain with an ascii space in it\n\n The vocab word with a space in it should have the correct number\n of dimensions read, with the space converted to nbsp\n \"\"\"\n test_txt_file = tempfile.NamedTemporaryFile(dir=f'{TEST_WORKING_DIR}/out', suffix=\".txt\", delete=False)\n try:\n test_txt_file.write(SPACE_PRETRAIN.encode())\n test_txt_file.close()\n\n pt = pretrain.Pretrain(vec_filename=test_txt_file.name, save_to_file=False)\n check_embedding(pt.emb)\n assert \"unban\\xa0mox\" in pt.vocab\n # this one also works because of the normalize_unit in vocab.py\n assert \"unban mox\" in pt.vocab\n finally:\n os.unlink(test_txt_file.name)\n\nNO_HEADER_PRETRAIN=\"\"\"\nunban 1 2 3 4\nmox 5 6 7 8\nopal 9 10 11 12\n\"\"\".strip()\n\ndef test_no_header():\n \"\"\"\n Check loading a pretrain with no rows,cols header\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tmpdir:\n filename = os.path.join(tmpdir, \"tiny.txt\")\n with open(filename, \"w\", encoding=\"utf-8\") as fout:\n fout.write(NO_HEADER_PRETRAIN)\n pt = pretrain.Pretrain(vec_filename=filename, save_to_file=False)\n check_embedding(pt.emb)\n\nUNK_PRETRAIN=\"\"\"\nunban 1 2 3 4\nmox 5 6 7 8\nopal 9 10 11 12\n -1 -1 -1 -1\n\"\"\".strip()\n\ndef test_no_header():\n \"\"\"\n Check loading a pretrain with at the end, like GloVe does\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tmpdir:\n filename = os.path.join(tmpdir, \"tiny.txt\")\n with open(filename, \"w\", encoding=\"utf-8\") as fout:\n fout.write(UNK_PRETRAIN)\n pt = pretrain.Pretrain(vec_filename=filename, save_to_file=False)\n check_embedding(pt.emb, unk=True)\n"
},
{
"path": "stanza/tests/common/test_relative_attn.py",
"content": "import pytest\n\nimport torch\n\nfrom stanza.models.common.relative_attn import RelativeAttention\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\n\ndef test_attn():\n foo = RelativeAttention(d_model=100, num_heads=2, window=8, dropout=0.0)\n bar = torch.randn(10, 13, 100)\n result = foo(bar)\n assert result.shape == bar.shape\n value = foo.value(bar)\n if not torch.allclose(result[:, -1, :], value[:, -1, :], atol=1e-06):\n raise ValueError(result[:, -1, :] - value[:, -1, :])\n assert torch.allclose(result[:, -1, :], value[:, -1, :], atol=1e-06)\n assert not torch.allclose(result[:, 0, :], value[:, 0, :])\n\n\ndef test_shorter_sequence():\n # originally this was failing because the batch was smaller than the window\n foo = RelativeAttention(d_model=20, num_heads=2, window=5, dropout=0.0)\n bar = torch.randn(10, 3, 20)\n result = foo(bar)\n assert result.shape == bar.shape\n\n value = foo.value(bar)\n if not torch.allclose(result[:, -1, :], value[:, -1, :], atol=1e-06):\n raise ValueError(result[:, -1, :] - value[:, -1, :])\n assert torch.allclose(result[:, -1, :], value[:, -1, :], atol=1e-06)\n assert not torch.allclose(result[:, 0, :], value[:, 0, :])\n\ndef test_reverse():\n foo = RelativeAttention(d_model=100, num_heads=2, window=8, reverse=True, dropout=0.0)\n bar = torch.randn(10, 13, 100)\n result = foo(bar)\n assert result.shape == bar.shape\n value = foo.value(bar)\n if not torch.allclose(result[:, 0, :], value[:, 0, :], atol=1e-06):\n raise ValueError(result[:, 0, :] - value[:, 0, :])\n assert torch.allclose(result[:, 0, :], value[:, 0, :], atol=1e-06)\n assert not torch.allclose(result[:, -1, :], value[:, -1, :])\n\n\n"
},
{
"path": "stanza/tests/common/test_short_name_to_treebank.py",
"content": "import pytest\n\nimport stanza\nfrom stanza.models.common import short_name_to_treebank\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\ndef test_short_name():\n assert short_name_to_treebank.short_name_to_treebank(\"en_ewt\") == \"UD_English-EWT\"\n\ndef test_canonical_name():\n assert short_name_to_treebank.canonical_treebank_name(\"UD_URDU-UDTB\") == \"UD_Urdu-UDTB\"\n assert short_name_to_treebank.canonical_treebank_name(\"ur_udtb\") == \"UD_Urdu-UDTB\"\n assert short_name_to_treebank.canonical_treebank_name(\"Unban_Mox_Opal\") == \"Unban_Mox_Opal\"\n"
},
{
"path": "stanza/tests/common/test_utils.py",
"content": "import lzma\nimport os\nimport tempfile\n\nimport pytest\n\nimport stanza\nimport stanza.models.common.utils as utils\nfrom stanza.tests import *\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\ndef test_wordvec_not_found():\n \"\"\"\n get_wordvec_file should fail if neither word2vec nor fasttext exists\n \"\"\"\n with tempfile.TemporaryDirectory(dir=f'{TEST_WORKING_DIR}/out') as temp_dir:\n with pytest.raises(FileNotFoundError):\n utils.get_wordvec_file(wordvec_dir=temp_dir, shorthand='en_foo')\n\n\ndef test_word2vec_xz():\n \"\"\"\n Test searching for word2vec and xz files\n \"\"\"\n with tempfile.TemporaryDirectory(dir=f'{TEST_WORKING_DIR}/out') as temp_dir:\n # make a fake directory for English word vectors\n word2vec_dir = os.path.join(temp_dir, 'word2vec', 'English')\n os.makedirs(word2vec_dir)\n\n # make a fake English word vector file\n fake_file = os.path.join(word2vec_dir, 'en.vectors.xz')\n fout = open(fake_file, 'w')\n fout.close()\n\n # get_wordvec_file should now find this fake file\n filename = utils.get_wordvec_file(wordvec_dir=temp_dir, shorthand='en_foo')\n assert filename == fake_file\n\ndef test_fasttext_txt():\n \"\"\"\n Test searching for fasttext and txt files\n \"\"\"\n with tempfile.TemporaryDirectory(dir=f'{TEST_WORKING_DIR}/out') as temp_dir:\n # make a fake directory for English word vectors\n fasttext_dir = os.path.join(temp_dir, 'fasttext', 'English')\n os.makedirs(fasttext_dir)\n\n # make a fake English word vector file\n fake_file = os.path.join(fasttext_dir, 'en.vectors.txt')\n fout = open(fake_file, 'w')\n fout.close()\n\n # get_wordvec_file should now find this fake file\n filename = utils.get_wordvec_file(wordvec_dir=temp_dir, shorthand='en_foo')\n assert filename == fake_file\n\ndef test_wordvec_type():\n \"\"\"\n If we supply our own wordvec type, get_wordvec_file should find that\n \"\"\"\n with tempfile.TemporaryDirectory(dir=f'{TEST_WORKING_DIR}/out') as temp_dir:\n # make a fake directory for English word vectors\n google_dir = os.path.join(temp_dir, 'google', 'English')\n os.makedirs(google_dir)\n\n # make a fake English word vector file\n fake_file = os.path.join(google_dir, 'en.vectors.txt')\n fout = open(fake_file, 'w')\n fout.close()\n\n # get_wordvec_file should now find this fake file\n filename = utils.get_wordvec_file(wordvec_dir=temp_dir, shorthand='en_foo', wordvec_type='google')\n assert filename == fake_file\n\n # this file won't be found using the normal defaults\n with pytest.raises(FileNotFoundError):\n utils.get_wordvec_file(wordvec_dir=temp_dir, shorthand='en_foo')\n\ndef test_sort_with_indices():\n data = [[1, 2, 3], [4, 5], [6]]\n ordered, orig_idx = utils.sort_with_indices(data, key=len)\n assert ordered == ([6], [4, 5], [1, 2, 3])\n assert orig_idx == (2, 1, 0)\n\n unsorted = utils.unsort(ordered, orig_idx)\n assert data == unsorted\n\ndef test_empty_sort_with_indices():\n ordered, orig_idx = utils.sort_with_indices([])\n assert len(ordered) == 0\n assert len(orig_idx) == 0\n\n unsorted = utils.unsort(ordered, orig_idx)\n assert [] == unsorted\n\n\ndef test_split_into_batches():\n data = []\n for i in range(5):\n data.append([\"Unban\", \"mox\", \"opal\", str(i)])\n\n data.append([\"Do\", \"n't\", \"ban\", \"Urza\", \"'s\", \"Saga\", \"that\", \"card\", \"is\", \"great\"])\n data.append([\"Ban\", \"Ragavan\"])\n\n # small batches will put one element in each interval\n batches = utils.split_into_batches(data, 5)\n assert batches == [(0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7)]\n\n # this one has a batch interrupted in the middle by a large element\n batches = utils.split_into_batches(data, 8)\n assert batches == [(0, 2), (2, 4), (4, 5), (5, 6), (6, 7)]\n\n # this one has the large element at the start of its own batch\n batches = utils.split_into_batches(data[1:], 8)\n assert batches == [(0, 2), (2, 4), (4, 5), (5, 6)]\n\n # overloading the test! assert that the key & reverse is working\n ordered, orig_idx = utils.sort_with_indices(data, key=len, reverse=True)\n assert [len(x) for x in ordered] == [10, 4, 4, 4, 4, 4, 2]\n\n # this has the large element at the start\n batches = utils.split_into_batches(ordered, 8)\n assert batches == [(0, 1), (1, 3), (3, 5), (5, 7)]\n\n # double check that unsort is working as expected\n assert data == utils.unsort(ordered, orig_idx)\n\n\ndef test_find_missing_tags():\n assert utils.find_missing_tags([\"O\", \"PER\", \"LOC\"], [\"O\", \"PER\", \"LOC\"]) == []\n assert utils.find_missing_tags([\"O\", \"PER\", \"LOC\"], [\"O\", \"PER\", \"LOC\", \"ORG\"]) == ['ORG']\n assert utils.find_missing_tags([[\"O\", \"PER\"], [\"O\", \"LOC\"]], [[\"O\", \"PER\"], [\"LOC\", \"ORG\"]]) == ['ORG']\n\n\ndef test_open_read_text():\n \"\"\"\n test that we can read either .xz or regular txt\n \"\"\"\n TEXT = \"this is a test\"\n with tempfile.TemporaryDirectory() as tempdir:\n # test text file\n filename = os.path.join(tempdir, \"foo.txt\")\n with open(filename, \"w\") as fout:\n fout.write(TEXT)\n with utils.open_read_text(filename) as fin:\n in_text = fin.read()\n assert TEXT == in_text\n\n assert fin.closed\n\n # the context should close the file when we throw an exception!\n try:\n with utils.open_read_text(filename) as finex:\n assert not finex.closed\n raise ValueError(\"unban mox opal!\")\n except ValueError:\n pass\n assert finex.closed\n\n # test xz file\n filename = os.path.join(tempdir, \"foo.txt.xz\")\n with lzma.open(filename, \"wt\") as fout:\n fout.write(TEXT)\n with utils.open_read_text(filename) as finxz:\n in_text = finxz.read()\n assert TEXT == in_text\n\n assert finxz.closed\n\n # the context should close the file when we throw an exception!\n try:\n with utils.open_read_text(filename) as finexxz:\n assert not finexxz.closed\n raise ValueError(\"unban mox opal!\")\n except ValueError:\n pass\n assert finexxz.closed\n\n\ndef test_checkpoint_name():\n \"\"\"\n Test some expected results for the checkpoint names\n \"\"\"\n # use os.path.split so that the test is agnostic of file separator on Linux or Windows\n checkpoint = utils.checkpoint_name(\"saved_models\", \"kk_oscar_forward_charlm.pt\", None)\n assert os.path.split(checkpoint) == (\"saved_models\", \"kk_oscar_forward_charlm_checkpoint.pt\")\n\n checkpoint = utils.checkpoint_name(\"saved_models\", \"kk_oscar_forward_charlm\", None)\n assert os.path.split(checkpoint) == (\"saved_models\", \"kk_oscar_forward_charlm_checkpoint\")\n\n checkpoint = utils.checkpoint_name(\"saved_models\", \"kk_oscar_forward_charlm\", \"othername.pt\")\n assert os.path.split(checkpoint) == (\"saved_models\", \"othername.pt\")\n\ndef test_punct_simplification():\n \"\"\"\n Test a punctuation simplification that should make it so unexpected\n question/exclamation marks types are processed into ? and !\n \"\"\"\n test = [[[\"!!!!\"],\n [\"‼‼‼‼\"],\n [\"????\"],\n [\"?!?!\"],\n [\"?？︖\"],\n [\"?foo\"],\n [\"bar!\"]]]\n test = utils.simplify_punct(test)\n expected = [[['!'], ['!'], ['?'], ['?'], ['?'], ['?foo'], ['bar!']]]\n assert test == expected\n\n"
},
{
"path": "stanza/tests/constituency/__init__.py",
"content": ""
},
{
"path": "stanza/tests/constituency/test_convert_arboretum.py",
"content": "\"\"\"\nTest a couple different classes of trees to check the output of the Arboretum conversion\n\nNote that the text has been removed\n\"\"\"\n\nimport os\nimport tempfile\n\nimport pytest\n\nfrom stanza.server import tsurgeon\nfrom stanza.tests import TEST_WORKING_DIR\nfrom stanza.utils.datasets.constituency import convert_arboretum\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\n\nPROJ_EXAMPLE=\"\"\"\n\n\t\n\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t \n\n\t\t\n\t\t\t\n\t\t\t\t\n\t\t\t \n\t\t\t\n\t\t\t\t\n\t\t\t\t\n\t\t\t\t\n\t\t\t\t\n\t\t\t\t\n\t\t\t \n\t\t\t\n\t\t\t\t\n\t\t\t\t\n\t\t\t \n\t\t\t\n\t\t\t\t\n\t\t\t\t\n\t\t\t \n\t\t\t\n\t\t\t\t\n\t\t\t\t\n\t\t\t\t\n\t\t\t \n\t\t \n\t \n \n\"\"\"\n\nNOT_FIX_NONPROJ_EXAMPLE=\"\"\"\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\"\"\"\n\n\nNONPROJ_EXAMPLE=\"\"\"\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\"\"\"\n\ndef test_projective_example():\n \"\"\"\n Test reading a basic tree, along with some further manipulations from the conversion program\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tempdir:\n test_name = os.path.join(tempdir, \"proj.xml\")\n with open(test_name, \"w\", encoding=\"utf-8\") as fout:\n fout.write(PROJ_EXAMPLE)\n sentences = convert_arboretum.read_xml_file(test_name)\n assert len(sentences) == 1\n\n tree, words = convert_arboretum.process_tree(sentences[0])\n expected_tree = \"(s (fcl (prop s2_1) (v-fin s2_2) (pron-pers s2_3) (adjp (adj s2_4) (pp (prp s2_5) (np (art s2_6) (adj s2_7) (n s2_8)))) (pu s2_9)))\"\n assert str(tree) == expected_tree\n assert [w.word for w in words.values()] == ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', '.']\n assert not convert_arboretum.word_sequence_missing_words(tree)\n with tsurgeon.Tsurgeon() as tsurgeon_processor:\n assert tree == convert_arboretum.check_words(tree, tsurgeon_processor)\n\n # check that the words can be replaced as expected\n replaced_tree = convert_arboretum.replace_words(tree, words)\n expected_tree = \"(s (fcl (prop A) (v-fin B) (pron-pers C) (adjp (adj D) (pp (prp E) (np (art F) (adj G) (n H)))) (pu .)))\"\n assert str(replaced_tree) == expected_tree\n assert convert_arboretum.split_underscores(replaced_tree) == replaced_tree\n\n # fake a word which should be split\n words['s2_1'] = words['s2_1']._replace(word='foo_bar')\n replaced_tree = convert_arboretum.replace_words(tree, words)\n expected_tree = \"(s (fcl (prop foo_bar) (v-fin B) (pron-pers C) (adjp (adj D) (pp (prp E) (np (art F) (adj G) (n H)))) (pu .)))\"\n assert str(replaced_tree) == expected_tree\n expected_tree = \"(s (fcl (np (prop foo) (prop bar)) (v-fin B) (pron-pers C) (adjp (adj D) (pp (prp E) (np (art F) (adj G) (n H)))) (pu .)))\"\n assert str(convert_arboretum.split_underscores(replaced_tree)) == expected_tree\n\n\ndef test_not_fix_example():\n \"\"\"\n Test that a non-projective tree which we don't have a heuristic for quietly fails\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tempdir:\n test_name = os.path.join(tempdir, \"nofix.xml\")\n with open(test_name, \"w\", encoding=\"utf-8\") as fout:\n fout.write(NOT_FIX_NONPROJ_EXAMPLE)\n sentences = convert_arboretum.read_xml_file(test_name)\n assert len(sentences) == 1\n\n tree, words = convert_arboretum.process_tree(sentences[0])\n assert not convert_arboretum.word_sequence_missing_words(tree)\n with tsurgeon.Tsurgeon() as tsurgeon_processor:\n assert convert_arboretum.check_words(tree, tsurgeon_processor) is None\n\n\ndef test_fix_proj_example():\n \"\"\"\n Test that a non-projective tree can be rearranged as expected\n\n Note that there are several other classes of non-proj tree we could test as well...\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tempdir:\n test_name = os.path.join(tempdir, \"fix.xml\")\n with open(test_name, \"w\", encoding=\"utf-8\") as fout:\n fout.write(NONPROJ_EXAMPLE)\n sentences = convert_arboretum.read_xml_file(test_name)\n assert len(sentences) == 1\n\n tree, words = convert_arboretum.process_tree(sentences[0])\n assert not convert_arboretum.word_sequence_missing_words(tree)\n # the 4 and 5 are moved inside the 3-6 node\n expected_orig = \"(s (fcl (advp (adv s9_1) (adv s9_2)) (vp (v-fin s9_3) (v-pcp2 s9_6)) (prop s9_4) (adv s9_5) (pp (prp s9_7) (np (num s9_8) (n s9_9))) (pu s9_10)))\"\n expected_proj = \"(s (fcl (advp (adv s9_1) (adv s9_2)) (vp (v-fin s9_3) (prop s9_4) (adv s9_5) (v-pcp2 s9_6)) (pp (prp s9_7) (np (num s9_8) (n s9_9))) (pu s9_10)))\"\n assert str(tree) == expected_orig\n with tsurgeon.Tsurgeon() as tsurgeon_processor:\n assert str(convert_arboretum.check_words(tree, tsurgeon_processor)) == expected_proj\n\n"
},
{
"path": "stanza/tests/constituency/test_convert_it_vit.py",
"content": "\"\"\"\nTest a couple different classes of trees to check the output of the VIT conversion\n\nA couple representative trees are included, but hopefully not enough\nto be a problem in terms of our license.\n\nOne of the tests is currently disabled as it relies on tregex & tsurgeon features\nnot yet released\n\"\"\"\n\nimport io\nimport os\nimport tempfile\n\nimport pytest\n\nfrom stanza.server import tsurgeon\nfrom stanza.utils.conll import CoNLL\nfrom stanza.utils.datasets.constituency import convert_it_vit\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\n# just a sample! don't sue us please\nCON_SAMPLE = \"\"\"\n#ID=sent_00002\tcp-[sp-[part-negli, sn-[sa-[ag-ultimi], nt-anni]], f-[sn-[art-la, n-dinamica, spd-[partd-dei, sn-[n-polo_di_attrazione]]], ibar-[ause-è, ausep-stata, savv-[savv-[avv-sempre], avv-più], vppt-caratterizzata], compin-[spda-[partda-dall, sn-[n-emergere, spd-[pd-di, sn-[art-una, sa-[ag-crescente], n-concorrenza, f2-[rel-che, f-[ibar-[clit-si, ause-è, avv-progressivamente, vppin-spostata], compin-[spda-[partda-dalle, sn-[sa-[ag-singole], n-imprese]], sp-[part-ai, sn-[n-sistemi, sa-[coord-[ag-economici, cong-e, ag-territoriali]]]], fp-[punt-',', sv5-[vgt-determinando, compt-[sn-[art-l_, nf-esigenza, spd-[pd-di, sn-[art-una, n-riconsiderazione, spd-[partd-dei, sn-[n-rapporti, sv3-[ppre-esistenti, compin-[sp-[p-tra, sn-[n-soggetti, sa-[ag-produttivi]]], cong-e, sn-[n-ambiente, f2-[sp-[p-in, sn-[relob-cui]], f-[sn-[deit-questi], ibar-[vin-operano, punto-.]]]]]]]]]]]]]]]]]]]]]]]]\n\n#ID=sent_00318\tdirsp-[fc-[congf-tuttavia, f-[sn-[sq-[ind-qualche], n-problema], ir_infl-[vsupir-potrebbe, vcl-esserci], compc-[clit-ci, sp-[p-per, sn-[art-la, n-commissione, sa-[ag-esteri], f2-[sp-[part-alla, relob-cui, sn-[n-presidenza]], f-[ibar-[vc-è], compc-[sn-[n-candidato], sn-[art-l, n-esponente, spd-[pd-di, sn-[mw-Alleanza, npro-Nazionale]], sn-[mw-Mirko, nh-Tremaglia]]]]]]]]]], dirs-':', f3-[sn-[art-una, n-candidatura, sc-[q-più, sa-[ppas-subìta], sc-[ccong-che, sa-[ppas-gradita]], compt-[spda-[partda-dalla, sn-[mw-Lega, npro-Nord, punt-',', f2-[rel-che, fc-[congf-tuttavia, f-[ir_infl-[vsupir-dovrebbe, vit-rispettare], compt-[sn-[art-gli, n-accordi]]]]]]]]]], punto-.]]\n\n#ID=sent_00589\tf-[sn-[art-l, n-ottimismo, spd-[pd-di, sn-[nh-Kantor]]], ir_infl-[vsupir-potrebbe, congf-però, vcl-rivelarsi], compc-[sn-[in-ancora, art-una, nt-volta], sa-[ag-prematuro]], punto-.]\n\"\"\"\n\nUD_SAMPLE = \"\"\"\n# sent_id = VIT-2\n# text = Negli ultimi anni la dinamica dei polo di attrazione è stata sempre più caratterizzata dall'emergere di una crescente concorrenza che si è progressivamente spostata dalle singole imprese ai sistemi economici e territoriali, determinando l'esigenza di una riconsiderazione dei rapporti esistenti tra soggetti produttivi e ambiente in cui questi operano.\n1-2\tNegli\t_\t_\t_\t_\t_\t_\t_\t_\n1\tIn\tin\tADP\tE\t_\t4\tcase\t_\t_\n2\tgli\til\tDET\tRD\tDefinite=Def|Gender=Masc|Number=Plur|PronType=Art\t4\tdet\t_\t_\n3\tultimi\tultimo\tADJ\tA\tGender=Masc|Number=Plur\t4\tamod\t_\t_\n4\tanni\tanno\tNOUN\tS\tGender=Masc|Number=Plur\t16\tobl\t_\t_\n5\tla\til\tDET\tRD\tDefinite=Def|Gender=Fem|Number=Sing|PronType=Art\t6\tdet\t_\t_\n6\tdinamica\tdinamica\tNOUN\tS\tGender=Fem|Number=Sing\t16\tnsubj:pass\t_\t_\n7-8\tdei\t_\t_\t_\t_\t_\t_\t_\t_\n7\tdi\tdi\tADP\tE\t_\t9\tcase\t_\t_\n8\ti\til\tDET\tRD\tDefinite=Def|Gender=Masc|Number=Plur|PronType=Art\t9\tdet\t_\t_\n9\tpolo\tpolo\tNOUN\tS\tGender=Masc|Number=Sing\t6\tnmod\t_\t_\n10\tdi\tdi\tADP\tE\t_\t11\tcase\t_\t_\n11\tattrazione\tattrazione\tNOUN\tS\tGender=Fem|Number=Sing\t9\tnmod\t_\t_\n12\tè\tessere\tAUX\tVA\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t16\taux\t_\t_\n13\tstata\tessere\tAUX\tVA\tGender=Fem|Number=Sing|Tense=Past|VerbForm=Part\t16\taux:pass\t_\t_\n14\tsempre\tsempre\tADV\tB\t_\t15\tadvmod\t_\t_\n15\tpiù\tpiù\tADV\tB\t_\t16\tadvmod\t_\t_\n16\tcaratterizzata\tcaratterizzare\tVERB\tV\tGender=Fem|Number=Sing|Tense=Past|VerbForm=Part\t0\troot\t_\t_\n17-18\tdall'\t_\t_\t_\t_\t_\t_\t_\tSpaceAfter=No\n17\tda\tda\tADP\tE\t_\t19\tcase\t_\t_\n18\tl'\til\tDET\tRD\tDefinite=Def|Number=Sing|PronType=Art\t19\tdet\t_\t_\n19\temergere\temergere\tNOUN\tS\tGender=Masc|Number=Sing\t16\tobl\t_\t_\n20\tdi\tdi\tADP\tE\t_\t23\tcase\t_\t_\n21\tuna\tuno\tDET\tRI\tDefinite=Ind|Gender=Fem|Number=Sing|PronType=Art\t23\tdet\t_\t_\n22\tcrescente\tcrescente\tADJ\tA\tNumber=Sing\t23\tamod\t_\t_\n23\tconcorrenza\tconcorrenza\tNOUN\tS\tGender=Fem|Number=Sing\t19\tnmod\t_\t_\n24\tche\tche\tPRON\tPR\tPronType=Rel\t28\tnsubj\t_\t_\n25\tsi\tsi\tPRON\tPC\tClitic=Yes|Person=3|PronType=Prs\t28\texpl\t_\t_\n26\tè\tessere\tAUX\tVA\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t28\taux\t_\t_\n27\tprogressivamente\tprogressivamente\tADV\tB\t_\t28\tadvmod\t_\t_\n28\tspostata\tspostare\tVERB\tV\tGender=Fem|Number=Sing|Tense=Past|VerbForm=Part\t23\tacl:relcl\t_\t_\n29-30\tdalle\t_\t_\t_\t_\t_\t_\t_\t_\n29\tda\tda\tADP\tE\t_\t32\tcase\t_\t_\n30\tle\til\tDET\tRD\tDefinite=Def|Gender=Fem|Number=Plur|PronType=Art\t32\tdet\t_\t_\n31\tsingole\tsingolo\tADJ\tA\tGender=Fem|Number=Plur\t32\tamod\t_\t_\n32\timprese\timpresa\tNOUN\tS\tGender=Fem|Number=Plur\t28\tobl\t_\t_\n33-34\tai\t_\t_\t_\t_\t_\t_\t_\t_\n33\ta\ta\tADP\tE\t_\t35\tcase\t_\t_\n34\ti\til\tDET\tRD\tDefinite=Def|Gender=Masc|Number=Plur|PronType=Art\t35\tdet\t_\t_\n35\tsistemi\tsistema\tNOUN\tS\tGender=Masc|Number=Plur\t28\tobl\t_\t_\n36\teconomici\teconomico\tADJ\tA\tGender=Masc|Number=Plur\t35\tamod\t_\t_\n37\te\te\tCCONJ\tCC\t_\t38\tcc\t_\t_\n38\tterritoriali\tterritoriale\tADJ\tA\tNumber=Plur\t36\tconj\t_\tSpaceAfter=No\n39\t,\t,\tPUNCT\tFF\t_\t28\tpunct\t_\t_\n40\tdeterminando\tdeterminare\tVERB\tV\tVerbForm=Ger\t28\tadvcl\t_\t_\n41\tl'\til\tDET\tRD\tDefinite=Def|Number=Sing|PronType=Art\t42\tdet\t_\tSpaceAfter=No\n42\tesigenza\tesigenza\tNOUN\tS\tGender=Fem|Number=Sing\t40\tobj\t_\t_\n43\tdi\tdi\tADP\tE\t_\t45\tcase\t_\t_\n44\tuna\tuno\tDET\tRI\tDefinite=Ind|Gender=Fem|Number=Sing|PronType=Art\t45\tdet\t_\t_\n45\triconsiderazione\triconsiderazione\tNOUN\tS\tGender=Fem|Number=Sing\t42\tnmod\t_\t_\n46-47\tdei\t_\t_\t_\t_\t_\t_\t_\t_\n46\tdi\tdi\tADP\tE\t_\t48\tcase\t_\t_\n47\ti\til\tDET\tRD\tDefinite=Def|Gender=Masc|Number=Plur|PronType=Art\t48\tdet\t_\t_\n48\trapporti\trapporto\tNOUN\tS\tGender=Masc|Number=Plur\t45\tnmod\t_\t_\n49\tesistenti\tesistente\tVERB\tV\tNumber=Plur\t48\tacl\t_\t_\n50\ttra\ttra\tADP\tE\t_\t51\tcase\t_\t_\n51\tsoggetti\tsoggetto\tNOUN\tS\tGender=Masc|Number=Plur\t49\tobl\t_\t_\n52\tproduttivi\tproduttivo\tADJ\tA\tGender=Masc|Number=Plur\t51\tamod\t_\t_\n53\te\te\tCCONJ\tCC\t_\t54\tcc\t_\t_\n54\tambiente\tambiente\tNOUN\tS\tGender=Masc|Number=Sing\t51\tconj\t_\t_\n55\tin\tin\tADP\tE\t_\t56\tcase\t_\t_\n56\tcui\tcui\tPRON\tPR\tPronType=Rel\t58\tobl\t_\t_\n57\tquesti\tquesto\tPRON\tPD\tGender=Masc|Number=Plur|PronType=Dem\t58\tnsubj\t_\t_\n58\toperano\toperare\tVERB\tV\tMood=Ind|Number=Plur|Person=3|Tense=Pres|VerbForm=Fin\t54\tacl:relcl\t_\tSpaceAfter=No\n59\t.\t.\tPUNCT\tFS\t_\t16\tpunct\t_\t_\n\n# sent_id = VIT-318\n# text = Tuttavia qualche problema potrebbe esserci per la commissione esteri alla cui presidenza è candidato l'esponente di Alleanza Nazionale Mirko Tremaglia: una candidatura più subìta che gradita dalla Lega Nord, che tuttavia dovrebbe rispettare gli accordi.\n1\tTuttavia\ttuttavia\tCCONJ\tCC\t_\t5\tcc\t_\t_\n2\tqualche\tqualche\tDET\tDI\tNumber=Sing|PronType=Ind\t3\tdet\t_\t_\n3\tproblema\tproblema\tNOUN\tS\tGender=Masc|Number=Sing\t5\tnsubj\t_\t_\n4\tpotrebbe\tpotere\tAUX\tVA\tMood=Cnd|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t5\taux\t_\t_\n5-6\tesserci\t_\t_\t_\t_\t_\t_\t_\t_\n5\tesser\tessere\tVERB\tV\tVerbForm=Inf\t0\troot\t_\t_\n6\tci\tci\tPRON\tPC\tClitic=Yes|Number=Plur|Person=1|PronType=Prs\t5\texpl\t_\t_\n7\tper\tper\tADP\tE\t_\t9\tcase\t_\t_\n8\tla\til\tDET\tRD\tDefinite=Def|Gender=Fem|Number=Sing|PronType=Art\t9\tdet\t_\t_\n9\tcommissione\tcommissione\tNOUN\tS\tGender=Fem|Number=Sing\t5\tobl\t_\t_\n10\testeri\testero\tADJ\tA\tGender=Masc|Number=Plur\t9\tamod\t_\t_\n11-12\talla\t_\t_\t_\t_\t_\t_\t_\t_\n11\ta\ta\tADP\tE\t_\t14\tcase\t_\t_\n12\tla\til\tDET\tRD\tDefinite=Def|Gender=Fem|Number=Sing|PronType=Art\t14\tdet\t_\t_\n13\tcui\tcui\tDET\tDR\tPronType=Rel\t14\tdet:poss\t_\t_\n14\tpresidenza\tpresidenza\tNOUN\tS\tGender=Fem|Number=Sing\t16\tobl\t_\t_\n15\tè\tessere\tAUX\tVA\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t16\taux:pass\t_\t_\n16\tcandidato\tcandidare\tVERB\tV\tGender=Masc|Number=Sing|Tense=Past|VerbForm=Part\t9\tacl:relcl\t_\t_\n17\tl'\til\tDET\tRD\tDefinite=Def|Number=Sing|PronType=Art\t18\tdet\t_\tSpaceAfter=No\n18\tesponente\tesponente\tNOUN\tS\tNumber=Sing\t16\tnsubj:pass\t_\t_\n19\tdi\tdi\tADP\tE\t_\t20\tcase\t_\t_\n20\tAlleanza\tAlleanza\tPROPN\tSP\t_\t18\tnmod\t_\t_\n21\tNazionale\tNazionale\tPROPN\tSP\t_\t20\tflat:name\t_\t_\n22\tMirko\tMirko\tPROPN\tSP\t_\t18\tnmod\t_\t_\n23\tTremaglia\tTremaglia\tPROPN\tSP\t_\t22\tflat:name\t_\tSpaceAfter=No\n24\t:\t:\tPUNCT\tFC\t_\t22\tpunct\t_\t_\n25\tuna\tuno\tDET\tRI\tDefinite=Ind|Gender=Fem|Number=Sing|PronType=Art\t26\tdet\t_\t_\n26\tcandidatura\tcandidatura\tNOUN\tS\tGender=Fem|Number=Sing\t22\tappos\t_\t_\n27\tpiù\tpiù\tADV\tB\t_\t28\tadvmod\t_\t_\n28\tsubìta\tsubire\tVERB\tV\tGender=Fem|Number=Sing|Tense=Past|VerbForm=Part\t26\tadvcl\t_\t_\n29\tche\tche\tCCONJ\tCC\t_\t30\tcc\t_\t_\n30\tgradita\tgradito\tADJ\tA\tGender=Fem|Number=Sing\t28\tamod\t_\t_\n31-32\tdalla\t_\t_\t_\t_\t_\t_\t_\t_\n31\tda\tda\tADP\tE\t_\t33\tcase\t_\t_\n32\tla\til\tDET\tRD\tDefinite=Def|Gender=Fem|Number=Sing|PronType=Art\t33\tdet\t_\t_\n33\tLega\tLega\tPROPN\tSP\t_\t28\tobl:agent\t_\t_\n34\tNord\tNord\tPROPN\tSP\t_\t33\tflat:name\t_\tSpaceAfter=No\n35\t,\t,\tPUNCT\tFC\t_\t33\tpunct\t_\t_\n36\tche\tche\tPRON\tPR\tPronType=Rel\t39\tnsubj\t_\t_\n37\ttuttavia\ttuttavia\tCCONJ\tCC\t_\t39\tcc\t_\t_\n38\tdovrebbe\tdovere\tAUX\tVM\tMood=Cnd|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t39\taux\t_\t_\n39\trispettare\trispettare\tVERB\tV\tVerbForm=Inf\t33\tacl:relcl\t_\t_\n40\tgli\til\tDET\tRD\tDefinite=Def|Gender=Masc|Number=Plur|PronType=Art\t41\tdet\t_\t_\n41\taccordi\taccordio\tNOUN\tS\tGender=Masc|Number=Plur\t39\tobj\t_\tSpaceAfter=No\n42\t.\t.\tPUNCT\tFS\t_\t5\tpunct\t_\t_\n\n# sent_id = VIT-591\n# text = L'ottimismo di Kantor potrebbe però rivelarsi ancora una volta prematuro.\n1\tL'\til\tDET\tRD\tDefinite=Def|Number=Sing|PronType=Art\t2\tdet\t_\tSpaceAfter=No\n2\tottimismo\tottimismo\tNOUN\tS\tGender=Masc|Number=Sing\t7\tnsubj\t_\t_\n3\tdi\tdi\tADP\tE\t_\t4\tcase\t_\t_\n4\tKantor\tKantor\tPROPN\tSP\t_\t2\tnmod\t_\t_\n5\tpotrebbe\tpotere\tAUX\tVM\tMood=Cnd|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t7\taux\t_\t_\n6\tperò\tperò\tADV\tB\t_\t7\tadvmod\t_\t_\n7-8\trivelarsi\t_\t_\t_\t_\t_\t_\t_\t_\n7\trivelar\trivelare\tVERB\tV\tVerbForm=Inf\t0\troot\t_\t_\n8\tsi\tsi\tPRON\tPC\tClitic=Yes|Person=3|PronType=Prs\t7\texpl\t_\t_\n9\tancora\tancora\tADV\tB\t_\t7\tadvmod\t_\t_\n10\tuna\tuno\tDET\tRI\tDefinite=Ind|Gender=Fem|Number=Sing|PronType=Art\t11\tdet\t_\t_\n11\tvolta\tvolta\tNOUN\tS\tGender=Fem|Number=Sing\t7\tobl\t_\t_\n12\tprematuro\tprematuro\tADJ\tA\tGender=Masc|Number=Sing\t7\txcomp\t_\tSpaceAfter=No\n13\t.\t.\tPUNCT\tFS\t_\t7\tpunct\t_\t_\n\"\"\"\n\n\ndef test_process_mwts():\n # dei appears multiple times\n # the verb/pron esserci will be ignored\n expected_mwts = {'Negli': ('In', 'gli'), 'dei': ('di', 'i'), \"dall'\": ('da', \"l'\"), 'dalle': ('da', 'le'), 'ai': ('a', 'i'), 'alla': ('a', 'la'), 'dalla': ('da', 'la')}\n\n ud_train_data = CoNLL.conll2doc(input_str=UD_SAMPLE)\n\n mwts = convert_it_vit.get_mwt(ud_train_data)\n assert expected_mwts == mwts\n\ndef test_raw_tree():\n con_sentences = convert_it_vit.read_constituency_sentences(io.StringIO(CON_SAMPLE))\n expected_ids = [\"#ID=sent_00002\", \"#ID=sent_00318\", \"#ID=sent_00589\"]\n expected_trees = [\"(ROOT (cp (sp (part negli) (sn (sa (ag ultimi)) (nt anni))) (f (sn (art la) (n dinamica) (spd (partd dei) (sn (n polo) (n di) (n attrazione)))) (ibar (ause è) (ausep stata) (savv (savv (avv sempre)) (avv più)) (vppt caratterizzata)) (compin (spda (partda dall) (sn (n emergere) (spd (pd di) (sn (art una) (sa (ag crescente)) (n concorrenza) (f2 (rel che) (f (ibar (clit si) (ause è) (avv progressivamente) (vppin spostata)) (compin (spda (partda dalle) (sn (sa (ag singole)) (n imprese))) (sp (part ai) (sn (n sistemi) (sa (coord (ag economici) (cong e) (ag territoriali))))) (fp (punt ,) (sv5 (vgt determinando) (compt (sn (art l') (nf esigenza) (spd (pd di) (sn (art una) (n riconsiderazione) (spd (partd dei) (sn (n rapporti) (sv3 (ppre esistenti) (compin (sp (p tra) (sn (n soggetti) (sa (ag produttivi)))) (cong e) (sn (n ambiente) (f2 (sp (p in) (sn (relob cui))) (f (sn (deit questi)) (ibar (vin operano) (punto .))))))))))))))))))))))))))\",\n \"(ROOT (dirsp (fc (congf tuttavia) (f (sn (sq (ind qualche)) (n problema)) (ir_infl (vsupir potrebbe) (vcl esserci)) (compc (clit ci) (sp (p per) (sn (art la) (n commissione) (sa (ag esteri)) (f2 (sp (part alla) (relob cui) (sn (n presidenza))) (f (ibar (vc è)) (compc (sn (n candidato)) (sn (art l) (n esponente) (spd (pd di) (sn (mw Alleanza) (npro Nazionale))) (sn (mw Mirko) (nh Tremaglia))))))))))) (dirs :) (f3 (sn (art una) (n candidatura) (sc (q più) (sa (ppas subìta)) (sc (ccong che) (sa (ppas gradita))) (compt (spda (partda dalla) (sn (mw Lega) (npro Nord) (punt ,) (f2 (rel che) (fc (congf tuttavia) (f (ir_infl (vsupir dovrebbe) (vit rispettare)) (compt (sn (art gli) (n accordi))))))))))) (punto .))))\",\n \"(ROOT (f (sn (art l) (n ottimismo) (spd (pd di) (sn (nh Kantor)))) (ir_infl (vsupir potrebbe) (congf però) (vcl rivelarsi)) (compc (sn (in ancora) (art una) (nt volta)) (sa (ag prematuro))) (punto .)))\"]\n assert len(con_sentences) == 3\n for sentence, expected_id, expected_tree in zip(con_sentences, expected_ids, expected_trees):\n assert sentence[0] == expected_id\n tree = convert_it_vit.raw_tree(sentence[1])\n assert str(tree) == expected_tree\n\ndef test_update_mwts():\n con_sentences = convert_it_vit.read_constituency_sentences(io.StringIO(CON_SAMPLE))\n ud_train_data = CoNLL.conll2doc(input_str=UD_SAMPLE)\n mwt_map = convert_it_vit.get_mwt(ud_train_data)\n expected_trees=[\"(ROOT (cp (sp (part In) (sn (art gli) (sa (ag ultimi)) (nt anni))) (f (sn (art la) (n dinamica) (spd (partd di) (sn (art i) (n polo) (n di) (n attrazione)))) (ibar (ause è) (ausep stata) (savv (savv (avv sempre)) (avv più)) (vppt caratterizzata)) (compin (spda (partda da) (sn (art l') (n emergere) (spd (pd di) (sn (art una) (sa (ag crescente)) (n concorrenza) (f2 (rel che) (f (ibar (clit si) (ause è) (avv progressivamente) (vppin spostata)) (compin (spda (partda da) (sn (art le) (sa (ag singole)) (n imprese))) (sp (part a) (sn (art i) (n sistemi) (sa (coord (ag economici) (cong e) (ag territoriali))))) (fp (punt ,) (sv5 (vgt determinando) (compt (sn (art l') (nf esigenza) (spd (pd di) (sn (art una) (n riconsiderazione) (spd (partd di) (sn (art i) (n rapporti) (sv3 (ppre esistenti) (compin (sp (p tra) (sn (n soggetti) (sa (ag produttivi)))) (cong e) (sn (n ambiente) (f2 (sp (p in) (sn (relob cui))) (f (sn (deit questi)) (ibar (vin operano) (punto .))))))))))))))))))))))))))\",\n \"(ROOT (dirsp (fc (congf tuttavia) (f (sn (sq (ind qualche)) (n problema)) (ir_infl (vsupir potrebbe) (vcl esserci)) (compc (clit ci) (sp (p per) (sn (art la) (n commissione) (sa (ag esteri)) (f2 (sp (part a) (art la) (relob cui) (sn (n presidenza))) (f (ibar (vc è)) (compc (sn (n candidato)) (sn (art l) (n esponente) (spd (pd di) (sn (mw Alleanza) (npro Nazionale))) (sn (mw Mirko) (nh Tremaglia))))))))))) (dirs :) (f3 (sn (art una) (n candidatura) (sc (q più) (sa (ppas subìta)) (sc (ccong che) (sa (ppas gradita))) (compt (spda (partda da) (sn (art la) (mw Lega) (npro Nord) (punt ,) (f2 (rel che) (fc (congf tuttavia) (f (ir_infl (vsupir dovrebbe) (vit rispettare)) (compt (sn (art gli) (n accordi))))))))))) (punto .))))\",\n \"(ROOT (f (sn (art l) (n ottimismo) (spd (pd di) (sn (nh Kantor)))) (ir_infl (vsupir potrebbe) (congf però) (vcl rivelarsi)) (compc (clit si) (sn (in ancora) (art una) (nt volta)) (sa (ag prematuro))) (punto .)))\"]\n with tsurgeon.Tsurgeon() as tsurgeon_processor:\n for con_sentence, ud_sentence, expected_tree in zip(con_sentences, ud_train_data.sentences, expected_trees):\n con_tree = convert_it_vit.raw_tree(con_sentence[1])\n updated_tree, _ = convert_it_vit.update_mwts_and_special_cases(con_tree, ud_sentence, mwt_map, tsurgeon_processor)\n assert str(updated_tree) == expected_tree\n\n\nCON_PERCENT_SAMPLE = \"\"\"\nID#sent_00020 f-[sn-[art-il, n-tesoro], ibar-[vt-mette], compt-[sp-[part-sul, sn-[n-mercato]], sn-[art-il, num-51%, sp-[p-a, sn-[num-2, n-lire]], sp-[p-per, sn-[n-azione]]]], punto-.]\nID#sent_00022 dirsp-[f3-[sn-[art-le, n-novità]], dirs-':', f3-[coord-[sn-[n-voto, spd-[pd-di, sn-[n-lista]]], cong-e, sn-[n-tetto, sp-[part-agli, sn-[n-acquisti]], sv3-[vppt-limitato, comppas-[sp-[part-allo, sn-[num-0/5%]]]]]], punto-.]]\nID#sent_00517 dirsp-[fc-[f-[sn-[art-l, n-aumento, sa-[ag-mensile], spd-[pd-di, sn-[nt-aprile]]], ibar-[ause-è, vppc-stato], compc-[sq-[q-dell_, sn-[num-1/3%]], sp-[p-contro, sn-[art-lo, num-0/7/0/8%, spd-[partd-degli, sn-[sa-[ag-ultimi], num-due, sn-[nt-mesi]]]]]]]]]\nID#sent_01117 fc-[f-[sn-[art-La, sa-[ag-crescente], n-ripresa, spd-[partd-dei, sn-[n-beni, spd-[pd-di, sn-[n-consumo]]]]], ibar-[vin-deriva], savv-[avv-esclusivamente], compin-[spda-[partda-dal, sn-[n-miglioramento, f2-[spd-[pd-di, sn-[relob-cui]], f-[ibar-[ausa-hanno, vppin-beneficiato], compin-[sn-[n-beni, coord-[sa-[ag-durevoli, fp-[par-'(', sn-[num-plus4/5%], par-')']], cong-e, sa-[ag-semidurevoli, fp-[par-'(', sn-[num-plus1/5%], par-')']]]]]]]]]]], punt-',', fs-[cosu-mentre, f-[sn-[art-i, n-beni, sa-[neg-non, ag-durevoli], fp-[par-'(', sn-[num-min1%], par-')']], ibar-[vt-accusano], cong-ancora, compt-[sn-[art-un, sa-[ag-evidente], n-ritardo]]]], punto-.]\n\"\"\"\n\nCON_PERCENT_LEAVES = [\n ['il', 'tesoro', 'mette', 'sul', 'mercato', 'il', '51', '%%', 'a', '2', 'lire', 'per', 'azione', '.'],\n ['le', 'novità', ':', 'voto', 'di', 'lista', 'e', 'tetto', 'agli', 'acquisti', 'limitato', 'allo', '0,5', '%%', '.'],\n ['l', 'aumento', 'mensile', 'di', 'aprile', 'è', 'stato', \"dell'\", '1,3', '%%', 'contro', 'lo', '0/7,0/8', '%%', 'degli', 'ultimi', 'due', 'mesi'],\n # the plus and min look bad, but they get cleaned up when merging with the UD version of the dataset\n ['La', 'crescente', 'ripresa', 'dei', 'beni', 'di', 'consumo', 'deriva', 'esclusivamente', 'dal', 'miglioramento', 'di', 'cui', 'hanno', 'beneficiato', 'beni', 'durevoli', '(', 'plus4,5', '%%', ')', 'e', 'semidurevoli', '(', 'plus1,5', '%%', ')', ',', 'mentre', 'i', 'beni', 'non', 'durevoli', '(', 'min1', '%%', ')', 'accusano', 'ancora', 'un', 'evidente', 'ritardo', '.']\n]\n\ndef test_read_percent():\n con_sentences = convert_it_vit.read_constituency_sentences(io.StringIO(CON_PERCENT_SAMPLE))\n assert len(con_sentences) == len(CON_PERCENT_LEAVES)\n for (_, raw_tree), expected_leaves in zip(con_sentences, CON_PERCENT_LEAVES):\n tree = convert_it_vit.raw_tree(raw_tree)\n words = tree.leaf_labels()\n if expected_leaves is None:\n print(words)\n else:\n assert words == expected_leaves\n"
},
{
"path": "stanza/tests/constituency/test_convert_starlang.py",
"content": "\"\"\"\nTest a couple different classes of trees to check the output of the Starlang conversion\n\"\"\"\n\nimport os\nimport tempfile\n\nimport pytest\n\nfrom stanza.utils.datasets.constituency import convert_starlang\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\nTREE=\"( (S (NP (NP {morphologicalAnalysis=bayan+NOUN+A3SG+PNON+NOM}{metaMorphemes=bayan}{turkish=Bayan}{english=Ms.}{semantics=TUR10-0396530}{namedEntity=PERSON}{propBank=ARG0$TUR10-0148580}{englishSemantics=ENG31-06352895-n}) (NP {morphologicalAnalysis=haag+NOUN+PROP+A3SG+PNON+NOM}{metaMorphemes=haag}{turkish=Haag}{english=Haag}{semantics=TUR10-0000000}{namedEntity=PERSON}{propBank=ARG0$TUR10-0148580})) (VP (NP {morphologicalAnalysis=elianti+NOUN+PROP+A3SG+PNON+NOM}{metaMorphemes=elianti}{turkish=Elianti}{english=Elianti}{semantics=TUR10-0000000}{namedEntity=NONE}{propBank=ARG1$TUR10-0148580}) (VP {morphologicalAnalysis=çal+VERB+POS+AOR+A3SG}{metaMorphemes=çal+Ar}{turkish=çalar}{english=plays}{semantics=TUR10-0148580}{namedEntity=NONE}{propBank=PREDICATE$TUR10-0148580}{englishSemantics=ENG31-01730049-v})) (. {morphologicalAnalysis=.+PUNC}{metaMorphemes=.}{metaMorphemesMoved=.}{turkish=.}{english=.}{semantics=TUR10-1081860}{namedEntity=NONE}{propBank=NONE})) )\"\n\ndef test_read_tree():\n \"\"\"\n Test a basic tree read\n \"\"\"\n tree = convert_starlang.read_tree(TREE)\n assert \"(ROOT (S (NP (NP Bayan) (NP Haag)) (VP (NP Elianti) (VP çalar)) (. .)))\" == str(tree)\n\ndef test_missing_word():\n \"\"\"\n Test that an error is thrown if the word is missing\n \"\"\"\n tree_text = TREE.replace(\"turkish=\", \"foo=\")\n with pytest.raises(ValueError):\n tree = convert_starlang.read_tree(tree_text)\n\ndef test_bad_label():\n \"\"\"\n Test that an unexpected label results in an error\n \"\"\"\n tree_text = TREE.replace(\"(S\", \"(s\")\n with pytest.raises(ValueError):\n tree = convert_starlang.read_tree(tree_text)\n"
},
{
"path": "stanza/tests/constituency/test_ensemble.py",
"content": "\"\"\"\nAdd a simple test of the Ensemble's inference path\n\nThis just reuses one model several times - that should still check the main loop, at least\n\"\"\"\n\nimport pytest\n\nfrom stanza import Pipeline\nfrom stanza.models.constituency import text_processing\nfrom stanza.models.constituency import tree_reader\nfrom stanza.models.constituency.ensemble import Ensemble, EnsembleTrainer\nfrom stanza.models.constituency.text_processing import parse_tokenized_sentences\n\nfrom stanza.tests import TEST_MODELS_DIR\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\n\n@pytest.fixture(scope=\"module\")\ndef pipeline():\n return Pipeline(dir=TEST_MODELS_DIR, lang=\"en\", processors=\"tokenize, pos, constituency\", tokenize_pretokenized=True)\n\n@pytest.fixture(scope=\"module\")\ndef saved_ensemble(tmp_path_factory, pipeline):\n tmp_path = tmp_path_factory.mktemp(\"ensemble\")\n\n # test the ensemble by reusing the same parser multiple times\n con_processor = pipeline.processors[\"constituency\"]\n model = con_processor._model\n args = dict(model.args)\n foundation_cache = pipeline.foundation_cache\n\n model_path = con_processor._config['model_path']\n # reuse the same model 3 times just to make sure the code paths are working\n filenames = [model_path, model_path, model_path]\n\n ensemble = EnsembleTrainer.from_files(args, filenames, foundation_cache=foundation_cache)\n save_path = tmp_path / \"ensemble.pt\"\n\n ensemble.save(save_path)\n return ensemble, save_path, args, foundation_cache\n\ndef check_basic_predictions(trees):\n predictions = [x.predictions for x in trees]\n assert len(predictions) == 2\n assert all(len(x) == 1 for x in predictions)\n trees = [x[0].tree for x in predictions]\n result = [\"{}\".format(tree) for tree in trees]\n expected = [\"(ROOT (S (NP (DT This)) (VP (VBZ is) (NP (DT a) (NN test)))))\",\n \"(ROOT (S (NP (DT This)) (VP (VBZ is) (NP (DT another) (NN test)))))\"]\n assert result == expected\n\ndef test_ensemble_inference(pipeline):\n # test the ensemble by reusing the same parser multiple times\n con_processor = pipeline.processors[\"constituency\"]\n model = con_processor._model\n args = dict(model.args)\n foundation_cache = pipeline.foundation_cache\n\n model_path = con_processor._config['model_path']\n # reuse the same model 3 times just to make sure the code paths are working\n filenames = [model_path, model_path, model_path]\n\n ensemble = EnsembleTrainer.from_files(args, filenames, foundation_cache=foundation_cache)\n ensemble = ensemble.model\n sentences = [[\"This\", \"is\", \"a\", \"test\"], [\"This\", \"is\", \"another\", \"test\"]]\n trees = parse_tokenized_sentences(args, ensemble, [pipeline], sentences)\n check_basic_predictions(trees)\n\ndef test_ensemble_save(saved_ensemble):\n \"\"\"\n Depending on the saved_ensemble fixture should be enough to ensure\n that the ensemble was correctly saved\n\n (loading is tested separately)\n \"\"\"\n\ndef test_ensemble_save_load(pipeline, saved_ensemble):\n _, save_path, args, foundation_cache = saved_ensemble\n ensemble = EnsembleTrainer.load(save_path, args, foundation_cache=foundation_cache)\n sentences = [[\"This\", \"is\", \"a\", \"test\"], [\"This\", \"is\", \"another\", \"test\"]]\n trees = parse_tokenized_sentences(args, ensemble.model, [pipeline], sentences)\n check_basic_predictions(trees)\n\ndef test_parse_text(tmp_path, pipeline, saved_ensemble):\n _, model_path, args, foundation_cache = saved_ensemble\n\n raw_file = str(tmp_path / \"test_input.txt\")\n with open(raw_file, \"w\") as fout:\n fout.write(\"This is a test\\nThis is another test\\n\")\n output_file = str(tmp_path / \"test_output.txt\")\n\n args = dict(args)\n args['tokenized_file'] = raw_file\n args['predict_file'] = output_file\n\n text_processing.load_model_parse_text(args, model_path, [pipeline])\n trees = tree_reader.read_treebank(output_file)\n trees = [\"{}\".format(x) for x in trees]\n expected_trees = [\"(ROOT (S (NP (DT This)) (VP (VBZ is) (NP (DT a) (NN test)))))\",\n \"(ROOT (S (NP (DT This)) (VP (VBZ is) (NP (DT another) (NN test)))))\"]\n assert trees == expected_trees\n\ndef test_pipeline(saved_ensemble):\n _, model_path, _, foundation_cache = saved_ensemble\n nlp = Pipeline(\"en\", processors=\"tokenize,pos,constituency\", constituency_model_path=str(model_path), foundation_cache=foundation_cache, download_method=None)\n doc = nlp(\"This is a test\")\n tree = \"{}\".format(doc.sentences[0].constituency)\n assert tree == \"(ROOT (S (NP (DT This)) (VP (VBZ is) (NP (DT a) (NN test)))))\"\n"
},
{
"path": "stanza/tests/constituency/test_in_order_compound_oracle.py",
"content": "import pytest\n\nfrom stanza.models.constituency import in_order_compound_oracle\nfrom stanza.models.constituency import tree_reader\nfrom stanza.models.constituency.parse_transitions import CloseConstituent, OpenConstituent, Shift, TransitionScheme\nfrom stanza.models.constituency.transition_sequence import build_treebank\n\nfrom stanza.tests.constituency.test_transition_sequence import reconstruct_tree\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\n# A sample tree from PTB with a triple unary transition (at a location other than root)\n# Here we test the incorrect closing of various brackets\nTRIPLE_UNARY_START_TREE = \"\"\"\n( (S\n (PRN\n (S\n (NP-SBJ (-NONE- *) )\n (VP (VB See) )))\n (, ,)\n (NP-SBJ\n (NP (DT the) (JJ other) (NN rule) )\n (PP (IN of)\n (NP (NN thumb) ))\n (PP (IN about)\n (NP (NN ballooning) )))))\n\"\"\"\n\nTREES = [TRIPLE_UNARY_START_TREE]\nTREEBANK = \"\\n\".join(TREES)\n\nROOT_LABELS = [\"ROOT\"]\n\n@pytest.fixture(scope=\"module\")\ndef trees():\n trees = tree_reader.read_trees(TREEBANK)\n trees = [t.prune_none().simplify_labels() for t in trees]\n assert len(trees) == len(TREES)\n\n return trees\n\n@pytest.fixture(scope=\"module\")\ndef gold_sequences(trees):\n gold_sequences = build_treebank(trees, TransitionScheme.IN_ORDER_COMPOUND)\n return gold_sequences\n\ndef get_repairs(gold_sequence, wrong_transition, repair_fn):\n \"\"\"\n Use the repair function and the wrong transition to iterate over the gold sequence\n\n Returns a list of possible repairs, one for each position in the sequence\n Repairs are tuples, (idx, seq)\n \"\"\"\n repairs = [(idx, repair_fn(gold_transition, wrong_transition, gold_sequence, idx, ROOT_LABELS, None, None))\n for idx, gold_transition in enumerate(gold_sequence)]\n repairs = [x for x in repairs if x[1] is not None]\n return repairs\n\ndef test_fix_shift_close():\n trees = tree_reader.read_trees(TRIPLE_UNARY_START_TREE)\n trees = [t.prune_none().simplify_labels() for t in trees]\n assert len(trees) == 1\n tree = trees[0]\n\n gold_sequences = build_treebank(trees, TransitionScheme.IN_ORDER_COMPOUND)\n\n # there are three places in this tree where a long bracket (more than 2 subtrees)\n # could theoretically be closed and then reopened\n repairs = get_repairs(gold_sequences[0], CloseConstituent(), in_order_compound_oracle.fix_shift_close_error)\n assert len(repairs) == 3\n\n expected_trees = [\"(ROOT (S (S (PRN (S (VP (VB See)))) (, ,)) (NP (NP (DT the) (JJ other) (NN rule)) (PP (IN of) (NP (NN thumb))) (PP (IN about) (NP (NN ballooning))))))\",\n \"(ROOT (S (PRN (S (VP (VB See)))) (, ,) (NP (NP (NP (DT the) (JJ other)) (NN rule)) (PP (IN of) (NP (NN thumb))) (PP (IN about) (NP (NN ballooning))))))\",\n \"(ROOT (S (PRN (S (VP (VB See)))) (, ,) (NP (NP (NP (DT the) (JJ other) (NN rule)) (PP (IN of) (NP (NN thumb)))) (PP (IN about) (NP (NN ballooning))))))\"]\n\n for repair, expected in zip(repairs, expected_trees):\n repaired_tree = reconstruct_tree(tree, repair[1], transition_scheme=TransitionScheme.IN_ORDER_COMPOUND)\n assert str(repaired_tree) == expected\n\ndef test_fix_open_close():\n trees = tree_reader.read_trees(TRIPLE_UNARY_START_TREE)\n trees = [t.prune_none().simplify_labels() for t in trees]\n assert len(trees) == 1\n tree = trees[0]\n\n gold_sequences = build_treebank(trees, TransitionScheme.IN_ORDER_COMPOUND)\n\n repairs = get_repairs(gold_sequences[0], CloseConstituent(), in_order_compound_oracle.fix_open_close_error)\n print(\"------------------\")\n for repair in repairs:\n print(repair)\n repaired_tree = reconstruct_tree(tree, repair[1], transition_scheme=TransitionScheme.IN_ORDER_COMPOUND)\n print(\"{:P}\".format(repaired_tree))\n"
},
{
"path": "stanza/tests/constituency/test_in_order_oracle.py",
"content": "import itertools\nimport pytest\n\nfrom stanza.models.constituency import parse_transitions\nfrom stanza.models.constituency import tree_reader\nfrom stanza.models.constituency.base_model import SimpleModel\nfrom stanza.models.constituency.in_order_oracle import *\nfrom stanza.models.constituency.parse_transitions import CloseConstituent, OpenConstituent, Shift, TransitionScheme\nfrom stanza.models.constituency.transition_sequence import build_treebank\n\nfrom stanza.tests import *\nfrom stanza.tests.constituency.test_transition_sequence import reconstruct_tree\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\n# A sample tree from PTB with a single unary transition (at a location other than root)\nSINGLE_UNARY_TREE = \"\"\"\n( (S\n (NP-SBJ-1 (DT A) (NN record) (NN date) )\n (VP (VBZ has) (RB n't)\n (VP (VBN been)\n (VP (VBN set)\n (NP (-NONE- *-1) ))))\n (. .) ))\n\"\"\"\n\n# [Shift, OpenConstituent(('NP-SBJ-1',)), Shift, Shift, CloseConstituent, OpenConstituent(('S',)), Shift, OpenConstituent(('VP',)), Shift, Shift, OpenConstituent(('VP',)), Shift, OpenConstituent(('VP',)), Shift, OpenConstituent(('NP',)), CloseConstituent, CloseConstituent, CloseConstituent, CloseConstituent, Shift, CloseConstituent, OpenConstituent(('ROOT',)), CloseConstituent]\n\n# A sample tree from PTB with a double unary transition (at a location other than root)\nDOUBLE_UNARY_TREE = \"\"\"\n( (S\n (NP-SBJ\n (NP (RB Not) (PDT all) (DT those) )\n (SBAR\n (WHNP-3 (WP who) )\n (S\n (NP-SBJ (-NONE- *T*-3) )\n (VP (VBD wrote) ))))\n (VP (VBP oppose)\n (NP (DT the) (NNS changes) ))\n (. .) ))\n\"\"\"\n\n# A sample tree from PTB with a triple unary transition (at a location other than root)\n# The triple unary is at the START of the next bracket, which affects how the\n# dynamic oracle repairs the transition sequence\nTRIPLE_UNARY_START_TREE = \"\"\"\n( (S\n (PRN\n (S\n (NP-SBJ (-NONE- *) )\n (VP (VB See) )))\n (, ,)\n (NP-SBJ\n (NP (DT the) (JJ other) (NN rule) )\n (PP (IN of)\n (NP (NN thumb) ))\n (PP (IN about)\n (NP (NN ballooning) )))))\n\"\"\"\n\n# A sample tree from PTB with a triple unary transition (at a location other than root)\n# The triple unary is at the END of the next bracket, which affects how the\n# dynamic oracle repairs the transition sequence\nTRIPLE_UNARY_END_TREE = \"\"\"\n( (S\n (NP (NNS optimists) )\n (VP (VBP expect) \n (S \n (NP-SBJ-4 (NNP Hong) (NNP Kong) )\n (VP (TO to) \n (VP (VB hum) \n (ADVP-CLR (RB along) )\n (SBAR-MNR (RB as) \n (S \n (NP-SBJ (-NONE- *-4) )\n (VP (-NONE- *?*) \n (ADVP-TMP (IN before) ))))))))))\n\"\"\"\n\nTREES = [SINGLE_UNARY_TREE, DOUBLE_UNARY_TREE, TRIPLE_UNARY_START_TREE, TRIPLE_UNARY_END_TREE]\nTREEBANK = \"\\n\".join(TREES)\n\nNOUN_PHRASE_TREE = \"\"\"\n( (NP\n (NP (NNP Chicago) (POS 's))\n (NNP Goodman)\n (NNP Theatre)))\n\"\"\"\n\nWIDE_NP_TREE = \"\"\"\n( (S\n (NP-SBJ (DT These) (NNS studies))\n (VP (VBP demonstrate)\n (SBAR (IN that)\n (S\n (NP-SBJ (NNS mice))\n (VP (VBP are)\n (NP-PRD\n (NP (DT a)\n (ADJP (JJ practical)\n (CC and)\n (JJ powerful))\n (JJ experimental) (NN system))\n (SBAR\n (WHADVP-2 (-NONE- *0*))\n (S\n (NP-SBJ (-NONE- *PRO*))\n (VP (TO to)\n (VP (VB study)\n (NP (DT the) (NN genetics)))))))))))))\n\"\"\"\n\nWIDE_TREES = [NOUN_PHRASE_TREE, WIDE_NP_TREE]\nWIDE_TREEBANK = \"\\n\".join(WIDE_TREES)\n\nROOT_LABELS = [\"ROOT\"]\n\ndef get_repairs(gold_sequence, wrong_transition, repair_fn):\n \"\"\"\n Use the repair function and the wrong transition to iterate over the gold sequence\n\n Returns a list of possible repairs, one for each position in the sequence\n Repairs are tuples, (idx, seq)\n \"\"\"\n repairs = [(idx, repair_fn(gold_transition, wrong_transition, gold_sequence, idx, ROOT_LABELS, None, None))\n for idx, gold_transition in enumerate(gold_sequence)]\n repairs = [x for x in repairs if x[1] is not None]\n return repairs\n\n@pytest.fixture(scope=\"module\")\ndef unary_trees():\n trees = tree_reader.read_trees(TREEBANK)\n trees = [t.prune_none().simplify_labels() for t in trees]\n assert len(trees) == len(TREES)\n\n return trees\n\n@pytest.fixture(scope=\"module\")\ndef gold_sequences(unary_trees):\n gold_sequences = build_treebank(unary_trees, TransitionScheme.IN_ORDER)\n return gold_sequences\n\n@pytest.fixture(scope=\"module\")\ndef wide_trees():\n trees = tree_reader.read_trees(WIDE_TREEBANK)\n trees = [t.prune_none().simplify_labels() for t in trees]\n assert len(trees) == len(WIDE_TREES)\n\n return trees\n\ndef test_wrong_open_root(gold_sequences):\n \"\"\"\n Test the results of the dynamic oracle on a few trees if the ROOT is mishandled.\n \"\"\"\n wrong_transition = OpenConstituent(\"S\")\n gold_transition = OpenConstituent(\"ROOT\")\n close_transition = CloseConstituent()\n\n for gold_sequence in gold_sequences:\n # each of the sequences should be ended with ROOT, Close\n assert gold_sequence[-2] == gold_transition\n\n repairs = get_repairs(gold_sequence, wrong_transition, fix_wrong_open_root_error)\n # there is only spot in the sequence with a ROOT, so there should\n # be exactly one location which affords a S/ROOT replacement\n assert len(repairs) == 1\n repair = repairs[0]\n\n # the repair should occur at the -2 position, which is where ROOT is\n assert repair[0] == len(gold_sequence) - 2\n # and the resulting list should have the wrong transition followed by a Close\n # to give the model another chance to close the tree\n expected = gold_sequence[:-2] + [wrong_transition, close_transition] + gold_sequence[-2:]\n assert repair[1] == expected\n\ndef test_missed_unary(gold_sequences):\n \"\"\"\n Test the repairs of an open/open error if it is effectively a skipped unary transition\n \"\"\"\n wrong_transition = OpenConstituent(\"S\")\n\n repairs = get_repairs(gold_sequences[0], wrong_transition, fix_wrong_open_unary_chain)\n assert len(repairs) == 0\n\n # here we are simulating picking NT-S instead of NT-VP\n # the DOUBLE_UNARY tree has one location where this is relevant, index 11\n repairs = get_repairs(gold_sequences[1], wrong_transition, fix_wrong_open_unary_chain)\n assert len(repairs) == 1\n assert repairs[0][0] == 11\n assert repairs[0][1] == gold_sequences[1][:11] + gold_sequences[1][13:]\n\n # the TRIPLE_UNARY_START tree has two locations where this is relevant\n # at index 1, the pattern goes (S (VP ...))\n # so choosing S instead of VP means you can skip the VP and only miss that one bracket\n # at index 5, the pattern goes (S (PRN (S (VP ...))) (...))\n # note that this is capturing a unary transition into a larger constituent\n # skipping the PRN is satisfactory\n repairs = get_repairs(gold_sequences[2], wrong_transition, fix_wrong_open_unary_chain)\n assert len(repairs) == 2\n assert repairs[0][0] == 1\n assert repairs[0][1] == gold_sequences[2][:1] + gold_sequences[2][3:]\n assert repairs[1][0] == 5\n assert repairs[1][1] == gold_sequences[2][:5] + gold_sequences[2][7:]\n\n # The TRIPLE_UNARY_END tree has 2 sections of tree for a total of 3 locations\n # where the repair might happen\n # Surprisingly the unary transition at the very start can only be\n # repaired by skipping it and using the outer S transition instead\n # The second repair overall (first repair in the second location)\n # should have a double skip to reach the S node\n repairs = get_repairs(gold_sequences[3], wrong_transition, fix_wrong_open_unary_chain)\n assert len(repairs) == 3\n assert repairs[0][0] == 1\n assert repairs[0][1] == gold_sequences[3][:1] + gold_sequences[3][3:]\n assert repairs[1][0] == 21\n assert repairs[1][1] == gold_sequences[3][:21] + gold_sequences[3][25:]\n assert repairs[2][0] == 23\n assert repairs[2][1] == gold_sequences[3][:23] + gold_sequences[3][25:]\n\n\ndef test_open_with_stuff(unary_trees, gold_sequences):\n wrong_transition = OpenConstituent(\"S\")\n expected_trees = [\n \"(ROOT (S (DT A) (NN record) (NN date) (VP (VBZ has) (RB n't) (VP (VBN been) (VP (VBN set)))) (. .)))\",\n \"(ROOT (S (NP (RB Not) (PDT all) (DT those)) (SBAR (WHNP (WP who)) (S (VP (VBD wrote)))) (VP (VBP oppose) (NP (DT the) (NNS changes))) (. .)))\",\n None,\n \"(ROOT (S (NP (NNS optimists)) (VP (VBP expect) (S (NNP Hong) (NNP Kong) (VP (TO to) (VP (VB hum) (ADVP (RB along)) (SBAR (RB as) (S (VP (ADVP (IN before)))))))))))\"\n ]\n\n for tree, gold_sequence, expected in zip(unary_trees, gold_sequences, expected_trees):\n repairs = get_repairs(gold_sequence, wrong_transition, fix_wrong_open_stuff_unary)\n if expected is None:\n assert len(repairs) == 0\n else:\n assert len(repairs) == 1\n result = reconstruct_tree(tree, repairs[0][1])\n assert str(result) == expected\n\ndef test_general_open(gold_sequences):\n wrong_transition = OpenConstituent(\"SBARQ\")\n\n for sequence in gold_sequences:\n repairs = get_repairs(sequence, wrong_transition, fix_wrong_open_general)\n assert len(repairs) == sum(isinstance(x, OpenConstituent) for x in sequence) - 1\n for repair in repairs:\n assert len(repair[1]) == len(sequence)\n assert repair[1][repair[0]] == wrong_transition\n assert repair[1][:repair[0]] == sequence[:repair[0]]\n assert repair[1][repair[0]+1:] == sequence[repair[0]+1:]\n\ndef test_missed_unary(unary_trees, gold_sequences):\n shift_transition = Shift()\n close_transition = CloseConstituent()\n\n expected_close_results = [\n [(12, 2)],\n [(11, 4), (13, 2)],\n # (NP NN thumb) and (NP NN ballooning) are both candidates for this repair\n [(18, 2), (24, 2)],\n [(21, 6), (23, 4), (25, 2)],\n ]\n\n expected_shift_results = [\n (),\n (),\n (),\n # (ADVP-CLR (RB along)) is followed by a shift\n [(16, 2)],\n ]\n\n for tree, sequence, expected_close, expected_shift in zip(unary_trees, gold_sequences, expected_close_results, expected_shift_results):\n repairs = get_repairs(sequence, close_transition, fix_missed_unary)\n assert len(repairs) == len(expected_close)\n for repair, (expected_idx, expected_len) in zip(repairs, expected_close):\n assert repair[0] == expected_idx\n assert repair[1] == sequence[:expected_idx] + sequence[expected_idx+expected_len:]\n\n repairs = get_repairs(sequence, shift_transition, fix_missed_unary)\n assert len(repairs) == len(expected_shift)\n for repair, (expected_idx, expected_len) in zip(repairs, expected_shift):\n assert repair[0] == expected_idx\n assert repair[1] == sequence[:expected_idx] + sequence[expected_idx+expected_len:]\n\ndef test_open_shift(unary_trees, gold_sequences):\n shift_transition = Shift()\n\n expected_repairs = [\n [(7, \"(ROOT (S (NP (DT A) (NN record) (NN date)) (VBZ has) (RB n't) (VP (VBN been) (VP (VBN set))) (. .)))\"),\n (10, \"(ROOT (S (NP (DT A) (NN record) (NN date)) (VP (VBZ has) (RB n't) (VBN been) (VP (VBN set))) (. .)))\")],\n [(7, \"(ROOT (S (NP (NP (RB Not) (PDT all) (DT those)) (WP who) (S (VP (VBD wrote)))) (VP (VBP oppose) (NP (DT the) (NNS changes))) (. .)))\"),\n (9, \"(ROOT (S (NP (NP (RB Not) (PDT all) (DT those)) (WHNP (WP who)) (S (VP (VBD wrote)))) (VP (VBP oppose) (NP (DT the) (NNS changes))) (. .)))\"),\n (19, \"(ROOT (S (NP (NP (RB Not) (PDT all) (DT those)) (SBAR (WHNP (WP who)) (S (VP (VBD wrote))))) (VBP oppose) (NP (DT the) (NNS changes)) (. .)))\"),\n (21, \"(ROOT (S (NP (NP (RB Not) (PDT all) (DT those)) (SBAR (WHNP (WP who)) (S (VP (VBD wrote))))) (VP (VBP oppose) (DT the) (NNS changes)) (. .)))\")],\n [(14, \"(ROOT (S (PRN (S (VP (VB See)))) (, ,) (NP (DT the) (JJ other) (NN rule)) (PP (IN of) (NP (NN thumb))) (PP (IN about) (NP (NN ballooning)))))\"),\n (16, \"(ROOT (S (PRN (S (VP (VB See)))) (, ,) (NP (NP (DT the) (JJ other) (NN rule)) (IN of) (NP (NN thumb)) (PP (IN about) (NP (NN ballooning))))))\"),\n (22, \"(ROOT (S (PRN (S (VP (VB See)))) (, ,) (NP (NP (DT the) (JJ other) (NN rule)) (PP (IN of) (NP (NN thumb))) (IN about) (NP (NN ballooning)))))\")],\n [(5, \"(ROOT (S (NP (NNS optimists)) (VBP expect) (S (NP (NNP Hong) (NNP Kong)) (VP (TO to) (VP (VB hum) (ADVP (RB along)) (SBAR (RB as) (S (VP (ADVP (IN before))))))))))\"),\n (10, \"(ROOT (S (NP (NNS optimists)) (VP (VBP expect) (NP (NNP Hong) (NNP Kong)) (VP (TO to) (VP (VB hum) (ADVP (RB along)) (SBAR (RB as) (S (VP (ADVP (IN before))))))))))\"),\n (12, \"(ROOT (S (NP (NNS optimists)) (VP (VBP expect) (S (NP (NNP Hong) (NNP Kong)) (TO to) (VP (VB hum) (ADVP (RB along)) (SBAR (RB as) (S (VP (ADVP (IN before))))))))))\"),\n (14, \"(ROOT (S (NP (NNS optimists)) (VP (VBP expect) (S (NP (NNP Hong) (NNP Kong)) (VP (TO to) (VB hum) (ADVP (RB along)) (SBAR (RB as) (S (VP (ADVP (IN before))))))))))\"),\n (19, \"(ROOT (S (NP (NNS optimists)) (VP (VBP expect) (S (NP (NNP Hong) (NNP Kong)) (VP (TO to) (VP (VB hum) (ADVP (RB along)) (RB as) (S (VP (ADVP (IN before))))))))))\")]\n ]\n\n for tree, sequence, expected in zip(unary_trees, gold_sequences, expected_repairs):\n repairs = get_repairs(sequence, shift_transition, fix_open_shift)\n assert len(repairs) == len(expected)\n for repair, (idx, expected_tree) in zip(repairs, expected):\n assert repair[0] == idx\n result_tree = reconstruct_tree(tree, repair[1])\n assert str(result_tree) == expected_tree\n\n\ndef test_open_close(unary_trees, gold_sequences):\n close_transition = CloseConstituent()\n\n expected_repairs = [\n [(7, \"(ROOT (S (S (NP (DT A) (NN record) (NN date)) (VBZ has)) (RB n't) (VP (VBN been) (VP (VBN set))) (. .)))\"),\n (10, \"(ROOT (S (NP (DT A) (NN record) (NN date)) (VP (VP (VBZ has) (RB n't) (VBN been)) (VP (VBN set))) (. .)))\")],\n # missed the WHNP. The surrounding SBAR cannot be created, either\n [(7, \"(ROOT (S (NP (NP (NP (RB Not) (PDT all) (DT those)) (WP who)) (S (VP (VBD wrote)))) (VP (VBP oppose) (NP (DT the) (NNS changes))) (. .)))\"),\n # missed the SBAR\n (9, \"(ROOT (S (NP (NP (NP (RB Not) (PDT all) (DT those)) (WHNP (WP who))) (S (VP (VBD wrote)))) (VP (VBP oppose) (NP (DT the) (NNS changes))) (. .)))\"),\n # missed the VP around \"oppose the changes\"\n (19, \"(ROOT (S (S (NP (NP (RB Not) (PDT all) (DT those)) (SBAR (WHNP (WP who)) (S (VP (VBD wrote))))) (VBP oppose)) (NP (DT the) (NNS changes)) (. .)))\"),\n # missed the NP in \"the changes\", looks pretty bad tbh\n (21, \"(ROOT (S (NP (NP (RB Not) (PDT all) (DT those)) (SBAR (WHNP (WP who)) (S (VP (VBD wrote))))) (VP (VP (VBP oppose) (DT the)) (NNS changes)) (. .)))\")],\n [(14, \"(ROOT (S (S (PRN (S (VP (VB See)))) (, ,) (NP (DT the) (JJ other) (NN rule))) (PP (IN of) (NP (NN thumb))) (PP (IN about) (NP (NN ballooning)))))\"),\n (16, \"(ROOT (S (PRN (S (VP (VB See)))) (, ,) (NP (NP (NP (DT the) (JJ other) (NN rule)) (IN of)) (NP (NN thumb)) (PP (IN about) (NP (NN ballooning))))))\"),\n (22, \"(ROOT (S (PRN (S (VP (VB See)))) (, ,) (NP (NP (NP (DT the) (JJ other) (NN rule)) (PP (IN of) (NP (NN thumb))) (IN about)) (NP (NN ballooning)))))\")],\n [(5, \"(ROOT (S (S (NP (NNS optimists)) (VBP expect)) (S (NP (NNP Hong) (NNP Kong)) (VP (TO to) (VP (VB hum) (ADVP (RB along)) (SBAR (RB as) (S (VP (ADVP (IN before))))))))))\"),\n (10, \"(ROOT (S (NP (NNS optimists)) (VP (VP (VBP expect) (NP (NNP Hong) (NNP Kong))) (VP (TO to) (VP (VB hum) (ADVP (RB along)) (SBAR (RB as) (S (VP (ADVP (IN before))))))))))\"),\n (12, \"(ROOT (S (NP (NNS optimists)) (VP (VBP expect) (S (S (NP (NNP Hong) (NNP Kong)) (TO to)) (VP (VB hum) (ADVP (RB along)) (SBAR (RB as) (S (VP (ADVP (IN before))))))))))\"),\n (14, \"(ROOT (S (NP (NNS optimists)) (VP (VBP expect) (S (NP (NNP Hong) (NNP Kong)) (VP (VP (TO to) (VB hum)) (ADVP (RB along)) (SBAR (RB as) (S (VP (ADVP (IN before))))))))))\"),\n (19, \"(ROOT (S (NP (NNS optimists)) (VP (VBP expect) (S (NP (NNP Hong) (NNP Kong)) (VP (TO to) (VP (VP (VB hum) (ADVP (RB along)) (RB as)) (S (VP (ADVP (IN before))))))))))\")]\n ]\n\n for tree, sequence, expected in zip(unary_trees, gold_sequences, expected_repairs):\n repairs = get_repairs(sequence, close_transition, fix_open_close)\n\n assert len(repairs) == len(expected)\n for repair, (idx, expected_tree) in zip(repairs, expected):\n assert repair[0] == idx\n result_tree = reconstruct_tree(tree, repair[1])\n assert str(result_tree) == expected_tree\n\ndef test_shift_close(unary_trees, gold_sequences):\n \"\"\"\n Test the fix for a shift -> close\n\n These errors can occur pretty much everywhere, and the fix is quite simple,\n so we only test a few cases.\n \"\"\"\n\n close_transition = CloseConstituent()\n\n expected_tree = \"(ROOT (S (NP (NP (DT A)) (NN record) (NN date)) (VP (VBZ has) (RB n't) (VP (VBN been) (VP (VBN set)))) (. .)))\"\n\n repairs = get_repairs(gold_sequences[0], close_transition, fix_shift_close)\n assert len(repairs) == 7\n result_tree = reconstruct_tree(unary_trees[0], repairs[0][1])\n assert str(result_tree) == expected_tree\n\n repairs = get_repairs(gold_sequences[1], close_transition, fix_shift_close)\n assert len(repairs) == 8\n\n repairs = get_repairs(gold_sequences[2], close_transition, fix_shift_close)\n assert len(repairs) == 8\n\n repairs = get_repairs(gold_sequences[3], close_transition, fix_shift_close)\n assert len(repairs) == 9\n for rep in repairs:\n if rep[0] == 16:\n # This one is special because it occurs as part of a unary\n # in other words, it should go unary, shift\n # and instead we are making it close where the unary should be\n # ... the unary would create \"(ADVP (RB along))\"\n expected_tree = \"(ROOT (S (NP (NNS optimists)) (VP (VBP expect) (S (NP (NNP Hong) (NNP Kong)) (VP (TO to) (VP (VP (VB hum) (RB along)) (SBAR (RB as) (S (VP (ADVP (IN before)))))))))))\"\n result_tree = reconstruct_tree(unary_trees[3], rep[1])\n assert str(result_tree) == expected_tree\n break\n else:\n raise AssertionError(\"Did not find an expected repair location\")\n\ndef test_close_open_shift_nested(unary_trees, gold_sequences):\n shift_transition = Shift()\n\n expected_trees = [{},\n {4: \"(ROOT (S (NP (RB Not) (PDT all) (DT those) (SBAR (WHNP (WP who)) (S (VP (VBD wrote))))) (VP (VBP oppose) (NP (DT the) (NNS changes))) (. .)))\"},\n {4: \"(ROOT (S (VP (VB See)) (, ,) (NP (NP (DT the) (JJ other) (NN rule)) (PP (IN of) (NP (NN thumb))) (PP (IN about) (NP (NN ballooning))))))\",\n 13: \"(ROOT (S (PRN (S (VP (VB See)))) (, ,) (NP (DT the) (JJ other) (NN rule) (PP (IN of) (NP (NN thumb))) (PP (IN about) (NP (NN ballooning))))))\"},\n {}]\n\n for tree, gold_sequence, expected in zip(unary_trees, gold_sequences, expected_trees):\n repairs = get_repairs(gold_sequence, shift_transition, fix_close_open_shift_nested)\n assert len(repairs) == len(expected)\n if len(expected) >= 1:\n for repair in repairs:\n assert repair[0] in expected.keys()\n result_tree = reconstruct_tree(tree, repair[1])\n assert str(result_tree) == expected[repair[0]]\n\ndef check_repairs(trees, gold_sequences, expected_trees, transition, repair_fn):\n for tree_idx, (gold_tree, gold_sequence, expected) in enumerate(zip(trees, gold_sequences, expected_trees)):\n repairs = get_repairs(gold_sequence, transition, repair_fn)\n if expected is not None:\n assert len(repairs) == len(expected)\n for repair in repairs:\n assert repair[0] in expected\n result_tree = reconstruct_tree(gold_tree, repair[1])\n assert str(result_tree) == expected[repair[0]]\n else:\n print(\"---------------------\")\n print(\"{:P}\".format(gold_tree))\n print(gold_sequence)\n #print(repairs)\n for repair in repairs:\n print(\"---------------------\")\n print(gold_sequence)\n print(repair[1])\n result_tree = reconstruct_tree(gold_tree, repair[1])\n print(\"{:P}\".format(gold_tree))\n print(\"{:P}\".format(result_tree))\n print(tree_idx)\n print(repair[0])\n print(result_tree)\n\ndef test_close_open_shift_unambiguous(unary_trees, gold_sequences):\n shift_transition = Shift()\n\n expected_trees = [{},\n {8: \"(ROOT (S (NP (NP (RB Not) (PDT all) (DT those)) (SBAR (WHNP (WP who) (S (VP (VBD wrote)))))) (VP (VBP oppose) (NP (DT the) (NNS changes))) (. .)))\"},\n {},\n {2: \"(ROOT (S (NP (NNS optimists) (VP (VBP expect) (S (NP (NNP Hong) (NNP Kong)) (VP (TO to) (VP (VB hum) (ADVP (RB along)) (SBAR (RB as) (S (VP (ADVP (IN before))))))))))))\",\n 9: \"(ROOT (S (NP (NNS optimists)) (VP (VBP expect) (S (NP (NNP Hong) (NNP Kong) (VP (TO to) (VP (VB hum) (ADVP (RB along)) (SBAR (RB as) (S (VP (ADVP (IN before))))))))))))\"}]\n check_repairs(unary_trees, gold_sequences, expected_trees, shift_transition, fix_close_open_shift_unambiguous_bracket)\n\ndef test_close_open_shift_ambiguous_early(unary_trees, gold_sequences):\n shift_transition = Shift()\n\n expected_trees = [{4: \"(ROOT (S (NP (DT A) (NN record) (NN date) (VP (VBZ has) (RB n't) (VP (VBN been) (VP (VBN set))))) (. .)))\"},\n {16: \"(ROOT (S (NP (NP (RB Not) (PDT all) (DT those)) (SBAR (WHNP (WP who)) (S (VP (VBD wrote)))) (VP (VBP oppose) (NP (DT the) (NNS changes)))) (. .)))\"},\n {2: \"(ROOT (S (PRN (S (VP (VB See) (, ,)))) (NP (NP (DT the) (JJ other) (NN rule)) (PP (IN of) (NP (NN thumb))) (PP (IN about) (NP (NN ballooning))))))\",\n 6: \"(ROOT (S (PRN (S (VP (VB See))) (, ,)) (NP (NP (DT the) (JJ other) (NN rule)) (PP (IN of) (NP (NN thumb))) (PP (IN about) (NP (NN ballooning))))))\"},\n {}]\n check_repairs(unary_trees, gold_sequences, expected_trees, shift_transition, fix_close_open_shift_ambiguous_bracket_early)\n\ndef test_close_open_shift_ambiguous_late(unary_trees, gold_sequences):\n shift_transition = Shift()\n\n expected_trees = [{4: \"(ROOT (S (NP (DT A) (NN record) (NN date) (VP (VBZ has) (RB n't) (VP (VBN been) (VP (VBN set)))) (. .))))\"},\n {16: \"(ROOT (S (NP (NP (RB Not) (PDT all) (DT those)) (SBAR (WHNP (WP who)) (S (VP (VBD wrote)))) (VP (VBP oppose) (NP (DT the) (NNS changes))) (. .))))\"},\n {2: \"(ROOT (S (PRN (S (VP (VB See) (, ,) (NP (NP (DT the) (JJ other) (NN rule)) (PP (IN of) (NP (NN thumb))) (PP (IN about) (NP (NN ballooning)))))))))\",\n 6: \"(ROOT (S (PRN (S (VP (VB See))) (, ,) (NP (NP (DT the) (JJ other) (NN rule)) (PP (IN of) (NP (NN thumb))) (PP (IN about) (NP (NN ballooning)))))))\"},\n {}]\n check_repairs(unary_trees, gold_sequences, expected_trees, shift_transition, fix_close_open_shift_ambiguous_bracket_late)\n\n\ndef test_close_shift_shift(unary_trees, wide_trees):\n \"\"\"\n Test that close -> shift works when there is a single block shifted after\n\n Includes a test specifically that there is no oracle action when there are two blocks after the missed close\n \"\"\"\n shift_transition = Shift()\n\n expected_trees = [{15: \"(ROOT (S (NP (DT A) (NN record) (NN date)) (VP (VBZ has) (RB n't) (VP (VBN been) (VP (VBN set))) (. .))))\"},\n {24: \"(ROOT (S (NP (NP (RB Not) (PDT all) (DT those)) (SBAR (WHNP (WP who)) (S (VP (VBD wrote))))) (VP (VBP oppose) (NP (DT the) (NNS changes)) (. .))))\"},\n {20: \"(ROOT (S (PRN (S (VP (VB See)))) (, ,) (NP (NP (DT the) (JJ other) (NN rule)) (PP (IN of) (NP (NN thumb)) (PP (IN about) (NP (NN ballooning)))))))\"},\n {17: \"(ROOT (S (NP (NNS optimists)) (VP (VBP expect) (S (NP (NNP Hong) (NNP Kong)) (VP (TO to) (VP (VB hum) (ADVP (RB along) (SBAR (RB as) (S (VP (ADVP (IN before))))))))))))\"},\n {},\n {}]\n\n test_trees = unary_trees + wide_trees\n gold_sequences = build_treebank(test_trees, TransitionScheme.IN_ORDER)\n\n check_repairs(test_trees, gold_sequences, expected_trees, shift_transition, fix_close_shift_shift_unambiguous)\n\n\ndef test_close_shift_shift_early(unary_trees, wide_trees):\n \"\"\"\n Test that close -> shift works when there are multiple blocks shifted after\n\n Also checks that the single block case is skipped, so as to keep them separate when testing\n\n A tree with the expected property was specifically added for this test\n \"\"\"\n shift_transition = Shift()\n\n test_trees = unary_trees + wide_trees\n gold_sequences = build_treebank(test_trees, TransitionScheme.IN_ORDER)\n\n expected_trees = [{},\n {},\n {},\n {},\n {},\n {21: \"(ROOT (S (NP (DT These) (NNS studies)) (VP (VBP demonstrate) (SBAR (IN that) (S (NP (NNS mice)) (VP (VBP are) (NP (NP (DT a) (ADJP (JJ practical) (CC and) (JJ powerful) (JJ experimental)) (NN system)) (SBAR (S (VP (TO to) (VP (VB study) (NP (DT the) (NN genetics)))))))))))))\"}]\n\n check_repairs(test_trees, gold_sequences, expected_trees, shift_transition, fix_close_shift_shift_ambiguous_early)\n\ndef test_close_shift_shift_late(unary_trees, wide_trees):\n \"\"\"\n Test that close -> shift works when there are multiple blocks shifted after\n\n Also checks that the single block case is skipped, so as to keep them separate when testing\n\n A tree with the expected property was specifically added for this test\n \"\"\"\n shift_transition = Shift()\n\n test_trees = unary_trees + wide_trees\n gold_sequences = build_treebank(test_trees, TransitionScheme.IN_ORDER)\n\n expected_trees = [{},\n {},\n {},\n {},\n {},\n {21: \"(ROOT (S (NP (DT These) (NNS studies)) (VP (VBP demonstrate) (SBAR (IN that) (S (NP (NNS mice)) (VP (VBP are) (NP (NP (DT a) (ADJP (JJ practical) (CC and) (JJ powerful) (JJ experimental) (NN system))) (SBAR (S (VP (TO to) (VP (VB study) (NP (DT the) (NN genetics)))))))))))))\"}]\n\n check_repairs(test_trees, gold_sequences, expected_trees, shift_transition, fix_close_shift_shift_ambiguous_late)\n"
},
{
"path": "stanza/tests/constituency/test_lstm_model.py",
"content": "import os\n\nimport pytest\nimport torch\n\nfrom stanza.models.common import pretrain\nfrom stanza.models.common.utils import set_random_seed\nfrom stanza.models.constituency import parse_transitions\nfrom stanza.tests import *\nfrom stanza.tests.constituency import test_parse_transitions\nfrom stanza.tests.constituency.test_trainer import build_trainer\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\n@pytest.fixture(scope=\"module\")\ndef pretrain_file():\n return f'{TEST_WORKING_DIR}/in/tiny_emb.pt'\n\ndef build_model(pretrain_file, *args):\n # By default, we turn off multistage, since that can turn off various other structures in the initial training\n args = ['--no_multistage', '--pattn_num_layers', '4', '--pattn_d_model', '256', '--hidden_size', '128', '--use_lattn'] + list(args)\n trainer = build_trainer(pretrain_file, *args)\n return trainer.model\n\n@pytest.fixture(scope=\"module\")\ndef unary_model(pretrain_file):\n return build_model(pretrain_file, \"--transition_scheme\", \"TOP_DOWN_UNARY\")\n\ndef test_initial_state(unary_model):\n test_parse_transitions.test_initial_state(unary_model)\n\ndef test_shift(pretrain_file):\n # TODO: might be good to include some tests specifically for shift\n # in the context of a model with unaries\n model = build_model(pretrain_file)\n test_parse_transitions.test_shift(model)\n\ndef test_unary(unary_model):\n test_parse_transitions.test_unary(unary_model)\n\ndef test_unary_requires_root(unary_model):\n test_parse_transitions.test_unary_requires_root(unary_model)\n\ndef test_open(unary_model):\n test_parse_transitions.test_open(unary_model)\n\ndef test_compound_open(pretrain_file):\n model = build_model(pretrain_file, '--transition_scheme', \"TOP_DOWN_COMPOUND\")\n test_parse_transitions.test_compound_open(model)\n\ndef test_in_order_open(pretrain_file):\n model = build_model(pretrain_file, '--transition_scheme', \"IN_ORDER\")\n test_parse_transitions.test_in_order_open(model)\n\ndef test_close(unary_model):\n test_parse_transitions.test_close(unary_model)\n\ndef run_forward_checks(model, num_states=1):\n \"\"\"\n Run a couple small transitions and a forward pass on the given model\n\n Results are not checked in any way. This function allows for\n testing that building models with various options results in a\n functional model.\n \"\"\"\n states = test_parse_transitions.build_initial_state(model, num_states)\n model(states)\n\n shift = parse_transitions.Shift()\n shifts = [shift for _ in range(num_states)]\n states = model.bulk_apply(states, shifts)\n model(states)\n\n open_transition = parse_transitions.OpenConstituent(\"NP\")\n open_transitions = [open_transition for _ in range(num_states)]\n assert open_transition.is_legal(states[0], model)\n states = model.bulk_apply(states, open_transitions)\n assert states[0].num_opens == 1\n model(states)\n\n states = model.bulk_apply(states, shifts)\n model(states)\n states = model.bulk_apply(states, shifts)\n model(states)\n assert states[0].num_opens == 1\n # now should have \"mox\", \"opal\" on the constituents\n\n close_transition = parse_transitions.CloseConstituent()\n close_transitions = [close_transition for _ in range(num_states)]\n assert close_transition.is_legal(states[0], model)\n states = model.bulk_apply(states, close_transitions)\n assert states[0].num_opens == 0\n\n model(states)\n\ndef test_unary_forward(unary_model):\n \"\"\"\n Checks that the forward pass doesn't crash when run after various operations\n\n Doesn't check the forward pass for making reasonable answers\n \"\"\"\n run_forward_checks(unary_model)\n\ndef test_lstm_forward(pretrain_file):\n model = build_model(pretrain_file)\n run_forward_checks(model, num_states=1)\n run_forward_checks(model, num_states=2)\n\ndef test_lstm_layers(pretrain_file):\n model = build_model(pretrain_file, '--num_lstm_layers', '1')\n run_forward_checks(model)\n model = build_model(pretrain_file, '--num_lstm_layers', '2')\n run_forward_checks(model)\n model = build_model(pretrain_file, '--num_lstm_layers', '3')\n run_forward_checks(model)\n\ndef test_multiple_output_forward(pretrain_file):\n \"\"\"\n Test a couple different sizes of output layers\n \"\"\"\n model = build_model(pretrain_file, '--num_output_layers', '1', '--num_lstm_layers', '2')\n run_forward_checks(model)\n\n model = build_model(pretrain_file, '--num_output_layers', '2', '--num_lstm_layers', '2')\n run_forward_checks(model)\n\n model = build_model(pretrain_file, '--num_output_layers', '3', '--num_lstm_layers', '2')\n run_forward_checks(model)\n\ndef test_no_tag_embedding_forward(pretrain_file):\n \"\"\"\n Test that the model continues to work if the tag embedding is turned on or off\n \"\"\"\n model = build_model(pretrain_file, '--tag_embedding_dim', '20')\n run_forward_checks(model)\n\n model = build_model(pretrain_file, '--tag_embedding_dim', '0')\n run_forward_checks(model)\n\ndef test_forward_combined_dummy(pretrain_file):\n \"\"\"\n Tests combined dummy and open node embeddings\n \"\"\"\n model = build_model(pretrain_file, '--combined_dummy_embedding')\n run_forward_checks(model)\n\n model = build_model(pretrain_file, '--no_combined_dummy_embedding')\n run_forward_checks(model)\n\ndef test_nonlinearity_init(pretrain_file):\n \"\"\"\n Tests that different initialization methods of the nonlinearities result in valid tensors\n \"\"\"\n model = build_model(pretrain_file, '--nonlinearity', 'relu')\n run_forward_checks(model)\n\n model = build_model(pretrain_file, '--nonlinearity', 'tanh')\n run_forward_checks(model)\n\n model = build_model(pretrain_file, '--nonlinearity', 'silu')\n run_forward_checks(model)\n\ndef test_forward_charlm(pretrain_file):\n \"\"\"\n Tests loading and running a charlm\n\n Note that this doesn't test the results of the charlm itself,\n just that the model is shaped correctly\n \"\"\"\n forward_charlm_path = os.path.join(TEST_MODELS_DIR, \"en\", \"forward_charlm\", \"1billion.pt\")\n backward_charlm_path = os.path.join(TEST_MODELS_DIR, \"en\", \"backward_charlm\", \"1billion.pt\")\n assert os.path.exists(forward_charlm_path), \"Need to download en test models (or update path to the forward charlm)\"\n assert os.path.exists(backward_charlm_path), \"Need to download en test models (or update path to the backward charlm)\"\n\n model = build_model(pretrain_file, '--charlm_forward_file', forward_charlm_path, '--charlm_backward_file', backward_charlm_path, '--sentence_boundary_vectors', 'none')\n run_forward_checks(model)\n\n model = build_model(pretrain_file, '--charlm_forward_file', forward_charlm_path, '--charlm_backward_file', backward_charlm_path, '--sentence_boundary_vectors', 'words')\n run_forward_checks(model)\n\ndef test_forward_bert(pretrain_file):\n \"\"\"\n Test on a tiny Bert, which hopefully does not take up too much disk space or memory\n \"\"\"\n bert_model = \"hf-internal-testing/tiny-bert\"\n\n model = build_model(pretrain_file, '--bert_model', bert_model)\n run_forward_checks(model)\n\n\ndef test_forward_xlnet(pretrain_file):\n \"\"\"\n Test on a tiny xlnet, which hopefully does not take up too much disk space or memory\n \"\"\"\n bert_model = \"hf-internal-testing/tiny-random-xlnet\"\n\n model = build_model(pretrain_file, '--bert_model', bert_model)\n run_forward_checks(model)\n\n\ndef test_forward_sentence_boundaries(pretrain_file):\n \"\"\"\n Test start & stop boundary vectors\n \"\"\"\n model = build_model(pretrain_file, '--sentence_boundary_vectors', 'everything')\n run_forward_checks(model)\n\n model = build_model(pretrain_file, '--sentence_boundary_vectors', 'words')\n run_forward_checks(model)\n\n model = build_model(pretrain_file, '--sentence_boundary_vectors', 'none')\n run_forward_checks(model)\n\ndef test_forward_constituency_composition(pretrain_file):\n \"\"\"\n Test different constituency composition functions\n \"\"\"\n model = build_model(pretrain_file, '--constituency_composition', 'bilstm')\n run_forward_checks(model, num_states=2)\n\n model = build_model(pretrain_file, '--constituency_composition', 'max')\n run_forward_checks(model, num_states=2)\n\n model = build_model(pretrain_file, '--constituency_composition', 'key')\n run_forward_checks(model, num_states=2)\n\n model = build_model(pretrain_file, '--constituency_composition', 'untied_key')\n run_forward_checks(model, num_states=2)\n\n model = build_model(pretrain_file, '--constituency_composition', 'untied_max')\n run_forward_checks(model, num_states=2)\n\n model = build_model(pretrain_file, '--constituency_composition', 'bilstm_max')\n run_forward_checks(model, num_states=2)\n\n model = build_model(pretrain_file, '--constituency_composition', 'tree_lstm')\n run_forward_checks(model, num_states=2)\n\n model = build_model(pretrain_file, '--constituency_composition', 'tree_lstm_cx')\n run_forward_checks(model, num_states=2)\n\n model = build_model(pretrain_file, '--constituency_composition', 'bigram')\n run_forward_checks(model, num_states=2)\n\n model = build_model(pretrain_file, '--constituency_composition', 'attn')\n run_forward_checks(model, num_states=2)\n\ndef test_forward_key_position(pretrain_file):\n \"\"\"\n Test KEY and UNTIED_KEY either with or without reduce_position\n \"\"\"\n model = build_model(pretrain_file, '--constituency_composition', 'untied_key', '--reduce_position', '0')\n run_forward_checks(model, num_states=2)\n\n model = build_model(pretrain_file, '--constituency_composition', 'untied_key', '--reduce_position', '32')\n run_forward_checks(model, num_states=2)\n\n model = build_model(pretrain_file, '--constituency_composition', 'key', '--reduce_position', '0')\n run_forward_checks(model, num_states=2)\n\n model = build_model(pretrain_file, '--constituency_composition', 'key', '--reduce_position', '32')\n run_forward_checks(model, num_states=2)\n\n\ndef test_forward_attn_hidden_size(pretrain_file):\n \"\"\"\n Test that when attn is used with hidden sizes not evenly divisible by reduce_heads, the model reconfigures the hidden_size\n \"\"\"\n model = build_model(pretrain_file, '--constituency_composition', 'attn', '--hidden_size', '129')\n assert model.hidden_size >= 129\n assert model.hidden_size % model.reduce_heads == 0\n run_forward_checks(model, num_states=2)\n\n model = build_model(pretrain_file, '--constituency_composition', 'attn', '--hidden_size', '129', '--reduce_heads', '10')\n assert model.hidden_size == 130\n assert model.reduce_heads == 10\n\ndef test_forward_partitioned_attention(pretrain_file):\n \"\"\"\n Test with & without partitioned attention layers\n \"\"\"\n model = build_model(pretrain_file, '--pattn_num_heads', '8', '--pattn_num_layers', '8')\n run_forward_checks(model)\n\n model = build_model(pretrain_file, '--pattn_num_heads', '0', '--pattn_num_layers', '0')\n run_forward_checks(model)\n\ndef test_forward_labeled_attention(pretrain_file):\n \"\"\"\n Test with & without labeled attention layers\n \"\"\"\n model = build_model(pretrain_file, '--lattn_d_proj', '64', '--lattn_d_l', '16')\n run_forward_checks(model)\n\n model = build_model(pretrain_file, '--lattn_d_proj', '0', '--lattn_d_l', '0')\n run_forward_checks(model)\n\n model = build_model(pretrain_file, '--lattn_d_proj', '64', '--lattn_d_l', '16', '--lattn_combined_input')\n run_forward_checks(model)\n\ndef test_lattn_partitioned(pretrain_file):\n model = build_model(pretrain_file, '--lattn_d_proj', '64', '--lattn_d_l', '16', '--lattn_partitioned')\n run_forward_checks(model)\n\n model = build_model(pretrain_file, '--lattn_d_proj', '64', '--lattn_d_l', '16', '--no_lattn_partitioned')\n run_forward_checks(model)\n\n\ndef test_lattn_projection(pretrain_file):\n \"\"\"\n Test with & without labeled attention layers\n \"\"\"\n with pytest.raises(ValueError):\n # this is too small\n model = build_model(pretrain_file, '--pattn_d_model', '1024', '--lattn_d_proj', '64', '--lattn_d_l', '16', '--lattn_d_input_proj', '256', '--lattn_partitioned')\n run_forward_checks(model)\n\n model = build_model(pretrain_file, '--pattn_d_model', '1024', '--lattn_d_proj', '64', '--lattn_d_l', '16', '--no_lattn_partitioned', '--lattn_d_input_proj', '256')\n run_forward_checks(model)\n\n model = build_model(pretrain_file, '--lattn_d_proj', '64', '--lattn_d_l', '16', '--lattn_d_input_proj', '768')\n run_forward_checks(model)\n\n # check that it works if we turn off the projection,\n # in case having it on beccomes the default\n model = build_model(pretrain_file, '--lattn_d_proj', '64', '--lattn_d_l', '16', '--lattn_d_input_proj', '0')\n run_forward_checks(model)\n\ndef test_forward_timing_choices(pretrain_file):\n \"\"\"\n Test different timing / position encodings\n \"\"\"\n model = build_model(pretrain_file, '--pattn_num_heads', '4', '--pattn_num_layers', '4', '--pattn_timing', 'sin')\n run_forward_checks(model)\n\n model = build_model(pretrain_file, '--pattn_num_heads', '4', '--pattn_num_layers', '4', '--pattn_timing', 'learned')\n run_forward_checks(model)\n\ndef test_transition_stack(pretrain_file):\n \"\"\"\n Test different transition stack types: lstm & attention\n \"\"\"\n model = build_model(pretrain_file,\n '--pattn_num_layers', '0', '--lattn_d_proj', '0',\n '--transition_stack', 'attn', '--transition_heads', '1')\n run_forward_checks(model)\n\n model = build_model(pretrain_file,\n '--pattn_num_layers', '0', '--lattn_d_proj', '0',\n '--transition_stack', 'attn', '--transition_heads', '4')\n run_forward_checks(model)\n\n model = build_model(pretrain_file,\n '--pattn_num_layers', '0', '--lattn_d_proj', '0',\n '--transition_stack', 'lstm')\n run_forward_checks(model)\n\ndef test_constituent_stack(pretrain_file):\n \"\"\"\n Test different constituent stack types: lstm & attention\n \"\"\"\n model = build_model(pretrain_file,\n '--pattn_num_layers', '0', '--lattn_d_proj', '0',\n '--constituent_stack', 'attn', '--constituent_heads', '1')\n run_forward_checks(model)\n\n model = build_model(pretrain_file,\n '--pattn_num_layers', '0', '--lattn_d_proj', '0',\n '--constituent_stack', 'attn', '--constituent_heads', '4')\n run_forward_checks(model)\n\n model = build_model(pretrain_file,\n '--pattn_num_layers', '0', '--lattn_d_proj', '0',\n '--constituent_stack', 'lstm')\n run_forward_checks(model)\n\ndef test_different_transition_sizes(pretrain_file):\n \"\"\"\n If the transition hidden size and embedding size are different, the model should still work\n \"\"\"\n model = build_model(pretrain_file,\n '--pattn_num_layers', '0', '--lattn_d_proj', '0',\n '--transition_embedding_dim', '10', '--transition_hidden_size', '10',\n '--sentence_boundary_vectors', 'everything')\n run_forward_checks(model)\n\n model = build_model(pretrain_file,\n '--pattn_num_layers', '0', '--lattn_d_proj', '0',\n '--transition_embedding_dim', '20', '--transition_hidden_size', '10',\n '--sentence_boundary_vectors', 'everything')\n run_forward_checks(model)\n\n model = build_model(pretrain_file,\n '--pattn_num_layers', '0', '--lattn_d_proj', '0',\n '--transition_embedding_dim', '10', '--transition_hidden_size', '20',\n '--sentence_boundary_vectors', 'everything')\n run_forward_checks(model)\n\n model = build_model(pretrain_file,\n '--pattn_num_layers', '0', '--lattn_d_proj', '0',\n '--transition_embedding_dim', '10', '--transition_hidden_size', '10',\n '--sentence_boundary_vectors', 'none')\n run_forward_checks(model)\n\n model = build_model(pretrain_file,\n '--pattn_num_layers', '0', '--lattn_d_proj', '0',\n '--transition_embedding_dim', '20', '--transition_hidden_size', '10',\n '--sentence_boundary_vectors', 'none')\n run_forward_checks(model)\n\n model = build_model(pretrain_file,\n '--pattn_num_layers', '0', '--lattn_d_proj', '0',\n '--transition_embedding_dim', '10', '--transition_hidden_size', '20',\n '--sentence_boundary_vectors', 'none')\n run_forward_checks(model)\n\ndef test_relative_attention(pretrain_file):\n model = build_model(pretrain_file, '--no_use_lattn', '--use_rattn', '--rattn_heads', '10', '--no_rattn_cat')\n run_forward_checks(model)\n\ndef test_relative_attention_cat(pretrain_file):\n model = build_model(pretrain_file, '--no_use_lattn', '--use_rattn', '--rattn_heads', '10', '--rattn_cat')\n run_forward_checks(model)\n cat_size = model.word_input_size\n\n model = build_model(pretrain_file, '--no_use_lattn', '--use_rattn', '--rattn_heads', '10', '--no_rattn_cat')\n run_forward_checks(model)\n no_cat_size = model.word_input_size\n assert cat_size > no_cat_size\n\ndef test_relative_attention_directional(pretrain_file):\n model = build_model(pretrain_file, '--no_use_lattn', '--use_rattn', '--rattn_heads', '10', '--no_rattn_forward', '--no_rattn_cat')\n run_forward_checks(model)\n\n model = build_model(pretrain_file, '--no_use_lattn', '--use_rattn', '--rattn_heads', '10', '--no_rattn_reverse', '--no_rattn_cat')\n run_forward_checks(model)\n\ndef test_relative_attention_sinks(pretrain_file):\n model = build_model(pretrain_file, '--no_use_lattn', '--use_rattn', '--rattn_heads', '10', '--no_rattn_cat', '--rattn_window', '2', '--rattn_sinks', '1')\n run_forward_checks(model)\n model = build_model(pretrain_file, '--no_use_lattn', '--use_rattn', '--rattn_heads', '10', '--no_rattn_cat', '--rattn_sinks', '1')\n run_forward_checks(model)\n model = build_model(pretrain_file, '--no_use_lattn', '--use_rattn', '--rattn_heads', '10', '--no_rattn_cat', '--rattn_window', '2', '--rattn_sinks', '2')\n run_forward_checks(model)\n model = build_model(pretrain_file, '--no_use_lattn', '--use_rattn', '--rattn_heads', '10', '--no_rattn_cat', '--rattn_sinks', '2')\n run_forward_checks(model)\n\ndef test_relative_attention_cat_sinks(pretrain_file):\n model = build_model(pretrain_file, '--no_use_lattn', '--use_rattn', '--rattn_heads', '10', '--rattn_cat', '--rattn_window', '2', '--rattn_sinks', '1')\n run_forward_checks(model)\n model = build_model(pretrain_file, '--no_use_lattn', '--use_rattn', '--rattn_heads', '10', '--rattn_cat', '--rattn_sinks', '1')\n run_forward_checks(model)\n model = build_model(pretrain_file, '--no_use_lattn', '--use_rattn', '--rattn_heads', '10', '--rattn_cat', '--rattn_window', '2', '--rattn_sinks', '2')\n run_forward_checks(model)\n model = build_model(pretrain_file, '--no_use_lattn', '--use_rattn', '--rattn_heads', '10', '--rattn_cat', '--rattn_sinks', '2')\n run_forward_checks(model)\n\ndef test_relative_attention_endpoint_sinks(pretrain_file):\n model = build_model(pretrain_file, '--no_use_lattn', '--use_rattn', '--rattn_heads', '10', '--rattn_use_endpoint_sinks', '--rattn_window', '2', '--rattn_sinks', '1')\n run_forward_checks(model)\n model = build_model(pretrain_file, '--no_use_lattn', '--use_rattn', '--rattn_heads', '10', '--rattn_use_endpoint_sinks', '--rattn_sinks', '1')\n run_forward_checks(model)\n model = build_model(pretrain_file, '--no_use_lattn', '--use_rattn', '--rattn_heads', '10', '--rattn_use_endpoint_sinks', '--rattn_window', '2', '--rattn_sinks', '2')\n run_forward_checks(model)\n model = build_model(pretrain_file, '--no_use_lattn', '--use_rattn', '--rattn_heads', '10', '--rattn_use_endpoint_sinks', '--rattn_sinks', '2')\n run_forward_checks(model)\n\ndef test_lstm_tree_forward(pretrain_file):\n \"\"\"\n Test the LSTM_TREE forward pass\n \"\"\"\n model = build_model(pretrain_file, '--num_tree_lstm_layers', '1', '--constituency_composition', 'tree_lstm')\n run_forward_checks(model)\n model = build_model(pretrain_file, '--num_tree_lstm_layers', '2', '--constituency_composition', 'tree_lstm')\n run_forward_checks(model)\n model = build_model(pretrain_file, '--num_tree_lstm_layers', '3', '--constituency_composition', 'tree_lstm')\n run_forward_checks(model)\n\ndef test_lstm_tree_cx_forward(pretrain_file):\n \"\"\"\n Test the LSTM_TREE_CX forward pass\n \"\"\"\n model = build_model(pretrain_file, '--num_tree_lstm_layers', '1', '--constituency_composition', 'tree_lstm_cx')\n run_forward_checks(model)\n model = build_model(pretrain_file, '--num_tree_lstm_layers', '2', '--constituency_composition', 'tree_lstm_cx')\n run_forward_checks(model)\n model = build_model(pretrain_file, '--num_tree_lstm_layers', '3', '--constituency_composition', 'tree_lstm_cx')\n run_forward_checks(model)\n\ndef test_maxout(pretrain_file):\n \"\"\"\n Test with and without maxout layers for output\n \"\"\"\n model = build_model(pretrain_file, '--maxout_k', '0')\n run_forward_checks(model)\n # check the output size & implicitly check the type\n # to check for a particularly silly bug\n assert model.output_layers[-1].weight.shape[0] == len(model.transitions)\n\n model = build_model(pretrain_file, '--maxout_k', '2')\n run_forward_checks(model)\n assert model.output_layers[-1].linear.weight.shape[0] == len(model.transitions) * 2\n\n model = build_model(pretrain_file, '--maxout_k', '3')\n run_forward_checks(model)\n assert model.output_layers[-1].linear.weight.shape[0] == len(model.transitions) * 3\n\ndef check_structure_test(pretrain_file, args1, args2):\n \"\"\"\n Test that the \"copy\" method copies the parameters from one model to another\n\n Also check that the copied models produce the same results\n \"\"\"\n set_random_seed(1000)\n other = build_model(pretrain_file, *args1)\n other.eval()\n\n set_random_seed(1001)\n model = build_model(pretrain_file, *args2)\n model.eval()\n\n assert not torch.allclose(model.delta_embedding.weight, other.delta_embedding.weight)\n assert not torch.allclose(model.output_layers[0].weight, other.output_layers[0].weight)\n\n model.copy_with_new_structure(other)\n\n assert torch.allclose(model.delta_embedding.weight, other.delta_embedding.weight)\n assert torch.allclose(model.output_layers[0].weight, other.output_layers[0].weight)\n # the norms will be the same, as the non-zero values are all the same\n assert torch.allclose(torch.linalg.norm(model.word_lstm.weight_ih_l0), torch.linalg.norm(other.word_lstm.weight_ih_l0))\n\n # now, check that applying one transition to an initial state\n # results in the same values in the output states for both models\n # as the pattn layer inputs are 0, the output values should be equal\n shift = [parse_transitions.Shift()]\n model_states = test_parse_transitions.build_initial_state(model, 1)\n model_states = model.bulk_apply(model_states, shift)\n\n other_states = test_parse_transitions.build_initial_state(other, 1)\n other_states = other.bulk_apply(other_states, shift)\n\n for i, j in zip(other_states[0].word_queue, model_states[0].word_queue):\n assert torch.allclose(i.hx, j.hx, atol=1e-07)\n for i, j in zip(other_states[0].transitions, model_states[0].transitions):\n assert torch.allclose(i.lstm_hx, j.lstm_hx)\n assert torch.allclose(i.lstm_cx, j.lstm_cx)\n for i, j in zip(other_states[0].constituents, model_states[0].constituents):\n assert (i.value is None) == (j.value is None)\n if i.value is not None:\n assert torch.allclose(i.value.tree_hx, j.value.tree_hx, atol=1e-07)\n assert torch.allclose(i.lstm_hx, j.lstm_hx)\n assert torch.allclose(i.lstm_cx, j.lstm_cx)\n\ndef test_copy_with_new_structure_same(pretrain_file):\n \"\"\"\n Test that copying the structure with no changes works as expected\n \"\"\"\n check_structure_test(pretrain_file,\n ['--pattn_num_layers', '0', '--lattn_d_proj', '0', '--hidden_size', '20', '--delta_embedding_dim', '10'],\n ['--pattn_num_layers', '0', '--lattn_d_proj', '0', '--hidden_size', '20', '--delta_embedding_dim', '10'])\n\ndef test_copy_with_new_structure_untied(pretrain_file):\n \"\"\"\n Test that copying the structure with no changes works as expected\n \"\"\"\n check_structure_test(pretrain_file,\n ['--pattn_num_layers', '0', '--lattn_d_proj', '0', '--hidden_size', '20', '--delta_embedding_dim', '10', '--constituency_composition', 'MAX'],\n ['--pattn_num_layers', '0', '--lattn_d_proj', '0', '--hidden_size', '20', '--delta_embedding_dim', '10', '--constituency_composition', 'UNTIED_MAX'])\n\ndef test_copy_with_new_structure_pattn(pretrain_file):\n check_structure_test(pretrain_file,\n ['--pattn_num_layers', '0', '--lattn_d_proj', '0', '--hidden_size', '20', '--delta_embedding_dim', '10'],\n ['--pattn_num_layers', '1', '--lattn_d_proj', '0', '--hidden_size', '20', '--delta_embedding_dim', '10', '--pattn_d_model', '20', '--pattn_num_heads', '2'])\n\ndef test_copy_with_new_structure_both(pretrain_file):\n check_structure_test(pretrain_file,\n ['--pattn_num_layers', '0', '--lattn_d_proj', '0', '--hidden_size', '20', '--delta_embedding_dim', '10'],\n ['--pattn_num_layers', '1', '--lattn_d_proj', '32', '--hidden_size', '20', '--delta_embedding_dim', '10', '--pattn_d_model', '20', '--pattn_num_heads', '2'])\n\ndef test_copy_with_new_structure_lattn(pretrain_file):\n check_structure_test(pretrain_file,\n ['--pattn_num_layers', '1', '--lattn_d_proj', '0', '--hidden_size', '20', '--delta_embedding_dim', '10', '--pattn_d_model', '20', '--pattn_num_heads', '2'],\n ['--pattn_num_layers', '1', '--lattn_d_proj', '32', '--hidden_size', '20', '--delta_embedding_dim', '10', '--pattn_d_model', '20', '--pattn_num_heads', '2'])\n\ndef test_parse_tagged_words(pretrain_file):\n \"\"\"\n Small test which doesn't check results, just execution\n \"\"\"\n model = build_model(pretrain_file)\n\n sentence = [(\"I\", \"PRP\"), (\"am\", \"VBZ\"), (\"Luffa\", \"NNP\")]\n\n # we don't expect a useful tree out of a random model\n # so we don't check the result\n # just check that it works without crashing\n result = model.parse_tagged_words([sentence], 10)\n assert len(result) == 1\n pts = [x for x in result[0].yield_preterminals()]\n\n for word, pt in zip(sentence, pts):\n assert pt.children[0].label == word[0]\n assert pt.label == word[1]\n"
},
{
"path": "stanza/tests/constituency/test_parse_transitions.py",
"content": "import pytest\n\nfrom stanza.models.constituency import parse_transitions\nfrom stanza.models.constituency.base_model import SimpleModel, UNARY_LIMIT\nfrom stanza.models.constituency.parse_transitions import TransitionScheme, Shift, CloseConstituent, OpenConstituent\nfrom stanza.tests import *\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\n\ndef build_initial_state(model, num_states=1):\n words = [\"Unban\", \"Mox\", \"Opal\"]\n tags = [\"VB\", \"NNP\", \"NNP\"]\n sentences = [list(zip(words, tags)) for _ in range(num_states)]\n\n states = model.initial_state_from_words(sentences)\n assert len(states) == num_states\n assert all(state.num_transitions == 0 for state in states)\n return states\n\ndef test_initial_state(model=None):\n if model is None:\n model = SimpleModel()\n states = build_initial_state(model)\n assert len(states) == 1\n state = states[0]\n\n assert state.sentence_length == 3\n assert state.num_opens == 0\n # each stack has a sentinel value at the end\n assert len(state.word_queue) == 5\n assert len(state.constituents) == 1\n assert len(state.transitions) == 1\n assert state.word_position == 0\n\ndef test_shift(model=None):\n if model is None:\n model = SimpleModel()\n state = build_initial_state(model)[0]\n\n open_transition = parse_transitions.OpenConstituent(\"ROOT\")\n state = open_transition.apply(state, model)\n open_transition = parse_transitions.OpenConstituent(\"S\")\n state = open_transition.apply(state, model)\n shift = parse_transitions.Shift()\n assert shift.is_legal(state, model)\n assert len(state.word_queue) == 5\n assert state.word_position == 0\n\n state = shift.apply(state, model)\n assert len(state.word_queue) == 5\n # 4 because of the dummy created by the opens\n assert len(state.constituents) == 4\n assert len(state.transitions) == 4\n assert shift.is_legal(state, model)\n assert state.word_position == 1\n assert not state.empty_word_queue()\n\n state = shift.apply(state, model)\n assert len(state.word_queue) == 5\n assert len(state.constituents) == 5\n assert len(state.transitions) == 5\n assert shift.is_legal(state, model)\n assert state.word_position == 2\n assert not state.empty_word_queue()\n\n state = shift.apply(state, model)\n assert len(state.word_queue) == 5\n assert len(state.constituents) == 6\n assert len(state.transitions) == 6\n assert not shift.is_legal(state, model)\n assert state.word_position == 3\n assert state.empty_word_queue()\n\n constituents = state.constituents\n assert model.get_top_constituent(constituents).children[0].label == 'Opal'\n constituents = constituents.pop()\n assert model.get_top_constituent(constituents).children[0].label == 'Mox'\n constituents = constituents.pop()\n assert model.get_top_constituent(constituents).children[0].label == 'Unban'\n\ndef test_initial_unary(model=None):\n # it doesn't make sense to start with a CompoundUnary\n if model is None:\n model = SimpleModel()\n\n state = build_initial_state(model)[0]\n unary = parse_transitions.CompoundUnary('ROOT', 'VP')\n assert unary.label == ('ROOT', 'VP',)\n assert not unary.is_legal(state, model)\n unary = parse_transitions.CompoundUnary('VP')\n assert unary.label == ('VP',)\n assert not unary.is_legal(state, model)\n\n\ndef test_unary(model=None):\n if model is None:\n model = SimpleModel()\n state = build_initial_state(model)[0]\n\n shift = parse_transitions.Shift()\n state = shift.apply(state, model)\n\n # this is technically the wrong parse but we're being lazy\n unary = parse_transitions.CompoundUnary('S', 'VP')\n assert unary.is_legal(state, model)\n state = unary.apply(state, model)\n assert not unary.is_legal(state, model)\n\n tree = model.get_top_constituent(state.constituents)\n assert tree.label == 'S'\n assert len(tree.children) == 1\n tree = tree.children[0]\n assert tree.label == 'VP'\n assert len(tree.children) == 1\n tree = tree.children[0]\n assert tree.label == 'VB'\n assert tree.is_preterminal()\n\ndef test_unary_requires_root(model=None):\n if model is None:\n model = SimpleModel(transition_scheme=TransitionScheme.TOP_DOWN_UNARY)\n state = build_initial_state(model)[0]\n\n open_transition = parse_transitions.OpenConstituent(\"S\")\n assert open_transition.is_legal(state, model)\n state = open_transition.apply(state, model)\n\n shift = parse_transitions.Shift()\n assert shift.is_legal(state, model)\n state = shift.apply(state, model)\n assert shift.is_legal(state, model)\n state = shift.apply(state, model)\n assert shift.is_legal(state, model)\n state = shift.apply(state, model)\n assert not shift.is_legal(state, model)\n\n close_transition = parse_transitions.CloseConstituent()\n assert close_transition.is_legal(state, model)\n state = close_transition.apply(state, model)\n assert not open_transition.is_legal(state, model)\n assert not close_transition.is_legal(state, model)\n\n np_unary = parse_transitions.CompoundUnary(\"NP\")\n assert not np_unary.is_legal(state, model)\n root_unary = parse_transitions.CompoundUnary(\"ROOT\")\n assert root_unary.is_legal(state, model)\n assert not state.finished(model)\n state = root_unary.apply(state, model)\n assert not root_unary.is_legal(state, model)\n\n assert state.finished(model)\n\ndef test_open(model=None):\n if model is None:\n model = SimpleModel()\n state = build_initial_state(model)[0]\n\n shift = parse_transitions.Shift()\n state = shift.apply(state, model)\n state = shift.apply(state, model)\n assert state.num_opens == 0\n\n open_transition = parse_transitions.OpenConstituent(\"VP\")\n assert open_transition.is_legal(state, model)\n state = open_transition.apply(state, model)\n assert open_transition.is_legal(state, model)\n assert state.num_opens == 1\n\n # check that it is illegal if there are too many opens already\n for i in range(20):\n state = open_transition.apply(state, model)\n assert not open_transition.is_legal(state, model)\n assert state.num_opens == 21\n\n # check that it is illegal if the state is out of words\n state = build_initial_state(model)[0]\n state = shift.apply(state, model)\n state = shift.apply(state, model)\n state = shift.apply(state, model)\n assert not open_transition.is_legal(state, model)\n\ndef test_compound_open(model=None):\n if model is None:\n model = SimpleModel()\n state = build_initial_state(model)[0]\n\n open_transition = parse_transitions.OpenConstituent(\"ROOT\", \"S\")\n assert open_transition.is_legal(state, model)\n shift = parse_transitions.Shift()\n close_transition = parse_transitions.CloseConstituent()\n\n state = open_transition.apply(state, model)\n state = shift.apply(state, model)\n state = shift.apply(state, model)\n state = shift.apply(state, model)\n state = close_transition.apply(state, model)\n\n tree = model.get_top_constituent(state.constituents)\n assert tree.label == 'ROOT'\n assert len(tree.children) == 1\n tree = tree.children[0]\n assert tree.label == 'S'\n assert len(tree.children) == 3\n assert tree.children[0].children[0].label == 'Unban'\n assert tree.children[1].children[0].label == 'Mox'\n assert tree.children[2].children[0].label == 'Opal'\n\ndef test_in_order_open(model=None):\n if model is None:\n model = SimpleModel(TransitionScheme.IN_ORDER)\n state = build_initial_state(model)[0]\n\n shift = parse_transitions.Shift()\n assert shift.is_legal(state, model)\n state = shift.apply(state, model)\n assert not shift.is_legal(state, model)\n\n open_vp = parse_transitions.OpenConstituent(\"VP\")\n assert open_vp.is_legal(state, model)\n state = open_vp.apply(state, model)\n assert not open_vp.is_legal(state, model)\n\n close_trans = parse_transitions.CloseConstituent()\n assert close_trans.is_legal(state, model)\n state = close_trans.apply(state, model)\n\n open_s = parse_transitions.OpenConstituent(\"S\")\n assert open_s.is_legal(state, model)\n state = open_s.apply(state, model)\n assert not open_vp.is_legal(state, model)\n\n # check that root transitions won't happen in the middle of a parse\n open_root = parse_transitions.OpenConstituent(\"ROOT\")\n assert not open_root.is_legal(state, model)\n\n # build (NP (NNP Mox) (NNP Opal))\n open_np = parse_transitions.OpenConstituent(\"NP\")\n assert shift.is_legal(state, model)\n state = shift.apply(state, model)\n assert open_np.is_legal(state, model)\n # make sure root can't happen in places where an arbitrary open is legal\n assert not open_root.is_legal(state, model)\n state = open_np.apply(state, model)\n assert shift.is_legal(state, model)\n state = shift.apply(state, model)\n assert close_trans.is_legal(state, model)\n state = close_trans.apply(state, model)\n\n assert close_trans.is_legal(state, model)\n state = close_trans.apply(state, model)\n\n assert open_root.is_legal(state, model)\n state = open_root.apply(state, model)\n\ndef test_too_many_unaries_close():\n \"\"\"\n This tests rejecting Close at the start of a sequence after too many unary transitions\n\n The model should reject doing multiple \"unaries\" - eg, Open then Close - in an IN_ORDER sequence\n \"\"\"\n model = SimpleModel(TransitionScheme.IN_ORDER)\n state = build_initial_state(model)[0]\n\n shift = parse_transitions.Shift()\n assert shift.is_legal(state, model)\n state = shift.apply(state, model)\n\n open_np = parse_transitions.OpenConstituent(\"NP\")\n close_trans = parse_transitions.CloseConstituent()\n for _ in range(UNARY_LIMIT):\n assert open_np.is_legal(state, model)\n state = open_np.apply(state, model)\n\n assert close_trans.is_legal(state, model)\n state = close_trans.apply(state, model)\n\n assert open_np.is_legal(state, model)\n state = open_np.apply(state, model)\n assert not close_trans.is_legal(state, model)\n\ndef test_too_many_unaries_open():\n \"\"\"\n This tests rejecting Open in the middle of a sequence after too many unary transitions\n\n The model should reject doing multiple \"unaries\" - eg, Open then Close - in an IN_ORDER sequence\n \"\"\"\n model = SimpleModel(TransitionScheme.IN_ORDER)\n state = build_initial_state(model)[0]\n\n shift = parse_transitions.Shift()\n assert shift.is_legal(state, model)\n state = shift.apply(state, model)\n\n open_np = parse_transitions.OpenConstituent(\"NP\")\n close_trans = parse_transitions.CloseConstituent()\n\n assert open_np.is_legal(state, model)\n state = open_np.apply(state, model)\n assert not open_np.is_legal(state, model)\n assert shift.is_legal(state, model)\n state = shift.apply(state, model)\n\n for _ in range(UNARY_LIMIT):\n assert open_np.is_legal(state, model)\n state = open_np.apply(state, model)\n\n assert close_trans.is_legal(state, model)\n state = close_trans.apply(state, model)\n\n assert not open_np.is_legal(state, model)\n\ndef test_close(model=None):\n if model is None:\n model = SimpleModel()\n\n # this one actually tests an entire subtree building\n state = build_initial_state(model)[0]\n\n open_transition_vp = parse_transitions.OpenConstituent(\"VP\")\n assert open_transition_vp.is_legal(state, model)\n state = open_transition_vp.apply(state, model)\n assert state.num_opens == 1\n\n shift = parse_transitions.Shift()\n assert shift.is_legal(state, model)\n state = shift.apply(state, model)\n\n open_transition_np = parse_transitions.OpenConstituent(\"NP\")\n assert open_transition_np.is_legal(state, model)\n state = open_transition_np.apply(state, model)\n assert state.num_opens == 2\n\n assert shift.is_legal(state, model)\n state = shift.apply(state, model)\n assert shift.is_legal(state, model)\n state = shift.apply(state, model)\n assert not shift.is_legal(state, model)\n assert state.num_opens == 2\n # now should have \"mox\", \"opal\" on the constituents\n\n close_transition = parse_transitions.CloseConstituent()\n assert close_transition.is_legal(state, model)\n state = close_transition.apply(state, model)\n assert state.num_opens == 1\n assert close_transition.is_legal(state, model)\n state = close_transition.apply(state, model)\n assert state.num_opens == 0\n assert not close_transition.is_legal(state, model)\n\n tree = model.get_top_constituent(state.constituents)\n assert tree.label == 'VP'\n assert len(tree.children) == 2\n tree = tree.children[1]\n assert tree.label == 'NP'\n assert len(tree.children) == 2\n assert tree.children[0].is_preterminal()\n assert tree.children[1].is_preterminal()\n assert tree.children[0].children[0].label == 'Mox'\n assert tree.children[1].children[0].label == 'Opal'\n\n # extra one for None at the start of the TreeStack\n assert len(state.constituents) == 2\n\n assert state.all_transitions(model) == [open_transition_vp, shift, open_transition_np, shift, shift, close_transition, close_transition]\n\ndef test_in_order_compound_finalize(model=None):\n \"\"\"\n Test the Finalize transition is only legal at the end of a sequence\n \"\"\"\n if model is None:\n model = SimpleModel(transition_scheme=TransitionScheme.IN_ORDER_COMPOUND)\n\n state = build_initial_state(model)[0]\n\n finalize = parse_transitions.Finalize(\"ROOT\")\n\n shift = parse_transitions.Shift()\n assert shift.is_legal(state, model)\n assert not finalize.is_legal(state, model)\n state = shift.apply(state, model)\n\n open_transition = parse_transitions.OpenConstituent(\"NP\")\n assert open_transition.is_legal(state, model)\n assert not finalize.is_legal(state, model)\n state = open_transition.apply(state, model)\n assert state.num_opens == 1\n\n assert shift.is_legal(state, model)\n assert not finalize.is_legal(state, model)\n state = shift.apply(state, model)\n assert shift.is_legal(state, model)\n assert not finalize.is_legal(state, model)\n state = shift.apply(state, model)\n\n close_transition = parse_transitions.CloseConstituent()\n assert close_transition.is_legal(state, model)\n state = close_transition.apply(state, model)\n assert state.num_opens == 0\n assert not close_transition.is_legal(state, model)\n assert finalize.is_legal(state, model)\n\n state = finalize.apply(state, model)\n assert not finalize.is_legal(state, model)\n tree = model.get_top_constituent(state.constituents)\n assert tree.label == 'ROOT'\n\ndef test_hashes():\n transitions = set()\n\n shift = parse_transitions.Shift()\n assert shift not in transitions\n transitions.add(shift)\n assert shift in transitions\n shift = parse_transitions.Shift()\n assert shift in transitions\n\n for i in range(5):\n transitions.add(shift)\n assert len(transitions) == 1\n\n unary = parse_transitions.CompoundUnary(\"asdf\")\n assert unary not in transitions\n transitions.add(unary)\n assert unary in transitions\n\n unary = parse_transitions.CompoundUnary(\"asdf\", \"zzzz\")\n assert unary not in transitions\n transitions.add(unary)\n transitions.add(unary)\n transitions.add(unary)\n unary = parse_transitions.CompoundUnary(\"asdf\", \"zzzz\")\n assert unary in transitions\n\n oc = parse_transitions.OpenConstituent(\"asdf\")\n assert oc not in transitions\n transitions.add(oc)\n assert oc in transitions\n transitions.add(oc)\n transitions.add(oc)\n assert len(transitions) == 4\n assert parse_transitions.OpenConstituent(\"asdf\") in transitions\n\n cc = parse_transitions.CloseConstituent()\n assert cc not in transitions\n transitions.add(cc)\n transitions.add(cc)\n transitions.add(cc)\n assert cc in transitions\n cc = parse_transitions.CloseConstituent()\n assert cc in transitions\n assert len(transitions) == 5\n\n\ndef test_sort():\n expected = []\n\n expected.append(parse_transitions.Shift())\n expected.append(parse_transitions.CloseConstituent())\n expected.append(parse_transitions.CompoundUnary(\"NP\"))\n expected.append(parse_transitions.CompoundUnary(\"NP\", \"VP\"))\n expected.append(parse_transitions.OpenConstituent(\"mox\"))\n expected.append(parse_transitions.OpenConstituent(\"opal\"))\n expected.append(parse_transitions.OpenConstituent(\"unban\"))\n\n transitions = set(expected)\n transitions = sorted(transitions)\n assert transitions == expected\n\ndef test_check_transitions():\n \"\"\"\n Test that check_transitions passes or fails a couple simple, small test cases\n \"\"\"\n transitions = {Shift(), CloseConstituent(), OpenConstituent(\"NP\"), OpenConstituent(\"VP\")}\n\n other = {Shift(), CloseConstituent(), OpenConstituent(\"NP\"), OpenConstituent(\"VP\")}\n parse_transitions.check_transitions(transitions, other, \"test\")\n\n # This will get a pass because it is a unary made out of existing unaries\n other = {Shift(), CloseConstituent(), OpenConstituent(\"NP\", \"VP\")}\n parse_transitions.check_transitions(transitions, other, \"test\")\n\n # This should fail\n with pytest.raises(RuntimeError):\n other = {Shift(), CloseConstituent(), OpenConstituent(\"NP\", \"ZP\")}\n parse_transitions.check_transitions(transitions, other, \"test\")\n"
},
{
"path": "stanza/tests/constituency/test_parse_tree.py",
"content": "import pytest\n\nfrom stanza.models.constituency.parse_tree import Tree\nfrom stanza.models.constituency import tree_reader\n\nfrom stanza.tests import *\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\ndef test_leaf_preterminal():\n foo = Tree(label=\"foo\")\n assert foo.is_leaf()\n assert not foo.is_preterminal()\n assert len(foo.children) == 0\n assert str(foo) == 'foo'\n\n bar = Tree(label=\"bar\", children=foo)\n assert not bar.is_leaf()\n assert bar.is_preterminal()\n assert len(bar.children) == 1\n assert str(bar) == \"(bar foo)\"\n\n baz = Tree(label=\"baz\", children=[bar])\n assert not baz.is_leaf()\n assert not baz.is_preterminal()\n assert len(baz.children) == 1\n assert str(baz) == \"(baz (bar foo))\"\n\n\ndef test_yield_preterminals():\n text = \"((S (VP (VB Unban)) (NP (NNP Mox) (NNP Opal))))\"\n trees = tree_reader.read_trees(text)\n\n preterminals = list(trees[0].yield_preterminals())\n assert len(preterminals) == 3\n assert str(preterminals) == \"[(VB Unban), (NNP Mox), (NNP Opal)]\"\n\ndef test_depth():\n text = \"(foo) ((S (VP (VB Unban)) (NP (NNP Mox) (NNP Opal))))\"\n trees = tree_reader.read_trees(text)\n assert trees[0].depth() == 0\n assert trees[1].depth() == 4\n\ndef test_unique_labels():\n \"\"\"\n Test getting the unique labels from a tree\n\n Assumes tree_reader works, which should be fine since it is tested elsewhere\n \"\"\"\n text=\"((SBARQ (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (IN in) (NP (DT this) (NN seat))))) (. ?))) ((SBARQ (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (IN in) (NP (DT this) (NN seat))))) (. ?)))\"\n\n trees = tree_reader.read_trees(text)\n\n labels = Tree.get_unique_constituent_labels(trees)\n expected = ['NP', 'PP', 'ROOT', 'SBARQ', 'SQ', 'VP', 'WHNP']\n assert labels == expected\n\ndef test_unique_tags():\n \"\"\"\n Test getting the unique tags from a tree\n \"\"\"\n text=\"((SBARQ (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (IN in) (NP (DT this) (NN seat))))) (. ?)))\"\n\n trees = tree_reader.read_trees(text)\n\n tags = Tree.get_unique_tags(trees)\n expected = ['.', 'DT', 'IN', 'NN', 'VBZ', 'WP']\n assert tags == expected\n\n\ndef test_unique_words():\n \"\"\"\n Test getting the unique words from a tree\n \"\"\"\n text=\"((SBARQ (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (IN in) (NP (DT this) (NN seat))))) (. ?)))\"\n\n trees = tree_reader.read_trees(text)\n\n words = Tree.get_unique_words(trees)\n expected = ['?', 'Who', 'in', 'seat', 'sits', 'this']\n assert words == expected\n\ndef test_rare_words():\n \"\"\"\n Test getting the unique words from a tree\n \"\"\"\n text=\"((SBARQ (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (IN in) (NP (DT this) (NN seat))))) (. ?))) ((SBARQ (NP (DT this) (NN seat)) (. ?)))\"\n\n trees = tree_reader.read_trees(text)\n\n words = Tree.get_rare_words(trees, 0.5)\n expected = ['Who', 'in', 'sits']\n assert words == expected\n\ndef test_common_words():\n \"\"\"\n Test getting the unique words from a tree\n \"\"\"\n text=\"((SBARQ (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (IN in) (NP (DT this) (NN seat))))) (. ?))) ((SBARQ (NP (DT this) (NN seat)) (. ?)))\"\n\n trees = tree_reader.read_trees(text)\n\n words = Tree.get_common_words(trees, 3)\n expected = ['?', 'seat', 'this']\n assert words == expected\n\ndef test_root_labels():\n text=\"( (SBARQ-FOO (WHNP-BAR (WP Who)) (SQ#ASDF (VP=1 (VBZ sits) (PP (IN in) (NP (DT this) (NN seat))))) (. ?)))\"\n trees = tree_reader.read_trees(text)\n assert [\"ROOT\"] == Tree.get_root_labels(trees)\n\n text=(\"( (SBARQ-FOO (WHNP-BAR (WP Who)) (SQ#ASDF (VP=1 (VBZ sits) (PP (IN in) (NP (DT this) (NN seat))))) (. ?)))\" +\n \"( (SBARQ-FOO (WHNP-BAR (WP Who)) (SQ#ASDF (VP=1 (VBZ sits) (PP (IN in) (NP (DT this) (NN seat))))) (. ?)))\" +\n \"( (SBARQ-FOO (WHNP-BAR (WP Who)) (SQ#ASDF (VP=1 (VBZ sits) (PP (IN in) (NP (DT this) (NN seat))))) (. ?)))\")\n trees = tree_reader.read_trees(text)\n assert [\"ROOT\"] == Tree.get_root_labels(trees)\n\n text=\"(FOO) (BAR)\"\n trees = tree_reader.read_trees(text)\n assert [\"BAR\", \"FOO\"] == Tree.get_root_labels(trees)\n\ndef test_prune_none():\n text=[\"((SBARQ (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (-NONE- in) (NP (DT this) (NN seat))))) (. ?)))\", # test one dead node\n \"((SBARQ (WHNP (-NONE- Who)) (SQ (VP (VBZ sits) (PP (IN in) (NP (DT this) (NN seat))))) (. ?)))\", # test recursive dead nodes\n \"((SBARQ (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (IN in) (NP (-NONE- this) (-NONE- seat))))) (. ?)))\"] # test all children dead\n expected=[\"(ROOT (SBARQ (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (NP (DT this) (NN seat))))) (. ?)))\",\n \"(ROOT (SBARQ (SQ (VP (VBZ sits) (PP (IN in) (NP (DT this) (NN seat))))) (. ?)))\",\n \"(ROOT (SBARQ (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (IN in)))) (. ?)))\"]\n\n for t, e in zip(text, expected):\n trees = tree_reader.read_trees(t)\n assert len(trees) == 1\n tree = trees[0].prune_none()\n assert e == str(tree)\n\ndef test_simplify_labels():\n text=\"( (SBARQ-FOO (WHNP-BAR (WP Who)) (SQ#ASDF (VP=1 (VBZ sits) (PP (IN in) (NP (DT this) (- -))))) (. ?)))\"\n expected = \"(ROOT (SBARQ (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (IN in) (NP (DT this) (- -))))) (. ?)))\"\n trees = tree_reader.read_trees(text)\n trees = [t.simplify_labels() for t in trees]\n assert len(trees) == 1\n assert expected == str(trees[0])\n\ndef test_remap_constituent_labels():\n text=\"(ROOT (SBARQ (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (IN in)))) (. ?)))\"\n expected=\"(ROOT (FOO (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (IN in)))) (. ?)))\"\n\n label_map = { \"SBARQ\": \"FOO\" }\n trees = tree_reader.read_trees(text)\n trees = [t.remap_constituent_labels(label_map) for t in trees]\n assert len(trees) == 1\n assert expected == str(trees[0])\n\ndef test_remap_constituent_words():\n text=\"(ROOT (SBARQ (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (IN in)))) (. ?)))\"\n expected=\"(ROOT (SBARQ (WHNP (WP unban)) (SQ (VP (VBZ mox) (PP (IN opal)))) (. ?)))\"\n\n word_map = { \"Who\": \"unban\", \"sits\": \"mox\", \"in\": \"opal\" }\n trees = tree_reader.read_trees(text)\n trees = [t.remap_words(word_map) for t in trees]\n assert len(trees) == 1\n assert expected == str(trees[0])\n\ndef test_replace_words():\n text=\"(ROOT (SBARQ (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (IN in)))) (. ?)))\"\n expected=\"(ROOT (SBARQ (WHNP (WP unban)) (SQ (VP (VBZ mox) (PP (IN opal)))) (. ?)))\"\n new_words = [\"unban\", \"mox\", \"opal\", \"?\"]\n\n trees = tree_reader.read_trees(text)\n assert len(trees) == 1\n tree = trees[0]\n new_tree = tree.replace_words(new_words)\n assert expected == str(new_tree)\n\n\ndef test_compound_constituents():\n # TODO: add skinny trees like this to the various transition tests\n text=\"((VP (VB Unban)))\"\n trees = tree_reader.read_trees(text)\n assert Tree.get_compound_constituents(trees) == [('ROOT', 'VP')]\n\n text=\"(ROOT (SBARQ (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (IN in)))) (. ?)))\"\n trees = tree_reader.read_trees(text)\n assert Tree.get_compound_constituents(trees) == [('PP',), ('ROOT', 'SBARQ'), ('SQ', 'VP'), ('WHNP',)]\n\n text=\"((VP (VB Unban))) (ROOT (SBARQ (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (IN in)))) (. ?)))\"\n trees = tree_reader.read_trees(text)\n assert Tree.get_compound_constituents(trees) == [('PP',), ('ROOT', 'SBARQ'), ('ROOT', 'VP'), ('SQ', 'VP'), ('WHNP',)]\n\ndef test_equals():\n \"\"\"\n Check one tree from the actual dataset for ==\n\n when built with compound Open, this didn't work because of a silly bug\n \"\"\"\n text = \"(ROOT (S (NP (DT The) (NNP Arizona) (NNPS Corporations) (NNP Commission)) (VP (VBD authorized) (NP (NP (DT an) (ADJP (CD 11.5)) (NN %) (NN rate) (NN increase)) (PP (IN at) (NP (NNP Tucson) (NNP Electric) (NNP Power) (NNP Co.))) (, ,) (UCP (ADJP (ADJP (RB substantially) (JJR lower)) (SBAR (IN than) (S (VP (VBN recommended) (NP (JJ last) (NN month)) (PP (IN by) (NP (DT a) (NN commission) (NN hearing) (NN officer))))))) (CC and) (NP (NP (QP (RB barely) (PDT half)) (DT the) (NN rise)) (VP (VBN sought) (PP (IN by) (NP (DT the) (NN utility)))))))) (. .)))\"\n\n trees = tree_reader.read_trees(text)\n assert len(trees) == 1\n tree = trees[0]\n\n assert tree == tree\n\n trees2 = tree_reader.read_trees(text)\n tree2 = trees2[0]\n\n assert tree is not tree2\n assert tree == tree2\n\n\n# This tree was causing the model to barf on CTB7,\n# although it turns out the problem was just the\n# depth of the unary, not the list\nCHINESE_LONG_LIST_TREE = \"\"\"\n(ROOT\n (IP\n (NP (NNP 证券法))\n (VP\n (PP\n (IN 对)\n (NP\n (DNP\n (NP\n (NP (NNP 中国))\n (NP\n (NN 证券)\n (NN 市场)))\n (DEC 的))\n (NP (NN 运作))))\n (, ，)\n (PP\n (PP\n (IN 从)\n (NP\n (NP (NN 股票))\n (NP (VV 发行) (EC 、) (VV 交易))))\n (, ，)\n (PP\n (VV 到)\n (NP\n (NP (NN 上市) (NN 公司) (NN 收购))\n (EC 、)\n (NP (NN 证券) (NN 交易所))\n (EC 、)\n (NP (NN 证券) (NN 公司))\n (EC 、)\n (NP (NN 登记) (NN 结算) (NN 机构))\n (EC 、)\n (NP (NN 交易) (NN 服务) (NN 机构))\n (EC 、)\n (NP (NN 证券业) (NN 协会))\n (EC 、)\n (NP (NN 证券) (NN 监督) (NN 管理) (NN 机构))\n (CC 和)\n (NP\n (DNP\n (NP (CP (CP (IP (VP (VV 违法))))))\n (DEC 的))\n (NP (NN 法律) (NN 责任))))))\n (ADVP (RB 都))\n (VP\n (VV 作)\n (AS 了)\n (NP\n (ADJP (JJ 详细))\n (NP (NN 规定)))))\n (. 。)))\n\"\"\"\n\nWEIRD_UNARY = \"\"\"\n (DNP\n (NP (CP (CP (IP (VP (ASDF\n (NP (NN 上市) (NN 公司) (NN 收购))\n (EC 、)\n (NP (NN 证券) (NN 交易所))\n (EC 、)\n (NP (NN 证券) (NN 公司))\n (EC 、)\n (NP (NN 登记) (NN 结算) (NN 机构))\n (EC 、)\n (NP (NN 交易) (NN 服务) (NN 机构))\n (EC 、)\n (NP (NN 证券业) (NN 协会))\n (EC 、)\n (NP (NN 证券) (NN 监督) (NN 管理) (NN 机构))))))))\n (DEC 的))\n\"\"\"\n\n\ndef test_count_unaries():\n trees = tree_reader.read_trees(CHINESE_LONG_LIST_TREE)\n assert len(trees) == 1\n assert trees[0].count_unary_depth() == 5\n\n trees = tree_reader.read_trees(WEIRD_UNARY)\n assert len(trees) == 1\n assert trees[0].count_unary_depth() == 5\n\ndef test_str_bracket_labels():\n text = \"((S (VP (VB Unban)) (NP (NNP Mox) (NNP Opal))))\"\n expected = \"(_ROOT (_S (_VP (_VB Unban )_VB )_VP (_NP (_NNP Mox )_NNP (_NNP Opal )_NNP )_NP )_S )_ROOT\"\n\n trees = tree_reader.read_trees(text)\n assert len(trees) == 1\n assert \"{:L}\".format(trees[0]) == expected\n\ndef test_all_leaves_are_preterminals():\n text = \"((S (VP (VB Unban)) (NP (NNP Mox) (NNP Opal))))\"\n trees = tree_reader.read_trees(text)\n assert len(trees) == 1\n assert trees[0].all_leaves_are_preterminals()\n\n text = \"((S (VP (VB Unban)) (NP (Mox) (NNP Opal))))\"\n trees = tree_reader.read_trees(text)\n assert len(trees) == 1\n assert not trees[0].all_leaves_are_preterminals()\n\ndef test_latex():\n \"\"\"\n Test the latex format for trees\n \"\"\"\n expected = \"\\\\Tree [.S [.NP Jennifer ] [.VP has [.NP nice antennae ] ] ]\"\n tree = \"(ROOT (S (NP (NNP Jennifer)) (VP (VBZ has) (NP (JJ nice) (NNS antennae)))))\"\n tree = tree_reader.read_trees(tree)[0]\n text = \"{:T}\".format(tree)\n assert text == expected\n\ndef test_pretty_print():\n \"\"\"\n Pretty print a couple trees - newlines & indentation\n \"\"\"\n text = \"(ROOT (S (VP (VB Unban)) (NP (NNP Mox) (NNP Opal)))) (ROOT (S (NP (DT The) (NNP Arizona) (NNPS Corporations) (NNP Commission)) (VP (VBD authorized) (NP (NP (DT an) (ADJP (CD 11.5)) (NN %) (NN rate) (NN increase)) (PP (IN at) (NP (NNP Tucson) (NNP Electric)))))))\"\n trees = tree_reader.read_trees(text)\n assert len(trees) == 2\n\n expected = \"\"\"(ROOT\n (S\n (VP (VB Unban))\n (NP (NNP Mox) (NNP Opal))))\n\"\"\"\n\n assert \"{:P}\".format(trees[0]) == expected\n\n expected = \"\"\"(ROOT\n (S\n (NP (DT The) (NNP Arizona) (NNPS Corporations) (NNP Commission))\n (VP\n (VBD authorized)\n (NP\n (NP\n (DT an)\n (ADJP (CD 11.5))\n (NN %)\n (NN rate)\n (NN increase))\n (PP\n (IN at)\n (NP (NNP Tucson) (NNP Electric)))))))\n\"\"\"\n assert \"{:P}\".format(trees[1]) == expected\n\n assert text == \"{:O} {:O}\".format(*trees)\n\ndef test_reverse():\n text = \"(ROOT (S (NP (PRP I)) (VP (VBP want) (S (VP (TO to) (VP (VB lick) (NP (NP (NNP Jennifer) (POS 's)) (NNS antennae))))))))\"\n trees = tree_reader.read_trees(text)\n assert len(trees) == 1\n reversed_tree = trees[0].reverse()\n assert str(reversed_tree) == \"(ROOT (S (VP (S (VP (VP (NP (NNS antennae) (NP (POS 's) (NNP Jennifer))) (VB lick)) (TO to))) (VBP want)) (NP (PRP I))))\"\n"
},
{
"path": "stanza/tests/constituency/test_positional_encoding.py",
"content": "import pytest\n\nimport torch\n\nfrom stanza import Pipeline\nfrom stanza.models.constituency.positional_encoding import SinusoidalEncoding, AddSinusoidalEncoding\n\nfrom stanza.tests import *\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\n\ndef test_positional_encoding():\n encoding = SinusoidalEncoding(model_dim=10, max_len=6)\n foo = encoding(torch.tensor([5]))\n assert foo.shape == (1, 10)\n # TODO: check the values\n\ndef test_resize():\n encoding = SinusoidalEncoding(model_dim=10, max_len=3)\n foo = encoding(torch.tensor([5]))\n assert foo.shape == (1, 10)\n\n\ndef test_arange():\n encoding = SinusoidalEncoding(model_dim=10, max_len=2)\n foo = encoding(torch.arange(4))\n assert foo.shape == (4, 10)\n assert encoding.max_len() == 4\n\ndef test_add():\n encoding = AddSinusoidalEncoding(d_model=10, max_len=4)\n x = torch.zeros(1, 4, 10)\n y = encoding(x)\n\n r = torch.randn(1, 4, 10)\n r2 = encoding(r)\n\n assert torch.allclose(r2 - r, y, atol=1e-07)\n\n r = torch.randn(2, 4, 10)\n r2 = encoding(r)\n\n assert torch.allclose(r2[0] - r[0], y, atol=1e-07)\n assert torch.allclose(r2[1] - r[1], y, atol=1e-07)\n"
},
{
"path": "stanza/tests/constituency/test_selftrain_vi_quad.py",
"content": "\"\"\"\nTest some of the methods in the vi_quad dataset\n\nUses a small section of the dataset as a test\n\"\"\"\n\nimport pytest\n\nfrom stanza.utils.datasets.constituency import selftrain_vi_quad\n\nfrom stanza.tests import *\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\nSAMPLE_TEXT = \"\"\"\n{\"version\": \"1.1\", \"data\": [{\"title\": \"Ph\\u1ea1m V\\u0103n \\u0110\\u1ed3ng\", \"paragraphs\": [{\"qas\": [{\"question\": \"T\\u00ean g\\u1ecdi n\\u00e0o \\u0111\\u01b0\\u1ee3c Ph\\u1ea1m V\\u0103n \\u0110\\u1ed3ng s\\u1eed d\\u1ee5ng khi l\\u00e0m Ph\\u00f3 ch\\u1ee7 nhi\\u1ec7m c\\u01a1 quan Bi\\u1ec7n s\\u1ef1 x\\u1ee9 t\\u1ea1i Qu\\u1ebf L\\u00e2m?\", \"answers\": [{\"answer_start\": 507, \"text\": \"L\\u00e2m B\\u00e1 Ki\\u1ec7t\"}], \"id\": \"uit_01__05272_0_1\"}, {\"question\": \"Ph\\u1ea1m V\\u0103n \\u0110\\u1ed3ng gi\\u1eef ch\\u1ee9c v\\u1ee5 g\\u00ec trong b\\u1ed9 m\\u00e1y Nh\\u00e0 n\\u01b0\\u1edbc C\\u1ed9ng h\\u00f2a X\\u00e3 h\\u1ed9i ch\\u1ee7 ngh\\u0129a Vi\\u1ec7t Nam?\", \"answers\": [{\"answer_start\": 60, \"text\": \"Th\\u1ee7 t\\u01b0\\u1edbng\"}], \"id\": \"uit_01__05272_0_2\"}, {\"question\": \"Giai \\u0111o\\u1ea1n n\\u0103m 1955-1976, Ph\\u1ea1m V\\u0103n \\u0110\\u1ed3ng n\\u1eafm gi\\u1eef ch\\u1ee9c v\\u1ee5 g\\u00ec?\", \"answers\": [{\"answer_start\": 245, \"text\": \"Th\\u1ee7 t\\u01b0\\u1edbng Ch\\u00ednh ph\\u1ee7 Vi\\u1ec7t Nam D\\u00e2n ch\\u1ee7 C\\u1ed9ng h\\u00f2a\"}], \"id\": \"uit_01__05272_0_3\"}], \"context\": \"Ph\\u1ea1m V\\u0103n \\u0110\\u1ed3ng (1 th\\u00e1ng 3 n\\u0103m 1906 \\u2013 29 th\\u00e1ng 4 n\\u0103m 2000) l\\u00e0 Th\\u1ee7 t\\u01b0\\u1edbng \\u0111\\u1ea7u ti\\u00ean c\\u1ee7a n\\u01b0\\u1edbc C\\u1ed9ng h\\u00f2a X\\u00e3 h\\u1ed9i ch\\u1ee7 ngh\\u0129a Vi\\u1ec7t Nam t\\u1eeb n\\u0103m 1976 (t\\u1eeb n\\u0103m 1981 g\\u1ecdi l\\u00e0 Ch\\u1ee7 t\\u1ecbch H\\u1ed9i \\u0111\\u1ed3ng B\\u1ed9 tr\\u01b0\\u1edfng) cho \\u0111\\u1ebfn khi ngh\\u1ec9 h\\u01b0u n\\u0103m 1987. Tr\\u01b0\\u1edbc \\u0111\\u00f3 \\u00f4ng t\\u1eebng gi\\u1eef ch\\u1ee9c v\\u1ee5 Th\\u1ee7 t\\u01b0\\u1edbng Ch\\u00ednh ph\\u1ee7 Vi\\u1ec7t Nam D\\u00e2n ch\\u1ee7 C\\u1ed9ng h\\u00f2a t\\u1eeb n\\u0103m 1955 \\u0111\\u1ebfn n\\u0103m 1976. \\u00d4ng l\\u00e0 v\\u1ecb Th\\u1ee7 t\\u01b0\\u1edbng Vi\\u1ec7t Nam t\\u1ea1i v\\u1ecb l\\u00e2u nh\\u1ea5t (1955\\u20131987). \\u00d4ng l\\u00e0 h\\u1ecdc tr\\u00f2, c\\u1ed9ng s\\u1ef1 c\\u1ee7a Ch\\u1ee7 t\\u1ecbch H\\u1ed3 Ch\\u00ed Minh. \\u00d4ng c\\u00f3 t\\u00ean g\\u1ecdi th\\u00e2n m\\u1eadt l\\u00e0 T\\u00f4, \\u0111\\u00e2y t\\u1eebng l\\u00e0 b\\u00ed danh c\\u1ee7a \\u00f4ng. \\u00d4ng c\\u00f2n c\\u00f3 t\\u00ean g\\u1ecdi l\\u00e0 L\\u00e2m B\\u00e1 Ki\\u1ec7t khi l\\u00e0m Ph\\u00f3 ch\\u1ee7 nhi\\u1ec7m c\\u01a1 quan Bi\\u1ec7n s\\u1ef1 x\\u1ee9 t\\u1ea1i Qu\\u1ebf L\\u00e2m (Ch\\u1ee7 nhi\\u1ec7m l\\u00e0 H\\u1ed3 H\\u1ecdc L\\u00e3m).\"}, {\"qas\": [{\"question\": \"S\\u1ef1 ki\\u1ec7n quan tr\\u1ecdng n\\u00e0o \\u0111\\u00e3 di\\u1ec5n ra v\\u00e0o ng\\u00e0y 20/7/1954?\", \"answers\": [{\"answer_start\": 364, \"text\": \"b\\u1ea3n Hi\\u1ec7p \\u0111\\u1ecbnh \\u0111\\u00ecnh ch\\u1ec9 chi\\u1ebfn s\\u1ef1 \\u1edf Vi\\u1ec7t Nam, Campuchia v\\u00e0 L\\u00e0o \\u0111\\u00e3 \\u0111\\u01b0\\u1ee3c k\\u00fd k\\u1ebft th\\u1eeba nh\\u1eadn t\\u00f4n tr\\u1ecdng \\u0111\\u1ed9c l\\u1eadp, ch\\u1ee7 quy\\u1ec1n, c\\u1ee7a n\\u01b0\\u1edbc Vi\\u1ec7t Nam, L\\u00e0o v\\u00e0 Campuchia\"}], \"id\": \"uit_01__05272_1_1\"}, {\"question\": \"Ch\\u1ee9c v\\u1ee5 m\\u00e0 Ph\\u1ea1m V\\u0103n \\u0110\\u1ed3ng \\u0111\\u1ea3m nhi\\u1ec7m t\\u1ea1i H\\u1ed9i ngh\\u1ecb Gen\\u00e8ve v\\u1ec1 \\u0110\\u00f4ng D\\u01b0\\u01a1ng?\", \"answers\": [{\"answer_start\": 33, \"text\": \"Tr\\u01b0\\u1edfng ph\\u00e1i \\u0111o\\u00e0n Ch\\u00ednh ph\\u1ee7\"}], \"id\": \"uit_01__05272_1_2\"}, {\"question\": \"H\\u1ed9i ngh\\u1ecb Gen\\u00e8ve v\\u1ec1 \\u0110\\u00f4ng D\\u01b0\\u01a1ng c\\u00f3 t\\u00ednh ch\\u1ea5t nh\\u01b0 th\\u1ebf n\\u00e0o?\", \"answers\": [{\"answer_start\": 262, \"text\": \"r\\u1ea5t c\\u0103ng th\\u1eb3ng v\\u00e0 ph\\u1ee9c t\\u1ea1p\"}], \"id\": \"uit_01__05272_1_3\"}]}]}]}\n\"\"\"\n\nEXPECTED = ['Tên gọi nào được Phạm Văn Đồng sử dụng khi làm Phó chủ nhiệm cơ quan Biện sự xứ tại Quế Lâm?', 'Phạm Văn Đồng giữ chức vụ gì trong bộ máy Nhà nước Cộng hòa Xã hội chủ nghĩa Việt Nam?', 'Giai đoạn năm 1955-1976, Phạm Văn Đồng nắm giữ chức vụ gì?', 'Sự kiện quan trọng nào đã diễn ra vào ngày 20/7/1954?', 'Chức vụ mà Phạm Văn Đồng đảm nhiệm tại Hội nghị Genève về Đông Dương?', 'Hội nghị Genève về Đông Dương có tính chất như thế nào?']\n\ndef test_read_file():\n results = selftrain_vi_quad.parse_quad(SAMPLE_TEXT)\n assert results == EXPECTED\n"
},
{
"path": "stanza/tests/constituency/test_text_processing.py",
"content": "\"\"\"\nRun through the various text processing methods for using the parser on text files / directories\n\nUses a simple tree where the parser should always get it right, but things could potentially go wrong\n\"\"\"\n\nimport glob\nimport os\nimport pytest\n\nfrom stanza import Pipeline\n\nfrom stanza.models.constituency import text_processing\nfrom stanza.models.constituency import tree_reader\nfrom stanza.tests import TEST_MODELS_DIR\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\n@pytest.fixture(scope=\"module\")\ndef pipeline():\n return Pipeline(dir=TEST_MODELS_DIR, lang=\"en\", processors=\"tokenize, pos, constituency\", tokenize_pretokenized=True)\n\ndef test_read_tokenized_file(tmp_path):\n filename = str(tmp_path / \"test_input.txt\")\n with open(filename, \"w\") as fout:\n # test that the underscore token comes back with spaces\n fout.write(\"This is a_small test\\nLine two\\n\")\n text, ids = text_processing.read_tokenized_file(filename)\n assert text == [['This', 'is', 'a small', 'test'], ['Line', 'two']]\n assert ids == [None, None]\n\ndef test_parse_tokenized_sentences(pipeline):\n con_processor = pipeline.processors[\"constituency\"]\n model = con_processor._model\n args = model.args\n\n sentences = [[\"This\", \"is\", \"a\", \"test\"]]\n trees = text_processing.parse_tokenized_sentences(args, model, [pipeline], sentences)\n predictions = [x.predictions for x in trees]\n assert len(predictions) == 1\n scored_trees = predictions[0]\n assert len(scored_trees) == 1\n result = \"{}\".format(scored_trees[0].tree)\n expected = \"(ROOT (S (NP (DT This)) (VP (VBZ is) (NP (DT a) (NN test)))))\"\n assert result == expected\n\ndef test_parse_text(tmp_path, pipeline):\n con_processor = pipeline.processors[\"constituency\"]\n model = con_processor._model\n args = model.args\n\n raw_file = str(tmp_path / \"test_input.txt\")\n with open(raw_file, \"w\") as fout:\n fout.write(\"This is a test\\nThis is another test\\n\")\n output_file = str(tmp_path / \"test_output.txt\")\n text_processing.parse_text(args, model, [pipeline], tokenized_file=raw_file, predict_file=output_file)\n\n trees = tree_reader.read_treebank(output_file)\n trees = [\"{}\".format(x) for x in trees]\n expected_trees = [\"(ROOT (S (NP (DT This)) (VP (VBZ is) (NP (DT a) (NN test)))))\",\n \"(ROOT (S (NP (DT This)) (VP (VBZ is) (NP (DT another) (NN test)))))\"]\n assert trees == expected_trees\n\ndef test_parse_dir(tmp_path, pipeline):\n con_processor = pipeline.processors[\"constituency\"]\n model = con_processor._model\n args = model.args\n\n raw_dir = str(tmp_path / \"input\")\n os.makedirs(raw_dir)\n raw_f1 = str(tmp_path / \"input\" / \"f1.txt\")\n raw_f2 = str(tmp_path / \"input\" / \"f2.txt\")\n output_dir = str(tmp_path / \"output\")\n\n with open(raw_f1, \"w\") as fout:\n fout.write(\"This is a test\")\n with open(raw_f2, \"w\") as fout:\n fout.write(\"This is another test\")\n\n text_processing.parse_dir(args, model, [pipeline], raw_dir, output_dir)\n output_files = sorted(glob.glob(os.path.join(output_dir, \"*\")))\n expected_trees = [\"(ROOT (S (NP (DT This)) (VP (VBZ is) (NP (DT a) (NN test)))))\",\n \"(ROOT (S (NP (DT This)) (VP (VBZ is) (NP (DT another) (NN test)))))\"]\n for output_file, expected_tree in zip(output_files, expected_trees):\n trees = tree_reader.read_treebank(output_file)\n assert len(trees) == 1\n assert \"{}\".format(trees[0]) == expected_tree\n\ndef test_parse_text(tmp_path, pipeline):\n con_processor = pipeline.processors[\"constituency\"]\n model = con_processor._model\n args = dict(model.args)\n\n model_path = con_processor._config['model_path']\n\n raw_file = str(tmp_path / \"test_input.txt\")\n with open(raw_file, \"w\") as fout:\n fout.write(\"This is a test\\nThis is another test\\n\")\n output_file = str(tmp_path / \"test_output.txt\")\n\n args['tokenized_file'] = raw_file\n args['predict_file'] = output_file\n\n text_processing.load_model_parse_text(args, model_path, [pipeline])\n trees = tree_reader.read_treebank(output_file)\n trees = [\"{}\".format(x) for x in trees]\n expected_trees = [\"(ROOT (S (NP (DT This)) (VP (VBZ is) (NP (DT a) (NN test)))))\",\n \"(ROOT (S (NP (DT This)) (VP (VBZ is) (NP (DT another) (NN test)))))\"]\n assert trees == expected_trees\n"
},
{
"path": "stanza/tests/constituency/test_top_down_oracle.py",
"content": "import pytest\n\nfrom stanza.models.constituency.base_model import SimpleModel\nfrom stanza.models.constituency.parse_transitions import Shift, OpenConstituent, CloseConstituent, TransitionScheme\nfrom stanza.models.constituency.top_down_oracle import *\nfrom stanza.models.constituency.transition_sequence import build_sequence\nfrom stanza.models.constituency.tree_reader import read_trees\n\nfrom stanza.tests.constituency.test_transition_sequence import reconstruct_tree\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\nOPEN_SHIFT_EXAMPLE_TREE = \"\"\"\n( (S\n (NP (NNP Jennifer) (NNP Sh\\'reyan))\n (VP (VBZ has)\n (NP (RB nice) (NNS antennae)))))\n\"\"\"\n\nOPEN_SHIFT_PROBLEM_TREE = \"\"\"\n(ROOT (S (NP (NP (NP (DT The) (`` ``) (JJ Thin) (NNP Man) ('' '') (NN series)) (PP (IN of) (NP (NNS movies)))) (, ,) (CONJP (RB as) (RB well) (IN as)) (NP (JJ many) (NNS others)) (, ,)) (VP (VBD based) (NP (PRP$ their) (JJ entire) (JJ comedic) (NN appeal)) (PP (IN on) (NP (NP (DT the) (NN star) (NNS detectives) (POS ')) (JJ witty) (NNS quips) (CC and) (NNS puns))) (SBAR (IN as) (S (NP (NP (JJ other) (NNS characters)) (PP (IN in) (NP (DT the) (NNS movies)))) (VP (VBD were) (VP (VBN murdered)))))) (. .)))\n\"\"\"\n\nROOT_LABELS = [\"ROOT\"]\n\ndef get_single_repair(gold_sequence, wrong_transition, repair_fn, idx, *args, **kwargs):\n return repair_fn(gold_sequence[idx], wrong_transition, gold_sequence, idx, ROOT_LABELS, None, None, *args, **kwargs)\n\ndef build_state(model, tree, num_transitions):\n transitions = build_sequence(tree, transition_scheme=TransitionScheme.TOP_DOWN)\n states = model.initial_state_from_gold_trees([tree], [transitions])\n for idx, t in enumerate(transitions[:num_transitions]):\n assert t.is_legal(states[0], model), \"Transition {} not legal at step {} in sequence {}\".format(t, idx, sequence)\n states = model.bulk_apply(states, [t])\n state = states[0]\n return state\n\ndef test_fix_open_shift():\n trees = read_trees(OPEN_SHIFT_EXAMPLE_TREE)\n assert len(trees) == 1\n tree = trees[0]\n\n transitions = build_sequence(tree, transition_scheme=TransitionScheme.TOP_DOWN)\n EXPECTED_ORIG = [OpenConstituent('ROOT'), OpenConstituent('S'), OpenConstituent('NP'), Shift(), Shift(), CloseConstituent(), OpenConstituent('VP'), Shift(), OpenConstituent('NP'), Shift(), Shift(), CloseConstituent(), CloseConstituent(), CloseConstituent(), CloseConstituent()]\n EXPECTED_FIX_EARLY = [OpenConstituent('ROOT'), OpenConstituent('S'), Shift(), Shift(), OpenConstituent('VP'), Shift(), OpenConstituent('NP'), Shift(), Shift(), CloseConstituent(), CloseConstituent(), CloseConstituent(), CloseConstituent()]\n EXPECTED_FIX_LATE = [OpenConstituent('ROOT'), OpenConstituent('S'), OpenConstituent('NP'), Shift(), Shift(), CloseConstituent(), OpenConstituent('VP'), Shift(), Shift(), Shift(), CloseConstituent(), CloseConstituent(), CloseConstituent()]\n\n assert transitions == EXPECTED_ORIG\n\n new_transitions = get_single_repair(transitions, Shift(), fix_one_open_shift, 2)\n assert new_transitions == EXPECTED_FIX_EARLY\n\n new_transitions = get_single_repair(transitions, Shift(), fix_one_open_shift, 8)\n assert new_transitions == EXPECTED_FIX_LATE\n\ndef test_fix_open_shift_observed_error():\n \"\"\"\n Ran into an error on this tree, need to fix it\n\n The problem is the multiple Open in a row all need to be removed when a Shift happens\n \"\"\"\n trees = read_trees(OPEN_SHIFT_PROBLEM_TREE)\n assert len(trees) == 1\n tree = trees[0]\n\n transitions = build_sequence(tree, transition_scheme=TransitionScheme.TOP_DOWN)\n new_transitions = get_single_repair(transitions, Shift(), fix_one_open_shift, 2)\n assert new_transitions is None\n\n new_transitions = get_single_repair(transitions, Shift(), fix_multiple_open_shift, 2)\n\n # Can break the expected transitions down like this:\n # [OpenConstituent(('ROOT',)), OpenConstituent(('S',)),\n # all gone: OpenConstituent(('NP',)), OpenConstituent(('NP',)), OpenConstituent(('NP',)),\n # Shift, Shift, Shift, Shift, Shift, Shift,\n # gone: CloseConstituent,\n # OpenConstituent(('PP',)), Shift, OpenConstituent(('NP',)), Shift, CloseConstituent, CloseConstituent,\n # gone: CloseConstituent,\n # Shift, OpenConstituent(('CONJP',)), Shift, Shift, Shift, CloseConstituent, OpenConstituent(('NP',)), Shift, Shift, CloseConstituent, Shift,\n # gone: CloseConstituent,\n # and then the rest:\n # OpenConstituent(('VP',)), Shift, OpenConstituent(('NP',)),\n # Shift, Shift, Shift, Shift, CloseConstituent,\n # OpenConstituent(('PP',)), Shift, OpenConstituent(('NP',)),\n # OpenConstituent(('NP',)), Shift, Shift, Shift, Shift,\n # CloseConstituent, Shift, Shift, Shift, Shift, CloseConstituent,\n # CloseConstituent, OpenConstituent(('SBAR',)), Shift,\n # OpenConstituent(('S',)), OpenConstituent(('NP',)),\n # OpenConstituent(('NP',)), Shift, Shift, CloseConstituent,\n # OpenConstituent(('PP',)), Shift, OpenConstituent(('NP',)),\n # Shift, Shift, CloseConstituent, CloseConstituent,\n # CloseConstituent, OpenConstituent(('VP',)), Shift,\n # OpenConstituent(('VP',)), Shift, CloseConstituent,\n # CloseConstituent, CloseConstituent, CloseConstituent,\n # CloseConstituent, Shift, CloseConstituent, CloseConstituent]\n expected_transitions = [OpenConstituent('ROOT'), OpenConstituent('S'), Shift(), Shift(), Shift(), Shift(), Shift(), Shift(), OpenConstituent('PP'), Shift(), OpenConstituent('NP'), Shift(), CloseConstituent(), CloseConstituent(), Shift(), OpenConstituent('CONJP'), Shift(), Shift(), Shift(), CloseConstituent(), OpenConstituent('NP'), Shift(), Shift(), CloseConstituent(), Shift(), OpenConstituent('VP'), Shift(), OpenConstituent('NP'), Shift(), Shift(), Shift(), Shift(), CloseConstituent(), OpenConstituent('PP'), Shift(), OpenConstituent('NP'), OpenConstituent('NP'), Shift(), Shift(), Shift(), Shift(), CloseConstituent(), Shift(), Shift(), Shift(), Shift(), CloseConstituent(), CloseConstituent(), OpenConstituent('SBAR'), Shift(), OpenConstituent('S'), OpenConstituent('NP'), OpenConstituent('NP'), Shift(), Shift(), CloseConstituent(), OpenConstituent('PP'), Shift(), OpenConstituent('NP'), Shift(), Shift(), CloseConstituent(), CloseConstituent(), CloseConstituent(), OpenConstituent('VP'), Shift(), OpenConstituent('VP'), Shift(), CloseConstituent(), CloseConstituent(), CloseConstituent(), CloseConstituent(), CloseConstituent(), Shift(), CloseConstituent(), CloseConstituent()]\n\n assert new_transitions == expected_transitions\n\ndef test_open_open_ambiguous_unary_fix():\n trees = read_trees(OPEN_SHIFT_EXAMPLE_TREE)\n assert len(trees) == 1\n tree = trees[0]\n\n transitions = build_sequence(tree, transition_scheme=TransitionScheme.TOP_DOWN)\n EXPECTED_ORIG = [OpenConstituent('ROOT'), OpenConstituent('S'), OpenConstituent('NP'), Shift(), Shift(), CloseConstituent(), OpenConstituent('VP'), Shift(), OpenConstituent('NP'), Shift(), Shift(), CloseConstituent(), CloseConstituent(), CloseConstituent(), CloseConstituent()]\n EXPECTED_FIX = [OpenConstituent('ROOT'), OpenConstituent('S'), OpenConstituent('VP'), OpenConstituent('NP'), Shift(), Shift(), CloseConstituent(), CloseConstituent(), OpenConstituent('VP'), Shift(), OpenConstituent('NP'), Shift(), Shift(), CloseConstituent(), CloseConstituent(), CloseConstituent(), CloseConstituent()]\n assert transitions == EXPECTED_ORIG\n new_transitions = get_single_repair(transitions, OpenConstituent('VP'), fix_open_open_ambiguous_unary, 2)\n assert new_transitions == EXPECTED_FIX\n\n\ndef test_open_open_ambiguous_later_fix():\n trees = read_trees(OPEN_SHIFT_EXAMPLE_TREE)\n assert len(trees) == 1\n tree = trees[0]\n\n transitions = build_sequence(tree, transition_scheme=TransitionScheme.TOP_DOWN)\n EXPECTED_ORIG = [OpenConstituent('ROOT'), OpenConstituent('S'), OpenConstituent('NP'), Shift(), Shift(), CloseConstituent(), OpenConstituent('VP'), Shift(), OpenConstituent('NP'), Shift(), Shift(), CloseConstituent(), CloseConstituent(), CloseConstituent(), CloseConstituent()]\n EXPECTED_FIX = [OpenConstituent('ROOT'), OpenConstituent('S'), OpenConstituent('VP'), OpenConstituent('NP'), Shift(), Shift(), CloseConstituent(), OpenConstituent('VP'), Shift(), OpenConstituent('NP'), Shift(), Shift(), CloseConstituent(), CloseConstituent(), CloseConstituent(), CloseConstituent(), CloseConstituent()]\n assert transitions == EXPECTED_ORIG\n new_transitions = get_single_repair(transitions, OpenConstituent('VP'), fix_open_open_ambiguous_later, 2)\n assert new_transitions == EXPECTED_FIX\n\n\nCLOSE_SHIFT_EXAMPLE_TREE = \"\"\"\n( (NP (DT a)\n (ADJP (NN stock) (HYPH -) (VBG picking))\n (NN tool)))\n\"\"\"\n\n# not intended to be a correct tree\nCLOSE_SHIFT_DEEP_EXAMPLE_TREE = \"\"\"\n( (NP (DT a)\n (VP (ADJP (NN stock) (HYPH -) (VBG picking)))\n (NN tool)))\n\"\"\"\n\n# not intended to be a correct tree\nCLOSE_SHIFT_OPEN_EXAMPLE_TREE = \"\"\"\n( (NP (DT a)\n (ADJP (NN stock) (HYPH -) (VBG picking))\n (NP (NN tool))))\n\"\"\"\n\nCLOSE_SHIFT_AMBIGUOUS_TREE = \"\"\"\n( (NP (DT a)\n (ADJP (NN stock) (HYPH -) (VBG picking))\n (NN tool)\n (NN foo)))\n\"\"\"\n\ndef test_fix_close_shift_ambiguous_immediate():\n \"\"\"\n Test the result when a close/shift error occurs and we want to close the new, incorrect constituent immediately\n \"\"\"\n trees = read_trees(CLOSE_SHIFT_AMBIGUOUS_TREE)\n assert len(trees) == 1\n tree = trees[0]\n\n transitions = build_sequence(tree, transition_scheme=TransitionScheme.TOP_DOWN)\n new_sequence = get_single_repair(transitions, transitions[8], fix_close_shift_ambiguous_later, 7)\n expected_original = [OpenConstituent('ROOT'), OpenConstituent('NP'), Shift(), OpenConstituent('ADJP'), Shift(), Shift(), Shift(), CloseConstituent(), Shift(), Shift(), CloseConstituent(), CloseConstituent()]\n expected_update = [OpenConstituent('ROOT'), OpenConstituent('NP'), Shift(), OpenConstituent('ADJP'), Shift(), Shift(), Shift(), Shift(), Shift(), CloseConstituent(), CloseConstituent(), CloseConstituent()]\n assert transitions == expected_original\n assert new_sequence == expected_update\n\ndef test_fix_close_shift_ambiguous_later():\n # test that the one with two shifts, which is ambiguous, gets rejected\n trees = read_trees(CLOSE_SHIFT_AMBIGUOUS_TREE)\n assert len(trees) == 1\n tree = trees[0]\n\n transitions = build_sequence(tree, transition_scheme=TransitionScheme.TOP_DOWN)\n new_sequence = get_single_repair(transitions, transitions[8], fix_close_shift_ambiguous_immediate, 7)\n expected_original = [OpenConstituent('ROOT'), OpenConstituent('NP'), Shift(), OpenConstituent('ADJP'), Shift(), Shift(), Shift(), CloseConstituent(), Shift(), Shift(), CloseConstituent(), CloseConstituent()]\n expected_update = [OpenConstituent('ROOT'), OpenConstituent('NP'), Shift(), OpenConstituent('ADJP'), Shift(), Shift(), Shift(), Shift(), CloseConstituent(), Shift(), CloseConstituent(), CloseConstituent()]\n assert transitions == expected_original\n assert new_sequence == expected_update\n\ndef test_oracle_with_optional_level():\n tree = read_trees(CLOSE_SHIFT_AMBIGUOUS_TREE)[0]\n gold_sequence = [OpenConstituent('ROOT'), OpenConstituent('NP'), Shift(), OpenConstituent('ADJP'), Shift(), Shift(), Shift(), CloseConstituent(), Shift(), Shift(), CloseConstituent(), CloseConstituent()]\n expected_update = [OpenConstituent('ROOT'), OpenConstituent('NP'), Shift(), OpenConstituent('ADJP'), Shift(), Shift(), Shift(), Shift(), CloseConstituent(), Shift(), CloseConstituent(), CloseConstituent()]\n\n transitions = build_sequence(tree, transition_scheme=TransitionScheme.TOP_DOWN)\n assert transitions == gold_sequence\n\n oracle = TopDownOracle(ROOT_LABELS, 1, \"\", \"\")\n\n model = SimpleModel(transition_scheme=TransitionScheme.TOP_DOWN_UNARY, root_labels=ROOT_LABELS)\n state = build_state(model, tree, 7)\n fix, new_sequence = oracle.fix_error(pred_transition=gold_sequence[8],\n model=model,\n state=state)\n assert fix is RepairType.OTHER_CLOSE_SHIFT\n assert new_sequence is None\n\n oracle = TopDownOracle(ROOT_LABELS, 1, \"CLOSE_SHIFT_AMBIGUOUS_IMMEDIATE_ERROR\", \"\")\n fix, new_sequence = oracle.fix_error(pred_transition=gold_sequence[8],\n model=model,\n state=state)\n assert fix is RepairType.CLOSE_SHIFT_AMBIGUOUS_IMMEDIATE_ERROR\n assert new_sequence == expected_update\n\n\ndef test_fix_close_shift():\n \"\"\"\n Test a tree of the kind we expect the close/shift to be able to get right\n \"\"\"\n trees = read_trees(CLOSE_SHIFT_EXAMPLE_TREE)\n assert len(trees) == 1\n tree = trees[0]\n\n transitions = build_sequence(tree, transition_scheme=TransitionScheme.TOP_DOWN)\n\n new_sequence = get_single_repair(transitions, transitions[8], fix_close_shift, 7)\n\n expected_original = [OpenConstituent('ROOT'), OpenConstituent('NP'), Shift(), OpenConstituent('ADJP'), Shift(), Shift(), Shift(), CloseConstituent(), Shift(), CloseConstituent(), CloseConstituent()]\n expected_update = [OpenConstituent('ROOT'), OpenConstituent('NP'), Shift(), OpenConstituent('ADJP'), Shift(), Shift(), Shift(), Shift(), CloseConstituent(), CloseConstituent(), CloseConstituent()]\n assert transitions == expected_original\n assert new_sequence == expected_update\n\n # test that the one with two shifts, which is ambiguous, gets rejected\n trees = read_trees(CLOSE_SHIFT_AMBIGUOUS_TREE)\n assert len(trees) == 1\n tree = trees[0]\n\n transitions = build_sequence(tree, transition_scheme=TransitionScheme.TOP_DOWN)\n new_sequence = get_single_repair(transitions, transitions[8], fix_close_shift, 7)\n assert new_sequence is None\n\ndef test_fix_close_shift_deeper_tree():\n \"\"\"\n Test a tree of the kind we expect the close/shift to be able to get right\n \"\"\"\n trees = read_trees(CLOSE_SHIFT_DEEP_EXAMPLE_TREE)\n assert len(trees) == 1\n tree = trees[0]\n\n transitions = build_sequence(tree, transition_scheme=TransitionScheme.TOP_DOWN)\n\n for count_opens in [True, False]:\n new_sequence = get_single_repair(transitions, transitions[10], fix_close_shift, 8, count_opens=count_opens)\n\n expected_original = [OpenConstituent('ROOT'), OpenConstituent('NP'), Shift(), OpenConstituent('VP'), OpenConstituent('ADJP'), Shift(), Shift(), Shift(), CloseConstituent(), CloseConstituent(), Shift(), CloseConstituent(), CloseConstituent()]\n expected_update = [OpenConstituent('ROOT'), OpenConstituent('NP'), Shift(), OpenConstituent('VP'), OpenConstituent('ADJP'), Shift(), Shift(), Shift(), Shift(), CloseConstituent(), CloseConstituent(), CloseConstituent(), CloseConstituent()]\n assert transitions == expected_original\n assert new_sequence == expected_update\n\ndef test_fix_close_shift_open_tree():\n \"\"\"\n We would like the close/shift to get this case right as well\n \"\"\"\n trees = read_trees(CLOSE_SHIFT_OPEN_EXAMPLE_TREE)\n assert len(trees) == 1\n tree = trees[0]\n\n transitions = build_sequence(tree, transition_scheme=TransitionScheme.TOP_DOWN)\n\n new_sequence = get_single_repair(transitions, transitions[9], fix_close_shift, 7, count_opens=False)\n assert new_sequence is None\n\n new_sequence = get_single_repair(transitions, transitions[9], fix_close_shift_with_opens, 7)\n\n expected_original = [OpenConstituent('ROOT'), OpenConstituent('NP'), Shift(), OpenConstituent('ADJP'), Shift(), Shift(), Shift(), CloseConstituent(), OpenConstituent('NP'), Shift(), CloseConstituent(), CloseConstituent(), CloseConstituent()]\n expected_update = [OpenConstituent('ROOT'), OpenConstituent('NP'), Shift(), OpenConstituent('ADJP'), Shift(), Shift(), Shift(), Shift(), CloseConstituent(), CloseConstituent(), CloseConstituent()]\n assert transitions == expected_original\n assert new_sequence == expected_update\n\nCLOSE_OPEN_EXAMPLE_TREE = \"\"\"\n( (VP (VBZ eat)\n (NP (NN spaghetti))\n (PP (IN with) (DT a) (NN fork))))\n\"\"\"\n\nCLOSE_OPEN_DIFFERENT_LABEL_TREE = \"\"\"\n( (VP (VBZ eat)\n (NP (NN spaghetti))\n (NP (DT a) (NN fork))))\n\"\"\"\n\nCLOSE_OPEN_TWO_LABELS_TREE = \"\"\"\n( (VP (VBZ eat)\n (NP (NN spaghetti))\n (PP (IN with) (DT a) (NN fork))\n (PP (IN in) (DT a) (NN restaurant))))\n\"\"\"\n\ndef test_fix_close_open():\n trees = read_trees(CLOSE_OPEN_EXAMPLE_TREE)\n assert len(trees) == 1\n tree = trees[0]\n\n transitions = build_sequence(tree, transition_scheme=TransitionScheme.TOP_DOWN)\n\n assert isinstance(transitions[5], CloseConstituent)\n assert transitions[6] == OpenConstituent(\"PP\")\n\n new_transitions = get_single_repair(transitions, transitions[6], fix_close_open_correct_open, 5)\n\n expected_original = [OpenConstituent('ROOT'), OpenConstituent('VP'), Shift(), OpenConstituent('NP'), Shift(), CloseConstituent(), OpenConstituent('PP'), Shift(), Shift(), Shift(), CloseConstituent(), CloseConstituent(), CloseConstituent()]\n expected_update = [OpenConstituent('ROOT'), OpenConstituent('VP'), Shift(), OpenConstituent('NP'), Shift(), OpenConstituent('PP'), Shift(), Shift(), Shift(), CloseConstituent(), CloseConstituent(), CloseConstituent(), CloseConstituent()]\n\n assert transitions == expected_original\n assert new_transitions == expected_update\n\ndef test_fix_close_open_invalid():\n for TREE in (CLOSE_OPEN_DIFFERENT_LABEL_TREE, CLOSE_OPEN_TWO_LABELS_TREE):\n trees = read_trees(TREE)\n assert len(trees) == 1\n tree = trees[0]\n\n transitions = build_sequence(tree, transition_scheme=TransitionScheme.TOP_DOWN)\n\n assert isinstance(transitions[5], CloseConstituent)\n assert isinstance(transitions[6], OpenConstituent)\n\n new_transitions = get_single_repair(transitions, OpenConstituent(\"PP\"), fix_close_open_correct_open, 5)\n assert new_transitions is None\n\ndef test_fix_close_open_ambiguous_immediate():\n \"\"\"\n Test that a fix for an ambiguous close/open works as expected\n \"\"\"\n trees = read_trees(CLOSE_OPEN_TWO_LABELS_TREE)\n assert len(trees) == 1\n tree = trees[0]\n\n transitions = build_sequence(tree, transition_scheme=TransitionScheme.TOP_DOWN)\n assert isinstance(transitions[5], CloseConstituent)\n assert isinstance(transitions[6], OpenConstituent)\n\n reconstructed = reconstruct_tree(tree, transitions, transition_scheme=TransitionScheme.TOP_DOWN)\n assert tree == reconstructed\n\n new_transitions = get_single_repair(transitions, OpenConstituent(\"PP\"), fix_close_open_correct_open, 5, check_close=False)\n reconstructed = reconstruct_tree(tree, new_transitions, transition_scheme=TransitionScheme.TOP_DOWN)\n\n expected = \"\"\"\n ( (VP (VBZ eat)\n (NP (NN spaghetti)\n (PP (IN with) (DT a) (NN fork)))\n (PP (IN in) (DT a) (NN restaurant))))\n \"\"\"\n expected = read_trees(expected)[0]\n assert reconstructed == expected\n\ndef test_fix_close_open_ambiguous_later():\n \"\"\"\n Test that a fix for an ambiguous close/open works as expected\n \"\"\"\n trees = read_trees(CLOSE_OPEN_TWO_LABELS_TREE)\n assert len(trees) == 1\n tree = trees[0]\n\n transitions = build_sequence(tree, transition_scheme=TransitionScheme.TOP_DOWN)\n assert isinstance(transitions[5], CloseConstituent)\n assert isinstance(transitions[6], OpenConstituent)\n\n reconstructed = reconstruct_tree(tree, transitions, transition_scheme=TransitionScheme.TOP_DOWN)\n assert tree == reconstructed\n\n new_transitions = get_single_repair(transitions, OpenConstituent(\"PP\"), fix_close_open_correct_open_ambiguous_later, 5, check_close=False)\n reconstructed = reconstruct_tree(tree, new_transitions, transition_scheme=TransitionScheme.TOP_DOWN)\n\n expected = \"\"\"\n ( (VP (VBZ eat)\n (NP (NN spaghetti)\n (PP (IN with) (DT a) (NN fork))\n (PP (IN in) (DT a) (NN restaurant)))))\n \"\"\"\n expected = read_trees(expected)[0]\n assert reconstructed == expected\n\n\nSHIFT_CLOSE_EXAMPLES = [\n (\"((S (NP (DT an) (NML (NNP Oct) (CD 19)) (NN review))))\", \"((S (NP (DT an) (NML (NNP Oct) (CD 19))) (NN review)))\", 8),\n (\"((S (NP (` `) (NP (DT The) (NN Misanthrope)) (` `) (PP (IN at) (NP (NNP Goodman) (NNP Theatre))))))\",\n \"((S (NP (` `) (NP (DT The)) (NN Misanthrope) (` `) (PP (IN at) (NP (NNP Goodman) (NNP Theatre))))))\", 6),\n (\"((S (NP (` `) (NP (DT The) (NN Misanthrope)) (` `) (PP (IN at) (NP (NNP Goodman) (NNP Theatre))))))\",\n \"((S (NP (` `) (NP (DT The) (NN Misanthrope))) (` `) (PP (IN at) (NP (NNP Goodman) (NNP Theatre)))))\", 8),\n (\"((S (NP (` `) (NP (DT The) (NN Misanthrope)) (` `) (PP (IN at) (NP (NNP Goodman) (NNP Theatre))))))\",\n \"((S (NP (` `) (NP (DT The) (NN Misanthrope)) (` `) (PP (IN at) (NP (NNP Goodman)) (NNP Theatre)))))\", 13),\n]\n\ndef test_shift_close():\n for idx, (orig_tree, expected_tree, shift_position) in enumerate(SHIFT_CLOSE_EXAMPLES):\n trees = read_trees(orig_tree)\n assert len(trees) == 1\n tree = trees[0]\n\n transitions = build_sequence(tree, transition_scheme=TransitionScheme.TOP_DOWN)\n if shift_position is None:\n print(transitions)\n continue\n\n assert isinstance(transitions[shift_position], Shift)\n new_transitions = get_single_repair(transitions, CloseConstituent(), fix_shift_close, shift_position)\n reconstructed = reconstruct_tree(tree, new_transitions, transition_scheme=TransitionScheme.TOP_DOWN)\n if expected_tree is None:\n print(transitions)\n print(new_transitions)\n\n print(\"{:P}\".format(reconstructed))\n else:\n expected_tree = read_trees(expected_tree)\n assert len(expected_tree) == 1\n expected_tree = expected_tree[0]\n\n assert reconstructed == expected_tree\n\ndef test_shift_open_ambiguous_unary():\n \"\"\"\n Test what happens if a Shift is turned into an Open in an ambiguous manner\n \"\"\"\n trees = read_trees(CLOSE_SHIFT_AMBIGUOUS_TREE)\n assert len(trees) == 1\n tree = trees[0]\n\n transitions = build_sequence(tree, transition_scheme=TransitionScheme.TOP_DOWN)\n expected_original = [OpenConstituent('ROOT'), OpenConstituent('NP'), Shift(), OpenConstituent('ADJP'), Shift(), Shift(), Shift(), CloseConstituent(), Shift(), Shift(), CloseConstituent(), CloseConstituent()]\n assert transitions == expected_original\n\n new_sequence = get_single_repair(transitions, OpenConstituent(\"ZZ\"), fix_shift_open_ambiguous_unary, 4)\n expected_updated = [OpenConstituent('ROOT'), OpenConstituent('NP'), Shift(), OpenConstituent('ADJP'), OpenConstituent('ZZ'), Shift(), CloseConstituent(), Shift(), Shift(), CloseConstituent(), Shift(), Shift(), CloseConstituent(), CloseConstituent()]\n assert new_sequence == expected_updated\n\ndef test_shift_open_ambiguous_later():\n \"\"\"\n Test what happens if a Shift is turned into an Open in an ambiguous manner\n \"\"\"\n trees = read_trees(CLOSE_SHIFT_AMBIGUOUS_TREE)\n assert len(trees) == 1\n tree = trees[0]\n\n transitions = build_sequence(tree, transition_scheme=TransitionScheme.TOP_DOWN)\n expected_original = [OpenConstituent('ROOT'), OpenConstituent('NP'), Shift(), OpenConstituent('ADJP'), Shift(), Shift(), Shift(), CloseConstituent(), Shift(), Shift(), CloseConstituent(), CloseConstituent()]\n assert transitions == expected_original\n\n new_sequence = get_single_repair(transitions, OpenConstituent(\"ZZ\"), fix_shift_open_ambiguous_later, 4)\n expected_updated = [OpenConstituent('ROOT'), OpenConstituent('NP'), Shift(), OpenConstituent('ADJP'), OpenConstituent('ZZ'), Shift(), Shift(), Shift(), CloseConstituent(), CloseConstituent(), Shift(), Shift(), CloseConstituent(), CloseConstituent()]\n assert new_sequence == expected_updated\n"
},
{
"path": "stanza/tests/constituency/test_trainer.py",
"content": "from collections import defaultdict\nimport logging\nimport pathlib\nimport tempfile\n\nimport pytest\nimport torch\nfrom torch import nn\nfrom torch import optim\n\nfrom stanza import Pipeline\n\nfrom stanza.models import constituency_parser\nfrom stanza.models.common import pretrain\nfrom stanza.models.common.bert_embedding import load_bert, load_tokenizer\nfrom stanza.models.common.foundation_cache import FoundationCache\nfrom stanza.models.common.utils import set_random_seed\nfrom stanza.models.constituency import lstm_model\nfrom stanza.models.constituency.parse_transitions import Transition\nfrom stanza.models.constituency import parser_training\nfrom stanza.models.constituency import trainer\nfrom stanza.models.constituency import tree_reader\nfrom stanza.tests import *\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\nlogger = logging.getLogger('stanza.constituency.trainer')\nlogger.setLevel(logging.WARNING)\n\nTREEBANK = \"\"\"\n( (S\n (VP (VBG Enjoying)\n (NP (PRP$ my) (JJ favorite) (NN Friday) (NN tradition)))\n (. .)))\n\n( (NP\n (VP (VBG Sitting)\n (PP (IN in)\n (NP (DT a) (RB stifling) (JJ hot) (NNP South) (NNP Station)))\n (VP (VBG waiting)\n (PP (IN for)\n (NP (PRP$ my) (JJ delayed) (NNP @MBTA) (NN train)))))\n (. .)))\n\n( (S\n (NP (PRP I))\n (VP\n (ADVP (RB really))\n (VBP hate)\n (NP (DT the) (NNP @MBTA)))))\n\n( (S\n (S (VP (VB Seek)))\n (CC and)\n (S (NP (PRP ye))\n (VP (MD shall)\n (VP (VB find))))\n (. .)))\n\"\"\"\n\ndef build_trainer(wordvec_pretrain_file, *args, treebank=TREEBANK):\n # TODO: build a fake embedding some other way?\n train_trees = tree_reader.read_trees(treebank)\n dev_trees = train_trees[-1:]\n silver_trees = []\n\n args = ['--wordvec_pretrain_file', wordvec_pretrain_file] + list(args)\n args = constituency_parser.parse_args(args)\n\n foundation_cache = FoundationCache()\n # might be None, unless we're testing loading an existing model\n model_load_name = args['load_name']\n\n model, _, _, _ = parser_training.build_trainer(args, train_trees, dev_trees, silver_trees, foundation_cache, model_load_name)\n assert isinstance(model.model, lstm_model.LSTMModel)\n return model\n\nclass TestTrainer:\n @pytest.fixture(scope=\"class\")\n def wordvec_pretrain_file(self):\n return f'{TEST_WORKING_DIR}/in/tiny_emb.pt'\n\n @pytest.fixture(scope=\"class\")\n def tiny_random_xlnet(self, tmp_path_factory):\n \"\"\"\n Download the tiny-random-xlnet model and make a concrete copy of it\n\n The issue here is that the \"random\" nature of the original\n makes it difficult or impossible to test that the values in\n the transformer don't change during certain operations.\n Saving a concrete instantiation of those random numbers makes\n it so we can test there is no difference when training only a\n subset of the layers, for example\n \"\"\"\n xlnet_name = 'hf-internal-testing/tiny-random-xlnet'\n xlnet_model, xlnet_tokenizer = load_bert(xlnet_name)\n path = str(tmp_path_factory.mktemp('tiny-random-xlnet'))\n xlnet_model.save_pretrained(path)\n xlnet_tokenizer.save_pretrained(path)\n return path\n\n @pytest.fixture(scope=\"class\")\n def tiny_random_bart(self, tmp_path_factory):\n \"\"\"\n Download the tiny-random-bart model and make a concrete copy of it\n\n Issue is the same as with tiny_random_xlnet\n \"\"\"\n bart_name = 'hf-internal-testing/tiny-random-bart'\n bart_model, bart_tokenizer = load_bert(bart_name)\n path = str(tmp_path_factory.mktemp('tiny-random-bart'))\n bart_model.save_pretrained(path)\n bart_tokenizer.save_pretrained(path)\n return path\n\n def test_initial_model(self, wordvec_pretrain_file):\n \"\"\"\n does nothing, just tests that the construction went okay\n \"\"\"\n args = ['wordvec_pretrain_file', wordvec_pretrain_file]\n build_trainer(wordvec_pretrain_file)\n\n\n def test_save_load_model(self, wordvec_pretrain_file):\n \"\"\"\n Just tests that saving and loading works without crashs.\n\n Currently no test of the values themselves\n (checks some fields to make sure they are regenerated correctly)\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tmpdirname:\n tr = build_trainer(wordvec_pretrain_file)\n transitions = tr.model.transitions\n\n # attempt saving\n filename = os.path.join(tmpdirname, \"parser.pt\")\n tr.save(filename)\n\n assert os.path.exists(filename)\n\n # load it back in\n tr2 = tr.load(filename)\n trans2 = tr2.model.transitions\n assert(transitions == trans2)\n assert all(isinstance(x, Transition) for x in trans2)\n\n def test_relearn_structure(self, wordvec_pretrain_file):\n \"\"\"\n Test that starting a trainer with --relearn_structure copies the old model\n \"\"\"\n\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tmpdirname:\n set_random_seed(1000)\n args = ['--pattn_num_layers', '0', '--lattn_d_proj', '0', '--hidden_size', '20', '--delta_embedding_dim', '10']\n tr = build_trainer(wordvec_pretrain_file, *args)\n\n # attempt saving\n filename = os.path.join(tmpdirname, \"parser.pt\")\n tr.save(filename)\n\n set_random_seed(1001)\n args = ['--pattn_num_layers', '1', '--lattn_d_proj', '0', '--hidden_size', '20', '--delta_embedding_dim', '10', '--relearn_structure', '--load_name', filename]\n tr2 = build_trainer(wordvec_pretrain_file, *args)\n\n assert torch.allclose(tr.model.delta_embedding.weight, tr2.model.delta_embedding.weight)\n assert torch.allclose(tr.model.output_layers[0].weight, tr2.model.output_layers[0].weight)\n # the norms will be the same, as the non-zero values are all the same\n assert torch.allclose(torch.linalg.norm(tr.model.word_lstm.weight_ih_l0), torch.linalg.norm(tr2.model.word_lstm.weight_ih_l0))\n\n def write_treebanks(self, tmpdirname):\n train_treebank_file = os.path.join(tmpdirname, \"train.mrg\")\n with open(train_treebank_file, 'w', encoding='utf-8') as fout:\n fout.write(TREEBANK)\n fout.write(TREEBANK)\n\n eval_treebank_file = os.path.join(tmpdirname, \"eval.mrg\")\n with open(eval_treebank_file, 'w', encoding='utf-8') as fout:\n fout.write(TREEBANK)\n\n return train_treebank_file, eval_treebank_file\n\n def training_args(self, wordvec_pretrain_file, tmpdirname, train_treebank_file, eval_treebank_file, *additional_args):\n # let's not make the model huge...\n args = ['--pattn_num_layers', '0', '--pattn_d_model', '128', '--lattn_d_proj', '0', '--use_lattn', '--hidden_size', '20', '--delta_embedding_dim', '10',\n '--wordvec_pretrain_file', wordvec_pretrain_file, '--data_dir', tmpdirname,\n '--save_dir', tmpdirname, '--save_name', 'test.pt', '--save_each_start', '0', '--save_each_name', os.path.join(tmpdirname, 'each_%02d.pt'),\n '--train_file', train_treebank_file, '--eval_file', eval_treebank_file,\n '--epoch_size', '6', '--train_batch_size', '3',\n '--shorthand', 'en_test']\n args = args + list(additional_args)\n args = constituency_parser.parse_args(args)\n # just in case we change the defaults in the future\n args['wandb'] = None\n return args\n\n def run_train_test(self, wordvec_pretrain_file, tmpdirname, num_epochs=5, extra_args=None, use_silver=False, exists_ok=False, foundation_cache=None):\n \"\"\"\n Runs a test of the trainer for a few iterations.\n\n Checks some basic properties of the saved model, but doesn't\n check for the accuracy of the results\n \"\"\"\n if extra_args is None:\n extra_args = []\n extra_args += ['--epochs', '%d' % num_epochs]\n\n train_treebank_file, eval_treebank_file = self.write_treebanks(tmpdirname)\n if use_silver:\n extra_args += ['--silver_file', str(eval_treebank_file)]\n args = self.training_args(wordvec_pretrain_file, tmpdirname, train_treebank_file, eval_treebank_file, *extra_args)\n\n each_name = args['save_each_name']\n if not exists_ok:\n assert not os.path.exists(args['save_name'])\n retag_pipeline = Pipeline(lang=\"en\", processors=\"tokenize, pos\", tokenize_pretokenized=True, dir=TEST_MODELS_DIR, foundation_cache=foundation_cache, download_method=None)\n trained_model = parser_training.train(args, None, [retag_pipeline])\n # check that hooks are in the model if expected\n for p in trained_model.model.parameters():\n if p.requires_grad:\n if args['grad_clipping'] is not None:\n assert len(p._backward_hooks) == 1\n else:\n assert p._backward_hooks is None\n\n # check that the model can be loaded back\n assert os.path.exists(args['save_name'])\n peft_name = trained_model.model.peft_name\n tr = trainer.Trainer.load(args['save_name'], load_optimizer=True, foundation_cache=retag_pipeline.foundation_cache, peft_name=trained_model.model.peft_name)\n assert tr.optimizer is not None\n assert tr.scheduler is not None\n assert tr.epochs_trained >= 1\n for p in tr.model.parameters():\n if p.requires_grad:\n assert p._backward_hooks is None\n\n tr = trainer.Trainer.load(args['checkpoint_save_name'], load_optimizer=True, foundation_cache=retag_pipeline.foundation_cache, peft_name=trained_model.model.peft_name)\n assert tr.optimizer is not None\n assert tr.scheduler is not None\n assert tr.epochs_trained == num_epochs\n\n for i in range(1, num_epochs+1):\n model_name = each_name % i\n assert os.path.exists(model_name)\n tr = trainer.Trainer.load(model_name, load_optimizer=True, foundation_cache=retag_pipeline.foundation_cache, peft_name=trained_model.model.peft_name)\n assert tr.epochs_trained == i\n assert tr.batches_trained == (4 * i if use_silver else 2 * i)\n\n return args, trained_model\n\n def test_train(self, wordvec_pretrain_file):\n \"\"\"\n Test the whole thing for a few iterations on the fake data\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tmpdirname:\n self.run_train_test(wordvec_pretrain_file, tmpdirname)\n\n def test_early_dropout(self, wordvec_pretrain_file):\n \"\"\"\n Test the whole thing for a few iterations on the fake data\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tmpdirname:\n args = ['--early_dropout', '3']\n _, model = self.run_train_test(wordvec_pretrain_file, tmpdirname, num_epochs=6, extra_args=args)\n model = model.model\n dropouts = [(name, module) for name, module in model.named_children() if isinstance(module, nn.Dropout)]\n assert len(dropouts) > 0, \"Didn't find any dropouts in the model!\"\n for name, module in dropouts:\n assert module.p == 0.0, \"Dropout module %s was not set to 0 with early_dropout\"\n\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tmpdirname:\n # test that when turned off, early_dropout doesn't happen\n args = ['--early_dropout', '-1']\n _, model = self.run_train_test(wordvec_pretrain_file, tmpdirname, num_epochs=6, extra_args=args)\n model = model.model\n dropouts = [(name, module) for name, module in model.named_children() if isinstance(module, nn.Dropout)]\n assert len(dropouts) > 0, \"Didn't find any dropouts in the model!\"\n if all(module.p == 0.0 for _, module in dropouts):\n raise AssertionError(\"All dropouts were 0 after training even though early_dropout was set to -1\")\n\n def test_train_silver(self, wordvec_pretrain_file):\n \"\"\"\n Test the whole thing for a few iterations on the fake data\n\n This tests that it works if you give it a silver file\n The check for the use of the silver data is that the\n number of batches trained should go up\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tmpdirname:\n self.run_train_test(wordvec_pretrain_file, tmpdirname, use_silver=True)\n\n def test_train_checkpoint(self, wordvec_pretrain_file):\n \"\"\"\n Test the whole thing for a few iterations, then restart\n\n This tests that the 5th iteration save file is not rewritten\n and that the iterations continue to 10\n\n TODO: could make it more robust by verifying that only 5 more\n epochs are trained. Perhaps a \"most recent epochs\" could be\n saved in the trainer\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tmpdirname:\n args, _ = self.run_train_test(wordvec_pretrain_file, tmpdirname, use_silver=False)\n save_5 = args['save_each_name'] % 5\n save_10 = args['save_each_name'] % 10\n assert os.path.exists(save_5)\n assert not os.path.exists(save_10)\n\n save_5_stat = pathlib.Path(save_5).stat()\n\n self.run_train_test(wordvec_pretrain_file, tmpdirname, num_epochs=10, use_silver=False, exists_ok=True)\n assert os.path.exists(save_5)\n assert os.path.exists(save_10)\n\n assert pathlib.Path(save_5).stat().st_mtime == save_5_stat.st_mtime\n\n def run_multistage_tests(self, wordvec_pretrain_file, tmpdirname, use_lattn, extra_args=None):\n train_treebank_file, eval_treebank_file = self.write_treebanks(tmpdirname)\n args = ['--multistage', '--pattn_num_layers', '1']\n if use_lattn:\n args += ['--lattn_d_proj', '16']\n if extra_args:\n args += extra_args\n args, _ = self.run_train_test(wordvec_pretrain_file, tmpdirname, num_epochs=8, extra_args=args)\n each_name = os.path.join(args['save_dir'], 'each_%02d.pt')\n\n word_input_sizes = defaultdict(list)\n for i in range(1, 9):\n model_name = each_name % i\n assert os.path.exists(model_name)\n tr = trainer.Trainer.load(model_name, load_optimizer=True)\n assert tr.epochs_trained == i\n word_input_sizes[tr.model.word_input_size].append(i)\n if use_lattn:\n # there should be three stages: no attn, pattn, pattn+lattn\n assert len(word_input_sizes) == 3\n word_input_keys = sorted(word_input_sizes.keys())\n assert word_input_sizes[word_input_keys[0]] == [1, 2, 3]\n assert word_input_sizes[word_input_keys[1]] == [4, 5]\n assert word_input_sizes[word_input_keys[2]] == [6, 7, 8]\n else:\n # with no lattn, there are two stages: no attn, pattn\n assert len(word_input_sizes) == 2\n word_input_keys = sorted(word_input_sizes.keys())\n assert word_input_sizes[word_input_keys[0]] == [1, 2, 3]\n assert word_input_sizes[word_input_keys[1]] == [4, 5, 6, 7, 8]\n\n def test_multistage_lattn(self, wordvec_pretrain_file):\n \"\"\"\n Test a multistage training for a few iterations on the fake data\n\n This should start with no pattn or lattn, have pattn in the middle, then lattn at the end\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tmpdirname:\n self.run_multistage_tests(wordvec_pretrain_file, tmpdirname, use_lattn=True)\n\n def test_multistage_no_lattn(self, wordvec_pretrain_file):\n \"\"\"\n Test a multistage training for a few iterations on the fake data\n\n This should start with no pattn or lattn, have pattn in the middle, then lattn at the end\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tmpdirname:\n self.run_multistage_tests(wordvec_pretrain_file, tmpdirname, use_lattn=False)\n\n def test_multistage_optimizer(self, wordvec_pretrain_file):\n \"\"\"\n Test that the correct optimizers are built for a multistage training process\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tmpdirname:\n extra_args = ['--optim', 'adamw']\n self.run_multistage_tests(wordvec_pretrain_file, tmpdirname, use_lattn=False, extra_args=extra_args)\n\n # check that the optimizers which get rebuilt when loading\n # the models are adadelta for the first half of the\n # multistage, then adamw\n each_name = os.path.join(tmpdirname, 'each_%02d.pt')\n for i in range(1, 3):\n model_name = each_name % i\n tr = trainer.Trainer.load(model_name, load_optimizer=True)\n assert tr.epochs_trained == i\n assert isinstance(tr.optimizer, optim.Adadelta)\n # double check that this is actually a valid test\n assert not isinstance(tr.optimizer, optim.AdamW)\n\n for i in range(4, 8):\n model_name = each_name % i\n tr = trainer.Trainer.load(model_name, load_optimizer=True)\n assert tr.epochs_trained == i\n assert isinstance(tr.optimizer, optim.AdamW)\n\n\n def test_grad_clip_hooks(self, wordvec_pretrain_file):\n \"\"\"\n Verify that grad clipping is not saved with the model, but is attached at training time\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tmpdirname:\n args = ['--grad_clipping', '25']\n self.run_train_test(wordvec_pretrain_file, tmpdirname, extra_args=args)\n\n def test_analyze_trees(self, wordvec_pretrain_file):\n test_str = \"(ROOT (S (NP (PRP I)) (VP (VBP wan) (S (VP (TO na) (VP (VB lick) (NP (NP (NNP Sh'reyan) (POS 's)) (NNS antennae)))))))) (ROOT (S (NP (DT This) (NN interface)) (VP (VBZ sucks))))\"\n\n test_tree = tree_reader.read_trees(test_str)\n assert len(test_tree) == 2\n\n args = ['--pattn_num_layers', '0', '--lattn_d_proj', '0', '--hidden_size', '20', '--delta_embedding_dim', '10']\n tr = build_trainer(wordvec_pretrain_file, *args)\n\n results = tr.model.analyze_trees(test_tree)\n assert len(results) == 2\n assert len(results[0].predictions) == 1\n assert results[0].predictions[0].tree == test_tree[0]\n assert results[0].state is not None\n assert isinstance(results[0].state.score, torch.Tensor)\n assert results[0].state.score.shape == torch.Size([])\n assert len(results[0].constituents) == 9\n assert results[0].constituents[-1].value == test_tree[0]\n # the way the results are built, the next-to-last entry\n # should be the thing just below the root\n assert results[0].constituents[-2].value == test_tree[0].children[0]\n\n assert len(results[1].predictions) == 1\n assert results[1].predictions[0].tree == test_tree[1]\n assert results[1].state is not None\n assert isinstance(results[1].state.score, torch.Tensor)\n assert results[1].state.score.shape == torch.Size([])\n assert len(results[1].constituents) == 4\n assert results[1].constituents[-1].value == test_tree[1]\n assert results[1].constituents[-2].value == test_tree[1].children[0]\n\n def bert_weights_allclose(self, bert_model, parser_model):\n \"\"\"\n Return True if all bert weights are close, False otherwise\n \"\"\"\n for name, parameter in bert_model.named_parameters():\n other_name = \"bert_model.\" + name\n other_parameter = parser_model.model.get_parameter(other_name)\n if not torch.allclose(parameter.cpu(), other_parameter.cpu()):\n return False\n return True\n\n def frozen_transformer_test(self, wordvec_pretrain_file, transformer_name):\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tmpdirname:\n foundation_cache = FoundationCache()\n args = ['--bert_model', transformer_name]\n args, trained_model = self.run_train_test(wordvec_pretrain_file, tmpdirname, extra_args=args, foundation_cache=foundation_cache)\n bert_model, bert_tokenizer = foundation_cache.load_bert(transformer_name)\n assert self.bert_weights_allclose(bert_model, trained_model)\n\n checkpoint = torch.load(args['save_name'], lambda storage, loc: storage, weights_only=True)\n params = checkpoint['params']\n # check that the bert model wasn't saved in the model\n assert all(not x.startswith(\"bert_model.\") for x in params['model'].keys())\n # make sure we're looking at the right thing\n assert any(x.startswith(\"output_layers.\") for x in params['model'].keys())\n\n # check that the cached model is used as expected when loading a bert model\n trained_model = trainer.Trainer.load(args['save_name'], foundation_cache=foundation_cache)\n assert trained_model.model.bert_model is bert_model\n\n def test_bert_frozen(self, wordvec_pretrain_file):\n \"\"\"\n Check that the parameters of the bert model don't change when training a basic model\n \"\"\"\n self.frozen_transformer_test(wordvec_pretrain_file, 'hf-internal-testing/tiny-bert')\n\n def test_xlnet_frozen(self, wordvec_pretrain_file, tiny_random_xlnet):\n \"\"\"\n Check that the parameters of an xlnet model don't change when training a basic model\n \"\"\"\n self.frozen_transformer_test(wordvec_pretrain_file, tiny_random_xlnet)\n\n def test_bart_frozen(self, wordvec_pretrain_file, tiny_random_bart):\n \"\"\"\n Check that the parameters of an xlnet model don't change when training a basic model\n \"\"\"\n self.frozen_transformer_test(wordvec_pretrain_file, tiny_random_bart)\n\n def test_bert_finetune_one_epoch(self, wordvec_pretrain_file):\n \"\"\"\n Check that the parameters the bert model DO change over a single training step\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tmpdirname:\n transformer_name = 'hf-internal-testing/tiny-bert'\n args = ['--bert_model', transformer_name, '--bert_finetune', '--optim', 'adadelta']\n args, trained_model = self.run_train_test(wordvec_pretrain_file, tmpdirname, num_epochs=1, extra_args=args)\n\n # check that the weights are different\n foundation_cache = FoundationCache()\n bert_model, bert_tokenizer = foundation_cache.load_bert(transformer_name)\n assert not self.bert_weights_allclose(bert_model, trained_model)\n\n # double check that a new bert is created instead of using the FoundationCache when the bert has been trained\n model_name = args['save_name']\n assert os.path.exists(model_name)\n no_finetune_args = self.training_args(wordvec_pretrain_file, tmpdirname, None, None, \"--no_bert_finetune\", \"--no_stage1_bert_finetune\", '--bert_model', transformer_name)\n tr = trainer.Trainer.load(model_name, args=no_finetune_args, foundation_cache=foundation_cache)\n assert tr.model.bert_model is not bert_model\n assert not self.bert_weights_allclose(bert_model, tr)\n assert self.bert_weights_allclose(trained_model.model.bert_model, tr)\n\n new_save_name = os.path.join(tmpdirname, \"test_resave_bert.pt\")\n assert not os.path.exists(new_save_name)\n tr.save(new_save_name, save_optimizer=False)\n tr2 = trainer.Trainer.load(new_save_name, args=no_finetune_args, foundation_cache=foundation_cache)\n # check that the resaved model included its finetuned bert weights\n assert tr2.model.bert_model is not bert_model\n # the finetuned bert weights should also be scheduled for saving the next time as well\n assert not tr2.model.is_unsaved_module(\"bert_model\")\n\n def finetune_transformer_test(self, wordvec_pretrain_file, transformer_name):\n \"\"\"\n Check that the parameters of the transformer DO change when using bert_finetune\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tmpdirname:\n args = ['--bert_model', transformer_name, '--bert_finetune', '--optim', 'adamw']\n args, trained_model = self.run_train_test(wordvec_pretrain_file, tmpdirname, extra_args=args)\n\n # check that the weights are different\n foundation_cache = FoundationCache()\n bert_model, bert_tokenizer = foundation_cache.load_bert(transformer_name)\n assert not self.bert_weights_allclose(bert_model, trained_model)\n\n # double check that a new bert is created instead of using the FoundationCache when the bert has been trained\n no_finetune_args = self.training_args(wordvec_pretrain_file, tmpdirname, None, None, \"--no_bert_finetune\", \"--no_stage1_bert_finetune\", '--bert_model', transformer_name)\n trained_model = trainer.Trainer.load(args['save_name'], args=no_finetune_args, foundation_cache=foundation_cache)\n assert not trained_model.model.args['bert_finetune']\n assert not trained_model.model.args['stage1_bert_finetune']\n assert trained_model.model.bert_model is not bert_model\n\n def test_bert_finetune(self, wordvec_pretrain_file):\n \"\"\"\n Check that the parameters of a bert model DO change when using bert_finetune\n \"\"\"\n self.finetune_transformer_test(wordvec_pretrain_file, 'hf-internal-testing/tiny-bert')\n\n def test_xlnet_finetune(self, wordvec_pretrain_file, tiny_random_xlnet):\n \"\"\"\n Check that the parameters of an xlnet model DO change when using bert_finetune\n \"\"\"\n self.finetune_transformer_test(wordvec_pretrain_file, tiny_random_xlnet)\n\n def test_stage1_bert_finetune(self, wordvec_pretrain_file):\n \"\"\"\n Check that the parameters the bert model DO change when using stage1_bert_finetune, but only for the first couple steps\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tmpdirname:\n bert_model_name = 'hf-internal-testing/tiny-bert'\n args = ['--bert_model', bert_model_name, '--stage1_bert_finetune', '--optim', 'adamw']\n # need to use num_epochs==6 so that epochs 1 and 2 are saved to be different\n # a test of 5 or less means that sometimes it will reload the params\n # at step 2 to get ready for the following iterations with adamw\n args, trained_model = self.run_train_test(wordvec_pretrain_file, tmpdirname, num_epochs=6, extra_args=args)\n\n # check that the weights are different\n foundation_cache = FoundationCache()\n bert_model, bert_tokenizer = foundation_cache.load_bert(bert_model_name)\n assert not self.bert_weights_allclose(bert_model, trained_model)\n\n # double check that a new bert is created instead of using the FoundationCache when the bert has been trained\n no_finetune_args = self.training_args(wordvec_pretrain_file, tmpdirname, None, None, \"--no_bert_finetune\", \"--no_stage1_bert_finetune\", '--bert_model', bert_model_name, '--optim', 'adamw')\n num_epochs = trained_model.model.args['epochs']\n each_name = os.path.join(tmpdirname, 'each_%02d.pt')\n for i in range(1, num_epochs+1):\n model_name = each_name % i\n assert os.path.exists(model_name)\n tr = trainer.Trainer.load(model_name, args=no_finetune_args, foundation_cache=foundation_cache)\n assert tr.model.bert_model is not bert_model\n assert not self.bert_weights_allclose(bert_model, tr)\n if i >= num_epochs // 2:\n assert self.bert_weights_allclose(trained_model.model.bert_model, tr)\n\n # verify that models 1 and 2 are saved to be different\n model_name_1 = each_name % 1\n model_name_2 = each_name % 2\n tr_1 = trainer.Trainer.load(model_name_1, args=no_finetune_args, foundation_cache=foundation_cache)\n tr_2 = trainer.Trainer.load(model_name_2, args=no_finetune_args, foundation_cache=foundation_cache)\n assert not self.bert_weights_allclose(tr_1.model.bert_model, tr_2)\n\n\n def one_layer_finetune_transformer_test(self, wordvec_pretrain_file, transformer_name):\n \"\"\"\n Check that the parameters the bert model DO change when using bert_finetune\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tmpdirname:\n args = ['--bert_model', transformer_name, '--bert_finetune', '--bert_finetune_layers', '1', '--optim', 'adamw', '--bert_finetune_layers', '1']\n args, trained_model = self.run_train_test(wordvec_pretrain_file, tmpdirname, extra_args=args)\n\n # check that the weights of the last layer are different,\n # but the weights of the earlier layers and\n # non-transformer-layers are the same\n foundation_cache = FoundationCache()\n bert_model, bert_tokenizer = foundation_cache.load_bert(transformer_name)\n assert bert_model.config.num_hidden_layers > 1\n layer_name = \"layer.%d.\" % (bert_model.config.num_hidden_layers - 1)\n for name, parameter in bert_model.named_parameters():\n other_name = \"bert_model.\" + name\n other_parameter = trained_model.model.get_parameter(other_name)\n if layer_name in name:\n if 'rel_attn.seg_embed' in name or 'rel_attn.r_s_bias' in name:\n # not sure why this happens for xlnet, just roll with it\n continue\n assert not torch.allclose(parameter.cpu(), other_parameter.cpu())\n else:\n assert torch.allclose(parameter.cpu(), other_parameter.cpu())\n\n def test_bert_finetune_one_layer(self, wordvec_pretrain_file):\n self.one_layer_finetune_transformer_test(wordvec_pretrain_file, 'hf-internal-testing/tiny-bert')\n\n def test_xlnet_finetune_one_layer(self, wordvec_pretrain_file, tiny_random_xlnet):\n self.one_layer_finetune_transformer_test(wordvec_pretrain_file, tiny_random_xlnet)\n\n def test_peft_finetune(self, tmp_path, wordvec_pretrain_file):\n transformer_name = 'hf-internal-testing/tiny-bert'\n args = ['--bert_model', transformer_name, '--bert_finetune', '--optim', 'adamw', '--use_peft']\n args, trained_model = self.run_train_test(wordvec_pretrain_file, str(tmp_path), extra_args=args)\n\n def test_peft_twostage_finetune(self, wordvec_pretrain_file):\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as tmpdirname:\n num_epochs = 6\n transformer_name = 'hf-internal-testing/tiny-bert'\n args = ['--bert_model', transformer_name, '--stage1_bert_finetune', '--optim', 'adamw', '--use_peft']\n args, trained_model = self.run_train_test(wordvec_pretrain_file, tmpdirname, num_epochs=num_epochs, extra_args=args)\n for epoch in range(num_epochs):\n filename_prev = args['save_each_name'] % epoch\n filename_next = args['save_each_name'] % (epoch+1)\n trainer_prev = trainer.Trainer.load(filename_prev, args=args, load_optimizer=False)\n trainer_next = trainer.Trainer.load(filename_next, args=args, load_optimizer=False)\n\n lora_names = [name for name, _ in trainer_prev.model.bert_model.named_parameters() if name.find(\"lora\") >= 0]\n if epoch < 2:\n assert not any(torch.allclose(trainer_prev.model.bert_model.get_parameter(name).cpu(),\n trainer_next.model.bert_model.get_parameter(name).cpu())\n for name in lora_names)\n elif epoch > 2:\n assert all(torch.allclose(trainer_prev.model.bert_model.get_parameter(name).cpu(),\n trainer_next.model.bert_model.get_parameter(name).cpu())\n for name in lora_names)\n"
},
{
"path": "stanza/tests/constituency/test_transformer_tree_stack.py",
"content": "import pytest\n\nimport torch\n\nfrom stanza.models.constituency.transformer_tree_stack import TransformerTreeStack\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\ndef test_initial_state():\n \"\"\"\n Test that the initial state has the expected shapes\n \"\"\"\n ts = TransformerTreeStack(3, 5, 0.0)\n initial = ts.initial_state()\n assert len(initial) == 1\n assert initial.value.output.shape == torch.Size([5])\n assert initial.value.key_stack.shape == torch.Size([1, 5])\n assert initial.value.value_stack.shape == torch.Size([1, 5])\n\ndef test_output():\n \"\"\"\n Test that you can get an expected output shape from the TTS\n \"\"\"\n ts = TransformerTreeStack(3, 5, 0.0)\n initial = ts.initial_state()\n out = ts.output(initial)\n assert out.shape == torch.Size([5])\n assert torch.allclose(initial.value.output, out)\n\ndef test_push_state_single():\n \"\"\"\n Test that stacks are being updated correctly when using a single stack\n\n Values of the attention are not verified, though\n \"\"\"\n ts = TransformerTreeStack(3, 5, 0.0)\n initial = ts.initial_state()\n rand_input = torch.randn(1, 3)\n stacks = ts.push_states([initial], [\"A\"], rand_input)\n stacks = ts.push_states(stacks, [\"B\"], rand_input)\n assert len(stacks) == 1\n assert len(stacks[0]) == 3\n assert stacks[0].value.value == \"B\"\n assert stacks[0].pop().value.value == \"A\"\n assert stacks[0].pop().pop().value.value is None\n\ndef test_push_state_same_length():\n \"\"\"\n Test that stacks are being updated correctly when using 3 stacks of the same length\n\n Values of the attention are not verified, though\n \"\"\"\n ts = TransformerTreeStack(3, 5, 0.0)\n initial = ts.initial_state()\n rand_input = torch.randn(3, 3)\n stacks = ts.push_states([initial, initial, initial], [\"A\", \"A\", \"A\"], rand_input)\n stacks = ts.push_states(stacks, [\"B\", \"B\", \"B\"], rand_input)\n stacks = ts.push_states(stacks, [\"C\", \"C\", \"C\"], rand_input)\n assert len(stacks) == 3\n for s in stacks:\n assert len(s) == 4\n assert s.value.key_stack.shape == torch.Size([4, 5])\n assert s.value.value_stack.shape == torch.Size([4, 5])\n assert s.value.value == \"C\"\n assert s.pop().value.value == \"B\"\n assert s.pop().pop().value.value == \"A\"\n assert s.pop().pop().pop().value.value is None\n\ndef test_push_state_different_length():\n \"\"\"\n Test what happens if stacks of different lengths are passed in\n \"\"\"\n ts = TransformerTreeStack(3, 5, 0.0)\n initial = ts.initial_state()\n rand_input = torch.randn(2, 3)\n one_step = ts.push_states([initial], [\"A\"], rand_input[0:1, :])[0]\n stacks = [one_step, initial]\n stacks = ts.push_states(stacks, [\"B\", \"C\"], rand_input)\n assert len(stacks) == 2\n assert len(stacks[0]) == 3\n assert len(stacks[1]) == 2\n assert stacks[0].pop().value.value == 'A'\n assert stacks[0].value.value == 'B'\n assert stacks[1].value.value == 'C'\n\n assert stacks[0].value.key_stack.shape == torch.Size([3, 5])\n assert stacks[1].value.key_stack.shape == torch.Size([2, 5])\n\ndef test_mask():\n \"\"\"\n Test that a mask prevents the softmax from picking up unwanted values\n \"\"\"\n ts = TransformerTreeStack(3, 5, 0.0)\n\n random_v = torch.tensor([[[0.1, 0.2, 0.3, 0.4, 0.5]]])\n double_v = random_v * 2\n value = torch.cat([random_v, double_v], axis=1)\n random_k = torch.randn(1, 1, 5)\n key = torch.cat([random_k, random_k], axis=1)\n query = torch.randn(1, 5)\n\n output = ts.attention(key, query, value)\n # when the two keys are equal, we expect the attention to be 50/50\n expected_output = (random_v + double_v) / 2\n assert torch.allclose(output, expected_output)\n\n # If the first entry is masked out, the second one should be the\n # only one represented\n mask = torch.zeros(1, 2, dtype=torch.bool)\n mask[0][0] = True\n output = ts.attention(key, query, value, mask)\n assert torch.allclose(output, double_v)\n\n # If the second entry is masked out, the first one should be the\n # only one represented\n mask = torch.zeros(1, 2, dtype=torch.bool)\n mask[0][1] = True\n output = ts.attention(key, query, value, mask)\n assert torch.allclose(output, random_v)\n\ndef test_position():\n \"\"\"\n Test that nothing goes horribly wrong when position encodings are used\n\n Does not actually test the results of the encodings\n \"\"\"\n ts = TransformerTreeStack(4, 5, 0.0, use_position=True)\n initial = ts.initial_state()\n assert len(initial) == 1\n assert initial.value.output.shape == torch.Size([5])\n assert initial.value.key_stack.shape == torch.Size([1, 5])\n assert initial.value.value_stack.shape == torch.Size([1, 5])\n\n rand_input = torch.randn(2, 4)\n one_step = ts.push_states([initial], [\"A\"], rand_input[0:1, :])[0]\n stacks = [one_step, initial]\n stacks = ts.push_states(stacks, [\"B\", \"C\"], rand_input)\n\ndef test_length_limit():\n \"\"\"\n Test that the length limit drops nodes as the length limit is exceeded\n \"\"\"\n ts = TransformerTreeStack(4, 5, 0.0, length_limit = 2)\n initial = ts.initial_state()\n assert len(initial) == 1\n assert initial.value.output.shape == torch.Size([5])\n assert initial.value.key_stack.shape == torch.Size([1, 5])\n assert initial.value.value_stack.shape == torch.Size([1, 5])\n\n data = torch.tensor([[0.1, 0.2, 0.3, 0.4]])\n stacks = ts.push_states([initial], [\"A\"], data)\n\n stacks = ts.push_states(stacks, [\"B\"], data)\n assert len(stacks) == 1\n assert len(stacks[0]) == 3\n assert stacks[0].value.key_stack.shape[0] == 3\n assert stacks[0].value.value_stack.shape[0] == 3\n\n stacks = ts.push_states(stacks, [\"C\"], data)\n assert len(stacks) == 1\n assert len(stacks[0]) == 4\n assert stacks[0].value.key_stack.shape[0] == 3\n assert stacks[0].value.value_stack.shape[0] == 3\n\n stacks = ts.push_states(stacks, [\"D\"], data)\n assert len(stacks) == 1\n assert len(stacks[0]) == 5\n assert stacks[0].value.key_stack.shape[0] == 3\n assert stacks[0].value.value_stack.shape[0] == 3\n\ndef test_two_heads():\n \"\"\"\n Test that the length limit drops nodes as the length limit is exceeded\n \"\"\"\n ts = TransformerTreeStack(4, 6, 0.0, num_heads = 2)\n initial = ts.initial_state()\n assert len(initial) == 1\n assert initial.value.output.shape == torch.Size([6])\n assert initial.value.key_stack.shape == torch.Size([1, 6])\n assert initial.value.value_stack.shape == torch.Size([1, 6])\n\n rand_input = torch.randn(2, 4)\n one_step = ts.push_states([initial], [\"A\"], rand_input[0:1, :])[0]\n stacks = [one_step, initial]\n stacks = ts.push_states(stacks, [\"B\", \"C\"], rand_input)\n assert len(stacks) == 2\n assert len(stacks[0]) == 3\n assert len(stacks[1]) == 2\n assert stacks[0].pop().value.value == 'A'\n assert stacks[0].value.value == 'B'\n assert stacks[1].value.value == 'C'\n\n assert stacks[0].value.key_stack.shape == torch.Size([3, 6])\n assert stacks[1].value.key_stack.shape == torch.Size([2, 6])\n\n"
},
{
"path": "stanza/tests/constituency/test_transition_sequence.py",
"content": "import pytest\nfrom stanza.models.constituency import parse_transitions\nfrom stanza.models.constituency import transition_sequence\nfrom stanza.models.constituency import tree_reader\nfrom stanza.models.constituency.base_model import SimpleModel, UNARY_LIMIT\nfrom stanza.models.constituency.parse_transitions import *\n\nfrom stanza.tests import *\nfrom stanza.tests.constituency.test_parse_tree import CHINESE_LONG_LIST_TREE\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\ndef reconstruct_tree(tree, sequence, transition_scheme=TransitionScheme.IN_ORDER, unary_limit=UNARY_LIMIT, reverse=False):\n \"\"\"\n Starting from a tree and a list of transitions, build the tree caused by the transitions\n \"\"\"\n model = SimpleModel(transition_scheme=transition_scheme, unary_limit=unary_limit, reverse_sentence=reverse)\n states = model.initial_state_from_gold_trees([tree])\n assert(len(states)) == 1\n assert states[0].num_transitions == 0\n\n # TODO: could fold this into parse_sentences (similar to verify_transitions in trainer.py)\n for idx, t in enumerate(sequence):\n assert t.is_legal(states[0], model), \"Transition {} not legal at step {} in sequence {}\".format(t, idx, sequence)\n states = model.bulk_apply(states, [t])\n\n result_tree = states[0].constituents.value\n if reverse:\n result_tree = result_tree.reverse()\n return result_tree\n\ndef check_reproduce_tree(transition_scheme):\n text=\"((SBARQ (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (IN in) (NP (DT this) (NN seat))))) (. ?)))\"\n trees = tree_reader.read_trees(text)\n\n model = SimpleModel(transition_scheme)\n transitions = transition_sequence.build_sequence(trees[0], transition_scheme)\n states = model.initial_state_from_gold_trees(trees)\n assert(len(states)) == 1\n state = states[0]\n assert state.num_transitions == 0\n\n for t in transitions:\n assert t.is_legal(state, model)\n state = t.apply(state, model)\n\n # one item for the final tree\n # one item for the sentinel at the end\n assert len(state.constituents) == 2\n # the transition sequence should put all of the words\n # from the buffer onto the tree\n # one spot left for the sentinel value\n assert len(state.word_queue) == 8\n assert state.sentence_length == 6\n assert state.word_position == state.sentence_length\n assert len(state.transitions) == len(transitions) + 1\n\n result_tree = state.constituents.value\n assert result_tree == trees[0]\n\ndef test_top_down_unary():\n check_reproduce_tree(transition_scheme=TransitionScheme.TOP_DOWN_UNARY)\n\ndef test_top_down_no_unary():\n check_reproduce_tree(transition_scheme=TransitionScheme.TOP_DOWN)\n\ndef test_in_order():\n check_reproduce_tree(transition_scheme=TransitionScheme.IN_ORDER)\n\ndef test_in_order_compound():\n check_reproduce_tree(transition_scheme=TransitionScheme.IN_ORDER_COMPOUND)\n\ndef test_in_order_unary():\n check_reproduce_tree(transition_scheme=TransitionScheme.IN_ORDER_UNARY)\n\ndef test_all_transitions():\n text=\"((SBARQ (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (IN in) (NP (DT this) (NN seat))))) (. ?)))\"\n trees = tree_reader.read_trees(text)\n model = SimpleModel()\n transitions = transition_sequence.build_treebank(trees)\n\n expected = [Shift(), CloseConstituent(), CompoundUnary(\"ROOT\"), CompoundUnary(\"SQ\"), CompoundUnary(\"WHNP\"), OpenConstituent(\"NP\"), OpenConstituent(\"PP\"), OpenConstituent(\"SBARQ\"), OpenConstituent(\"VP\")]\n assert transition_sequence.all_transitions(transitions) == expected\n\n\ndef test_all_transitions_no_unary():\n text=\"((SBARQ (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (IN in) (NP (DT this) (NN seat))))) (. ?)))\"\n trees = tree_reader.read_trees(text)\n model = SimpleModel()\n transitions = transition_sequence.build_treebank(trees, transition_scheme=TransitionScheme.TOP_DOWN)\n\n expected = [Shift(), CloseConstituent(), OpenConstituent(\"NP\"), OpenConstituent(\"PP\"), OpenConstituent(\"ROOT\"), OpenConstituent(\"SBARQ\"), OpenConstituent(\"SQ\"), OpenConstituent(\"VP\"), OpenConstituent(\"WHNP\")]\n assert transition_sequence.all_transitions(transitions) == expected\n\ndef test_top_down_compound_unary():\n text = \"(ROOT (S (NP (DT The) (NNP Arizona) (NNPS Corporations) (NNP Commission)) (VP (VBD authorized) (NP (NP (DT an) (ADJP (CD 11.5)) (NN %) (NN rate) (NN increase)) (PP (IN at) (NP (NNP Tucson) (NNP Electric) (NNP Power) (NNP Co.))) (, ,) (UCP (ADJP (ADJP (RB substantially) (JJR lower)) (SBAR (IN than) (S (VP (VBN recommended) (NP (JJ last) (NN month)) (PP (IN by) (NP (DT a) (NN commission) (NN hearing) (NN officer))))))) (CC and) (NP (NP (QP (RB barely) (PDT half)) (DT the) (NN rise)) (VP (VBN sought) (PP (IN by) (NP (DT the) (NN utility)))))))) (. .)))\"\n\n trees = tree_reader.read_trees(text)\n assert len(trees) == 1\n\n model = SimpleModel()\n transitions = transition_sequence.build_sequence(trees[0], transition_scheme=TransitionScheme.TOP_DOWN_COMPOUND)\n\n states = model.initial_state_from_gold_trees(trees)\n assert len(states) == 1\n state = states[0]\n\n for t in transitions:\n assert t.is_legal(state, model)\n state = t.apply(state, model)\n\n result = model.get_top_constituent(state.constituents)\n assert trees[0] == result\n\n\ndef test_chinese_tree():\n trees = tree_reader.read_trees(CHINESE_LONG_LIST_TREE)\n\n transitions = transition_sequence.build_treebank(trees, transition_scheme=TransitionScheme.TOP_DOWN)\n redone = reconstruct_tree(trees[0], transitions[0], transition_scheme=TransitionScheme.TOP_DOWN)\n assert redone == trees[0]\n\n transitions = transition_sequence.build_treebank(trees, transition_scheme=TransitionScheme.IN_ORDER)\n with pytest.raises(AssertionError):\n redone = reconstruct_tree(trees[0], transitions[0], transition_scheme=TransitionScheme.IN_ORDER)\n\n redone = reconstruct_tree(trees[0], transitions[0], transition_scheme=TransitionScheme.IN_ORDER, unary_limit=6)\n assert redone == trees[0]\n\n\ndef test_chinese_tree_reversed():\n \"\"\"\n test that the reversed transitions also work\n \"\"\"\n trees = tree_reader.read_trees(CHINESE_LONG_LIST_TREE)\n\n transitions = transition_sequence.build_treebank(trees, transition_scheme=TransitionScheme.TOP_DOWN, reverse=True)\n redone = reconstruct_tree(trees[0], transitions[0], transition_scheme=TransitionScheme.TOP_DOWN, reverse=True)\n assert redone == trees[0]\n\n with pytest.raises(AssertionError):\n # turn off reverse - it should fail to rebuild the tree\n redone = reconstruct_tree(trees[0], transitions[0], transition_scheme=TransitionScheme.TOP_DOWN)\n assert redone == trees[0]\n\n transitions = transition_sequence.build_treebank(trees, transition_scheme=TransitionScheme.IN_ORDER, reverse=True)\n with pytest.raises(AssertionError):\n redone = reconstruct_tree(trees[0], transitions[0], transition_scheme=TransitionScheme.IN_ORDER, reverse=True)\n\n redone = reconstruct_tree(trees[0], transitions[0], transition_scheme=TransitionScheme.IN_ORDER, unary_limit=6, reverse=True)\n assert redone == trees[0]\n\n with pytest.raises(AssertionError):\n # turn off reverse - it should fail to rebuild the tree\n redone = reconstruct_tree(trees[0], transitions[0], transition_scheme=TransitionScheme.IN_ORDER, unary_limit=6)\n assert redone == trees[0]\n"
},
{
"path": "stanza/tests/constituency/test_tree_reader.py",
"content": "import pytest\nfrom stanza.models.constituency import tree_reader\nfrom stanza.models.constituency.tree_reader import MixedTreeError, UnclosedTreeError, UnlabeledTreeError\n\nfrom stanza.tests import *\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\ndef test_simple():\n \"\"\"\n Tests reading two simple trees from the same text\n \"\"\"\n text = \"(VB Unban) (NNP Opal)\"\n trees = tree_reader.read_trees(text)\n assert len(trees) == 2\n assert trees[0].is_preterminal()\n assert trees[0].label == 'VB'\n assert trees[0].children[0].label == 'Unban'\n assert trees[1].is_preterminal()\n assert trees[1].label == 'NNP'\n assert trees[1].children[0].label == 'Opal'\n\ndef test_newlines():\n \"\"\"\n The same test should work if there are newlines\n \"\"\"\n text = \"(VB Unban)\\n\\n(NNP Opal)\"\n trees = tree_reader.read_trees(text)\n assert len(trees) == 2\n\ndef test_parens():\n \"\"\"\n Parens should be escaped in the tree files and escaped when written\n \"\"\"\n text = \"(-LRB- -LRB-) (-RRB- -RRB-)\"\n trees = tree_reader.read_trees(text)\n assert len(trees) == 2\n\n assert trees[0].label == '-LRB-'\n assert trees[0].children[0].label == '('\n assert \"{}\".format(trees[0]) == '(-LRB- -LRB-)'\n\n assert trees[1].label == '-RRB-'\n assert trees[1].children[0].label == ')'\n assert \"{}\".format(trees[1]) == '(-RRB- -RRB-)'\n\ndef test_complicated():\n \"\"\"\n A more complicated tree that should successfully read\n \"\"\"\n text=\"( (SBARQ (WHNP (WP Who)) (SQ (VP (VBZ sits) (PP (IN in) (NP (DT this) (NN seat))))) (. ?)))\"\n trees = tree_reader.read_trees(text)\n assert len(trees) == 1\n tree = trees[0]\n assert not tree.is_leaf()\n assert not tree.is_preterminal()\n assert tree.label == 'ROOT'\n assert len(tree.children) == 1\n assert tree.children[0].label == 'SBARQ'\n assert len(tree.children[0].children) == 3\n assert [x.label for x in tree.children[0].children] == ['WHNP', 'SQ', '.']\n # etc etc\n\ndef test_one_word():\n \"\"\"\n Check that one node trees are correctly read\n\n probably not super relevant for the parsing use case\n \"\"\"\n text=\"(FOO) (BAR)\"\n trees = tree_reader.read_trees(text)\n assert len(trees) == 2\n\n assert trees[0].is_leaf()\n assert trees[0].label == 'FOO'\n\n assert trees[1].is_leaf()\n assert trees[1].label == 'BAR'\n\ndef test_missing_close_parens():\n \"\"\"\n Test the unclosed error condition\n \"\"\"\n text = \"(Foo) \\n (Bar \\n zzz\"\n try:\n trees = tree_reader.read_trees(text)\n raise AssertionError(\"Expected an exception\")\n except UnclosedTreeError as e:\n assert e.line_num == 1\n\ndef test_mixed_tree():\n \"\"\"\n Test the mixed error condition\n \"\"\"\n text = \"(Foo) \\n (Bar) \\n (Unban (Mox) Opal)\"\n try:\n trees = tree_reader.read_trees(text)\n raise AssertionError(\"Expected an exception\")\n except MixedTreeError as e:\n assert e.line_num == 2\n\n trees = tree_reader.read_trees(text, broken_ok=True)\n assert len(trees) == 3\n\ndef test_unlabeled_tree():\n \"\"\"\n Test the unlabeled error condition\n \"\"\"\n text = \"(ROOT ((Foo) (Bar)))\"\n try:\n trees = tree_reader.read_trees(text)\n raise AssertionError(\"Expected an exception\")\n except UnlabeledTreeError as e:\n assert e.line_num == 0\n\n trees = tree_reader.read_trees(text, broken_ok=True)\n assert len(trees) == 1\n\n \n"
},
{
"path": "stanza/tests/constituency/test_tree_stack.py",
"content": "import pytest\n\nfrom stanza.models.constituency.tree_stack import TreeStack\n\nfrom stanza.tests import *\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\ndef test_simple():\n stack = TreeStack(value=5, parent=None, length=1)\n stack = stack.push(3)\n stack = stack.push(1)\n\n expected_values = [1, 3, 5]\n for value in expected_values:\n assert stack.value == value\n stack = stack.pop()\n assert stack is None\n\ndef test_iter():\n stack = TreeStack(value=5, parent=None, length=1)\n stack = stack.push(3)\n stack = stack.push(1)\n\n stack_list = list(stack)\n assert list(stack) == [1, 3, 5]\n\ndef test_str():\n stack = TreeStack(value=5, parent=None, length=1)\n stack = stack.push(3)\n stack = stack.push(1)\n\n assert str(stack) == \"TreeStack(1, 3, 5)\"\n\ndef test_len():\n stack = TreeStack(value=5, parent=None, length=1)\n assert len(stack) == 1\n\n stack = stack.push(3)\n stack = stack.push(1)\n assert len(stack) == 3\n\ndef test_long_len():\n \"\"\"\n Original stack had a bug where this took exponential time...\n \"\"\"\n stack = TreeStack(value=0, parent=None, length=1)\n for i in range(1, 40):\n stack = stack.push(i)\n assert len(stack) == 40\n"
},
{
"path": "stanza/tests/constituency/test_utils.py",
"content": "import pytest\n\nfrom stanza import Pipeline\nfrom stanza.models.constituency import tree_reader\nfrom stanza.models.constituency import utils\n\nfrom stanza.tests import *\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\n\n@pytest.fixture(scope=\"module\")\ndef pipeline():\n return Pipeline(dir=TEST_MODELS_DIR, lang=\"en\", processors=\"tokenize, pos\", tokenize_pretokenized=True)\n\n\n\ndef test_xpos_retag(pipeline):\n \"\"\"\n Test using the English tagger that trees will be correctly retagged by read_trees using xpos\n \"\"\"\n text = \"((S (VP (X Find)) (NP (X Mox) (X Opal)))) ((S (NP (X Ragavan)) (VP (X steals) (NP (X important) (X cards)))))\"\n expected = \"((S (VP (VB Find)) (NP (NNP Mox) (NNP Opal)))) ((S (NP (NNP Ragavan)) (VP (VBZ steals) (NP (JJ important) (NNS cards)))))\"\n\n trees = tree_reader.read_trees(text)\n\n new_trees = utils.retag_trees(trees, [pipeline], xpos=True)\n assert new_trees == tree_reader.read_trees(expected)\n\n\n\ndef test_upos_retag(pipeline):\n \"\"\"\n Test using the English tagger that trees will be correctly retagged by read_trees using upos\n \"\"\"\n text = \"((S (VP (X Find)) (NP (X Mox) (X Opal)))) ((S (NP (X Ragavan)) (VP (X steals) (NP (X important) (X cards)))))\"\n expected = \"((S (VP (VERB Find)) (NP (PROPN Mox) (PROPN Opal)))) ((S (NP (PROPN Ragavan)) (VP (VERB steals) (NP (ADJ important) (NOUN cards)))))\"\n\n trees = tree_reader.read_trees(text)\n\n new_trees = utils.retag_trees(trees, [pipeline], xpos=False)\n assert new_trees == tree_reader.read_trees(expected)\n\n\ndef test_replace_tags():\n \"\"\"\n Test the underlying replace_tags method\n\n Also tests that the method throws exceptions when it is supposed to\n \"\"\"\n text = \"((S (VP (X Find)) (NP (X Mox) (X Opal))))\"\n expected = \"((S (VP (A Find)) (NP (B Mox) (C Opal))))\"\n\n trees = tree_reader.read_trees(text)\n\n new_tags = [\"A\", \"B\", \"C\"]\n new_tree = trees[0].replace_tags(new_tags)\n\n assert new_tree == tree_reader.read_trees(expected)[0]\n\n with pytest.raises(ValueError):\n new_tags = [\"A\", \"B\"]\n new_tree = trees[0].replace_tags(new_tags)\n\n with pytest.raises(ValueError):\n new_tags = [\"A\", \"B\", \"C\", \"D\"]\n new_tree = trees[0].replace_tags(new_tags)\n\n"
},
{
"path": "stanza/tests/constituency/test_vietnamese.py",
"content": "\"\"\"\nA few tests for Vietnamese parsing, which has some difficulties related to spaces in words\n\nTechnically some other languages can have this, too, like that one French token\n\"\"\"\n\nimport os\nimport tempfile\n\nimport pytest\n\nfrom stanza.models.common import pretrain\nfrom stanza.models.constituency import tree_reader\n\nfrom stanza.tests.constituency.test_trainer import build_trainer\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\nVI_TREEBANK = '(ROOT (S-TTL (NP (\" \") (N-H Đảo) (Np Đài Loan) (\" \") (PP (E-H ở) (NP (N-H đồng bằng) (NP (N-H sông) (Np Cửu Long))))) (. .)))'\n\nVI_TREEBANK_UNDERSCORE = '(ROOT (S-TTL (NP (\" \") (N-H Đảo) (Np Đài_Loan) (\" \") (PP (E-H ở) (NP (N-H đồng_bằng) (NP (N-H sông) (Np Cửu_Long))))) (. .)))'\n\nVI_TREEBANK_SIMPLE = '(ROOT (S (NP (\" \") (N Đảo) (Np Đài Loan) (\" \") (PP (E ở) (NP (N đồng bằng) (NP (N sông) (Np Cửu Long))))) (. .)))'\n\nVI_TREEBANK_PAREN = '(ROOT (S-TTL (NP (PUNCT -LRB-) (N-H Đảo) (Np Đài Loan) (PUNCT -RRB-) (PP (E-H ở) (NP (N-H đồng bằng) (NP (N-H sông) (Np Cửu Long))))) (. .)))'\nVI_TREEBANK_VLSP = '\\n(S-TTL (NP (PUNCT LBKT) (N-H Đảo) (Np Đài_Loan) (PUNCT RBKT) (PP (E-H ở) (NP (N-H đồng_bằng) (NP (N-H sông) (Np Cửu_Long))))) (. .))\\n '\nVI_TREEBANK_VLSP_50 = '\\n(S-TTL (NP (PUNCT LBKT) (N-H Đảo) (Np Đài_Loan) (PUNCT RBKT) (PP (E-H ở) (NP (N-H đồng_bằng) (NP (N-H sông) (Np Cửu_Long))))) (. .))\\n '\nVI_TREEBANK_VLSP_100 = '\\n(S-TTL (NP (PUNCT LBKT) (N-H Đảo) (Np Đài_Loan) (PUNCT RBKT) (PP (E-H ở) (NP (N-H đồng_bằng) (NP (N-H sông) (Np Cửu_Long))))) (. .))\\n '\n\nEXPECTED_LABELED_BRACKETS = '(_ROOT (_S (_NP (_\" \" )_\" (_N Đảo )_N (_Np Đài_Loan )_Np (_\" \" )_\" (_PP (_E ở )_E (_NP (_N đồng_bằng )_N (_NP (_N sông )_N (_Np Cửu_Long )_Np )_NP )_NP )_PP )_NP (_. . )_. )_S )_ROOT'\n\n\ndef test_read_vi_tree():\n \"\"\"\n Test that an individual tree with spaces in the leaves is being processed as we expect\n \"\"\"\n text = VI_TREEBANK.split(\"\\n\")[0]\n trees = tree_reader.read_trees(text)\n assert len(trees) == 1\n assert str(trees[0]) == text\n # this is the first NP\n # the third node of that NP, eg (Np Đài Loan)\n node = trees[0].children[0].children[0].children[2]\n assert node.is_preterminal()\n assert node.children[0].label == \"Đài Loan\"\n\nVI_EMBEDDING = \"\"\"\n4 4\nĐảo 0.11 0.21 0.31 0.41\nĐài Loan 0.12 0.22 0.32 0.42\nđồng bằng 0.13 0.23 0.33 0.43\nsông 0.14 0.24 0.34 0.44\n\"\"\".strip()\n\ndef test_vi_embedding():\n \"\"\"\n Test that a VI embedding's words are correctly found when processing trees\n \"\"\"\n text = VI_TREEBANK.split(\"\\n\")[0]\n trees = tree_reader.read_trees(text)\n words = set(trees[0].leaf_labels())\n\n with tempfile.TemporaryDirectory() as tempdir:\n emb_filename = os.path.join(tempdir, \"emb.txt\")\n pt_filename = os.path.join(tempdir, \"emb.pt\")\n with open(emb_filename, \"w\", encoding=\"utf-8\") as fout:\n fout.write(VI_EMBEDDING)\n pt = pretrain.Pretrain(filename=pt_filename, vec_filename=emb_filename, save_to_file=True)\n pt.load()\n\n trainer = build_trainer(pt_filename)\n model = trainer.model\n\n assert model.num_words_known(words) == 4\n\n\ndef test_space_formatting():\n \"\"\"\n By default, spaces are left as spaces, but there is a format option to change spaces\n \"\"\"\n text = VI_TREEBANK.split(\"\\n\")[0]\n trees = tree_reader.read_trees(text)\n assert len(trees) == 1\n assert str(trees[0]) == text\n\n assert \"{}\".format(trees[0]) == VI_TREEBANK\n assert \"{:_O}\".format(trees[0]) == VI_TREEBANK_UNDERSCORE\n\ndef test_vlsp_formatting():\n text = VI_TREEBANK_PAREN.split(\"\\n\")[0]\n trees = tree_reader.read_trees(text)\n assert len(trees) == 1\n assert str(trees[0]) == text\n\n assert \"{:_V}\".format(trees[0]) == VI_TREEBANK_VLSP\n trees[0].tree_id = 50\n assert \"{:_Vi}\".format(trees[0]) == VI_TREEBANK_VLSP_50\n trees[0].tree_id = 100\n assert \"{:_Vi}\".format(trees[0]) == VI_TREEBANK_VLSP_100\n\n empty = tree_reader.read_trees(\"(ROOT)\")[0]\n with pytest.raises(ValueError):\n \"{:V}\".format(empty)\n\n branches = tree_reader.read_trees(\"(ROOT (1) (2) (3))\")[0]\n with pytest.raises(ValueError):\n \"{:V}\".format(branches)\n\ndef test_language_formatting():\n \"\"\"\n Test turning the parse tree into a 'language' for GPT\n \"\"\"\n text = VI_TREEBANK.split(\"\\n\")[0]\n trees = tree_reader.read_trees(text)\n trees = [t.prune_none().simplify_labels() for t in trees]\n assert len(trees) == 1\n assert str(trees[0]) == VI_TREEBANK_SIMPLE\n\n text = \"{:L}\".format(trees[0])\n assert text == EXPECTED_LABELED_BRACKETS\n\n"
},
{
"path": "stanza/tests/datasets/__init__.py",
"content": ""
},
{
"path": "stanza/tests/datasets/coref/__init__.py",
"content": ""
},
{
"path": "stanza/tests/datasets/coref/test_hebrew_iahlt.py",
"content": "import pytest\n\nfrom stanza import Pipeline\nfrom stanza.tests import TEST_MODELS_DIR\nfrom stanza.utils.datasets.coref.convert_hebrew_iahlt import extract_doc\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\n@pytest.fixture(scope=\"module\")\ndef tokenizer():\n pipe = Pipeline(lang=\"he\", processors=\"tokenize\", dir=TEST_MODELS_DIR, download_method=None)\n return pipe\n\nTEXT = \"\"\"\n\n\n\nמבולבלים? גם אנחנו: למסעדנים והמלצרים יש עוד סימני שאלה על הטיפים\n\nהפער בין פסיקת בית הדין לעבודה לבין פסיקה קודמת של בג\"ץ, משאיר את הענף בחוסר וודאות, וה -1 בינואר כבר מעבר לפינה . \"מבחינתי , הייתי מוסיף לתפריט תוספת שירות של 17% \", אמר בעלים של מסעדה בשדרות\n\nברשות המיסים מסתפקים במסר עמום באשר לכוונותיהם לאור פסק דין הטיפים שצפוי להיכנס לתוקפו ב-1 בינואר . על פי פרשנותם המקצועית , הבהירו, יש מקום לחייב את כספי הטיפים במע\"מ , \"עם זאת, הרשות עדין בוחנת את הסוגיה וטרם התקבלה החלטה אופרטיבית בעניין \". ואיך אמורים המסעדנים להיערך בינתיים ליישום הפסיקה ולמחזור השנה הבאה ? ביום חמישי יפגשו אנשי ארגון 'מסעדנים חזקים ביחד' עם מנהל רשות המיסים ערן יעקב, וידרשו תשובות ברורות.\n\n\"אני עדיין לא מדבר עם העובדים שלי , ואני גם לא יודע איך להיערך החל מעוד שבועיים\", אמר ל'דבר ראשון' ניר שוחט, הבעלים של מסעדת סושי מוטו בשדרות ומוסיף כי יהיה קשה להתאים את הפסיקה למציאות בשטח . \"אף אחד לא יודע. יש המון סתירות – עורך הדין אומר דבר אחד ורואה החשבון דבר אחר. עדיין לא הצליחו להבין את החוק לאשורו \".\n\n\"מבחינתי , הייתי מוסיף לתפריט תוספת שירות של 17% . זה יגלם גם את המע\"מ והטיפים ומזה אני אשלם למלצרים . די כבר עם הטיפים האלה , מספיק.\"\n\"\"\"\n\nCLUSTER = {'metadata': {'name': 'המסעדנים', 'entity': 'person'}, 'mentions': [[28, 35, {}], [572, 581, {}]]}\n\ndef test_extract_doc(tokenizer):\n doc = {'text': TEXT,\n 'clusters': [CLUSTER],\n 'metadata': {\n 'doc_id': 'test'\n }\n }\n extracted = extract_doc(tokenizer, [doc])\n assert len(extracted) == 1\n assert len(extracted[0].coref_spans) == 2\n assert extracted[0].coref_spans[1] == [(0, 4, 4)]\n assert extracted[0].coref_spans[6] == [(0, 3, 4)]\n"
},
{
"path": "stanza/tests/datasets/ner/__init__.py",
"content": ""
},
{
"path": "stanza/tests/datasets/ner/test_prepare_ner_file.py",
"content": "\"\"\"\nTest some simple conversions of NER bio files\n\"\"\"\n\nimport pytest\n\nimport json\n\nfrom stanza.models.common.doc import Document\nfrom stanza.utils.datasets.ner.prepare_ner_file import process_dataset\n\nBIO_1 = \"\"\"\nJennifer\tB-PERSON\nSh'reyan\tI-PERSON\nhas\tO\nlovely\tO\nantennae\tO\n\"\"\".strip()\n\nBIO_2 = \"\"\"\nbut\tO\nI\tO\ndon't\tO\nlike\tO\nthe\tO\nway\tO\nJennifer\tB-PERSON\ntreated\tO\nBeckett\tB-PERSON\non\tO\nthe\tO\nCerritos\tB-LOCATION\n\"\"\".strip()\n\ndef check_json_file(doc, raw_text, expected_sentences, expected_tokens):\n raw_sentences = raw_text.strip().split(\"\\n\\n\")\n assert len(raw_sentences) == expected_sentences\n if isinstance(expected_tokens, int):\n expected_tokens = [expected_tokens]\n for raw_sentence, expected_len in zip(raw_sentences, expected_tokens):\n assert len(raw_sentence.strip().split(\"\\n\")) == expected_len\n\n assert len(doc.sentences) == expected_sentences\n for sentence, expected_len in zip(doc.sentences, expected_tokens):\n assert len(sentence.tokens) == expected_len\n for sentence, raw_sentence in zip(doc.sentences, raw_sentences):\n for token, line in zip(sentence.tokens, raw_sentence.strip().split(\"\\n\")):\n word, tag = line.strip().split()\n assert token.text == word\n assert token.ner == tag\n\ndef write_and_convert(tmp_path, raw_text):\n bio_file = tmp_path / \"test.bio\"\n with open(bio_file, \"w\", encoding=\"utf-8\") as fout:\n fout.write(raw_text)\n\n json_file = tmp_path / \"json.bio\"\n process_dataset(bio_file, json_file)\n\n with open(json_file) as fin:\n doc = Document(json.load(fin))\n\n return doc\n\ndef run_test(tmp_path, raw_text, expected_sentences, expected_tokens):\n doc = write_and_convert(tmp_path, raw_text)\n check_json_file(doc, raw_text, expected_sentences, expected_tokens)\n\ndef test_simple(tmp_path):\n run_test(tmp_path, BIO_1, 1, 5)\n\ndef test_ner_at_end(tmp_path):\n run_test(tmp_path, BIO_2, 1, 12)\n\ndef test_two_sentences(tmp_path):\n raw_text = BIO_1 + \"\\n\\n\" + BIO_2\n run_test(tmp_path, raw_text, 2, [5, 12])\n"
},
{
"path": "stanza/tests/datasets/ner/test_utils.py",
"content": "\"\"\"\nTest the utils file of the NER dataset processing\n\"\"\"\n\nimport pytest\n\nfrom stanza.utils.datasets.ner.utils import list_doc_entities\nfrom stanza.tests.datasets.ner.test_prepare_ner_file import BIO_1, BIO_2, write_and_convert\n\ndef test_list_doc_entities(tmp_path):\n \"\"\"\n Test the function which lists all of the entities in a doc\n \"\"\"\n doc = write_and_convert(tmp_path, BIO_1)\n entities = list_doc_entities(doc)\n expected = [(('Jennifer', \"Sh'reyan\"), 'PERSON')]\n assert expected == entities\n\n doc = write_and_convert(tmp_path, BIO_2)\n entities = list_doc_entities(doc)\n expected = [(('Jennifer',), 'PERSON'), (('Beckett',), 'PERSON'), (('Cerritos',), 'LOCATION')]\n assert expected == entities \n\n doc = write_and_convert(tmp_path, \"\\n\\n\".join([BIO_1, BIO_2]))\n entities = list_doc_entities(doc)\n expected = [(('Jennifer', \"Sh'reyan\"), 'PERSON'), (('Jennifer',), 'PERSON'), (('Beckett',), 'PERSON'), (('Cerritos',), 'LOCATION')]\n assert expected == entities\n\n doc = write_and_convert(tmp_path, \"\\n\\n\".join([BIO_1, BIO_1, BIO_2]))\n entities = list_doc_entities(doc)\n expected = [(('Jennifer', \"Sh'reyan\"), 'PERSON'), (('Jennifer', \"Sh'reyan\"), 'PERSON'), (('Jennifer',), 'PERSON'), (('Beckett',), 'PERSON'), (('Cerritos',), 'LOCATION')]\n assert expected == entities\n\n\n"
},
{
"path": "stanza/tests/datasets/test_common.py",
"content": "\"\"\"\nTest conllu manipulating routines in stanza/utils/dataset/common.py\n\"\"\"\n\nimport pytest\n\n\nfrom stanza.utils.datasets.common import maybe_add_fake_dependencies\n# from stanza.tests import *\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\nDEPS_EXAMPLE=\"\"\"\n# text = Sh'reyan's antennae are hella thicc\n1\tSh'reyan\tSh'reyan\tPROPN\tNNP\tNumber=Sing\t3\tnmod:poss\t3:nmod:poss\tSpaceAfter=No\n2\t's\t's\tPART\tPOS\t_\t1\tcase\t1:case\t_\n3\tantennae\tantenna\tNOUN\tNNS\tNumber=Plur\t6\tnsubj\t6:nsubj\t_\n4\tare\tbe\tVERB\tVBP\tMood=Ind|Tense=Pres|VerbForm=Fin\t6\tcop\t6:cop\t_\n5\thella\thella\tADV\tRB\t_\t6\tadvmod\t6:advmod\t_\n6\tthicc\tthicc\tADJ\tJJ\tDegree=Pos\t0\troot\t0:root\t_\n\"\"\".strip().split(\"\\n\")\n\n\nONLY_ROOT_EXAMPLE=\"\"\"\n# text = Sh'reyan's antennae are hella thicc\n1\tSh'reyan\tSh'reyan\tPROPN\tNNP\tNumber=Sing\t_\t_\t_\tSpaceAfter=No\n2\t's\t's\tPART\tPOS\t_\t_\t_\t_\t_\n3\tantennae\tantenna\tNOUN\tNNS\tNumber=Plur\t_\t_\t_\t_\n4\tare\tbe\tVERB\tVBP\tMood=Ind|Tense=Pres|VerbForm=Fin\t_\t_\t_\t_\n5\thella\thella\tADV\tRB\t_\t_\t_\t_\t_\n6\tthicc\tthicc\tADJ\tJJ\tDegree=Pos\t0\troot\t0:root\t_\n\"\"\".strip().split(\"\\n\")\n\nONLY_ROOT_EXPECTED=\"\"\"\n# text = Sh'reyan's antennae are hella thicc\n1\tSh'reyan\tSh'reyan\tPROPN\tNNP\tNumber=Sing\t6\tdep\t_\tSpaceAfter=No\n2\t's\t's\tPART\tPOS\t_\t1\tdep\t_\t_\n3\tantennae\tantenna\tNOUN\tNNS\tNumber=Plur\t1\tdep\t_\t_\n4\tare\tbe\tVERB\tVBP\tMood=Ind|Tense=Pres|VerbForm=Fin\t1\tdep\t_\t_\n5\thella\thella\tADV\tRB\t_\t1\tdep\t_\t_\n6\tthicc\tthicc\tADJ\tJJ\tDegree=Pos\t0\troot\t0:root\t_\n\"\"\".strip().split(\"\\n\")\n\nNO_DEPS_EXAMPLE=\"\"\"\n# text = Sh'reyan's antennae are hella thicc\n1\tSh'reyan\tSh'reyan\tPROPN\tNNP\tNumber=Sing\t_\t_\t_\tSpaceAfter=No\n2\t's\t's\tPART\tPOS\t_\t_\t_\t_\t_\n3\tantennae\tantenna\tNOUN\tNNS\tNumber=Plur\t_\t_\t_\t_\n4\tare\tbe\tVERB\tVBP\tMood=Ind|Tense=Pres|VerbForm=Fin\t_\t_\t_\t_\n5\thella\thella\tADV\tRB\t_\t_\t_\t_\t_\n6\tthicc\tthicc\tADJ\tJJ\tDegree=Pos\t_\t_\t_\t_\n\"\"\".strip().split(\"\\n\")\n\nNO_DEPS_EXPECTED=\"\"\"\n# text = Sh'reyan's antennae are hella thicc\n1\tSh'reyan\tSh'reyan\tPROPN\tNNP\tNumber=Sing\t0\troot\t_\tSpaceAfter=No\n2\t's\t's\tPART\tPOS\t_\t1\tdep\t_\t_\n3\tantennae\tantenna\tNOUN\tNNS\tNumber=Plur\t1\tdep\t_\t_\n4\tare\tbe\tVERB\tVBP\tMood=Ind|Tense=Pres|VerbForm=Fin\t1\tdep\t_\t_\n5\thella\thella\tADV\tRB\t_\t1\tdep\t_\t_\n6\tthicc\tthicc\tADJ\tJJ\tDegree=Pos\t1\tdep\t_\t_\n\"\"\".strip().split(\"\\n\")\n\n\ndef test_fake_deps_no_change():\n result = maybe_add_fake_dependencies(DEPS_EXAMPLE)\n assert result == DEPS_EXAMPLE\n\ndef test_fake_deps_all_tokens():\n result = maybe_add_fake_dependencies(NO_DEPS_EXAMPLE)\n assert result == NO_DEPS_EXPECTED\n\n\ndef test_fake_deps_only_root():\n result = maybe_add_fake_dependencies(ONLY_ROOT_EXAMPLE)\n assert result == ONLY_ROOT_EXPECTED\n"
},
{
"path": "stanza/tests/datasets/test_vietnamese_renormalization.py",
"content": "import pytest\nimport os\n\nfrom stanza.utils.datasets.vietnamese import renormalize\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\ndef test_replace_all():\n text = \"SỌAmple tụy test file\"\n expected = \"SOẠmple tuỵ test file\"\n\n assert renormalize.replace_all(text) == expected\n\ndef test_replace_file(tmp_path):\n text = \"SỌAmple tụy test file\"\n expected = \"SOẠmple tuỵ test file\"\n\n orig = tmp_path / \"orig.txt\"\n converted = tmp_path / \"converted.txt\"\n\n with open(orig, \"w\", encoding=\"utf-8\") as fout:\n for i in range(10):\n fout.write(text)\n fout.write(\"\\n\")\n\n renormalize.convert_file(orig, converted)\n\n assert os.path.exists(converted)\n with open(converted, encoding=\"utf-8\") as fin:\n lines = fin.readlines()\n\n assert len(lines) == 10\n for i in lines:\n assert i.strip() == expected\n \n"
},
{
"path": "stanza/tests/depparse/__init__.py",
"content": ""
},
{
"path": "stanza/tests/depparse/test_depparse_data.py",
"content": "\"\"\"\nTest some pieces of the depparse dataloader\n\"\"\"\nimport pytest\nfrom stanza.models import parser\nfrom stanza.models.depparse.data import data_to_batches, DataLoader\nfrom stanza.utils.conll import CoNLL\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\ndef make_fake_data(*lengths):\n data = []\n for i, length in enumerate(lengths):\n word = chr(ord('A') + i)\n chunk = [[word] * length]\n data.append(chunk)\n return data\n\ndef check_batches(batched_data, expected_sizes, expected_order):\n for chunk, size in zip(batched_data, expected_sizes):\n assert sum(len(x[0]) for x in chunk) == size\n word_order = []\n for chunk in batched_data:\n for sentence in chunk:\n word_order.append(sentence[0][0])\n assert word_order == expected_order\n\ndef test_data_to_batches_eval_mode():\n \"\"\"\n Tests the chunking of batches in eval_mode\n\n A few options are tested, such as whether or not to sort and the maximum sentence size\n \"\"\"\n data = make_fake_data(1, 2, 3)\n batched_data = data_to_batches(data, batch_size=5, eval_mode=True, sort_during_eval=True, min_length_to_batch_separately=None)\n check_batches(batched_data[0], [5, 1], ['C', 'B', 'A'])\n\n data = make_fake_data(1, 2, 6)\n batched_data = data_to_batches(data, batch_size=5, eval_mode=True, sort_during_eval=True, min_length_to_batch_separately=None)\n check_batches(batched_data[0], [6, 3], ['C', 'B', 'A'])\n\n data = make_fake_data(3, 2, 1)\n batched_data = data_to_batches(data, batch_size=5, eval_mode=True, sort_during_eval=True, min_length_to_batch_separately=None)\n check_batches(batched_data[0], [5, 1], ['A', 'B', 'C'])\n\n data = make_fake_data(3, 5, 2)\n batched_data = data_to_batches(data, batch_size=5, eval_mode=True, sort_during_eval=True, min_length_to_batch_separately=None)\n check_batches(batched_data[0], [5, 5], ['B', 'A', 'C'])\n\n data = make_fake_data(3, 5, 2)\n batched_data = data_to_batches(data, batch_size=5, eval_mode=True, sort_during_eval=False, min_length_to_batch_separately=3)\n check_batches(batched_data[0], [3, 5, 2], ['A', 'B', 'C'])\n\n data = make_fake_data(4, 1, 1)\n batched_data = data_to_batches(data, batch_size=5, eval_mode=True, sort_during_eval=False, min_length_to_batch_separately=3)\n check_batches(batched_data[0], [4, 2], ['A', 'B', 'C'])\n\n data = make_fake_data(1, 4, 1)\n batched_data = data_to_batches(data, batch_size=5, eval_mode=True, sort_during_eval=False, min_length_to_batch_separately=3)\n check_batches(batched_data[0], [1, 4, 1], ['A', 'B', 'C'])\n\n\nEWT_PUNCT_SAMPLE = \"\"\"\n# sent_id = weblog-blogspot.com_alaindewitt_20040929103700_ENG_20040929_103700-0048\n# text = Bush asked for permission to go to Alabama to work on a Senate campaign.\n1\tBush\tBush\tPROPN\tNNP\tNumber=Sing\t2\tnsubj\t2:nsubj\t_\n2\tasked\task\tVERB\tVBD\tMood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin\t0\troot\t0:root\t_\n3\tfor\tfor\tADP\tIN\t_\t4\tcase\t4:case\t_\n4\tpermission\tpermission\tNOUN\tNN\tNumber=Sing\t2\tobl\t2:obl:for\t_\n5\tto\tto\tPART\tTO\t_\t6\tmark\t6:mark\t_\n6\tgo\tgo\tVERB\tVB\tVerbForm=Inf\t4\tacl\t4:acl:to\t_\n7\tto\tto\tADP\tIN\t_\t8\tcase\t8:case\t_\n8\tAlabama\tAlabama\tPROPN\tNNP\tNumber=Sing\t6\tobl\t6:obl:to\t_\n9\tto\tto\tPART\tTO\t_\t10\tmark\t10:mark\t_\n10\twork\twork\tVERB\tVB\tVerbForm=Inf\t6\tadvcl\t6:advcl:to\t_\n11\ton\ton\tADP\tIN\t_\t14\tcase\t14:case\t_\n12\ta\ta\tDET\tDT\tDefinite=Ind|PronType=Art\t14\tdet\t14:det\t_\n13\tSenate\tSenate\tPROPN\tNNP\tNumber=Sing\t14\tcompound\t14:compound\t_\n14\tcampaign\tcampaign\tNOUN\tNN\tNumber=Sing\t10\tobl\t10:obl:on\tSpaceAfter=No\n15\t!!!!!\t!\tPUNCT\t.\t_\t2\tpunct\t2:punct\t_\n\n# sent_id = weblog-blogspot.com_alaindewitt_20040929103700_ENG_20040929_103700-0049\n# text = His superior officers said OK.\n1\tHis\this\tPRON\tPRP$\tCase=Gen|Gender=Masc|Number=Sing|Person=3|Poss=Yes|PronType=Prs\t3\tnmod:poss\t3:nmod:poss\t_\n2\tsuperior\tsuperior\tADJ\tJJ\tDegree=Pos\t3\tamod\t3:amod\t_\n3\tofficers\tofficer\tNOUN\tNNS\tNumber=Plur\t4\tnsubj\t4:nsubj\t_\n4\tsaid\tsay\tVERB\tVBD\tMood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin\t0\troot\t0:root\t_\n5\tOK\tok\tINTJ\tUH\t_\t4\tobj\t4:obj\tSpaceAfter=No\n6\t?????\t?\tPUNCT\t.\t_\t4\tpunct\t4:punct\t_\n\"\"\"\n\n\ndef test_punct_simplification():\n \"\"\"\n Test a punctuation simplification that should make it so unexpected\n question/exclamation marks types are processed into ? and !\n \"\"\"\n sample = CoNLL.conll2doc(input_str=EWT_PUNCT_SAMPLE)\n\n args = parser.parse_args(args=[\"--batch_size\", \"1000\", \"--shorthand\", \"en_test\"])\n data = DataLoader(sample, 5000, args, None)\n\n batches = [batch for batch in data]\n assert batches[0][-1] == [['Bush', 'asked', 'for', 'permission', 'to', 'go', 'to', 'Alabama', 'to', 'work', 'on', 'a', 'Senate', 'campaign', '!'],\n ['His', 'superior', 'officers', 'said', 'OK', '?']]\n\n\nif __name__ == '__main__':\n test_data_to_batches()\n\n"
},
{
"path": "stanza/tests/depparse/test_parser.py",
"content": "\"\"\"\nRun the tagger for a couple iterations on some fake data\n\nUses a couple sentences of UD_English-EWT as training/dev data\n\"\"\"\n\nimport os\nimport pytest\nimport zipfile\n\nimport torch\n\nfrom stanza.models import parser\nfrom stanza.models.common import pretrain\nfrom stanza.models.depparse.trainer import Trainer\nfrom stanza.tests import TEST_WORKING_DIR\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\nTRAIN_DATA = \"\"\"\n# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0003\n# text = DPA: Iraqi authorities announced that they had busted up 3 terrorist cells operating in Baghdad.\n1\tDPA\tDPA\tPROPN\tNNP\tNumber=Sing\t0\troot\t0:root\tSpaceAfter=No\n2\t:\t:\tPUNCT\t:\t_\t1\tpunct\t1:punct\t_\n3\tIraqi\tIraqi\tADJ\tJJ\tDegree=Pos\t4\tamod\t4:amod\t_\n4\tauthorities\tauthority\tNOUN\tNNS\tNumber=Plur\t5\tnsubj\t5:nsubj\t_\n5\tannounced\tannounce\tVERB\tVBD\tMood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin\t1\tparataxis\t1:parataxis\t_\n6\tthat\tthat\tSCONJ\tIN\t_\t9\tmark\t9:mark\t_\n7\tthey\tthey\tPRON\tPRP\tCase=Nom|Number=Plur|Person=3|PronType=Prs\t9\tnsubj\t9:nsubj\t_\n8\thad\thave\tAUX\tVBD\tMood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin\t9\taux\t9:aux\t_\n9\tbusted\tbust\tVERB\tVBN\tTense=Past|VerbForm=Part\t5\tccomp\t5:ccomp\t_\n10\tup\tup\tADP\tRP\t_\t9\tcompound:prt\t9:compound:prt\t_\n11\t3\t3\tNUM\tCD\tNumForm=Digit|NumType=Card\t13\tnummod\t13:nummod\t_\n12\tterrorist\tterrorist\tADJ\tJJ\tDegree=Pos\t13\tamod\t13:amod\t_\n13\tcells\tcell\tNOUN\tNNS\tNumber=Plur\t9\tobj\t9:obj\t_\n14\toperating\toperate\tVERB\tVBG\tVerbForm=Ger\t13\tacl\t13:acl\t_\n15\tin\tin\tADP\tIN\t_\t16\tcase\t16:case\t_\n16\tBaghdad\tBaghdad\tPROPN\tNNP\tNumber=Sing\t14\tobl\t14:obl:in\tSpaceAfter=No\n17\t.\t.\tPUNCT\t.\t_\t1\tpunct\t1:punct\t_\n\n# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0004\n# text = Two of them were being run by 2 officials of the Ministry of the Interior!\n1\tTwo\ttwo\tNUM\tCD\tNumForm=Word|NumType=Card\t6\tnsubj:pass\t6:nsubj:pass\t_\n2\tof\tof\tADP\tIN\t_\t3\tcase\t3:case\t_\n3\tthem\tthey\tPRON\tPRP\tCase=Acc|Number=Plur|Person=3|PronType=Prs\t1\tnmod\t1:nmod:of\t_\n4\twere\tbe\tAUX\tVBD\tMood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin\t6\taux\t6:aux\t_\n5\tbeing\tbe\tAUX\tVBG\tVerbForm=Ger\t6\taux:pass\t6:aux:pass\t_\n6\trun\trun\tVERB\tVBN\tTense=Past|VerbForm=Part|Voice=Pass\t0\troot\t0:root\t_\n7\tby\tby\tADP\tIN\t_\t9\tcase\t9:case\t_\n8\t2\t2\tNUM\tCD\tNumForm=Digit|NumType=Card\t9\tnummod\t9:nummod\t_\n9\tofficials\tofficial\tNOUN\tNNS\tNumber=Plur\t6\tobl\t6:obl:by\t_\n10\tof\tof\tADP\tIN\t_\t12\tcase\t12:case\t_\n11\tthe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t12\tdet\t12:det\t_\n12\tMinistry\tMinistry\tPROPN\tNNP\tNumber=Sing\t9\tnmod\t9:nmod:of\t_\n13\tof\tof\tADP\tIN\t_\t15\tcase\t15:case\t_\n14\tthe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t15\tdet\t15:det\t_\n15\tInterior\tInterior\tPROPN\tNNP\tNumber=Sing\t12\tnmod\t12:nmod:of\tSpaceAfter=No\n16\t!\t!\tPUNCT\t.\t_\t6\tpunct\t6:punct\t_\n\n\"\"\".lstrip()\n\n\nDEV_DATA = \"\"\"\n1\tFrom\tfrom\tADP\tIN\t_\t3\tcase\t3:case\t_\n2\tthe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t3\tdet\t3:det\t_\n3\tAP\tAP\tPROPN\tNNP\tNumber=Sing\t4\tobl\t4:obl:from\t_\n4\tcomes\tcome\tVERB\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t0\troot\t0:root\t_\n5\tthis\tthis\tDET\tDT\tNumber=Sing|PronType=Dem\t6\tdet\t6:det\t_\n6\tstory\tstory\tNOUN\tNN\tNumber=Sing\t4\tnsubj\t4:nsubj\t_\n7\t:\t:\tPUNCT\t:\t_\t4\tpunct\t4:punct\t_\n\n\"\"\".lstrip()\n\n\n\nclass TestParser:\n @pytest.fixture(scope=\"class\")\n def wordvec_pretrain_file(self):\n return f'{TEST_WORKING_DIR}/in/tiny_emb.pt'\n\n def run_training(self, tmp_path, wordvec_pretrain_file, train_text, dev_text, augment_nopunct=False, extra_args=None, zip_train_data=False):\n \"\"\"\n Run the training for a few iterations, load & return the model\n \"\"\"\n train_file = str(tmp_path / \"train.zip\") if zip_train_data else str(tmp_path / \"train.conllu\")\n dev_file = str(tmp_path / \"dev.conllu\")\n pred_file = str(tmp_path / \"pred.conllu\")\n\n save_name = \"test_parser.pt\"\n save_file = str(tmp_path / save_name)\n\n if zip_train_data:\n with zipfile.ZipFile(train_file, \"w\") as zout:\n with zout.open('train.conllu', 'w') as fout:\n fout.write(train_text.encode())\n else:\n with open(train_file, \"w\", encoding=\"utf-8\") as fout:\n fout.write(train_text)\n\n with open(dev_file, \"w\", encoding=\"utf-8\") as fout:\n fout.write(dev_text)\n\n args = [\"--wordvec_pretrain_file\", wordvec_pretrain_file,\n \"--train_file\", train_file,\n \"--eval_file\", dev_file,\n \"--output_file\", pred_file,\n \"--log_step\", \"10\",\n \"--eval_interval\", \"20\",\n \"--max_steps\", \"100\",\n \"--shorthand\", \"en_test\",\n \"--save_dir\", str(tmp_path),\n \"--save_name\", save_name,\n # in case we are doing a bert test\n \"--bert_start_finetuning\", \"10\",\n \"--bert_warmup_steps\", \"10\",\n \"--lang\", \"en\"]\n if not augment_nopunct:\n args.extend([\"--augment_nopunct\", \"0.0\"])\n if extra_args is not None:\n args = args + extra_args\n trainer, _ = parser.main(args)\n\n assert os.path.exists(save_file)\n pt = pretrain.Pretrain(wordvec_pretrain_file)\n # test loading the saved model\n saved_model = Trainer(pretrain=pt, model_file=save_file)\n return trainer\n\n def test_train(self, tmp_path, wordvec_pretrain_file):\n \"\"\"\n Simple test of a few 'epochs' of tagger training\n \"\"\"\n self.run_training(tmp_path, wordvec_pretrain_file, TRAIN_DATA, DEV_DATA)\n\n def test_arc_embedding(self, tmp_path, wordvec_pretrain_file):\n \"\"\"\n Simple test w/ and w/o arc embedding\n \"\"\"\n self.run_training(tmp_path, wordvec_pretrain_file, TRAIN_DATA, DEV_DATA, extra_args=['--use_arc_embedding'])\n\n def test_no_arc_embedding(self, tmp_path, wordvec_pretrain_file):\n \"\"\"\n Simple test w/ and w/o arc embedding\n \"\"\"\n self.run_training(tmp_path, wordvec_pretrain_file, TRAIN_DATA, DEV_DATA, extra_args=['--no_use_arc_embedding'])\n\n def test_zipfile_train(self, tmp_path, wordvec_pretrain_file):\n \"\"\"\n Simple test of a few 'epochs' of tagger training with a zipfile\n \"\"\"\n self.run_training(tmp_path, wordvec_pretrain_file, TRAIN_DATA, DEV_DATA, zip_train_data=True)\n\n def test_with_bert_nlayers(self, tmp_path, wordvec_pretrain_file):\n self.run_training(tmp_path, wordvec_pretrain_file, TRAIN_DATA, DEV_DATA, extra_args=['--bert_model', 'hf-internal-testing/tiny-bert', '--bert_hidden_layers', '2'])\n\n def test_with_bert_finetuning(self, tmp_path, wordvec_pretrain_file):\n trainer = self.run_training(tmp_path, wordvec_pretrain_file, TRAIN_DATA, DEV_DATA, extra_args=['--bert_model', 'hf-internal-testing/tiny-bert', '--bert_finetune', '--bert_hidden_layers', '2'])\n assert 'bert_optimizer' in trainer.optimizer.keys()\n assert 'bert_scheduler' in trainer.scheduler.keys()\n\n def test_with_bert_finetuning_resaved(self, tmp_path, wordvec_pretrain_file):\n \"\"\"\n Check that if we save, then load, then save a model with a finetuned bert, that bert isn't lost\n \"\"\"\n trainer = self.run_training(tmp_path, wordvec_pretrain_file, TRAIN_DATA, DEV_DATA, extra_args=['--bert_model', 'hf-internal-testing/tiny-bert', '--bert_finetune', '--bert_hidden_layers', '2'])\n assert 'bert_optimizer' in trainer.optimizer.keys()\n assert 'bert_scheduler' in trainer.scheduler.keys()\n\n save_name = trainer.args['save_name']\n filename = tmp_path / save_name\n assert os.path.exists(filename)\n checkpoint = torch.load(filename, lambda storage, loc: storage, weights_only=True)\n assert any(x.startswith(\"bert_model\") for x in checkpoint['model'].keys())\n\n # Test loading the saved model, saving it, and still having bert in it\n # even if we have set bert_finetune to False for this incarnation\n pt = pretrain.Pretrain(wordvec_pretrain_file)\n args = {\"bert_finetune\": False}\n saved_model = Trainer(pretrain=pt, model_file=filename, args=args)\n\n saved_model.save(filename)\n\n # This is the part that would fail if the force_bert_saved option did not exist\n checkpoint = torch.load(filename, lambda storage, loc: storage, weights_only=True)\n assert any(x.startswith(\"bert_model\") for x in checkpoint['model'].keys())\n\n def test_with_peft(self, tmp_path, wordvec_pretrain_file):\n trainer = self.run_training(tmp_path, wordvec_pretrain_file, TRAIN_DATA, DEV_DATA, extra_args=['--bert_model', 'hf-internal-testing/tiny-bert', '--bert_finetune', '--bert_hidden_layers', '2', '--use_peft'])\n assert 'bert_optimizer' in trainer.optimizer.keys()\n assert 'bert_scheduler' in trainer.scheduler.keys()\n\n def test_single_optimizer_checkpoint(self, tmp_path, wordvec_pretrain_file):\n trainer = self.run_training(tmp_path, wordvec_pretrain_file, TRAIN_DATA, DEV_DATA, extra_args=['--optim', 'adam'])\n\n save_dir = trainer.args['save_dir']\n save_name = trainer.args['save_name']\n checkpoint_name = trainer.args[\"checkpoint_save_name\"]\n\n assert os.path.exists(os.path.join(save_dir, save_name))\n assert checkpoint_name is not None\n assert os.path.exists(checkpoint_name)\n\n assert len(trainer.optimizer) == 1\n for opt in trainer.optimizer.values():\n assert isinstance(opt, torch.optim.Adam)\n\n pt = pretrain.Pretrain(wordvec_pretrain_file)\n checkpoint = Trainer(args=trainer.args, pretrain=pt, model_file=checkpoint_name)\n assert checkpoint.optimizer is not None\n assert len(checkpoint.optimizer) == 1\n for opt in checkpoint.optimizer.values():\n assert isinstance(opt, torch.optim.Adam)\n\n def test_two_optimizers_checkpoint(self, tmp_path, wordvec_pretrain_file):\n trainer = self.run_training(tmp_path, wordvec_pretrain_file, TRAIN_DATA, DEV_DATA, extra_args=['--optim', 'adam', '--second_optim', 'sgd', '--second_optim_start_step', '40'])\n\n save_dir = trainer.args['save_dir']\n save_name = trainer.args['save_name']\n checkpoint_name = trainer.args[\"checkpoint_save_name\"]\n\n assert os.path.exists(os.path.join(save_dir, save_name))\n assert checkpoint_name is not None\n assert os.path.exists(checkpoint_name)\n\n assert len(trainer.optimizer) == 1\n for opt in trainer.optimizer.values():\n assert isinstance(opt, torch.optim.SGD)\n\n pt = pretrain.Pretrain(wordvec_pretrain_file)\n checkpoint = Trainer(args=trainer.args, pretrain=pt, model_file=checkpoint_name)\n assert checkpoint.optimizer is not None\n assert len(checkpoint.optimizer) == 1\n for opt in trainer.optimizer.values():\n assert isinstance(opt, torch.optim.SGD)\n\n"
},
{
"path": "stanza/tests/langid/__init__.py",
"content": ""
},
{
"path": "stanza/tests/langid/test_langid.py",
"content": "\"\"\"\nBasic tests of langid module\n\"\"\"\n\nimport pytest\n\nfrom stanza.models.common.doc import Document\nfrom stanza.pipeline.core import Pipeline\nfrom stanza.pipeline.langid_processor import LangIDProcessor\nfrom stanza.tests import TEST_MODELS_DIR\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\n#pytestmark = pytest.mark.skip\n\n@pytest.fixture(scope=\"module\")\ndef basic_multilingual():\n return Pipeline(dir=TEST_MODELS_DIR, lang='multilingual', processors=\"langid\")\n\n@pytest.fixture(scope=\"module\")\ndef enfr_multilingual():\n return Pipeline(dir=TEST_MODELS_DIR, lang=\"multilingual\", processors=\"langid\", langid_lang_subset=[\"en\", \"fr\"])\n\n@pytest.fixture(scope=\"module\")\ndef en_multilingual():\n return Pipeline(dir=TEST_MODELS_DIR, lang=\"multilingual\", processors=\"langid\", langid_lang_subset=[\"en\"])\n\n@pytest.fixture(scope=\"module\")\ndef clean_multilingual():\n return Pipeline(dir=TEST_MODELS_DIR, lang=\"multilingual\", processors=\"langid\", langid_clean_text=True)\n\ndef test_langid(basic_multilingual):\n \"\"\"\n Basic test of language identification\n \"\"\"\n english_text = \"This is an English sentence.\"\n french_text = \"C'est une phrase française.\"\n docs = [english_text, french_text]\n\n docs = [Document([], text=text) for text in docs]\n basic_multilingual(docs)\n predictions = [doc.lang for doc in docs]\n assert predictions == [\"en\", \"fr\"]\n\ndef test_langid_benchmark(basic_multilingual):\n \"\"\"\n Run lang id model on 500 examples, confirm reasonable accuracy.\n \"\"\"\n examples = [\n {\"text\": \"contingentiam in naturalibus causis.\", \"label\": \"la\"},\n {\"text\": \"I jak opowiadał nieżyjący już pan Czesław\", \"label\": \"pl\"},\n {\"text\": \"Sonera gilt seit längerem als Übernahmekandidat\", \"label\": \"de\"},\n {\"text\": \"与银类似，汞也可以与空气中的硫化氢反应。\", \"label\": \"zh-hans\"},\n {\"text\": \"contradictionem implicat.\", \"label\": \"la\"},\n {\"text\": \"Bis zu Prozent gingen die Offerten etwa im\", \"label\": \"de\"},\n {\"text\": \"inneren Sicherheit vorgeschlagene Ausweitung der\", \"label\": \"de\"},\n {\"text\": \"Multimedia-PDA mit Mini-Tastatur\", \"label\": \"de\"},\n {\"text\": \"Ponášalo sa to na rovnicu o dvoch neznámych.\", \"label\": \"sk\"},\n {\"text\": \"이처럼 앞으로 심판의 그 날에 다시 올 메시아가 예수 그리스도이며 , 그는 모든 인류의\", \"label\": \"ko\"},\n {\"text\": \"Die Arbeitsgruppe bedauert , dass der weit über\", \"label\": \"de\"},\n {\"text\": \"И только раз довелось поговорить с ним не вполне\", \"label\": \"ru\"},\n {\"text\": \"de a-l lovi cu piciorul și conștiința că era\", \"label\": \"ro\"},\n {\"text\": \"relación coas pretensións do demandante e que, nos\", \"label\": \"gl\"},\n {\"text\": \"med petdeset in sedemdeset\", \"label\": \"sl\"},\n {\"text\": \"Catalunya; el Consell Comarcal del Vallès Oriental\", \"label\": \"ca\"},\n {\"text\": \"kunnen worden.\", \"label\": \"nl\"},\n {\"text\": \"Witkin je ve většině ohledů zcela jiný.\", \"label\": \"cs\"},\n {\"text\": \"lernen, so zu agieren, dass sie positive oder auch\", \"label\": \"de\"},\n {\"text\": \"olurmuş...\", \"label\": \"tr\"},\n {\"text\": \"sarcasmo de Altman, desde as «peruas» que discutem\", \"label\": \"pt\"},\n {\"text\": \"خلاف فوجداری مقدمہ درج کرے۔\", \"label\": \"ur\"},\n {\"text\": \"Norddal kommune :\", \"label\": \"no\"},\n {\"text\": \"dem Windows-.-Zeitalter , soll in diesem Jahr\", \"label\": \"de\"},\n {\"text\": \"przeklętych ucieleśniają mit poety-cygana,\", \"label\": \"pl\"},\n {\"text\": \"We do not believe the suspect has ties to this\", \"label\": \"en\"},\n {\"text\": \"groziņu pīšanu.\", \"label\": \"lv\"},\n {\"text\": \"Senior Vice-President David M. Thomas möchte\", \"label\": \"de\"},\n {\"text\": \"neomylně vybral nějakou knihu a začetl se.\", \"label\": \"cs\"},\n {\"text\": \"Statt dessen darf beispielsweise der Browser des\", \"label\": \"de\"},\n {\"text\": \"outubro, alcançando R $ bilhões em .\", \"label\": \"pt\"},\n {\"text\": \"(Porte, ), as it does other disciplines\", \"label\": \"en\"},\n {\"text\": \"uskupení se mylně domnívaly, že podporu\", \"label\": \"cs\"},\n {\"text\": \"Übernahme von Next Ende an dem System herum , das\", \"label\": \"de\"},\n {\"text\": \"No podemos decir a la Hacienda que los alemanes\", \"label\": \"es\"},\n {\"text\": \"и рѣста еи братья\", \"label\": \"orv\"},\n {\"text\": \"الذي اتخذ قرارا بتجميد اعلان الدولة الفلسطينية\", \"label\": \"ar\"},\n {\"text\": \"uurides Rootsi sõjaarhiivist toodud . sajandi\", \"label\": \"et\"},\n {\"text\": \"selskapets penger til å pusse opp sin enebolig på\", \"label\": \"no\"},\n {\"text\": \"средней полосе и севернее в Ярославской,\", \"label\": \"ru\"},\n {\"text\": \"il-massa żejda fil-ġemgħat u superġemgħat ta'\", \"label\": \"mt\"},\n {\"text\": \"The Global Beauties on internetilehekülg, mida\", \"label\": \"et\"},\n {\"text\": \"이스라엘 인들은 하나님이 그 큰 팔을 펴 이집트 인들을 치는 것을 보고 하나님을 두려워하며\", \"label\": \"ko\"},\n {\"text\": \"Snad ještě dodejme jeden ekonomický argument.\", \"label\": \"cs\"},\n {\"text\": \"Spalio d. vykusiame pirmajame rinkimų ture\", \"label\": \"lt\"},\n {\"text\": \"und schlechter Journalismus ein gutes Geschäft .\", \"label\": \"de\"},\n {\"text\": \"Du sodiečiai sėdi ant potvynio apsemtų namų stogo.\", \"label\": \"lt\"},\n {\"text\": \"цей є автентичним.\", \"label\": \"uk\"},\n {\"text\": \"Și îndegrabă fu cu îngerul mulțime de șireaguri\", \"label\": \"ro\"},\n {\"text\": \"sobra personal cualificado.\", \"label\": \"es\"},\n {\"text\": \"Tako se u Njemačkoj dvije trećine liječnika služe\", \"label\": \"hr\"},\n {\"text\": \"Dual-Athlon-Chipsatz noch in diesem Jahr\", \"label\": \"de\"},\n {\"text\": \"यहां तक कि चीन के चीफ ऑफ जनरल स्टाफ भी भारत का\", \"label\": \"hi\"},\n {\"text\": \"Li forestier du mont avale\", \"label\": \"fro\"},\n {\"text\": \"Netzwerken für Privatanwender zu bewundern .\", \"label\": \"de\"},\n {\"text\": \"만해는 승적을 가진 중이 결혼할 수 없다는 불교의 계율을 시대에 맞지 않는 것으로 보았다\", \"label\": \"ko\"},\n {\"text\": \"balance and weight distribution but not really for\", \"label\": \"en\"},\n {\"text\": \"og så e # tente vi opp den om morgonen å sfyrte\", \"label\": \"nn\"},\n {\"text\": \"변화는 의심의 여지가 없는 것이지만 반면에 진화는 논쟁의 씨앗이다 .\", \"label\": \"ko\"},\n {\"text\": \"puteare fac aceastea.\", \"label\": \"ro\"},\n {\"text\": \"Waitt seine Führungsmannschaft nicht dem\", \"label\": \"de\"},\n {\"text\": \"juhtimisega, tulid sealt.\", \"label\": \"et\"},\n {\"text\": \"Veränderungen .\", \"label\": \"de\"},\n {\"text\": \"banda en el Bayer Leverkusen de la Bundesliga de\", \"label\": \"es\"},\n {\"text\": \"В туже зиму посла всеволодъ сн҃а своѥго ст҃ослава\", \"label\": \"orv\"},\n {\"text\": \"пославъ приведе я мастеры ѿ грекъ\", \"label\": \"orv\"},\n {\"text\": \"En un nou escenari difícil d'imaginar fa poques\", \"label\": \"ca\"},\n {\"text\": \"καὶ γὰρ τινὲς αὐτοὺς εὐεργεσίαι εἶχον ἐκ Κροίσου\", \"label\": \"grc\"},\n {\"text\": \"직접적인 관련이 있다 .\", \"label\": \"ko\"},\n {\"text\": \"가까운 듯하면서도 멀다 .\", \"label\": \"ko\"},\n {\"text\": \"Er bietet ein ähnliches Leistungsniveau und\", \"label\": \"de\"},\n {\"text\": \"民都洛水牛是獨居的，並不會以群族聚居。\", \"label\": \"zh-hant\"},\n {\"text\": \"την τρομοκρατία.\", \"label\": \"el\"},\n {\"text\": \"hurbiltzen diren neurrian.\", \"label\": \"eu\"},\n {\"text\": \"Ah dimenticavo, ma tutta sta caciara per fare un\", \"label\": \"it\"},\n {\"text\": \"На первом этапе (-) прошла так называемая\", \"label\": \"ru\"},\n {\"text\": \"of games are on the market.\", \"label\": \"en\"},\n {\"text\": \"находится Мост дружбы, соединяющий узбекский и\", \"label\": \"ru\"},\n {\"text\": \"lessié je voldroie que li saint fussent aporté\", \"label\": \"fro\"},\n {\"text\": \"Дошла очередь и до Гималаев.\", \"label\": \"ru\"},\n {\"text\": \"vzácným suknem táhly pouští, si jednou chtěl do\", \"label\": \"cs\"},\n {\"text\": \"E no terceiro tipo sitúa a familias (%), nos que a\", \"label\": \"gl\"},\n {\"text\": \"وجابت دوريات امريكية وعراقية شوارع المدينة، فيما\", \"label\": \"ar\"},\n {\"text\": \"Jeg har bodd her i år .\", \"label\": \"no\"},\n {\"text\": \"Pohrozil, že odbory zostří postoj, pokud se\", \"label\": \"cs\"},\n {\"text\": \"tinham conseguido.\", \"label\": \"pt\"},\n {\"text\": \"Nicht-Erkrankten einen Anfangsverdacht für einen\", \"label\": \"de\"},\n {\"text\": \"permanece em aberto.\", \"label\": \"pt\"},\n {\"text\": \"questi possono promettere rendimenti fino a un\", \"label\": \"it\"},\n {\"text\": \"Tema juurutatud kahevedurisüsteemita oleksid\", \"label\": \"et\"},\n {\"text\": \"Поведение внешне простой игрушки оказалось\", \"label\": \"ru\"},\n {\"text\": \"Bundesländern war vom Börsenverein des Deutschen\", \"label\": \"de\"},\n {\"text\": \"acció, 'a mesura que avanci l'estiu, amb l'augment\", \"label\": \"ca\"},\n {\"text\": \"Dove trovare queste risorse? Jay Naidoo, ministro\", \"label\": \"it\"},\n {\"text\": \"essas gordurinhas.\", \"label\": \"pt\"},\n {\"text\": \"Im zweiten Schritt sollen im übernächsten Jahr\", \"label\": \"de\"},\n {\"text\": \"allveelaeva pole enam vaja, kuna külm sõda on läbi\", \"label\": \"et\"},\n {\"text\": \"उपद्रवी दुकानों को लूटने के साथ ही उनमें आग लगा\", \"label\": \"hi\"},\n {\"text\": \"@user nella sfortuna sei fortunata ..\", \"label\": \"it\"},\n {\"text\": \"математических школ в виде грозовых туч.\", \"label\": \"ru\"},\n {\"text\": \"No cambiaremos nunca nuestra forma de jugar por un\", \"label\": \"es\"},\n {\"text\": \"dla tej klasy ani wymogów minimalnych, z wyjątkiem\", \"label\": \"pl\"},\n {\"text\": \"en todo el mundo, mientras que en España consiguió\", \"label\": \"es\"},\n {\"text\": \"политики считать надежное обеспечение военной\", \"label\": \"ru\"},\n {\"text\": \"gogoratzen du, genio alemana delakoaren\", \"label\": \"eu\"},\n {\"text\": \"Бычий глаз.\", \"label\": \"ru\"},\n {\"text\": \"Opeření se v pravidelných obdobích obnovuje\", \"label\": \"cs\"},\n {\"text\": \"I no és només la seva, es tracta d'una resposta\", \"label\": \"ca\"},\n {\"text\": \"오경을 가르쳤다 .\", \"label\": \"ko\"},\n {\"text\": \"Nach der so genannten Start-up-Periode vergibt die\", \"label\": \"de\"},\n {\"text\": \"Saulista huomasi jo lapsena , että hänellä on\", \"label\": \"fi\"},\n {\"text\": \"Министерство культуры сочло нецелесообразным, и\", \"label\": \"ru\"},\n {\"text\": \"znepřátelené tábory v Tádžikistánu předseda\", \"label\": \"cs\"},\n {\"text\": \"καὶ ἦν ὁ λαὸς προσδοκῶν τὸν Ζαχαρίαν καὶ ἐθαύμαζον\", \"label\": \"grc\"},\n {\"text\": \"Вечером, в продукте, этот же человек говорил о\", \"label\": \"ru\"},\n {\"text\": \"lugar á formación de xuizos máis complexos.\", \"label\": \"gl\"},\n {\"text\": \"cheaper, in the end?\", \"label\": \"en\"},\n {\"text\": \"الوزارة في شأن صفقات بيع الشركات العامة التي تم\", \"label\": \"ar\"},\n {\"text\": \"tärkeintä elämässäni .\", \"label\": \"fi\"},\n {\"text\": \"Виконання Мінських угод було заблоковано Росією та\", \"label\": \"uk\"},\n {\"text\": \"Aby szybko rozpoznać żołnierzy desantu, należy\", \"label\": \"pl\"},\n {\"text\": \"Bankengeschäfte liegen vorn , sagte Strothmann .\", \"label\": \"de\"},\n {\"text\": \"продолжение работы.\", \"label\": \"ru\"},\n {\"text\": \"Metro AG plant Online-Offensive\", \"label\": \"de\"},\n {\"text\": \"nu vor veni, și să vor osîndi, aceia nu pot porni\", \"label\": \"ro\"},\n {\"text\": \"Ich denke , es geht in Wirklichkeit darum , NT bei\", \"label\": \"de\"},\n {\"text\": \"de turism care încasează contravaloarea\", \"label\": \"ro\"},\n {\"text\": \"Aurkaria itotzea da helburua, baloia lapurtu eta\", \"label\": \"eu\"},\n {\"text\": \"com a centre de formació en Tecnologies de la\", \"label\": \"ca\"},\n {\"text\": \"oportet igitur quod omne agens in agendo intendat\", \"label\": \"la\"},\n {\"text\": \"Jerzego Andrzejewskiego, oparty na chińskich\", \"label\": \"pl\"},\n {\"text\": \"sau một vài câu chuyện xã giao không dính dáng tới\", \"label\": \"vi\"},\n {\"text\": \"что экономическому прорыву жесткий авторитарный\", \"label\": \"ru\"},\n {\"text\": \"DRAM-Preisen scheinen DSPs ein\", \"label\": \"de\"},\n {\"text\": \"Jos dajan nubbái: Mana!\", \"label\": \"sme\"},\n {\"text\": \"toți carii ascultară de el să răsipiră.\", \"label\": \"ro\"},\n {\"text\": \"odpowiedzialności, które w systemie własności\", \"label\": \"pl\"},\n {\"text\": \"Dvomesečno potovanje do Mollenda v Peruju je\", \"label\": \"sl\"},\n {\"text\": \"d'entre les agències internacionals.\", \"label\": \"ca\"},\n {\"text\": \"Fahrzeugzugangssysteme gefertigt und an viele\", \"label\": \"de\"},\n {\"text\": \"in an answer to the sharers' petition in Cuthbert\", \"label\": \"en\"},\n {\"text\": \"Europa-Domain per Verordnung zu regeln .\", \"label\": \"de\"},\n {\"text\": \"#Balotelli. Su ebay prezzi stracciati per Silvio\", \"label\": \"it\"},\n {\"text\": \"Ne na košickém trávníku, ale už včera v letadle se\", \"label\": \"cs\"},\n {\"text\": \"zaměstnanosti a investičních strategií.\", \"label\": \"cs\"},\n {\"text\": \"Tatínku, udělej den\", \"label\": \"cs\"},\n {\"text\": \"frecuencia con Mary.\", \"label\": \"es\"},\n {\"text\": \"Свеаборге.\", \"label\": \"ru\"},\n {\"text\": \"opatření slovenské strany o certifikaci nejvíce\", \"label\": \"cs\"},\n {\"text\": \"En todas me decían: 'Espera que hagamos un estudio\", \"label\": \"es\"},\n {\"text\": \"Die Demonstration sollte nach Darstellung der\", \"label\": \"de\"},\n {\"text\": \"Ci vorrà un assoluto rigore se dietro i disavanzi\", \"label\": \"it\"},\n {\"text\": \"Tatínku, víš, že Honzovi odešla maminka?\", \"label\": \"cs\"},\n {\"text\": \"Die Anzahl der Rechner wuchs um % auf und die\", \"label\": \"de\"},\n {\"text\": \"האמריקאית על אדמת סעודיה עלולה לסבך את ישראל, אין\", \"label\": \"he\"},\n {\"text\": \"Volán Egyesülés, a Közlekedési Főfelügyelet is.\", \"label\": \"hu\"},\n {\"text\": \"Schejbala, který stejnou hru s velkým úspěchem\", \"label\": \"cs\"},\n {\"text\": \"depends on the data type of the field.\", \"label\": \"en\"},\n {\"text\": \"Umsatzwarnung zu Wochenbeginn zeitweise auf ein\", \"label\": \"de\"},\n {\"text\": \"niin heti nukun .\", \"label\": \"fi\"},\n {\"text\": \"Mobilfunkunternehmen gegen die Anwendung der so\", \"label\": \"de\"},\n {\"text\": \"sapessi le intenzioni del governo Monti e dell'UE\", \"label\": \"it\"},\n {\"text\": \"Di chi è figlia Martine Aubry?\", \"label\": \"it\"},\n {\"text\": \"avec le reste du monde.\", \"label\": \"fr\"},\n {\"text\": \"Այդ մաքոքը ինքնին նոր չէ, աշխարհը արդեն մի քանի\", \"label\": \"hy\"},\n {\"text\": \"și în cazul destrămării cenaclului.\", \"label\": \"ro\"},\n {\"text\": \"befriedigen kann , und ohne die auftretenden\", \"label\": \"de\"},\n {\"text\": \"Κύκνον τ̓ ἐξεναρεῖν καὶ ἀπὸ κλυτὰ τεύχεα δῦσαι.\", \"label\": \"grc\"},\n {\"text\": \"færdiguddannede.\", \"label\": \"da\"},\n {\"text\": \"Schmidt war Sohn eines Rittergutsbesitzers.\", \"label\": \"de\"},\n {\"text\": \"и вдаша попадь ѡпрати\", \"label\": \"orv\"},\n {\"text\": \"cine nu știe învățătură”.\", \"label\": \"ro\"},\n {\"text\": \"détacha et cette dernière tenta de tuer le jeune\", \"label\": \"fr\"},\n {\"text\": \"Der har saka også ei lengre forhistorie.\", \"label\": \"nn\"},\n {\"text\": \"Pieprz roztłuc w moździerzu, dodać do pasty,\", \"label\": \"pl\"},\n {\"text\": \"Лежа за гребнем оврага, как за бруствером, Ушаков\", \"label\": \"ru\"},\n {\"text\": \"gesucht habe, vielen Dank nochmals!\", \"label\": \"de\"},\n {\"text\": \"инструментальных сталей, повышения\", \"label\": \"ru\"},\n {\"text\": \"im Halbfinale Patrick Smith und im Finale dann\", \"label\": \"de\"},\n {\"text\": \"البنوك التريث في منح تسهيلات جديدة لمنتجي حديد\", \"label\": \"ar\"},\n {\"text\": \"una bolsa ventral, la cual se encuentra debajo de\", \"label\": \"es\"},\n {\"text\": \"za SETimes.\", \"label\": \"sr\"},\n {\"text\": \"de Irak, a un piloto italiano que había violado el\", \"label\": \"es\"},\n {\"text\": \"Er könne sich nicht erklären , wie die Zeitung auf\", \"label\": \"de\"},\n {\"text\": \"Прохорова.\", \"label\": \"ru\"},\n {\"text\": \"la democrazia perde sulla tecnocrazia? #\", \"label\": \"it\"},\n {\"text\": \"entre ambas instituciones, confirmó al medio que\", \"label\": \"es\"},\n {\"text\": \"Austlandet, vart det funne om lag førti\", \"label\": \"nn\"},\n {\"text\": \"уровнями власти.\", \"label\": \"ru\"},\n {\"text\": \"Dá tedy primáři úplatek, a často ne malý.\", \"label\": \"cs\"},\n {\"text\": \"brillantes del acto, al llevar a cabo en el\", \"label\": \"es\"},\n {\"text\": \"eee druga zadeva je majhen priročen gre kamorkoli\", \"label\": \"sl\"},\n {\"text\": \"Das ATX-Board paßt in herkömmliche PC-ATX-Gehäuse\", \"label\": \"de\"},\n {\"text\": \"Za vodné bylo v prvním pololetí zaplaceno v ČR\", \"label\": \"cs\"},\n {\"text\": \"Даже на полсантиметра.\", \"label\": \"ru\"},\n {\"text\": \"com la del primer tinent d'alcalde en funcions,\", \"label\": \"ca\"},\n {\"text\": \"кількох оповідань в цілості — щось на зразок того\", \"label\": \"uk\"},\n {\"text\": \"sed ad divitias congregandas, vel superfluum\", \"label\": \"la\"},\n {\"text\": \"Norma Talmadge, spela mot Valentino i en version\", \"label\": \"sv\"},\n {\"text\": \"Dlatego chciał się jej oświadczyć w niezwykłym\", \"label\": \"pl\"},\n {\"text\": \"будут выступать на одинаковых снарядах.\", \"label\": \"ru\"},\n {\"text\": \"Orang-orang terbunuh di sana.\", \"label\": \"id\"},\n {\"text\": \"لدى رايت شقيق اسمه أوسكار, وهو يعمل كرسام للكتب\", \"label\": \"ar\"},\n {\"text\": \"Wirklichkeit verlagerten und kaum noch\", \"label\": \"de\"},\n {\"text\": \"как перемешивают костяшки перед игрой в домино, и\", \"label\": \"ru\"},\n {\"text\": \"В средине дня, когда солнце светило в нашу\", \"label\": \"ru\"},\n {\"text\": \"d'aventure aux rôles de jeune romantique avec une\", \"label\": \"fr\"},\n {\"text\": \"My teď hledáme organizace, jež by s námi chtěly\", \"label\": \"cs\"},\n {\"text\": \"Urteilsfähigkeit einbüßen , wenn ich eigene\", \"label\": \"de\"},\n {\"text\": \"sua appartenenza anche a voci diverse da quella in\", \"label\": \"it\"},\n {\"text\": \"Aufträge dieses Jahr verdoppeln werden .\", \"label\": \"de\"},\n {\"text\": \"M.E.: Miała szanse mnie odnaleźć, gdyby naprawdę\", \"label\": \"pl\"},\n {\"text\": \"secundum contactum virtutis, cum careat dimensiva\", \"label\": \"la\"},\n {\"text\": \"ezinbestekoa dela esan zuen.\", \"label\": \"eu\"},\n {\"text\": \"Anek hurbiltzeko eskatzen zion besaulkitik, eta\", \"label\": \"eu\"},\n {\"text\": \"perfectius alio videat, quamvis uterque videat\", \"label\": \"la\"},\n {\"text\": \"Die Strecke war anspruchsvoll und führte unter\", \"label\": \"de\"},\n {\"text\": \"саморазоблачительным уроком, западные СМИ не\", \"label\": \"ru\"},\n {\"text\": \"han representerer radikal islamisme .\", \"label\": \"no\"},\n {\"text\": \"Què s'hi respira pel que fa a la reforma del\", \"label\": \"ca\"},\n {\"text\": \"previsto para também ser desconstruido.\", \"label\": \"pt\"},\n {\"text\": \"Ὠκεανοῦ βαθυκόλποις ἄνθεά τ̓ αἰνυμένην, ῥόδα καὶ\", \"label\": \"grc\"},\n {\"text\": \"para jovens de a anos nos Cieps.\", \"label\": \"pt\"},\n {\"text\": \"संघर्ष को अंजाम तक पहुंचाने का ऐलान किया है ।\", \"label\": \"hi\"},\n {\"text\": \"objeví i u nás.\", \"label\": \"cs\"},\n {\"text\": \"kvitteringer.\", \"label\": \"da\"},\n {\"text\": \"This report is no exception.\", \"label\": \"en\"},\n {\"text\": \"Разлепват доносниците до избирателните списъци\", \"label\": \"bg\"},\n {\"text\": \"anderem ihre Bewegungsfreiheit in den USA\", \"label\": \"de\"},\n {\"text\": \"Ñu tegoon ca kaw gor ña ay njotti bopp yu kenn\", \"label\": \"wo\"},\n {\"text\": \"Struktur kann beispielsweise der Schwerpunkt mehr\", \"label\": \"de\"},\n {\"text\": \"% la velocidad permitida, la sanción es muy grave.\", \"label\": \"es\"},\n {\"text\": \"Teles-Einstieg in ADSL-Markt\", \"label\": \"de\"},\n {\"text\": \"ettekäändeks liiga suure osamaksu.\", \"label\": \"et\"},\n {\"text\": \"als Indiz für die geänderte Marktpolitik des\", \"label\": \"de\"},\n {\"text\": \"quod quidem aperte consequitur ponentes\", \"label\": \"la\"},\n {\"text\": \"de negociación para el próximo de junio.\", \"label\": \"es\"},\n {\"text\": \"Tyto důmyslné dekorace doznaly v poslední době\", \"label\": \"cs\"},\n {\"text\": \"največjega uspeha doslej.\", \"label\": \"sl\"},\n {\"text\": \"Paul Allen je jedan od suosnivača Interval\", \"label\": \"hr\"},\n {\"text\": \"Federal (Seac / DF) eo Sindicato das Empresas de\", \"label\": \"pt\"},\n {\"text\": \"Quartal mit . Mark gegenüber dem gleichen Quartal\", \"label\": \"de\"},\n {\"text\": \"otros clubes y del Barça B saldrán varios\", \"label\": \"es\"},\n {\"text\": \"Jaskula (Pol.) -\", \"label\": \"cs\"},\n {\"text\": \"umožnily říci, že je možné přejít k mnohem\", \"label\": \"cs\"},\n {\"text\": \"اعلن الجنرال تومي فرانكس قائد القوات الامريكية\", \"label\": \"ar\"},\n {\"text\": \"Telekom-Chef Ron Sommer und der Vorstandssprecher\", \"label\": \"de\"},\n {\"text\": \"My, jako průmyslový a finanční holding, můžeme\", \"label\": \"cs\"},\n {\"text\": \"voorlichting onder andere betrekking kan hebben:\", \"label\": \"nl\"},\n {\"text\": \"Hinrichtung geistig Behinderter applaudiert oder\", \"label\": \"de\"},\n {\"text\": \"wie beispielsweise Anzahl erzielte Klicks ,\", \"label\": \"de\"},\n {\"text\": \"Intel-PC-SDRAM-Spezifikation in der Version . (\", \"label\": \"de\"},\n {\"text\": \"plângere în termen de zile de la comunicarea\", \"label\": \"ro\"},\n {\"text\": \"и Испания ще изгубят втория си комисар в ЕК.\", \"label\": \"bg\"},\n {\"text\": \"इसके चलते इस आदिवासी जनजाति का क्षरण हो रहा है ।\", \"label\": \"hi\"},\n {\"text\": \"aunque se mostró contrario a establecer un\", \"label\": \"es\"},\n {\"text\": \"des letzten Jahres von auf Millionen Euro .\", \"label\": \"de\"},\n {\"text\": \"Ankara se također poziva da u cijelosti ratificira\", \"label\": \"hr\"},\n {\"text\": \"herunterlädt .\", \"label\": \"de\"},\n {\"text\": \"стрессовую ситуацию для организма, каковой\", \"label\": \"ru\"},\n {\"text\": \"Státního shromáždění (parlamentu).\", \"label\": \"cs\"},\n {\"text\": \"diskutieren , ob und wie dieser Dienst weiterhin\", \"label\": \"de\"},\n {\"text\": \"Verbindungen zu FPÖ-nahen Polizisten gepflegt und\", \"label\": \"de\"},\n {\"text\": \"Pražského volebního lídra ovšem nevybírá Miloš\", \"label\": \"cs\"},\n {\"text\": \"Nach einem Bericht der Washington Post bleibt das\", \"label\": \"de\"},\n {\"text\": \"للوضع آنذاك، لكني في قرارة نفسي كنت سعيداً لما\", \"label\": \"ar\"},\n {\"text\": \"не желаят запазването на статуквото.\", \"label\": \"bg\"},\n {\"text\": \"Offenburg gewesen .\", \"label\": \"de\"},\n {\"text\": \"ἐὰν ὑμῖν εἴπω οὐ μὴ πιστεύσητε\", \"label\": \"grc\"},\n {\"text\": \"all'odiato compagno di squadra Prost, il quale\", \"label\": \"it\"},\n {\"text\": \"historischen Gänselieselbrunnens.\", \"label\": \"de\"},\n {\"text\": \"למידע מלווייני הריגול האמריקאיים העוקבים אחר\", \"label\": \"he\"},\n {\"text\": \"οὐδὲν ἄρα διαφέρεις Ἀμάσιος τοῦ Ἠλείου, ὃν\", \"label\": \"grc\"},\n {\"text\": \"movementos migratorios.\", \"label\": \"gl\"},\n {\"text\": \"Handy und ein Spracherkennungsprogramm sämtliche\", \"label\": \"de\"},\n {\"text\": \"Kümne aasta jooksul on Eestisse ohjeldamatult\", \"label\": \"et\"},\n {\"text\": \"H.G. Bücknera.\", \"label\": \"pl\"},\n {\"text\": \"protiv krijumčarenja, ili pak traženju ukidanja\", \"label\": \"hr\"},\n {\"text\": \"Topware-Anteile mehrere Millionen Mark gefordert\", \"label\": \"de\"},\n {\"text\": \"Maar de mensen die nu over Van Dijk bij FC Twente\", \"label\": \"nl\"},\n {\"text\": \"poidan experimentar as percepcións do interesado,\", \"label\": \"gl\"},\n {\"text\": \"Miał przecież w kieszeni nóż.\", \"label\": \"pl\"},\n {\"text\": \"Avšak žádná z nich nepronikla za hranice přímé\", \"label\": \"cs\"},\n {\"text\": \"esim. helpottamalla luottoja muiden\", \"label\": \"fi\"},\n {\"text\": \"Podle předběžných výsledků zvítězila v\", \"label\": \"cs\"},\n {\"text\": \"Nicht nur das Web-Frontend , auch die\", \"label\": \"de\"},\n {\"text\": \"Regierungsinstitutionen oder Universitäten bei\", \"label\": \"de\"},\n {\"text\": \"Խուլեն Լոպետեգիին, պատճառաբանելով, որ վերջինս\", \"label\": \"hy\"},\n {\"text\": \"Афганистана, где в последние дни идут ожесточенные\", \"label\": \"ru\"},\n {\"text\": \"лѧхове же не идоша\", \"label\": \"orv\"},\n {\"text\": \"Mit Hilfe von IBMs Chip-Management-Systemen sollen\", \"label\": \"de\"},\n {\"text\": \", als Manager zu Telefonica zu wechseln .\", \"label\": \"de\"},\n {\"text\": \"którym zajmuje się człowiek, zmienia go i pozwala\", \"label\": \"pl\"},\n {\"text\": \"činí kyperských liber, to je asi USD.\", \"label\": \"cs\"},\n {\"text\": \"Studienplätze getauscht werden .\", \"label\": \"de\"},\n {\"text\": \"учёных, орнитологов признают вид.\", \"label\": \"ru\"},\n {\"text\": \"acordare a concediilor prevăzute de legislațiile\", \"label\": \"ro\"},\n {\"text\": \"at større innsats for fornybar, berekraftig energi\", \"label\": \"nn\"},\n {\"text\": \"Politiet veit ikkje kor mange personar som deltok\", \"label\": \"nn\"},\n {\"text\": \"offentligheten av unge , sinte menn som har\", \"label\": \"no\"},\n {\"text\": \"însuși în jurul lapunei, care încet DISPARE în\", \"label\": \"ro\"},\n {\"text\": \"O motivo da decisão é evitar uma sobrecarga ainda\", \"label\": \"pt\"},\n {\"text\": \"El Apostolado de la prensa contribuye en modo\", \"label\": \"es\"},\n {\"text\": \"Teltow ( Kreis Teltow-Fläming ) ist Schmitt einer\", \"label\": \"de\"},\n {\"text\": \"grozījumus un iesniegt tos Apvienoto Nāciju\", \"label\": \"lv\"},\n {\"text\": \"Gestalt einer deutschen Nationalmannschaft als\", \"label\": \"de\"},\n {\"text\": \"D überholt zu haben , konterte am heutigen Montag\", \"label\": \"de\"},\n {\"text\": \"Softwarehersteller Oracle hat im dritten Quartal\", \"label\": \"de\"},\n {\"text\": \"Během nich se ekonomické podmínky mohou radikálně\", \"label\": \"cs\"},\n {\"text\": \"Dziki kot w górach zeskakuje z kamienia.\", \"label\": \"pl\"},\n {\"text\": \"Ačkoliv ligový nováček prohrál, opět potvrdil, že\", \"label\": \"cs\"},\n {\"text\": \"des Tages , Portraits internationaler Stars sowie\", \"label\": \"de\"},\n {\"text\": \"Communicator bekannt wurde .\", \"label\": \"de\"},\n {\"text\": \"τῷ δ’ ἄρα καὶ αὐτῷ ἡ γυνή ἐπίτεξ ἐοῦσα πᾶσαν\", \"label\": \"grc\"},\n {\"text\": \"Triadú tenia, mentre redactava 'Dies de memòria',\", \"label\": \"ca\"},\n {\"text\": \"دستهجمعی در درخشندگی ماه سیمگون زمزمه ستاینده و\", \"label\": \"fa\"},\n {\"text\": \"Книгу, наполненную мелочной заботой об одежде,\", \"label\": \"ru\"},\n {\"text\": \"putares canem leporem persequi.\", \"label\": \"la\"},\n {\"text\": \"В дальнейшем эта яркость слегка померкла, но в\", \"label\": \"ru\"},\n {\"text\": \"offizielles Verfahren gegen die Telekom\", \"label\": \"de\"},\n {\"text\": \"podrían haber sido habitantes de la Península\", \"label\": \"es\"},\n {\"text\": \"Grundlage für dieses Verfahren sind spezielle\", \"label\": \"de\"},\n {\"text\": \"Rechtsausschuß vorgelegten Entwurf der Richtlinie\", \"label\": \"de\"},\n {\"text\": \"Im so genannten Portalgeschäft sei das Unternehmen\", \"label\": \"de\"},\n {\"text\": \"ⲏ ⲉⲓϣⲁⲛϥⲓ ⲛⲉⲓⲇⲱⲗⲟⲛ ⲉⲧϩⲙⲡⲉⲕⲏⲓ ⲙⲏ ⲉⲓⲛⲁϣϩⲱⲡ ⲟⲛ ⲙⲡⲣⲏ\", \"label\": \"cop\"},\n {\"text\": \"juego podían matar a cualquier herbívoro, pero\", \"label\": \"es\"},\n {\"text\": \"Nach Angaben von Axent nutzen Unternehmen aus der\", \"label\": \"de\"},\n {\"text\": \"hrdiny Havlovy Zahradní slavnosti (premiéra ) se\", \"label\": \"cs\"},\n {\"text\": \"Een zin van heb ik jou daar\", \"label\": \"nl\"},\n {\"text\": \"hat sein Hirn an der CeBIT-Kasse vergessen .\", \"label\": \"de\"},\n {\"text\": \"καὶ τοὺς ἐκπλαγέντας οὐκ ἔχειν ἔτι ἐλεγχομένους\", \"label\": \"grc\"},\n {\"text\": \"nachgewiesenen langfristigen Kosten , sowie den im\", \"label\": \"de\"},\n {\"text\": \"jučer nakon četiri dana putovanja u Helsinki.\", \"label\": \"hr\"},\n {\"text\": \"pašto paslaugos teikėjas gali susitarti su\", \"label\": \"lt\"},\n {\"text\": \"В результате, эти золотые кадры переходят из одной\", \"label\": \"ru\"},\n {\"text\": \"द फाइव-ईयर एंगेजमेंट में अभिनय किया जिसमें जैसन\", \"label\": \"hi\"},\n {\"text\": \"výpis o počtu akcií.\", \"label\": \"cs\"},\n {\"text\": \"Enfin, elles arrivent à un pavillon chinois\", \"label\": \"fr\"},\n {\"text\": \"Tentu saja, tren yang berhubungandengan\", \"label\": \"id\"},\n {\"text\": \"Arbeidarpartiet og SV har sikra seg fleirtal mot\", \"label\": \"nn\"},\n {\"text\": \"eles: 'Tudo isso está errado' , disse um\", \"label\": \"pt\"},\n {\"text\": \"The islands are in their own time zone, minutes\", \"label\": \"en\"},\n {\"text\": \"Auswahl debütierte er am .\", \"label\": \"de\"},\n {\"text\": \"Bu komisyonlar, arazilerini satın almak için\", \"label\": \"tr\"},\n {\"text\": \"Geschütze gegen Redmond aufgefahren .\", \"label\": \"de\"},\n {\"text\": \"Time scything the hours, but at the top, over the\", \"label\": \"en\"},\n {\"text\": \"Di musim semi , berharap mengadaptasi Tintin untuk\", \"label\": \"id\"},\n {\"text\": \"крупнейшей геополитической катастрофой XX века.\", \"label\": \"ru\"},\n {\"text\": \"Rajojen avaaminen ei suju ongelmitta .\", \"label\": \"fi\"},\n {\"text\": \"непроницаемым, как для СССР.\", \"label\": \"ru\"},\n {\"text\": \"Ma non mancano le polemiche.\", \"label\": \"it\"},\n {\"text\": \"Internet als Ort politischer Diskussion und auch\", \"label\": \"de\"},\n {\"text\": \"incomplets.\", \"label\": \"ca\"},\n {\"text\": \"Su padre luchó al lado de Luis Moya, primer Jefe\", \"label\": \"es\"},\n {\"text\": \"informazione.\", \"label\": \"it\"},\n {\"text\": \"Primacom bietet für Telekom-Kabelnetz\", \"label\": \"de\"},\n {\"text\": \"Oświadczenie prezydencji w imieniu Unii\", \"label\": \"pl\"},\n {\"text\": \"foran rattet i familiens gamle Baleno hvis døra på\", \"label\": \"no\"},\n {\"text\": \"[speaker:laughter]\", \"label\": \"sl\"},\n {\"text\": \"Dog med langt mindre utstyr med seg.\", \"label\": \"nn\"},\n {\"text\": \"dass es nicht schon mit der anfänglichen\", \"label\": \"de\"},\n {\"text\": \"इस पर दोनों पक्षों में नोकझोंक शुरू हो गई ।\", \"label\": \"hi\"},\n {\"text\": \"کے ترجمان منیش تیواری اور دگ وجئے سنگھ نے بھی یہ\", \"label\": \"ur\"},\n {\"text\": \"dell'Assemblea Costituente che posseggono i\", \"label\": \"it\"},\n {\"text\": \"и аште вьси съблазнѧтъ сѧ нъ не азъ\", \"label\": \"cu\"},\n {\"text\": \"In Irvine hat auch das Logistikunternehmen Atlas\", \"label\": \"de\"},\n {\"text\": \"законодательных норм, принимаемых существующей\", \"label\": \"ru\"},\n {\"text\": \"Κροίσῳ προτείνων τὰς χεῖρας ἐπικατασφάξαι μιν\", \"label\": \"grc\"},\n {\"text\": \"МИНУСЫ: ИНФЛЯЦИЯ И КРИЗИС В ЖИВОТНОВОДСТВЕ.\", \"label\": \"ru\"},\n {\"text\": \"unterschiedlicher Meinung .\", \"label\": \"de\"},\n {\"text\": \"Jospa joku ystävällinen sielu auttaisi kassieni\", \"label\": \"fi\"},\n {\"text\": \"Añadió que, en el futuro se harán otros\", \"label\": \"es\"},\n {\"text\": \"Sessiz tonlama hem Fince, hem de Kuzey Sami\", \"label\": \"tr\"},\n {\"text\": \"nicht ihnen gehört und sie nicht alles , was sie\", \"label\": \"de\"},\n {\"text\": \"Etelästä Kuivajärveen laskee Tammelan Liesjärvestä\", \"label\": \"fi\"},\n {\"text\": \"ICANNs Vorsitzender Vint Cerf warb mit dem Hinweis\", \"label\": \"de\"},\n {\"text\": \"Norsk politikk frå til kan dermed, i\", \"label\": \"nn\"},\n {\"text\": \"Głosowało posłów.\", \"label\": \"pl\"},\n {\"text\": \"Danny Jones -- smithjones@ev.net\", \"label\": \"en\"},\n {\"text\": \"sebeuvědomění moderní civilizace sehrála lučavka\", \"label\": \"cs\"},\n {\"text\": \"относительно спокойный сон: тому гарантия\", \"label\": \"ru\"},\n {\"text\": \"A halte voiz prist li pedra a crïer\", \"label\": \"fro\"},\n {\"text\": \"آنها امیدوارند این واکسن بهزودی در دسترس بیماران\", \"label\": \"fa\"},\n {\"text\": \"vlastní důstojnou vousatou tváří.\", \"label\": \"cs\"},\n {\"text\": \"ora aprire la strada a nuove cause e alimentare il\", \"label\": \"it\"},\n {\"text\": \"Die Zahl der Vielleser nahm von auf Prozent zu ,\", \"label\": \"de\"},\n {\"text\": \"Finanzvorstand von Hotline-Dienstleister InfoGenie\", \"label\": \"de\"},\n {\"text\": \"entwickeln .\", \"label\": \"de\"},\n {\"text\": \"incolumità pubblica.\", \"label\": \"it\"},\n {\"text\": \"lehtija televisiomainonta\", \"label\": \"fi\"},\n {\"text\": \"joistakin kohdista eri mieltä.\", \"label\": \"fi\"},\n {\"text\": \"Hlavně anglická nezávislá scéna, Dead Can Dance,\", \"label\": \"cs\"},\n {\"text\": \"pásmech od do bodů bodové stupnice.\", \"label\": \"cs\"},\n {\"text\": \"Zu Beginn des Ersten Weltkrieges zählte das\", \"label\": \"de\"},\n {\"text\": \"Així van sorgir, damunt els antics cementiris,\", \"label\": \"ca\"},\n {\"text\": \"In manchem Gedicht der spätern Alten, wie zum\", \"label\": \"de\"},\n {\"text\": \"gaweihaida jah insandida in þana fairƕu jus qiþiþ\", \"label\": \"got\"},\n {\"text\": \"Beides sollte gelöscht werden!\", \"label\": \"de\"},\n {\"text\": \"modifiqués la seva petició inicial de anys de\", \"label\": \"ca\"},\n {\"text\": \"В день открытия симпозиума состоялась закладка\", \"label\": \"ru\"},\n {\"text\": \"tõestatud.\", \"label\": \"et\"},\n {\"text\": \"ἵππῳ πίπτει αὐτοῦ ταύτῃ\", \"label\": \"grc\"},\n {\"text\": \"bisher nie enttäuscht!\", \"label\": \"de\"},\n {\"text\": \"De bohte ollu tuollárat ja suttolaččat ja\", \"label\": \"sme\"},\n {\"text\": \"Klarsignal från röstlängdsläsaren, tre tryck i\", \"label\": \"sv\"},\n {\"text\": \"Tvůrcem nového termínu je Joseph Fisher.\", \"label\": \"cs\"},\n {\"text\": \"Nie miałem czasu na reakcję twierdzi Norbert,\", \"label\": \"pl\"},\n {\"text\": \"potentia Schöpfer.\", \"label\": \"de\"},\n {\"text\": \"Un poquito caro, pero vale mucho la pena;\", \"label\": \"es\"},\n {\"text\": \"οὔ τε γὰρ ἴφθιμοι Λύκιοι Δαναῶν ἐδύναντο τεῖχος\", \"label\": \"grc\"},\n {\"text\": \"vajec, sladového výtažku a některých vitamínových\", \"label\": \"cs\"},\n {\"text\": \"Настоящие герои, те, чьи истории потом\", \"label\": \"ru\"},\n {\"text\": \"praesumptio:\", \"label\": \"la\"},\n {\"text\": \"Olin justkui nende vastutusel.\", \"label\": \"et\"},\n {\"text\": \"Jokainen keinahdus tuo lähemmäksi hetkeä jolloin\", \"label\": \"fi\"},\n {\"text\": \"ekonomicky výhodných způsobů odvodnění těžkých,\", \"label\": \"cs\"},\n {\"text\": \"Poprvé ve své historii dokázala v kvalifikaci pro\", \"label\": \"cs\"},\n {\"text\": \"zpracovatelského a spotřebního průmyslu bude nutné\", \"label\": \"cs\"},\n {\"text\": \"Windows CE zu integrieren .\", \"label\": \"de\"},\n {\"text\": \"Armangué, a través d'un decret, ordenés l'aturada\", \"label\": \"ca\"},\n {\"text\": \"to, co nás Evropany spojuje, než to, co nás od\", \"label\": \"cs\"},\n {\"text\": \"ergänzt durch einen gesetzlich verankertes\", \"label\": \"de\"},\n {\"text\": \"Насчитал, что с начала года всего три дня были\", \"label\": \"ru\"},\n {\"text\": \"Borisovu tražeći od njega da prihvati njenu\", \"label\": \"sr\"},\n {\"text\": \"la presenza di ben veleni diversi: . chili di\", \"label\": \"it\"},\n {\"text\": \"καὶ τῶν ἐκλεκτῶν ἀγγέλων ἵνα ταῦτα φυλάξῃς χωρὶς\", \"label\": \"grc\"},\n {\"text\": \"pretraživale obližnju bolnicu i stambene zgrade u\", \"label\": \"hr\"},\n {\"text\": \"An rund Katzen habe Wolf seine Spiele getestet ,\", \"label\": \"de\"},\n {\"text\": \"investigating since March.\", \"label\": \"en\"},\n {\"text\": \"Tonböden (Mullböden).\", \"label\": \"de\"},\n {\"text\": \"Stálý dopisovatel LN v SRN Bedřich Utitz\", \"label\": \"cs\"},\n {\"text\": \"červnu předložené smlouvy.\", \"label\": \"cs\"},\n {\"text\": \"πνεύματι ᾧ ἐλάλει\", \"label\": \"grc\"},\n {\"text\": \".%의 신장세를 보였다.\", \"label\": \"ko\"},\n {\"text\": \"Foae verde, foi de nuc, Prin pădure, prin colnic,\", \"label\": \"ro\"},\n {\"text\": \"διαπέμψας ἄλλους ἄλλῃ τοὺς μὲν ἐς Δελφοὺς ἰέναι\", \"label\": \"grc\"},\n {\"text\": \"المسلمين أو أي تيار سياسي طالما عمل ذلك التيار في\", \"label\": \"ar\"},\n {\"text\": \"As informações são da Dow Jones.\", \"label\": \"pt\"},\n {\"text\": \"Milliarde DM ausgestattet sein .\", \"label\": \"de\"},\n {\"text\": \"De utgår fortfarande från att kvinnans jämlikhet\", \"label\": \"sv\"},\n {\"text\": \"Sneeuw maakte in Davos bij de voorbereiding een\", \"label\": \"nl\"},\n {\"text\": \"De ahí que en este mercado puedan negociarse\", \"label\": \"es\"},\n {\"text\": \"intenzívnějšímu sbírání a studiu.\", \"label\": \"cs\"},\n {\"text\": \"और औसकर ४.० पैकेज का प्रयोग किया गया है ।\", \"label\": \"hi\"},\n {\"text\": \"Adipati Kuningan karena Kuningan menjadi bagian\", \"label\": \"id\"},\n {\"text\": \"Svako je bar jednom poželeo da mašine prosto umeju\", \"label\": \"sr\"},\n {\"text\": \"Im vergangenen Jahr haben die Regierungen einen\", \"label\": \"de\"},\n {\"text\": \"durat motus, aliquid fit et non est;\", \"label\": \"la\"},\n {\"text\": \"Dominować będą piosenki do tekstów Edwarda\", \"label\": \"pl\"},\n {\"text\": \"beantwortet .\", \"label\": \"de\"},\n {\"text\": \"О гуманитариях было кому рассказывать, а вот за\", \"label\": \"ru\"},\n {\"text\": \"Helsingin kaupunki riitautti vuokrasopimuksen\", \"label\": \"fi\"},\n {\"text\": \"chợt tan biến.\", \"label\": \"vi\"},\n {\"text\": \"avtomobil ločuje od drugih.\", \"label\": \"sl\"},\n {\"text\": \"Congress has proven itself ineffective as a body.\", \"label\": \"en\"},\n {\"text\": \"मैक्सिको ने इस तरह का शो इस समय आयोजित करने का\", \"label\": \"hi\"},\n {\"text\": \"No minimum order amount.\", \"label\": \"en\"},\n {\"text\": \"Convertassa .\", \"label\": \"fi\"},\n {\"text\": \"Как это можно сделать?\", \"label\": \"ru\"},\n {\"text\": \"tha mi creidsinn gu robh iad ceart cho saor shuas\", \"label\": \"gd\"},\n {\"text\": \"실제 일제는 이런 만해의 논리를 묵살하고 한반도를 침략한 다음 , 이어 만주를 침략하고\", \"label\": \"ko\"},\n {\"text\": \"Da un semplice richiamo all'ordine fino a grandi\", \"label\": \"it\"},\n {\"text\": \"pozoruhodný nejen po umělecké stránce, jež\", \"label\": \"cs\"},\n {\"text\": \"La comida y el servicio aprueban.\", \"label\": \"es\"},\n {\"text\": \"again, connected not with each other but to the\", \"label\": \"en\"},\n {\"text\": \"Protokol výslovně stanoví, že nikdo nemůže být\", \"label\": \"cs\"},\n {\"text\": \"ఒక విషయం అడగాలని ఉంది .\", \"label\": \"te\"},\n {\"text\": \"Безгранично почитая дирекцию, ловя на лету каждое\", \"label\": \"ru\"},\n {\"text\": \"rovnoběžných růstových vrstev, zůstávají krychlové\", \"label\": \"cs\"},\n {\"text\": \"प्रवेश और पूर्व प्रधानमंत्री लाल बहादुर शास्त्री\", \"label\": \"hi\"},\n {\"text\": \"Bronzen medaille in de Europese marathon.\", \"label\": \"nl\"},\n {\"text\": \"- gadu vecumā viņi to nesaprot.\", \"label\": \"lv\"},\n {\"text\": \"Realizó sus estudios primarios en la Escuela Julia\", \"label\": \"es\"},\n {\"text\": \"cuartos de final, su clasificación para la final a\", \"label\": \"es\"},\n {\"text\": \"Sem si pro něho přiletí americký raketoplán, na\", \"label\": \"cs\"},\n {\"text\": \"Way to go!\", \"label\": \"en\"},\n {\"text\": \"gehört der neuen SPD-Führung unter Parteichef\", \"label\": \"de\"},\n {\"text\": \"Somit simuliert der Player mit einer GByte-Platte\", \"label\": \"de\"},\n {\"text\": \"Berufung auf kommissionsnahe Kreise , die bereits\", \"label\": \"de\"},\n {\"text\": \"Dist Clarïen\", \"label\": \"fro\"},\n {\"text\": \"Schon nach den Gerüchten , die Telekom wolle den\", \"label\": \"de\"},\n {\"text\": \"Software von NetObjects ist nach Angaben des\", \"label\": \"de\"},\n {\"text\": \"si enim per legem iustitia ergo Christus gratis\", \"label\": \"la\"},\n {\"text\": \"ducerent in ipsam magis quam in corpus christi,\", \"label\": \"la\"},\n {\"text\": \"Neustar-Melbourne-IT-Partnerschaft NeuLevel .\", \"label\": \"de\"},\n {\"text\": \"forderte dagegen seine drastische Verschärfung.\", \"label\": \"de\"},\n {\"text\": \"pemmican på hundrede forskellige måder.\", \"label\": \"da\"},\n {\"text\": \"Lehån, själv matematiklärare, visar hur den nya\", \"label\": \"sv\"},\n {\"text\": \"I highly recommend his shop.\", \"label\": \"en\"},\n {\"text\": \"verità, giovani fedeli prostratevi #amen\", \"label\": \"it\"},\n {\"text\": \"उत्तर प्रदेश के अध्यक्ष पद से हटाए गए विनय कटियार\", \"label\": \"hi\"},\n {\"text\": \"() روزی مےں کشادگی ہوتی ہے۔\", \"label\": \"ur\"},\n {\"text\": \"Prozessorgeschäft profitieren kann , stellen\", \"label\": \"de\"},\n {\"text\": \"školy začalo počítat pytle s moukou a zjistilo, že\", \"label\": \"cs\"},\n {\"text\": \"प्रभावशाली पर गैर सरकारी लोगों के घरों में भी\", \"label\": \"hi\"},\n {\"text\": \"geschichtslos , oder eine Farce , wie sich\", \"label\": \"de\"},\n {\"text\": \"Ústrednými mocnosťami v marci však spôsobilo, že\", \"label\": \"sk\"},\n {\"text\": \"التسليح بدون مبرر، واستمرار الأضرار الناجمة عن فرض\", \"label\": \"ar\"},\n {\"text\": \"Například Pedagogická fakulta Univerzity Karlovy\", \"label\": \"cs\"},\n {\"text\": \"nostris ut eriperet nos de praesenti saeculo\", \"label\": \"la\"}]\n \n docs = [Document([], text=example[\"text\"]) for example in examples]\n gold_labels = [example[\"label\"] for example in examples]\n basic_multilingual(docs)\n accuracy = sum([(doc.lang == label) for doc,label in zip(docs,gold_labels)])/len(docs)\n assert accuracy >= 0.98\n\n\ndef test_text_cleaning(basic_multilingual, clean_multilingual):\n \"\"\"\n Basic test of cleaning text\n \"\"\"\n docs = [\"Bonjour le monde! #thisisfrench #ilovefrance\",\n \"Bonjour le monde! https://t.co/U0Zjp3tusD\"]\n docs = [Document([], text=text) for text in docs]\n \n basic_multilingual(docs)\n assert [doc.lang for doc in docs] == [\"it\", \"it\"]\n \n assert clean_multilingual.processors[\"langid\"]._clean_text\n clean_multilingual(docs)\n assert [doc.lang for doc in docs] == [\"fr\", \"fr\"]\n\ndef test_emoji_cleaning():\n TEXT = [\"Sh'reyan has nice antennae :thumbs_up:\",\n \"This is🐱 a cat\"]\n EXPECTED = [\"Sh'reyan has nice antennae\",\n \"This is a cat\"]\n for text, expected in zip(TEXT, EXPECTED):\n assert LangIDProcessor.clean_text(text) == expected\n\ndef test_lang_subset(basic_multilingual, enfr_multilingual, en_multilingual):\n \"\"\"\n Basic test of restricting output to subset of languages\n \"\"\"\n docs = [\"Bonjour le monde! #thisisfrench #ilovefrance\",\n \"Bonjour le monde! https://t.co/U0Zjp3tusD\"]\n docs = [Document([], text=text) for text in docs]\n\n basic_multilingual(docs)\n assert [doc.lang for doc in docs] == [\"it\", \"it\"]\n\n assert enfr_multilingual.processors[\"langid\"]._model.lang_subset == [\"en\", \"fr\"]\n enfr_multilingual(docs)\n assert [doc.lang for doc in docs] == [\"fr\", \"fr\"]\n\n assert en_multilingual.processors[\"langid\"]._model.lang_subset == [\"en\"]\n en_multilingual(docs)\n assert [doc.lang for doc in docs] == [\"en\", \"en\"]\n\ndef test_lang_subset_unlikely_language(en_multilingual):\n \"\"\"\n Test that the language subset masking chooses a legal language, even if all legal languages are supa unlikely\n \"\"\"\n sentences = [\"你好\" * 200]\n docs = [Document([], text=text) for text in sentences]\n en_multilingual(docs)\n assert [doc.lang for doc in docs] == [\"en\"]\n\n processor = en_multilingual.processors['langid']\n model = processor._model\n text_tensor = processor._text_to_tensor(sentences)\n en_idx = model.tag_to_idx['en']\n predictions = model(text_tensor)\n assert predictions[0, en_idx] < 0, \"If this test fails, then regardless of how unlikely it was, the model is predicting the input string is possibly English. Update the test by picking a different combination of languages & input\"\n\n"
},
{
"path": "stanza/tests/langid/test_multilingual.py",
"content": "\"\"\"\nTests specifically for the MultilingualPipeline\n\"\"\"\n\nfrom collections import defaultdict\n\nimport pytest\n\nfrom stanza.pipeline.multilingual import MultilingualPipeline\n\nfrom stanza.tests import TEST_MODELS_DIR\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\ndef run_multilingual_pipeline(en_has_dependencies=True, fr_has_dependencies=True, **kwargs):\n english_text = \"This is an English sentence.\"\n english_words = [\"This\", \"is\", \"an\", \"English\", \"sentence\", \".\"]\n english_deps_gold = \"\\n\".join((\n \"('This', 5, 'nsubj')\",\n \"('is', 5, 'cop')\",\n \"('an', 5, 'det')\",\n \"('English', 5, 'amod')\",\n \"('sentence', 0, 'root')\",\n \"('.', 5, 'punct')\"\n ))\n if not en_has_dependencies:\n english_deps_gold = \"\"\n\n french_text = \"C'est une phrase française.\"\n french_words = [\"C'\", \"est\", \"une\", \"phrase\", \"française\", \".\"]\n french_deps_gold = \"\\n\".join((\n \"(\\\"C'\\\", 4, 'nsubj')\",\n \"('est', 4, 'cop')\",\n \"('une', 4, 'det')\",\n \"('phrase', 0, 'root')\",\n \"('française', 4, 'amod')\",\n \"('.', 4, 'punct')\"\n ))\n if not fr_has_dependencies:\n french_deps_gold = \"\"\n\n if 'lang_configs' in kwargs:\n nlp = MultilingualPipeline(model_dir=TEST_MODELS_DIR, download_method=None, **kwargs)\n else:\n lang_configs = {\"en\": {\"processors\": \"tokenize,pos,lemma,depparse\"},\n \"fr\": {\"processors\": \"tokenize,pos,lemma,depparse\"}}\n nlp = MultilingualPipeline(model_dir=TEST_MODELS_DIR, download_method=None, lang_configs=lang_configs, **kwargs)\n docs = [english_text, french_text]\n docs = nlp(docs)\n\n assert docs[0].lang == \"en\"\n assert len(docs[0].sentences) == 1\n assert [x.text for x in docs[0].sentences[0].words] == english_words\n assert docs[0].sentences[0].dependencies_string() == english_deps_gold\n\n assert len(docs[1].sentences) == 1\n assert docs[1].lang == \"fr\"\n assert [x.text for x in docs[1].sentences[0].words] == french_words\n assert docs[1].sentences[0].dependencies_string() == french_deps_gold\n\n\ndef test_multilingual_pipeline():\n \"\"\"\n Basic test of multilingual pipeline\n \"\"\"\n run_multilingual_pipeline()\n\ndef test_multilingual_pipeline_small_cache():\n \"\"\"\n Test with the cache size 1\n \"\"\"\n run_multilingual_pipeline(max_cache_size=1)\n\n\ndef test_multilingual_config():\n \"\"\"\n Test with only tokenize for the EN pipeline\n \"\"\"\n lang_configs = {\n \"en\": {\"processors\": \"tokenize\"}\n }\n\n run_multilingual_pipeline(en_has_dependencies=False, lang_configs=lang_configs)\n\ndef test_multilingual_processors_limited():\n \"\"\"\n Test loading an available subset of processors\n \"\"\"\n run_multilingual_pipeline(en_has_dependencies=False, fr_has_dependencies=False, lang_configs={}, processors=\"tokenize\")\n run_multilingual_pipeline(en_has_dependencies=True, fr_has_dependencies=False, lang_configs={\"en\": {\"processors\": \"tokenize,pos,lemma,depparse\"}}, processors=\"tokenize\")\n # this should not fail, as it will drop the zzzzzzzzzz processor for the languages which don't have it\n run_multilingual_pipeline(en_has_dependencies=False, fr_has_dependencies=False, lang_configs={}, processors=\"tokenize,zzzzzzzzzz\")\n\n\ndef test_defaultdict_config():\n \"\"\"\n Test that you can pass in a defaultdict for the lang_configs argument\n \"\"\"\n lang_configs = defaultdict(lambda: dict(processors=\"tokenize\"))\n run_multilingual_pipeline(en_has_dependencies=False, fr_has_dependencies=False, lang_configs=lang_configs)\n\n lang_configs = defaultdict(lambda: dict(processors=\"tokenize\"))\n lang_configs[\"en\"] = {\"processors\": \"tokenize,pos,lemma,depparse\"}\n run_multilingual_pipeline(en_has_dependencies=True, fr_has_dependencies=False, lang_configs=lang_configs)\n"
},
{
"path": "stanza/tests/lemma/__init__.py",
"content": ""
},
{
"path": "stanza/tests/lemma/test_data.py",
"content": "\"\"\"\nTest a couple basic data functions, such as processing a doc for its lemmas\n\"\"\"\n\nimport pytest\n\nfrom stanza.models.common.doc import Document\nfrom stanza.models.lemma.data import DataLoader\nfrom stanza.utils.conll import CoNLL\n\nTRAIN_DATA = \"\"\"\n# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0003\n# text = DPA: Iraqi authorities announced that they had busted up 3 terrorist cells operating in Baghdad.\n1\tDPA\tDPA\tPROPN\tNNP\tNumber=Sing\t0\troot\t0:root\tSpaceAfter=No\n2\t:\t:\tPUNCT\t:\t_\t1\tpunct\t1:punct\t_\n3\tIraqi\tIraqi\tADJ\tJJ\tDegree=Pos\t4\tamod\t4:amod\t_\n4\tauthorities\tauthority\tNOUN\tNNS\tNumber=Plur\t5\tnsubj\t5:nsubj\t_\n5\tannounced\tannounce\tVERB\tVBD\tMood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin\t1\tparataxis\t1:parataxis\t_\n6\tthat\tthat\tSCONJ\tIN\t_\t9\tmark\t9:mark\t_\n7\tthey\tthey\tPRON\tPRP\tCase=Nom|Number=Plur|Person=3|PronType=Prs\t9\tnsubj\t9:nsubj\t_\n8\thad\thave\tAUX\tVBD\tMood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin\t9\taux\t9:aux\t_\n9\tbusted\tbust\tVERB\tVBN\tTense=Past|VerbForm=Part\t5\tccomp\t5:ccomp\t_\n10\tup\tup\tADP\tRP\t_\t9\tcompound:prt\t9:compound:prt\t_\n11\t3\t3\tNUM\tCD\tNumForm=Digit|NumType=Card\t13\tnummod\t13:nummod\t_\n12\tterrorist\tterrorist\tADJ\tJJ\tDegree=Pos\t13\tamod\t13:amod\t_\n13\tcells\tcell\tNOUN\tNNS\tNumber=Plur\t9\tobj\t9:obj\t_\n14\toperating\toperate\tVERB\tVBG\tVerbForm=Ger\t13\tacl\t13:acl\t_\n15\tin\tin\tADP\tIN\t_\t16\tcase\t16:case\t_\n16\tBaghdad\tBaghdad\tPROPN\tNNP\tNumber=Sing\t14\tobl\t14:obl:in\tSpaceAfter=No\n17\t.\t.\tPUNCT\t.\t_\t1\tpunct\t1:punct\t_\n\n# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0004\n# text = Two of them were being run by 2 officials of the Ministry of the Interior!\n1\tTwo\ttwo\tNUM\tCD\tNumForm=Word|NumType=Card\t6\tnsubj:pass\t6:nsubj:pass\t_\n2\tof\tof\tADP\tIN\t_\t3\tcase\t3:case\t_\n3\tthem\tthey\tPRON\tPRP\tCase=Acc|Number=Plur|Person=3|PronType=Prs\t1\tnmod\t1:nmod:of\t_\n4\twere\tbe\tAUX\tVBD\tMood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin\t6\taux\t6:aux\t_\n5\tbeing\tbe\tAUX\tVBG\tVerbForm=Ger\t6\taux:pass\t6:aux:pass\t_\n6\trun\trun\tVERB\tVBN\tTense=Past|VerbForm=Part|Voice=Pass\t0\troot\t0:root\t_\n7\tby\tby\tADP\tIN\t_\t9\tcase\t9:case\t_\n8\t2\t2\tNUM\tCD\tNumForm=Digit|NumType=Card\t9\tnummod\t9:nummod\t_\n9\tofficials\tofficial\tNOUN\tNNS\tNumber=Plur\t6\tobl\t6:obl:by\t_\n10\tof\tof\tADP\tIN\t_\t12\tcase\t12:case\t_\n11\tthe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t12\tdet\t12:det\t_\n12\tMinistry\tMinistry\tPROPN\tNNP\tNumber=Sing\t9\tnmod\t9:nmod:of\t_\n13\tof\tof\tADP\tIN\t_\t15\tcase\t15:case\t_\n14\tthe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t15\tdet\t15:det\t_\n15\tInterior\tInterior\tPROPN\tNNP\tNumber=Sing\t12\tnmod\t12:nmod:of\tSpaceAfter=No\n16\t!\t!\tPUNCT\t.\t_\t6\tpunct\t6:punct\t_\n\n\"\"\".lstrip()\n\nGOESWITH_DATA = \"\"\"\n# sent_id = email-enronsent27_01-0041\n# newpar id = email-enronsent27_01-p0005\n# text = Ken Rice@ENRON COMMUNICATIONS\n1\tKen\tkenrice@enroncommunications\tX\tGW\tTypo=Yes\t0\troot\t0:root\t_\n2\tRice@ENRON\t_\tX\tGW\t_\t1\tgoeswith\t1:goeswith\t_\n3\tCOMMUNICATIONS\t_\tX\tADD\t_\t1\tgoeswith\t1:goeswith\t_\n\n\"\"\".lstrip()\n\nCORRECT_FORM_DATA = \"\"\"\n# sent_id = weblog-blogspot.com_healingiraq_20040409053012_ENG_20040409_053012-0019\n# text = They are targetting ambulances\n1\tThey\tthey\tPRON\tPRP\tCase=Nom|Number=Plur|Person=3|PronType=Prs\t3\tnsubj\t3:nsubj\t_\n2\tare\tbe\tAUX\tVBP\tMood=Ind|Number=Plur|Person=3|Tense=Pres|VerbForm=Fin\t3\taux\t3:aux\t_\n3\ttargetting\ttarget\tVERB\tVBG\tTense=Pres|Typo=Yes|VerbForm=Part\t0\troot\t0:root\tCorrectForm=targeting\n4\tambulances\tambulance\tNOUN\tNNS\tNumber=Plur\t3\tobj\t3:obj\tSpaceAfter=No\n\"\"\"\n\nBLANKS_DATA = \"\"\"\n# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0018\n# text = Guerrillas killed an engineer, Asi Ali, from Tikrit.\n1\tGuerrillas\t_\tNOUN\tNNS\tNumber=Plur\t2\tnsubj\t2:nsubj\t_\n2\tkilled\t_\tVERB\tVBD\tMood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin\t0\troot\t0:root\t_\n3\tan\ta\tDET\tDT\tDefinite=Ind|PronType=Art\t4\tdet\t4:det\t_\n4\tengineer\t_\tNOUN\tNN\tNumber=Sing\t2\tobj\t2:obj\tSpaceAfter=No\n\n\"\"\".lstrip()\n\n\ndef test_load_document():\n train_doc = CoNLL.conll2doc(input_str=TRAIN_DATA)\n data = DataLoader.load_doc(train_doc, caseless=False, skip_blank_lemmas=False, evaluation=True)\n assert len(data) == 33 # meticulously counted by hand\n assert all(len(x) == 3 for x in data)\n\n data = DataLoader.load_doc(train_doc, caseless=False, skip_blank_lemmas=False, evaluation=False)\n assert len(data) == 33\n assert all(len(x) == 3 for x in data)\n\ndef test_load_goeswith():\n raw_data = TRAIN_DATA + GOESWITH_DATA\n train_doc = CoNLL.conll2doc(input_str=raw_data)\n data = DataLoader.load_doc(train_doc, caseless=False, skip_blank_lemmas=False, evaluation=True)\n assert len(data) == 36 # will be the same as in test_load_document with three additional words\n assert all(len(x) == 3 for x in data)\n\n data = DataLoader.load_doc(train_doc, caseless=False, skip_blank_lemmas=False, evaluation=False)\n assert len(data) == 33 # will be the same as in test_load_document, but with the trailing 3 GOESWITH removed\n assert all(len(x) == 3 for x in data)\n\ndef test_correct_form():\n raw_data = TRAIN_DATA + CORRECT_FORM_DATA\n train_doc = CoNLL.conll2doc(input_str=raw_data)\n data = DataLoader.load_doc(train_doc, caseless=False, skip_blank_lemmas=False, evaluation=True)\n assert len(data) == 37\n # the 'targeting' correction should not be applied if evaluation=True\n # when evaluation=False, then the CorrectForms will be applied\n assert not any(x[0] == 'targeting' for x in data)\n\n data = DataLoader.load_doc(train_doc, caseless=False, skip_blank_lemmas=False, evaluation=False)\n assert len(data) == 38 # the same, but with an extra row so the model learns both 'targetting' and 'targeting'\n assert any(x[0] == 'targeting' for x in data)\n assert any(x[0] == 'targetting' for x in data)\n\ndef test_load_blank():\n raw_data = TRAIN_DATA + BLANKS_DATA\n train_doc = CoNLL.conll2doc(input_str=raw_data)\n data = DataLoader.load_doc(train_doc, caseless=False, skip_blank_lemmas=False, evaluation=False)\n assert len(data) == 37 # will be the same as in test_load_document with FOUR additional words\n assert all(len(x) == 3 for x in data)\n\n data = DataLoader.load_doc(train_doc, caseless=False, skip_blank_lemmas=True, evaluation=False)\n assert len(data) == 34 # will be the same as in test_load_document, but one extra word is added. others were blank\n assert all(len(x) == 3 for x in data)\n\n"
},
{
"path": "stanza/tests/lemma/test_lemma_trainer.py",
"content": "\"\"\"\nTest a couple basic functions - load & save an existing model\n\"\"\"\n\nimport pytest\n\nimport glob\nimport os\nimport tempfile\n\nimport torch\n\nfrom stanza.models import lemmatizer\nfrom stanza.models.lemma import trainer\nfrom stanza.tests import *\nfrom stanza.utils.training.common import choose_lemma_charlm, build_charlm_args\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\n@pytest.fixture(scope=\"module\")\ndef english_model():\n models_path = os.path.join(TEST_MODELS_DIR, \"en\", \"lemma\", \"*\")\n models = glob.glob(models_path)\n # we expect at least one English model downloaded for the tests\n assert len(models) >= 1, \"No English lemma models downloaded during setup! Please make sure to run the setup script.\"\n for model_file in models:\n if \"nocharlm\" in model_file:\n return trainer.Trainer(model_file=model_file)\n raise FileNotFoundError(\"Should have downloaded the nocharlm English lemmatizer during setup. Please rerun the setup script.\")\n\ndef test_load_model(english_model):\n \"\"\"\n Does nothing, just tests that loading works\n \"\"\"\n\ndef test_save_load_model(english_model):\n \"\"\"\n Load, save, and load again\n \"\"\"\n with tempfile.TemporaryDirectory() as tempdir:\n save_file = os.path.join(tempdir, \"resaved\", \"lemma.pt\")\n english_model.save(save_file)\n reloaded = trainer.Trainer(model_file=save_file)\n\nTRAIN_DATA = \"\"\"\n# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0003\n# text = DPA: Iraqi authorities announced that they had busted up 3 terrorist cells operating in Baghdad.\n1\tDPA\tDPA\tPROPN\tNNP\tNumber=Sing\t0\troot\t0:root\tSpaceAfter=No\n2\t:\t:\tPUNCT\t:\t_\t1\tpunct\t1:punct\t_\n3\tIraqi\tIraqi\tADJ\tJJ\tDegree=Pos\t4\tamod\t4:amod\t_\n4\tauthorities\tauthority\tNOUN\tNNS\tNumber=Plur\t5\tnsubj\t5:nsubj\t_\n5\tannounced\tannounce\tVERB\tVBD\tMood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin\t1\tparataxis\t1:parataxis\t_\n6\tthat\tthat\tSCONJ\tIN\t_\t9\tmark\t9:mark\t_\n7\tthey\tthey\tPRON\tPRP\tCase=Nom|Number=Plur|Person=3|PronType=Prs\t9\tnsubj\t9:nsubj\t_\n8\thad\thave\tAUX\tVBD\tMood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin\t9\taux\t9:aux\t_\n9\tbusted\tbust\tVERB\tVBN\tTense=Past|VerbForm=Part\t5\tccomp\t5:ccomp\t_\n10\tup\tup\tADP\tRP\t_\t9\tcompound:prt\t9:compound:prt\t_\n11\t3\t3\tNUM\tCD\tNumForm=Digit|NumType=Card\t13\tnummod\t13:nummod\t_\n12\tterrorist\tterrorist\tADJ\tJJ\tDegree=Pos\t13\tamod\t13:amod\t_\n13\tcells\tcell\tNOUN\tNNS\tNumber=Plur\t9\tobj\t9:obj\t_\n14\toperating\toperate\tVERB\tVBG\tVerbForm=Ger\t13\tacl\t13:acl\t_\n15\tin\tin\tADP\tIN\t_\t16\tcase\t16:case\t_\n16\tBaghdad\tBaghdad\tPROPN\tNNP\tNumber=Sing\t14\tobl\t14:obl:in\tSpaceAfter=No\n17\t.\t.\tPUNCT\t.\t_\t1\tpunct\t1:punct\t_\n\n# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0004\n# text = Two of them were being run by 2 officials of the Ministry of the Interior!\n1\tTwo\ttwo\tNUM\tCD\tNumForm=Word|NumType=Card\t6\tnsubj:pass\t6:nsubj:pass\t_\n2\tof\tof\tADP\tIN\t_\t3\tcase\t3:case\t_\n3\tthem\tthey\tPRON\tPRP\tCase=Acc|Number=Plur|Person=3|PronType=Prs\t1\tnmod\t1:nmod:of\t_\n4\twere\tbe\tAUX\tVBD\tMood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin\t6\taux\t6:aux\t_\n5\tbeing\tbe\tAUX\tVBG\tVerbForm=Ger\t6\taux:pass\t6:aux:pass\t_\n6\trun\trun\tVERB\tVBN\tTense=Past|VerbForm=Part|Voice=Pass\t0\troot\t0:root\t_\n7\tby\tby\tADP\tIN\t_\t9\tcase\t9:case\t_\n8\t2\t2\tNUM\tCD\tNumForm=Digit|NumType=Card\t9\tnummod\t9:nummod\t_\n9\tofficials\tofficial\tNOUN\tNNS\tNumber=Plur\t6\tobl\t6:obl:by\t_\n10\tof\tof\tADP\tIN\t_\t12\tcase\t12:case\t_\n11\tthe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t12\tdet\t12:det\t_\n12\tMinistry\tMinistry\tPROPN\tNNP\tNumber=Sing\t9\tnmod\t9:nmod:of\t_\n13\tof\tof\tADP\tIN\t_\t15\tcase\t15:case\t_\n14\tthe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t15\tdet\t15:det\t_\n15\tInterior\tInterior\tPROPN\tNNP\tNumber=Sing\t12\tnmod\t12:nmod:of\tSpaceAfter=No\n16\t!\t!\tPUNCT\t.\t_\t6\tpunct\t6:punct\t_\n\n\"\"\".lstrip()\n\nDEV_DATA = \"\"\"\n1\tFrom\tfrom\tADP\tIN\t_\t3\tcase\t3:case\t_\n2\tthe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t3\tdet\t3:det\t_\n3\tAP\tAP\tPROPN\tNNP\tNumber=Sing\t4\tobl\t4:obl:from\t_\n4\tcomes\tcome\tVERB\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t0\troot\t0:root\t_\n5\tthis\tthis\tDET\tDT\tNumber=Sing|PronType=Dem\t6\tdet\t6:det\t_\n6\tstory\tstory\tNOUN\tNN\tNumber=Sing\t4\tnsubj\t4:nsubj\t_\n7\t:\t:\tPUNCT\t:\t_\t4\tpunct\t4:punct\t_\n\n\"\"\".lstrip()\n\nclass TestLemmatizer:\n @pytest.fixture(scope=\"class\")\n def charlm_args(self):\n charlm = choose_lemma_charlm(\"en\", \"test\", \"default\")\n charlm_args = build_charlm_args(\"en\", charlm, model_dir=TEST_MODELS_DIR)\n return charlm_args\n\n\n def run_training(self, tmp_path, train_text, dev_text, extra_args=None):\n \"\"\"\n Run the training for a few iterations, load & return the model\n \"\"\"\n pred_file = str(tmp_path / \"pred.conllu\")\n\n save_name = \"test_tagger.pt\"\n save_file = str(tmp_path / save_name)\n\n train_file = str(tmp_path / \"train.conllu\")\n with open(train_file, \"w\", encoding=\"utf-8\") as fout:\n fout.write(train_text)\n\n dev_file = str(tmp_path / \"dev.conllu\")\n with open(dev_file, \"w\", encoding=\"utf-8\") as fout:\n fout.write(dev_text)\n\n args = [\"--train_file\", train_file,\n \"--eval_file\", dev_file,\n \"--output_file\", pred_file,\n \"--num_epoch\", \"2\",\n \"--log_step\", \"10\",\n \"--save_dir\", str(tmp_path),\n \"--save_name\", save_name,\n \"--shorthand\", \"en_test\"]\n if extra_args is not None:\n args = args + extra_args\n lemmatizer.main(args)\n\n assert os.path.exists(save_file)\n saved_model = trainer.Trainer(model_file=save_file)\n return saved_model\n\n def test_basic_train(self, tmp_path):\n \"\"\"\n Simple test of a few 'epochs' of lemmatizer training\n \"\"\"\n self.run_training(tmp_path, TRAIN_DATA, DEV_DATA)\n\n def test_charlm_train(self, tmp_path, charlm_args):\n \"\"\"\n Simple test of a few 'epochs' of lemmatizer training\n \"\"\"\n saved_model = self.run_training(tmp_path, TRAIN_DATA, DEV_DATA, extra_args=charlm_args)\n\n # check that the charlm wasn't saved in here\n args = saved_model.args\n save_name = os.path.join(args['save_dir'], args['save_name'])\n checkpoint = torch.load(save_name, lambda storage, loc: storage, weights_only=True)\n assert not any(x.startswith(\"contextual_embedding\") for x in checkpoint['model'].keys())\n"
},
{
"path": "stanza/tests/lemma/test_lowercase.py",
"content": "import pytest\n\nfrom stanza.models.lemmatizer import all_lowercase\nfrom stanza.utils.conll import CoNLL\n\nLATIN_CONLLU = \"\"\"\n# sent_id = train-s1\n# text = unde et philosophus dicit felicitatem esse operationem perfectam.\n# reference = ittb-scg-s4203\n1\tunde\tunde\tADV\tO4\tAdvType=Loc|PronType=Rel\t4\tadvmod:lmod\t_\t_\n2\tet\tet\tCCONJ\tO4\t_\t3\tadvmod:emph\t_\t_\n3\tphilosophus\tphilosophus\tNOUN\tB1|grn1|casA|gen1\tCase=Nom|Gender=Masc|InflClass=IndEurO|Number=Sing\t4\tnsubj\t_\t_\n4\tdicit\tdico\tVERB\tN3|modA|tem1|gen6\tAspect=Imp|InflClass=LatX|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin|Voice=Act\t0\troot\t_\tTraditionalMood=Indicativus|TraditionalTense=Praesens\n5\tfelicitatem\tfelicitas\tNOUN\tC1|grn1|casD|gen2\tCase=Acc|Gender=Fem|InflClass=IndEurX|Number=Sing\t7\tnsubj\t_\t_\n6\tesse\tsum\tAUX\tN3|modH|tem1\tAspect=Imp|Tense=Pres|VerbForm=Inf\t7\tcop\t_\t_\n7\toperationem\toperatio\tNOUN\tC1|grn1|casD|gen2|vgr1\tCase=Acc|Gender=Fem|InflClass=IndEurX|Number=Sing\t4\tccomp\t_\t_\n8\tperfectam\tperfectus\tADJ\tA1|grn1|casD|gen2\tCase=Acc|Gender=Fem|InflClass=IndEurA|Number=Sing\t7\tamod\t_\tSpaceAfter=No\n9\t.\t.\tPUNCT\tPunc\t_\t4\tpunct\t_\t_\n\n# sent_id = train-s2\n# text = perfectio autem operationis dependet ex quatuor.\n# reference = ittb-scg-s4204\n1\tperfectio\tperfectio\tNOUN\tC1|grn1|casA|gen2\tCase=Nom|Gender=Fem|InflClass=IndEurX|Number=Sing\t4\tnsubj\t_\t_\n2\tautem\tautem\tPART\tO4\t_\t4\tdiscourse\t_\t_\n3\toperationis\toperatio\tNOUN\tC1|grn1|casB|gen2|vgr1\tCase=Gen|Gender=Fem|InflClass=IndEurX|Number=Sing\t1\tnmod\t_\t_\n4\tdependet\tdependeo\tVERB\tK3|modA|tem1|gen6\tAspect=Imp|InflClass=LatE|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin|Voice=Act\t0\troot\t_\tTraditionalMood=Indicativus|TraditionalTense=Praesens\n5\tex\tex\tADP\tS4|vgr2\t_\t6\tcase\t_\t_\n6\tquatuor\tquattuor\tNUM\tG1|gen3|vgr1\tNumForm=Word|NumType=Card\t4\tobl:arg\t_\tSpaceAfter=No\n7\t.\t.\tPUNCT\tPunc\t_\t4\tpunct\t_\t_\n\"\"\".lstrip()\n\nENG_CONLLU = \"\"\"\n# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0007\n# text = You wonder if he was manipulating the market with his bombing targets.\n1\tYou\tyou\tPRON\tPRP\tCase=Nom|Person=2|PronType=Prs\t2\tnsubj\t2:nsubj\t_\n2\twonder\twonder\tVERB\tVBP\tMood=Ind|Number=Sing|Person=2|Tense=Pres|VerbForm=Fin\t0\troot\t0:root\t_\n3\tif\tif\tSCONJ\tIN\t_\t6\tmark\t6:mark\t_\n4\the\the\tPRON\tPRP\tCase=Nom|Gender=Masc|Number=Sing|Person=3|PronType=Prs\t6\tnsubj\t6:nsubj\t_\n5\twas\tbe\tAUX\tVBD\tMood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin\t6\taux\t6:aux\t_\n6\tmanipulating\tmanipulate\tVERB\tVBG\tTense=Pres|VerbForm=Part\t2\tccomp\t2:ccomp\t_\n7\tthe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t8\tdet\t8:det\t_\n8\tmarket\tmarket\tNOUN\tNN\tNumber=Sing\t6\tobj\t6:obj\t_\n9\twith\twith\tADP\tIN\t_\t12\tcase\t12:case\t_\n10\this\this\tPRON\tPRP$\tCase=Gen|Gender=Masc|Number=Sing|Person=3|Poss=Yes|PronType=Prs\t12\tnmod:poss\t12:nmod:poss\t_\n11\tbombing\tbombing\tNOUN\tNN\tNumber=Sing\t12\tcompound\t12:compound\t_\n12\ttargets\ttarget\tNOUN\tNNS\tNumber=Plur\t6\tobl\t6:obl:with\tSpaceAfter=No\n13\t.\t.\tPUNCT\t.\t_\t2\tpunct\t2:punct\t_\n\"\"\".lstrip()\n\n\ndef test_all_lowercase():\n doc = CoNLL.conll2doc(input_str=LATIN_CONLLU)\n assert all_lowercase(doc)\n\ndef test_not_all_lowercase():\n doc = CoNLL.conll2doc(input_str=ENG_CONLLU)\n assert not all_lowercase(doc)\n"
},
{
"path": "stanza/tests/lemma_classifier/__init__.py",
"content": ""
},
{
"path": "stanza/tests/lemma_classifier/test_data_preparation.py",
"content": "import os\n\nimport pytest\n\nimport stanza.models.lemma_classifier.utils as utils\nimport stanza.utils.datasets.prepare_lemma_classifier as prepare_lemma_classifier\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\nEWT_ONE_SENTENCE = \"\"\"\n# sent_id = weblog-blogspot.com_rigorousintuition_20050518101500_ENG_20050518_101500-0002\n# newpar id = weblog-blogspot.com_rigorousintuition_20050518101500_ENG_20050518_101500-p0002\n# text = Here's a Miami Herald interview\n1-2\tHere's\t_\t_\t_\t_\t_\t_\t_\t_\n1\tHere\there\tADV\tRB\tPronType=Dem\t0\troot\t0:root\t_\n2\t's\tbe\tAUX\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t1\tcop\t1:cop\t_\n3\ta\ta\tDET\tDT\tDefinite=Ind|PronType=Art\t6\tdet\t6:det\t_\n4\tMiami\tMiami\tPROPN\tNNP\tNumber=Sing\t5\tcompound\t5:compound\t_\n5\tHerald\tHerald\tPROPN\tNNP\tNumber=Sing\t6\tcompound\t6:compound\t_\n6\tinterview\tinterview\tNOUN\tNN\tNumber=Sing\t1\tnsubj\t1:nsubj\t_\n\"\"\".lstrip()\n\n\nEWT_TRAIN_SENTENCES = \"\"\"\n# sent_id = weblog-blogspot.com_rigorousintuition_20050518101500_ENG_20050518_101500-0002\n# newpar id = weblog-blogspot.com_rigorousintuition_20050518101500_ENG_20050518_101500-p0002\n# text = Here's a Miami Herald interview\n1-2\tHere's\t_\t_\t_\t_\t_\t_\t_\t_\n1\tHere\there\tADV\tRB\tPronType=Dem\t0\troot\t0:root\t_\n2\t's\tbe\tAUX\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t1\tcop\t1:cop\t_\n3\ta\ta\tDET\tDT\tDefinite=Ind|PronType=Art\t6\tdet\t6:det\t_\n4\tMiami\tMiami\tPROPN\tNNP\tNumber=Sing\t5\tcompound\t5:compound\t_\n5\tHerald\tHerald\tPROPN\tNNP\tNumber=Sing\t6\tcompound\t6:compound\t_\n6\tinterview\tinterview\tNOUN\tNN\tNumber=Sing\t1\tnsubj\t1:nsubj\t_\n\n# sent_id = weblog-blogspot.com_rigorousintuition_20050518101500_ENG_20050518_101500-0027\n# text = But Posada's nearly 80 years old\n1\tBut\tbut\tCCONJ\tCC\t_\t7\tcc\t7:cc\t_\n2-3\tPosada's\t_\t_\t_\t_\t_\t_\t_\t_\n2\tPosada\tPosada\tPROPN\tNNP\tNumber=Sing\t7\tnsubj\t7:nsubj\t_\n3\t's\tbe\tAUX\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t7\tcop\t7:cop\t_\n4\tnearly\tnearly\tADV\tRB\t_\t5\tadvmod\t5:advmod\t_\n5\t80\t80\tNUM\tCD\tNumForm=Digit|NumType=Card\t6\tnummod\t6:nummod\t_\n6\tyears\tyear\tNOUN\tNNS\tNumber=Plur\t7\tobl:npmod\t7:obl:npmod\t_\n7\told\told\tADJ\tJJ\tDegree=Pos\t0\troot\t0:root\tSpaceAfter=No\n\n# sent_id = weblog-blogspot.com_rigorousintuition_20050518101500_ENG_20050518_101500-0067\n# newpar id = weblog-blogspot.com_rigorousintuition_20050518101500_ENG_20050518_101500-p0011\n# text = Now that's a post I can relate to.\n1\tNow\tnow\tADV\tRB\t_\t5\tadvmod\t5:advmod\t_\n2-3\tthat's\t_\t_\t_\t_\t_\t_\t_\t_\n2\tthat\tthat\tPRON\tDT\tNumber=Sing|PronType=Dem\t5\tnsubj\t5:nsubj\t_\n3\t's\tbe\tAUX\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t5\tcop\t5:cop\t_\n4\ta\ta\tDET\tDT\tDefinite=Ind|PronType=Art\t5\tdet\t5:det\t_\n5\tpost\tpost\tNOUN\tNN\tNumber=Sing\t0\troot\t0:root\t_\n6\tI\tI\tPRON\tPRP\tCase=Nom|Number=Sing|Person=1|PronType=Prs\t8\tnsubj\t8:nsubj\t_\n7\tcan\tcan\tAUX\tMD\tVerbForm=Fin\t8\taux\t8:aux\t_\n8\trelate\trelate\tVERB\tVB\tVerbForm=Inf\t5\tacl:relcl\t5:acl:relcl\t_\n9\tto\tto\tADP\tIN\t_\t8\tobl\t8:obl\tSpaceAfter=No\n10\t.\t.\tPUNCT\t.\t_\t5\tpunct\t5:punct\t_\n\n# sent_id = weblog-blogspot.com_rigorousintuition_20050518101500_ENG_20050518_101500-0073\n# newpar id = weblog-blogspot.com_rigorousintuition_20050518101500_ENG_20050518_101500-p0012\n# text = hey that's a great blog\n1\they\they\tINTJ\tUH\t_\t6\tdiscourse\t6:discourse\t_\n2-3\tthat's\t_\t_\t_\t_\t_\t_\t_\t_\n2\tthat\tthat\tPRON\tDT\tNumber=Sing|PronType=Dem\t6\tnsubj\t6:nsubj\t_\n3\t's\tbe\tAUX\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t6\tcop\t6:cop\t_\n4\ta\ta\tDET\tDT\tDefinite=Ind|PronType=Art\t6\tdet\t6:det\t_\n5\tgreat\tgreat\tADJ\tJJ\tDegree=Pos\t6\tamod\t6:amod\t_\n6\tblog\tblog\tNOUN\tNN\tNumber=Sing\t0\troot\t0:root\tSpaceAfter=No\n\n# sent_id = weblog-blogspot.com_rigorousintuition_20050518101500_ENG_20050518_101500-0089\n# text = And It's Not Hard To Do\n1\tAnd\tand\tCCONJ\tCC\t_\t5\tcc\t5:cc\t_\n2-3\tIt's\t_\t_\t_\t_\t_\t_\t_\t_\n2\tIt\tit\tPRON\tPRP\tCase=Nom|Gender=Neut|Number=Sing|Person=3|PronType=Prs\t5\texpl\t5:expl\t_\n3\t's\tbe\tAUX\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t5\tcop\t5:cop\t_\n4\tNot\tnot\tPART\tRB\t_\t5\tadvmod\t5:advmod\t_\n5\tHard\thard\tADJ\tJJ\tDegree=Pos\t0\troot\t0:root\t_\n6\tTo\tto\tPART\tTO\t_\t7\tmark\t7:mark\t_\n7\tDo\tdo\tVERB\tVB\tVerbForm=Inf\t5\tcsubj\t5:csubj\tSpaceAfter=No\n\n# sent_id = weblog-blogspot.com_rigorousintuition_20060511134300_ENG_20060511_134300-0029\n# text = Meanwhile, a decision's been reached\n1\tMeanwhile\tmeanwhile\tADV\tRB\t_\t7\tadvmod\t7:advmod\tSpaceAfter=No\n2\t,\t,\tPUNCT\t,\t_\t1\tpunct\t1:punct\t_\n3\ta\ta\tDET\tDT\tDefinite=Ind|PronType=Art\t4\tdet\t4:det\t_\n4-5\tdecision's\t_\t_\t_\t_\t_\t_\t_\t_\n4\tdecision\tdecision\tNOUN\tNN\tNumber=Sing\t7\tnsubj:pass\t7:nsubj:pass\t_\n5\t's\thave\tAUX\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t7\taux\t7:aux\t_\n6\tbeen\tbe\tAUX\tVBN\tTense=Past|VerbForm=Part\t7\taux:pass\t7:aux:pass\t_\n7\treached\treach\tVERB\tVBN\tTense=Past|VerbForm=Part|Voice=Pass\t0\troot\t0:root\t_\n\n# sent_id = weblog-blogspot.com_rigorousintuition_20060511134300_ENG_20060511_134300-0138\n# text = It's become a guardian of morality\n1-2\tIt's\t_\t_\t_\t_\t_\t_\t_\t_\n1\tIt\tit\tPRON\tPRP\tCase=Nom|Gender=Neut|Number=Sing|Person=3|PronType=Prs\t3\tnsubj\t3:nsubj|5:nsubj:xsubj\t_\n2\t's\thave\tAUX\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t3\taux\t3:aux\t_\n3\tbecome\tbecome\tVERB\tVBN\tTense=Past|VerbForm=Part\t0\troot\t0:root\t_\n4\ta\ta\tDET\tDT\tDefinite=Ind|PronType=Art\t5\tdet\t5:det\t_\n5\tguardian\tguardian\tNOUN\tNN\tNumber=Sing\t3\txcomp\t3:xcomp\t_\n6\tof\tof\tADP\tIN\t_\t7\tcase\t7:case\t_\n7\tmorality\tmorality\tNOUN\tNN\tNumber=Sing\t5\tnmod\t5:nmod:of\t_\n\n# sent_id = email-enronsent15_01-0018\n# text = It's got its own bathroom and tv\n1-2\tIt's\t_\t_\t_\t_\t_\t_\t_\t_\n1\tIt\tit\tPRON\tPRP\tCase=Nom|Gender=Neut|Number=Sing|Person=3|PronType=Prs\t3\tnsubj\t3:nsubj|13:nsubj\t_\n2\t's\thave\tAUX\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t3\taux\t3:aux\t_\n3\tgot\tget\tVERB\tVBN\tTense=Past|VerbForm=Part\t0\troot\t0:root\t_\n4\tits\tits\tPRON\tPRP$\tCase=Gen|Gender=Neut|Number=Sing|Person=3|Poss=Yes|PronType=Prs\t6\tnmod:poss\t6:nmod:poss\t_\n5\town\town\tADJ\tJJ\tDegree=Pos\t6\tamod\t6:amod\t_\n6\tbathroom\tbathroom\tNOUN\tNN\tNumber=Sing\t3\tobj\t3:obj\t_\n7\tand\tand\tCCONJ\tCC\t_\t8\tcc\t8:cc\t_\n8\ttv\tTV\tNOUN\tNN\tNumber=Sing\t6\tconj\t3:obj|6:conj:and\tSpaceAfter=No\n\n# sent_id = newsgroup-groups.google.com_alt.animals.cat_01ff709c4bf2c60c_ENG_20040418_040100-0022\n# text = It's also got the website\n1-2\tIt's\t_\t_\t_\t_\t_\t_\t_\t_\n1\tIt\tit\tPRON\tPRP\tCase=Nom|Gender=Neut|Number=Sing|Person=3|PronType=Prs\t4\tnsubj\t4:nsubj\t_\n2\t's\thave\tAUX\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t4\taux\t4:aux\t_\n3\talso\talso\tADV\tRB\t_\t4\tadvmod\t4:advmod\t_\n4\tgot\tget\tVERB\tVBN\tTense=Past|VerbForm=Part\t0\troot\t0:root\t_\n5\tthe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t6\tdet\t6:det\t_\n6\twebsite\twebsite\tNOUN\tNN\tNumber=Sing\t4\tobj\t4:obj|12:obl\t_\n\"\"\".lstrip()\n\n\n# from the train set, actually\nEWT_DEV_SENTENCES = \"\"\"\n# sent_id = answers-20111108104724AAuBUR7_ans-0044\n# text = He's only exhibited weight loss and some muscle atrophy\n1-2\tHe's\t_\t_\t_\t_\t_\t_\t_\t_\n1\tHe\the\tPRON\tPRP\tCase=Nom|Gender=Masc|Number=Sing|Person=3|PronType=Prs\t4\tnsubj\t4:nsubj\t_\n2\t's\thave\tAUX\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t4\taux\t4:aux\t_\n3\tonly\tonly\tADV\tRB\t_\t4\tadvmod\t4:advmod\t_\n4\texhibited\texhibit\tVERB\tVBN\tTense=Past|VerbForm=Part\t0\troot\t0:root\t_\n5\tweight\tweight\tNOUN\tNN\tNumber=Sing\t6\tcompound\t6:compound\t_\n6\tloss\tloss\tNOUN\tNN\tNumber=Sing\t4\tobj\t4:obj\t_\n7\tand\tand\tCCONJ\tCC\t_\t10\tcc\t10:cc\t_\n8\tsome\tsome\tDET\tDT\tPronType=Ind\t10\tdet\t10:det\t_\n9\tmuscle\tmuscle\tNOUN\tNN\tNumber=Sing\t10\tcompound\t10:compound\t_\n10\tatrophy\tatrophy\tNOUN\tNN\tNumber=Sing\t6\tconj\t4:obj|6:conj:and\tSpaceAfter=No\n\n# sent_id = weblog-blogspot.com_rigorousintuition_20060511134300_ENG_20060511_134300-0097\n# text = It's a good thing too.\n1-2\tIt's\t_\t_\t_\t_\t_\t_\t_\t_\n1\tIt\tit\tPRON\tPRP\tCase=Nom|Gender=Neut|Number=Sing|Person=3|PronType=Prs\t5\tnsubj\t5:nsubj\t_\n2\t's\tbe\tAUX\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t5\tcop\t5:cop\t_\n3\ta\ta\tDET\tDT\tDefinite=Ind|PronType=Art\t5\tdet\t5:det\t_\n4\tgood\tgood\tADJ\tJJ\tDegree=Pos\t5\tamod\t5:amod\t_\n5\tthing\tthing\tNOUN\tNN\tNumber=Sing\t0\troot\t0:root\t_\n6\ttoo\ttoo\tADV\tRB\t_\t5\tadvmod\t5:advmod\tSpaceAfter=No\n7\t.\t.\tPUNCT\t.\t_\t5\tpunct\t5:punct\t_\n\"\"\".lstrip()\n\n# from the train set, actually\nEWT_TEST_SENTENCES = \"\"\"\n# sent_id = reviews-162422-0015\n# text = He said he's had a long and bad day.\n1\tHe\the\tPRON\tPRP\tCase=Nom|Gender=Masc|Number=Sing|Person=3|PronType=Prs\t2\tnsubj\t2:nsubj\t_\n2\tsaid\tsay\tVERB\tVBD\tMood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin\t0\troot\t0:root\t_\n3-4\the's\t_\t_\t_\t_\t_\t_\t_\t_\n3\the\the\tPRON\tPRP\tCase=Nom|Gender=Masc|Number=Sing|Person=3|PronType=Prs\t5\tnsubj\t5:nsubj\t_\n4\t's\thave\tAUX\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t5\taux\t5:aux\t_\n5\thad\thave\tVERB\tVBN\tTense=Past|VerbForm=Part\t2\tccomp\t2:ccomp\t_\n6\ta\ta\tDET\tDT\tDefinite=Ind|PronType=Art\t10\tdet\t10:det\t_\n7\tlong\tlong\tADJ\tJJ\tDegree=Pos\t10\tamod\t10:amod\t_\n8\tand\tand\tCCONJ\tCC\t_\t9\tcc\t9:cc\t_\n9\tbad\tbad\tADJ\tJJ\tDegree=Pos\t7\tconj\t7:conj:and|10:amod\t_\n10\tday\tday\tNOUN\tNN\tNumber=Sing\t5\tobj\t5:obj\tSpaceAfter=No\n11\t.\t.\tPUNCT\t.\t_\t2\tpunct\t2:punct\t_\n\n# sent_id = weblog-blogspot.com_rigorousintuition_20060511134300_ENG_20060511_134300-0100\n# text = What's a few dead soldiers\n1-2\tWhat's\t_\t_\t_\t_\t_\t_\t_\t_\n1\tWhat\twhat\tPRON\tWP\tPronType=Int\t6\tnsubj\t6:nsubj\t_\n2\t's\tbe\tAUX\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t6\tcop\t6:cop\t_\n3\ta\ta\tDET\tDT\tDefinite=Ind|PronType=Art\t6\tdet\t6:det\t_\n4\tfew\tfew\tADJ\tJJ\tDegree=Pos\t6\tamod\t6:amod\t_\n5\tdead\tdead\tADJ\tJJ\tDegree=Pos\t6\tamod\t6:amod\t_\n6\tsoldiers\tsoldier\tNOUN\tNNS\tNumber=Plur\t0\troot\t0:root\t_\n\"\"\"\n\ndef write_test_dataset(tmp_path, texts, datasets):\n ud_path = tmp_path / \"ud\"\n input_path = ud_path / \"UD_English-EWT\"\n output_path = tmp_path / \"data\" / \"lemma_classifier\"\n\n os.makedirs(input_path, exist_ok=True)\n\n for text, dataset in zip(texts, datasets):\n sample_file = input_path / (\"en_ewt-ud-%s.conllu\" % dataset)\n with open(sample_file, \"w\", encoding=\"utf-8\") as fout:\n fout.write(text)\n\n paths = {\"UDBASE\": ud_path,\n \"LEMMA_CLASSIFIER_DATA_DIR\": output_path}\n\n return paths\n\ndef write_english_test_dataset(tmp_path):\n texts = (EWT_TRAIN_SENTENCES, EWT_DEV_SENTENCES, EWT_TEST_SENTENCES)\n datasets = prepare_lemma_classifier.SECTIONS\n return write_test_dataset(tmp_path, texts, datasets)\n\ndef convert_english_dataset(tmp_path):\n paths = write_english_test_dataset(tmp_path)\n converted_files = prepare_lemma_classifier.process_treebank(paths, \"en_ewt\", \"'s\", \"AUX\", \"be|have\")\n assert len(converted_files) == 3\n\n return converted_files\n\ndef test_convert_one_sentence(tmp_path):\n texts = [EWT_ONE_SENTENCE]\n datasets = [\"train\"]\n paths = write_test_dataset(tmp_path, texts, datasets)\n\n converted_files = prepare_lemma_classifier.process_treebank(paths, \"en_ewt\", \"'s\", \"AUX\", \"be|have\", [\"train\"])\n assert len(converted_files) == 1\n\n dataset = utils.Dataset(converted_files[0], get_counts=True, batch_size=10, shuffle=False)\n\n assert len(dataset) == 1\n assert dataset.label_decoder == {'be': 0}\n id_to_upos = {y: x for x, y in dataset.upos_to_id.items()}\n\n for text_batches, _, upos_batches, _ in dataset:\n assert text_batches == [['Here', \"'s\", 'a', 'Miami', 'Herald', 'interview']]\n upos = [id_to_upos[x] for x in upos_batches[0]]\n assert upos == ['ADV', 'AUX', 'DET', 'PROPN', 'PROPN', 'NOUN']\n\ndef test_convert_dataset(tmp_path):\n converted_files = convert_english_dataset(tmp_path)\n\n dataset = utils.Dataset(converted_files[0], get_counts=True, batch_size=10, shuffle=False)\n\n assert len(dataset) == 1\n label_decoder = dataset.label_decoder\n assert len(label_decoder) == 2\n assert \"be\" in label_decoder\n assert \"have\" in label_decoder\n for text_batches, _, _, _ in dataset:\n assert len(text_batches) == 9\n\n dataset = utils.Dataset(converted_files[1], get_counts=True, batch_size=10, shuffle=False)\n assert len(dataset) == 1\n for text_batches, _, _, _ in dataset:\n assert len(text_batches) == 2\n\n dataset = utils.Dataset(converted_files[2], get_counts=True, batch_size=10, shuffle=False)\n assert len(dataset) == 1\n for text_batches, _, _, _ in dataset:\n assert len(text_batches) == 2\n\n"
},
{
"path": "stanza/tests/lemma_classifier/test_training.py",
"content": "import glob\nimport os\n\nimport pytest\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\nfrom stanza.models.lemma_classifier import train_lstm_model\nfrom stanza.models.lemma_classifier import train_transformer_model\nfrom stanza.models.lemma_classifier.base_model import LemmaClassifier\nfrom stanza.models.lemma_classifier.evaluate_models import evaluate_model\n\nfrom stanza.tests import TEST_WORKING_DIR\nfrom stanza.tests.lemma_classifier.test_data_preparation import convert_english_dataset\n\n@pytest.fixture(scope=\"module\")\ndef pretrain_file():\n return f'{TEST_WORKING_DIR}/in/tiny_emb.pt'\n\ndef test_train_lstm(tmp_path, pretrain_file):\n converted_files = convert_english_dataset(tmp_path)\n\n save_name = str(tmp_path / 'lemma.pt')\n\n train_file = converted_files[0]\n eval_file = converted_files[1]\n train_args = ['--wordvec_pretrain_file', pretrain_file,\n '--save_name', save_name,\n '--train_file', train_file,\n '--eval_file', eval_file]\n trainer = train_lstm_model.main(train_args)\n\n evaluate_model(trainer.model, eval_file)\n # test that loading the model works\n model = LemmaClassifier.load(save_name, None)\n\ndef test_train_transformer(tmp_path, pretrain_file):\n converted_files = convert_english_dataset(tmp_path)\n\n save_name = str(tmp_path / 'lemma.pt')\n\n train_file = converted_files[0]\n eval_file = converted_files[1]\n train_args = ['--bert_model', 'hf-internal-testing/tiny-bert',\n '--save_name', save_name,\n '--train_file', train_file,\n '--eval_file', eval_file]\n trainer = train_transformer_model.main(train_args)\n\n evaluate_model(trainer.model, eval_file)\n\n # test that loading the model works\n model = LemmaClassifier.load(save_name, None)\n"
},
{
"path": "stanza/tests/morphseg/__init__.py",
"content": ""
},
{
"path": "stanza/tests/morphseg/conftest.py",
"content": "\"\"\"\nShared pytest fixtures and configuration\n\"\"\"\n\nimport pytest\nfrom morphseg import MorphemeSegmenter\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\n@pytest.fixture(scope=\"session\")\ndef english_segmenter():\n \"\"\"\n Load English segmenter once for the entire test session\n \"\"\"\n return MorphemeSegmenter('en')\n\n\n@pytest.fixture(scope=\"session\")\ndef all_segmenters():\n \"\"\"\n Load all supported language segmenters\n \"\"\"\n segmenters = {}\n for lang in MorphemeSegmenter.PRETRAINED_MODEL_LANGS:\n segmenters[lang] = MorphemeSegmenter(lang)\n return segmenters\n\n\ndef pytest_configure(config):\n \"\"\"\n Custom pytest configuration\n \"\"\"\n config.addinivalue_line(\n \"markers\", \"slow: marks tests as slow (deselect with '-m \\\"not slow\\\"')\"\n )\n config.addinivalue_line(\n \"markers\", \"multilingual: marks tests that test multiple languages\"\n )\n"
},
{
"path": "stanza/tests/morphseg/test_integration.py",
"content": "\"\"\"\nIntegration tests for morphseg\n\"\"\"\n\nimport pytest\nfrom morphseg import MorphemeSegmenter\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\nclass TestIntegration:\n\n def test_full_pipeline(self):\n \"\"\"Test complete segmentation pipeline\"\"\"\n segmenter = MorphemeSegmenter('en')\n\n text = \"According to all known laws of aviation, there is no way a bee should be able to fly.\"\n result = segmenter.segment(text, output_string=False)\n\n # Should segment multiple words\n assert len(result) > 10\n\n # Each word should have at least one morpheme\n for word_morphemes in result:\n assert len(word_morphemes) >= 1\n\n def test_consistency_across_modes(self):\n \"\"\"Test that list and string output modes are consistent\"\"\"\n segmenter = MorphemeSegmenter('en')\n\n words = ['running', 'dogs', 'aviation']\n\n for word in words:\n list_result = segmenter.segment(word, output_string=False)\n string_result = segmenter.segment(word, output_string=True, delimiter=' @@')\n\n # String result should be reconstructable from list result\n expected_string = ' @@'.join(list_result[0])\n assert string_result == expected_string, \\\n f\"List and string outputs don't match for '{word}'\"\n\n def test_unicode_handling(self):\n \"\"\"Test handling of unicode characters\"\"\"\n segmenter = MorphemeSegmenter('fr')\n\n text = \"café résumé\"\n result = segmenter.segment(text, output_string=False)\n\n assert isinstance(result, list)\n assert len(result) >= 1\n\n def test_mixed_case(self):\n \"\"\"Test handling of mixed case input\"\"\"\n segmenter = MorphemeSegmenter('en')\n\n # Should normalize to lowercase\n result1 = segmenter.segment('Running', output_string=False)\n result2 = segmenter.segment('RUNNING', output_string=False)\n result3 = segmenter.segment('running', output_string=False)\n\n # All should produce the same result\n assert result1 == result2 == result3\n"
},
{
"path": "stanza/tests/morphseg/test_morpheme_segmenter.py",
"content": "\"\"\"\nTests for MorphemeSegmenter class\n\"\"\"\n\nimport pytest\nfrom morphseg import MorphemeSegmenter\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\nclass TestMorphemeSegmenter:\n\n @pytest.fixture(scope=\"class\")\n def english_segmenter(self):\n \"\"\"Load English model once for all tests\"\"\"\n return MorphemeSegmenter('en')\n\n def test_basic_segmentation(self, english_segmenter):\n \"\"\"Test basic morpheme segmentation\"\"\"\n result = english_segmenter.segment('running', output_string=False)\n\n assert isinstance(result, list)\n assert len(result) == 1\n assert isinstance(result[0], list)\n assert len(result[0]) >= 1\n\n def test_multiple_words(self, english_segmenter):\n \"\"\"Test segmentation of multiple words\"\"\"\n result = english_segmenter.segment('running quickly', output_string=False)\n\n assert isinstance(result, list)\n assert len(result) == 2\n for segmentation in result:\n assert isinstance(segmentation, list)\n assert len(segmentation) >= 1\n\n def test_known_segmentations(self, english_segmenter):\n \"\"\"Test known morpheme segmentations\"\"\"\n test_cases = {\n 'dogs': ['dog', 's'],\n 'aviation': ['aviate', 'ion'],\n 'known': ['know', 'n'],\n }\n\n for word, expected in test_cases.items():\n result = english_segmenter.segment(word, output_string=False)\n assert result[0] == expected, f\"Expected {expected}, got {result[0]} for '{word}'\"\n\n def test_output_string_mode(self, english_segmenter):\n \"\"\"Test string output mode\"\"\"\n result = english_segmenter.segment('running quickly', output_string=True)\n\n assert isinstance(result, str)\n assert ' @@' in result # Default delimiter\n\n def test_custom_delimiter(self, english_segmenter):\n \"\"\"Test custom delimiter in output\"\"\"\n result = english_segmenter.segment('running', output_string=True, delimiter='-')\n\n assert isinstance(result, str)\n assert '-' in result or result == 'running' # May be unsegmented\n\n def test_empty_input(self, english_segmenter):\n \"\"\"Test handling of empty input\"\"\"\n result = english_segmenter.segment('', output_string=False)\n assert result == []\n\n result = english_segmenter.segment('', output_string=True)\n assert result == \"\"\n\n def test_single_character(self, english_segmenter):\n \"\"\"Test single character input\"\"\"\n result = english_segmenter.segment('a', output_string=False)\n\n assert isinstance(result, list)\n assert len(result) == 1\n assert result[0] == ['a']\n\n def test_punctuation(self, english_segmenter):\n \"\"\"Test handling of punctuation\"\"\"\n result = english_segmenter.segment('Hello, world!', output_string=False)\n\n assert isinstance(result, list)\n # Should segment only words, not punctuation\n assert len(result) > 0\n\n\nclass TestDeterminism:\n \"\"\"\n Tests to ensure predictions are deterministic\n \"\"\"\n\n def test_deterministic_predictions(self):\n \"\"\"Test that same input produces same output consistently\"\"\"\n segmenter = MorphemeSegmenter('en')\n\n test_words = ['running', 'dogs', 'quickly', 'aviation']\n\n for word in test_words:\n results = []\n for _ in range(5):\n result = segmenter.segment(word, output_string=False)\n results.append(result)\n\n # All results should be identical\n for i in range(1, len(results)):\n assert results[i] == results[0], \\\n f\"Non-deterministic results for '{word}': {results[0]} vs {results[i]}\"\n\n def test_deterministic_batch(self):\n \"\"\"Test determinism with batch processing\"\"\"\n segmenter = MorphemeSegmenter('en')\n\n text = \"The dogs are running quickly through the fields.\"\n\n results = []\n for _ in range(3):\n result = segmenter.segment(text, output_string=False)\n results.append(result)\n\n # All results should be identical\n for i in range(1, len(results)):\n assert results[i] == results[0], \\\n f\"Non-deterministic batch results: {results[0]} vs {results[i]}\"\n\n\nclass TestMultilingual:\n\n @pytest.mark.parametrize(\"lang\", ['cs', 'en', 'es', 'fr', 'hu', 'it', 'la', 'ru'])\n def test_language_loading(self, lang):\n \"\"\"Test that all supported languages can be loaded\"\"\"\n segmenter = MorphemeSegmenter(lang)\n assert segmenter.lang == lang\n assert segmenter.sequence_labeller is not None\n\n @pytest.mark.parametrize(\"lang,word\", [\n ('en', 'running'),\n ('es', 'corriendo'),\n ('fr', 'rapidement'),\n ('ru', 'бегущий'), # Russian instead of German\n ])\n def test_multilingual_segmentation(self, lang, word):\n \"\"\"Test segmentation across languages\"\"\"\n if lang not in MorphemeSegmenter.PRETRAINED_MODEL_LANGS:\n pytest.skip(f\"Language {lang} not supported\")\n\n segmenter = MorphemeSegmenter(lang)\n result = segmenter.segment(word, output_string=False)\n\n assert isinstance(result, list)\n assert len(result) >= 1\n\n\nclass TestErrorHandling:\n\n def test_invalid_language(self):\n \"\"\"Test handling of invalid language code\"\"\"\n with pytest.warns(UserWarning):\n segmenter = MorphemeSegmenter('invalid_lang')\n assert segmenter.sequence_labeller is None\n\n def test_invalid_input_type(self):\n \"\"\"Test handling of invalid input types\"\"\"\n segmenter = MorphemeSegmenter('en')\n\n with pytest.raises(ValueError, match=\"Input sequence must be a string\"):\n segmenter.segment(123)\n\n with pytest.raises(ValueError, match=\"Input sequence must be a string\"):\n segmenter.segment(['not', 'a', 'string'])\n\n def test_invalid_output_string_type(self):\n \"\"\"Test handling of invalid output_string parameter\"\"\"\n segmenter = MorphemeSegmenter('en')\n\n with pytest.raises(ValueError, match=\"output_string must be a boolean\"):\n segmenter.segment('test', output_string='yes')\n\n def test_invalid_delimiter_type(self):\n \"\"\"Test handling of invalid delimiter parameter\"\"\"\n segmenter = MorphemeSegmenter('en')\n\n with pytest.raises(ValueError, match=\"Delimiter must be a string\"):\n segmenter.segment('test', delimiter=123)\n\n def test_model_not_trained(self):\n \"\"\"Test error when using untrained model\"\"\"\n segmenter = MorphemeSegmenter('en')\n segmenter.sequence_labeller = None\n\n with pytest.raises(RuntimeError, match=\"Model not trained\"):\n segmenter.segment('test')\n\n\nclass TestModelState:\n \"\"\"\n Tests to ensure model is in correct state\n \"\"\"\n\n def test_model_in_eval_mode(self):\n \"\"\"Test that loaded model is in eval mode\"\"\"\n segmenter = MorphemeSegmenter('en')\n\n # Check that model is in eval mode\n assert not segmenter.sequence_labeller.model.model.training, \\\n \"Model should be in eval mode after loading\"\n\n def test_model_stays_in_eval_mode(self):\n \"\"\"Test that model stays in eval mode after predictions\"\"\"\n segmenter = MorphemeSegmenter('en')\n\n # Make several predictions\n for _ in range(3):\n segmenter.segment('running', output_string=False)\n\n # Model should still be in eval mode\n assert not segmenter.sequence_labeller.model.model.training, \\\n \"Model should remain in eval mode after predictions\"\n"
},
{
"path": "stanza/tests/morphseg/test_stanza_integration.py",
"content": "\"\"\"\nIntegration tests for Stanza MorphSeg Processor\nTests the morpheme segmentation processor within the Stanza pipeline\n\"\"\"\n\nimport pytest\nimport stanza\nfrom stanza.models.common.doc import Document\nfrom stanza.tests import TEST_MODELS_DIR\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\nclass TestMorphSegProcessor:\n \"\"\"Tests for the MorphSeg processor in Stanza pipeline\"\"\"\n\n @pytest.fixture(scope=\"class\")\n def en_pipeline(self):\n \"\"\"Create English pipeline with morphseg processor\"\"\"\n return stanza.Pipeline(\n lang='en',\n processors='tokenize,morphseg',\n model_dir=TEST_MODELS_DIR,\n download_method=None\n )\n\n def test_processor_loads(self, en_pipeline):\n \"\"\"Test that morphseg processor loads successfully\"\"\"\n assert 'morphseg' in en_pipeline.processors\n assert en_pipeline.processors['morphseg'] is not None\n\n def test_basic_segmentation(self, en_pipeline):\n \"\"\"Test basic morpheme segmentation through pipeline\"\"\"\n doc = en_pipeline(\"running\")\n\n assert len(doc.sentences) == 1\n assert len(doc.sentences[0].words) == 1\n\n word = doc.sentences[0].words[0]\n assert hasattr(word, 'morphemes')\n assert isinstance(word.morphemes, list)\n assert len(word.morphemes) >= 1\n\n def test_known_segmentations(self, en_pipeline):\n \"\"\"Test known morpheme segmentations\"\"\"\n # Note: These are actual segmentations from the en2 model\n # Some words may be unsegmented depending on the model\n test_cases = {\n 'dogs': ['dog', 's'],\n 'aviation': ['aviate', 'ion'],\n 'known': ['know', 'n'],\n }\n\n for word_text, expected in test_cases.items():\n doc = en_pipeline(word_text)\n word = doc.sentences[0].words[0]\n assert word.morphemes == expected, \\\n f\"Expected {expected}, got {word.morphemes} for '{word_text}'\"\n\n def test_segmentation_consistency(self, en_pipeline):\n \"\"\"Test that segmentation is consistent and produces valid output\"\"\"\n words = ['running', 'quickly', 'walked', 'playing']\n\n for word_text in words:\n doc = en_pipeline(word_text)\n word = doc.sentences[0].words[0]\n\n # Should have morphemes attribute\n assert hasattr(word, 'morphemes')\n assert isinstance(word.morphemes, list)\n assert len(word.morphemes) >= 1\n\n # All morphemes should be strings\n for morpheme in word.morphemes:\n assert isinstance(morpheme, str)\n assert len(morpheme) > 0\n\n def test_multiple_words(self, en_pipeline):\n \"\"\"Test segmentation of multiple words in a sentence\"\"\"\n doc = en_pipeline(\"The dogs are running quickly.\")\n\n # Check that all words have morphemes attribute\n for sentence in doc.sentences:\n for word in sentence.words:\n assert hasattr(word, 'morphemes')\n assert isinstance(word.morphemes, list)\n assert len(word.morphemes) >= 1\n\n def test_punctuation_handling(self, en_pipeline):\n \"\"\"Test that punctuation is handled correctly\"\"\"\n doc = en_pipeline(\"Hello, world!\")\n\n # All tokens should have morphemes, including punctuation\n for sentence in doc.sentences:\n for word in sentence.words:\n assert hasattr(word, 'morphemes')\n # Punctuation should be unsegmented\n if word.text in [',', '!', '.']:\n assert word.morphemes == [word.text]\n\n def test_long_text(self, en_pipeline):\n \"\"\"Test processing of longer text\"\"\"\n text = \"According to all known laws of aviation, there is no way a bee should be able to fly.\"\n doc = en_pipeline(text)\n\n # Should have multiple sentences or one long sentence\n assert len(doc.sentences) >= 1\n\n # Count words with morpheme segmentation\n total_words = sum(len(sent.words) for sent in doc.sentences)\n assert total_words > 10\n\n # All words should have morphemes\n for sentence in doc.sentences:\n for word in sentence.words:\n assert hasattr(word, 'morphemes')\n\n def test_empty_input(self, en_pipeline):\n \"\"\"Test handling of empty input\"\"\"\n doc = en_pipeline(\"\")\n assert len(doc.sentences) == 0\n\n def test_single_character(self, en_pipeline):\n \"\"\"Test single character input\"\"\"\n doc = en_pipeline(\"I\")\n\n assert len(doc.sentences) == 1\n word = doc.sentences[0].words[0]\n assert word.morphemes == ['i'] # Normalized to lowercase\n\n def test_morphemes_attribute_persistence(self, en_pipeline):\n \"\"\"Test that morphemes attribute persists through pipeline\"\"\"\n doc = en_pipeline(\"running quickly\")\n\n # Store morphemes\n morphemes_list = []\n for sentence in doc.sentences:\n for word in sentence.words:\n morphemes_list.append(word.morphemes)\n\n # Access again to ensure persistence\n for i, sentence in enumerate(doc.sentences):\n for j, word in enumerate(sentence.words):\n assert hasattr(word, 'morphemes')\n assert word.morphemes is not None\n\n\nclass TestMultilingualMorphSeg:\n \"\"\"Test morpheme segmentation across different languages\"\"\"\n\n @pytest.mark.parametrize(\"lang,text,expected_word\", [\n ('en', 'running', 'running'),\n ('es', 'corriendo', 'corriendo'),\n ('fr', 'rapidement', 'rapidement'),\n ('cs', 'běžící', 'běžící'),\n ('it', 'correndo', 'correndo'),\n ])\n def test_multilingual_support(self, lang, text, expected_word):\n \"\"\"Test that different languages can be processed\"\"\"\n try:\n nlp = stanza.Pipeline(\n lang=lang,\n processors='tokenize,morphseg',\n download_method=None\n )\n doc = nlp(text)\n\n assert len(doc.sentences) >= 1\n assert len(doc.sentences[0].words) >= 1\n\n word = doc.sentences[0].words[0]\n assert hasattr(word, 'morphemes')\n assert isinstance(word.morphemes, list)\n\n except Exception as e:\n pytest.skip(f\"Language {lang} not available: {e}\")\n\n\nclass TestMorphSegWithOtherProcessors:\n \"\"\"Test morphseg processor in combination with other processors\"\"\"\n\n def test_with_mwt(self):\n \"\"\"Test morphseg with MWT processor\"\"\"\n nlp = stanza.Pipeline(\n lang='en',\n processors='tokenize,mwt,morphseg',\n download_method=None\n )\n\n doc = nlp(\"The dogs are running.\")\n\n for sentence in doc.sentences:\n for word in sentence.words:\n assert hasattr(word, 'morphemes')\n\n def test_with_pos(self):\n \"\"\"Test morphseg with POS tagging\"\"\"\n try:\n nlp = stanza.Pipeline(\n lang='en',\n processors='tokenize,pos,morphseg',\n download_method=None\n )\n\n doc = nlp(\"running quickly\")\n\n for sentence in doc.sentences:\n for word in sentence.words:\n # Should have both POS and morphemes\n assert hasattr(word, 'morphemes')\n assert hasattr(word, 'upos') or hasattr(word, 'xpos')\n\n except Exception as e:\n pytest.skip(f\"POS processor not available: {e}\")\n\n def test_with_lemma(self):\n \"\"\"Test morphseg with lemmatization\"\"\"\n try:\n nlp = stanza.Pipeline(\n lang='en',\n processors='tokenize,pos,lemma,morphseg',\n download_method=None\n )\n\n doc = nlp(\"The dogs were running quickly.\")\n\n for sentence in doc.sentences:\n for word in sentence.words:\n # Should have both lemma and morphemes\n assert hasattr(word, 'morphemes')\n assert hasattr(word, 'lemma')\n\n except Exception as e:\n pytest.skip(f\"Lemma processor not available: {e}\")\n\n\nclass TestMorphSegDeterminism:\n \"\"\"Test that morphseg processor produces deterministic results\"\"\"\n\n def test_deterministic_results(self):\n \"\"\"Test that same input produces same output\"\"\"\n nlp = stanza.Pipeline(\n lang='en',\n processors='tokenize,morphseg',\n download_method=None\n )\n\n text = \"running dogs aviation\"\n\n results = []\n for _ in range(3):\n doc = nlp(text)\n morphemes = [word.morphemes for sent in doc.sentences for word in sent.words]\n results.append(morphemes)\n\n # All results should be identical\n for i in range(1, len(results)):\n assert results[i] == results[0], \\\n f\"Non-deterministic results: {results[0]} vs {results[i]}\"\n\n def test_batch_determinism(self):\n \"\"\"Test determinism with batch processing\"\"\"\n nlp = stanza.Pipeline(\n lang='en',\n processors='tokenize,morphseg',\n download_method=None\n )\n\n texts = [\n \"The dogs are running.\",\n \"Aviation is amazing.\",\n \"Known facts are helpful.\"\n ]\n\n # Process multiple times\n all_results = []\n for _ in range(2):\n batch_results = []\n for text in texts:\n doc = nlp(text)\n morphemes = [word.morphemes for sent in doc.sentences for word in sent.words]\n batch_results.append(morphemes)\n all_results.append(batch_results)\n\n # Results should be identical\n assert all_results[0] == all_results[1]\n\n\nclass TestMorphSegEdgeCases:\n \"\"\"Test edge cases and special inputs\"\"\"\n\n @pytest.fixture(scope=\"class\")\n def en_pipeline(self):\n \"\"\"Create English pipeline\"\"\"\n return stanza.Pipeline(\n lang='en',\n processors='tokenize,morphseg',\n download_method=None\n )\n\n def test_numbers(self, en_pipeline):\n \"\"\"Test handling of numbers\"\"\"\n doc = en_pipeline(\"123 456\")\n\n for sentence in doc.sentences:\n for word in sentence.words:\n assert hasattr(word, 'morphemes')\n\n def test_mixed_case(self, en_pipeline):\n \"\"\"Test mixed case handling\"\"\"\n # Should normalize to same result\n doc1 = en_pipeline(\"Running\")\n doc2 = en_pipeline(\"RUNNING\")\n doc3 = en_pipeline(\"running\")\n\n morphemes1 = doc1.sentences[0].words[0].morphemes\n morphemes2 = doc2.sentences[0].words[0].morphemes\n morphemes3 = doc3.sentences[0].words[0].morphemes\n\n assert morphemes1 == morphemes2 == morphemes3\n\n def test_unicode_characters(self, en_pipeline):\n \"\"\"Test handling of unicode characters\"\"\"\n doc = en_pipeline(\"café résumé\")\n\n for sentence in doc.sentences:\n for word in sentence.words:\n assert hasattr(word, 'morphemes')\n assert isinstance(word.morphemes, list)\n\n def test_special_characters(self, en_pipeline):\n \"\"\"Test handling of special characters\"\"\"\n doc = en_pipeline(\"test@example.com $100 50%\")\n\n for sentence in doc.sentences:\n for word in sentence.words:\n assert hasattr(word, 'morphemes')\n\n def test_very_long_word(self, en_pipeline):\n \"\"\"Test handling of very long words\"\"\"\n long_word = \"antidisestablishmentarianism\"\n doc = en_pipeline(long_word)\n\n word = doc.sentences[0].words[0]\n assert hasattr(word, 'morphemes')\n assert len(word.morphemes) >= 1\n\n def test_repeated_words(self, en_pipeline):\n \"\"\"Test handling of repeated words\"\"\"\n doc = en_pipeline(\"running running running\")\n\n # All instances should have same segmentation\n morphemes_list = [word.morphemes for word in doc.sentences[0].words]\n assert morphemes_list[0] == morphemes_list[1] == morphemes_list[2]\n\n def test_whitespace_handling(self, en_pipeline):\n \"\"\"Test handling of various whitespace\"\"\"\n doc = en_pipeline(\"word1 word2\\tword3\\nword4\")\n\n # Should properly segment all words despite whitespace\n word_count = sum(len(sent.words) for sent in doc.sentences)\n assert word_count >= 4\n\n for sentence in doc.sentences:\n for word in sentence.words:\n assert hasattr(word, 'morphemes')\n\n\nclass TestMorphSegConfiguration:\n \"\"\"Test different configurations of morphseg processor\"\"\"\n\n def test_custom_model_path(self):\n \"\"\"Test loading with custom model path configuration\"\"\"\n # Test that the configuration accepts model_path parameter\n # Using default behavior (no custom path)\n nlp = stanza.Pipeline(\n lang='en',\n processors='tokenize,morphseg',\n download_method=None\n )\n doc = nlp(\"testing\")\n assert len(doc.sentences) > 0\n assert hasattr(doc.sentences[0].words[0], 'morphemes')\n\n def test_custom_model_path_with_file(self):\n \"\"\"Test loading with an actual custom model file path\"\"\"\n # This test would require a custom model file to exist\n # Skip if no custom model is available\n pytest.skip(\"Custom model path test requires a specific model file\")\n\n def test_processor_requirements(self):\n \"\"\"Test that morphseg requires tokenize\"\"\"\n # MorphSeg requires TOKENIZE processor\n nlp = stanza.Pipeline(\n lang='en',\n processors='tokenize,morphseg',\n download_method=None\n )\n\n # Verify tokenize is present\n assert 'tokenize' in nlp.processors or 'tokenize' in str(nlp.processors)\n\n\nclass TestMorphSegOutputFormat:\n \"\"\"Test output format of morpheme segmentations\"\"\"\n\n @pytest.fixture(scope=\"class\")\n def en_pipeline(self):\n \"\"\"Create English pipeline\"\"\"\n return stanza.Pipeline(\n lang='en',\n processors='tokenize,morphseg',\n download_method=None\n )\n\n def test_morphemes_is_list(self, en_pipeline):\n \"\"\"Test that morphemes attribute is always a list\"\"\"\n doc = en_pipeline(\"The dogs are running quickly.\")\n\n for sentence in doc.sentences:\n for word in sentence.words:\n assert isinstance(word.morphemes, list)\n\n def test_morphemes_are_strings(self, en_pipeline):\n \"\"\"Test that all morphemes are strings\"\"\"\n doc = en_pipeline(\"The dogs are running quickly.\")\n\n for sentence in doc.sentences:\n for word in sentence.words:\n for morpheme in word.morphemes:\n assert isinstance(morpheme, str)\n\n def test_morphemes_non_empty(self, en_pipeline):\n \"\"\"Test that morphemes list is never empty\"\"\"\n doc = en_pipeline(\"The dogs are running quickly.\")\n\n for sentence in doc.sentences:\n for word in sentence.words:\n assert len(word.morphemes) >= 1\n\n def test_unsegmented_words(self, en_pipeline):\n \"\"\"Test that unsegmented words have single morpheme\"\"\"\n # Words like 'the', 'is', 'a' typically don't segment\n doc = en_pipeline(\"The dog is a pet.\")\n\n for sentence in doc.sentences:\n for word in sentence.words:\n # Even if unsegmented, should have the word itself as morpheme\n if len(word.morphemes) == 1:\n # The single morpheme should match the normalized word\n assert isinstance(word.morphemes[0], str)\n\n\nclass TestMorphSegRepeatedly:\n \"\"\"Test repeated processing of multiple documents\"\"\"\n\n def test_sequential_document_processing(self):\n \"\"\"Test processing multiple documents one after another\"\"\"\n nlp = stanza.Pipeline(\n lang='en',\n processors='tokenize,morphseg',\n download_method=None\n )\n\n texts = [\n \"The dogs are running.\",\n \"Aviation is fascinating.\",\n \"Programming requires patience.\"\n ]\n\n for text in texts:\n doc = nlp(text)\n for sentence in doc.sentences:\n for word in sentence.words:\n assert hasattr(word, 'morphemes')\n assert isinstance(word.morphemes, list)\n\n def test_multi_sentence_document(self):\n \"\"\"Test processing a document with multiple sentences (internal batching)\"\"\"\n nlp = stanza.Pipeline(\n lang='en',\n processors='tokenize,morphseg',\n download_method=None\n )\n\n doc = nlp(\"The dogs are running. Aviation is fascinating. Programming requires patience.\")\n\n assert len(doc.sentences) == 3\n\n for sentence in doc.sentences:\n for word in sentence.words:\n assert hasattr(word, 'morphemes')\n assert isinstance(word.morphemes, list)\n"
},
{
"path": "stanza/tests/mwt/__init__.py",
"content": ""
},
{
"path": "stanza/tests/mwt/test_character_classifier.py",
"content": "import os\nimport pytest\n\nfrom stanza.models import mwt_expander\nfrom stanza.models.mwt.character_classifier import CharacterClassifier\nfrom stanza.models.mwt.data import DataLoader\nfrom stanza.models.mwt.trainer import Trainer\nfrom stanza.utils.conll import CoNLL\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\nENG_TRAIN = \"\"\"\n# text = Elena's motorcycle tour\n1-2\tElena's\t_\t_\t_\t_\t_\t_\t_\t_\n1\tElena\tElena\tPROPN\tNNP\tNumber=Sing\t4\tnmod:poss\t4:nmod:poss\t_\n2\t's\t's\tPART\tPOS\t_\t1\tcase\t1:case\t_\n3\tmotorcycle\tmotorcycle\tNOUN\tNN\tNumber=Sing\t4\tcompound\t4:compound\t_\n4\ttour\ttour\tNOUN\tNN\tNumber=Sing\t0\troot\t0:root\t_\n\n\n# text = women's reproductive health\n1-2\twomen's\t_\t_\t_\t_\t_\t_\t_\t_\n1\twomen\twoman\tNOUN\tNNS\tNumber=Plur\t4\tnmod:poss\t4:nmod:poss\t_\n2\t's\t's\tPART\tPOS\t_\t1\tcase\t1:case\t_\n3\treproductive\treproductive\tADJ\tJJ\tDegree=Pos\t4\tamod\t4:amod\t_\n4\thealth\thealth\tNOUN\tNN\tNumber=Sing\t0\troot\t0:root\tSpaceAfter=No\n\n\n# text = The Chernobyl Children's Project\n1\tThe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t3\tdet\t3:det\t_\n2\tChernobyl\tChernobyl\tPROPN\tNNP\tNumber=Sing\t3\tcompound\t3:compound\t_\n3-4\tChildren's\t_\t_\t_\t_\t_\t_\t_\t_\n3\tChildren\tChildren\tPROPN\tNNP\tNumber=Sing\t5\tnmod:poss\t5:nmod:poss\t_\n4\t's\t's\tPART\tPOS\t_\t3\tcase\t3:case\t_\n5\tProject\tProject\tPROPN\tNNP\tNumber=Sing\t0\troot\t0:root\t_\n\n\"\"\".lstrip()\n\nENG_DEV = \"\"\"\n# text = The Chernobyl Children's Project\n1\tThe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t3\tdet\t3:det\t_\n2\tChernobyl\tChernobyl\tPROPN\tNNP\tNumber=Sing\t3\tcompound\t3:compound\t_\n3-4\tChildren's\t_\t_\t_\t_\t_\t_\t_\t_\n3\tChildren\tChildren\tPROPN\tNNP\tNumber=Sing\t5\tnmod:poss\t5:nmod:poss\t_\n4\t's\t's\tPART\tPOS\t_\t3\tcase\t3:case\t_\n5\tProject\tProject\tPROPN\tNNP\tNumber=Sing\t0\troot\t0:root\t_\n\n\"\"\".lstrip()\n\ndef test_train(tmp_path):\n test_train = str(os.path.join(tmp_path, \"en_test.train.conllu\"))\n with open(test_train, \"w\") as fout:\n fout.write(ENG_TRAIN)\n\n test_dev = str(os.path.join(tmp_path, \"en_test.dev.conllu\"))\n with open(test_dev, \"w\") as fout:\n fout.write(ENG_DEV)\n\n test_output = str(os.path.join(tmp_path, \"en_test.dev.pred.conllu\"))\n model_name = \"en_test_mwt.pt\"\n\n args = [\n \"--data_dir\", str(tmp_path),\n \"--train_file\", test_train,\n \"--eval_file\", test_dev,\n \"--gold_file\", test_dev,\n \"--lang\", \"en\",\n \"--shorthand\", \"en_test\",\n \"--output_file\", test_output,\n \"--save_dir\", str(tmp_path),\n \"--save_name\", model_name,\n \"--num_epoch\", \"10\",\n ]\n\n mwt_expander.main(args=args)\n\n model = Trainer(model_file=os.path.join(tmp_path, model_name))\n assert model.model is not None\n assert isinstance(model.model, CharacterClassifier)\n\n doc = CoNLL.conll2doc(input_str=ENG_DEV)\n dataloader = DataLoader(doc, 10, model.args, vocab=model.vocab, evaluation=True, expand_unk_vocab=True)\n preds = []\n for i, batch in enumerate(dataloader.to_loader()):\n assert i == 0 # there should only be one batch\n preds += model.predict(batch, never_decode_unk=True, vocab=dataloader.vocab)\n assert len(preds) == 1\n # it is possible to make a version of the test where this happens almost every time\n # for example, running for 100 epochs makes the model succeed 30 times in a row\n # (never saw a failure)\n # but the one time that failure happened, it would be really annoying\n #assert preds[0] == \"Children 's\"\n"
},
{
"path": "stanza/tests/mwt/test_english_corner_cases.py",
"content": "\"\"\"\nTest a couple English MWT corner cases which might be more widely applicable to other MWT languages\n\n- unknown English character doesn't result in bizarre splits\n- Casing or CASING doesn't get lost in the dictionary lookup\n\nIn the English UD datasets, the MWT are composed exactly of the\nsubwords, so the MWT model should be chopping up the input text rather\nthan generating new text.\n\nFurthermore, SHE'S and She's should be split \"SHE 'S\" and \"She 's\" respectively\n\"\"\"\n\nimport pytest\nimport stanza\n\nfrom stanza.tests import TEST_MODELS_DIR\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\ndef test_mwt_unknown_char():\n pipeline = stanza.Pipeline(processors='tokenize,mwt', dir=TEST_MODELS_DIR, lang='en', download_method=None)\n\n mwt_trainer = pipeline.processors['mwt']._trainer\n\n assert mwt_trainer.args['force_exact_pieces']\n\n # find a letter 'i' which isn't in the training data\n # the MWT model should still recognize a possessive containing this letter\n assert \"i\" in mwt_trainer.vocab\n for letter in \"ĩîíìī\":\n if letter not in mwt_trainer.vocab:\n break\n else:\n raise AssertionError(\"Need to update the MWT test - all of the non-standard letters 'i' are now in the MWT vocab\")\n\n word = \"Jenn\" + letter + \"fer\"\n possessive = word + \"'s\"\n text = \"I wanna lick \" + possessive + \" antennae\"\n doc = pipeline(text)\n assert doc.sentences[0].tokens[1].text == 'wanna'\n assert len(doc.sentences[0].tokens[1].words) == 2\n assert \"\".join(x.text for x in doc.sentences[0].tokens[1].words) == 'wanna'\n\n assert doc.sentences[0].tokens[3].text == possessive\n assert len(doc.sentences[0].tokens[3].words) == 2\n assert \"\".join(x.text for x in doc.sentences[0].tokens[3].words) == possessive\n\n\ndef test_english_mwt_casing():\n \"\"\"\n Test that for a word where the lowercase split is known, the correct casing is still used\n\n Once upon a time, the logic used in the MWT expander would split\n SHE'S -> she 's\n\n which is a very surprising tokenization to people expecting\n the original text in the output document\n \"\"\"\n pipeline = stanza.Pipeline(processors='tokenize,mwt', dir=TEST_MODELS_DIR, lang='en', download_method=None)\n\n mwt_trainer = pipeline.processors['mwt']._trainer\n for i in range(1, 20):\n # many test cases follow this pattern for some reason,\n # so we should proactively look for a test case which hasn't\n # made its way into the MWT dictionary\n unknown_name = \"jennife\" + \"r\" * i + \"'s\"\n if unknown_name not in mwt_trainer.expansion_dict and unknown_name.upper() not in mwt_trainer.expansion_dict:\n unknown_name = unknown_name.upper()\n break\n else:\n raise AssertionError(\"Need a new heuristic for the unknown word in the English MWT!\")\n\n # this SHOULD show up in the expansion dict\n assert \"she's\" in mwt_trainer.expansion_dict, \"Expected |she's| to be in the English MWT expansion dict... perhaps find a different test case\"\n\n text = [x.text for x in pipeline(\"JENNIFER HAS NICE ANTENNAE\").sentences[0].words]\n assert text == ['JENNIFER', 'HAS', 'NICE', 'ANTENNAE']\n\n text = [x.text for x in pipeline(unknown_name + \" GOT NICE ANTENNAE\").sentences[0].words]\n assert text == [unknown_name[:-2], \"'S\", 'GOT', 'NICE', 'ANTENNAE']\n\n text = [x.text for x in pipeline(\"SHE'S GOT NICE ANTENNAE\").sentences[0].words]\n assert text == ['SHE', \"'S\", 'GOT', 'NICE', 'ANTENNAE']\n\n text = [x.text for x in pipeline(\"She's GOT NICE ANTENNAE\").sentences[0].words]\n assert text == ['She', \"'s\", 'GOT', 'NICE', 'ANTENNAE']\n\n"
},
{
"path": "stanza/tests/mwt/test_prepare_mwt.py",
"content": "\nimport pytest\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\nfrom stanza.utils.datasets.prepare_mwt_treebank import check_mwt_composition\n\nSAMPLE_GOOD_TEXT = \"\"\"\n# sent_id = weblog-typepad.com_ripples_20040407125600_ENG_20040407_125600-0057\n# text = The Chernobyl Children's Project (http://www.adiccp.org/home/default.asp) offers several ways to help the children of that region.\n1\tThe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t3\tdet\t3:det\t_\n2\tChernobyl\tChernobyl\tPROPN\tNNP\tNumber=Sing\t3\tcompound\t3:compound\t_\n3-4\tChildren's\t_\t_\t_\t_\t_\t_\t_\t_\n3\tChildren\tChildren\tPROPN\tNNP\tNumber=Sing\t5\tnmod:poss\t5:nmod:poss\t_\n4\t's\t's\tPART\tPOS\t_\t3\tcase\t3:case\t_\n5\tProject\tProject\tPROPN\tNNP\tNumber=Sing\t9\tnsubj\t9:nsubj\t_\n6\t(\t(\tPUNCT\t-LRB-\t_\t7\tpunct\t7:punct\tSpaceAfter=No\n7\thttp://www.adiccp.org/home/default.asp\thttp://www.adiccp.org/home/default.asp\tX\tADD\t_\t5\tappos\t5:appos\tSpaceAfter=No\n8\t)\t)\tPUNCT\t-RRB-\t_\t7\tpunct\t7:punct\t_\n9\toffers\toffer\tVERB\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t0\troot\t0:root\t_\n10\tseveral\tseveral\tADJ\tJJ\tDegree=Pos\t11\tamod\t11:amod\t_\n11\tways\tway\tNOUN\tNNS\tNumber=Plur\t9\tobj\t9:obj\t_\n12\tto\tto\tPART\tTO\t_\t13\tmark\t13:mark\t_\n13\thelp\thelp\tVERB\tVB\tVerbForm=Inf\t11\tacl\t11:acl:to\t_\n14\tthe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t15\tdet\t15:det\t_\n15\tchildren\tchild\tNOUN\tNNS\tNumber=Plur\t13\tobj\t13:obj\t_\n16\tof\tof\tADP\tIN\t_\t18\tcase\t18:case\t_\n17\tthat\tthat\tDET\tDT\tNumber=Sing|PronType=Dem\t18\tdet\t18:det\t_\n18\tregion\tregion\tNOUN\tNN\tNumber=Sing\t15\tnmod\t15:nmod:of\tSpaceAfter=No\n19\t.\t.\tPUNCT\t.\t_\t9\tpunct\t9:punct\t_\n\"\"\".lstrip()\n\nSAMPLE_BAD_TEXT = \"\"\"\n# sent_id = weblog-typepad.com_ripples_20040407125600_ENG_20040407_125600-0057\n# text = The Chernobyl Children's Project (http://www.adiccp.org/home/default.asp) offers several ways to help the children of that region.\n1\tThe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t3\tdet\t3:det\t_\n2\tChernobyl\tChernobyl\tPROPN\tNNP\tNumber=Sing\t3\tcompound\t3:compound\t_\n3-4\tChildren's\t_\t_\t_\t_\t_\t_\t_\t_\n3\tChildrez\tChildren\tPROPN\tNNP\tNumber=Sing\t5\tnmod:poss\t5:nmod:poss\t_\n4\t's\t's\tPART\tPOS\t_\t3\tcase\t3:case\t_\n5\tProject\tProject\tPROPN\tNNP\tNumber=Sing\t9\tnsubj\t9:nsubj\t_\n6\t(\t(\tPUNCT\t-LRB-\t_\t7\tpunct\t7:punct\tSpaceAfter=No\n7\thttp://www.adiccp.org/home/default.asp\thttp://www.adiccp.org/home/default.asp\tX\tADD\t_\t5\tappos\t5:appos\tSpaceAfter=No\n8\t)\t)\tPUNCT\t-RRB-\t_\t7\tpunct\t7:punct\t_\n9\toffers\toffer\tVERB\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t0\troot\t0:root\t_\n10\tseveral\tseveral\tADJ\tJJ\tDegree=Pos\t11\tamod\t11:amod\t_\n11\tways\tway\tNOUN\tNNS\tNumber=Plur\t9\tobj\t9:obj\t_\n12\tto\tto\tPART\tTO\t_\t13\tmark\t13:mark\t_\n13\thelp\thelp\tVERB\tVB\tVerbForm=Inf\t11\tacl\t11:acl:to\t_\n14\tthe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t15\tdet\t15:det\t_\n15\tchildren\tchild\tNOUN\tNNS\tNumber=Plur\t13\tobj\t13:obj\t_\n16\tof\tof\tADP\tIN\t_\t18\tcase\t18:case\t_\n17\tthat\tthat\tDET\tDT\tNumber=Sing|PronType=Dem\t18\tdet\t18:det\t_\n18\tregion\tregion\tNOUN\tNN\tNumber=Sing\t15\tnmod\t15:nmod:of\tSpaceAfter=No\n19\t.\t.\tPUNCT\t.\t_\t9\tpunct\t9:punct\t_\n\"\"\".lstrip()\n\ndef test_check_mwt_composition(tmp_path):\n mwt_file = tmp_path / \"good.mwt\"\n with open(mwt_file, \"w\", encoding=\"utf-8\") as fout:\n fout.write(SAMPLE_GOOD_TEXT)\n check_mwt_composition(mwt_file)\n\n mwt_file = tmp_path / \"bad.mwt\"\n with open(mwt_file, \"w\", encoding=\"utf-8\") as fout:\n fout.write(SAMPLE_BAD_TEXT)\n with pytest.raises(ValueError):\n check_mwt_composition(mwt_file)\n"
},
{
"path": "stanza/tests/mwt/test_utils.py",
"content": "\"\"\"\nTest the MWT resplitting of preexisting tokens without word splits\n\"\"\"\n\nimport pytest\n\nimport stanza\nfrom stanza.models.mwt.utils import resplit_mwt\n\nfrom stanza.tests import TEST_MODELS_DIR\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\n@pytest.fixture(scope=\"module\")\ndef pipeline():\n \"\"\"\n A reusable pipeline with the NER module\n \"\"\"\n return stanza.Pipeline(\"en\", dir=TEST_MODELS_DIR, processors=\"tokenize,mwt\", package=\"gum\")\n\n\ndef test_resplit_keep_tokens(pipeline):\n \"\"\"\n Test splitting with enforced token boundaries\n \"\"\"\n tokens = [[\"I\", \"can't\", \"believe\", \"it\"], [\"I can't\", \"sleep\"]]\n doc = resplit_mwt(tokens, pipeline)\n assert len(doc.sentences) == 2\n assert len(doc.sentences[0].tokens) == 4\n assert len(doc.sentences[0].tokens[1].words) == 2\n assert doc.sentences[0].tokens[1].words[0].text == \"ca\"\n assert doc.sentences[0].tokens[1].words[1].text == \"n't\"\n\n assert len(doc.sentences[1].tokens) == 2\n # updated GUM MWT splits \"I can't\" into three segments\n # the way we want, \"I - ca - n't\"\n # previously it would split \"I - can - 't\"\n assert len(doc.sentences[1].tokens[0].words) == 3\n assert doc.sentences[1].tokens[0].words[0].text == \"I\"\n assert doc.sentences[1].tokens[0].words[1].text == \"ca\"\n assert doc.sentences[1].tokens[0].words[2].text == \"n't\"\n\n\ndef test_resplit_no_keep_tokens(pipeline):\n \"\"\"\n Test splitting without enforced token boundaries\n \"\"\"\n tokens = [[\"I\", \"can't\", \"believe\", \"it\"], [\"I can't\", \"sleep\"]]\n doc = resplit_mwt(tokens, pipeline, keep_tokens=False)\n assert len(doc.sentences) == 2\n assert len(doc.sentences[0].tokens) == 4\n assert len(doc.sentences[0].tokens[1].words) == 2\n assert doc.sentences[0].tokens[1].words[0].text == \"ca\"\n assert doc.sentences[0].tokens[1].words[1].text == \"n't\"\n\n assert len(doc.sentences[1].tokens) == 3\n assert len(doc.sentences[1].tokens[1].words) == 2\n assert doc.sentences[1].tokens[1].words[0].text == \"ca\"\n assert doc.sentences[1].tokens[1].words[1].text == \"n't\"\n"
},
{
"path": "stanza/tests/ner/__init__.py",
"content": ""
},
{
"path": "stanza/tests/ner/test_bsf_2_beios.py",
"content": "\"\"\"\nTests the conversion code for the lang_uk NER dataset\n\"\"\"\n\nimport unittest\nfrom stanza.utils.datasets.ner.convert_bsf_to_beios import convert_bsf, parse_bsf, BsfInfo\n\nimport pytest\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\nclass TestBsf2Beios(unittest.TestCase):\n \n def test_empty_markup(self):\n res = convert_bsf('', '')\n self.assertEqual('', res)\n\n def test_1line_markup(self):\n data = 'тележурналіст Василь'\n bsf_markup = 'T1\tPERS 14 20\tВасиль'\n expected = '''тележурналіст O\nВасиль S-PERS'''\n self.assertEqual(expected, convert_bsf(data, bsf_markup))\n\n def test_1line_follow_markup(self):\n data = 'тележурналіст Василь .'\n bsf_markup = 'T1\tPERS 14 20\tВасиль'\n expected = '''тележурналіст O\nВасиль S-PERS\n. O'''\n self.assertEqual(expected, convert_bsf(data, bsf_markup))\n\n def test_1line_2tok_markup(self):\n data = 'тележурналіст Василь Нагірний .'\n bsf_markup = 'T1\tPERS 14 29\tВасиль Нагірний'\n expected = '''тележурналіст O\nВасиль B-PERS\nНагірний E-PERS\n. O'''\n self.assertEqual(expected, convert_bsf(data, bsf_markup))\n\n def test_1line_Long_tok_markup(self):\n data = 'А в музеї Гуцульщини і Покуття можна '\n bsf_markup = 'T12\tORG 4 30\tмузеї Гуцульщини і Покуття'\n expected = '''А O\nв O\nмузеї B-ORG\nГуцульщини I-ORG\nі I-ORG\nПокуття E-ORG\nможна O'''\n self.assertEqual(expected, convert_bsf(data, bsf_markup))\n\n def test_2line_2tok_markup(self):\n data = '''тележурналіст Василь Нагірний .\nВ івано-франківському видавництві «Лілея НВ» вийшла друком'''\n bsf_markup = '''T1\tPERS 14 29\tВасиль Нагірний\nT2\tORG 67 75\tЛілея НВ'''\n expected = '''тележурналіст O\nВасиль B-PERS\nНагірний E-PERS\n. O\n\n\nВ O\nівано-франківському O\nвидавництві O\n« O\nЛілея B-ORG\nНВ E-ORG\n» O\nвийшла O\nдруком O'''\n self.assertEqual(expected, convert_bsf(data, bsf_markup))\n\n def test_real_markup(self):\n data = '''Через напіввоєнний стан в Україні та збільшення телефонних терористичних погроз українці купуватимуть sim-карти тільки за паспортами .\nПро це повідомив начальник управління зв'язків зі ЗМІ адміністрації Держспецзв'язку Віталій Кукса .\nВін зауважив , що днями відомство опублікує проект змін до правил надання телекомунікаційних послуг , де будуть прописані норми ідентифікації громадян .\nАбонентів , які на сьогодні вже мають sim-карту , за словами Віталія Кукси , реєструватимуть , коли ті звертатимуться в службу підтримки свого оператора мобільного зв'язку .\nОднак мобільні оператори побоюються , що таке нововведення помітно зменшить продаж стартових пакетів , адже спеціалізовані магазини є лише у містах .\nВідтак купити сімку в невеликих населених пунктах буде неможливо .\nКрім того , нова процедура ідентифікації абонентів вимагатиме від операторів мобільного зв'язку додаткових витрат .\n- Близько 90 % українських абонентів - це абоненти передоплати .\nЯкщо мова буде йти навіть про поетапну їх ідентифікацію , зробити це буде складно , довго і дорого .\nМобільним операторам доведеться йти на чималі витрати , пов'язані з укладанням і зберіганням договорів , веденням баз даних , - розповіла « Економічній правді » начальник відділу зв'язків з громадськістю « МТС-Україна » Вікторія Рубан .\n'''\n bsf_markup = '''T1\tLOC 26 33\tУкраїні\nT2\tORG 203 218\tДержспецзв'язку\nT3\tPERS 219 232\tВіталій Кукса\nT4\tPERS 449 462\tВіталія Кукси\nT5\tORG 1201 1219\tЕкономічній правді\nT6\tORG 1267 1278\tМТС-Україна\nT7\tPERS 1281 1295\tВікторія Рубан\n'''\n expected = '''Через O\nнапіввоєнний O\nстан O\nв O\nУкраїні S-LOC\nта O\nзбільшення O\nтелефонних O\nтерористичних O\nпогроз O\nукраїнці O\nкупуватимуть O\nsim-карти O\nтільки O\nза O\nпаспортами O\n. O\n\n\nПро O\nце O\nповідомив O\nначальник O\nуправління O\nзв'язків O\nзі O\nЗМІ O\nадміністрації O\nДержспецзв'язку S-ORG\nВіталій B-PERS\nКукса E-PERS\n. O\n\n\nВін O\nзауважив O\n, O\nщо O\nднями O\nвідомство O\nопублікує O\nпроект O\nзмін O\nдо O\nправил O\nнадання O\nтелекомунікаційних O\nпослуг O\n, O\nде O\nбудуть O\nпрописані O\nнорми O\nідентифікації O\nгромадян O\n. O\n\n\nАбонентів O\n, O\nякі O\nна O\nсьогодні O\nвже O\nмають O\nsim-карту O\n, O\nза O\nсловами O\nВіталія B-PERS\nКукси E-PERS\n, O\nреєструватимуть O\n, O\nколи O\nті O\nзвертатимуться O\nв O\nслужбу O\nпідтримки O\nсвого O\nоператора O\nмобільного O\nзв'язку O\n. O\n\n\nОднак O\nмобільні O\nоператори O\nпобоюються O\n, O\nщо O\nтаке O\nнововведення O\nпомітно O\nзменшить O\nпродаж O\nстартових O\nпакетів O\n, O\nадже O\nспеціалізовані O\nмагазини O\nє O\nлише O\nу O\nмістах O\n. O\n\n\nВідтак O\nкупити O\nсімку O\nв O\nневеликих O\nнаселених O\nпунктах O\nбуде O\nнеможливо O\n. O\n\n\nКрім O\nтого O\n, O\nнова O\nпроцедура O\nідентифікації O\nабонентів O\nвимагатиме O\nвід O\nоператорів O\nмобільного O\nзв'язку O\nдодаткових O\nвитрат O\n. O\n\n\n- O\nБлизько O\n90 O\n% O\nукраїнських O\nабонентів O\n- O\nце O\nабоненти O\nпередоплати O\n. O\n\n\nЯкщо O\nмова O\nбуде O\nйти O\nнавіть O\nпро O\nпоетапну O\nїх O\nідентифікацію O\n, O\nзробити O\nце O\nбуде O\nскладно O\n, O\nдовго O\nі O\nдорого O\n. O\n\n\nМобільним O\nоператорам O\nдоведеться O\nйти O\nна O\nчималі O\nвитрати O\n, O\nпов'язані O\nз O\nукладанням O\nі O\nзберіганням O\nдоговорів O\n, O\nведенням O\nбаз O\nданих O\n, O\n- O\nрозповіла O\n« O\nЕкономічній B-ORG\nправді E-ORG\n» O\nначальник O\nвідділу O\nзв'язків O\nз O\nгромадськістю O\n« O\nМТС-Україна S-ORG\n» O\nВікторія B-PERS\nРубан E-PERS\n. O'''\n self.assertEqual(expected, convert_bsf(data, bsf_markup))\n\n\nclass TestBsf(unittest.TestCase):\n\n def test_empty_bsf(self):\n self.assertEqual(parse_bsf(''), [])\n\n def test_empty2_bsf(self):\n self.assertEqual(parse_bsf(' \\n \\n'), [])\n\n def test_1line_bsf(self):\n bsf = 'T1\tPERS 103 118\tВасиль Нагірний'\n res = parse_bsf(bsf)\n expected = BsfInfo('T1', 'PERS', 103, 118, 'Василь Нагірний')\n self.assertEqual(len(res), 1)\n self.assertEqual(res, [expected])\n\n def test_2line_bsf(self):\n bsf = '''T9\tPERS 778 783\tКарла\nT10\tMISC 814 819\tміста'''\n res = parse_bsf(bsf)\n expected = [BsfInfo('T9', 'PERS', 778, 783, 'Карла'),\n BsfInfo('T10', 'MISC', 814, 819, 'міста')]\n self.assertEqual(len(res), 2)\n self.assertEqual(res, expected)\n\n def test_multiline_bsf(self):\n bsf = '''T3\tPERS 220 235\tАндрієм Кіщуком\nT4\tMISC 251 285\tА .\nKubler .\nСвітло і тіні маестро\nT5\tPERS 363 369\tКіблер'''\n res = parse_bsf(bsf)\n expected = [BsfInfo('T3', 'PERS', 220, 235, 'Андрієм Кіщуком'),\n BsfInfo('T4', 'MISC', 251, 285, '''А .\nKubler .\nСвітло і тіні маестро'''),\n BsfInfo('T5', 'PERS', 363, 369, 'Кіблер')]\n self.assertEqual(len(res), len(expected))\n self.assertEqual(res, expected)\n\n\nif __name__ == '__main__':\n unittest.main()\n"
},
{
"path": "stanza/tests/ner/test_bsf_2_iob.py",
"content": "\"\"\"\nTests the conversion code for the lang_uk NER dataset\n\"\"\"\n\nimport unittest\nfrom stanza.utils.datasets.ner.convert_bsf_to_beios import convert_bsf, parse_bsf, BsfInfo\n\nimport pytest\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\nclass TestBsf2Iob(unittest.TestCase):\n\n def test_1line_follow_markup_iob(self):\n data = 'тележурналіст Василь .'\n bsf_markup = 'T1\tPERS 14 20\tВасиль'\n expected = '''тележурналіст O\nВасиль B-PERS\n. O'''\n self.assertEqual(expected, convert_bsf(data, bsf_markup, converter='iob'))\n\n def test_1line_2tok_markup_iob(self):\n data = 'тележурналіст Василь Нагірний .'\n bsf_markup = 'T1\tPERS 14 29\tВасиль Нагірний'\n expected = '''тележурналіст O\nВасиль B-PERS\nНагірний I-PERS\n. O'''\n self.assertEqual(expected, convert_bsf(data, bsf_markup, converter='iob'))\n\n def test_1line_Long_tok_markup_iob(self):\n data = 'А в музеї Гуцульщини і Покуття можна '\n bsf_markup = 'T12\tORG 4 30\tмузеї Гуцульщини і Покуття'\n expected = '''А O\nв O\nмузеї B-ORG\nГуцульщини I-ORG\nі I-ORG\nПокуття I-ORG\nможна O'''\n self.assertEqual(expected, convert_bsf(data, bsf_markup, converter='iob'))\n\n def test_2line_2tok_markup_iob(self):\n data = '''тележурналіст Василь Нагірний .\nВ івано-франківському видавництві «Лілея НВ» вийшла друком'''\n bsf_markup = '''T1\tPERS 14 29\tВасиль Нагірний\nT2\tORG 67 75\tЛілея НВ'''\n expected = '''тележурналіст O\nВасиль B-PERS\nНагірний I-PERS\n. O\n\n\nВ O\nівано-франківському O\nвидавництві O\n« O\nЛілея B-ORG\nНВ I-ORG\n» O\nвийшла O\nдруком O'''\n self.assertEqual(expected, convert_bsf(data, bsf_markup, converter='iob'))\n\n def test_all_multiline_iob(self):\n data = '''його книжечка «А .\nKubler .\nСвітло і тіні маестро» .\nПричому'''\n bsf_markup = '''T4\tMISC 15 49\tА .\nKubler .\nСвітло і тіні маестро\n'''\n expected = '''його O\nкнижечка O\n« O\nА B-MISC\n. I-MISC\nKubler I-MISC\n. I-MISC\nСвітло I-MISC\nі I-MISC\nтіні I-MISC\nмаестро I-MISC\n» O\n. O\n\n\nПричому O'''\n self.assertEqual(expected, convert_bsf(data, bsf_markup, converter='iob'))\n\n\nif __name__ == '__main__':\n unittest.main()\n"
},
{
"path": "stanza/tests/ner/test_combine_ner_datasets.py",
"content": "import json\nimport os\nimport pytest\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\nfrom stanza.models.common.doc import Document\nfrom stanza.tests.ner.test_ner_training import write_temp_file, EN_TRAIN_BIO, EN_DEV_BIO\nfrom stanza.utils.datasets.ner import combine_ner_datasets\n\n\ndef test_combine(tmp_path):\n \"\"\"\n Test that if we write two short datasets and combine them, we get back\n one slightly longer dataset\n\n To simplify matters, we just use the same input text with longer\n amounts of text for each shard.\n \"\"\"\n SHARDS = (\"train\", \"dev\", \"test\")\n for s_num, shard in enumerate(SHARDS):\n t1_json = tmp_path / (\"en_t1.%s.json\" % shard)\n # eg, 1x, 2x, 3x the test data from test_ner_training\n write_temp_file(t1_json, \"\\n\\n\".join([EN_TRAIN_BIO] * (s_num + 1)))\n\n t2_json = tmp_path / (\"en_t2.%s.json\" % shard)\n write_temp_file(t2_json, \"\\n\\n\".join([EN_DEV_BIO] * (s_num + 1)))\n\n args = [\"--output_dataset\", \"en_c\", \"en_t1\", \"en_t2\", \"--input_dir\", str(tmp_path), \"--output_dir\", str(tmp_path)]\n combine_ner_datasets.main(args)\n\n for s_num, shard in enumerate(SHARDS):\n filename = tmp_path / (\"en_c.%s.json\" % shard)\n assert os.path.exists(filename)\n\n with open(filename, encoding=\"utf-8\") as fin:\n doc = Document(json.load(fin))\n assert len(doc.sentences) == (s_num + 1) * 3\n\n"
},
{
"path": "stanza/tests/ner/test_convert_amt.py",
"content": "\"\"\"\nTest some of the functions used for converting an AMT json to a Stanza json\n\"\"\"\n\n\nimport os\n\nimport pytest\n\nimport stanza\nfrom stanza.utils.datasets.ner import convert_amt\n\nfrom stanza.tests import TEST_MODELS_DIR\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\nTEXT = \"Jennifer Sh'reyan has lovely antennae.\"\n\ndef fake_label(label, start_char, end_char):\n return {'label': label,\n 'startOffset': start_char,\n 'endOffset': end_char}\n\nLABELS = [\n fake_label('Person', 0, 8),\n fake_label('Person', 9, 17),\n fake_label('Person', 0, 17),\n fake_label('Andorian', 0, 8),\n fake_label('Appendage', 29, 37),\n fake_label('Person', 1, 8),\n fake_label('Person', 0, 7),\n fake_label('Person', 0, 9),\n fake_label('Appendage', 29, 38),\n]\n\ndef fake_labels(*indices):\n return [LABELS[x] for x in indices]\n\ndef fake_docs(*indices):\n return [(TEXT, fake_labels(*indices))]\n\ndef test_remove_nesting():\n \"\"\"\n Test a few orders on nested items to make sure the desired results are coming back\n \"\"\"\n # this should be unchanged\n result = convert_amt.remove_nesting(fake_docs(0, 1))\n assert result == fake_docs(0, 1)\n\n # this should be returned sorted\n result = convert_amt.remove_nesting(fake_docs(0, 4, 1))\n assert result == fake_docs(0, 1, 4)\n\n # this should just have one copy\n result = convert_amt.remove_nesting(fake_docs(0, 0))\n assert result == fake_docs(0)\n \n # outer one preferred\n result = convert_amt.remove_nesting(fake_docs(0, 2))\n assert result == fake_docs(2)\n result = convert_amt.remove_nesting(fake_docs(1, 2))\n assert result == fake_docs(2)\n result = convert_amt.remove_nesting(fake_docs(5, 2))\n assert result == fake_docs(2)\n # order doesn't matter\n result = convert_amt.remove_nesting(fake_docs(0, 4, 2))\n assert result == fake_docs(2, 4)\n result = convert_amt.remove_nesting(fake_docs(2, 4, 0))\n assert result == fake_docs(2, 4)\n \n # first one preferred\n result = convert_amt.remove_nesting(fake_docs(0, 3))\n assert result == fake_docs(0)\n result = convert_amt.remove_nesting(fake_docs(3, 0))\n assert result == fake_docs(3)\n\ndef test_process_doc():\n nlp = stanza.Pipeline(dir=TEST_MODELS_DIR, processors=\"tokenize\", download_method=None)\n\n def check_results(doc, *expected):\n ner = [x[1] for x in doc[0]]\n assert ner == list(expected)\n\n # test a standard case of all the values lining up\n doc = convert_amt.process_doc(TEXT, fake_labels(2, 4), nlp)\n check_results(doc, \"B-Person\", \"I-Person\", \"O\", \"O\", \"B-Appendage\", \"O\")\n\n # test a slightly wrong start index\n doc = convert_amt.process_doc(TEXT, fake_labels(5, 1, 4), nlp)\n check_results(doc, \"B-Person\", \"B-Person\", \"O\", \"O\", \"B-Appendage\", \"O\")\n\n # test a slightly wrong end index\n doc = convert_amt.process_doc(TEXT, fake_labels(6, 1, 4), nlp)\n check_results(doc, \"B-Person\", \"B-Person\", \"O\", \"O\", \"B-Appendage\", \"O\")\n\n # test a slightly wronger end index\n doc = convert_amt.process_doc(TEXT, fake_labels(7, 4), nlp)\n check_results(doc, \"B-Person\", \"O\", \"O\", \"O\", \"B-Appendage\", \"O\")\n\n # test a period at the end of a text - should not be captured\n doc = convert_amt.process_doc(TEXT, fake_labels(7, 8), nlp)\n check_results(doc, \"B-Person\", \"O\", \"O\", \"O\", \"B-Appendage\", \"O\")\n\n \n"
},
{
"path": "stanza/tests/ner/test_convert_nkjp.py",
"content": "import pytest\n\nimport io\nimport os\nimport xml.etree.ElementTree as ET\n\nfrom stanza.utils.datasets.ner.convert_nkjp import MORPH_FILE, NER_FILE, extract_entities_from_subfolder, extract_entities_from_sentence, extract_unassigned_subfolder_entities\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\nEXPECTED_ENTITIES = {\n '1-p': {\n '1.39-s': [{'ent_id': 'named_1.39-s_n1', 'index': 0, 'orth': 'Sił Zbrojnych', 'ner_type': 'orgName', 'ner_subtype': None, 'targets': ['1.37-seg', '1.38-seg']}],\n '1.56-s': [],\n '1.79-s': []\n },\n '2-p': {\n '2.30-s': [],\n '2.45-s': []\n },\n '3-p': {\n '3.70-s': []\n }\n}\n\n\n@pytest.fixture(scope=\"module\")\ndef dataset(tmp_path_factory):\n dataset_path = tmp_path_factory.mktemp(\"nkjp_dataset\")\n sample_path = dataset_path / \"sample\"\n os.mkdir(sample_path)\n ann_path = sample_path / NER_FILE\n with open(ann_path, \"w\", encoding=\"utf-8\") as fout:\n fout.write(SAMPLE_ANN)\n morph_path = sample_path / MORPH_FILE\n with open(morph_path, \"w\", encoding=\"utf-8\") as fout:\n fout.write(SAMPLE_MORPHO)\n return dataset_path\n\nEXPECTED_TOKENS = [\n {'seg_id': '1.1-seg', 'i': 0, 'orth': '2', 'text': '2', 'tag': '_', 'ner': 'O', 'ner_subtype': None},\n {'seg_id': '1.37-seg', 'i': 36, 'orth': 'Sił', 'text': 'Sił', 'tag': '_', 'ner': 'B-orgName', 'ner_subtype': None},\n {'seg_id': '1.38-seg', 'i': 37, 'orth': 'Zbrojnych', 'text': 'Zbrojnych', 'tag': '_', 'ner': 'I-orgName', 'ner_subtype': None},\n]\n\ndef test_extract_entities_from_subfolder(dataset):\n entities = extract_entities_from_subfolder(\"sample\", dataset)\n assert len(entities) == 1\n assert len(entities['1-p']) == 1\n assert len(entities['1-p']['1.39-s']) == 39\n assert entities['1-p']['1.39-s']['1.1-seg'] == EXPECTED_TOKENS[0]\n assert entities['1-p']['1.39-s']['1.37-seg'] == EXPECTED_TOKENS[1]\n assert entities['1-p']['1.39-s']['1.38-seg'] == EXPECTED_TOKENS[2]\n\n\ndef test_extract_unassigned(dataset):\n entities = extract_unassigned_subfolder_entities(\"sample\", dataset)\n assert entities == EXPECTED_ENTITIES\n\nSENTENCE_SAMPLE = \"\"\"\n \n \n \n \n \n \n \n Sił Zbrojnych \n \n \n Siły Zbrojne \n \n \n \n \n \n \n \n \n \n\"\"\".strip()\n\n\nEMPTY_SENTENCE = \"\"\"\"\"\"\n\ndef test_extract_entities_from_sentence():\n rt = ET.fromstring(SENTENCE_SAMPLE)\n entities = extract_entities_from_sentence(rt)\n assert entities == EXPECTED_ENTITIES['1-p']['1.39-s']\n\n rt = ET.fromstring(EMPTY_SENTENCE)\n entities = extract_entities_from_sentence(rt)\n assert entities == []\n\n\n\n# picked completely at random, one sample file for testing:\n# 610-1-000248/ann_named.xml\n# only the first sentence is used in the morpho file\nSAMPLE_ANN = \"\"\"\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n Sił Zbrojnych \n \n \n Siły Zbrojne \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n
\n \n \n
\n \n \n \n \n\"\"\".lstrip()\n\n\n\nSAMPLE_MORPHO = \"\"\"\n\n\n \n \n \n \n \n \n \n \n \n \n \n 2 \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n 2 \n \n \n \n \n \n \n \n \n \n \n \n \n \n 2:adj:sg:nom:n:pos \n \n \n \n \n \n \n \n \n . \n \n \n \n \n \n \n \n \n . \n \n \n \n \n \n \n \n \n \n \n \n \n \n .:interp \n \n \n \n \n \n \n \n \n Wezwanie \n \n \n \n \n \n wezwanie \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n wezwać \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n wezwanie:subst:sg:acc:n \n \n \n \n \n \n \n \n \n , \n \n \n \n \n \n \n \n \n , \n \n \n \n \n \n \n \n \n \n \n \n \n \n ,:interp \n \n \n \n \n \n \n \n \n o \n \n \n \n \n \n o \n \n \n \n \n \n \n \n \n \n \n o \n \n \n \n \n \n \n \n \n \n \n \n \n \n ojciec \n \n \n \n \n \n \n \n \n \n \n \n \n \n o:prep:loc \n \n \n \n \n \n \n \n \n którym \n \n \n \n \n \n który \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n który:adj:sg:loc:n:pos \n \n \n \n \n \n \n \n \n mowa \n \n \n \n \n \n mowa \n \n \n \n \n \n \n \n \n \n \n \n \n \n mowa:subst:sg:nom:f \n \n \n \n \n \n \n \n \n w \n \n \n \n \n \n w \n \n \n \n \n \n \n \n \n \n \n \n \n \n wiek \n \n \n \n \n \n \n \n \n \n \n wielki \n \n \n \n \n \n \n \n \n \n \n wiersz \n \n \n \n \n \n \n \n \n \n \n wieś \n \n \n \n \n \n \n \n \n \n \n wyspa \n \n \n \n \n \n \n \n \n \n \n \n \n \n w:prep:loc:nwok \n \n \n \n \n \n \n \n \n ust \n \n \n \n \n \n usta \n \n \n \n \n \n \n \n \n \n \n ustęp \n \n \n \n \n \n \n \n \n \n \n \n \n \n ustęp:brev:pun \n \n \n \n \n \n \n \n \n . \n \n \n \n \n \n \n \n \n . \n \n \n \n \n \n \n \n \n \n \n \n \n \n .:interp \n \n \n \n \n \n \n \n \n 1 \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n 1 \n \n \n \n \n \n \n \n \n \n \n \n \n \n 1:adj:sg:loc:m3:pos \n \n \n \n \n \n \n \n \n , \n \n \n \n \n \n \n \n \n , \n \n \n \n \n \n \n \n \n \n \n \n \n \n ,:interp \n \n \n \n \n \n \n \n \n doręcza \n \n \n \n \n \n doręczać \n \n \n \n \n \n \n \n \n \n \n doręcze \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n doręczać:fin:sg:ter:imperf \n \n \n \n \n \n \n \n \n się \n \n \n \n \n \n się \n \n \n \n \n \n \n \n \n \n \n \n \n \n się:qub \n \n \n \n \n \n \n \n \n na \n \n \n \n \n \n na \n \n \n \n \n \n \n \n \n \n \n na \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n na:prep:acc \n \n \n \n \n \n \n \n \n czternaście \n \n \n \n \n \n czternaście \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n czternaście:num:pl:acc:m3:rec \n \n \n \n \n \n \n \n \n dni \n \n \n \n \n \n dni \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n dzień \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n dzień:subst:pl:gen:m3 \n \n \n \n \n \n \n \n \n przed \n \n \n \n \n \n przed \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n przed:prep:inst:nwok \n \n \n \n \n \n \n \n \n terminem \n \n \n \n \n \n termin \n \n \n \n \n \n \n \n \n \n \n \n \n \n termin:subst:sg:inst:m3 \n \n \n \n \n \n \n \n \n wykonania \n \n \n \n \n \n wykonanie \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n wykonać \n \n \n \n \n \n \n \n \n \n \n \n \n \n wykonać:ger:sg:gen:n:perf:aff \n \n \n \n \n \n \n \n \n świadczenia \n \n \n \n \n \n świadczenie \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n świadczyć \n \n \n \n \n \n \n \n \n \n \n \n \n \n świadczenie:subst:sg:gen:n \n \n \n \n \n \n \n \n \n , \n \n \n \n \n \n \n \n \n , \n \n \n \n \n \n \n \n \n \n \n \n \n \n ,:interp \n \n \n \n \n \n \n \n \n z \n \n \n \n \n \n z \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n z \n \n \n \n \n \n \n \n \n \n \n zeszyt \n \n \n \n \n \n \n \n \n \n \n \n \n \n z:prep:inst:nwok \n \n \n \n \n \n \n \n \n wyjątkiem \n \n \n \n \n \n wyjątek \n \n \n \n \n \n \n \n \n \n \n \n \n \n wyjątek:subst:sg:inst:m3 \n \n \n \n \n \n \n \n \n przypadków \n \n \n \n \n \n przypadek \n \n \n \n \n \n \n \n \n \n \n \n \n \n przypadek:subst:pl:gen:m3 \n \n \n \n \n \n \n \n \n , \n \n \n \n \n \n \n \n \n , \n \n \n \n \n \n \n \n \n \n \n \n \n \n ,:interp \n \n \n \n \n \n \n \n \n w \n \n \n \n \n \n w \n \n \n \n \n \n \n \n \n \n \n \n \n \n wiek \n \n \n \n \n \n \n \n \n \n \n wielki \n \n \n \n \n \n \n \n \n \n \n wiersz \n \n \n \n \n \n \n \n \n \n \n wieś \n \n \n \n \n \n \n \n \n \n \n wyspa \n \n \n \n \n \n \n \n \n \n \n \n \n \n w:prep:loc:nwok \n \n \n \n \n \n \n \n \n których \n \n \n \n \n \n który \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n który:adj:pl:loc:m3:pos \n \n \n \n \n \n \n \n \n wykonanie \n \n \n \n \n \n wykonanie \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n wykonać \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n wykonać:ger:sg:nom:n:perf:aff \n \n \n \n \n \n \n \n \n świadczenia \n \n \n \n \n \n świadczenie \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n świadczyć \n \n \n \n \n \n \n \n \n \n \n \n \n \n świadczenie:subst:sg:gen:n \n \n \n \n \n \n \n \n \n następuje \n \n \n \n \n \n następować \n \n \n \n \n \n \n \n \n \n \n \n \n \n następować:fin:sg:ter:imperf \n \n \n \n \n \n \n \n \n w \n \n \n \n \n \n w \n \n \n \n \n \n \n \n \n \n \n \n \n \n wiek \n \n \n \n \n \n \n \n \n \n \n wielki \n \n \n \n \n \n \n \n \n \n \n wiersz \n \n \n \n \n \n \n \n \n \n \n wieś \n \n \n \n \n \n \n \n \n \n \n wyspa \n \n \n \n \n \n \n \n \n \n \n \n \n \n w:prep:loc:nwok \n \n \n \n \n \n \n \n \n celu \n \n \n \n \n \n Cela \n \n \n \n \n \n \n \n \n \n \n cel \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n cel:subst:sg:loc:m3 \n \n \n \n \n \n \n \n \n sprawdzenia \n \n \n \n \n \n sprawdzić \n \n \n \n \n \n \n \n \n \n \n \n \n \n sprawdzić:ger:sg:gen:n:perf:aff \n \n \n \n \n \n \n \n \n gotowości \n \n \n \n \n \n gotowość \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n gotowość:subst:sg:gen:f \n \n \n \n \n \n \n \n \n mobilizacyjnej \n \n \n \n \n \n mobilizacyjny \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n mobilizacyjny:adj:sg:gen:f:pos \n \n \n \n \n \n \n \n \n Sił \n \n \n \n \n \n siła \n \n \n \n \n \n \n \n \n \n \n siły \n \n \n \n \n \n \n \n \n \n \n \n \n \n siła:subst:pl:gen:f \n \n \n \n \n \n \n \n \n Zbrojnych \n \n \n \n \n \n zbrojny \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n zbrojny:adj:pl:gen:f:pos \n \n \n \n \n \n \n \n \n . \n \n \n \n \n \n \n \n \n . \n \n \n \n \n \n \n \n \n \n \n \n \n \n .:interp \n \n \n \n \n \n \n
\n \n \n \n \n\"\"\".lstrip()\n"
},
{
"path": "stanza/tests/ner/test_convert_starlang_ner.py",
"content": "\"\"\"\nTest a couple different classes of trees to check the output of the Starlang conversion for NER\n\"\"\"\n\nimport os\nimport tempfile\n\nimport pytest\n\nfrom stanza.utils.datasets.ner import convert_starlang_ner\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\nTREE=\"( (S (NP (NP {morphologicalAnalysis=bayan+NOUN+A3SG+PNON+NOM}{metaMorphemes=bayan}{turkish=Bayan}{english=Ms.}{semantics=TUR10-0396530}{namedEntity=PERSON}{propBank=ARG0$TUR10-0148580}{englishSemantics=ENG31-06352895-n}) (NP {morphologicalAnalysis=haag+NOUN+PROP+A3SG+PNON+NOM}{metaMorphemes=haag}{turkish=Haag}{english=Haag}{semantics=TUR10-0000000}{namedEntity=PERSON}{propBank=ARG0$TUR10-0148580})) (VP (NP {morphologicalAnalysis=elianti+NOUN+PROP+A3SG+PNON+NOM}{metaMorphemes=elianti}{turkish=Elianti}{english=Elianti}{semantics=TUR10-0000000}{namedEntity=NONE}{propBank=ARG1$TUR10-0148580}) (VP {morphologicalAnalysis=çal+VERB+POS+AOR+A3SG}{metaMorphemes=çal+Ar}{turkish=çalar}{english=plays}{semantics=TUR10-0148580}{namedEntity=NONE}{propBank=PREDICATE$TUR10-0148580}{englishSemantics=ENG31-01730049-v})) (. {morphologicalAnalysis=.+PUNC}{metaMorphemes=.}{metaMorphemesMoved=.}{turkish=.}{english=.}{semantics=TUR10-1081860}{namedEntity=NONE}{propBank=NONE})) )\"\n\ndef test_read_tree():\n \"\"\"\n Test a basic tree read\n \"\"\"\n sentence = convert_starlang_ner.read_tree(TREE)\n expected = [('Bayan', 'PERSON'), ('Haag', 'PERSON'), ('Elianti', 'O'), ('çalar', 'O'), ('.', 'O')]\n assert sentence == expected\n\n"
},
{
"path": "stanza/tests/ner/test_data.py",
"content": "import json\nimport pytest\n\nfrom stanza.models import ner_tagger\nfrom stanza.models.common.doc import Document\nfrom stanza.models.ner.data import DataLoader\nfrom stanza.tests import TEST_WORKING_DIR\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\n\nONE_SENTENCE = \"\"\"\n[\n [\n {\n \"text\": \"EU\",\n \"ner\": \"B-ORG\"\n },\n {\n \"text\": \"rejects\",\n \"ner\": \"O\"\n },\n {\n \"text\": \"German\",\n \"ner\": \"B-MISC\"\n },\n {\n \"text\": \"call\",\n \"ner\": \"O\"\n },\n {\n \"text\": \"to\",\n \"ner\": \"O\"\n },\n {\n \"text\": \"boycott\",\n \"ner\": \"O\"\n },\n {\n \"text\": \"Mox\",\n \"ner\": \"B-MISC\"\n },\n {\n \"text\": \"Opal\",\n \"ner\": \"I-MISC\"\n },\n {\n \"text\": \".\",\n \"ner\": \"O\"\n }\n ]\n]\n\"\"\"\n\n@pytest.fixture(scope=\"module\")\ndef pretrain_file():\n return f'{TEST_WORKING_DIR}/in/tiny_emb.pt'\n\n\n@pytest.fixture(scope=\"module\")\ndef one_sentence_json_path(tmpdir_factory):\n filename = tmpdir_factory.mktemp('data').join(\"sentence.json\")\n with open(filename, 'w') as fout:\n fout.write(ONE_SENTENCE)\n return filename\n\n\ndef test_build_vocab(pretrain_file, one_sentence_json_path, tmp_path):\n \"\"\"\n Test that when loading a data file, we get back \n \"\"\"\n args = ner_tagger.parse_args([\"--wordvec_pretrain_file\", pretrain_file])\n pt = ner_tagger.load_pretrain(args)\n\n with open(one_sentence_json_path) as fin:\n train_doc = Document(json.load(fin))\n\n train_batch = DataLoader(train_doc, args['batch_size'], args, pt, vocab=None, evaluation=False, scheme=args.get('train_scheme'), max_batch_words=args['max_batch_words'])\n\n vocab = train_batch.vocab\n pt_words = list(vocab['word'])\n assert pt_words == ['', '', '', '', 'unban', 'mox', 'opal']\n delta_words = list(vocab['delta'])\n assert delta_words == ['', '', '', '', 'eu', 'rejects', 'german', 'call', 'to', 'boycott', 'mox', 'opal', '.']\n tags = list(vocab['tag'])\n assert tags == [[''], [''], [], [''], ['S-ORG'], ['O'], ['S-MISC'], ['B-MISC'], ['E-MISC']]\n\n\ndef test_build_vocab_ignore_repeats(pretrain_file, one_sentence_json_path, tmp_path):\n \"\"\"\n Test that when loading a data file, we get back \n \"\"\"\n args = ner_tagger.parse_args([\"--wordvec_pretrain_file\", pretrain_file, \"--emb_finetune_known_only\"])\n pt = ner_tagger.load_pretrain(args)\n\n with open(one_sentence_json_path) as fin:\n train_doc = Document(json.load(fin))\n\n train_batch = DataLoader(train_doc, args['batch_size'], args, pt, vocab=None, evaluation=False, scheme=args.get('train_scheme'), max_batch_words=args['max_batch_words'])\n\n vocab = train_batch.vocab\n pt_words = list(vocab['word'])\n assert pt_words == ['', '', '', '', 'unban', 'mox', 'opal']\n delta_words = list(vocab['delta'])\n assert delta_words == ['', '', '', '', 'mox', 'opal']\n tags = list(vocab['tag'])\n assert tags == [[''], [''], [], [''], ['S-ORG'], ['O'], ['S-MISC'], ['B-MISC'], ['E-MISC']]\n"
},
{
"path": "stanza/tests/ner/test_from_conllu.py",
"content": "import pytest\n\nfrom stanza import Pipeline\nfrom stanza.utils.conll import CoNLL\nfrom stanza.tests import TEST_MODELS_DIR\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\ndef test_from_conllu():\n \"\"\"\n If the doc does not have the entire text available, make sure it still safely processes the text\n\n Test case supplied from user - see issue #1428\n \"\"\"\n pipe = Pipeline(\"en\", dir=TEST_MODELS_DIR, processors=\"tokenize,ner\", download_method=None)\n doc = pipe(\"In February, I traveled to Seattle. Dr. Pritchett gave me a new hip\")\n ents = [x.text for x in doc.ents]\n # the default NER model ought to find these three\n assert ents == ['February', 'Seattle', 'Pritchett']\n\n doc_conllu = \"{:C}\\n\\n\".format(doc)\n doc = CoNLL.conll2doc(input_str=doc_conllu)\n pipe = Pipeline(\"en\", dir=TEST_MODELS_DIR, processors=\"tokenize,ner\", tokenize_pretokenized=True, download_method=None)\n pipe(doc)\n ents = [x.text for x in doc.ents]\n # this should still work when processed from a CoNLLu document\n # the bug previously caused a crash because the text to construct\n # the entities was not available, since the Document wouldn't have\n # the entire document text available\n assert ents == ['February', 'Seattle', 'Pritchett']\n"
},
{
"path": "stanza/tests/ner/test_models_ner_scorer.py",
"content": "\"\"\"\nSimple test of the scorer module for NER\n\"\"\"\n\nimport pytest\nimport stanza\n\nfrom stanza.tests import *\nfrom stanza.models.ner.scorer import score_by_token, score_by_entity\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\ndef test_ner_scorer():\n pred_sequences = [['O', 'S-LOC', 'O', 'O', 'B-PER', 'E-PER'],\n ['O', 'S-MISC', 'O', 'E-ORG', 'O', 'B-PER', 'I-PER', 'E-PER']]\n gold_sequences = [['O', 'B-LOC', 'E-LOC', 'O', 'B-PER', 'E-PER'],\n ['O', 'S-MISC', 'B-ORG', 'E-ORG', 'O', 'B-PER', 'E-PER', 'S-LOC']]\n \n token_p, token_r, token_f, confusion = score_by_token(pred_sequences, gold_sequences)\n assert pytest.approx(token_p, abs=0.00001) == 0.625\n assert pytest.approx(token_r, abs=0.00001) == 0.5\n assert pytest.approx(token_f, abs=0.00001) == 0.55555\n\n entity_p, entity_r, entity_f, entity_f1 = score_by_entity(pred_sequences, gold_sequences)\n assert pytest.approx(entity_p, abs=0.00001) == 0.4\n assert pytest.approx(entity_r, abs=0.00001) == 0.33333\n assert pytest.approx(entity_f, abs=0.00001) == 0.36363\n assert entity_f1 == {'LOC': 0.0, 'MISC': 1.0, 'ORG': 0.0, 'PER': 0.5}\n"
},
{
"path": "stanza/tests/ner/test_ner_tagger.py",
"content": "\"\"\"\nBasic testing of the NER tagger.\n\"\"\"\n\nimport os\nimport pytest\nimport stanza\n\nfrom stanza.tests import *\nfrom stanza.models import ner_tagger\nfrom stanza.utils.confusion import confusion_to_macro_f1\nimport stanza.utils.datasets.ner.prepare_ner_file as prepare_ner_file\nfrom stanza.utils.training.run_ner import build_pretrain_args\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\nEN_DOC = \"Chris Manning is a good man. He works in Stanford University.\"\n\nEN_DOC_GOLD = \"\"\"\n\n\n\"\"\".strip()\n\nEN_BIO = \"\"\"\nChris B-PERSON\nManning E-PERSON\nis O\na O\ngood O\nman O\n. O\n\nHe O\nworks O\nin O\nStanford B-ORG\nUniversity E-ORG\n. O\n\"\"\".strip().replace(\" \", \"\\t\")\n\nEN_EXPECTED_OUTPUT = \"\"\"\nChris B-PERSON B-PERSON\nManning E-PERSON E-PERSON\nis O O\na O O\ngood O O\nman O O\n. O O\n\nHe O O\nworks O O\nin O O\nStanford B-ORG B-ORG\nUniversity E-ORG E-ORG\n. O O\n\"\"\".strip().replace(\" \", \"\\t\")\n\n\ndef test_ner():\n nlp = stanza.Pipeline(**{'processors': 'tokenize,ner', 'dir': TEST_MODELS_DIR, 'lang': 'en', 'logging_level': 'error'})\n doc = nlp(EN_DOC)\n assert EN_DOC_GOLD == '\\n'.join([ent.pretty_print() for ent in doc.ents])\n\ndef test_evaluate(tmp_path):\n \"\"\"\n This simple example should have a 1.0 f1 for the ontonote model\n \"\"\"\n package = \"ontonotes-ww-multi_charlm\"\n model_path = os.path.join(TEST_MODELS_DIR, \"en\", \"ner\", package + \".pt\")\n assert os.path.exists(model_path), \"The {} model should be downloaded as part of setup.py\".format(package)\n\n os.makedirs(tmp_path, exist_ok=True)\n\n test_bio_filename = tmp_path / \"test.bio\"\n test_json_filename = tmp_path / \"test.json\"\n test_output_filename = tmp_path / \"output.bio\"\n with open(test_bio_filename, \"w\", encoding=\"utf-8\") as fout:\n fout.write(EN_BIO)\n\n prepare_ner_file.process_dataset(test_bio_filename, test_json_filename)\n\n args = [\"--save_name\", str(model_path),\n \"--eval_file\", str(test_json_filename),\n \"--eval_output_file\", str(test_output_filename),\n \"--mode\", \"predict\"]\n args = args + build_pretrain_args(\"en\", package, model_dir=TEST_MODELS_DIR, extra_args=[])\n args = ner_tagger.parse_args(args=args)\n confusion = ner_tagger.evaluate(args)\n assert confusion_to_macro_f1(confusion) == pytest.approx(1.0)\n\n with open(test_output_filename, encoding=\"utf-8\") as fin:\n results = fin.read().strip()\n\n assert results == EN_EXPECTED_OUTPUT\n"
},
{
"path": "stanza/tests/ner/test_ner_trainer.py",
"content": "import pytest\n\nfrom stanza.tests import *\n\nfrom stanza.models.ner import trainer\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\ndef test_fix_singleton_tags():\n TESTS = [\n ([\"O\"], [\"O\"]),\n ([\"B-PER\"], [\"S-PER\"]),\n ([\"B-PER\", \"I-PER\"], [\"B-PER\", \"E-PER\"]),\n ([\"B-PER\", \"O\", \"B-PER\"], [\"S-PER\", \"O\", \"S-PER\"]),\n ([\"B-PER\", \"B-PER\", \"I-PER\"], [\"S-PER\", \"B-PER\", \"E-PER\"]),\n ([\"B-PER\", \"I-PER\", \"O\", \"B-PER\"], [\"B-PER\", \"E-PER\", \"O\", \"S-PER\"]),\n ([\"B-PER\", \"B-PER\", \"I-PER\", \"B-PER\"], [\"S-PER\", \"B-PER\", \"E-PER\", \"S-PER\"]),\n ([\"B-PER\", \"I-ORG\", \"O\", \"B-PER\"], [\"S-PER\", \"S-ORG\", \"O\", \"S-PER\"]),\n ([\"B-PER\", \"I-PER\", \"E-PER\", \"O\", \"B-PER\", \"E-PER\"], [\"B-PER\", \"I-PER\", \"E-PER\", \"O\", \"B-PER\", \"E-PER\"]),\n ([\"S-PER\", \"B-PER\", \"E-PER\"], [\"S-PER\", \"B-PER\", \"E-PER\"]),\n ([\"E-PER\"], [\"S-PER\"]),\n ([\"E-PER\", \"O\", \"E-PER\"], [\"S-PER\", \"O\", \"S-PER\"]),\n ([\"B-PER\", \"E-ORG\", \"O\", \"B-PER\"], [\"S-PER\", \"S-ORG\", \"O\", \"S-PER\"]),\n ([\"I-PER\", \"I-PER\", \"E-PER\", \"O\", \"B-PER\", \"E-PER\"], [\"B-PER\", \"I-PER\", \"E-PER\", \"O\", \"B-PER\", \"E-PER\"]),\n ([\"B-PER\", \"I-PER\", \"I-PER\", \"O\", \"B-PER\", \"E-PER\"], [\"B-PER\", \"I-PER\", \"E-PER\", \"O\", \"B-PER\", \"E-PER\"]),\n ([\"B-PER\", \"I-PER\", \"E-PER\", \"O\", \"I-PER\", \"E-PER\"], [\"B-PER\", \"I-PER\", \"E-PER\", \"O\", \"B-PER\", \"E-PER\"]),\n ([\"B-PER\", \"I-PER\", \"E-PER\", \"O\", \"B-PER\", \"I-PER\"], [\"B-PER\", \"I-PER\", \"E-PER\", \"O\", \"B-PER\", \"E-PER\"]),\n ([\"I-PER\", \"I-PER\", \"I-PER\", \"O\", \"I-PER\", \"I-PER\"], [\"B-PER\", \"I-PER\", \"E-PER\", \"O\", \"B-PER\", \"E-PER\"]),\n ]\n \n for unfixed, expected in TESTS:\n assert trainer.fix_singleton_tags(unfixed) == expected, \"Error converting {} to {}\".format(unfixed, expected)\n"
},
{
"path": "stanza/tests/ner/test_ner_training.py",
"content": "import json\nimport logging\nimport os\nimport warnings\n\nimport pytest\nimport torch\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\nfrom stanza.models import ner_tagger\nfrom stanza.models.ner.trainer import Trainer\nfrom stanza.tests import TEST_WORKING_DIR\nfrom stanza.utils.datasets.ner.prepare_ner_file import process_dataset\n\nlogger = logging.getLogger('stanza')\n\nEN_TRAIN_BIO = \"\"\"\nChris B-PERSON\nManning E-PERSON\nis O\na O\ngood O\nman O\n. O\n\nHe O\nworks O\nin O\nStanford B-ORG\nUniversity E-ORG\n. O\n\"\"\".lstrip().replace(\" \", \"\\t\")\n\nEN_DEV_BIO = \"\"\"\nChris B-PERSON\nManning E-PERSON\nis O\npart O\nof O\nComputer B-ORG\nScience E-ORG\n\"\"\".lstrip().replace(\" \", \"\\t\")\n\nEN_TRAIN_2TAG = \"\"\"\nChris B-PERSON B-PER\nManning E-PERSON E-PER\nis O O\na O O\ngood O O\nman O O\n. O O\n\nHe O O\nworks O O\nin O O\nStanford B-ORG B-ORG\nUniversity E-ORG B-ORG\n. O O\n\"\"\".strip().replace(\" \", \"\\t\")\n\nEN_TRAIN_2TAG_EMPTY2 = \"\"\"\nChris B-PERSON -\nManning E-PERSON -\nis O -\na O -\ngood O -\nman O -\n. O -\n\nHe O -\nworks O -\nin O -\nStanford B-ORG -\nUniversity E-ORG -\n. O -\n\"\"\".strip().replace(\" \", \"\\t\")\n\nEN_DEV_2TAG = \"\"\"\nChris B-PERSON B-PER\nManning E-PERSON E-PER\nis O O\npart O O\nof O O\nComputer B-ORG B-ORG\nScience E-ORG E-ORG\n\"\"\".strip().replace(\" \", \"\\t\")\n\n@pytest.fixture(scope=\"module\")\ndef pretrain_file():\n return f'{TEST_WORKING_DIR}/in/tiny_emb.pt'\n\ndef write_temp_file(filename, bio_data):\n bio_filename = os.path.splitext(filename)[0] + \".bio\"\n with open(bio_filename, \"w\", encoding=\"utf-8\") as fout:\n fout.write(bio_data)\n process_dataset(bio_filename, filename)\n\ndef write_temp_2tag(filename, bio_data):\n doc = []\n sentences = bio_data.split(\"\\n\\n\")\n for sentence in sentences:\n doc.append([])\n for word in sentence.split(\"\\n\"):\n text, tags = word.split(\"\\t\", maxsplit=1)\n doc[-1].append({\n \"text\": text,\n \"multi_ner\": tags.split()\n })\n\n with open(filename, \"w\", encoding=\"utf-8\") as fout:\n json.dump(doc, fout)\n\ndef get_args(tmp_path, pretrain_file, train_json, dev_json, *extra_args):\n save_dir = tmp_path / \"models\"\n args = [\"--data_dir\", str(tmp_path),\n \"--wordvec_pretrain_file\", pretrain_file,\n \"--train_file\", str(train_json),\n \"--eval_file\", str(dev_json),\n \"--shorthand\", \"en_test\",\n \"--max_steps\", \"100\",\n \"--eval_interval\", \"40\",\n \"--save_dir\", str(save_dir)]\n\n args = args + list(extra_args)\n return args\n\ndef run_two_tag_training(pretrain_file, tmp_path, *extra_args, train_data=EN_TRAIN_2TAG):\n train_json = tmp_path / \"en_test.train.json\"\n write_temp_2tag(train_json, train_data)\n\n dev_json = tmp_path / \"en_test.dev.json\"\n write_temp_2tag(dev_json, EN_DEV_2TAG)\n\n args = get_args(tmp_path, pretrain_file, train_json, dev_json, *extra_args)\n return ner_tagger.main(args)\n\ndef test_basic_two_tag_training(pretrain_file, tmp_path):\n trainer = run_two_tag_training(pretrain_file, tmp_path)\n assert len(trainer.model.tag_clfs) == 2\n assert len(trainer.model.crits) == 2\n assert len(trainer.vocab['tag'].lens()) == 2\n\ndef test_two_tag_training_backprop(pretrain_file, tmp_path):\n \"\"\"\n Test that the training is backproping both tags\n\n We can do this by using the \"finetune\" mechanism and verifying\n that the output tensors are different\n \"\"\"\n trainer = run_two_tag_training(pretrain_file, tmp_path)\n\n # first, need to save the final model before restarting\n # (alternatively, could reload the final checkpoint)\n trainer.save(os.path.join(trainer.args['save_dir'], trainer.args['save_name']))\n new_trainer = run_two_tag_training(pretrain_file, tmp_path, \"--finetune\")\n\n assert len(trainer.model.tag_clfs) == 2\n assert len(new_trainer.model.tag_clfs) == 2\n for old_clf, new_clf in zip(trainer.model.tag_clfs, new_trainer.model.tag_clfs):\n assert not torch.allclose(old_clf.weight, new_clf.weight)\n\ndef test_two_tag_training_c2_backprop(pretrain_file, tmp_path):\n \"\"\"\n Test that the training is backproping only one tag if one column is blank\n\n We can do this by using the \"finetune\" mechanism and verifying\n that the output tensors are different in just the first column\n \"\"\"\n trainer = run_two_tag_training(pretrain_file, tmp_path)\n\n # first, need to save the final model before restarting\n # (alternatively, could reload the final checkpoint)\n trainer.save(os.path.join(trainer.args['save_dir'], trainer.args['save_name']))\n new_trainer = run_two_tag_training(pretrain_file, tmp_path, \"--finetune\", train_data=EN_TRAIN_2TAG_EMPTY2)\n\n assert len(trainer.model.tag_clfs) == 2\n assert len(new_trainer.model.tag_clfs) == 2\n assert not torch.allclose(trainer.model.tag_clfs[0].weight, new_trainer.model.tag_clfs[0].weight)\n assert torch.allclose(trainer.model.tag_clfs[1].weight, new_trainer.model.tag_clfs[1].weight)\n\ndef test_connected_two_tag_training(pretrain_file, tmp_path):\n trainer = run_two_tag_training(pretrain_file, tmp_path, \"--connect_output_layers\")\n assert len(trainer.model.tag_clfs) == 2\n assert len(trainer.model.crits) == 2\n assert len(trainer.vocab['tag'].lens()) == 2\n\n # this checks that with the connected output layers,\n # the second output layer has its size increased\n # by the number of tags known to the first output layer\n assert trainer.model.tag_clfs[1].weight.shape[1] == trainer.vocab['tag'].lens()[0] + trainer.model.tag_clfs[0].weight.shape[1]\n\ndef run_training(pretrain_file, tmp_path, *extra_args):\n train_json = tmp_path / \"en_test.train.json\"\n write_temp_file(train_json, EN_TRAIN_BIO)\n\n dev_json = tmp_path / \"en_test.dev.json\"\n write_temp_file(dev_json, EN_DEV_BIO)\n\n args = get_args(tmp_path, pretrain_file, train_json, dev_json, *extra_args)\n return ner_tagger.main(args)\n\n\ndef test_train_model_gpu(pretrain_file, tmp_path):\n \"\"\"\n Briefly train an NER model (no expectation of correctness) and check that it is on the GPU\n \"\"\"\n trainer = run_training(pretrain_file, tmp_path)\n if not torch.cuda.is_available():\n warnings.warn(\"Cannot check that the NER model is on the GPU, since GPU is not available\")\n return\n\n model = trainer.model\n device = next(model.parameters()).device\n assert str(device).startswith(\"cuda\")\n\n\ndef test_train_model_cpu(pretrain_file, tmp_path):\n \"\"\"\n Briefly train an NER model (no expectation of correctness) and check that it is on the GPU\n \"\"\"\n trainer = run_training(pretrain_file, tmp_path, \"--cpu\")\n\n model = trainer.model\n device = next(model.parameters()).device\n assert str(device).startswith(\"cpu\")\n\ndef model_file_has_bert(filename):\n checkpoint = torch.load(filename, lambda storage, loc: storage, weights_only=True)\n return any(x.startswith(\"bert_model.\") for x in checkpoint['model'].keys())\n\ndef test_with_bert(pretrain_file, tmp_path):\n trainer = run_training(pretrain_file, tmp_path, '--bert_model', 'hf-internal-testing/tiny-bert')\n model_file = os.path.join(trainer.args['save_dir'], trainer.args['save_name'])\n assert not model_file_has_bert(model_file)\n\ndef test_with_bert_finetune(pretrain_file, tmp_path):\n trainer = run_training(pretrain_file, tmp_path, '--bert_model', 'hf-internal-testing/tiny-bert', '--bert_finetune')\n model_file = os.path.join(trainer.args['save_dir'], trainer.args['save_name'])\n assert model_file_has_bert(model_file)\n\n foo_save_filename = os.path.join(tmp_path, \"foo_\" + trainer.args['save_name'])\n bar_save_filename = os.path.join(tmp_path, \"bar_\" + trainer.args['save_name'])\n trainer.save(foo_save_filename)\n assert model_file_has_bert(foo_save_filename)\n\n # TODO: technically this should still work if we turn off bert finetuning when reloading\n reloaded_trainer = Trainer(args=trainer.args, model_file=foo_save_filename)\n reloaded_trainer.save(bar_save_filename)\n assert model_file_has_bert(bar_save_filename)\n\ndef test_with_peft_finetune(pretrain_file, tmp_path):\n # TODO: check that the peft tensors are moving when training?\n trainer = run_training(pretrain_file, tmp_path, '--bert_model', 'hf-internal-testing/tiny-bert', '--use_peft')\n model_file = os.path.join(trainer.args['save_dir'], trainer.args['save_name'])\n checkpoint = torch.load(model_file, lambda storage, loc: storage, weights_only=True)\n assert 'bert_lora' in checkpoint\n assert not any(x.startswith(\"bert_model.\") for x in checkpoint['model'].keys())\n\n # test loading\n reloaded_trainer = Trainer(args=trainer.args, model_file=model_file)\n"
},
{
"path": "stanza/tests/ner/test_ner_utils.py",
"content": "import pytest\n\nfrom stanza.tests import *\n\nfrom stanza.models.common.vocab import EMPTY\nfrom stanza.models.ner import utils\n\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\nWORDS = [[\"Unban\", \"Mox\", \"Opal\"], [\"Ragavan\", \"is\", \"red\"], [\"Urza\", \"Lord\", \"High\", \"Artificer\", \"goes\", \"infinite\", \"with\", \"Thopter\", \"Sword\"]]\nBIO_TAGS = [[\"O\", \"B-ART\", \"I-ART\"], [\"B-MONKEY\", \"O\", \"B-COLOR\"], [\"B-PER\", \"I-PER\", \"I-PER\", \"I-PER\", \"O\", \"O\", \"O\", \"B-WEAPON\", \"B-WEAPON\"]]\nBIO_U_TAGS = [[\"O\", \"B_ART\", \"I_ART\"], [\"B_MONKEY\", \"O\", \"B_COLOR\"], [\"B_PER\", \"I_PER\", \"I_PER\", \"I_PER\", \"O\", \"O\", \"O\", \"B_WEAPON\", \"B_WEAPON\"]]\nBIOES_TAGS = [[\"O\", \"B-ART\", \"E-ART\"], [\"S-MONKEY\", \"O\", \"S-COLOR\"], [\"B-PER\", \"I-PER\", \"I-PER\", \"E-PER\", \"O\", \"O\", \"O\", \"S-WEAPON\", \"S-WEAPON\"]]\n# note the problem with not using BIO tags - the consecutive tags for thopter/sword get treated as one item\nBASIC_TAGS = [[\"O\", \"ART\", \"ART\"], [\"MONKEY\", \"O\", \"COLOR\"], [ \"PER\", \"PER\", \"PER\", \"PER\", \"O\", \"O\", \"O\", \"WEAPON\", \"WEAPON\"]]\nBASIC_BIOES = [[\"O\", \"B-ART\", \"E-ART\"], [\"S-MONKEY\", \"O\", \"S-COLOR\"], [\"B-PER\", \"I-PER\", \"I-PER\", \"E-PER\", \"O\", \"O\", \"O\", \"B-WEAPON\", \"E-WEAPON\"]]\nALT_BIO = [[\"O\", \"B-MANA\", \"I-MANA\"], [\"B-CRE\", \"O\", \"O\"], [\"B-CRE\", \"I-CRE\", \"I-CRE\", \"I-CRE\", \"O\", \"O\", \"O\", \"B-ART\", \"B-ART\"]]\nALT_BIOES = [[\"O\", \"B-MANA\", \"E-MANA\"], [\"S-CRE\", \"O\", \"O\"], [\"B-CRE\", \"I-CRE\", \"I-CRE\", \"E-CRE\", \"O\", \"O\", \"O\", \"S-ART\", \"S-ART\"]]\nNONE_BIO = [[\"O\", \"B-MANA\", \"I-MANA\"], [None, None, None], [\"B-CRE\", \"I-CRE\", \"I-CRE\", \"I-CRE\", \"O\", \"O\", \"O\", \"B-ART\", \"B-ART\"]]\nNONE_BIOES = [[\"O\", \"B-MANA\", \"E-MANA\"], [None, None, None], [\"B-CRE\", \"I-CRE\", \"I-CRE\", \"E-CRE\", \"O\", \"O\", \"O\", \"S-ART\", \"S-ART\"]]\nEMPTY_BIO = [[\"O\", \"B-MANA\", \"I-MANA\"], [EMPTY, EMPTY, EMPTY], [\"B-CRE\", \"I-CRE\", \"I-CRE\", \"I-CRE\", \"O\", \"O\", \"O\", \"B-ART\", \"B-ART\"]]\n\ndef test_normalize_empty_tags():\n sentences = [[(word[0], (word[1],)) for word in zip(*sentence)] for sentence in zip(WORDS, NONE_BIO)]\n new_sentences = utils.normalize_empty_tags(sentences)\n expected = [[(word[0], (word[1],)) for word in zip(*sentence)] for sentence in zip(WORDS, EMPTY_BIO)]\n assert new_sentences == expected\n\ndef check_reprocessed_tags(words, input_tags, expected_tags):\n sentences = [list(zip(x, y)) for x, y in zip(words, input_tags)]\n retagged = utils.process_tags(sentences=sentences, scheme=\"bioes\")\n # process_tags selectively returns tuples or strings based on the input\n # so we don't need to fiddle with the expected output format here\n expected_retagged = [list(zip(x, y)) for x, y in zip(words, expected_tags)]\n assert retagged == expected_retagged\n\ndef test_process_tags_bio():\n check_reprocessed_tags(WORDS, BIO_TAGS, BIOES_TAGS)\n # check that the alternate version is correct as well\n # that way we can independently check the two layer version\n check_reprocessed_tags(WORDS, ALT_BIO, ALT_BIOES)\n\ndef test_process_tags_with_none():\n # if there is a block of tags with None in them, the Nones should be skipped over\n check_reprocessed_tags(WORDS, NONE_BIO, NONE_BIOES)\n\ndef merge_tags(*tags):\n merged_tags = [[tuple(x) for x in zip(*sentences)] # combine tags such as (\"O\", \"O\"), (\"B-ART\", \"B-MANA\"), ...\n for sentences in zip(*tags)] # ... for each set of sentences\n return merged_tags\n\ndef test_combined_tags_bio():\n bio_tags = merge_tags(BIO_TAGS, ALT_BIO)\n expected = merge_tags(BIOES_TAGS, ALT_BIOES)\n check_reprocessed_tags(WORDS, bio_tags, expected)\n\ndef test_combined_tags_mixed():\n bio_tags = merge_tags(BIO_TAGS, ALT_BIOES)\n expected = merge_tags(BIOES_TAGS, ALT_BIOES)\n check_reprocessed_tags(WORDS, bio_tags, expected)\n\ndef test_process_tags_basic():\n check_reprocessed_tags(WORDS, BASIC_TAGS, BASIC_BIOES)\n\ndef test_process_tags_bioes():\n \"\"\"\n This one should not change, naturally\n \"\"\"\n check_reprocessed_tags(WORDS, BIOES_TAGS, BIOES_TAGS)\n check_reprocessed_tags(WORDS, BASIC_BIOES, BASIC_BIOES)\n\ndef run_flattened(fn, tags):\n return fn([x for x in y for y in tags])\n\ndef test_check_bio():\n assert utils.is_bio_scheme([x for y in BIO_TAGS for x in y])\n assert not utils.is_bio_scheme([x for y in BIOES_TAGS for x in y])\n assert not utils.is_bio_scheme([x for y in BASIC_TAGS for x in y])\n assert not utils.is_bio_scheme([x for y in BASIC_BIOES for x in y])\n\ndef test_check_basic():\n assert not utils.is_basic_scheme([x for y in BIO_TAGS for x in y])\n assert not utils.is_basic_scheme([x for y in BIOES_TAGS for x in y])\n assert utils.is_basic_scheme([x for y in BASIC_TAGS for x in y])\n assert not utils.is_basic_scheme([x for y in BASIC_BIOES for x in y])\n\ndef test_underscores():\n \"\"\"\n Check that the methods work if the inputs are underscores instead of dashes\n \"\"\"\n assert not utils.is_basic_scheme([x for y in BIO_U_TAGS for x in y])\n check_reprocessed_tags(WORDS, BIO_U_TAGS, BIOES_TAGS)\n\ndef test_merge_tags():\n \"\"\"\n Check a few versions of the tag sequence merging\n \"\"\"\n seq1 = [ \"O\", \"O\", \"O\", \"B-FOO\", \"E-FOO\", \"O\"]\n seq2 = [ \"S-FOO\", \"O\", \"B-FOO\", \"E-FOO\", \"O\", \"O\"]\n seq3 = [ \"B-FOO\", \"E-FOO\", \"B-FOO\", \"E-FOO\", \"O\", \"O\"]\n seq_err = [ \"O\", \"B-FOO\", \"O\", \"B-FOO\", \"E-FOO\", \"O\"]\n seq_err2 = [ \"O\", \"B-FOO\", \"O\", \"B-FOO\", \"B-FOO\", \"O\"]\n seq_err3 = [ \"O\", \"B-FOO\", \"O\", \"B-FOO\", \"I-FOO\", \"O\"]\n seq_err4 = [ \"O\", \"B-FOO\", \"O\", \"B-FOO\", \"I-FOO\", \"I-FOO\"]\n\n result = utils.merge_tags(seq1, seq2)\n expected = [ \"S-FOO\", \"O\", \"O\", \"B-FOO\", \"E-FOO\", \"O\"]\n assert result == expected\n\n result = utils.merge_tags(seq2, seq1)\n expected = [ \"S-FOO\", \"O\", \"B-FOO\", \"E-FOO\", \"O\", \"O\"]\n assert result == expected\n\n result = utils.merge_tags(seq1, seq3)\n expected = [ \"B-FOO\", \"E-FOO\", \"O\", \"B-FOO\", \"E-FOO\", \"O\"]\n assert result == expected\n\n with pytest.raises(ValueError):\n result = utils.merge_tags(seq1, seq_err)\n\n with pytest.raises(ValueError):\n result = utils.merge_tags(seq1, seq_err2)\n\n with pytest.raises(ValueError):\n result = utils.merge_tags(seq1, seq_err3)\n\n with pytest.raises(ValueError):\n result = utils.merge_tags(seq1, seq_err4)\n\n"
},
{
"path": "stanza/tests/ner/test_pay_amt_annotators.py",
"content": "\"\"\"\nSimple test for tracking AMT annotator work\n\"\"\"\n\nimport os\nimport zipfile\n\nimport pytest\n\nfrom stanza.tests import TEST_WORKING_DIR\nfrom stanza.utils.ner import paying_annotators\n\nDATA_SOURCE = os.path.join(TEST_WORKING_DIR, \"in\", \"aws_annotations.zip\")\n\n@pytest.fixture(scope=\"module\")\ndef completed_amt_job_metadata(tmp_path_factory):\n assert os.path.exists(DATA_SOURCE)\n unzip_path = tmp_path_factory.mktemp(\"amt_test\")\n input_path = unzip_path / \"ner\" / \"aws_labeling_copy\"\n with zipfile.ZipFile(DATA_SOURCE, 'r') as zin:\n zin.extractall(unzip_path)\n return input_path\n\ndef test_amt_annotator_track(completed_amt_job_metadata):\n workers = {\n \"7efc17ac-3397-4472-afe5-89184ad145d0\": \"Worker1\",\n \"afce8c28-969c-4e73-a20f-622ef122f585\": \"Worker2\",\n \"91f6236e-63c6-4a84-8fd6-1efbab6dedab\": \"Worker3\",\n \"6f202e93-e6b6-4e1d-8f07-0484b9a9093a\": \"Worker4\",\n \"2b674d33-f656-44b0-8f90-d70a1ab71ec2\": \"Worker5\"\n } # map AMT annotator subs to relevant identifier\n\n tracked_work = paying_annotators.track_tasks(completed_amt_job_metadata, workers)\n assert tracked_work == {'Worker4': 20, 'Worker5': 20, 'Worker2': 3, 'Worker3': 16}\n\n\ndef test_amt_annotator_track_no_map(completed_amt_job_metadata):\n sub_to_count = paying_annotators.track_tasks(completed_amt_job_metadata)\n assert sub_to_count == {'6f202e93-e6b6-4e1d-8f07-0484b9a9093a': 20, '2b674d33-f656-44b0-8f90-d70a1ab71ec2': 20,\n 'afce8c28-969c-4e73-a20f-622ef122f585': 3, '91f6236e-63c6-4a84-8fd6-1efbab6dedab': 16}\n\n\ndef main():\n test_amt_annotator_track()\n test_amt_annotator_track_no_map()\n\n\nif __name__ == \"__main__\":\n main()\n print(\"TESTS COMPLETED!\")\n"
},
{
"path": "stanza/tests/ner/test_split_wikiner.py",
"content": "\"\"\"\nRuns a few tests on the split_wikiner file\n\"\"\"\n\nimport os\nimport tempfile\n\nimport pytest\n\nfrom stanza.utils.datasets.ner import split_wikiner\n\nfrom stanza.tests import *\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\n# two sentences from the Italian dataset, split into many pieces\n# to test the splitting functionality\nFBK_SAMPLE = \"\"\"\nIl\tO\nPapa\tO\nsi\tO\naggrava\tO\n\nLe\tO\ncondizioni\tO\ndi\tO\n\nPapa\tO\nGiovanni\tPER\nPaolo\tPER\nII\tPER\nsi\tO\n\nsono\tO\naggravate\tO\nin\tO\nil\tO\ncorso\tO\n\ndi\tO\nla\tO\ngiornata\tO\ndi\tO\ngiovedì\tO\n.\tO\n\nIl\tO\nportavoce\tO\nNavarro\tPER\nValls\tPER\n\nha\tO\ndichiarato\tO\nche\tO\n\nil\tO\nSanto\tO\nPadre\tO\n\nin\tO\nla\tO\ngiornata\tO\n\ndi\tO\noggi\tO\nè\tO\nstato\tO\n\ncolpito\tO\nda\tO\nuna\tO\naffezione\tO\n\naltamente\tO\nfebbrile\tO\nprovocata\tO\nda\tO\nuna\tO\n\ninfezione\tO\ndocumentata\tO\n\ndi\tO\nle\tO\nvie\tO\nurinarie\tO\n.\tO\n\nA\tO\nil\tO\nmomento\tO\n\nnon\tO\nè\tO\nprevisto\tO\nil\tO\nricovero\tO\n\na\tO\nil\tO\nPoliclinico\tLOC\nGemelli\tLOC\n,\tO\n\ncome\tO\nha\tO\nprecisato\tO\nil\tO\n\nresponsabile\tO\ndi\tO\nil\tO\ndipartimento\tO\n\ndi\tO\nemergenza\tO\nprofessor\tO\nRodolfo\tPER\nProietti\tPER\n.\tO\n\"\"\"\n\n\ndef test_read_sentences():\n with tempfile.TemporaryDirectory() as tempdir:\n raw_filename = os.path.join(tempdir, \"raw.tsv\")\n with open(raw_filename, \"w\") as fout:\n fout.write(FBK_SAMPLE)\n\n sentences = split_wikiner.read_sentences(raw_filename, \"utf-8\")\n assert len(sentences) == 20\n text = [[\"\\t\".join(word) for word in sent] for sent in sentences]\n text = [\"\\n\".join(sent) for sent in text]\n text = \"\\n\\n\".join(text)\n assert FBK_SAMPLE.strip() == text\n\ndef test_write_sentences():\n with tempfile.TemporaryDirectory() as tempdir:\n raw_filename = os.path.join(tempdir, \"raw.tsv\")\n with open(raw_filename, \"w\") as fout:\n fout.write(FBK_SAMPLE)\n\n sentences = split_wikiner.read_sentences(raw_filename, \"utf-8\")\n copy_filename = os.path.join(tempdir, \"copy.tsv\")\n split_wikiner.write_sentences_to_file(sentences, copy_filename)\n\n sent2 = split_wikiner.read_sentences(raw_filename, \"utf-8\")\n assert sent2 == sentences\n\ndef run_split_wikiner(expected_train=14, expected_dev=3, expected_test=3, **kwargs):\n \"\"\"\n Runs a test using various parameters to check the results of the splitting process\n \"\"\"\n with tempfile.TemporaryDirectory() as indir:\n raw_filename = os.path.join(indir, \"raw.tsv\")\n with open(raw_filename, \"w\") as fout:\n fout.write(FBK_SAMPLE)\n\n with tempfile.TemporaryDirectory() as outdir:\n split_wikiner.split_wikiner(outdir, raw_filename, **kwargs)\n\n train_file = os.path.join(outdir, \"it_fbk.train.bio\")\n dev_file = os.path.join(outdir, \"it_fbk.dev.bio\")\n test_file = os.path.join(outdir, \"it_fbk.test.bio\")\n\n assert os.path.exists(train_file)\n assert os.path.exists(dev_file)\n if kwargs[\"test_section\"]:\n assert os.path.exists(test_file)\n else:\n assert not os.path.exists(test_file)\n\n train_sent = split_wikiner.read_sentences(train_file, \"utf-8\")\n dev_sent = split_wikiner.read_sentences(dev_file, \"utf-8\")\n assert len(train_sent) == expected_train\n assert len(dev_sent) == expected_dev\n if kwargs[\"test_section\"]:\n test_sent = split_wikiner.read_sentences(test_file, \"utf-8\")\n assert len(test_sent) == expected_test\n else:\n test_sent = []\n\n if kwargs[\"shuffle\"]:\n orig_sents = sorted(split_wikiner.read_sentences(raw_filename, \"utf-8\"))\n split_sents = sorted(train_sent + dev_sent + test_sent)\n else:\n orig_sents = split_wikiner.read_sentences(raw_filename, \"utf-8\")\n split_sents = train_sent + dev_sent + test_sent\n assert orig_sents == split_sents\n\ndef test_no_shuffle_split():\n run_split_wikiner(prefix=\"it_fbk\", shuffle=False, test_section=True)\n\ndef test_shuffle_split():\n run_split_wikiner(prefix=\"it_fbk\", shuffle=True, test_section=True)\n\ndef test_resize():\n run_split_wikiner(expected_train=12, expected_dev=2, expected_test=6, train_fraction=0.6, dev_fraction=0.1, prefix=\"it_fbk\", shuffle=True, test_section=True)\n\ndef test_no_test_split():\n run_split_wikiner(expected_train=17, train_fraction=0.85, prefix=\"it_fbk\", shuffle=False, test_section=False)\n\n"
},
{
"path": "stanza/tests/ner/test_suc3.py",
"content": "\"\"\"\nTests the conversion code for the SUC3 NER dataset\n\"\"\"\n\nimport os\nimport tempfile\nfrom zipfile import ZipFile\n\nimport pytest\npytestmark = [pytest.mark.travis, pytest.mark.pipeline]\n\nimport stanza.utils.datasets.ner.suc_conll_to_iob as suc_conll_to_iob\n\nTEST_CONLL = \"\"\"\n1\tDen\tden\tPN\tPN\tUTR|SIN|DEF|SUB/OBJ\t_\t_\t_\t_\tO\t_\tac01b-030:2328\n2\tGud\tGud\tPM\tPM\tNOM\t_\t_\t_\t_\tB\tmyth\tac01b-030:2329\n3\tgiver\tgiva\tVB\tVB\tPRS|AKT\t_\t_\t_\t_\tO\t_\tac01b-030:2330\n4\tämbetet\tämbete\tNN\tNN\tNEU|SIN|DEF|NOM\t_\t_\t_\t_\tO\t_\tac01b-030:2331\n5\tfår\tfå\tVB\tVB\tPRS|AKT\t_\t_\t_\t_\tO\t_\tac01b-030:2332\n6\tockså\tockså\tAB\tAB\t\t_\t_\t_\t_\tO\t_\tac01b-030:2333\n7\tförståndet\tförstånd\tNN\tNN\tNEU|SIN|DEF|NOM\t_\t_\t_\t_\tO\t_\tac01b-030:2334\n8\t.\t.\tMAD\tMAD\t\t_\t_\t_\t_\tO\t_\tac01b-030:2335\n\n1\tHan\than\tPN\tPN\tUTR|SIN|DEF|SUB\t_\t_\t_\t_\tO\t_\taa01a-017:227\n2\tberättar\tberätta\tVB\tVB\tPRS|AKT\t_\t_\t_\t_\tO\t_\taa01a-017:228\n3\tanekdoten\tanekdot\tNN\tNN\tUTR|SIN|DEF|NOM\t_\t_\t_\t_\tO\t_\taa01a-017:229\n4\tsom\tsom\tHP\tHP\t-|-|-\t_\t_\t_\t_\tO\t_\taa01a-017:230\n5\tFN-medlaren\tFN-medlare\tNN\tNN\tUTR|SIN|DEF|NOM\t_\t_\t_\t_\tO\t_\taa01a-017:231\n6\tBrian\tBrian\tPM\tPM\tNOM\t_\t_\t_\t_\tB\tperson\taa01a-017:232\n7\tUrquhart\tUrquhart\tPM\tPM\tNOM\t_\t_\t_\t_\tI\tperson\taa01a-017:233\n8\tmyntat\tmynta\tVB\tVB\tSUP|AKT\t_\t_\t_\t_\tO\t_\taa01a-017:234\n9\t:\t:\tMAD\tMAD\t\t_\t_\t_\t_\tO\t_\taa01a-017:235\n\"\"\"\n\nEXPECTED_IOB = \"\"\"\nDen\tO\nGud\tB-myth\ngiver\tO\nämbetet\tO\nfår\tO\nockså\tO\nförståndet\tO\n.\tO\n\nHan\tO\nberättar\tO\nanekdoten\tO\nsom\tO\nFN-medlaren\tO\nBrian\tB-person\nUrquhart\tI-person\nmyntat\tO\n:\tO\n\"\"\"\n\ndef test_read_zip():\n \"\"\"\n Test creating a fake zip file, then converting it to an .iob file\n \"\"\"\n with tempfile.TemporaryDirectory() as tempdir:\n zip_name = os.path.join(tempdir, \"test.zip\")\n in_filename = \"conll\"\n with ZipFile(zip_name, \"w\") as zout:\n with zout.open(in_filename, \"w\") as fout:\n fout.write(TEST_CONLL.encode())\n\n out_filename = os.path.join(tempdir, \"iob\")\n num = suc_conll_to_iob.extract_from_zip(zip_name, in_filename, out_filename)\n assert num == 2\n\n with open(out_filename) as fin:\n result = fin.read()\n assert EXPECTED_IOB.strip() == result.strip()\n\ndef test_read_raw():\n \"\"\"\n Test a direct text file conversion w/o the zip file\n \"\"\"\n with tempfile.TemporaryDirectory() as tempdir:\n in_filename = os.path.join(tempdir, \"test.txt\")\n with open(in_filename, \"w\", encoding=\"utf-8\") as fout:\n fout.write(TEST_CONLL)\n\n out_filename = os.path.join(tempdir, \"iob\")\n with open(in_filename, encoding=\"utf-8\") as fin, open(out_filename, \"w\", encoding=\"utf-8\") as fout:\n num = suc_conll_to_iob.extract(fin, fout)\n assert num == 2\n\n with open(out_filename) as fin:\n result = fin.read()\n assert EXPECTED_IOB.strip() == result.strip()\n"
},
{
"path": "stanza/tests/pipeline/__init__.py",
"content": ""
},
{
"path": "stanza/tests/pipeline/pipeline_device_tests.py",
"content": "\"\"\"\nUtility methods to check that all processors are on the expected device\n\nRefactored since it can be used for multiple pipelines\n\"\"\"\n\nimport warnings\n\nimport torch\n\ndef check_on_gpu(pipeline):\n \"\"\"\n Check that the processors are all on the GPU and that basic execution works\n \"\"\"\n if not torch.cuda.is_available():\n warnings.warn(\"Unable to run the test that checks the pipeline is on the GPU, as there is no GPU available!\")\n return\n\n for name, proc in pipeline.processors.items():\n if proc.trainer is not None:\n device = next(proc.trainer.model.parameters()).device\n else:\n device = next(proc._model.parameters()).device\n\n assert str(device).startswith(\"cuda\"), \"Processor %s was not on the GPU\" % name\n\n # just check that there are no cpu/cuda tensor conflicts\n # when running on the GPU\n pipeline(\"This is a small test\")\n\ndef check_on_cpu(pipeline):\n \"\"\"\n Check that the processors are all on the CPU and that basic execution works\n \"\"\"\n for name, proc in pipeline.processors.items():\n if proc.trainer is not None:\n device = next(proc.trainer.model.parameters()).device\n else:\n device = next(proc._model.parameters()).device\n\n assert str(device).startswith(\"cpu\"), \"Processor %s was not on the CPU\" % name\n\n # just check that there are no cpu/cuda tensor conflicts\n # when running on the CPU\n pipeline(\"This is a small test\")\n"
},
{
"path": "stanza/tests/pipeline/test_arabic_pipeline.py",
"content": "\"\"\"\nSmall test of loading the Arabic pipeline\n\nThe main goal is to check that nothing goes wrong with RtL languages,\nbut incidentally this would have caught a bug where the xpos tags\nwere split into individual pieces instead of reassembled as expected\n\"\"\"\n\nimport pytest\nimport stanza\n\nfrom stanza.tests import TEST_MODELS_DIR\n\npytestmark = pytest.mark.pipeline\n\ndef test_arabic_pos_pipeline():\n pipe = stanza.Pipeline(**{'processors': 'tokenize,pos', 'dir': TEST_MODELS_DIR, 'download_method': None, 'lang': 'ar'})\n text = \"ولم يتم اعتقال احد بحسب المتحدث باسم الشرطة.\"\n\n doc = pipe(text)\n # the first token translates to \"and not\", seems common enough\n # that we should be able to rely on it having a stable MWT and tag\n\n assert len(doc.sentences) == 1\n assert doc.sentences[0].tokens[0].text == \"ولم\"\n assert doc.sentences[0].words[0].xpos == \"C---------\"\n assert doc.sentences[0].words[1].xpos == \"F---------\"\n"
},
{
"path": "stanza/tests/pipeline/test_core.py",
"content": "import pytest\nimport shutil\nimport tempfile\n\nimport stanza\n\nfrom stanza.tests import *\n\nfrom stanza.pipeline import core\nfrom stanza.resources.common import get_md5, load_resources_json\n\npytestmark = pytest.mark.pipeline\n\ndef test_pretagged():\n \"\"\"\n Test that the pipeline does or doesn't build if pos is left out and pretagged is specified\n \"\"\"\n nlp = stanza.Pipeline(lang='en', dir=TEST_MODELS_DIR, processors=\"tokenize,pos,lemma,depparse\")\n with pytest.raises(core.PipelineRequirementsException):\n nlp = stanza.Pipeline(lang='en', dir=TEST_MODELS_DIR, processors=\"tokenize,lemma,depparse\")\n nlp = stanza.Pipeline(lang='en', dir=TEST_MODELS_DIR, processors=\"tokenize,lemma,depparse\", depparse_pretagged=True)\n nlp = stanza.Pipeline(lang='en', dir=TEST_MODELS_DIR, processors=\"tokenize,lemma,depparse\", pretagged=True)\n # test that the module specific flag overrides the general flag\n nlp = stanza.Pipeline(lang='en', dir=TEST_MODELS_DIR, processors=\"tokenize,lemma,depparse\", depparse_pretagged=True, pretagged=False)\n\ndef test_download_missing_ner_model():\n \"\"\"\n Test that the pipeline will automatically download missing models\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as test_dir:\n stanza.download(\"en\", model_dir=test_dir, processors=\"tokenize\", package=\"combined\", verbose=False)\n pipe = stanza.Pipeline(\"en\", model_dir=test_dir, processors=\"tokenize,ner\", package={\"ner\": (\"ontonotes_charlm\")})\n\n assert sorted(os.listdir(test_dir)) == ['en', 'resources.json']\n en_dir = os.path.join(test_dir, 'en')\n en_dir_listing = sorted(os.listdir(en_dir))\n assert en_dir_listing == ['backward_charlm', 'forward_charlm', 'mwt', 'ner', 'pretrain', 'tokenize']\n assert os.listdir(os.path.join(en_dir, 'ner')) == ['ontonotes_charlm.pt']\n\n\ndef test_download_missing_resources():\n \"\"\"\n Test that the pipeline will automatically download missing models\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as test_dir:\n pipe = stanza.Pipeline(\"en\", model_dir=test_dir, processors=\"tokenize,ner\", package={\"tokenize\": \"combined\", \"ner\": \"ontonotes_charlm\"})\n\n assert sorted(os.listdir(test_dir)) == ['en', 'resources.json']\n en_dir = os.path.join(test_dir, 'en')\n en_dir_listing = sorted(os.listdir(en_dir))\n assert en_dir_listing == ['backward_charlm', 'forward_charlm', 'mwt', 'ner', 'pretrain', 'tokenize']\n assert os.listdir(os.path.join(en_dir, 'ner')) == ['ontonotes_charlm.pt']\n\n\ndef test_download_resources_overwrites():\n \"\"\"\n Test that the DOWNLOAD_RESOURCES method overwrites an existing resources.json\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as test_dir:\n pipe = stanza.Pipeline(\"en\", model_dir=test_dir, processors=\"tokenize\", package={\"tokenize\": \"combined\"})\n\n assert sorted(os.listdir(test_dir)) == ['en', 'resources.json']\n resources_path = os.path.join(test_dir, 'resources.json')\n mod_time = os.path.getmtime(resources_path)\n\n pipe = stanza.Pipeline(\"en\", model_dir=test_dir, processors=\"tokenize\", package={\"tokenize\": \"combined\"})\n new_mod_time = os.path.getmtime(resources_path)\n assert mod_time != new_mod_time\n\ndef test_reuse_resources_overwrites():\n \"\"\"\n Test that the REUSE_RESOURCES method does *not* overwrite an existing resources.json\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as test_dir:\n pipe = stanza.Pipeline(\"en\",\n download_method=core.DownloadMethod.REUSE_RESOURCES,\n model_dir=test_dir,\n processors=\"tokenize\",\n package={\"tokenize\": \"combined\"})\n\n assert sorted(os.listdir(test_dir)) == ['en', 'resources.json']\n resources_path = os.path.join(test_dir, 'resources.json')\n mod_time = os.path.getmtime(resources_path)\n\n pipe = stanza.Pipeline(\"en\",\n download_method=core.DownloadMethod.REUSE_RESOURCES,\n model_dir=test_dir,\n processors=\"tokenize\",\n package={\"tokenize\": \"combined\"})\n new_mod_time = os.path.getmtime(resources_path)\n assert mod_time == new_mod_time\n\n\ndef test_download_not_repeated():\n \"\"\"\n Test that a model is only downloaded once if it already matches the expected model from the resources file\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as test_dir:\n stanza.download(\"en\", model_dir=test_dir, processors=\"tokenize\", package=\"combined\")\n\n assert sorted(os.listdir(test_dir)) == ['en', 'resources.json']\n en_dir = os.path.join(test_dir, 'en')\n en_dir_listing = sorted(os.listdir(en_dir))\n assert en_dir_listing == ['mwt', 'tokenize']\n tokenize_path = os.path.join(en_dir, \"tokenize\", \"combined.pt\")\n mod_time = os.path.getmtime(tokenize_path)\n\n pipe = stanza.Pipeline(\"en\", model_dir=test_dir, processors=\"tokenize\", package={\"tokenize\": \"combined\"})\n assert os.path.getmtime(tokenize_path) == mod_time\n\ndef test_download_none():\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as test_dir:\n stanza.download(\"it\", model_dir=test_dir, processors=\"tokenize\", package=\"combined\")\n stanza.download(\"it\", model_dir=test_dir, processors=\"tokenize\", package=\"vit\")\n\n it_dir = os.path.join(test_dir, 'it')\n it_dir_listing = sorted(os.listdir(it_dir))\n assert sorted(it_dir_listing) == ['mwt', 'tokenize']\n combined_path = os.path.join(it_dir, \"tokenize\", \"combined.pt\")\n vit_path = os.path.join(it_dir, \"tokenize\", \"vit.pt\")\n\n assert os.path.exists(combined_path)\n assert os.path.exists(vit_path)\n\n combined_md5 = get_md5(combined_path)\n vit_md5 = get_md5(vit_path)\n # check that the models are different\n # otherwise the test is not testing anything\n assert combined_md5 != vit_md5\n\n shutil.copyfile(vit_path, combined_path)\n assert get_md5(combined_path) == vit_md5\n\n pipe = stanza.Pipeline(\"it\", model_dir=test_dir, processors=\"tokenize\", package={\"tokenize\": \"combined\"}, download_method=None)\n assert get_md5(combined_path) == vit_md5\n\n pipe = stanza.Pipeline(\"it\", model_dir=test_dir, processors=\"tokenize\", package={\"tokenize\": \"combined\"})\n assert get_md5(combined_path) != vit_md5\n\n\ndef check_download_method_updates(download_method):\n \"\"\"\n Run a single test of creating a pipeline with a given download_method, checking that the model is updated\n \"\"\"\n with tempfile.TemporaryDirectory(dir=TEST_WORKING_DIR) as test_dir:\n stanza.download(\"en\", model_dir=test_dir, processors=\"tokenize\", package=\"combined\")\n\n assert sorted(os.listdir(test_dir)) == ['en', 'resources.json']\n en_dir = os.path.join(test_dir, 'en')\n en_dir_listing = sorted(os.listdir(en_dir))\n assert en_dir_listing == ['mwt', 'tokenize']\n tokenize_path = os.path.join(en_dir, \"tokenize\", \"combined.pt\")\n\n with open(tokenize_path, \"w\") as fout:\n fout.write(\"Unban mox opal!\")\n mod_time = os.path.getmtime(tokenize_path)\n\n pipe = stanza.Pipeline(\"en\", model_dir=test_dir, processors=\"tokenize\", package={\"tokenize\": \"combined\"}, download_method=download_method)\n assert os.path.getmtime(tokenize_path) != mod_time\n\ndef test_download_fixed():\n \"\"\"\n Test that a model is fixed if the existing model doesn't match the md5sum\n \"\"\"\n for download_method in (core.DownloadMethod.REUSE_RESOURCES, core.DownloadMethod.DOWNLOAD_RESOURCES):\n check_download_method_updates(download_method)\n\ndef test_download_strings():\n \"\"\"\n Same as the test of the download_method, but tests that the pipeline works for string download_method\n \"\"\"\n for download_method in (\"reuse_resources\", \"download_resources\"):\n check_download_method_updates(download_method)\n\ndef test_limited_pipeline():\n \"\"\"\n Test loading a pipeline, but then only using a couple processors\n \"\"\"\n pipe = stanza.Pipeline(processors=\"tokenize,pos,lemma,depparse,ner\", dir=TEST_MODELS_DIR)\n doc = pipe(\"John Bauer works at Stanford\")\n assert all(word.upos is not None for sentence in doc.sentences for word in sentence.words)\n assert all(token.ner is not None for sentence in doc.sentences for token in sentence.tokens)\n\n doc = pipe(\"John Bauer works at Stanford\", processors=[\"tokenize\",\"pos\"])\n assert all(word.upos is not None for sentence in doc.sentences for word in sentence.words)\n assert not any(token.ner is not None for sentence in doc.sentences for token in sentence.tokens)\n\n doc = pipe(\"John Bauer works at Stanford\", processors=\"tokenize\")\n assert not any(word.upos is not None for sentence in doc.sentences for word in sentence.words)\n assert not any(token.ner is not None for sentence in doc.sentences for token in sentence.tokens)\n\n doc = pipe(\"John Bauer works at Stanford\", processors=\"tokenize,ner\")\n assert not any(word.upos is not None for sentence in doc.sentences for word in sentence.words)\n assert all(token.ner is not None for sentence in doc.sentences for token in sentence.tokens)\n\n with pytest.raises(ValueError):\n # this should fail\n doc = pipe(\"John Bauer works at Stanford\", processors=\"tokenize,depparse\")\n\n@pytest.fixture(scope=\"module\")\ndef unknown_language_name():\n resources = load_resources_json(model_dir=TEST_MODELS_DIR)\n name = \"en\"\n while name in resources:\n name = name + \"z\"\n assert name != \"en\"\n return name\n\ndef test_empty_unknown_language(unknown_language_name):\n \"\"\"\n Check that there is an error for trying to load an unknown language\n \"\"\"\n with pytest.raises(ValueError):\n pipe = stanza.Pipeline(unknown_language_name, model_dir=TEST_MODELS_DIR, download_method=None)\n\ndef test_unknown_language_tokenizer(unknown_language_name):\n \"\"\"\n Test that loading tokenize works for an unknown language\n \"\"\"\n base_pipe = stanza.Pipeline(\"en\", dir=TEST_MODELS_DIR, processors=\"tokenize\", download_method=None)\n # even if we one day add MWT to English, the tokenizer by itself should still work\n tokenize_processor = base_pipe.processors[\"tokenize\"]\n\n pipe=stanza.Pipeline(unknown_language_name,\n processors=\"tokenize\",\n allow_unknown_language=True,\n tokenize_model_path=tokenize_processor.config['model_path'],\n download_method=None)\n doc = pipe(\"This is a test\")\n words = [x.text for x in doc.sentences[0].words]\n assert words == ['This', 'is', 'a', 'test']\n\n\ndef test_unknown_language_mwt(unknown_language_name):\n \"\"\"\n Test that loading tokenize & mwt works for an unknown language\n \"\"\"\n base_pipe = stanza.Pipeline(\"fr\", dir=TEST_MODELS_DIR, processors=\"tokenize,mwt\", download_method=None)\n assert len(base_pipe.processors) == 2\n tokenize_processor = base_pipe.processors[\"tokenize\"]\n mwt_processor = base_pipe.processors[\"mwt\"]\n\n pipe=stanza.Pipeline(unknown_language_name,\n model_dir=TEST_MODELS_DIR,\n processors=\"tokenize,mwt\",\n allow_unknown_language=True,\n tokenize_model_path=tokenize_processor.config['model_path'],\n mwt_model_path=mwt_processor.config['model_path'],\n download_method=None)\n"
},
{
"path": "stanza/tests/pipeline/test_decorators.py",
"content": "\"\"\"\nBasic tests of the depparse processor boolean flags\n\"\"\"\nimport pytest\n\nimport stanza\nfrom stanza.models.common.doc import Document\nfrom stanza.pipeline.core import PipelineRequirementsException\nfrom stanza.pipeline.processor import Processor, ProcessorVariant, register_processor, register_processor_variant, ProcessorRegisterException\nfrom stanza.utils.conll import CoNLL\nfrom stanza.tests import *\n\npytestmark = pytest.mark.pipeline\n\n# data for testing\nEN_DOC = \"This is a test sentence. This is another!\"\n\nEN_DOC_LOWERCASE_TOKENS = ''']>\n]>\n]>\n]>\n]>\n]>\n\n]>\n]>\n]>\n]>'''\n\nEN_DOC_LOL_TOKENS = ''']>\n]>\n]>\n]>\n]>\n]>\n]>\n]>'''\n\nEN_DOC_COOL_LEMMAS = ''']>\n]>\n]>\n]>\n]>\n]>\n\n]>\n]>\n]>\n]>'''\n\n@register_processor(\"lowercase\")\nclass LowercaseProcessor(Processor):\n ''' Processor that lowercases all text '''\n _requires = set(['tokenize'])\n _provides = set(['lowercase'])\n\n def __init__(self, config, pipeline, device):\n pass\n\n def _set_up_model(self, *args):\n pass\n\n def process(self, doc):\n doc.text = doc.text.lower()\n for sent in doc.sentences:\n for tok in sent.tokens:\n tok.text = tok.text.lower()\n\n for word in sent.words:\n word.text = word.text.lower()\n\n return doc\n\ndef test_register_processor():\n nlp = stanza.Pipeline(dir=TEST_MODELS_DIR, lang='en', processors='tokenize,lowercase', download_method=None)\n doc = nlp(EN_DOC)\n assert EN_DOC_LOWERCASE_TOKENS == '\\n\\n'.join(sent.tokens_string() for sent in doc.sentences)\n\ndef test_register_nonprocessor():\n with pytest.raises(ProcessorRegisterException):\n @register_processor(\"nonprocessor\")\n class NonProcessor:\n pass\n\n@register_processor_variant(\"tokenize\", \"lol\")\nclass LOLTokenizer(ProcessorVariant):\n ''' An alternative tokenizer that splits text by space and replaces all tokens with LOL '''\n\n def __init__(self, lang):\n pass\n\n def process(self, text):\n sentence = [{'id': (i+1, ), 'text': 'LOL'} for i, tok in enumerate(text.split())]\n return Document([sentence], text)\n\ndef test_register_processor_variant():\n nlp = stanza.Pipeline(dir=TEST_MODELS_DIR, lang='en', processors={\"tokenize\": \"lol\"}, package=None, download_method=None)\n doc = nlp(EN_DOC)\n assert EN_DOC_LOL_TOKENS == '\\n\\n'.join(sent.tokens_string() for sent in doc.sentences)\n\n@register_processor_variant(\"lemma\", \"cool\")\nclass CoolLemmatizer(ProcessorVariant):\n ''' An alternative lemmatizer that lemmatizes every word to \"cool\". '''\n\n OVERRIDE = True\n\n def __init__(self, lang):\n pass\n\n def process(self, document):\n for sentence in document.sentences:\n for word in sentence.words:\n word.lemma = \"cool\"\n\n return document\n\ndef test_register_processor_variant_with_override():\n nlp = stanza.Pipeline(dir=TEST_MODELS_DIR, lang='en', processors={\"tokenize\": \"combined\", \"pos\": \"combined\", \"lemma\": \"cool\"}, package=None, download_method=None)\n doc = nlp(EN_DOC)\n result = '\\n\\n'.join(sent.tokens_string() for sent in doc.sentences)\n assert EN_DOC_COOL_LEMMAS == result\n\ndef test_register_nonprocessor_variant():\n with pytest.raises(ProcessorRegisterException):\n @register_processor_variant(\"tokenize\", \"nonvariant\")\n class NonVariant:\n pass\n"
},
{
"path": "stanza/tests/pipeline/test_depparse.py",
"content": "\"\"\"\nBasic tests of the depparse processor boolean flags\n\"\"\"\nimport gc\n\nimport pytest\n\nimport stanza\nfrom stanza.pipeline.core import PipelineRequirementsException\nfrom stanza.utils.conll import CoNLL\nfrom stanza.tests import *\n\npytestmark = pytest.mark.pipeline\n\n# data for testing\nEN_DOC = \"Barack Obama was born in Hawaii. He was elected president in 2008. Obama attended Harvard.\"\n\nEN_DOC_CONLLU_PRETAGGED = \"\"\"\n1\tBarack\tBarack\tPROPN\tNNP\tNumber=Sing\t0\t_\t_\t_\n2\tObama\tObama\tPROPN\tNNP\tNumber=Sing\t1\t_\t_\t_\n3\twas\tbe\tAUX\tVBD\tMood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin\t2\t_\t_\t_\n4\tborn\tbear\tVERB\tVBN\tTense=Past|VerbForm=Part|Voice=Pass\t3\t_\t_\t_\n5\tin\tin\tADP\tIN\t_\t4\t_\t_\t_\n6\tHawaii\tHawaii\tPROPN\tNNP\tNumber=Sing\t5\t_\t_\t_\n7\t.\t.\tPUNCT\t.\t_\t6\t_\t_\t_\n\n1\tHe\the\tPRON\tPRP\tCase=Nom|Gender=Masc|Number=Sing|Person=3|PronType=Prs\t0\t_\t_\t_\n2\twas\tbe\tAUX\tVBD\tMood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin\t1\t_\t_\t_\n3\telected\telect\tVERB\tVBN\tTense=Past|VerbForm=Part|Voice=Pass\t2\t_\t_\t_\n4\tpresident\tpresident\tPROPN\tNNP\tNumber=Sing\t3\t_\t_\t_\n5\tin\tin\tADP\tIN\t_\t4\t_\t_\t_\n6\t2008\t2008\tNUM\tCD\tNumType=Card\t5\t_\t_\t_\n7\t.\t.\tPUNCT\t.\t_\t6\t_\t_\t_\n\n1\tObama\tObama\tPROPN\tNNP\tNumber=Sing\t0\t_\t_\t_\n2\tattended\tattend\tVERB\tVBD\tMood=Ind|Tense=Past|VerbForm=Fin\t1\t_\t_\t_\n3\tHarvard\tHarvard\tPROPN\tNNP\tNumber=Sing\t2\t_\t_\t_\n4\t.\t.\tPUNCT\t.\t_\t3\t_\t_\t_\n\n\n\"\"\".lstrip()\n\nEN_DOC_DEPENDENCY_PARSES_GOLD = \"\"\"\n('Barack', 4, 'nsubj:pass')\n('Obama', 1, 'flat')\n('was', 4, 'aux:pass')\n('born', 0, 'root')\n('in', 6, 'case')\n('Hawaii', 4, 'obl')\n('.', 4, 'punct')\n\n('He', 3, 'nsubj:pass')\n('was', 3, 'aux:pass')\n('elected', 0, 'root')\n('president', 3, 'xcomp')\n('in', 6, 'case')\n('2008', 3, 'obl')\n('.', 3, 'punct')\n\n('Obama', 2, 'nsubj')\n('attended', 0, 'root')\n('Harvard', 2, 'obj')\n('.', 2, 'punct')\n\"\"\".strip()\n\n@pytest.fixture(scope=\"module\")\ndef en_depparse_pipeline():\n nlp = stanza.Pipeline(dir=TEST_MODELS_DIR, lang='en', processors='tokenize,pos,lemma,depparse')\n gc.collect()\n return nlp\n\ndef test_depparse(en_depparse_pipeline):\n doc = en_depparse_pipeline(EN_DOC)\n assert EN_DOC_DEPENDENCY_PARSES_GOLD == '\\n\\n'.join([sent.dependencies_string() for sent in doc.sentences])\n\n\ndef test_depparse_with_pretagged_doc():\n nlp = stanza.Pipeline(**{'processors': 'depparse', 'dir': TEST_MODELS_DIR, 'lang': 'en',\n 'depparse_pretagged': True})\n\n doc = CoNLL.conll2doc(input_str=EN_DOC_CONLLU_PRETAGGED)\n processed_doc = nlp(doc)\n\n assert EN_DOC_DEPENDENCY_PARSES_GOLD == '\\n\\n'.join(\n [sent.dependencies_string() for sent in processed_doc.sentences])\n\n\ndef test_raises_requirements_exception_if_pretagged_not_passed():\n with pytest.raises(PipelineRequirementsException):\n stanza.Pipeline(**{'processors': 'depparse', 'dir': TEST_MODELS_DIR, 'lang': 'en'})\n"
},
{
"path": "stanza/tests/pipeline/test_english_pipeline.py",
"content": "\"\"\"\nBasic testing of the English pipeline\n\"\"\"\n\nimport pytest\nimport stanza\nfrom stanza.utils.conll import CoNLL\nfrom stanza.models.common.doc import Document\n\nfrom stanza.tests import *\nfrom stanza.tests.pipeline.pipeline_device_tests import check_on_gpu, check_on_cpu\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\n# data for testing\nEN_DOC = \"Barack Obama was born in Hawaii. He was elected president in 2008. Obama attended Harvard.\"\n\nEN_DOCS = [\"Barack Obama was born in Hawaii.\", \"He was elected president in 2008.\", \"Obama attended Harvard.\"]\n\nEN_DOC_TOKENS_GOLD = \"\"\"\n]>\n]>\n]>\n]>\n]>\n]>\n]>\n\n]>\n]>\n]>\n]>\n]>\n]>\n]>\n\n]>\n]>\n]>\n]>\n\"\"\".strip()\n\nEN_DOC_WORDS_GOLD = \"\"\"\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\"\"\".strip()\n\nEN_DOC_DEPENDENCY_PARSES_GOLD = \"\"\"\n('Barack', 4, 'nsubj:pass')\n('Obama', 1, 'flat')\n('was', 4, 'aux:pass')\n('born', 0, 'root')\n('in', 6, 'case')\n('Hawaii', 4, 'obl')\n('.', 4, 'punct')\n\n('He', 3, 'nsubj:pass')\n('was', 3, 'aux:pass')\n('elected', 0, 'root')\n('president', 3, 'xcomp')\n('in', 6, 'case')\n('2008', 3, 'obl')\n('.', 3, 'punct')\n\n('Obama', 2, 'nsubj')\n('attended', 0, 'root')\n('Harvard', 2, 'obj')\n('.', 2, 'punct')\n\"\"\".strip()\n\nEN_DOC_CONLLU_GOLD = \"\"\"\n# text = Barack Obama was born in Hawaii.\n# sent_id = 0\n# constituency = (ROOT (S (NP (NNP Barack) (NNP Obama)) (VP (VBD was) (VP (VBN born) (PP (IN in) (NP (NNP Hawaii))))) (. .)))\n# sentiment = 1\n1\tBarack\tBarack\tPROPN\tNNP\tNumber=Sing\t4\tnsubj:pass\t_\tstart_char=0|end_char=6|ner=B-PERSON\n2\tObama\tObama\tPROPN\tNNP\tNumber=Sing\t1\tflat\t_\tstart_char=7|end_char=12|ner=E-PERSON\n3\twas\tbe\tAUX\tVBD\tMood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin\t4\taux:pass\t_\tstart_char=13|end_char=16|ner=O\n4\tborn\tbear\tVERB\tVBN\tTense=Past|VerbForm=Part|Voice=Pass\t0\troot\t_\tstart_char=17|end_char=21|ner=O\n5\tin\tin\tADP\tIN\t_\t6\tcase\t_\tstart_char=22|end_char=24|ner=O\n6\tHawaii\tHawaii\tPROPN\tNNP\tNumber=Sing\t4\tobl\t_\tSpaceAfter=No|start_char=25|end_char=31|ner=S-GPE\n7\t.\t.\tPUNCT\t.\t_\t4\tpunct\t_\tSpacesAfter=\\\\s\\\\s|start_char=31|end_char=32|ner=O\n\n# text = He was elected president in 2008.\n# sent_id = 1\n# constituency = (ROOT (S (NP (PRP He)) (VP (VBD was) (VP (VBN elected) (S (NP (NN president))) (PP (IN in) (NP (CD 2008))))) (. .)))\n# sentiment = 1\n1\tHe\the\tPRON\tPRP\tCase=Nom|Gender=Masc|Number=Sing|Person=3|PronType=Prs\t3\tnsubj:pass\t_\tstart_char=34|end_char=36|ner=O\n2\twas\tbe\tAUX\tVBD\tMood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin\t3\taux:pass\t_\tstart_char=37|end_char=40|ner=O\n3\telected\telect\tVERB\tVBN\tTense=Past|VerbForm=Part|Voice=Pass\t0\troot\t_\tstart_char=41|end_char=48|ner=O\n4\tpresident\tpresident\tNOUN\tNN\tNumber=Sing\t3\txcomp\t_\tstart_char=49|end_char=58|ner=O\n5\tin\tin\tADP\tIN\t_\t6\tcase\t_\tstart_char=59|end_char=61|ner=O\n6\t2008\t2008\tNUM\tCD\tNumForm=Digit|NumType=Card\t3\tobl\t_\tSpaceAfter=No|start_char=62|end_char=66|ner=S-DATE\n7\t.\t.\tPUNCT\t.\t_\t3\tpunct\t_\tSpacesAfter=\\\\s\\\\s|start_char=66|end_char=67|ner=O\n\n# text = Obama attended Harvard.\n# sent_id = 2\n# constituency = (ROOT (S (NP (NNP Obama)) (VP (VBD attended) (NP (NNP Harvard))) (. .)))\n# sentiment = 1\n1\tObama\tObama\tPROPN\tNNP\tNumber=Sing\t2\tnsubj\t_\tstart_char=69|end_char=74|ner=S-PERSON\n2\tattended\tattend\tVERB\tVBD\tMood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin\t0\troot\t_\tstart_char=75|end_char=83|ner=O\n3\tHarvard\tHarvard\tPROPN\tNNP\tNumber=Sing\t2\tobj\t_\tSpaceAfter=No|start_char=84|end_char=91|ner=S-ORG\n4\t.\t.\tPUNCT\t.\t_\t2\tpunct\t_\tSpaceAfter=No|start_char=91|end_char=92|ner=O\n\"\"\".strip()\n\nEN_DOC_CONLLU_GOLD_MULTIDOC = \"\"\"\n# text = Barack Obama was born in Hawaii.\n# sent_id = 0\n# constituency = (ROOT (S (NP (NNP Barack) (NNP Obama)) (VP (VBD was) (VP (VBN born) (PP (IN in) (NP (NNP Hawaii))))) (. .)))\n# sentiment = 1\n1\tBarack\tBarack\tPROPN\tNNP\tNumber=Sing\t4\tnsubj:pass\t_\tstart_char=0|end_char=6|ner=B-PERSON\n2\tObama\tObama\tPROPN\tNNP\tNumber=Sing\t1\tflat\t_\tstart_char=7|end_char=12|ner=E-PERSON\n3\twas\tbe\tAUX\tVBD\tMood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin\t4\taux:pass\t_\tstart_char=13|end_char=16|ner=O\n4\tborn\tbear\tVERB\tVBN\tTense=Past|VerbForm=Part|Voice=Pass\t0\troot\t_\tstart_char=17|end_char=21|ner=O\n5\tin\tin\tADP\tIN\t_\t6\tcase\t_\tstart_char=22|end_char=24|ner=O\n6\tHawaii\tHawaii\tPROPN\tNNP\tNumber=Sing\t4\tobl\t_\tSpaceAfter=No|start_char=25|end_char=31|ner=S-GPE\n7\t.\t.\tPUNCT\t.\t_\t4\tpunct\t_\tSpaceAfter=No|start_char=31|end_char=32|ner=O\n\n# text = He was elected president in 2008.\n# sent_id = 1\n# constituency = (ROOT (S (NP (PRP He)) (VP (VBD was) (VP (VBN elected) (S (NP (NN president))) (PP (IN in) (NP (CD 2008))))) (. .)))\n# sentiment = 1\n1\tHe\the\tPRON\tPRP\tCase=Nom|Gender=Masc|Number=Sing|Person=3|PronType=Prs\t3\tnsubj:pass\t_\tstart_char=0|end_char=2|ner=O\n2\twas\tbe\tAUX\tVBD\tMood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin\t3\taux:pass\t_\tstart_char=3|end_char=6|ner=O\n3\telected\telect\tVERB\tVBN\tTense=Past|VerbForm=Part|Voice=Pass\t0\troot\t_\tstart_char=7|end_char=14|ner=O\n4\tpresident\tpresident\tNOUN\tNN\tNumber=Sing\t3\txcomp\t_\tstart_char=15|end_char=24|ner=O\n5\tin\tin\tADP\tIN\t_\t6\tcase\t_\tstart_char=25|end_char=27|ner=O\n6\t2008\t2008\tNUM\tCD\tNumForm=Digit|NumType=Card\t3\tobl\t_\tSpaceAfter=No|start_char=28|end_char=32|ner=S-DATE\n7\t.\t.\tPUNCT\t.\t_\t3\tpunct\t_\tSpaceAfter=No|start_char=32|end_char=33|ner=O\n\n# text = Obama attended Harvard.\n# sent_id = 2\n# constituency = (ROOT (S (NP (NNP Obama)) (VP (VBD attended) (NP (NNP Harvard))) (. .)))\n# sentiment = 1\n1\tObama\tObama\tPROPN\tNNP\tNumber=Sing\t2\tnsubj\t_\tstart_char=0|end_char=5|ner=S-PERSON\n2\tattended\tattend\tVERB\tVBD\tMood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin\t0\troot\t_\tstart_char=6|end_char=14|ner=O\n3\tHarvard\tHarvard\tPROPN\tNNP\tNumber=Sing\t2\tobj\t_\tSpaceAfter=No|start_char=15|end_char=22|ner=S-ORG\n4\t.\t.\tPUNCT\t.\t_\t2\tpunct\t_\tSpaceAfter=No|start_char=22|end_char=23|ner=O\n\"\"\".strip()\n\nPRETOKENIZED_TEXT = \"Jennifer has lovely blue antennae .\"\n\nPRETOKENIZED_PIECES = [PRETOKENIZED_TEXT.split()]\n\nEXPECTED_TOKENIZED_ONLY_CONLLU = \"\"\"\n# text = Jennifer has lovely blue antennae .\n# sent_id = 0\n1\tJennifer\t_\t_\t_\t_\t0\t_\t_\tstart_char=0|end_char=8\n2\thas\t_\t_\t_\t_\t1\t_\t_\tstart_char=9|end_char=12\n3\tlovely\t_\t_\t_\t_\t2\t_\t_\tstart_char=13|end_char=19\n4\tblue\t_\t_\t_\t_\t3\t_\t_\tstart_char=20|end_char=24\n5\tantennae\t_\t_\t_\t_\t4\t_\t_\tstart_char=25|end_char=33\n6\t.\t_\t_\t_\t_\t5\t_\t_\tSpaceAfter=No|start_char=34|end_char=35\n\"\"\".strip()\n\nEXPECTED_PRETOKENIZED_CONLLU = \"\"\"\n# text = Jennifer has lovely blue antennae .\n# sent_id = 0\n# constituency = (ROOT (S (NP (NNP Jennifer)) (VP (VBZ has) (NP (JJ lovely) (JJ blue) (NNS antennae))) (. .)))\n# sentiment = 2\n1\tJennifer\tJennifer\tPROPN\tNNP\tNumber=Sing\t2\tnsubj\t_\tstart_char=0|end_char=8|ner=S-PERSON\n2\thas\thave\tVERB\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t0\troot\t_\tstart_char=9|end_char=12|ner=O\n3\tlovely\tlovely\tADJ\tJJ\tDegree=Pos\t5\tamod\t_\tstart_char=13|end_char=19|ner=O\n4\tblue\tblue\tADJ\tJJ\tDegree=Pos\t5\tamod\t_\tstart_char=20|end_char=24|ner=O\n5\tantennae\tantenna\tNOUN\tNNS\tNumber=Plur\t2\tobj\t_\tstart_char=25|end_char=33|ner=O\n6\t.\t.\tPUNCT\t.\t_\t2\tpunct\t_\tSpaceAfter=No|start_char=34|end_char=35|ner=O\n\"\"\".strip()\n\nclass TestEnglishPipeline:\n @pytest.fixture(scope=\"class\")\n def pipeline(self):\n return stanza.Pipeline(dir=TEST_MODELS_DIR, download_method=None)\n\n @pytest.fixture(scope=\"class\")\n def pretokenized_pipeline(self):\n return stanza.Pipeline(dir=TEST_MODELS_DIR, tokenize_pretokenized=True, download_method=None)\n\n @pytest.fixture(scope=\"class\")\n def tokenizer_pipeline(self):\n return stanza.Pipeline(dir=TEST_MODELS_DIR, processors=\"tokenize\", download_method=None)\n\n @pytest.fixture(scope=\"class\")\n def processed_doc(self, pipeline):\n \"\"\" Document created by running full English pipeline on a few sentences \"\"\"\n return pipeline(EN_DOC)\n\n def test_text(self, processed_doc):\n assert processed_doc.text == EN_DOC\n\n\n def test_conllu(self, processed_doc):\n assert \"{:C}\".format(processed_doc) == EN_DOC_CONLLU_GOLD\n\n def test_process_conllu(self, pipeline):\n \"\"\"\n Process a conllu text directly\n\n This can use the pipeline which still uses tokenization, as\n process_conllu skips the tokenize and mwt processors\n \"\"\"\n doc = pipeline.process_conllu(EN_DOC_CONLLU_GOLD)\n result = \"{:C}\".format(doc)\n assert result == EN_DOC_CONLLU_GOLD\n\n def test_tokens(self, processed_doc):\n assert \"\\n\\n\".join([sent.tokens_string() for sent in processed_doc.sentences]) == EN_DOC_TOKENS_GOLD\n\n\n def test_words(self, processed_doc):\n assert \"\\n\\n\".join([sent.words_string() for sent in processed_doc.sentences]) == EN_DOC_WORDS_GOLD\n\n\n def test_dependency_parse(self, processed_doc):\n assert \"\\n\\n\".join([sent.dependencies_string() for sent in processed_doc.sentences]) == \\\n EN_DOC_DEPENDENCY_PARSES_GOLD\n\n def test_empty(self, pipeline):\n # make sure that various models handle the degenerate empty case\n pipeline(\"\")\n pipeline(\"--\")\n\n def test_bulk_process(self, pipeline):\n \"\"\" Double check that the bulk_process method in Pipeline converts documents as expected \"\"\"\n # it should process strings\n processed = pipeline.bulk_process(EN_DOCS)\n assert \"\\n\\n\".join([\"{:C}\".format(doc) for doc in processed]) == EN_DOC_CONLLU_GOLD_MULTIDOC\n\n # it should pass Documents through successfully\n docs = [Document([], text=t) for t in EN_DOCS]\n processed = pipeline.bulk_process(docs)\n assert \"\\n\\n\".join([\"{:C}\".format(doc) for doc in processed]) == EN_DOC_CONLLU_GOLD_MULTIDOC\n\n def test_empty_bulk_process(self, pipeline):\n \"\"\" Previously we had a bug where an empty document list would cause a crash \"\"\"\n processed = pipeline.bulk_process([])\n assert processed == []\n\n def test_pretokenized(self, pretokenized_pipeline, tokenizer_pipeline):\n doc = pretokenized_pipeline(PRETOKENIZED_PIECES)\n conllu = \"{:C}\".format(doc).strip()\n assert conllu == EXPECTED_PRETOKENIZED_CONLLU\n\n doc = tokenizer_pipeline(PRETOKENIZED_TEXT)\n conllu = \"{:C}\".format(doc).strip()\n assert conllu == EXPECTED_TOKENIZED_ONLY_CONLLU\n\n # putting a doc with tokens into the pipeline should also work\n reparsed = pretokenized_pipeline(doc)\n conllu = \"{:C}\".format(reparsed).strip()\n assert conllu == EXPECTED_PRETOKENIZED_CONLLU\n\n def test_bulk_pretokenized(self, pretokenized_pipeline, tokenizer_pipeline):\n doc = tokenizer_pipeline(PRETOKENIZED_TEXT)\n conllu = \"{:C}\".format(doc).strip()\n assert conllu == EXPECTED_TOKENIZED_ONLY_CONLLU\n\n docs = pretokenized_pipeline([doc, doc])\n assert len(docs) == 2\n for doc in docs:\n conllu = \"{:C}\".format(doc).strip()\n assert conllu == EXPECTED_PRETOKENIZED_CONLLU\n\n def test_conll2doc_pretokenized(self, pretokenized_pipeline):\n doc = CoNLL.conll2doc(input_str=EXPECTED_TOKENIZED_ONLY_CONLLU)\n # this was bug from version 1.10.1 sent to us from a user\n # the pretokenized tokenize_processor would try to whitespace tokenize a document\n # even if the document already had sentences & words & stuff\n # not only would that be wrong if the text wouldn't whitespace tokenize into the words\n # (such as with punctuation and SpaceAfter=No),\n # it wouldn't even work in the case of conll2doc, since the document.text wasn't set\n docs = pretokenized_pipeline([doc, doc])\n assert len(docs) == 2\n for doc in docs:\n conllu = \"{:C}\".format(doc).strip()\n assert conllu == EXPECTED_PRETOKENIZED_CONLLU\n\n def test_stream(self, pipeline):\n \"\"\" Test the streaming interface to the Pipeline \"\"\"\n # Test all of the documents in one batch\n # (the default batch size is significantly more than |EN_DOCS|)\n processed = [doc for doc in pipeline.stream(EN_DOCS)]\n assert \"\\n\\n\".join([\"{:C}\".format(doc) for doc in processed]) == EN_DOC_CONLLU_GOLD_MULTIDOC\n\n # It should also work on an iterator rather than an iterable\n processed = [doc for doc in pipeline.stream(iter(EN_DOCS))]\n assert \"\\n\\n\".join([\"{:C}\".format(doc) for doc in processed]) == EN_DOC_CONLLU_GOLD_MULTIDOC\n\n # Stream one at a time\n processed = [doc for doc in pipeline.stream(EN_DOCS, batch_size=1)]\n processed = [\"{:C}\".format(doc) for doc in processed]\n assert \"\\n\\n\".join(processed) == EN_DOC_CONLLU_GOLD_MULTIDOC\n\n @pytest.fixture(scope=\"class\")\n def processed_multidoc(self, pipeline):\n \"\"\" Document created by running full English pipeline on a few sentences \"\"\"\n docs = [Document([], text=t) for t in EN_DOCS]\n return pipeline(docs)\n\n def test_conllu_multidoc(self, processed_multidoc):\n assert \"\\n\\n\".join([\"{:C}\".format(doc) for doc in processed_multidoc]) == EN_DOC_CONLLU_GOLD_MULTIDOC\n\n def test_tokens_multidoc(self, processed_multidoc):\n assert \"\\n\\n\".join([sent.tokens_string() for processed_doc in processed_multidoc for sent in processed_doc.sentences]) == EN_DOC_TOKENS_GOLD\n\n\n def test_words_multidoc(self, processed_multidoc):\n assert \"\\n\\n\".join([sent.words_string() for processed_doc in processed_multidoc for sent in processed_doc.sentences]) == EN_DOC_WORDS_GOLD\n\n def test_sentence_indices_multidoc(self, processed_multidoc):\n sentences = [sent for doc in processed_multidoc for sent in doc.sentences]\n for sent_idx, sentence in enumerate(sentences):\n assert sent_idx == sentence.index\n\n def test_dependency_parse_multidoc(self, processed_multidoc):\n assert \"\\n\\n\".join([sent.dependencies_string() for processed_doc in processed_multidoc for sent in processed_doc.sentences]) == \\\n EN_DOC_DEPENDENCY_PARSES_GOLD\n\n\n @pytest.fixture(scope=\"class\")\n def processed_multidoc_variant(self):\n \"\"\" Document created by running full English pipeline on a few sentences \"\"\"\n docs = [Document([], text=t) for t in EN_DOCS]\n nlp = stanza.Pipeline(dir=TEST_MODELS_DIR, processors={'tokenize': 'spacy'})\n return nlp(docs)\n\n def test_dependency_parse_multidoc_variant(self, processed_multidoc_variant):\n assert \"\\n\\n\".join([sent.dependencies_string() for processed_doc in processed_multidoc_variant for sent in processed_doc.sentences]) == \\\n EN_DOC_DEPENDENCY_PARSES_GOLD\n\n def test_constituency_parser(self):\n nlp = stanza.Pipeline(dir=TEST_MODELS_DIR, processors=\"tokenize,pos,constituency\")\n doc = nlp(\"This is a test\")\n assert str(doc.sentences[0].constituency) == '(ROOT (S (NP (DT This)) (VP (VBZ is) (NP (DT a) (NN test)))))'\n\n def test_on_gpu(self, pipeline):\n \"\"\"\n The default pipeline should have all the models on the GPU\n \"\"\"\n check_on_gpu(pipeline)\n\n def test_on_cpu(self):\n \"\"\"\n Create a pipeline on the CPU, check that all the models on CPU\n \"\"\"\n pipeline = stanza.Pipeline(\"en\", dir=TEST_MODELS_DIR, use_gpu=False)\n check_on_cpu(pipeline)\n"
},
{
"path": "stanza/tests/pipeline/test_french_pipeline.py",
"content": "\"\"\"\nBasic testing of French pipeline\n\nThe benefit of this test is to verify that the bulk processing works\nfor languages with MWT in them\n\"\"\"\n\nimport pytest\nimport stanza\nfrom stanza.models.common.doc import Document\n\nfrom stanza.tests import *\nfrom stanza.tests.pipeline.pipeline_device_tests import check_on_gpu, check_on_cpu\n\npytestmark = pytest.mark.pipeline\n\n\nFR_MWT_SENTENCE = \"Alors encore inconnu du grand public, Emmanuel Macron devient en 2014 ministre de l'Économie, de \" \\\n \"l'Industrie et du Numérique.\"\n\nEXPECTED_RESULT = \"\"\"\n[\n [\n {\n \"id\": 1,\n \"text\": \"Alors\",\n \"lemma\": \"alors\",\n \"upos\": \"ADV\",\n \"head\": 3,\n \"deprel\": \"advmod\",\n \"start_char\": 0,\n \"end_char\": 5\n },\n {\n \"id\": 2,\n \"text\": \"encore\",\n \"lemma\": \"encore\",\n \"upos\": \"ADV\",\n \"head\": 3,\n \"deprel\": \"advmod\",\n \"start_char\": 6,\n \"end_char\": 12\n },\n {\n \"id\": 3,\n \"text\": \"inconnu\",\n \"lemma\": \"inconnu\",\n \"upos\": \"ADJ\",\n \"feats\": \"Gender=Masc|Number=Sing\",\n \"head\": 11,\n \"deprel\": \"advcl\",\n \"start_char\": 13,\n \"end_char\": 20\n },\n {\n \"id\": [\n 4,\n 5\n ],\n \"text\": \"du\",\n \"start_char\": 21,\n \"end_char\": 23\n },\n {\n \"id\": 4,\n \"text\": \"de\",\n \"lemma\": \"de\",\n \"upos\": \"ADP\",\n \"head\": 7,\n \"deprel\": \"case\"\n },\n {\n \"id\": 5,\n \"text\": \"le\",\n \"lemma\": \"le\",\n \"upos\": \"DET\",\n \"feats\": \"Definite=Def|Gender=Masc|Number=Sing|PronType=Art\",\n \"head\": 7,\n \"deprel\": \"det\"\n },\n {\n \"id\": 6,\n \"text\": \"grand\",\n \"lemma\": \"grand\",\n \"upos\": \"ADJ\",\n \"feats\": \"Gender=Masc|Number=Sing\",\n \"head\": 7,\n \"deprel\": \"amod\",\n \"start_char\": 24,\n \"end_char\": 29\n },\n {\n \"id\": 7,\n \"text\": \"public\",\n \"lemma\": \"public\",\n \"upos\": \"NOUN\",\n \"feats\": \"Gender=Masc|Number=Sing\",\n \"head\": 3,\n \"deprel\": \"obl:arg\",\n \"start_char\": 30,\n \"end_char\": 36,\n \"misc\": \"SpaceAfter=No\"\n },\n {\n \"id\": 8,\n \"text\": \",\",\n \"lemma\": \",\",\n \"upos\": \"PUNCT\",\n \"head\": 3,\n \"deprel\": \"punct\",\n \"start_char\": 36,\n \"end_char\": 37\n },\n {\n \"id\": 9,\n \"text\": \"Emmanuel\",\n \"lemma\": \"Emmanuel\",\n \"upos\": \"PROPN\",\n \"head\": 11,\n \"deprel\": \"nsubj\",\n \"start_char\": 38,\n \"end_char\": 46\n },\n {\n \"id\": 10,\n \"text\": \"Macron\",\n \"lemma\": \"Macron\",\n \"upos\": \"PROPN\",\n \"head\": 9,\n \"deprel\": \"flat:name\",\n \"start_char\": 47,\n \"end_char\": 53\n },\n {\n \"id\": 11,\n \"text\": \"devient\",\n \"lemma\": \"devenir\",\n \"upos\": \"VERB\",\n \"feats\": \"Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\",\n \"head\": 0,\n \"deprel\": \"root\",\n \"start_char\": 54,\n \"end_char\": 61\n },\n {\n \"id\": 12,\n \"text\": \"en\",\n \"lemma\": \"en\",\n \"upos\": \"ADP\",\n \"head\": 13,\n \"deprel\": \"case\",\n \"start_char\": 62,\n \"end_char\": 64\n },\n {\n \"id\": 13,\n \"text\": \"2014\",\n \"lemma\": \"2014\",\n \"upos\": \"NUM\",\n \"feats\": \"Number=Plur\",\n \"head\": 11,\n \"deprel\": \"obl:mod\",\n \"start_char\": 65,\n \"end_char\": 69\n },\n {\n \"id\": 14,\n \"text\": \"ministre\",\n \"lemma\": \"ministre\",\n \"upos\": \"NOUN\",\n \"feats\": \"Gender=Masc|Number=Sing\",\n \"head\": 11,\n \"deprel\": \"xcomp\",\n \"start_char\": 70,\n \"end_char\": 78\n },\n {\n \"id\": 15,\n \"text\": \"de\",\n \"lemma\": \"de\",\n \"upos\": \"ADP\",\n \"head\": 17,\n \"deprel\": \"case\",\n \"start_char\": 79,\n \"end_char\": 81\n },\n {\n \"id\": 16,\n \"text\": \"l'\",\n \"lemma\": \"le\",\n \"upos\": \"DET\",\n \"feats\": \"Definite=Def|Number=Sing|PronType=Art\",\n \"head\": 17,\n \"deprel\": \"det\",\n \"start_char\": 82,\n \"end_char\": 84,\n \"misc\": \"SpaceAfter=No\"\n },\n {\n \"id\": 17,\n \"text\": \"Économie\",\n \"lemma\": \"économie\",\n \"upos\": \"NOUN\",\n \"feats\": \"Gender=Fem|Number=Sing\",\n \"head\": 14,\n \"deprel\": \"nmod\",\n \"start_char\": 84,\n \"end_char\": 92,\n \"misc\": \"SpaceAfter=No\"\n },\n {\n \"id\": 18,\n \"text\": \",\",\n \"lemma\": \",\",\n \"upos\": \"PUNCT\",\n \"head\": 21,\n \"deprel\": \"punct\",\n \"start_char\": 92,\n \"end_char\": 93\n },\n {\n \"id\": 19,\n \"text\": \"de\",\n \"lemma\": \"de\",\n \"upos\": \"ADP\",\n \"head\": 21,\n \"deprel\": \"case\",\n \"start_char\": 94,\n \"end_char\": 96\n },\n {\n \"id\": 20,\n \"text\": \"l'\",\n \"lemma\": \"le\",\n \"upos\": \"DET\",\n \"feats\": \"Definite=Def|Number=Sing|PronType=Art\",\n \"head\": 21,\n \"deprel\": \"det\",\n \"start_char\": 97,\n \"end_char\": 99,\n \"misc\": \"SpaceAfter=No\"\n },\n {\n \"id\": 21,\n \"text\": \"Industrie\",\n \"lemma\": \"industrie\",\n \"upos\": \"NOUN\",\n \"feats\": \"Gender=Fem|Number=Sing\",\n \"head\": 17,\n \"deprel\": \"conj\",\n \"start_char\": 99,\n \"end_char\": 108\n },\n {\n \"id\": 22,\n \"text\": \"et\",\n \"lemma\": \"et\",\n \"upos\": \"CCONJ\",\n \"head\": 25,\n \"deprel\": \"cc\",\n \"start_char\": 109,\n \"end_char\": 111\n },\n {\n \"id\": [\n 23,\n 24\n ],\n \"text\": \"du\",\n \"start_char\": 112,\n \"end_char\": 114\n },\n {\n \"id\": 23,\n \"text\": \"de\",\n \"lemma\": \"de\",\n \"upos\": \"ADP\",\n \"head\": 25,\n \"deprel\": \"case\"\n },\n {\n \"id\": 24,\n \"text\": \"le\",\n \"lemma\": \"le\",\n \"upos\": \"DET\",\n \"feats\": \"Definite=Def|Gender=Masc|Number=Sing|PronType=Art\",\n \"head\": 25,\n \"deprel\": \"det\"\n },\n {\n \"id\": 25,\n \"text\": \"Numérique\",\n \"lemma\": \"numérique\",\n \"upos\": \"NOUN\",\n \"feats\": \"Gender=Masc|Number=Sing\",\n \"head\": 17,\n \"deprel\": \"conj\",\n \"start_char\": 115,\n \"end_char\": 124,\n \"misc\": \"SpaceAfter=No\"\n },\n {\n \"id\": 26,\n \"text\": \".\",\n \"lemma\": \".\",\n \"upos\": \"PUNCT\",\n \"head\": 11,\n \"deprel\": \"punct\",\n \"start_char\": 124,\n \"end_char\": 125,\n \"misc\": \"SpaceAfter=No\"\n }\n ]\n]\n\"\"\"\n\nclass TestFrenchPipeline:\n @pytest.fixture(scope=\"class\")\n def pipeline(self):\n \"\"\" Create a pipeline with French models \"\"\"\n pipeline = stanza.Pipeline(processors='tokenize,mwt,pos,lemma,depparse', dir=TEST_MODELS_DIR, lang='fr')\n return pipeline\n\n def test_single(self, pipeline):\n doc = pipeline(FR_MWT_SENTENCE)\n compare_ignoring_whitespace(str(doc), EXPECTED_RESULT)\n\n def test_bulk(self, pipeline):\n NUM_DOCS = 10\n raw_text = [FR_MWT_SENTENCE] * NUM_DOCS\n raw_doc = [Document([], text=doccontent) for doccontent in raw_text]\n\n result = pipeline(raw_doc)\n\n assert len(result) == NUM_DOCS\n for doc in result:\n compare_ignoring_whitespace(str(doc), EXPECTED_RESULT)\n assert len(doc.sentences) == 1\n assert doc.num_words == 26\n assert doc.num_tokens == 24\n\n def test_on_gpu(self, pipeline):\n \"\"\"\n The default pipeline should have all the models on the GPU\n \"\"\"\n check_on_gpu(pipeline)\n\n def test_on_cpu(self):\n \"\"\"\n Create a pipeline on the CPU, check that all the models on CPU\n \"\"\"\n pipeline = stanza.Pipeline(\"fr\", dir=TEST_MODELS_DIR, use_gpu=False)\n check_on_cpu(pipeline)\n"
},
{
"path": "stanza/tests/pipeline/test_lemmatizer.py",
"content": "\"\"\"\nBasic testing of lemmatization\n\"\"\"\n\nimport pytest\nimport stanza\n\nfrom stanza.tests import *\nfrom stanza.models.common.doc import TEXT, UPOS, LEMMA\n\npytestmark = pytest.mark.pipeline\n\nEN_DOC = \"Joe Smith was born in California.\"\n\nEN_DOC_IDENTITY_GOLD = \"\"\"\nJoe Joe\nSmith Smith\nwas was\nborn born\nin in\nCalifornia California\n. .\n\"\"\".strip()\n\nEN_DOC_LEMMATIZER_MODEL_GOLD = \"\"\"\nJoe Joe\nSmith Smith\nwas be\nborn bear\nin in\nCalifornia California\n. .\n\"\"\".strip()\n\n\ndef test_identity_lemmatizer():\n nlp = stanza.Pipeline(**{'processors': 'tokenize,lemma', 'dir': TEST_MODELS_DIR, 'lang': 'en', 'lemma_use_identity': True}, download_method=None)\n doc = nlp(EN_DOC)\n word_lemma_pairs = []\n for w in doc.iter_words():\n word_lemma_pairs += [f\"{w.text} {w.lemma}\"]\n assert EN_DOC_IDENTITY_GOLD == \"\\n\".join(word_lemma_pairs)\n\ndef test_full_lemmatizer():\n nlp = stanza.Pipeline(**{'processors': 'tokenize,pos,lemma', 'dir': TEST_MODELS_DIR, 'lang': 'en'}, download_method=None)\n doc = nlp(EN_DOC)\n word_lemma_pairs = []\n for w in doc.iter_words():\n word_lemma_pairs += [f\"{w.text} {w.lemma}\"]\n assert EN_DOC_LEMMATIZER_MODEL_GOLD == \"\\n\".join(word_lemma_pairs)\n\ndef find_unknown_word(lemmatizer, base):\n for i in range(10):\n base = base + \"z\"\n if base not in lemmatizer.word_dict and all(x[0] != base for x in lemmatizer.composite_dict.keys()):\n return base\n raise RuntimeError(\"wtf?\")\n\ndef test_store_results():\n nlp = stanza.Pipeline(**{'processors': 'tokenize,pos,lemma', 'dir': TEST_MODELS_DIR, 'lang': 'en'}, lemma_store_results=True, download_method=None)\n lemmatizer = nlp.processors[\"lemma\"]._trainer\n\n az = find_unknown_word(lemmatizer, \"a\")\n bz = find_unknown_word(lemmatizer, \"b\")\n cz = find_unknown_word(lemmatizer, \"c\")\n\n # try sentences with the order long, short\n doc = nlp(\"I found an \" + az + \" in my \" + bz + \". It was a \" + cz)\n stuff = doc.get([TEXT, UPOS, LEMMA])\n assert len(stuff) == 12\n assert stuff[3][0] == az\n assert stuff[6][0] == bz\n assert stuff[11][0] == cz\n\n assert lemmatizer.composite_dict[(az, stuff[3][1])] == stuff[3][2]\n assert lemmatizer.composite_dict[(bz, stuff[6][1])] == stuff[6][2]\n assert lemmatizer.composite_dict[(cz, stuff[11][1])] == stuff[11][2]\n\n doc2 = nlp(\"I found an \" + az + \" in my \" + bz + \". It was a \" + cz)\n stuff2 = doc2.get([TEXT, UPOS, LEMMA])\n\n assert stuff == stuff2\n\n dz = find_unknown_word(lemmatizer, \"d\")\n ez = find_unknown_word(lemmatizer, \"e\")\n fz = find_unknown_word(lemmatizer, \"f\")\n\n # try sentences with the order long, short\n doc = nlp(\"It was a \" + dz + \". I found an \" + ez + \" in my \" + fz)\n stuff = doc.get([TEXT, UPOS, LEMMA])\n assert len(stuff) == 12\n assert stuff[3][0] == dz\n assert stuff[8][0] == ez\n assert stuff[11][0] == fz\n\n assert lemmatizer.composite_dict[(dz, stuff[3][1])] == stuff[3][2]\n assert lemmatizer.composite_dict[(ez, stuff[8][1])] == stuff[8][2]\n assert lemmatizer.composite_dict[(fz, stuff[11][1])] == stuff[11][2]\n\n doc2 = nlp(\"It was a \" + dz + \". I found an \" + ez + \" in my \" + fz)\n stuff2 = doc2.get([TEXT, UPOS, LEMMA])\n\n assert stuff == stuff2\n\n assert az not in lemmatizer.word_dict\n\ndef test_caseless_lemmatizer():\n \"\"\"\n Test that setting the lemmatizer as caseless at Pipeline time lowercases the text\n \"\"\"\n nlp = stanza.Pipeline('en', processors='tokenize,pos,lemma', model_dir=TEST_MODELS_DIR, download_method=None)\n # the capital letter here should throw off the lemmatizer & it won't remove the plural\n # although weirdly the current English model *does* lowercase the A\n doc = nlp(\"Here is an Excerpt\")\n assert doc.sentences[0].words[-1].lemma == 'excerpt'\n\n nlp = stanza.Pipeline('en', processors='tokenize,pos,lemma', model_dir=TEST_MODELS_DIR, download_method=None, lemma_caseless=True)\n # with the model set to lowercasing, the word will be treated as if it were 'antennae'\n doc = nlp(\"Here is an Excerpt\")\n assert doc.sentences[0].words[-1].lemma == 'Excerpt'\n\ndef test_latin_caseless_lemmatizer():\n \"\"\"\n Test the Latin caseless lemmatizer\n \"\"\"\n nlp = stanza.Pipeline('la', package='ittb', processors='tokenize,pos,lemma', model_dir=TEST_MODELS_DIR, download_method=None)\n lemmatizer = nlp.processors['lemma']\n assert lemmatizer.config['caseless']\n\n doc = nlp(\"Quod Erat Demonstrandum\")\n expected_lemmas = \"qui sum demonstro\".split()\n assert len(doc.sentences) == 1\n assert len(doc.sentences[0].words) == 3\n for word, expected in zip(doc.sentences[0].words, expected_lemmas):\n assert word.lemma == expected\n\ndef test_contextual_lemmatizer():\n nlp = stanza.Pipeline('en', processors='tokenize,pos,lemma', model_dir=TEST_MODELS_DIR, package={\"lemma\": \"default_accurate\"}, download_method=\"reuse_resources\")\n lemmatizer = nlp.processors['lemma']._trainer\n # the accurate model should have a 's classifier\n assert len(lemmatizer.contextual_lemmatizers) > 0\n # ideally the doc would have 'have' as the lemma for the second\n # word, but maybe it's not always accurate. actually, it works\n # fine at the time of this test\n doc = nlp(\"He's added a contextual lemmatizer\")\n"
},
{
"path": "stanza/tests/pipeline/test_pipeline_constituency_processor.py",
"content": "import gc\nimport pytest\nimport stanza\nfrom stanza.models.common.foundation_cache import FoundationCache\n\nfrom stanza.tests import *\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\n# data for testing\nTEST_TEXT = \"This is a test. Another sentence. Are these sorted?\"\n\nTEST_TOKENS = [[\"This\", \"is\", \"a\", \"test\", \".\"], [\"Another\", \"sentence\", \".\"], [\"Are\", \"these\", \"sorted\", \"?\"]]\n\n@pytest.fixture(scope=\"module\")\ndef foundation_cache():\n # the test suite sometimes winds up holding on to GPU memory for too long,\n # resulting in an OOM error\n # occasionally calling gc.collect() will help\n gc.collect()\n return FoundationCache()\n\ndef check_results(doc):\n assert len(doc.sentences) == len(TEST_TOKENS)\n for sentence, expected in zip(doc.sentences, TEST_TOKENS):\n assert sentence.constituency.leaf_labels() == expected\n\ndef test_sorted_big_batch(foundation_cache):\n pipe = stanza.Pipeline(\"en\", model_dir=TEST_MODELS_DIR, processors=\"tokenize,pos,constituency\", foundation_cache=foundation_cache, download_method=None)\n doc = pipe(TEST_TEXT)\n check_results(doc)\n\ndef test_comments(foundation_cache):\n \"\"\"\n Test that the pipeline is creating constituency comments\n \"\"\"\n pipe = stanza.Pipeline(\"en\", model_dir=TEST_MODELS_DIR, processors=\"tokenize,pos,constituency\", foundation_cache=foundation_cache, download_method=None)\n doc = pipe(TEST_TEXT)\n check_results(doc)\n for sentence in doc.sentences:\n assert any(x.startswith(\"# constituency = \") for x in sentence.comments)\n doc.sentences[0].constituency = \"asdf\"\n assert \"# constituency = asdf\" in doc.sentences[0].comments\n for sentence in doc.sentences:\n assert len([x for x in sentence.comments if x.startswith(\"# constituency\")]) == 1\n\ndef test_illegal_batch_size(foundation_cache):\n stanza.Pipeline(\"en\", model_dir=TEST_MODELS_DIR, processors=\"tokenize,pos\", constituency_batch_size=\"zzz\", foundation_cache=foundation_cache, download_method=None)\n with pytest.raises(ValueError):\n stanza.Pipeline(\"en\", model_dir=TEST_MODELS_DIR, processors=\"tokenize,pos,constituency\", constituency_batch_size=\"zzz\", foundation_cache=foundation_cache, download_method=None)\n\ndef test_sorted_one_batch(foundation_cache):\n pipe = stanza.Pipeline(\"en\", model_dir=TEST_MODELS_DIR, processors=\"tokenize,pos,constituency\", constituency_batch_size=1, foundation_cache=foundation_cache, download_method=None)\n doc = pipe(TEST_TEXT)\n check_results(doc)\n\ndef test_sorted_two_batch(foundation_cache):\n pipe = stanza.Pipeline(\"en\", model_dir=TEST_MODELS_DIR, processors=\"tokenize,pos,constituency\", constituency_batch_size=2, foundation_cache=foundation_cache, download_method=None)\n doc = pipe(TEST_TEXT)\n check_results(doc)\n\ndef test_get_constituents(foundation_cache):\n pipe = stanza.Pipeline(\"en\", model_dir=TEST_MODELS_DIR, processors=\"tokenize,pos,constituency\", foundation_cache=foundation_cache, download_method=None)\n assert \"SBAR\" in pipe.processors[\"constituency\"].get_constituents()\n"
},
{
"path": "stanza/tests/pipeline/test_pipeline_depparse_processor.py",
"content": "\"\"\"\nBasic testing of part of speech tagging\n\"\"\"\n\nimport pytest\nimport stanza\nfrom stanza.models.common.vocab import VOCAB_PREFIX\n\nfrom stanza.tests import TEST_MODELS_DIR\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\nclass TestClassifier:\n @pytest.fixture(scope=\"class\")\n def english_depparse(self):\n \"\"\"\n Get a depparse_processor for English\n \"\"\"\n nlp = stanza.Pipeline(**{'processors': 'tokenize,pos,lemma,depparse', 'dir': TEST_MODELS_DIR, 'lang': 'en'})\n assert 'depparse' in nlp.processors\n return nlp.processors['depparse']\n\n def test_get_known_relations(self, english_depparse):\n \"\"\"\n Test getting the known relations from a processor.\n\n Doesn't test that all the relations exist, since who knows what will change in the future\n \"\"\"\n relations = english_depparse.get_known_relations()\n assert len(relations) > 5\n assert 'case' in relations\n for i in VOCAB_PREFIX:\n assert i not in relations\n"
},
{
"path": "stanza/tests/pipeline/test_pipeline_mwt_expander.py",
"content": "\"\"\"\nBasic testing of multi-word-token expansion\n\"\"\"\n\nimport pytest\nimport stanza\n\nfrom stanza.tests import *\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\n# mwt data for testing\nFR_MWT_SENTENCE = \"Alors encore inconnu du grand public, Emmanuel Macron devient en 2014 ministre de l'Économie, de \" \\\n \"l'Industrie et du Numérique.\"\n\n\nFR_MWT_TOKEN_TO_WORDS_GOLD = \"\"\"\ntoken: Alors \t\twords: []\ntoken: encore \t\twords: []\ntoken: inconnu \t\twords: []\ntoken: du \t\twords: [, ]\ntoken: grand \t\twords: []\ntoken: public \t\twords: []\ntoken: , \t\twords: []\ntoken: Emmanuel \t\twords: []\ntoken: Macron \t\twords: []\ntoken: devient \t\twords: []\ntoken: en \t\twords: []\ntoken: 2014 \t\twords: []\ntoken: ministre \t\twords: []\ntoken: de \t\twords: []\ntoken: l' \t\twords: []\ntoken: Économie \t\twords: []\ntoken: , \t\twords: []\ntoken: de \t\twords: []\ntoken: l' \t\twords: []\ntoken: Industrie\t\twords: []\ntoken: et \t\twords: []\ntoken: du \t\twords: [, ]\ntoken: Numérique\t\twords: []\ntoken: . \t\twords: []\n\"\"\".strip()\n\nFR_MWT_WORD_TO_TOKEN_GOLD = \"\"\"\nword: Alors \t\ttoken parent:1-Alors\nword: encore \t\ttoken parent:2-encore\nword: inconnu \t\ttoken parent:3-inconnu\nword: de \t\ttoken parent:4-5-du\nword: le \t\ttoken parent:4-5-du\nword: grand \t\ttoken parent:6-grand\nword: public \t\ttoken parent:7-public\nword: , \t\ttoken parent:8-,\nword: Emmanuel \t\ttoken parent:9-Emmanuel\nword: Macron \t\ttoken parent:10-Macron\nword: devient \t\ttoken parent:11-devient\nword: en \t\ttoken parent:12-en\nword: 2014 \t\ttoken parent:13-2014\nword: ministre \t\ttoken parent:14-ministre\nword: de \t\ttoken parent:15-de\nword: l' \t\ttoken parent:16-l'\nword: Économie \t\ttoken parent:17-Économie\nword: , \t\ttoken parent:18-,\nword: de \t\ttoken parent:19-de\nword: l' \t\ttoken parent:20-l'\nword: Industrie\t\ttoken parent:21-Industrie\nword: et \t\ttoken parent:22-et\nword: de \t\ttoken parent:23-24-du\nword: le \t\ttoken parent:23-24-du\nword: Numérique\t\ttoken parent:25-Numérique\nword: . \t\ttoken parent:26-.\n\"\"\".strip()\n\n\ndef test_mwt():\n pipeline = stanza.Pipeline(processors='tokenize,mwt', dir=TEST_MODELS_DIR, lang='fr', download_method=None)\n doc = pipeline(FR_MWT_SENTENCE)\n token_to_words = \"\\n\".join(\n [f'token: {token.text.ljust(9)}\\t\\twords: [{\", \".join([word.pretty_print() for word in token.words])}]' for sent in doc.sentences for token in sent.tokens]\n ).strip()\n word_to_token = \"\\n\".join(\n [f'word: {word.text.ljust(9)}\\t\\ttoken parent:{\"-\".join([str(x) for x in word.parent.id])}-{word.parent.text}'\n for sent in doc.sentences for word in sent.words]).strip()\n assert token_to_words == FR_MWT_TOKEN_TO_WORDS_GOLD\n assert word_to_token == FR_MWT_WORD_TO_TOKEN_GOLD\n\ndef test_unknown_character():\n \"\"\"\n The MWT processor has a mechanism to temporarily add unknown characters to the vocab\n\n Here we check that it is properly adding the characters from a test case a user sent us\n \"\"\"\n pipeline = stanza.Pipeline(processors='tokenize,mwt', dir=TEST_MODELS_DIR, lang='en', download_method=None)\n text = \"Björkängshallen's\"\n mwt_processor = pipeline.processors[\"mwt\"]\n trainer = mwt_processor.trainer\n # verify that the test case is still valid\n # (perhaps an updated MWT model will have all of these characters in the future)\n assert not all(x in trainer.vocab._unit2id for x in text)\n doc = pipeline(text)\n batch = mwt_processor.build_batch(doc)\n # the vocab used in this batch should have the missing characters\n assert all(x in batch.vocab._unit2id for x in text)\n\ndef test_unknown_word():\n \"\"\"\n Test a word which wasn't in the MWT training data\n\n The seq2seq model for MWT was randomly hallucinating, but with the\n CharacterClassifier, it should be able to process unusual MWT\n without hallucinations\n \"\"\"\n pipe = stanza.Pipeline(processors='tokenize,mwt', dir=TEST_MODELS_DIR, lang='en', download_method=None)\n doc = pipe(\"I read the newspaper's report.\")\n assert len(doc.sentences) == 1\n assert len(doc.sentences[0].tokens) == 6\n assert len(doc.sentences[0].tokens[3].words) == 2\n assert doc.sentences[0].tokens[3].words[0].text == 'newspaper'\n\n # double check that this is something unknown to the model\n mwt_processor = pipe.processors[\"mwt\"]\n trainer = mwt_processor.trainer\n expansion = trainer.dict_expansion(\"newspaper's\")\n assert expansion is None\n"
},
{
"path": "stanza/tests/pipeline/test_pipeline_ner_processor.py",
"content": "\nimport pytest\nimport stanza\nfrom stanza.utils.conll import CoNLL\nfrom stanza.models.common.doc import Document\n\nfrom stanza.tests import *\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\n# data for testing\nEN_DOCS = [\"Barack Obama was born in Hawaii.\", \"He was elected president in 2008.\", \"Obama attended Harvard.\"]\n\nEXPECTED_ENTS = [[{\n \"text\": \"Barack Obama\",\n \"type\": \"PERSON\",\n \"start_char\": 0,\n \"end_char\": 12\n}, {\n \"text\": \"Hawaii\",\n \"type\": \"GPE\",\n \"start_char\": 25,\n \"end_char\": 31\n}],\n[{\n \"text\": \"2008\",\n \"type\": \"DATE\",\n \"start_char\": 28,\n \"end_char\": 32\n}],\n[{\n \"text\": \"Obama\",\n \"type\": \"PERSON\",\n \"start_char\": 0,\n \"end_char\": 5\n}, {\n \"text\": \"Harvard\",\n \"type\": \"ORG\",\n \"start_char\": 15,\n \"end_char\": 22\n}]]\n\n\ndef check_entities_equal(doc, expected):\n \"\"\"\n Checks that the entities of a doc are equal to the given list of maps\n \"\"\"\n assert len(doc.ents) == len(expected)\n for doc_entity, expected_entity in zip(doc.ents, expected):\n for k in expected_entity:\n assert getattr(doc_entity, k) == expected_entity[k]\n\nclass TestNERProcessor:\n @pytest.fixture(scope=\"class\")\n def pipeline(self):\n \"\"\"\n A reusable pipeline with the NER module\n \"\"\"\n return stanza.Pipeline(dir=TEST_MODELS_DIR, processors=\"tokenize,ner\")\n\n @pytest.fixture(scope=\"class\")\n def processed_doc(self, pipeline):\n \"\"\" Document created by running full English pipeline on a few sentences \"\"\"\n return [pipeline(text) for text in EN_DOCS]\n\n\n @pytest.fixture(scope=\"class\")\n def processed_bulk(self, pipeline):\n \"\"\" Document created by running full English pipeline on a few sentences \"\"\"\n docs = [Document([], text=t) for t in EN_DOCS]\n return pipeline(docs)\n\n def test_bulk_ents(self, processed_bulk):\n assert len(processed_bulk) == len(EXPECTED_ENTS)\n for doc, expected in zip(processed_bulk, EXPECTED_ENTS):\n check_entities_equal(doc, expected)\n\n def test_ents(self, processed_doc):\n assert len(processed_doc) == len(EXPECTED_ENTS)\n for doc, expected in zip(processed_doc, EXPECTED_ENTS):\n check_entities_equal(doc, expected)\n\nEXPECTED_MULTI_ENTS = [{\n \"text\": \"John Bauer\",\n \"type\": \"PERSON\",\n \"start_char\": 0,\n \"end_char\": 10\n}, {\n \"text\": \"Stanford\",\n \"type\": \"ORG\",\n \"start_char\": 20,\n \"end_char\": 28\n}, {\n \"text\": \"hip arthritis\",\n \"type\": \"DISEASE\",\n \"start_char\": 37,\n \"end_char\": 50\n}, {\n \"text\": \"Chris Manning\",\n \"type\": \"PERSON\",\n \"start_char\": 66,\n \"end_char\": 79\n}]\n\n\nEXPECTED_MULTI_NER = [\n [('O', 'B-PERSON'),\n ('O', 'E-PERSON'),\n ('O', 'O'),\n ('O', 'O'),\n ('O', 'S-ORG'),\n ('O', 'O'),\n ('O', 'O'),\n ('B-DISEASE', 'O'),\n ('E-DISEASE', 'O'),\n ('O', 'O')],\n [('O', 'O'),\n ('O', 'O'),\n ('O', 'O'),\n ('O', 'B-PERSON'),\n ('O', 'E-PERSON'),]]\n\n\n\nclass TestMultiNERProcessor:\n @pytest.fixture(scope=\"class\")\n def pipeline(self):\n \"\"\"\n A reusable pipeline with TWO ner models\n \"\"\"\n return stanza.Pipeline(dir=TEST_MODELS_DIR, processors=\"tokenize,ner\", package={\"ner\": [\"ncbi_disease\", \"ontonotes_charlm\"]})\n\n def test_multi_example(self, pipeline):\n doc = pipeline(\"John Bauer works at Stanford and has hip arthritis. He works for Chris Manning\")\n check_entities_equal(doc, EXPECTED_MULTI_ENTS)\n\n def test_multi_ner(self, pipeline):\n \"\"\"\n Test that multiple NER labels are correctly assigned in tuples\n \"\"\"\n doc = pipeline(\"John Bauer works at Stanford and has hip arthritis. He works for Chris Manning\")\n multi_ner = [[token.multi_ner for token in sentence.tokens] for sentence in doc.sentences]\n assert multi_ner == EXPECTED_MULTI_NER\n\n def test_known_tags(self, pipeline):\n assert pipeline.processors[\"ner\"].get_known_tags() == [\"DISEASE\"]\n assert len(pipeline.processors[\"ner\"].get_known_tags(1)) == 18\n"
},
{
"path": "stanza/tests/pipeline/test_pipeline_pos_processor.py",
"content": "\"\"\"\nBasic testing of part of speech tagging\n\"\"\"\n\nimport pytest\nimport stanza\n\nfrom stanza.tests import *\n\npytestmark = pytest.mark.pipeline\n\nEN_DOC = \"Joe Smith was born in California.\"\n\nEN_DOC_GOLD = \"\"\"\n]>\n]>\n]>\n]>\n]>\n]>\n]>\n\"\"\".strip()\n\n@pytest.fixture(scope=\"module\")\ndef pos_pipeline():\n return stanza.Pipeline(**{'processors': 'tokenize,pos', 'dir': TEST_MODELS_DIR, 'download_method': None, 'lang': 'en'})\n\ndef test_part_of_speech(pos_pipeline):\n doc = pos_pipeline(EN_DOC)\n assert EN_DOC_GOLD == '\\n\\n'.join([sent.tokens_string() for sent in doc.sentences])\n\ndef test_get_known_xpos(pos_pipeline):\n tags = pos_pipeline.processors['pos'].get_known_xpos()\n # make sure we have xpos...\n assert 'DT' in tags\n # ... and not upos\n assert 'DET' not in tags\n\ndef test_get_known_upos(pos_pipeline):\n tags = pos_pipeline.processors['pos'].get_known_upos()\n # make sure we have upos...\n assert 'DET' in tags\n # ... and not xpos\n assert 'DT' not in tags\n\n\ndef test_get_known_feats(pos_pipeline):\n feats = pos_pipeline.processors['pos'].get_known_feats()\n # I appreciate how self-referential the Abbr feat is\n assert 'Abbr' in feats\n assert 'Yes' in feats['Abbr']\n"
},
{
"path": "stanza/tests/pipeline/test_pipeline_sentiment_processor.py",
"content": "import gc\n\nimport pytest\nimport stanza\nfrom stanza.utils.conll import CoNLL\nfrom stanza.models.common.doc import Document\n\nfrom stanza.tests import *\n\npytestmark = [pytest.mark.pipeline, pytest.mark.travis]\n\n# data for testing\nEN_DOCS = [\"Ragavan is terrible and should go away.\", \"Today is okay.\", \"Urza's Saga is great.\"]\n\nEN_DOC = \" \".join(EN_DOCS)\n\nEXPECTED = [0, 1, 2]\n\nclass TestSentimentPipeline:\n @pytest.fixture(scope=\"class\")\n def pipeline(self):\n \"\"\"\n A reusable pipeline with the NER module\n \"\"\"\n gc.collect()\n return stanza.Pipeline(dir=TEST_MODELS_DIR, processors=\"tokenize,sentiment\")\n\n def test_simple(self, pipeline):\n results = []\n for text in EN_DOCS:\n doc = pipeline(text)\n assert len(doc.sentences) == 1\n results.append(doc.sentences[0].sentiment)\n assert EXPECTED == results\n\n def test_multiple_sentences(self, pipeline):\n doc = pipeline(EN_DOC)\n assert len(doc.sentences) == 3\n results = [sentence.sentiment for sentence in doc.sentences]\n assert EXPECTED == results\n\n def test_empty_text(self, pipeline):\n \"\"\"\n Test empty text and a text which might get reduced to empty text by removing dashes\n \"\"\"\n doc = pipeline(\"\")\n assert len(doc.sentences) == 0\n\n doc = pipeline(\"--\")\n assert len(doc.sentences) == 1\n"
},
{
"path": "stanza/tests/pipeline/test_requirements.py",
"content": "\"\"\"\nTest the requirements functionality for processors\n\"\"\"\n\nimport pytest\nimport stanza\n\nfrom stanza.pipeline.core import PipelineRequirementsException\nfrom stanza.pipeline.processor import ProcessorRequirementsException\nfrom stanza.tests import *\n\npytestmark = pytest.mark.pipeline\n\ndef check_exception_vals(req_exception, req_exception_vals):\n \"\"\"\n Check the values of a ProcessorRequirementsException against a dict of expected values.\n :param req_exception: the ProcessorRequirementsException to evaluate\n :param req_exception_vals: expected values for the ProcessorRequirementsException\n :return: None\n \"\"\"\n assert isinstance(req_exception, ProcessorRequirementsException)\n assert req_exception.processor_type == req_exception_vals['processor_type']\n assert req_exception.processors_list == req_exception_vals['processors_list']\n assert req_exception.err_processor.requires == req_exception_vals['requires']\n\n\ndef test_missing_requirements():\n \"\"\"\n Try to build several pipelines with bad configs and check thrown exceptions against gold exceptions.\n :return: None\n \"\"\"\n # list of (bad configs, list of gold ProcessorRequirementsExceptions that should be thrown) pairs\n bad_config_lists = [\n # missing tokenize\n (\n # input config\n {'processors': 'pos,depparse', 'dir': TEST_MODELS_DIR, 'lang': 'en'},\n # 2 expected exceptions\n [\n {'processor_type': 'POSProcessor', 'processors_list': ['pos', 'depparse'], 'provided_reqs': set([]),\n 'requires': set(['tokenize'])},\n {'processor_type': 'DepparseProcessor', 'processors_list': ['pos', 'depparse'],\n 'provided_reqs': set([]), 'requires': set(['tokenize','pos', 'lemma'])}\n ]\n ),\n # no pos when lemma_pos set to True; for english mwt should not be included in the loaded processor list\n (\n # input config\n {'processors': 'tokenize,mwt,lemma', 'dir': TEST_MODELS_DIR, 'lang': 'en', 'lemma_pos': True},\n # 1 expected exception\n [\n {'processor_type': 'LemmaProcessor', 'processors_list': ['tokenize', 'mwt', 'lemma'],\n 'provided_reqs': set(['tokenize', 'mwt']), 'requires': set(['tokenize', 'pos'])}\n ]\n )\n ]\n # try to build each bad config, catch exceptions, check against gold\n pipeline_fails = 0\n for bad_config, gold_exceptions in bad_config_lists:\n try:\n stanza.Pipeline(**bad_config)\n except PipelineRequirementsException as e:\n pipeline_fails += 1\n assert isinstance(e, PipelineRequirementsException)\n assert len(e.processor_req_fails) == len(gold_exceptions)\n for processor_req_e, gold_exception in zip(e.processor_req_fails,gold_exceptions):\n # compare the thrown ProcessorRequirementsExceptions against gold\n check_exception_vals(processor_req_e, gold_exception)\n # check pipeline building failed twice\n assert pipeline_fails == 2\n\n\n"
},
{
"path": "stanza/tests/pipeline/test_tokenizer.py",
"content": "\"\"\"\nBasic testing of tokenization\n\"\"\"\n\nimport pytest\nimport stanza\n\nfrom stanza.tests import *\n\npytestmark = pytest.mark.pipeline\n\nEN_DOC = \"Joe Smith lives in California. Joe's favorite food is pizza. He enjoys going to the beach.\"\nEN_DOC_WITH_EXTRA_WHITESPACE = \"Joe Smith \\n lives in\\n California. Joe's favorite food \\tis pizza. \\t\\t\\tHe enjoys \\t\\tgoing to the beach.\"\nEN_DOC_GOLD_TOKENS = \"\"\"\n]>\n]>\n]>\n]>\n]>\n]>\n\n, ]>\n]>\n]>\n]>\n]>\n]>\n\n]>\n]>\n]>\n]>\n]>\n]>\n]>\n\"\"\".strip()\n\n# spaCy doesn't have MWT\nEN_DOC_SPACY_TOKENS = \"\"\"\n]>\n]>\n]>\n]>\n]>\n]>\n\n]>\n]>\n]>\n]>\n]>\n]>\n]>\n\n]>\n]>\n]>\n]>\n]>\n]>\n]>\n\"\"\".strip()\nEN_DOC_POSTPROCESSOR_TOKENS_LIST = [['Joe', 'Smith', 'lives', 'in', 'California', '.'], [(\"Joe's\", True), 'favorite', 'food', 'is', 'pizza', '.'], ['He', 'enjoys', 'going', 'to', 'the', 'beach', '.']]\nEN_DOC_POSTPROCESSOR_COMBINED_LIST = [['Joe', 'Smith', 'lives', 'in', 'California', '.'], ['Joe', \"'s\", 'favorite', 'food', 'is', 'pizza', '.'], ['He', 'enjoys', 'going', \"to the beach\", '.']]\n\nEN_DOC_POSTPROCESSOR_COMBINED_TOKENS = \"\"\"\n]>\n]>\n]>\n]>\n]>\n]>\n\n]>\n]>\n]>\n]>\n]>\n]>\n]>\n\n]>\n]>\n]>\n]>\n]>\n\"\"\"\n\n# ensure that the entry above has spaces somewhere to test that spaces work in between tokens\n\nEN_DOC_GOLD_NOSSPLIT_TOKENS = \"\"\"\n]>\n]>\n]>\n]>\n]>\n]>\n]>\n]>\n]>\n]>\n]>\n]>\n]>\n]>\n]>\n ![](\"https://github.com/stanfordnlp/stanza/raw/dev/images/stanza-logo.png\"){kind=logo}
![\"Run](\"https://github.com/stanfordnlp/stanza/actions/workflows/stanza-tests.yaml/badge.svg\"){kind=icon}
\n ![\"PyPI](\"https://img.shields.io/pypi/v/stanza?color=blue\")
\n ![\"Conda](\"https://img.shields.io/conda/vn/stanfordnlp/stanza?color=blue&label=conda\")
\n ![\"Python](\"https://img.shields.io/pypi/pyversions/stanza?colorB=blue\")
\n