Repository: ShannonAI/CorefQA Branch: master Commit: b50ff7d0bd1e Files: 45 Total size: 757.5 KB Directory structure: gitextract_ddvr5dia/ ├── .gitignore ├── README.md ├── bert/ │ ├── __init__.py │ ├── modeling.py │ ├── optimization.py │ └── tokenization.py ├── conll-2012/ │ └── scorer/ │ └── v8.01/ │ ├── README.txt │ ├── scorer.bat │ └── scorer.pl ├── data_utils/ │ ├── config_utils.py │ ├── conll.py │ ├── lowercase_vocab.txt │ └── uppercase_vocab.txt ├── func_builders/ │ ├── input_fn_builder.py │ └── model_fn_builder.py ├── logs/ │ └── corefqa_log.txt ├── models/ │ ├── corefqa.py │ └── mention_proposal.py ├── requirements.txt ├── run/ │ ├── build_dataset_to_tfrecord.py │ ├── run_corefqa.py │ ├── run_mention_proposal.py │ ├── run_squad.py │ └── transform_spanbert_pytorch_to_tf.py ├── scripts/ │ ├── data/ │ │ ├── download_pretrained_mlm.sh │ │ ├── generate_tfrecord_dataset.sh │ │ ├── preprocess_ontonotes_annfiles.sh │ │ └── transform_ckpt_pytorch_to_tf.sh │ └── models/ │ ├── corefqa_gpu.sh │ ├── corefqa_tpu.sh │ ├── mention_gpu.sh │ ├── mention_tpu.sh │ ├── quoref_tpu.sh │ └── squad_tpu.sh ├── tests/ │ ├── bitwise_and.py │ ├── construct_label.py │ ├── cumsum.py │ ├── gather.py │ ├── model_fn.py │ ├── tile_repeat.py │ └── tpu_operation.py └── utils/ ├── load_pytorch_to_tf.py ├── metrics.py ├── radam.py └── util.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class sftp-config.json .DS_Store *.py.swp # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ ================================================ FILE: README.md ================================================ # CorefQA: Coreference Resolution as Query-based Span Prediction The repository contains the code of the recent research advances in [Shannon.AI](http://www.shannonai.com). Please post github issues or email xiaoya_li@shannonai.com for relevant questions. **CorefQA: Coreference Resolution as Query-based Span Prediction**
Wei Wu, Fei Wang, Arianna Yuan, Fei Wu and Jiwei Li
In ACL 2020. [paper](https://arxiv.org/abs/1911.01746)
If you find this repo helpful, please cite the following: ```latex @article{wu2019coreference, title={Coreference Resolution as Query-based Span Prediction}, author={Wu, Wei and Wang, Fei and Yuan, Arianna and Wu, Fei and Li, Jiwei}, journal={arXiv preprint arXiv:1911.01746}, year={2019} } ``` ## Contents - [Overview](#overview) - [Hardware Requirements](#hardware-requirements) - [Install Package Dependencies](#install-package-dependencies) - [Data Preprocess](#data-preprocess) - [Download Pretrained MLM](#download-pretrained-mlm) - [Training](#training) - [Finetune the SpanBERT Model on the Combination of Squad and Quoref Datasets](#finetune-the-spanbert-model-on-the-combination-of-squad-and-quoref-datasets) - [Train the CorefQA Model on the CoNLL-2012 Coreference Resolution Task](#train-the-corefqa-model-on-the-conll-2012-coreference-resolution-task) - [Evaluation and Prediction](#evaluation-and-prediction) - [Download the Final CorefQA Model](#download-the-final-corefqa-model) - [Descriptions of Directories](#descriptions-of-directories) - [Acknowledgement](#acknowledgement) - [Useful Materials](#useful-materials) - [Contact](#contact) ## Overview The model introduces +3.5 (83.1) F1 performance boost over previous SOTA coreference models on the CoNLL benchmark. The current codebase is written in Tensorflow. We plan to release the PyTorch version soon. The current code version only supports training on TPUs and testing on GPUs (due to the annoying features of TF and TPUs). You thus have to bear the trouble of transferring all saved checkpoints from TPUs to GPUs for evaluation (we will fix this soon). Please follow the parameter setting in the log directionary to reproduce the performance. | Model | F1 (%) | | -------------- |:------:| | Previous SOTA (Joshi et al., 2019a) | 79.6 | | CorefQA + SpanBERT-large | 83.1 | ## Hardware Requirements TPU for training: Cloud TPU v3-8 device (128G memory) with Tensorflow 1.15 Python 3.5 GPU for evaluation: with CUDA 10.0 Tensorflow 1.15 Python 3.5 ## Install Package Dependencies ```shell $ python3 -m pip install --user virtualenv $ virtualenv --python=python3.5 ~/corefqa_venv $ source ~/corefqa_venv/bin/activate $ cd CorefQA $ pip install -r requirements.txt # If you are using TPU, please run the following commands: $ pip install --upgrade google-api-python-client $ pip install --upgrade oauth2client ``` ## Data Preprocess 1) Download the offical released [Ontonotes 5.0 (LDC2013T19)](https://catalog.ldc.upenn.edu/LDC2013T19).
2) Preprocess Ontonotes5 annotations files for the CoNLL-2012 coreference resolution task.
Run the command with **Python 2** `bash ./scripts/data/preprocess_ontonotes_annfiles.sh `
and it will create `{train/dev/test}.{language}.v4_gold_conll` files in the directory ``.
`` can be `english`, `arabic` or `chinese`. In this paper, we set `` to `english`.
If you want to use **Python 3**, please refer to the [guideline](https://github.com/huggingface/neuralcoref/blob/master/neuralcoref/train/training.md#get-the-data)
3) Generate TFRecord files for experiments.
Run the command with **Python 3** `bash ./scripts/data/generate_tfrecord_dataset.sh ` and it will create `{train/dev/test}.overlap.corefqa.{language}.tfrecord` files in the directory ``.
## Download Pretrained MLM In our experiments, we used pretrained mask language models to initialize the mention_proposal and corefqa models. 1) Download the pretrained models.
Run `bash ./scripts/data/download_pretrained_mlm.sh ` to download and unzip the pretrained mlm models.
`` shoule take the value of `[bert_base, bert_large, spanbert_base, spanbert_large, bert_tiny]`. - `bert_base, bert_large, spanbert_base, spanbert_large` are trained with a cased(upppercase and lowercase tokens) vocabulary. Should use the cased train/dev/test coreference datasets. - `bert_tiny` is trained with a uncased(lowercase tokens) vocabulary. We use the tinyBERT model for fast debugging. Should use the uncased train/dev/test coreference datasets.
2) Transform SpanBERT from `Pytorch` to `Tensorflow`.
After downloading `bert_` to `_tf_dir>` and `spanbert_` to `_pytorch_dir>`, you can start transforming the SpanBERT model to Tensorflow and the model is saved to the directory ``. `` should take the value of `[base, large]`.
We need to tranform the SpanBERT checkpoints from Pytorch to TF because the offical relased models were trained with Pytorch. Run `bash ./scripts/data/transform_ckpt_pytorch_to_tf.sh _pytorch_dir> _tf_dir> ` and the `` in TF will be saved in ``. - `` should take the value of `[spanbert_base, spanbert_large]`. - `` indicates that the `bert_model.ckpt` in the `_tf_dir>` should have the same scale(base, large) to the `bert_model.bin` in `_pytorch_dir>`. ## Training Follow the pipeline described in the paper, you need to:
1) load a pretrained SpanBERT model.
2) finetune the SpanBERT model on the combination of Squad and Quoref datasets.
3) pretrain the mention proposal model on the coref dataset.
4) jointly train the mention proposal model and the mention linking model.
**Notice:** We provide the options of both pretraining these models yourself and loading the our pretrained models for 2) and 3).
### Finetune the SpanBERT Model on the Combination of Squad and Quoref Datasets We finetune the SpanBERT model on the [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) and [Quoref](https://allennlp.org/quoref) QA tasks for data augmentation before the coreference resolution task. 1. You can directly download the pretrained model on the datasets. Download Data Augmentation Models on Squad and Quoref [link](https://www.dropbox.com/s/lqjc6kfe0w34jt0/finetune_spanbert_large_squad2.tar.gz?dl=0)
Run `./scripts/data/download_squad2_finetune_model.sh ` to download finetuned SpanBERT on SQuAD2.0.
The `` should take the value of `[base, large]`.
The `` is the path to save finetuned spanbert on SQuAD2.0 datasets.
2. Or start to finetune the SpanBERT model on QA tasks yourself. - Download SQuAD 2.0 [train](https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json) and [dev](https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json) sets. - Download Quoref [train and dev](https://quoref-dataset.s3-us-west-2.amazonaws.com/train_and_dev/quoref-train-dev-v0.1.zip) sets. - Finetune the SpanBERT model on Google Could V3-8 TPU. For Squad 2.0, Run the script in [./script/model/squad_tpu.sh](https://github.com/ShannonAI/CorefQA/blob/master/scripts/models/squad_tpu.sh) ```bash REPO_PATH=/home/shannon/coref-tf export TPU_NAME=tf-tpu export PYTHONPATH="$PYTHONPATH:$REPO_PATH" SQUAD_DIR=gs://qa_tasks/squad2 BERT_DIR=gs://pretrained_mlm_checkpoint/spanbert_large_tf OUTPUT_DIR=gs://corefqa_output_squad/spanbert_large_squad2_2e-5 python3 ${REPO_PATH}/run/run_squad.py \ --vocab_file=$BERT_DIR/vocab.txt \ --bert_config_file=$BERT_DIR/bert_config.json \ --init_checkpoint=$BERT_DIR/bert_model.ckpt \ --do_train=True \ --train_file=$SQUAD_DIR/train-v2.0.json \ --do_predict=True \ --predict_file=$SQUAD_DIR/dev-v2.0.json \ --train_batch_size=8 \ --learning_rate=2e-5 \ --num_train_epochs=4.0 \ --max_seq_length=384 \ --do_lower_case=False \ --doc_stride=128 \ --output_dir=${OUTPUT_DIR} \ --use_tpu=True \ --tpu_name=$TPU_NAME \ --version_2_with_negative=True ``` After getting the best model (choose based on the performance on dev set) on `SQuAD2.0`, you should start finetuning the saved model on `Quoref`.
Run the script in [./script/model/quoref_tpu.sh](https://github.com/ShannonAI/CorefQA/blob/master/scripts/models/quoref_tpu.sh) ```bash REPO_PATH=/home/shannon/coref-tf export TPU_NAME=tf-tpu export PYTHONPATH="$PYTHONPATH:$REPO_PATH" QUOREF_DIR=gs://qa_tasks/quoref BERT_DIR=gs://corefqa_output_squad/panbert_large_squad2_2e-5 OUTPUT_DIR=gs://corefqa_output_quoref/spanbert_large_squad2_best_quoref_3e-5 python3 ${REPO_PATH}/run_quoref.py \ --vocab_file=$BERT_DIR/vocab.txt \ --bert_config_file=$BERT_DIR/bert_config.json \ --init_checkpoint=$BERT_DIR/best_bert_model.ckpt \ --do_train=True \ --train_file=$QUOREF_DIR/quoref-train-v0.1.json \ --do_predict=True \ --predict_file=$QUOREF_DIR/quoref-dev-v0.1.json \ --train_batch_size=8 \ --learning_rate=3e-5 \ --num_train_epochs=5 \ --max_seq_length=384 \ --do_lower_case=False \ --doc_stride=128 \ --output_dir=${OUTPUT_DIR} \ --use_tpu=True \ --tpu_name=$TPU_NAME ``` We use the best model (choose based on the performance on DEV set) on `Quoref` to initialize the CorefQA Model. ### Train the CorefQA Model on the CoNLL-2012 Coreference Resolution Task 1.1 Your can you can download the pre-trained mention proposal model (including [model](https://storage.googleapis.com/public_model_checkpoints/mention_proposal/model.ckpt-22000.data-00000-of-00001), [meta](https://storage.googleapis.com/public_model_checkpoints/mention_proposal/model.ckpt-22000.meta) and [index](https://storage.googleapis.com/public_model_checkpoints/mention_proposal/model.ckpt-22000.index)). 1.2. Or train the mention proposal model yourself. The script can be found in [./script/model/mention_tpu.sh](https://github.com/ShannonAI/CorefQA/blob/master/scripts/models/mention_tpu.sh). ```bash REPO_PATH=/home/shannon/coref-tf export PYTHONPATH="$PYTHONPATH:$REPO_PATH" export TPU_NAME=tf-tpu export TPU_ZONE=europe-west4-a export GCP_PROJECT=xiaoyli-20-01-4820 BERT_DIR=gs://corefqa_output_quoref/spanbert_large_squad2_best_quoref_1e-5 DATA_DIR=gs://corefqa_data/final_overlap_384_6 OUTPUT_DIR=gs://corefqa_output_mention_proposal/squad_quoref_large_384_6_1e5_8_0.2 python3 ${REPO_PATH}/run/run_mention_proposal.py \ --output_dir=$OUTPUT_DIR \ --bert_config_file=$BERT_DIR/bert_config.json \ --init_checkpoint=$BERT_DIR/bert_model.ckpt \ --vocab_file=$BERT_DIR/vocab.txt \ --logfile_path=$OUTPUT_DIR/train.log \ --num_epochs=8 \ --keep_checkpoint_max=50 \ --save_checkpoints_steps=500 \ --train_file=$DATA_DIR/train.corefqa.english.tfrecord \ --dev_file=$DATA_DIR/dev.corefqa.english.tfrecord \ --test_file=$DATA_DIR/test.corefqa.english.tfrecord \ --do_train=True \ --do_eval=False \ --do_predict=False \ --learning_rate=1e-5 \ --dropout_rate=0.2 \ --mention_threshold=0.5 \ --hidden_size=1024 \ --num_docs=5604 \ --window_size=384 \ --num_window=6 \ --max_num_mention=60 \ --start_end_share=False \ --loss_start_ratio=0.3 \ --loss_end_ratio=0.3 \ --loss_span_ratio=0.3 \ --use_tpu=True \ --tpu_name=$TPU_NAME \ --tpu_zone=$TPU_ZONE \ --gcp_project=$GCP_PROJECT \ --num_tpu_cores=1 \ --seed=2333 ``` 2. Jointly train the mention proposal model and linking model on CoNLL-12.
After getting the best mention proposal model on the dev set, start jointly training the mention proposal and linking tasks. Run and the script can be found in [./script/model/corefqa_tpu.sh](https://github.com/ShannonAI/CorefQA/blob/master/scripts/models/corefqa_tpu.sh) ```bash REPO_PATH=/home/shannon/coref-tf export PYTHONPATH="$PYTHONPATH:$REPO_PATH" export TPU_NAME=tf-tpu export TPU_ZONE=europe-west4-a export GCP_PROJECT=xiaoyli-20-01-4820 BERT_DIR=gs://corefqa_output_mention_proposal/output_bertlarge DATA_DIR=gs://corefqa_data/final_overlap_384_6 OUTPUT_DIR=gs://corefqa_output_corefqa/squad_quoref_mention_large_384_6_8e4_8_0.2 python3 ${REPO_PATH}/run/run_corefqa.py \ --output_dir=$OUTPUT_DIR \ --bert_config_file=$BERT_DIR/bert_config.json \ --init_checkpoint=$BERT_DIR/best_bert_model.ckpt \ --vocab_file=$BERT_DIR/vocab.txt \ --logfile_path=$OUTPUT_DIR/train.log \ --num_epochs=8 \ --keep_checkpoint_max=50 \ --save_checkpoints_steps=500 \ --train_file=$DATA_DIR/train.corefqa.english.tfrecord \ --dev_file=$DATA_DIR/dev.corefqa.english.tfrecord \ --test_file=$DATA_DIR/test.corefqa.english.tfrecord \ --do_train=True \ --do_eval=False \ --do_predict=False \ --learning_rate=8e-4 \ --dropout_rate=0.2 \ --mention_threshold=0.5 \ --hidden_size=1024 \ --num_docs=5604 \ --window_size=384 \ --num_window=6 \ --max_num_mention=50 \ --start_end_share=False \ --max_span_width=10 \ --max_candiate_mentions=100 \ --top_span_ratio=0.2 \ --max_top_antecedents=30 \ --max_query_len=150 \ --max_context_len=150 \ --sec_qa_mention_score=False \ --use_tpu=True \ --tpu_name=$TPU_NAME \ --tpu_zone=$TPU_ZONE \ --gcp_project=$GCP_PROJECT \ --num_tpu_cores=1 \ --seed=2333 ``` ## Evaluation and Prediction Currently, the evaluation is conducted on a set of saved checkpoints after the training process, and DO NOT support evaluation during training. Please transfer all checkpoints (the output directory is set `--output_dir=` when running the `run_.py`) from TPUs to GPUs for evaluation. This can be achieved by downloading the output directory from the Google Cloud Storage.
The performance on the test set is obtained by using the model achieving the highest F1-score on the dev set.
Set `--do_eval=True`、 `--do_train=False` and `--do_predict=False` to `run_.py` and start the evaluation process on a set of saved checkpoints. And other parameters should be the same with the training process. `` should take the value of `[mention_proposal, corefqa]`.
The codebase also provides the option of evaluating a single model/checkpoint. Please set `--do_eval=False`、 `--do_train=False` and `--do_predict=True` to `run_.py` with the checkpoint path `--eval_checkpoint=`. `` should take the value of `[mention_proposal, corefqa]`.
## Download the Final CorefQA Model You can download the final CorefQA model at [link](https://drive.google.com/file/d/1RPYsS2dDxYyii7-3NkBNG7VtuA96NBLf/view?usp=sharing) and follow the instructions in the prediciton to obtain the score reported in the paper. ## Descriptions of Directories Name | Descriptions ----------- | ------------- bert | BERT modules (model,tokenizer,optimization) ref to the `google-research/bert` repository. conll-2012 | offical evaluation scripts for CoNLL2012 shared task. data_utils | modules for processing training data. func_builders | the input dataloader and model constructor for CorefQA. logs | the log files in our experiments. models | an implementation of CorefQA/MentionProposal models based on TF. run | modules for data preparation and training models. scripts/data | scripts for data preparation and loading pretrained models. scripts/models | scripts for {train/evaluate} {mention_proposal/corefqa} models on {TPU/GPU}. utils | modules including metrics、optimizers. ## Acknowledgement Many thanks to `Yuxian Meng` and the previous work `https://github.com/mandarjoshi90/coref`. ## Useful Materials - TPU Quick Start [link](https://cloud.google.com/tpu/docs/quickstart) - TPU Available Operations [link](https://cloud.google.com/tpu/docs/tensorflow-ops) ## Contact Feel free to discuss papers/code with us through issues/emails! ================================================ FILE: bert/__init__.py ================================================ # coding=utf-8 # Copyright 2018 The Google AI Language Team Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: bert/modeling.py ================================================ # coding=utf-8 # Copyright 2018 The Google AI Language Team Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """The main BERT model and related functions.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function import collections import copy import json import math import re import six import tensorflow as tf class BertConfig(object): """Configuration for `BertModel`.""" def __init__(self, vocab_size, hidden_size=768, num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072, hidden_act="gelu", hidden_dropout_prob=0.1, attention_probs_dropout_prob=0.1, max_position_embeddings=512, type_vocab_size=16, initializer_range=0.02): """Constructs BertConfig. Args: vocab_size: Vocabulary size of `inputs_ids` in `BertModel`. hidden_size: Size of the encoder layers and the pooler layer. num_hidden_layers: Number of hidden layers in the Transformer encoder. num_attention_heads: Number of attention heads for each attention layer in the Transformer encoder. intermediate_size: The size of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. hidden_act: The non-linear activation function (function or string) in the encoder and pooler. hidden_dropout_prob: The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. attention_probs_dropout_prob: The dropout ratio for the attention probabilities. max_position_embeddings: The maximum sequence length that this model might ever be used with. Typically set this to something large just in case (e.g., 512 or 1024 or 2048). type_vocab_size: The vocabulary size of the `token_type_ids` passed into `BertModel`. initializer_range: The stdev of the truncated_normal_initializer for initializing all weight matrices. """ self.vocab_size = vocab_size self.hidden_size = hidden_size self.num_hidden_layers = num_hidden_layers self.num_attention_heads = num_attention_heads self.hidden_act = hidden_act self.intermediate_size = intermediate_size self.hidden_dropout_prob = hidden_dropout_prob self.attention_probs_dropout_prob = attention_probs_dropout_prob self.max_position_embeddings = max_position_embeddings self.type_vocab_size = type_vocab_size self.initializer_range = initializer_range @classmethod def from_dict(cls, json_object): """Constructs a `BertConfig` from a Python dictionary of parameters.""" config = BertConfig(vocab_size=None) for (key, value) in six.iteritems(json_object): config.__dict__[key] = value return config @classmethod def from_json_file(cls, json_file): """Constructs a `BertConfig` from a json file of parameters.""" with tf.gfile.GFile(json_file, "r") as reader: text = reader.read() return cls.from_dict(json.loads(text)) def to_dict(self): """Serializes this instance to a Python dictionary.""" output = copy.deepcopy(self.__dict__) return output def to_json_string(self): """Serializes this instance to a JSON string.""" return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" class BertModel(object): """BERT model ("Bidirectional Embedding Representations from a Transformer"). Example usage: ```python # Already been converted into WordPiece token ids input_ids = tf.constant([[31, 51, 99], [15, 5, 0]]) input_mask = tf.constant([[1, 1, 1], [1, 1, 0]]) token_type_ids = tf.constant([[0, 0, 1], [0, 2, 0]]) config = modeling.BertConfig(vocab_size=32000, hidden_size=512, num_hidden_layers=8, num_attention_heads=6, intermediate_size=1024) model = modeling.BertModel(config=config, is_training=True, input_ids=input_ids, input_mask=input_mask, token_type_ids=token_type_ids) label_embeddings = tf.get_variable(...) pooled_output = model.get_pooled_output() logits = tf.matmul(pooled_output, label_embeddings) ... ``` """ def __init__(self, config, is_training, input_ids, input_mask=None, token_type_ids=None, use_one_hot_embeddings=True, scope=None): """Constructor for BertModel. Args: config: `BertConfig` instance. is_training: bool. rue for training model, false for eval model. Controls whether dropout will be applied. input_ids: int32 Tensor of shape [batch_size, seq_length]. input_mask: (optional) int32 Tensor of shape [batch_size, seq_length]. token_type_ids: (optional) int32 Tensor of shape [batch_size, seq_length]. use_one_hot_embeddings: (optional) bool. Whether to use one-hot word embeddings or tf.embedding_lookup() for the word embeddings. On the TPU, it is must faster if this is True, on the CPU or GPU, it is faster if this is False. scope: (optional) variable scope. Defaults to "bert". Raises: ValueError: The config is invalid or one of the input tensor shapes is invalid. """ config = copy.deepcopy(config) config.hidden_dropout_prob = tf.to_float(is_training) * config.hidden_dropout_prob config.attention_probs_dropout_prob = tf.to_float(is_training) * config.attention_probs_dropout_prob # config.hidden_dropout_prob = tf.Print(config.hidden_dropout_prob, [config.hidden_dropout_prob], 'hdden') # if not is_training: # config.hidden_dropout_prob = 0.0 # config.attention_probs_dropout_prob = 0.0 input_shape = get_shape_list(input_ids, expected_rank=2) batch_size = input_shape[0] seq_length = input_shape[1] if input_mask is None: input_mask = tf.ones(shape=[batch_size, seq_length], dtype=tf.int32) if token_type_ids is None: token_type_ids = tf.zeros(shape=[batch_size, seq_length], dtype=tf.int32) with tf.variable_scope(scope, default_name="bert", reuse=tf.AUTO_REUSE): with tf.variable_scope("embeddings"): # Perform embedding lookup on the word ids. (self.embedding_output, self.embedding_table) = embedding_lookup( input_ids=input_ids, vocab_size=config.vocab_size, embedding_size=config.hidden_size, initializer_range=config.initializer_range, word_embedding_name="word_embeddings", use_one_hot_embeddings=use_one_hot_embeddings) # Add positional embeddings and token type embeddings, then layer # normalize and perform dropout. self.embedding_output = embedding_postprocessor( input_tensor=self.embedding_output, use_token_type=True, token_type_ids=token_type_ids, token_type_vocab_size=config.type_vocab_size, token_type_embedding_name="token_type_embeddings", use_position_embeddings=True, position_embedding_name="position_embeddings", initializer_range=config.initializer_range, max_position_embeddings=config.max_position_embeddings, dropout_prob=config.hidden_dropout_prob) with tf.variable_scope("encoder"): # This converts a 2D mask of shape [batch_size, seq_length] to a 3D # mask of shape [batch_size, seq_length, seq_length] which is used # for the attention scores. attention_mask = create_attention_mask_from_input_mask( input_ids, input_mask) # Run the stacked transformer. # `sequence_output` shape = [batch_size, seq_length, hidden_size]. self.all_encoder_layers = transformer_model( input_tensor=self.embedding_output, attention_mask=attention_mask, hidden_size=config.hidden_size, num_hidden_layers=config.num_hidden_layers, num_attention_heads=config.num_attention_heads, intermediate_size=config.intermediate_size, intermediate_act_fn=get_activation(config.hidden_act), hidden_dropout_prob=config.hidden_dropout_prob, attention_probs_dropout_prob=config.attention_probs_dropout_prob, initializer_range=config.initializer_range, do_return_all_layers=True) self.sequence_output = self.all_encoder_layers[-1] # The "pooler" converts the encoded sequence tensor of shape # [batch_size, seq_length, hidden_size] to a tensor of shape # [batch_size, hidden_size]. This is necessary for segment-level # (or segment-pair-level) classification tasks where we need a fixed # dimensional representation of the segment. with tf.variable_scope("pooler"): # We "pool" the model by simply taking the hidden state corresponding # to the first token. We assume that this has been pre-trained first_token_tensor = tf.squeeze(self.sequence_output[:, 0:1, :], axis=1) self.pooled_output = tf.layers.dense( first_token_tensor, config.hidden_size, activation=tf.tanh, kernel_initializer=create_initializer(config.initializer_range)) def get_pooled_output(self): return self.pooled_output def get_sequence_output(self): """Gets final hidden layer of encoder. Returns: float Tensor of shape [batch_size, seq_length, hidden_size] corresponding to the final hidden of the transformer encoder. """ return self.sequence_output def get_all_encoder_layers(self): return self.all_encoder_layers def get_embedding_output(self): """Gets output of the embedding lookup (i.e., input to the transformer). Returns: float Tensor of shape [batch_size, seq_length, hidden_size] corresponding to the output of the embedding layer, after summing the word embeddings with the positional embeddings and the token type embeddings, then performing layer normalization. This is the input to the transformer. """ return self.embedding_output def get_embedding_table(self): return self.embedding_table def gelu(input_tensor): """Gaussian Error Linear Unit. This is a smoother version of the RELU. Original paper: https://arxiv.org/abs/1606.08415 Args: input_tensor: float Tensor to perform activation. Returns: `input_tensor` with the GELU activation applied. """ cdf = 0.5 * (1.0 + tf.erf(input_tensor / tf.sqrt(2.0))) return input_tensor * cdf def get_activation(activation_string): """Maps a string to a Python function, e.g., "relu" => `tf.nn.relu`. Args: activation_string: String name of the activation function. Returns: A Python function corresponding to the activation function. If `activation_string` is None, empty, or "linear", this will return None. If `activation_string` is not a string, it will return `activation_string`. Raises: ValueError: The `activation_string` does not correspond to a known activation. """ # We assume that anything that"s not a string is already an activation # function, so we just return it. if not isinstance(activation_string, six.string_types): return activation_string if not activation_string: return None act = activation_string.lower() if act == "linear": return None elif act == "relu": return tf.nn.relu elif act == "gelu": return gelu elif act == "tanh": return tf.tanh else: raise ValueError("Unsupported activation: %s" % act) def get_assignment_map_from_checkpoint(tvars, init_checkpoint): """Compute the union of the current variables and checkpoint variables.""" initialized_variable_names = {} name_to_variable = collections.OrderedDict() for var in tvars: name = var.name m = re.match("^(.*):\\d+$", name) if m is not None: name = m.group(1) name_to_variable[name] = var init_vars = tf.train.list_variables(init_checkpoint) # checkpoint variables, assignment_map = collections.OrderedDict() for x in init_vars: (name, var) = (x[0], x[1]) if name not in name_to_variable: continue assignment_map[name] = name initialized_variable_names[name] = 1 initialized_variable_names[name + ":0"] = 1 return assignment_map, initialized_variable_names def dropout(input_tensor, dropout_prob): """Perform dropout. Args: input_tensor: float Tensor. dropout_prob: Python float. The probability of dropping out a value (NOT of *keeping* a dimension as in `tf.nn.dropout`). Returns: A version of `input_tensor` with dropout applied. """ if dropout_prob is None or dropout_prob == 0.0: return input_tensor output = tf.nn.dropout(input_tensor, 1.0 - dropout_prob) return output def layer_norm(input_tensor, name=None): """Run layer normalization on the last dimension of the tensor.""" return tf.contrib.layers.layer_norm( inputs=input_tensor, begin_norm_axis=-1, begin_params_axis=-1, scope=name) def layer_norm_and_dropout(input_tensor, dropout_prob, name=None): """Runs layer normalization followed by dropout.""" output_tensor = layer_norm(input_tensor, name) output_tensor = dropout(output_tensor, dropout_prob) return output_tensor def create_initializer(initializer_range=0.02): """Creates a `truncated_normal_initializer` with the given range.""" return tf.truncated_normal_initializer(stddev=initializer_range) def embedding_lookup(input_ids, vocab_size, embedding_size=128, initializer_range=0.02, word_embedding_name="word_embeddings", use_one_hot_embeddings=False): """Looks up words embeddings for id tensor. Args: input_ids: int32 Tensor of shape [batch_size, seq_length] containing word ids. vocab_size: int. Size of the embedding vocabulary. embedding_size: int. Width of the word embeddings. initializer_range: float. Embedding initialization range. word_embedding_name: string. Name of the embedding table. use_one_hot_embeddings: bool. If True, use one-hot method for word embeddings. If False, use `tf.nn.embedding_lookup()`. One hot is better for TPUs. Returns: float Tensor of shape [batch_size, seq_length, embedding_size]. """ # This function assumes that the input is of shape [batch_size, seq_length, # num_inputs]. # # If the input is a 2D tensor of shape [batch_size, seq_length], we # reshape to [batch_size, seq_length, 1]. if input_ids.shape.ndims == 2: input_ids = tf.expand_dims(input_ids, axis=[-1]) embedding_table = tf.get_variable( name=word_embedding_name, shape=[vocab_size, embedding_size], initializer=create_initializer(initializer_range)) if use_one_hot_embeddings: flat_input_ids = tf.reshape(input_ids, [-1]) one_hot_input_ids = tf.one_hot(flat_input_ids, depth=vocab_size) output = tf.matmul(one_hot_input_ids, embedding_table) else: output = tf.nn.embedding_lookup(embedding_table, input_ids) input_shape = get_shape_list(input_ids) output = tf.reshape(output, input_shape[0:-1] + [input_shape[-1] * embedding_size]) return (output, embedding_table) def embedding_postprocessor(input_tensor, use_token_type=False, token_type_ids=None, token_type_vocab_size=16, token_type_embedding_name="token_type_embeddings", use_position_embeddings=True, position_embedding_name="position_embeddings", initializer_range=0.02, max_position_embeddings=512, dropout_prob=0.1): """Performs various post-processing on a word embedding tensor. Args: input_tensor: float Tensor of shape [batch_size, seq_length, embedding_size]. use_token_type: bool. Whether to add embeddings for `token_type_ids`. token_type_ids: (optional) int32 Tensor of shape [batch_size, seq_length]. Must be specified if `use_token_type` is True. token_type_vocab_size: int. The vocabulary size of `token_type_ids`. token_type_embedding_name: string. The name of the embedding table variable for token type ids. use_position_embeddings: bool. Whether to add position embeddings for the position of each token in the sequence. position_embedding_name: string. The name of the embedding table variable for positional embeddings. initializer_range: float. Range of the weight initialization. max_position_embeddings: int. Maximum sequence length that might ever be used with this model. This can be longer than the sequence length of input_tensor, but cannot be shorter. dropout_prob: float. Dropout probability applied to the final output tensor. Returns: float tensor with same shape as `input_tensor`. Raises: ValueError: One of the tensor shapes or input values is invalid. """ input_shape = get_shape_list(input_tensor, expected_rank=3) batch_size = input_shape[0] seq_length = input_shape[1] width = input_shape[2] output = input_tensor if use_token_type: if token_type_ids is None: raise ValueError("`token_type_ids` must be specified if" "`use_token_type` is True.") token_type_table = tf.get_variable( name=token_type_embedding_name, shape=[token_type_vocab_size, width], initializer=create_initializer(initializer_range)) # This vocab will be small so we always do one-hot here, since it is always # faster for a small vocabulary. flat_token_type_ids = tf.reshape(token_type_ids, [-1]) one_hot_ids = tf.one_hot(flat_token_type_ids, depth=token_type_vocab_size) token_type_embeddings = tf.matmul(one_hot_ids, token_type_table) token_type_embeddings = tf.reshape(token_type_embeddings, [batch_size, seq_length, width]) output += token_type_embeddings if use_position_embeddings: assert_op = tf.assert_less_equal(seq_length, max_position_embeddings) with tf.control_dependencies([assert_op]): full_position_embeddings = tf.get_variable( name=position_embedding_name, shape=[max_position_embeddings, width], initializer=create_initializer(initializer_range)) # Since the position embedding table is a learned variable, we create it # using a (long) sequence length `max_position_embeddings`. The actual # sequence length might be shorter than this, for faster training of # tasks that do not have long sequences. # # So `full_position_embeddings` is effectively an embedding table # for position [0, 1, 2, ..., max_position_embeddings-1], and the current # sequence has positions [0, 1, 2, ... seq_length-1], so we can just # perform a slice. position_embeddings = tf.slice(full_position_embeddings, [0, 0], [seq_length, -1]) num_dims = len(output.shape.as_list()) # Only the last two dimensions are relevant (`seq_length` and `width`), so # we broadcast among the first dimensions, which is typically just # the batch size. position_broadcast_shape = [] for _ in range(num_dims - 2): position_broadcast_shape.append(1) position_broadcast_shape.extend([seq_length, width]) position_embeddings = tf.reshape(position_embeddings, position_broadcast_shape) output += position_embeddings output = layer_norm_and_dropout(output, dropout_prob) return output def create_attention_mask_from_input_mask(from_tensor, to_mask): """Create 3D attention mask from a 2D tensor mask. Args: from_tensor: 2D or 3D Tensor of shape [batch_size, from_seq_length, ...]. to_mask: int32 Tensor of shape [batch_size, to_seq_length]. Returns: float Tensor of shape [batch_size, from_seq_length, to_seq_length]. """ from_shape = get_shape_list(from_tensor, expected_rank=[2, 3]) batch_size = from_shape[0] from_seq_length = from_shape[1] to_shape = get_shape_list(to_mask, expected_rank=2) to_seq_length = to_shape[1] to_mask = tf.cast( tf.reshape(to_mask, [batch_size, 1, to_seq_length]), tf.float32) # We don't assume that `from_tensor` is a mask (although it could be). We # don't actually care if we attend *from* padding tokens (only *to* padding) # tokens so we create a tensor of all ones. # # `broadcast_ones` = [batch_size, from_seq_length, 1] broadcast_ones = tf.ones( shape=[batch_size, from_seq_length, 1], dtype=tf.float32) # Here we broadcast along two dimensions to create the mask. mask = broadcast_ones * to_mask return mask def attention_layer(from_tensor, to_tensor, attention_mask=None, num_attention_heads=1, size_per_head=512, query_act=None, key_act=None, value_act=None, attention_probs_dropout_prob=0.0, initializer_range=0.02, do_return_2d_tensor=False, batch_size=None, from_seq_length=None, to_seq_length=None): """Performs multi-headed attention from `from_tensor` to `to_tensor`. This is an implementation of multi-headed attention based on "Attention is all you Need". If `from_tensor` and `to_tensor` are the same, then this is self-attention. Each timestep in `from_tensor` attends to the corresponding sequence in `to_tensor`, and returns a fixed-with vector. This function first projects `from_tensor` into a "query" tensor and `to_tensor` into "key" and "value" tensors. These are (effectively) a list of tensors of length `num_attention_heads`, where each tensor is of shape [batch_size, seq_length, size_per_head]. Then, the query and key tensors are dot-producted and scaled. These are softmaxed to obtain attention probabilities. The value tensors are then interpolated by these probabilities, then concatenated back to a single tensor and returned. In practice, the multi-headed attention are done with transposes and reshapes rather than actual separate tensors. Args: from_tensor: float Tensor of shape [batch_size, from_seq_length, from_width]. to_tensor: float Tensor of shape [batch_size, to_seq_length, to_width]. attention_mask: (optional) int32 Tensor of shape [batch_size, from_seq_length, to_seq_length]. The values should be 1 or 0. The attention scores will effectively be set to -infinity for any positions in the mask that are 0, and will be unchanged for positions that are 1. num_attention_heads: int. Number of attention heads. size_per_head: int. Size of each attention head. query_act: (optional) Activation function for the query transform. key_act: (optional) Activation function for the key transform. value_act: (optional) Activation function for the value transform. attention_probs_dropout_prob: (optional) float. Dropout probability of the attention probabilities. initializer_range: float. Range of the weight initializer. do_return_2d_tensor: bool. If True, the output will be of shape [batch_size * from_seq_length, num_attention_heads * size_per_head]. If False, the output will be of shape [batch_size, from_seq_length, num_attention_heads * size_per_head]. batch_size: (Optional) int. If the input is 2D, this might be the batch size of the 3D version of the `from_tensor` and `to_tensor`. from_seq_length: (Optional) If the input is 2D, this might be the seq length of the 3D version of the `from_tensor`. to_seq_length: (Optional) If the input is 2D, this might be the seq length of the 3D version of the `to_tensor`. Returns: float Tensor of shape [batch_size, from_seq_length, num_attention_heads * size_per_head]. (If `do_return_2d_tensor` is true, this will be of shape [batch_size * from_seq_length, num_attention_heads * size_per_head]). Raises: ValueError: Any of the arguments or tensor shapes are invalid. """ def transpose_for_scores(input_tensor, batch_size, num_attention_heads, seq_length, width): output_tensor = tf.reshape( input_tensor, [batch_size, seq_length, num_attention_heads, width]) output_tensor = tf.transpose(output_tensor, [0, 2, 1, 3]) return output_tensor from_shape = get_shape_list(from_tensor, expected_rank=[2, 3]) to_shape = get_shape_list(to_tensor, expected_rank=[2, 3]) if len(from_shape) != len(to_shape): raise ValueError( "The rank of `from_tensor` must match the rank of `to_tensor`.") if len(from_shape) == 3: batch_size = from_shape[0] from_seq_length = from_shape[1] to_seq_length = to_shape[1] elif len(from_shape) == 2: if (batch_size is None or from_seq_length is None or to_seq_length is None): raise ValueError( "When passing in rank 2 tensors to attention_layer, the values " "for `batch_size`, `from_seq_length`, and `to_seq_length` " "must all be specified.") # Scalar dimensions referenced here: # B = batch size (number of sequences) # F = `from_tensor` sequence length # T = `to_tensor` sequence length # N = `num_attention_heads` # H = `size_per_head` from_tensor_2d = reshape_to_matrix(from_tensor) to_tensor_2d = reshape_to_matrix(to_tensor) # `query_layer` = [B*F, N*H] query_layer = tf.layers.dense( from_tensor_2d, num_attention_heads * size_per_head, activation=query_act, name="query", kernel_initializer=create_initializer(initializer_range)) # `key_layer` = [B*T, N*H] key_layer = tf.layers.dense( to_tensor_2d, num_attention_heads * size_per_head, activation=key_act, name="key", kernel_initializer=create_initializer(initializer_range)) # `value_layer` = [B*T, N*H] value_layer = tf.layers.dense( to_tensor_2d, num_attention_heads * size_per_head, activation=value_act, name="value", kernel_initializer=create_initializer(initializer_range)) # `query_layer` = [B, N, F, H] query_layer = transpose_for_scores(query_layer, batch_size, num_attention_heads, from_seq_length, size_per_head) # `key_layer` = [B, N, T, H] key_layer = transpose_for_scores(key_layer, batch_size, num_attention_heads, to_seq_length, size_per_head) # Take the dot product between "query" and "key" to get the raw # attention scores. # `attention_scores` = [B, N, F, T] attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) attention_scores = tf.multiply(attention_scores, 1.0 / math.sqrt(float(size_per_head))) if attention_mask is not None: # `attention_mask` = [B, 1, F, T] attention_mask = tf.expand_dims(attention_mask, axis=[1]) # Since attention_mask is 1.0 for positions we want to attend and 0.0 for # masked positions, this operation will create a tensor which is 0.0 for # positions we want to attend and -10000.0 for masked positions. adder = (1.0 - tf.cast(attention_mask, tf.float32)) * -10000.0 # Since we are adding it to the raw scores before the softmax, this is # effectively the same as removing these entirely. attention_scores += adder # Normalize the attention scores to probabilities. # `attention_probs` = [B, N, F, T] attention_probs = tf.nn.softmax(attention_scores) # This is actually dropping out entire tokens to attend to, which might # seem a bit unusual, but is taken from the original Transformer paper. attention_probs = dropout(attention_probs, attention_probs_dropout_prob) # `value_layer` = [B, T, N, H] value_layer = tf.reshape( value_layer, [batch_size, to_seq_length, num_attention_heads, size_per_head]) # `value_layer` = [B, N, T, H] value_layer = tf.transpose(value_layer, [0, 2, 1, 3]) # `context_layer` = [B, N, F, H] context_layer = tf.matmul(attention_probs, value_layer) # `context_layer` = [B, F, N, H] context_layer = tf.transpose(context_layer, [0, 2, 1, 3]) if do_return_2d_tensor: # `context_layer` = [B*F, N*V] context_layer = tf.reshape( context_layer, [batch_size * from_seq_length, num_attention_heads * size_per_head]) else: # `context_layer` = [B, F, N*V] context_layer = tf.reshape( context_layer, [batch_size, from_seq_length, num_attention_heads * size_per_head]) return context_layer def transformer_model(input_tensor, attention_mask=None, hidden_size=768, num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072, intermediate_act_fn=gelu, hidden_dropout_prob=0.1, attention_probs_dropout_prob=0.1, initializer_range=0.02, do_return_all_layers=False): """Multi-headed, multi-layer Transformer from "Attention is All You Need". This is almost an exact implementation of the original Transformer encoder. See the original paper: https://arxiv.org/abs/1706.03762 Also see: https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/models/transformer.py Args: input_tensor: float Tensor of shape [batch_size, seq_length, hidden_size]. attention_mask: (optional) int32 Tensor of shape [batch_size, seq_length, seq_length], with 1 for positions that can be attended to and 0 in positions that should not be. hidden_size: int. Hidden size of the Transformer. num_hidden_layers: int. Number of layers (blocks) in the Transformer. num_attention_heads: int. Number of attention heads in the Transformer. intermediate_size: int. The size of the "intermediate" (a.k.a., feed forward) layer. intermediate_act_fn: function. The non-linear activation function to apply to the output of the intermediate/feed-forward layer. hidden_dropout_prob: float. Dropout probability for the hidden layers. attention_probs_dropout_prob: float. Dropout probability of the attention probabilities. initializer_range: float. Range of the initializer (stddev of truncated normal). do_return_all_layers: Whether to also return all layers or just the final layer. Returns: float Tensor of shape [batch_size, seq_length, hidden_size], the final hidden layer of the Transformer. Raises: ValueError: A Tensor shape or parameter is invalid. """ if hidden_size % num_attention_heads != 0: raise ValueError( "The hidden size (%d) is not a multiple of the number of attention " "heads (%d)" % (hidden_size, num_attention_heads)) attention_head_size = int(hidden_size / num_attention_heads) input_shape = get_shape_list(input_tensor, expected_rank=3) batch_size = input_shape[0] seq_length = input_shape[1] input_width = input_shape[2] # The Transformer performs sum residuals on all layers so the input needs # to be the same as the hidden size. if input_width != hidden_size: raise ValueError("The width of the input tensor (%d) != hidden size (%d)" % (input_width, hidden_size)) # We keep the representation as a 2D tensor to avoid re-shaping it back and # forth from a 3D tensor to a 2D tensor. Re-shapes are normally free on # the GPU/CPU but may not be free on the TPU, so we want to minimize them to # help the optimizer. prev_output = reshape_to_matrix(input_tensor) all_layer_outputs = [] for layer_idx in range(num_hidden_layers): with tf.variable_scope("layer_%d" % layer_idx): layer_input = prev_output with tf.variable_scope("attention"): attention_heads = [] with tf.variable_scope("self"): attention_head = attention_layer( from_tensor=layer_input, to_tensor=layer_input, attention_mask=attention_mask, num_attention_heads=num_attention_heads, size_per_head=attention_head_size, attention_probs_dropout_prob=attention_probs_dropout_prob, initializer_range=initializer_range, do_return_2d_tensor=True, batch_size=batch_size, from_seq_length=seq_length, to_seq_length=seq_length) attention_heads.append(attention_head) attention_output = None if len(attention_heads) == 1: attention_output = attention_heads[0] else: # In the case where we have other sequences, we just concatenate # them to the self-attention head before the projection. attention_output = tf.concat(attention_heads, axis=-1) # Run a linear projection of `hidden_size` then add a residual # with `layer_input`. with tf.variable_scope("output"): attention_output = tf.layers.dense( attention_output, hidden_size, kernel_initializer=create_initializer(initializer_range)) attention_output = dropout(attention_output, hidden_dropout_prob) attention_output = layer_norm(attention_output + layer_input) # The activation is only applied to the "intermediate" hidden layer. with tf.variable_scope("intermediate"): intermediate_output = tf.layers.dense( attention_output, intermediate_size, activation=intermediate_act_fn, kernel_initializer=create_initializer(initializer_range)) # Down-project back to `hidden_size` then add the residual. with tf.variable_scope("output"): layer_output = tf.layers.dense( intermediate_output, hidden_size, kernel_initializer=create_initializer(initializer_range)) layer_output = dropout(layer_output, hidden_dropout_prob) layer_output = layer_norm(layer_output + attention_output) prev_output = layer_output all_layer_outputs.append(layer_output) if do_return_all_layers: final_outputs = [] for layer_output in all_layer_outputs: final_output = reshape_from_matrix(layer_output, input_shape) final_outputs.append(final_output) return final_outputs else: final_output = reshape_from_matrix(prev_output, input_shape) return final_output def get_shape_list(tensor, expected_rank=None, name=None): """Returns a list of the shape of tensor, preferring static dimensions. Args: tensor: A tf.Tensor object to find the shape of. expected_rank: (optional) int. The expected rank of `tensor`. If this is specified and the `tensor` has a different rank, and exception will be thrown. name: Optional name of the tensor for the error message. Returns: A list of dimensions of the shape of tensor. All static dimensions will be returned as python integers, and dynamic dimensions will be returned as tf.Tensor scalars. """ if name is None: name = tensor.name if expected_rank is not None: assert_rank(tensor, expected_rank, name) shape = tensor.shape.as_list() non_static_indexes = [] for (index, dim) in enumerate(shape): if dim is None: non_static_indexes.append(index) if not non_static_indexes: return shape dyn_shape = tf.shape(tensor) for index in non_static_indexes: shape[index] = dyn_shape[index] return shape def reshape_to_matrix(input_tensor): """Reshapes a >= rank 2 tensor to a rank 2 tensor (i.e., a matrix).""" ndims = input_tensor.shape.ndims if ndims < 2: raise ValueError("Input tensor must have at least rank 2. Shape = %s" % (input_tensor.shape)) if ndims == 2: return input_tensor width = input_tensor.shape[-1] output_tensor = tf.reshape(input_tensor, [-1, width]) return output_tensor def reshape_from_matrix(output_tensor, orig_shape_list): """Reshapes a rank 2 tensor back to its original rank >= 2 tensor.""" if len(orig_shape_list) == 2: return output_tensor output_shape = get_shape_list(output_tensor) orig_dims = orig_shape_list[0:-1] width = output_shape[-1] return tf.reshape(output_tensor, orig_dims + [width]) def assert_rank(tensor, expected_rank, name=None): """Raises an exception if the tensor rank is not of the expected rank. Args: tensor: A tf.Tensor to check the rank of. expected_rank: Python integer or list of integers, expected rank. name: Optional name of the tensor for the error message. Raises: ValueError: If the expected shape doesn't match the actual shape. """ if name is None: name = tensor.name expected_rank_dict = {} if isinstance(expected_rank, six.integer_types): expected_rank_dict[expected_rank] = True else: for x in expected_rank: expected_rank_dict[x] = True actual_rank = tensor.shape.ndims if actual_rank not in expected_rank_dict: scope_name = tf.get_variable_scope().name raise ValueError( "For the tensor `%s` in scope `%s`, the actual rank " "`%d` (shape = %s) is not equal to the expected rank `%s`" % (name, scope_name, actual_rank, str(tensor.shape), str(expected_rank))) ================================================ FILE: bert/optimization.py ================================================ # coding=utf-8 # Copyright 2018 The Google AI Language Team Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Functions and classes related to optimization (weight updates).""" from __future__ import absolute_import from __future__ import division from __future__ import print_function import re import tensorflow as tf def create_optimizer(loss, init_lr, num_train_steps, num_warmup_steps, use_tpu): """Creates an optimizer training op.""" global_step = tf.train.get_or_create_global_step() learning_rate = tf.constant(value=init_lr, shape=[], dtype=tf.float32) # Implements linear decay of the learning rate. learning_rate = tf.train.polynomial_decay( learning_rate, global_step, num_train_steps, end_learning_rate=0.0, power=1.0, cycle=False) # Implements linear warmup. I.e., if global_step < num_warmup_steps, the # learning rate will be `global_step/num_warmup_steps * init_lr`. if num_warmup_steps: global_steps_int = tf.cast(global_step, tf.int32) warmup_steps_int = tf.constant(num_warmup_steps, dtype=tf.int32) global_steps_float = tf.cast(global_steps_int, tf.float32) warmup_steps_float = tf.cast(warmup_steps_int, tf.float32) warmup_percent_done = global_steps_float / warmup_steps_float warmup_learning_rate = init_lr * warmup_percent_done is_warmup = tf.cast(global_steps_int < warmup_steps_int, tf.float32) learning_rate = ( (1.0 - is_warmup) * learning_rate + is_warmup * warmup_learning_rate) # It is recommended that you use this optimizer for fine tuning, since this # is how the model was trained (note that the Adam m/v variables are NOT # loaded from init_checkpoint.) optimizer = AdamWeightDecayOptimizer( learning_rate=learning_rate, weight_decay_rate=0.01, beta_1=0.9, beta_2=0.999, epsilon=1e-6, exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"]) if use_tpu: optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer) tvars = tf.trainable_variables() grads = tf.gradients(loss, tvars) # This is how the model was pre-trained. (grads, _) = tf.clip_by_global_norm(grads, clip_norm=1.0) train_op = optimizer.apply_gradients( zip(grads, tvars), global_step=global_step) # Normally the global step update is done inside of `apply_gradients`. # However, `AdamWeightDecayOptimizer` doesn't do this. But if you use # a different optimizer, you should probably take this line out. new_global_step = global_step + 1 train_op = tf.group(train_op, [global_step.assign(new_global_step)]) return train_op class AdamWeightDecayOptimizer(tf.train.Optimizer): """A basic Adam optimizer that includes "correct" L2 weight decay.""" def __init__(self, learning_rate, weight_decay_rate=0.0, beta_1=0.9, beta_2=0.999, epsilon=1e-6, exclude_from_weight_decay=None, name="AdamWeightDecayOptimizer"): """Constructs a AdamWeightDecayOptimizer.""" super(AdamWeightDecayOptimizer, self).__init__(False, name) self.learning_rate = learning_rate self.weight_decay_rate = weight_decay_rate self.beta_1 = beta_1 self.beta_2 = beta_2 self.epsilon = epsilon self.exclude_from_weight_decay = exclude_from_weight_decay def apply_gradients(self, grads_and_vars, global_step=None, name=None): """See base class.""" assignments = [] for (grad, param) in grads_and_vars: if grad is None or param is None: continue param_name = self._get_variable_name(param.name) m = tf.get_variable( name=param_name + "/adam_m", shape=param.shape.as_list(), dtype=tf.float32, trainable=False, initializer=tf.zeros_initializer()) v = tf.get_variable( name=param_name + "/adam_v", shape=param.shape.as_list(), dtype=tf.float32, trainable=False, initializer=tf.zeros_initializer()) # Standard Adam update. next_m = ( tf.multiply(self.beta_1, m) + tf.multiply(1.0 - self.beta_1, grad)) next_v = ( tf.multiply(self.beta_2, v) + tf.multiply(1.0 - self.beta_2, tf.square(grad))) update = next_m / (tf.sqrt(next_v) + self.epsilon) # Just adding the square of the weights to the loss function is *not* # the correct way of using L2 regularization/weight decay with Adam, # since that will interact with the m and v parameters in strange ways. # # Instead we want ot decay the weights in a manner that doesn't interact # with the m/v parameters. This is equivalent to adding the square # of the weights to the loss with plain (non-momentum) SGD. if self._do_use_weight_decay(param_name): update += self.weight_decay_rate * param update_with_lr = self.learning_rate * update next_param = param - update_with_lr assignments.extend( [param.assign(next_param), m.assign(next_m), v.assign(next_v)]) return tf.group(*assignments, name=name) def _do_use_weight_decay(self, param_name): """Whether to use L2 weight decay for `param_name`.""" if not self.weight_decay_rate: return False if self.exclude_from_weight_decay: for r in self.exclude_from_weight_decay: if re.search(r, param_name) is not None: return False return True def _get_variable_name(self, param_name): """Get the variable name from the tensor name.""" m = re.match("^(.*):\\d+$", param_name) if m is not None: param_name = m.group(1) return param_name ================================================ FILE: bert/tokenization.py ================================================ # coding=utf-8 # Copyright 2018 The Google AI Language Team Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tokenization classes.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function import collections import re import unicodedata import six import tensorflow as tf def validate_case_matches_checkpoint(do_lower_case, init_checkpoint): """Checks whether the casing config is consistent with the checkpoint name.""" # The casing has to be passed in by the user and there is no explicit check # as to whether it matches the checkpoint. The casing information probably # should have been stored in the bert_config.json file, but it's not, so # we have to heuristically detect it to validate. if not init_checkpoint: return m = re.match("^.*?([A-Za-z0-9_-]+)/bert_model.ckpt", init_checkpoint) if m is None: return model_name = m.group(1) lower_models = [ "uncased_L-24_H-1024_A-16", "uncased_L-12_H-768_A-12", "multilingual_L-12_H-768_A-12", "chinese_L-12_H-768_A-12" ] cased_models = [ "cased_L-12_H-768_A-12", "cased_L-24_H-1024_A-16", "multi_cased_L-12_H-768_A-12" ] is_bad_config = False if model_name in lower_models and not do_lower_case: is_bad_config = True actual_flag = "False" case_name = "lowercased" opposite_flag = "True" if model_name in cased_models and do_lower_case: is_bad_config = True actual_flag = "True" case_name = "cased" opposite_flag = "False" if is_bad_config: raise ValueError( "You passed in `--do_lower_case=%s` with `--init_checkpoint=%s`. " "However, `%s` seems to be a %s model, so you " "should pass in `--do_lower_case=%s` so that the fine-tuning matches " "how the model was pre-training. If this error is wrong, please " "just comment out this check." % (actual_flag, init_checkpoint, model_name, case_name, opposite_flag)) def convert_to_unicode(text): """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" if six.PY3: if isinstance(text, str): return text elif isinstance(text, bytes): return text.decode("utf-8", "ignore") else: raise ValueError("Unsupported string type: %s" % (type(text))) elif six.PY2: if isinstance(text, str): return text.decode("utf-8", "ignore") elif isinstance(text, unicode): return text else: raise ValueError("Unsupported string type: %s" % (type(text))) else: raise ValueError("Not running on Python2 or Python 3?") def printable_text(text): """Returns text encoded in a way suitable for print or `tf.logging`.""" # These functions want `str` for both Python2 and Python3, but in one case # it's a Unicode string and in the other it's a byte string. if six.PY3: if isinstance(text, str): return text elif isinstance(text, bytes): return text.decode("utf-8", "ignore") else: raise ValueError("Unsupported string type: %s" % (type(text))) elif six.PY2: if isinstance(text, str): return text elif isinstance(text, unicode): return text.encode("utf-8") else: raise ValueError("Unsupported string type: %s" % (type(text))) else: raise ValueError("Not running on Python2 or Python 3?") def load_vocab(vocab_file): """Loads a vocabulary file into a dictionary.""" vocab = collections.OrderedDict() index = 0 with tf.gfile.GFile(vocab_file, "r") as reader: while True: token = convert_to_unicode(reader.readline()) if not token: break token = token.strip() vocab[token] = index index += 1 return vocab def convert_by_vocab(vocab, items): """Converts a sequence of [tokens|ids] using the vocab.""" output = [] for item in items: output.append(vocab[item]) return output def convert_tokens_to_ids(vocab, tokens): return convert_by_vocab(vocab, tokens) def convert_ids_to_tokens(inv_vocab, ids): return convert_by_vocab(inv_vocab, ids) def whitespace_tokenize(text): """Runs basic whitespace cleaning and splitting on a piece of text.""" text = text.strip() if not text: return [] tokens = text.split() return tokens class FullTokenizer(object): """Runs end-to-end tokenziation.""" def __init__(self, vocab_file, do_lower_case=True): self.vocab = load_vocab(vocab_file) self.inv_vocab = {v: k for k, v in self.vocab.items()} self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) def tokenize(self, text): split_tokens = [] for token in self.basic_tokenizer.tokenize(text): for sub_token in self.wordpiece_tokenizer.tokenize(token): split_tokens.append(sub_token) return split_tokens def convert_tokens_to_ids(self, tokens): return convert_by_vocab(self.vocab, tokens) def convert_ids_to_tokens(self, ids): return convert_by_vocab(self.inv_vocab, ids) class BasicTokenizer(object): """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" def __init__(self, do_lower_case=True): """Constructs a BasicTokenizer. Args: do_lower_case: Whether to lower case the input. """ self.do_lower_case = do_lower_case def tokenize(self, text): """Tokenizes a piece of text.""" text = convert_to_unicode(text) text = self._clean_text(text) # This was added on November 1st, 2018 for the multilingual and Chinese # models. This is also applied to the English models now, but it doesn't # matter since the English models were not trained on any Chinese data # and generally don't have any Chinese data in them (there are Chinese # characters in the vocabulary because Wikipedia does have some Chinese # words in the English Wikipedia.). text = self._tokenize_chinese_chars(text) orig_tokens = whitespace_tokenize(text) split_tokens = [] for token in orig_tokens: if self.do_lower_case: token = token.lower() token = self._run_strip_accents(token) split_tokens.extend(self._run_split_on_punc(token)) output_tokens = whitespace_tokenize(" ".join(split_tokens)) return output_tokens def _run_strip_accents(self, text): """Strips accents from a piece of text.""" text = unicodedata.normalize("NFD", text) output = [] for char in text: cat = unicodedata.category(char) if cat == "Mn": continue output.append(char) return "".join(output) def _run_split_on_punc(self, text): """Splits punctuation on a piece of text.""" chars = list(text) i = 0 start_new_word = True output = [] while i < len(chars): char = chars[i] if _is_punctuation(char): output.append([char]) start_new_word = True else: if start_new_word: output.append([]) start_new_word = False output[-1].append(char) i += 1 return ["".join(x) for x in output] def _tokenize_chinese_chars(self, text): """Adds whitespace around any CJK character.""" output = [] for char in text: cp = ord(char) if self._is_chinese_char(cp): output.append(" ") output.append(char) output.append(" ") else: output.append(char) return "".join(output) def _is_chinese_char(self, cp): """Checks whether CP is the codepoint of a CJK character.""" # This defines a "chinese character" as anything in the CJK Unicode block: # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) # # Note that the CJK Unicode block is NOT all Japanese and Korean characters, # despite its name. The modern Korean Hangul alphabet is a different block, # as is Japanese Hiragana and Katakana. Those alphabets are used to write # space-separated words, so they are not treated specially and handled # like the all of the other languages. if ((cp >= 0x4E00 and cp <= 0x9FFF) or # (cp >= 0x3400 and cp <= 0x4DBF) or # (cp >= 0x20000 and cp <= 0x2A6DF) or # (cp >= 0x2A700 and cp <= 0x2B73F) or # (cp >= 0x2B740 and cp <= 0x2B81F) or # (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) or # (cp >= 0x2F800 and cp <= 0x2FA1F)): # return True return False def _clean_text(self, text): """Performs invalid character removal and whitespace cleanup on text.""" output = [] for char in text: cp = ord(char) if cp == 0 or cp == 0xfffd or _is_control(char): continue if _is_whitespace(char): output.append(" ") else: output.append(char) return "".join(output) class WordpieceTokenizer(object): """Runs WordPiece tokenziation.""" def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=200): self.vocab = vocab self.unk_token = unk_token self.max_input_chars_per_word = max_input_chars_per_word def tokenize(self, text): """Tokenizes a piece of text into its word pieces. This uses a greedy longest-match-first algorithm to perform tokenization using the given vocabulary. For example: input = "unaffable" output = ["un", "##aff", "##able"] Args: text: A single token or whitespace separated tokens. This should have already been passed through `BasicTokenizer. Returns: A list of wordpiece tokens. """ text = convert_to_unicode(text) output_tokens = [] for token in whitespace_tokenize(text): chars = list(token) if len(chars) > self.max_input_chars_per_word: output_tokens.append(self.unk_token) continue is_bad = False start = 0 sub_tokens = [] while start < len(chars): end = len(chars) cur_substr = None while start < end: substr = "".join(chars[start:end]) if start > 0: substr = "##" + substr if substr in self.vocab: cur_substr = substr break end -= 1 if cur_substr is None: is_bad = True break sub_tokens.append(cur_substr) start = end if is_bad: output_tokens.append(self.unk_token) else: output_tokens.extend(sub_tokens) return output_tokens def _is_whitespace(char): """Checks whether `chars` is a whitespace character.""" # \t, \n, and \r are technically contorl characters but we treat them # as whitespace since they are generally considered as such. if char == " " or char == "\t" or char == "\n" or char == "\r": return True cat = unicodedata.category(char) if cat == "Zs": return True return False def _is_control(char): """Checks whether `chars` is a control character.""" # These are technically control characters but we count them as whitespace # characters. if char == "\t" or char == "\n" or char == "\r": return False cat = unicodedata.category(char) if cat.startswith("C"): return True return False def _is_punctuation(char): """Checks whether `chars` is a punctuation character.""" cp = ord(char) # We treat all non-letter/number ASCII as punctuation. # Characters such as "^", "$", and "`" are not in the Unicode # Punctuation class but we treat them as punctuation anyways, for # consistency. if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): return True cat = unicodedata.category(char) if cat.startswith("P"): return True return False ================================================ FILE: conll-2012/scorer/v8.01/README.txt ================================================ NAME CorScorer: Perl package for scoring coreference resolution systems using different metrics. VERSION v8.01 -- reference implementations of MUC, B-cubed, CEAF and BLANC metrics. CHANGES SINCE v8.0 - fixed a bug that crashed the BLANC scorer when a duplicate singleton mention was present in the response. INSTALLATION Requirements: 1. Perl: downloadable from http://perl.org 2. Algorithm-Munkres: included in this package and downloadable from CPAN http://search.cpan.org/~tpederse/Algorithm-Munkres-0.08 USE This package is distributed with two scripts to execute the scorer from the command line. Windows (tm): scorer.bat Linux: scorer.pl SYNOPSIS use CorScorer; $metric = 'ceafm'; # Scores the whole dataset &CorScorer::Score($metric, $keys_file, $response_file); # Scores one file &CorScorer::Score($metric, $keys_file, $response_file, $name); INPUT metric: the metric desired to score the results: muc: MUCScorer (Vilain et al, 1995) bcub: B-Cubed (Bagga and Baldwin, 1998) ceafm: CEAF (Luo et al., 2005) using mention-based similarity ceafe: CEAF (Luo et al., 2005) using entity-based similarity blanc: BLANC (Luo et al., 2014) BLANC metric for gold and predicted mentions all: uses all the metrics to score keys_file: file with expected coreference chains in CoNLL-2011/2012 format response_file: file with output of coreference system (CoNLL-2011/2012 format) name: [optional] the name of the document to score. If name is not given, all the documents in the dataset will be scored. If given name is "none" then all the documents are scored but only total results are shown. OUTPUT The score subroutine returns an array with four values in this order: 1) Recall numerator 2) Recall denominator 3) Precision numerator 4) Precision denominator Also recall, precision and F1 are printed in the standard output when variable $VERBOSE is not null. Final scores: Recall = recall_numerator / recall_denominator Precision = precision_numerator / precision_denominator F1 = 2 * Recall * Precision / (Recall + Precision) Identification of mentions An scorer for identification of mentions (recall, precision and F1) is also included. Mentions from system response are compared with key mentions. This version performs strict mention matching as was used in the CoNLL-2011 and 2012 shared tasks. AUTHORS Emili Sapena, Universitat Politècnica de Catalunya, http://www.lsi.upc.edu/~esapena, esapena lsi.upc.edu Sameer Pradhan, sameer.pradhan childrens.harvard.edu Sebastian Martschat, sebastian.martschat h-its.org Xiaoqiang Luo, xql google.com COPYRIGHT AND LICENSE Copyright (C) 2009-2011, Emili Sapena esapena lsi.upc.edu 2011-2014, Sameer Pradhan sameer.pradhan childrens.harvard.edu This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. ================================================ FILE: conll-2012/scorer/v8.01/scorer.bat ================================================ @rem = '--*-Perl-*-- @echo off if "%OS%" == "Windows_NT" goto WinNT perl -x -S "%0" %1 %2 %3 %4 %5 %6 %7 %8 %9 goto endofperl :WinNT perl -x -S %0 %* if NOT "%COMSPEC%" == "%SystemRoot%\system32\cmd.exe" goto endofperl if %errorlevel% == 9009 echo You do not have Perl in your PATH. if errorlevel 1 goto script_failed_so_exit_with_non_zero_val 2>nul goto endofperl @rem '; #!perl #line 15 BEGIN { $d = $0; $d =~ s/\/[^\/][^\/]*$//g; push(@INC, $d."/lib"); } use strict; use CorScorer; if (@ARGV < 3) { print q| use: scorer.bat [name] metric: the metric desired to score the results: muc: MUCScorer (Vilain et al, 1995) bcub: B-Cubed (Bagga and Baldwin, 1998) ceafm: CEAF (Luo et al, 2005) using mention-based similarity ceafe: CEAF (Luo et al, 2005) using entity-based similarity all: uses all the metrics to score keys_file: file with expected coreference chains in SemEval format response_file: file with output of coreference system (SemEval format) name: [optional] the name of the document to score. If name is not given, all the documents in the dataset will be scored. If given name is "none" then all the documents are scored but only total results are shown. |; exit; } my $metric = shift (@ARGV); if ($metric !~ /^(muc|bcub|ceafm|ceafe|all)/i) { print "Invalid metric\n"; exit; } if ($metric eq 'all') { foreach my $m ('muc', 'bcub', 'ceafm', 'ceafe') { print "\nMETRIC $m:\n"; &CorScorer::Score( $m, @ARGV ); } } else { &CorScorer::Score( $metric, @ARGV ); } __END__ :endofperl ================================================ FILE: conll-2012/scorer/v8.01/scorer.pl ================================================ #!/usr/bin/perl BEGIN { $d = $0; $d =~ s/\/[^\/][^\/]*$//g; if ($d eq $0) { unshift(@INC, "lib"); } else { unshift(@INC, $d . "/lib"); } } use strict; use CorScorer; if (@ARGV < 3) { print q| use: scorer.pl [name] metric: the metric desired to score the results: muc: MUCScorer (Vilain et al, 1995) bcub: B-Cubed (Bagga and Baldwin, 1998) ceafm: CEAF (Luo et al, 2005) using mention-based similarity ceafe: CEAF (Luo et al, 2005) using entity-based similarity blanc: BLANC all: uses all the metrics to score keys_file: file with expected coreference chains in SemEval format response_file: file with output of coreference system (SemEval format) name: [optional] the name of the document to score. If name is not given, all the documents in the dataset will be scored. If given name is "none" then all the documents are scored but only total results are shown. |; exit; } my $metric = shift(@ARGV); if ($metric !~ /^(muc|bcub|ceafm|ceafe|blanc|all)/i) { print "Invalid metric\n"; exit; } if ($metric eq 'all') { foreach my $m ('muc', 'bcub', 'ceafm', 'ceafe', 'blanc') { print "\nMETRIC $m:\n"; &CorScorer::Score($m, @ARGV); } } else { &CorScorer::Score($metric, @ARGV); } ================================================ FILE: data_utils/config_utils.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # author: xiaoy li # description: # config utils for the mention proposal and corefqa import os import json import tensorflow as tf class ModelConfig(object): def __init__(self, tf_flags, output_dir, model_sign="model"): key_value_pairs = tf_flags.flag_values_dict() for item_key, item_value in key_value_pairs.items(): self.__dict__[item_key] = item_value self.output_dir = output_dir config_path = os.path.join(self.output_dir, "{}_config.json".format(model_sign)) def logging_configs(self): tf.logging.info("$*$"*30) tf.logging.info("****** print model configs : ******") tf.logging.info("$*$"*30) for item_key, item_value in self.__dict__.items(): tf.logging.info("{} : {}".format(str(item_key), str(item_value))) ================================================ FILE: data_utils/conll.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # author: xiaoy li # description: # use the offical conll-2012 evaluation scripts to evaluate the trained model and save the evaluation results into files. import os import re import collections import operator import subprocess import tempfile REPO_PATH = "/".join(os.path.realpath(__file__).split("/")[:-2]) BEGIN_DOCUMENT_REGEX = re.compile(r"#begin document \((.*)\); part (\d+)") COREF_RESULTS_REGEX = re.compile( r".*Coreference: Recall: \([0-9.]+ / [0-9.]+\) ([0-9.]+)%\tPrecision: \([0-9.]+ / [0-9.]+\) ([0-9.]+)%\tF1: ([0-9.]+)%.*", re.DOTALL) def get_doc_key(doc_id, part): return "{}_{}".format(doc_id, int(part)) def output_conll(input_file, output_file, predictions, subtoken_map): prediction_map = {} for doc_key, clusters in predictions.items(): start_map = collections.defaultdict(list) end_map = collections.defaultdict(list) word_map = collections.defaultdict(list) for cluster_id, mentions in enumerate(clusters): for start, end in mentions: start, end = subtoken_map[doc_key][start], subtoken_map[doc_key][end] if start == end: word_map[start].append(cluster_id) else: start_map[start].append((cluster_id, end)) end_map[end].append((cluster_id, start)) for k, v in start_map.items(): start_map[k] = [cluster_id for cluster_id, end in sorted(v, key=operator.itemgetter(1), reverse=True)] for k, v in end_map.items(): end_map[k] = [cluster_id for cluster_id, start in sorted(v, key=operator.itemgetter(1), reverse=True)] prediction_map[doc_key] = (start_map, end_map, word_map) word_index = 0 for line in input_file.readlines(): row = line.split() if len(row) == 0: output_file.write("\n") elif row[0].startswith("#"): begin_match = re.match(BEGIN_DOCUMENT_REGEX, line) if begin_match: doc_key = get_doc_key(begin_match.group(1), begin_match.group(2)) start_map, end_map, word_map = prediction_map[doc_key] word_index = 0 output_file.write(line) output_file.write("\n") else: assert get_doc_key(row[0], row[1]) == doc_key coref_list = [] if word_index in end_map: for cluster_id in end_map[word_index]: coref_list.append("{})".format(cluster_id)) if word_index in word_map: for cluster_id in word_map[word_index]: coref_list.append("({})".format(cluster_id)) if word_index in start_map: for cluster_id in start_map[word_index]: coref_list.append("({}".format(cluster_id)) if len(coref_list) == 0: row[-1] = "-" else: row[-1] = "|".join(coref_list) output_file.write(" ".join(row)) output_file.write("\n") word_index += 1 def official_conll_eval(gold_path, predicted_path, metric, official_stdout=False): cmd = ["perl", os.path.join(REPO_PATH, "conll-2012/scorer/v8.01/scorer.pl"), metric, gold_path, predicted_path, "none"] process = subprocess.Popen(cmd, stdout=subprocess.PIPE) stdout, stderr = process.communicate() process.wait() stdout = stdout.decode("utf-8") if stderr is not None: print(stderr) if official_stdout: print("Official result for {}".format(metric)) print(stdout) coref_results_match = re.match(COREF_RESULTS_REGEX, stdout) recall = float(coref_results_match.group(1)) precision = float(coref_results_match.group(2)) f1 = float(coref_results_match.group(3)) return {"r": recall, "p": precision, "f": f1} def evaluate_conll(gold_path, predictions, subtoken_maps, official_stdout=False): with tempfile.NamedTemporaryFile(delete=False, mode="w") as prediction_file: with open(gold_path, "r") as gold_file: output_conll(gold_file, prediction_file, predictions, subtoken_maps) print("Predicted conll file: {}".format(prediction_file.name)) return {m: official_conll_eval(gold_file.name, prediction_file.name, m, official_stdout) for m in ("muc", "bcub", "ceafe")} ================================================ FILE: data_utils/lowercase_vocab.txt ================================================ [PAD] [unused0] [unused1] [unused2] [unused3] [unused4] [unused5] [unused6] [unused7] [unused8] [unused9] [unused10] [unused11] [unused12] [unused13] [unused14] [unused15] [unused16] [unused17] [unused18] [unused19] [unused20] [unused21] [unused22] [unused23] [unused24] [unused25] [unused26] [unused27] [unused28] [unused29] [unused30] [unused31] [unused32] [unused33] [unused34] [unused35] [unused36] [unused37] [unused38] [unused39] [unused40] [unused41] [unused42] [unused43] [unused44] [unused45] [unused46] [unused47] [unused48] [unused49] [unused50] [unused51] [unused52] [unused53] [unused54] [unused55] [unused56] [unused57] [unused58] [unused59] [unused60] [unused61] [unused62] [unused63] [unused64] [unused65] [unused66] [unused67] [unused68] [unused69] [unused70] [unused71] [unused72] [unused73] [unused74] [unused75] [unused76] [unused77] [unused78] [unused79] [unused80] [unused81] [unused82] [unused83] [unused84] [unused85] [unused86] [unused87] [unused88] [unused89] [unused90] [unused91] [unused92] [unused93] [unused94] [unused95] [unused96] [unused97] [unused98] [UNK] [CLS] [SEP] [MASK] [unused99] [unused100] [unused101] [unused102] [unused103] [unused104] [unused105] [unused106] [unused107] [unused108] [unused109] [unused110] [unused111] [unused112] [unused113] [unused114] [unused115] [unused116] [unused117] [unused118] [unused119] [unused120] [unused121] [unused122] [unused123] [unused124] [unused125] [unused126] [unused127] [unused128] [unused129] [unused130] [unused131] [unused132] [unused133] [unused134] [unused135] [unused136] [unused137] [unused138] [unused139] [unused140] [unused141] [unused142] [unused143] [unused144] [unused145] [unused146] [unused147] [unused148] [unused149] [unused150] [unused151] [unused152] [unused153] [unused154] [unused155] [unused156] [unused157] [unused158] [unused159] [unused160] [unused161] [unused162] [unused163] [unused164] [unused165] [unused166] [unused167] [unused168] [unused169] [unused170] [unused171] [unused172] [unused173] [unused174] [unused175] [unused176] [unused177] [unused178] [unused179] [unused180] [unused181] [unused182] [unused183] [unused184] [unused185] [unused186] [unused187] [unused188] [unused189] [unused190] [unused191] [unused192] [unused193] [unused194] [unused195] [unused196] [unused197] [unused198] [unused199] [unused200] [unused201] [unused202] [unused203] [unused204] [unused205] [unused206] [unused207] [unused208] [unused209] [unused210] [unused211] [unused212] [unused213] [unused214] [unused215] [unused216] [unused217] [unused218] [unused219] [unused220] [unused221] [unused222] [unused223] [unused224] [unused225] [unused226] [unused227] [unused228] [unused229] [unused230] [unused231] [unused232] [unused233] [unused234] [unused235] [unused236] [unused237] [unused238] [unused239] [unused240] [unused241] [unused242] [unused243] [unused244] [unused245] [unused246] [unused247] [unused248] [unused249] [unused250] [unused251] [unused252] [unused253] [unused254] [unused255] [unused256] [unused257] [unused258] [unused259] [unused260] [unused261] [unused262] [unused263] [unused264] [unused265] [unused266] [unused267] [unused268] [unused269] [unused270] [unused271] [unused272] [unused273] [unused274] [unused275] [unused276] [unused277] [unused278] [unused279] [unused280] [unused281] [unused282] [unused283] [unused284] [unused285] [unused286] [unused287] [unused288] [unused289] [unused290] [unused291] [unused292] [unused293] [unused294] [unused295] [unused296] [unused297] [unused298] [unused299] [unused300] [unused301] [unused302] [unused303] [unused304] [unused305] [unused306] [unused307] [unused308] [unused309] [unused310] [unused311] [unused312] [unused313] [unused314] [unused315] [unused316] [unused317] [unused318] [unused319] [unused320] [unused321] [unused322] [unused323] [unused324] [unused325] [unused326] [unused327] [unused328] [unused329] [unused330] [unused331] [unused332] [unused333] [unused334] [unused335] [unused336] [unused337] [unused338] [unused339] [unused340] [unused341] [unused342] [unused343] [unused344] [unused345] [unused346] [unused347] [unused348] [unused349] [unused350] [unused351] [unused352] [unused353] [unused354] [unused355] [unused356] [unused357] [unused358] [unused359] [unused360] [unused361] [unused362] [unused363] [unused364] [unused365] [unused366] [unused367] [unused368] [unused369] [unused370] [unused371] [unused372] [unused373] [unused374] [unused375] [unused376] [unused377] [unused378] [unused379] [unused380] [unused381] [unused382] [unused383] [unused384] [unused385] [unused386] [unused387] [unused388] [unused389] [unused390] [unused391] [unused392] [unused393] [unused394] [unused395] [unused396] [unused397] [unused398] [unused399] [unused400] [unused401] [unused402] [unused403] [unused404] [unused405] [unused406] [unused407] [unused408] [unused409] [unused410] [unused411] [unused412] [unused413] [unused414] [unused415] [unused416] [unused417] [unused418] [unused419] [unused420] [unused421] [unused422] [unused423] [unused424] [unused425] [unused426] [unused427] [unused428] [unused429] [unused430] [unused431] [unused432] [unused433] [unused434] [unused435] [unused436] [unused437] [unused438] [unused439] [unused440] [unused441] [unused442] [unused443] [unused444] [unused445] [unused446] [unused447] [unused448] [unused449] [unused450] [unused451] [unused452] [unused453] [unused454] [unused455] [unused456] [unused457] [unused458] [unused459] [unused460] [unused461] [unused462] [unused463] [unused464] [unused465] [unused466] [unused467] [unused468] [unused469] [unused470] [unused471] [unused472] [unused473] [unused474] [unused475] [unused476] [unused477] [unused478] [unused479] [unused480] [unused481] [unused482] [unused483] [unused484] [unused485] [unused486] [unused487] [unused488] [unused489] [unused490] [unused491] [unused492] [unused493] [unused494] [unused495] [unused496] [unused497] [unused498] [unused499] [unused500] [unused501] [unused502] [unused503] [unused504] [unused505] [unused506] [unused507] [unused508] [unused509] [unused510] [unused511] [unused512] [unused513] [unused514] [unused515] [unused516] [unused517] [unused518] [unused519] [unused520] [unused521] [unused522] [unused523] [unused524] [unused525] [unused526] [unused527] [unused528] [unused529] [unused530] [unused531] [unused532] [unused533] [unused534] [unused535] [unused536] [unused537] [unused538] [unused539] [unused540] [unused541] [unused542] [unused543] [unused544] [unused545] [unused546] [unused547] [unused548] [unused549] [unused550] [unused551] [unused552] [unused553] [unused554] [unused555] [unused556] [unused557] [unused558] [unused559] [unused560] [unused561] [unused562] [unused563] [unused564] [unused565] [unused566] [unused567] [unused568] [unused569] [unused570] [unused571] [unused572] [unused573] [unused574] [unused575] [unused576] [unused577] [unused578] [unused579] [unused580] [unused581] [unused582] [unused583] [unused584] [unused585] [unused586] [unused587] [unused588] [unused589] [unused590] [unused591] [unused592] [unused593] [unused594] [unused595] [unused596] [unused597] [unused598] [unused599] [unused600] [unused601] [unused602] [unused603] [unused604] [unused605] [unused606] [unused607] [unused608] [unused609] [unused610] [unused611] [unused612] [unused613] [unused614] [unused615] [unused616] [unused617] [unused618] [unused619] [unused620] [unused621] [unused622] [unused623] [unused624] [unused625] [unused626] [unused627] [unused628] [unused629] [unused630] [unused631] [unused632] [unused633] [unused634] [unused635] [unused636] [unused637] [unused638] [unused639] [unused640] [unused641] [unused642] [unused643] [unused644] [unused645] [unused646] [unused647] [unused648] [unused649] [unused650] [unused651] [unused652] [unused653] [unused654] [unused655] [unused656] [unused657] [unused658] [unused659] [unused660] [unused661] [unused662] [unused663] [unused664] [unused665] [unused666] [unused667] [unused668] [unused669] [unused670] [unused671] [unused672] [unused673] [unused674] [unused675] [unused676] [unused677] [unused678] [unused679] [unused680] [unused681] [unused682] [unused683] [unused684] [unused685] [unused686] [unused687] [unused688] [unused689] [unused690] [unused691] [unused692] [unused693] [unused694] [unused695] [unused696] [unused697] [unused698] [unused699] [unused700] [unused701] [unused702] [unused703] [unused704] [unused705] [unused706] [unused707] [unused708] [unused709] [unused710] [unused711] [unused712] [unused713] [unused714] [unused715] [unused716] [unused717] [unused718] [unused719] [unused720] [unused721] [unused722] [unused723] [unused724] [unused725] [unused726] [unused727] [unused728] [unused729] [unused730] [unused731] [unused732] [unused733] [unused734] [unused735] [unused736] [unused737] [unused738] [unused739] [unused740] [unused741] [unused742] [unused743] [unused744] [unused745] [unused746] [unused747] [unused748] [unused749] [unused750] [unused751] [unused752] [unused753] [unused754] [unused755] [unused756] [unused757] [unused758] [unused759] [unused760] [unused761] [unused762] [unused763] [unused764] [unused765] [unused766] [unused767] [unused768] [unused769] [unused770] [unused771] [unused772] [unused773] [unused774] [unused775] [unused776] [unused777] [unused778] [unused779] [unused780] [unused781] [unused782] [unused783] [unused784] [unused785] [unused786] [unused787] [unused788] [unused789] [unused790] [unused791] [unused792] [unused793] [unused794] [unused795] [unused796] [unused797] [unused798] [unused799] [unused800] [unused801] [unused802] [unused803] [unused804] [unused805] [unused806] [unused807] [unused808] [unused809] [unused810] [unused811] [unused812] [unused813] [unused814] [unused815] [unused816] [unused817] [unused818] [unused819] [unused820] [unused821] [unused822] [unused823] [unused824] [unused825] [unused826] [unused827] [unused828] [unused829] [unused830] [unused831] [unused832] [unused833] [unused834] [unused835] [unused836] [unused837] [unused838] [unused839] [unused840] [unused841] [unused842] [unused843] [unused844] [unused845] [unused846] [unused847] [unused848] [unused849] [unused850] [unused851] [unused852] [unused853] [unused854] [unused855] [unused856] [unused857] [unused858] [unused859] [unused860] [unused861] [unused862] [unused863] [unused864] [unused865] [unused866] [unused867] [unused868] [unused869] [unused870] [unused871] [unused872] [unused873] [unused874] [unused875] [unused876] [unused877] [unused878] [unused879] [unused880] [unused881] [unused882] [unused883] [unused884] [unused885] [unused886] [unused887] [unused888] [unused889] [unused890] [unused891] [unused892] [unused893] [unused894] [unused895] [unused896] [unused897] [unused898] [unused899] [unused900] [unused901] [unused902] [unused903] [unused904] [unused905] [unused906] [unused907] [unused908] [unused909] [unused910] [unused911] [unused912] [unused913] [unused914] [unused915] [unused916] [unused917] [unused918] [unused919] [unused920] [unused921] [unused922] [unused923] [unused924] [unused925] [unused926] [unused927] [unused928] [unused929] [unused930] [unused931] [unused932] [unused933] [unused934] [unused935] [unused936] [unused937] [unused938] [unused939] [unused940] [unused941] [unused942] [unused943] [unused944] [unused945] [unused946] [unused947] [unused948] [unused949] [unused950] [unused951] [unused952] [unused953] [unused954] [unused955] [unused956] [unused957] [unused958] [unused959] [unused960] [unused961] [unused962] [unused963] [unused964] [unused965] [unused966] [unused967] [unused968] [unused969] [unused970] [unused971] [unused972] [unused973] [unused974] [unused975] [unused976] [unused977] [unused978] [unused979] [unused980] [unused981] [unused982] [unused983] [unused984] [unused985] [unused986] [unused987] [unused988] [unused989] [unused990] [unused991] [unused992] [unused993] ! " # $ % & ' ( ) * + , - . / 0 1 2 3 4 5 6 7 8 9 : ; < = > ? @ [ \ ] ^ _ ` a b c d e f g h i j k l m n o p q r s t u v w x y z { | } ~ ¡ ¢ £ ¤ ¥ ¦ § ¨ © ª « ¬ ® ° ± ² ³ ´ µ ¶ · ¹ º » ¼ ½ ¾ ¿ × ß æ ð ÷ ø þ đ ħ ı ł ŋ œ ƒ ɐ ɑ ɒ ɔ ɕ ə ɛ ɡ ɣ ɨ ɪ ɫ ɬ ɯ ɲ ɴ ɹ ɾ ʀ ʁ ʂ ʃ ʉ ʊ ʋ ʌ ʎ ʐ ʑ ʒ ʔ ʰ ʲ ʳ ʷ ʸ ʻ ʼ ʾ ʿ ˈ ː ˡ ˢ ˣ ˤ α β γ δ ε ζ η θ ι κ λ μ ν ξ ο π ρ ς σ τ υ φ χ ψ ω а б в г д е ж з и к л м н о п р с т у ф х ц ч ш щ ъ ы ь э ю я ђ є і ј љ њ ћ ӏ ա բ գ դ ե թ ի լ կ հ մ յ ն ո պ ս վ տ ր ւ ք ־ א ב ג ד ה ו ז ח ט י ך כ ל ם מ ן נ ס ע ף פ ץ צ ק ר ש ת ، ء ا ب ة ت ث ج ح خ د ذ ر ز س ش ص ض ط ظ ع غ ـ ف ق ك ل م ن ه و ى ي ٹ پ چ ک گ ں ھ ہ ی ے अ आ उ ए क ख ग च ज ट ड ण त थ द ध न प ब भ म य र ल व श ष स ह ा ि ी ो । ॥ ং অ আ ই উ এ ও ক খ গ চ ছ জ ট ড ণ ত থ দ ধ ন প ব ভ ম য র ল শ ষ স হ া ি ী ে க ச ட த ந ன ப ம ய ர ல ள வ ா ி ு ே ை ನ ರ ಾ ක ය ර ල ව ා ก ง ต ท น พ ม ย ร ล ว ส อ า เ ་ ། ག ང ད ན པ བ མ འ ར ལ ས မ ა ბ გ დ ე ვ თ ი კ ლ მ ნ ო რ ს ტ უ ᄀ ᄂ ᄃ ᄅ ᄆ ᄇ ᄉ ᄊ ᄋ ᄌ ᄎ ᄏ ᄐ ᄑ ᄒ ᅡ ᅢ ᅥ ᅦ ᅧ ᅩ ᅪ ᅭ ᅮ ᅯ ᅲ ᅳ ᅴ ᅵ ᆨ ᆫ ᆯ ᆷ ᆸ ᆼ ᴬ ᴮ ᴰ ᴵ ᴺ ᵀ ᵃ ᵇ ᵈ ᵉ ᵍ ᵏ ᵐ ᵒ ᵖ ᵗ ᵘ ᵢ ᵣ ᵤ ᵥ ᶜ ᶠ ‐ ‑ ‒ – — ― ‖ ‘ ’ ‚ “ ” „ † ‡ • … ‰ ′ ″ › ‿ ⁄ ⁰ ⁱ ⁴ ⁵ ⁶ ⁷ ⁸ ⁹ ⁺ ⁻ ⁿ ₀ ₁ ₂ ₃ ₄ ₅ ₆ ₇ ₈ ₉ ₊ ₍ ₎ ₐ ₑ ₒ ₓ ₕ ₖ ₗ ₘ ₙ ₚ ₛ ₜ ₤ ₩ € ₱ ₹ ℓ № ℝ ™ ⅓ ⅔ ← ↑ → ↓ ↔ ↦ ⇄ ⇌ ⇒ ∂ ∅ ∆ ∇ ∈ − ∗ ∘ √ ∞ ∧ ∨ ∩ ∪ ≈ ≡ ≤ ≥ ⊂ ⊆ ⊕ ⊗ ⋅ ─ │ ■ ▪ ● ★ ☆ ☉ ♠ ♣ ♥ ♦ ♭ ♯ ⟨ ⟩ ⱼ ⺩ ⺼ ⽥ 、 。 〈 〉 《 》 「 」 『 』 〜 あ い う え お か き く け こ さ し す せ そ た ち っ つ て と な に ぬ ね の は ひ ふ へ ほ ま み む め も や ゆ よ ら り る れ ろ を ん ァ ア ィ イ ウ ェ エ オ カ キ ク ケ コ サ シ ス セ タ チ ッ ツ テ ト ナ ニ ノ ハ ヒ フ ヘ ホ マ ミ ム メ モ ャ ュ ョ ラ リ ル レ ロ ワ ン ・ ー 一 三 上 下 不 世 中 主 久 之 也 事 二 五 井 京 人 亻 仁 介 代 仮 伊 会 佐 侍 保 信 健 元 光 八 公 内 出 分 前 劉 力 加 勝 北 区 十 千 南 博 原 口 古 史 司 合 吉 同 名 和 囗 四 国 國 土 地 坂 城 堂 場 士 夏 外 大 天 太 夫 奈 女 子 学 宀 宇 安 宗 定 宣 宮 家 宿 寺 將 小 尚 山 岡 島 崎 川 州 巿 帝 平 年 幸 广 弘 張 彳 後 御 德 心 忄 志 忠 愛 成 我 戦 戸 手 扌 政 文 新 方 日 明 星 春 昭 智 曲 書 月 有 朝 木 本 李 村 東 松 林 森 楊 樹 橋 歌 止 正 武 比 氏 民 水 氵 氷 永 江 沢 河 治 法 海 清 漢 瀬 火 版 犬 王 生 田 男 疒 発 白 的 皇 目 相 省 真 石 示 社 神 福 禾 秀 秋 空 立 章 竹 糹 美 義 耳 良 艹 花 英 華 葉 藤 行 街 西 見 訁 語 谷 貝 貴 車 軍 辶 道 郎 郡 部 都 里 野 金 鈴 镇 長 門 間 阝 阿 陳 陽 雄 青 面 風 食 香 馬 高 龍 龸 fi fl ！ （ ） ， － ． ／ ： ？ ～ the of and in to was he is as for on with that it his by at from her ##s she you had an were but be this are not my they one which or have him me first all also their has up who out been when after there into new two its ##a time would no what about said we over then other so more ##e can if like back them only some could ##i where just ##ing during before ##n do ##o made school through than now years most world may between down well three ##d year while will ##ed ##r ##y later ##t city under around did such being used state people part know against your many second university both national ##er these don known off way until re how even get head ... didn ##ly team american because de ##l born united film since still long work south us became any high again day family see right man eyes house season war states including took life north same each called name much place however go four group another found won area here going 10 away series left home music best make hand number company several never last john 000 very album take end good too following released game played little began district ##m old want those side held own early county ll league use west ##u face think ##es 2010 government ##h march came small general town june ##on line based something ##k september thought looked along international 2011 air july club went january october our august april york 12 few 2012 2008 east show member college 2009 father public ##us come men five set station church ##c next former november room party located december 2013 age got 2007 ##g system let love 2006 though every 2014 look song water century without body black night within great women single ve building large population river named band white started ##an once 15 20 should 18 2015 service top built british open death king moved local times children february book why 11 door need president order final road wasn although due major died village third knew 2016 asked turned st wanted say ##p together received main son served different ##en behind himself felt members power football law voice play ##in near park history 30 having 2005 16 ##man saw mother ##al army point front help english street art late hands games award ##ia young 14 put published country division across told 13 often ever french london center six red 2017 led days include light 25 find tell among species really according central half 2004 form original gave office making enough lost full opened must included live given german player run business woman community cup might million land 2000 court development 17 short round ii km seen class story always become sure research almost director council la ##2 career things using island ##z couldn car ##is 24 close force ##1 better free support control field students 2003 education married ##b nothing worked others record big inside level anything continued give james ##3 military established non returned feel does title written thing feet william far co association hard already 2002 ##ra championship human western 100 ##na department hall role various production 21 19 heart 2001 living fire version ##ers ##f television royal ##4 produced working act case society region present radio period looking least total keep england wife program per brother mind special 22 ##le am works soon ##6 political george services taken created ##7 further able reached david union joined upon done important social information either ##ic ##x appeared position ground lead rock dark election 23 board france hair course arms site police girl instead real sound ##v words moment ##te someone ##8 summer project announced san less wrote past followed ##5 blue founded al finally india taking records america ##ne 1999 design considered northern god stop battle toward european outside described track today playing language 28 call 26 heard professional low australia miles california win yet green ##ie trying blood ##ton southern science maybe everything match square 27 mouth video race recorded leave above ##9 daughter points space 1998 museum change middle common ##0 move tv post ##ta lake seven tried elected closed ten paul minister ##th months start chief return canada person sea release similar modern brought rest hit formed mr ##la 1997 floor event doing thomas 1996 robert care killed training star week needed turn finished railway rather news health sent example ran term michael coming currently yes forces despite gold areas 50 stage fact 29 dead says popular 2018 originally germany probably developed result pulled friend stood money running mi signed word songs child eventually met tour average teams minutes festival current deep kind 1995 decided usually eastern seemed ##ness episode bed added table indian private charles route available idea throughout centre addition appointed style 1994 books eight construction press mean wall friends remained schools study ##ch ##um institute oh chinese sometimes events possible 1992 australian type brown forward talk process food debut seat performance committee features character arts herself else lot strong russian range hours peter arm ##da morning dr sold ##ry quickly directed 1993 guitar china ##w 31 list ##ma performed media uk players smile ##rs myself 40 placed coach province towards wouldn leading whole boy official designed grand census ##el europe attack japanese henry 1991 ##re ##os cross getting alone action lower network wide washington japan 1990 hospital believe changed sister ##ar hold gone sir hadn ship ##ka studies academy shot rights below base bad involved kept largest ##ist bank future especially beginning mark movement section female magazine plan professor lord longer ##ian sat walked hill actually civil energy model families size thus aircraft completed includes data captain ##or fight vocals featured richard bridge fourth 1989 officer stone hear ##ism means medical groups management self lips competition entire lived technology leaving federal tournament bit passed hot independent awards kingdom mary spent fine doesn reported ##ling jack fall raised itself stay true studio 1988 sports replaced paris systems saint leader theatre whose market capital parents spanish canadian earth ##ity cut degree writing bay christian awarded natural higher bill ##as coast provided previous senior ft valley organization stopped onto countries parts conference queen security interest saying allowed master earlier phone matter smith winning try happened moving campaign los ##ley breath nearly mid 1987 certain girls date italian african standing fell artist ##ted shows deal mine industry 1986 ##ng everyone republic provide collection library student ##ville primary owned older via heavy 1st makes ##able attention anyone africa ##ri stated length ended fingers command staff skin foreign opening governor okay medal kill sun cover job 1985 introduced chest hell feeling ##ies success meet reason standard meeting novel 1984 trade source buildings ##land rose guy goal ##ur chapter native husband previously unit limited entered weeks producer operations mountain takes covered forced related roman complete successful key texas cold ##ya channel 1980 traditional films dance clear approximately 500 nine van prince question active tracks ireland regional silver author personal sense operation ##ine economic 1983 holding twenty isbn additional speed hour edition regular historic places whom shook movie km² secretary prior report chicago read foundation view engine scored 1982 units ask airport property ready immediately lady month listed contract ##de manager themselves lines ##ki navy writer meant ##ts runs ##ro practice championships singer glass commission required forest starting culture generally giving access attended test couple stand catholic martin caught executive ##less eye ##ey thinking chair quite shoulder 1979 hope decision plays defeated municipality whether structure offered slowly pain ice direction ##ion paper mission 1981 mostly 200 noted individual managed nature lives plant ##ha helped except studied computer figure relationship issue significant loss die smiled gun ago highest 1972 ##am male bring goals mexico problem distance commercial completely location annual famous drive 1976 neck 1978 surface caused italy understand greek highway wrong hotel comes appearance joseph double issues musical companies castle income review assembly bass initially parliament artists experience 1974 particular walk foot engineering talking window dropped ##ter miss baby boys break 1975 stars edge remember policy carried train stadium bar sex angeles evidence ##ge becoming assistant soviet 1977 upper step wing 1970 youth financial reach ##ll actor numerous ##se ##st nodded arrived ##ation minute ##nt believed sorry complex beautiful victory associated temple 1968 1973 chance perhaps metal ##son 1945 bishop ##et lee launched particularly tree le retired subject prize contains yeah theory empire ##ce suddenly waiting trust recording ##to happy terms camp champion 1971 religious pass zealand names 2nd port ancient tom corner represented watch legal anti justice cause watched brothers 45 material changes simply response louis fast ##ting answer 60 historical 1969 stories straight create feature increased rate administration virginia el activities cultural overall winner programs basketball legs guard beyond cast doctor mm flight results remains cost effect winter ##ble larger islands problems chairman grew commander isn 1967 pay failed selected hurt fort box regiment majority journal 35 edward plans ##ke ##ni shown pretty irish characters directly scene likely operated allow spring ##j junior matches looks mike houses fellow ##tion beach marriage ##ham ##ive rules oil 65 florida expected nearby congress sam peace recent iii wait subsequently cell ##do variety serving agreed please poor joe pacific attempt wood democratic piece prime ##ca rural mile touch appears township 1964 1966 soldiers ##men ##ized 1965 pennsylvania closer fighting claimed score jones physical editor ##ous filled genus specific sitting super mom ##va therefore supported status fear cases store meaning wales minor spain tower focus vice frank follow parish separate golden horse fifth remaining branch 32 presented stared ##id uses secret forms ##co baseball exactly ##ck choice note discovered travel composed truth russia ball color kiss dad wind continue ring referred numbers digital greater ##ns metres slightly direct increase 1960 responsible crew rule trees troops ##no broke goes individuals hundred weight creek sleep memory defense provides ordered code value jewish windows 1944 safe judge whatever corps realized growing pre ##ga cities alexander gaze lies spread scott letter showed situation mayor transport watching workers extended ##li expression normal ##ment chart multiple border ##ba host ##ner daily mrs walls piano ##ko heat cannot ##ate earned products drama era authority seasons join grade ##io sign difficult machine 1963 territory mainly ##wood stations squadron 1962 stepped iron 19th ##led serve appear sky speak broken charge knowledge kilometres removed ships article campus simple ##ty pushed britain ##ve leaves recently cd soft boston latter easy acquired poland ##sa quality officers presence planned nations mass broadcast jean share image influence wild offer emperor electric reading headed ability promoted yellow ministry 1942 throat smaller politician ##by latin spoke cars williams males lack pop 80 ##ier acting seeing consists ##ti estate 1961 pressure johnson newspaper jr chris olympics online conditions beat elements walking vote ##field needs carolina text featuring global block shirt levels francisco purpose females et dutch duke ahead gas twice safety serious turning highly lieutenant firm maria amount mixed daniel proposed perfect agreement affairs 3rd seconds contemporary paid 1943 prison save kitchen label administrative intended constructed academic nice teacher races 1956 formerly corporation ben nation issued shut 1958 drums housing victoria seems opera 1959 graduated function von mentioned picked build recognized shortly protection picture notable exchange elections 1980s loved percent racing fish elizabeth garden volume hockey 1941 beside settled ##ford 1940 competed replied drew 1948 actress marine scotland steel glanced farm steve 1957 risk tonight positive magic singles effects gray screen dog ##ja residents bus sides none secondary literature polish destroyed flying founder households 1939 lay reserve usa gallery ##ler 1946 industrial younger approach appearances urban ones 1950 finish avenue powerful fully growth page honor jersey projects advanced revealed basic 90 infantry pair equipment visit 33 evening search grant effort solo treatment buried republican primarily bottom owner 1970s israel gives jim dream bob remain spot 70 notes produce champions contact ed soul accepted ways del ##ally losing split price capacity basis trial questions ##ina 1955 20th guess officially memorial naval initial ##ization whispered median engineer ##ful sydney ##go columbia strength 300 1952 tears senate 00 card asian agent 1947 software 44 draw warm supposed com pro ##il transferred leaned ##at candidate escape mountains asia potential activity entertainment seem traffic jackson murder 36 slow product orchestra haven agency bbc taught website comedy unable storm planning albums rugby environment scientific grabbed protect ##hi boat typically 1954 1953 damage principal divided dedicated mount ohio ##berg pick fought driver ##der empty shoulders sort thank berlin prominent account freedom necessary efforts alex headquarters follows alongside des simon andrew suggested operating learning steps 1949 sweet technical begin easily 34 teeth speaking settlement scale ##sh renamed ray max enemy semi joint compared ##rd scottish leadership analysis offers georgia pieces captured animal deputy guest organized ##lin tony combined method challenge 1960s huge wants battalion sons rise crime types facilities telling path 1951 platform sit 1990s ##lo tells assigned rich pull ##ot commonly alive ##za letters concept conducted wearing happen bought becomes holy gets ocean defeat languages purchased coffee occurred titled ##q declared applied sciences concert sounds jazz brain ##me painting fleet tax nick ##ius michigan count animals leaders episodes ##line content ##den birth ##it clubs 64 palace critical refused fair leg laughed returning surrounding participated formation lifted pointed connected rome medicine laid taylor santa powers adam tall shared focused knowing yards entrance falls ##wa calling ##ad sources chosen beneath resources yard ##ite nominated silence zone defined ##que gained thirty 38 bodies moon ##ard adopted christmas widely register apart iran premier serves du unknown parties ##les generation ##ff continues quick fields brigade quiet teaching clothes impact weapons partner flat theater supreme 1938 37 relations ##tor plants suffered 1936 wilson kids begins ##age 1918 seats armed internet models worth laws 400 communities classes background knows thanks quarter reaching humans carry killing format kong hong setting 75 architecture disease railroad inc possibly wish arthur thoughts harry doors density ##di crowd illinois stomach tone unique reports anyway ##ir liberal der vehicle thick dry drug faced largely facility theme holds creation strange colonel ##mi revolution bell politics turns silent rail relief independence combat shape write determined sales learned 4th finger oxford providing 1937 heritage fiction situated designated allowing distribution hosted ##est sight interview estimated reduced ##ria toronto footballer keeping guys damn claim motion sport sixth stayed ##ze en rear receive handed twelve dress audience granted brazil ##well spirit ##ated noticed etc olympic representative eric tight trouble reviews drink vampire missing roles ranked newly household finals wave critics ##ee phase massachusetts pilot unlike philadelphia bright guns crown organizations roof 42 respectively clearly tongue marked circle fox korea bronze brian expanded sexual supply yourself inspired labour fc ##ah reference vision draft connection brand reasons 1935 classic driving trip jesus cells entry 1920 neither trail claims atlantic orders labor nose afraid identified intelligence calls cancer attacked passing stephen positions imperial grey jason 39 sunday 48 swedish avoid extra uncle message covers allows surprise materials fame hunter ##ji 1930 citizens figures davis environmental confirmed shit titles di performing difference acts attacks ##ov existing votes opportunity nor shop entirely trains opposite pakistan ##pa develop resulted representatives actions reality pressed ##ish barely wine conversation faculty northwest ends documentary nuclear stock grace sets eat alternative ##ps bag resulting creating surprised cemetery 1919 drop finding sarah cricket streets tradition ride 1933 exhibition target ear explained rain composer injury apartment municipal educational occupied netherlands clean billion constitution learn 1914 maximum classical francis lose opposition jose ontario bear core hills rolled ending drawn permanent fun ##tes ##lla lewis sites chamber ryan ##way scoring height 1934 ##house lyrics staring 55 officials 1917 snow oldest ##tic orange ##ger qualified interior apparently succeeded thousand dinner lights existence fans heavily 41 greatest conservative send bowl plus enter catch ##un economy duty 1929 speech authorities princess performances versions shall graduate pictures effective remembered poetry desk crossed starring starts passenger sharp ##ant acres ass weather falling rank fund supporting check adult publishing heads cm southeast lane ##burg application bc ##ura les condition transfer prevent display ex regions earl federation cool relatively answered besides 1928 obtained portion ##town mix ##ding reaction liked dean express peak 1932 ##tte counter religion chain rare miller convention aid lie vehicles mobile perform squad wonder lying crazy sword ##ping attempted centuries weren philosophy category ##ize anna interested 47 sweden wolf frequently abandoned kg literary alliance task entitled ##ay threw promotion factory tiny soccer visited matt fm achieved 52 defence internal persian 43 methods ##ging arrested otherwise cambridge programming villages elementary districts rooms criminal conflict worry trained 1931 attempts waited signal bird truck subsequent programme ##ol ad 49 communist details faith sector patrick carrying laugh ##ss controlled korean showing origin fuel evil 1927 ##ent brief identity darkness address pool missed publication web planet ian anne wings invited ##tt briefly standards kissed ##be ideas climate causing walter worse albert articles winners desire aged northeast dangerous gate doubt 1922 wooden multi ##ky poet rising funding 46 communications communication violence copies prepared ford investigation skills 1924 pulling electronic ##ak ##ial ##han containing ultimately offices singing understanding restaurant tomorrow fashion christ ward da pope stands 5th flow studios aired commissioned contained exist fresh americans ##per wrestling approved kid employed respect suit 1925 angel asking increasing frame angry selling 1950s thin finds ##nd temperature statement ali explain inhabitants towns extensive narrow 51 jane flowers images promise somewhere object fly closely ##ls 1912 bureau cape 1926 weekly presidential legislative 1921 ##ai ##au launch founding ##ny 978 ##ring artillery strike un institutions roll writers landing chose kevin anymore pp ##ut attorney fit dan billboard receiving agricultural breaking sought dave admitted lands mexican ##bury charlie specifically hole iv howard credit moscow roads accident 1923 proved wear struck hey guards stuff slid expansion 1915 cat anthony ##kin melbourne opposed sub southwest architect failure plane 1916 ##ron map camera tank listen regarding wet introduction metropolitan link ep fighter inch grown gene anger fixed buy dvd khan domestic worldwide chapel mill functions examples ##head developing 1910 turkey hits pocket antonio papers grow unless circuit 18th concerned attached journalist selection journey converted provincial painted hearing aren bands negative aside wondered knight lap survey ma ##ow noise billy ##ium shooting guide bedroom priest resistance motor homes sounded giant ##mer 150 scenes equal comic patients hidden solid actual bringing afternoon touched funds wedding consisted marie canal sr kim treaty turkish recognition residence cathedral broad knees incident shaped fired norwegian handle cheek contest represent ##pe representing beauty ##sen birds advantage emergency wrapped drawing notice pink broadcasting ##ong somehow bachelor seventh collected registered establishment alan assumed chemical personnel roger retirement jeff portuguese wore tied device threat progress advance ##ised banks hired manchester nfl teachers structures forever ##bo tennis helping saturday sale applications junction hip incorporated neighborhood dressed ceremony ##ds influenced hers visual stairs decades inner kansas hung hoped gain scheduled downtown engaged austria clock norway certainly pale protected 1913 victor employees plate putting surrounded ##ists finishing blues tropical ##ries minnesota consider philippines accept 54 retrieved 1900 concern anderson properties institution gordon successfully vietnam ##dy backing outstanding muslim crossing folk producing usual demand occurs observed lawyer educated ##ana kelly string pleasure budget items quietly colorado philip typical ##worth derived 600 survived asks mental ##ide 56 jake jews distinguished ltd 1911 sri extremely 53 athletic loud thousands worried shadow transportation horses weapon arena importance users tim objects contributed dragon douglas aware senator johnny jordan sisters engines flag investment samuel shock capable clark row wheel refers session familiar biggest wins hate maintained drove hamilton request expressed injured underground churches walker wars tunnel passes stupid agriculture softly cabinet regarded joining indiana ##ea ##ms push dates spend behavior woods protein gently chase morgan mention burning wake combination occur mirror leads jimmy indeed impossible singapore paintings covering ##nes soldier locations attendance sell historian wisconsin invasion argued painter diego changing egypt ##don experienced inches ##ku missouri vol grounds spoken switzerland ##gan reform rolling ha forget massive resigned burned allen tennessee locked values improved ##mo wounded universe sick dating facing pack purchase user ##pur moments ##ul merged anniversary 1908 coal brick understood causes dynasty queensland establish stores crisis promote hoping views cards referee extension ##si raise arizona improve colonial formal charged ##rt palm lucky hide rescue faces 95 feelings candidates juan ##ell goods 6th courses weekend 59 luke cash fallen ##om delivered affected installed carefully tries swiss hollywood costs lincoln responsibility ##he shore file proper normally maryland assistance jump constant offering friendly waters persons realize contain trophy 800 partnership factor 58 musicians cry bound oregon indicated hero houston medium ##ure consisting somewhat ##ara 57 cycle ##che beer moore frederick gotten eleven worst weak approached arranged chin loan universal bond fifteen pattern disappeared ##ney translated ##zed lip arab capture interests insurance ##chi shifted cave prix warning sections courts coat plot smell feed golf favorite maintain knife vs voted degrees finance quebec opinion translation manner ruled operate productions choose musician discovery confused tired separated stream techniques committed attend ranking kings throw passengers measure horror fan mining sand danger salt calm decade dam require runner ##ik rush associate greece ##ker rivers consecutive matthew ##ski sighed sq documents steam edited closing tie accused 1905 ##ini islamic distributed directors organisation bruce 7th breathing mad lit arrival concrete taste 08 composition shaking faster amateur adjacent stating 1906 twin flew ##ran tokyo publications ##tone obviously ridge storage 1907 carl pages concluded desert driven universities ages terminal sequence borough 250 constituency creative cousin economics dreams margaret notably reduce montreal mode 17th ears saved jan vocal ##ica 1909 andy ##jo riding roughly threatened ##ise meters meanwhile landed compete repeated grass czech regularly charges tea sudden appeal ##ung solution describes pierre classification glad parking ##ning belt physics 99 rachel add hungarian participate expedition damaged gift childhood 85 fifty ##red mathematics jumped letting defensive mph ##ux ##gh testing ##hip hundreds shoot owners matters smoke israeli kentucky dancing mounted grandfather emma designs profit argentina ##gs truly li lawrence cole begun detroit willing branches smiling decide miami enjoyed recordings ##dale poverty ethnic gay ##bi gary arabic 09 accompanied ##one ##ons fishing determine residential acid ##ary alice returns starred mail ##ang jonathan strategy ##ue net forty cook businesses equivalent commonwealth distinct ill ##cy seriously ##ors ##ped shift harris replace rio imagine formula ensure ##ber additionally scheme conservation occasionally purposes feels favor ##and ##ore 1930s contrast hanging hunt movies 1904 instruments victims danish christopher busy demon sugar earliest colony studying balance duties ##ks belgium slipped carter 05 visible stages iraq fifa ##im commune forming zero 07 continuing talked counties legend bathroom option tail clay daughters afterwards severe jaw visitors ##ded devices aviation russell kate ##vi entering subjects ##ino temporary swimming forth smooth ghost audio bush operates rocks movements signs eddie ##tz ann voices honorary 06 memories dallas pure measures racial promised 66 harvard ceo 16th parliamentary indicate benefit flesh dublin louisiana 1902 1901 patient sleeping 1903 membership coastal medieval wanting element scholars rice 62 limit survive makeup rating definitely collaboration obvious ##tan boss ms baron birthday linked soil diocese ##lan ncaa ##mann offensive shell shouldn waist ##tus plain ross organ resolution manufacturing adding relative kennedy 98 whilst moth marketing gardens crash 72 heading partners credited carlos moves cable ##zi marshall ##out depending bottle represents rejected responded existed 04 jobs denmark lock ##ating treated graham routes talent commissioner drugs secure tests reign restored photography ##gi contributions oklahoma designer disc grin seattle robin paused atlanta unusual ##gate praised las laughing satellite hungary visiting ##sky interesting factors deck poems norman ##water stuck speaker rifle domain premiered ##her dc comics actors 01 reputation eliminated 8th ceiling prisoners script ##nce leather austin mississippi rapidly admiral parallel charlotte guilty tools gender divisions fruit ##bs laboratory nelson fantasy marry rapid aunt tribe requirements aspects suicide amongst adams bone ukraine abc kick sees edinburgh clothing column rough gods hunting broadway gathered concerns ##ek spending ty 12th snapped requires solar bones cavalry ##tta iowa drinking waste index franklin charity thompson stewart tip flash landscape friday enjoy singh poem listening ##back eighth fred differences adapted bomb ukrainian surgery corporate masters anywhere ##more waves odd sean portugal orleans dick debate kent eating puerto cleared 96 expect cinema 97 guitarist blocks electrical agree involving depth dying panel struggle ##ged peninsula adults novels emerged vienna metro debuted shoes tamil songwriter meets prove beating instance heaven scared sending marks artistic passage superior 03 significantly shopping ##tive retained ##izing malaysia technique cheeks ##ola warren maintenance destroy extreme allied 120 appearing ##yn fill advice alabama qualifying policies cleveland hat battery smart authors 10th soundtrack acted dated lb glance equipped coalition funny outer ambassador roy possibility couples campbell dna loose ethan supplies 1898 gonna 88 monster ##res shake agents frequency springs dogs practices 61 gang plastic easier suggests gulf blade exposed colors industries markets pan nervous electoral charts legislation ownership ##idae mac appointment shield copy assault socialist abbey monument license throne employment jay 93 replacement charter cloud powered suffering accounts oak connecticut strongly wright colour crystal 13th context welsh networks voiced gabriel jerry ##cing forehead mp ##ens manage schedule totally remix ##ii forests occupation print nicholas brazilian strategic vampires engineers 76 roots seek correct instrumental und alfred backed hop ##des stanley robinson traveled wayne welcome austrian achieve 67 exit rates 1899 strip whereas ##cs sing deeply adventure bobby rick jamie careful components cap useful personality knee ##shi pushing hosts 02 protest ca ottoman symphony ##sis 63 boundary 1890 processes considering considerable tons ##work ##ft ##nia cooper trading dear conduct 91 illegal apple revolutionary holiday definition harder ##van jacob circumstances destruction ##lle popularity grip classified liverpool donald baltimore flows seeking honour approval 92 mechanical till happening statue critic increasingly immediate describe commerce stare ##ster indonesia meat rounds boats baker orthodox depression formally worn naked claire muttered sentence 11th emily document 77 criticism wished vessel spiritual bent virgin parker minimum murray lunch danny printed compilation keyboards false blow belonged 68 raising 78 cutting ##board pittsburgh ##up 9th shadows 81 hated indigenous jon 15th barry scholar ah ##zer oliver ##gy stick susan meetings attracted spell romantic ##ver ye 1895 photo demanded customers ##ac 1896 logan revival keys modified commanded jeans ##ious upset raw phil detective hiding resident vincent ##bly experiences diamond defeating coverage lucas external parks franchise helen bible successor percussion celebrated il lift profile clan romania ##ied mills ##su nobody achievement shrugged fault 1897 rhythm initiative breakfast carbon 700 69 lasted violent 74 wound ken killer gradually filmed °c dollars processing 94 remove criticized guests sang chemistry ##vin legislature disney ##bridge uniform escaped integrated proposal purple denied liquid karl influential morris nights stones intense experimental twisted 71 84 ##ld pace nazi mitchell ny blind reporter newspapers 14th centers burn basin forgotten surviving filed collections monastery losses manual couch description appropriate merely tag missions sebastian restoration replacing triple 73 elder julia warriors benjamin julian convinced stronger amazing declined versus merchant happens output finland bare barbara absence ignored dawn injuries ##port producers ##ram 82 luis ##ities kw admit expensive electricity nba exception symbol ##ving ladies shower sheriff characteristics ##je aimed button ratio effectively summit angle jury bears foster vessels pants executed evans dozen advertising kicked patrol 1889 competitions lifetime principles athletics ##logy birmingham sponsored 89 rob nomination 1893 acoustic ##sm creature longest ##tra credits harbor dust josh ##so territories milk infrastructure completion thailand indians leon archbishop ##sy assist pitch blake arrangement girlfriend serbian operational hence sad scent fur dj sessions hp refer rarely ##ora exists 1892 ##ten scientists dirty penalty burst portrait seed 79 pole limits rival 1894 stable alpha grave constitutional alcohol arrest flower mystery devil architectural relationships greatly habitat ##istic larry progressive remote cotton ##ics ##ok preserved reaches ##ming cited 86 vast scholarship decisions cbs joy teach 1885 editions knocked eve searching partly participation gap animated fate excellent ##ett na 87 alternate saints youngest ##ily climbed ##ita ##tors suggest ##ct discussion staying choir lakes jacket revenue nevertheless peaked instrument wondering annually managing neil 1891 signing terry ##ice apply clinical brooklyn aim catherine fuck farmers figured ninth pride hugh evolution ordinary involvement comfortable shouted tech encouraged taiwan representation sharing ##lia ##em panic exact cargo competing fat cried 83 1920s occasions pa cabin borders utah marcus ##isation badly muscles ##ance victorian transition warner bet permission ##rin slave terrible similarly shares seth uefa possession medals benefits colleges lowered perfectly mall transit ##ye ##kar publisher ##ened harrison deaths elevation ##ae asleep machines sigh ash hardly argument occasion parent leo decline 1888 contribution ##ua concentration 1000 opportunities hispanic guardian extent emotions hips mason volumes bloody controversy diameter steady mistake phoenix identify violin ##sk departure richmond spin funeral enemies 1864 gear literally connor random sergeant grab confusion 1865 transmission informed op leaning sacred suspended thinks gates portland luck agencies yours hull expert muscle layer practical sculpture jerusalem latest lloyd statistics deeper recommended warrior arkansas mess supports greg eagle 1880 recovered rated concerts rushed ##ano stops eggs files premiere keith ##vo delhi turner pit affair belief paint ##zing mate ##ach ##ev victim ##ology withdrew bonus styles fled ##ud glasgow technologies funded nbc adaptation ##ata portrayed cooperation supporters judges bernard justin hallway ralph ##ick graduating controversial distant continental spider bite ##ho recognize intention mixing ##ese egyptian bow tourism suppose claiming tiger dominated participants vi ##ru nurse partially tape ##rum psychology ##rn essential touring duo voting civilian emotional channels ##king apparent hebrew 1887 tommy carrier intersection beast hudson ##gar ##zo lab nova bench discuss costa ##ered detailed behalf drivers unfortunately obtain ##lis rocky ##dae siege friendship honey ##rian 1861 amy hang posted governments collins respond wildlife preferred operator ##po laura pregnant videos dennis suspected boots instantly weird automatic businessman alleged placing throwing ph mood 1862 perry venue jet remainder ##lli ##ci passion biological boyfriend 1863 dirt buffalo ron segment fa abuse ##era genre thrown stroke colored stress exercise displayed ##gen struggled ##tti abroad dramatic wonderful thereafter madrid component widespread ##sed tale citizen todd monday 1886 vancouver overseas forcing crying descent ##ris discussed substantial ranks regime 1870 provinces switch drum zane ted tribes proof lp cream researchers volunteer manor silk milan donated allies venture principle delivery enterprise ##ves ##ans bars traditionally witch reminded copper ##uk pete inter links colin grinned elsewhere competitive frequent ##oy scream ##hu tension texts submarine finnish defending defend pat detail 1884 affiliated stuart themes villa periods tool belgian ruling crimes answers folded licensed resort demolished hans lucy 1881 lion traded photographs writes craig ##fa trials generated beth noble debt percentage yorkshire erected ss viewed grades confidence ceased islam telephone retail ##ible chile m² roberts sixteen ##ich commented hampshire innocent dual pounds checked regulations afghanistan sung rico liberty assets bigger options angels relegated tribute wells attending leaf ##yan butler romanian forum monthly lisa patterns gmina ##tory madison hurricane rev ##ians bristol ##ula elite valuable disaster democracy awareness germans freyja ##ins loop absolutely paying populations maine sole prayer spencer releases doorway bull ##ani lover midnight conclusion ##sson thirteen lily mediterranean ##lt nhl proud sample ##hill drummer guinea ##ova murphy climb ##ston instant attributed horn ain railways steven ##ao autumn ferry opponent root traveling secured corridor stretched tales sheet trinity cattle helps indicates manhattan murdered fitted 1882 gentle grandmother mines shocked vegas produces ##light caribbean ##ou belong continuous desperate drunk historically trio waved raf dealing nathan bat murmured interrupted residing scientist pioneer harold aaron ##net delta attempting minority mini believes chorus tend lots eyed indoor load shots updated jail ##llo concerning connecting wealth ##ved slaves arrive rangers sufficient rebuilt ##wick cardinal flood muhammad whenever relation runners moral repair viewers arriving revenge punk assisted bath fairly breathe lists innings illustrated whisper nearest voters clinton ties ultimate screamed beijing lions andre fictional gathering comfort radar suitable dismissed hms ban pine wrist atmosphere voivodeship bid timber ##ned ##nan giants ##ane cameron recovery uss identical categories switched serbia laughter noah ensemble therapy peoples touching ##off locally pearl platforms everywhere ballet tables lanka herbert outdoor toured derek 1883 spaces contested swept 1878 exclusive slight connections ##dra winds prisoner collective bangladesh tube publicly wealthy thai ##ys isolated select ##ric insisted pen fortune ticket spotted reportedly animation enforcement tanks 110 decides wider lowest owen ##time nod hitting ##hn gregory furthermore magazines fighters solutions ##ery pointing requested peru reed chancellor knights mask worker eldest flames reduction 1860 volunteers ##tis reporting ##hl wire advisory endemic origins settlers pursue knock consumer 1876 eu compound creatures mansion sentenced ivan deployed guitars frowned involves mechanism kilometers perspective shops maps terminus duncan alien fist bridges ##pers heroes fed derby swallowed ##ros patent sara illness characterized adventures slide hawaii jurisdiction ##op organised ##side adelaide walks biology se ##ties rogers swing tightly boundaries ##rie prepare implementation stolen ##sha certified colombia edwards garage ##mm recalled ##ball rage harm nigeria breast ##ren furniture pupils settle ##lus cuba balls client alaska 21st linear thrust celebration latino genetic terror ##cia ##ening lightning fee witness lodge establishing skull ##ique earning hood ##ei rebellion wang sporting warned missile devoted activist porch worship fourteen package 1871 decorated ##shire housed ##ock chess sailed doctors oscar joan treat garcia harbour jeremy ##ire traditions dominant jacques ##gon ##wan relocated 1879 amendment sized companion simultaneously volleyball spun acre increases stopping loves belongs affect drafted tossed scout battles 1875 filming shoved munich tenure vertical romance pc ##cher argue ##ical craft ranging www opens honest tyler yesterday virtual ##let muslims reveal snake immigrants radical screaming speakers firing saving belonging ease lighting prefecture blame farmer hungry grows rubbed beam sur subsidiary ##cha armenian sao dropping conventional ##fer microsoft reply qualify spots 1867 sweat festivals ##ken immigration physician discover exposure sandy explanation isaac implemented ##fish hart initiated connect stakes presents heights householder pleased tourist regardless slip closest ##ction surely sultan brings riley preparation aboard slammed baptist experiment ongoing interstate organic playoffs ##ika 1877 130 ##tar hindu error tours tier plenty arrangements talks trapped excited sank ho athens 1872 denver welfare suburb athletes trick diverse belly exclusively yelled 1868 ##med conversion ##ette 1874 internationally computers conductor abilities sensitive hello dispute measured globe rocket prices amsterdam flights tigers inn municipalities emotion references 3d ##mus explains airlines manufactured pm archaeological 1873 interpretation devon comment ##ites settlements kissing absolute improvement suite impressed barcelona sullivan jefferson towers jesse julie ##tin ##lu grandson hi gauge regard rings interviews trace raymond thumb departments burns serial bulgarian scores demonstrated ##ix 1866 kyle alberta underneath romanized ##ward relieved acquisition phrase cliff reveals han cuts merger custom ##dar nee gilbert graduation ##nts assessment cafe difficulty demands swung democrat jennifer commons 1940s grove ##yo completing focuses sum substitute bearing stretch reception ##py reflected essentially destination pairs ##ched survival resource ##bach promoting doubles messages tear ##down ##fully parade florence harvey incumbent partial framework 900 pedro frozen procedure olivia controls ##mic shelter personally temperatures ##od brisbane tested sits marble comprehensive oxygen leonard ##kov inaugural iranian referring quarters attitude ##ivity mainstream lined mars dakota norfolk unsuccessful ##° explosion helicopter congressional ##sing inspector bitch seal departed divine ##ters coaching examination punishment manufacturer sink columns unincorporated signals nevada squeezed dylan dining photos martial manuel eighteen elevator brushed plates ministers ivy congregation ##len slept specialized taxes curve restricted negotiations likes statistical arnold inspiration execution bold intermediate significance margin ruler wheels gothic intellectual dependent listened eligible buses widow syria earn cincinnati collapsed recipient secrets accessible philippine maritime goddess clerk surrender breaks playoff database ##ified ##lon ideal beetle aspect soap regulation strings expand anglo shorter crosses retreat tough coins wallace directions pressing ##oon shipping locomotives comparison topics nephew ##mes distinction honors travelled sierra ibn ##over fortress sa recognised carved 1869 clients ##dan intent ##mar coaches describing bread ##ington beaten northwestern ##ona merit youtube collapse challenges em historians objective submitted virus attacking drake assume ##ere diseases marc stem leeds ##cus ##ab farming glasses ##lock visits nowhere fellowship relevant carries restaurants experiments 101 constantly bases targets shah tenth opponents verse territorial ##ira writings corruption ##hs instruction inherited reverse emphasis ##vic employee arch keeps rabbi watson payment uh ##ala nancy ##tre venice fastest sexy banned adrian properly ruth touchdown dollar boards metre circles edges favour comments ok travels liberation scattered firmly ##ular holland permitted diesel kenya den originated ##ral demons resumed dragged rider ##rus servant blinked extend torn ##ias ##sey input meal everybody cylinder kinds camps ##fe bullet logic ##wn croatian evolved healthy fool chocolate wise preserve pradesh ##ess respective 1850 ##ew chicken artificial gross corresponding convicted cage caroline dialogue ##dor narrative stranger mario br christianity failing trent commanding buddhist 1848 maurice focusing yale bike altitude ##ering mouse revised ##sley veteran ##ig pulls theology crashed campaigns legion ##ability drag excellence customer cancelled intensity excuse ##lar liga participating contributing printing ##burn variable ##rk curious bin legacy renaissance ##my symptoms binding vocalist dancer ##nie grammar gospel democrats ya enters sc diplomatic hitler ##ser clouds mathematical quit defended oriented ##heim fundamental hardware impressive equally convince confederate guilt chuck sliding ##ware magnetic narrowed petersburg bulgaria otto phd skill ##ama reader hopes pitcher reservoir hearts automatically expecting mysterious bennett extensively imagined seeds monitor fix ##ative journalism struggling signature ranch encounter photographer observation protests ##pin influences ##hr calendar ##all cruz croatia locomotive hughes naturally shakespeare basement hook uncredited faded theories approaches dare phillips filling fury obama ##ain efficient arc deliver min raid breeding inducted leagues efficiency axis montana eagles ##ked supplied instructions karen picking indicating trap anchor practically christians tomb vary occasional electronics lords readers newcastle faint innovation collect situations engagement 160 claude mixture ##feld peer tissue logo lean ##ration °f floors ##ven architects reducing ##our ##ments rope 1859 ottawa ##har samples banking declaration proteins resignation francois saudi advocate exhibited armor twins divorce ##ras abraham reviewed jo temporarily matrix physically pulse curled ##ena difficulties bengal usage ##ban annie riders certificate ##pi holes warsaw distinctive jessica ##mon mutual 1857 customs circular eugene removal loaded mere vulnerable depicted generations dame heir enormous lightly climbing pitched lessons pilots nepal ram google preparing brad louise renowned ##₂ liam ##ably plaza shaw sophie brilliant bills ##bar ##nik fucking mainland server pleasant seized veterans jerked fail beta brush radiation stored warmth southeastern nate sin raced berkeley joke athlete designation trunk ##low roland qualification archives heels artwork receives judicial reserves ##bed woke installation abu floating fake lesser excitement interface concentrated addressed characteristic amanda saxophone monk auto ##bus releasing egg dies interaction defender ce outbreak glory loving ##bert sequel consciousness http awake ski enrolled ##ress handling rookie brow somebody biography warfare amounts contracts presentation fabric dissolved challenged meter psychological lt elevated rally accurate ##tha hospitals undergraduate specialist venezuela exhibit shed nursing protestant fluid structural footage jared consistent prey ##ska succession reflect exile lebanon wiped suspect shanghai resting integration preservation marvel variant pirates sheep rounded capita sailing colonies manuscript deemed variations clarke functional emerging boxing relaxed curse azerbaijan heavyweight nickname editorial rang grid tightened earthquake flashed miguel rushing ##ches improvements boxes brooks 180 consumption molecular felix societies repeatedly variation aids civic graphics professionals realm autonomous receiver delayed workshop militia chairs trump canyon ##point harsh extending lovely happiness ##jan stake eyebrows embassy wellington hannah ##ella sony corners bishops swear cloth contents xi namely commenced 1854 stanford nashville courage graphic commitment garrison ##bin hamlet clearing rebels attraction literacy cooking ruins temples jenny humanity celebrate hasn freight sixty rebel bastard ##art newton ##ada deer ##ges ##ching smiles delaware singers ##ets approaching assists flame ##ph boulevard barrel planted ##ome pursuit ##sia consequences posts shallow invitation rode depot ernest kane rod concepts preston topic chambers striking blast arrives descendants montgomery ranges worlds ##lay ##ari span chaos praise ##ag fewer 1855 sanctuary mud fbi ##ions programmes maintaining unity harper bore handsome closure tournaments thunder nebraska linda facade puts satisfied argentine dale cork dome panama ##yl 1858 tasks experts ##ates feeding equation ##las ##ida ##tu engage bryan ##ax um quartet melody disbanded sheffield blocked gasped delay kisses maggie connects ##non sts poured creator publishers ##we guided ellis extinct hug gaining ##ord complicated ##bility poll clenched investigate ##use thereby quantum spine cdp humor kills administered semifinals ##du encountered ignore ##bu commentary ##maker bother roosevelt 140 plains halfway flowing cultures crack imprisoned neighboring airline ##ses ##view ##mate ##ec gather wolves marathon transformed ##ill cruise organisations carol punch exhibitions numbered alarm ratings daddy silently ##stein queens colours impression guidance liu tactical ##rat marshal della arrow ##ings rested feared tender owns bitter advisor escort ##ides spare farms grants ##ene dragons encourage colleagues cameras ##und sucked pile spirits prague statements suspension landmark fence torture recreation bags permanently survivors pond spy predecessor bombing coup ##og protecting transformation glow ##lands ##book dug priests andrea feat barn jumping ##chen ##ologist ##con casualties stern auckland pipe serie revealing ba ##bel trevor mercy spectrum yang consist governing collaborated possessed epic comprises blew shane ##ack lopez honored magical sacrifice judgment perceived hammer mtv baronet tune das missionary sheets 350 neutral oral threatening attractive shade aims seminary ##master estates 1856 michel wounds refugees manufacturers ##nic mercury syndrome porter ##iya ##din hamburg identification upstairs purse widened pause cared breathed affiliate santiago prevented celtic fisher 125 recruited byzantine reconstruction farther ##mp diet sake au spite sensation ##ert blank separation 105 ##hon vladimir armies anime ##lie accommodate orbit cult sofia archive ##ify ##box founders sustained disorder honours northeastern mia crops violet threats blanket fires canton followers southwestern prototype voyage assignment altered moderate protocol pistol ##eo questioned brass lifting 1852 math authored ##ual doug dimensional dynamic ##san 1851 pronounced grateful quest uncomfortable boom presidency stevens relating politicians chen barrier quinn diana mosque tribal cheese palmer portions sometime chester treasure wu bend download millions reforms registration ##osa consequently monitoring ate preliminary brandon invented ps eaten exterior intervention ports documented log displays lecture sally favourite ##itz vermont lo invisible isle breed ##ator journalists relay speaks backward explore midfielder actively stefan procedures cannon blond kenneth centered servants chains libraries malcolm essex henri slavery ##hal facts fairy coached cassie cats washed cop ##fi announcement item 2000s vinyl activated marco frontier growled curriculum ##das loyal accomplished leslie ritual kenny ##00 vii napoleon hollow hybrid jungle stationed friedrich counted ##ulated platinum theatrical seated col rubber glen 1840 diversity healing extends id provisions administrator columbus ##oe tributary te assured org ##uous prestigious examined lectures grammy ronald associations bailey allan essays flute believing consultant proceedings travelling 1853 kit kerala yugoslavia buddy methodist ##ith burial centres batman ##nda discontinued bo dock stockholm lungs severely ##nk citing manga ##ugh steal mumbai iraqi robot celebrity bride broadcasts abolished pot joel overhead franz packed reconnaissance johann acknowledged introduce handled doctorate developments drinks alley palestine ##nis ##aki proceeded recover bradley grain patch afford infection nationalist legendary ##ath interchange virtually gen gravity exploration amber vital wishes powell doctrine elbow screenplay ##bird contribute indonesian pet creates ##com enzyme kylie discipline drops manila hunger ##ien layers suffer fever bits monica keyboard manages ##hood searched appeals ##bad testament grande reid ##war beliefs congo ##ification ##dia si requiring ##via casey 1849 regret streak rape depends syrian sprint pound tourists upcoming pub ##xi tense ##els practiced echo nationwide guild motorcycle liz ##zar chiefs desired elena bye precious absorbed relatives booth pianist ##mal citizenship exhausted wilhelm ##ceae ##hed noting quarterback urge hectares ##gue ace holly ##tal blonde davies parked sustainable stepping twentieth airfield galaxy nest chip ##nell tan shaft paulo requirement ##zy paradise tobacco trans renewed vietnamese ##cker ##ju suggesting catching holmes enjoying md trips colt holder butterfly nerve reformed cherry bowling trailer carriage goodbye appreciate toy joshua interactive enabled involve ##kan collar determination bunch facebook recall shorts superintendent episcopal frustration giovanni nineteenth laser privately array circulation ##ovic armstrong deals painful permit discrimination ##wi aires retiring cottage ni ##sta horizon ellen jamaica ripped fernando chapters playstation patron lecturer navigation behaviour genes georgian export solomon rivals swift seventeen rodriguez princeton independently sox 1847 arguing entity casting hank criteria oakland geographic milwaukee reflection expanding conquest dubbed ##tv halt brave brunswick doi arched curtis divorced predominantly somerset streams ugly zoo horrible curved buenos fierce dictionary vector theological unions handful stability chan punjab segments ##lly altar ignoring gesture monsters pastor ##stone thighs unexpected operators abruptly coin compiled associates improving migration pin ##ose compact collegiate reserved ##urs quarterfinals roster restore assembled hurry oval ##cies 1846 flags martha ##del victories sharply ##rated argues deadly neo drawings symbols performer ##iel griffin restrictions editing andrews java journals arabia compositions dee pierce removing hindi casino runway civilians minds nasa hotels ##zation refuge rent retain potentially conferences suburban conducting ##tto ##tions ##tle descended massacre ##cal ammunition terrain fork souls counts chelsea durham drives cab ##bank perth realizing palestinian finn simpson ##dal betty ##ule moreover particles cardinals tent evaluation extraordinary ##oid inscription ##works wednesday chloe maintains panels ashley trucks ##nation cluster sunlight strikes zhang ##wing dialect canon ##ap tucked ##ws collecting ##mas ##can ##sville maker quoted evan franco aria buying cleaning eva closet provision apollo clinic rat ##ez necessarily ac ##gle ##ising venues flipped cent spreading trustees checking authorized ##sco disappointed ##ado notion duration trumpet hesitated topped brussels rolls theoretical hint define aggressive repeat wash peaceful optical width allegedly mcdonald strict copyright ##illa investors mar jam witnesses sounding miranda michelle privacy hugo harmony ##pp valid lynn glared nina 102 headquartered diving boarding gibson ##ncy albanian marsh routine dealt enhanced er intelligent substance targeted enlisted discovers spinning observations pissed smoking rebecca capitol visa varied costume seemingly indies compensation surgeon thursday arsenal westminster suburbs rid anglican ##ridge knots foods alumni lighter fraser whoever portal scandal ##ray gavin advised instructor flooding terrorist ##ale teenage interim senses duck teen thesis abby eager overcome ##ile newport glenn rises shame ##cc prompted priority forgot bomber nicolas protective 360 cartoon katherine breeze lonely trusted henderson richardson relax banner candy palms remarkable ##rio legends cricketer essay ordained edmund rifles trigger ##uri ##away sail alert 1830 audiences penn sussex siblings pursued indianapolis resist rosa consequence succeed avoided 1845 ##ulation inland ##tie ##nna counsel profession chronicle hurried ##una eyebrow eventual bleeding innovative cure ##dom committees accounting con scope hardy heather tenor gut herald codes tore scales wagon ##oo luxury tin prefer fountain triangle bonds darling convoy dried traced beings troy accidentally slam findings smelled joey lawyers outcome steep bosnia configuration shifting toll brook performers lobby philosophical construct shrine aggregate boot cox phenomenon savage insane solely reynolds lifestyle ##ima nationally holdings consideration enable edgar mo mama ##tein fights relegation chances atomic hub conjunction awkward reactions currency finale kumar underwent steering elaborate gifts comprising melissa veins reasonable sunshine chi solve trails inhabited elimination ethics huh ana molly consent apartments layout marines ##ces hunters bulk ##oma hometown ##wall ##mont cracked reads neighbouring withdrawn admission wingspan damned anthology lancashire brands batting forgive cuban awful ##lyn 104 dimensions imagination ##ade dante ##ship tracking desperately goalkeeper ##yne groaned workshops confident burton gerald milton circus uncertain slope copenhagen sophia fog philosopher portraits accent cycling varying gripped larvae garrett specified scotia mature luther kurt rap ##kes aerial 750 ferdinand heated es transported ##shan safely nonetheless ##orn ##gal motors demanding ##sburg startled ##brook ally generate caps ghana stained demo mentions beds ap afterward diary ##bling utility ##iro richards 1837 conspiracy conscious shining footsteps observer cyprus urged loyalty developer probability olive upgraded gym miracle insects graves 1844 ourselves hydrogen amazon katie tickets poets ##pm planes ##pan prevention witnessed dense jin randy tang warehouse monroe bang archived elderly investigations alec granite mineral conflicts controlling aboriginal carlo ##zu mechanics stan stark rhode skirt est ##berry bombs respected ##horn imposed limestone deny nominee memphis grabbing disabled ##als amusement aa frankfurt corn referendum varies slowed disk firms unconscious incredible clue sue ##zhou twist ##cio joins idaho chad developers computing destroyer 103 mortal tucker kingston choices yu carson 1800 os whitney geneva pretend dimension staged plateau maya ##une freestyle ##bc rovers hiv ##ids tristan classroom prospect ##hus honestly diploma lied thermal auxiliary feast unlikely iata ##tel morocco pounding treasury lithuania considerably 1841 dish 1812 geological matching stumbled destroying marched brien advances cake nicole belle settling measuring directing ##mie tuesday bassist capabilities stunned fraud torpedo ##list ##phone anton wisdom surveillance ruined ##ulate lawsuit healthcare theorem halls trend aka horizontal dozens acquire lasting swim hawk gorgeous fees vicinity decrease adoption tactics ##ography pakistani ##ole draws ##hall willie burke heath algorithm integral powder elliott brigadier jackie tate varieties darker ##cho lately cigarette specimens adds ##ree ##ensis ##inger exploded finalist cia murders wilderness arguments nicknamed acceptance onwards manufacture robertson jets tampa enterprises blog loudly composers nominations 1838 ai malta inquiry automobile hosting viii rays tilted grief museums strategies furious euro equality cohen poison surrey wireless governed ridiculous moses ##esh ##room vanished ##ito barnes attract morrison istanbul ##iness absent rotation petition janet ##logical satisfaction custody deliberately observatory comedian surfaces pinyin novelist strictly canterbury oslo monks embrace ibm jealous photograph continent dorothy marina doc excess holden allegations explaining stack avoiding lance storyline majesty poorly spike dos bradford raven travis classics proven voltage pillow fists butt 1842 interpreted ##car 1839 gage telegraph lens promising expelled casual collector zones ##min silly nintendo ##kh ##bra downstairs chef suspicious afl flies vacant uganda pregnancy condemned lutheran estimates cheap decree saxon proximity stripped idiot deposits contrary presenter magnus glacier im offense edwin ##ori upright ##long bolt ##ois toss geographical ##izes environments delicate marking abstract xavier nails windsor plantation occurring equity saskatchewan fears drifted sequences vegetation revolt ##stic 1843 sooner fusion opposing nato skating 1836 secretly ruin lease ##oc edit ##nne flora anxiety ruby ##ological ##mia tel bout taxi emmy frost rainbow compounds foundations rainfall assassination nightmare dominican ##win achievements deserve orlando intact armenia ##nte calgary valentine 106 marion proclaimed theodore bells courtyard thigh gonzalez console troop minimal monte everyday ##ence ##if supporter terrorism buck openly presbyterian activists carpet ##iers rubbing uprising ##yi cute conceived legally ##cht millennium cello velocity ji rescued cardiff 1835 rex concentrate senators beard rendered glowing battalions scouts competitors sculptor catalogue arctic ion raja bicycle wow glancing lawn ##woman gentleman lighthouse publish predicted calculated ##val variants ##gne strain ##ui winston deceased ##nus touchdowns brady caleb sinking echoed crush hon blessed protagonist hayes endangered magnitude editors ##tine estimate responsibilities ##mel backup laying consumed sealed zurich lovers frustrated ##eau ahmed kicking mit treasurer 1832 biblical refuse terrified pump agrees genuine imprisonment refuses plymouth ##hen lou ##nen tara trembling antarctic ton learns ##tas crap crucial faction atop ##borough wrap lancaster odds hopkins erik lyon ##eon bros ##ode snap locality tips empress crowned cal acclaimed chuckled ##ory clara sends mild towel ##fl ##day ##а wishing assuming interviewed ##bal ##die interactions eden cups helena ##lf indie beck ##fire batteries filipino wizard parted ##lam traces ##born rows idol albany delegates ##ees ##sar discussions ##ex notre instructed belgrade highways suggestion lauren possess orientation alexandria abdul beats salary reunion ludwig alright wagner intimate pockets slovenia hugged brighton merchants cruel stole trek slopes repairs enrollment politically underlying promotional counting boeing ##bb isabella naming ##и keen bacteria listing separately belfast ussr 450 lithuanian anybody ribs sphere martinez cock embarrassed proposals fragments nationals ##fs ##wski premises fin 1500 alpine matched freely bounded jace sleeve ##af gaming pier populated evident ##like frances flooded ##dle frightened pour trainer framed visitor challenging pig wickets ##fold infected email ##pes arose ##aw reward ecuador oblast vale ch shuttle ##usa bach rankings forbidden cornwall accordance salem consumers bruno fantastic toes machinery resolved julius remembering propaganda iceland bombardment tide contacts wives ##rah concerto macdonald albania implement daisy tapped sudan helmet angela mistress ##lic crop sunk finest ##craft hostile ##ute ##tsu boxer fr paths adjusted habit ballot supervision soprano ##zen bullets wicked sunset regiments disappear lamp performs app ##gia ##oa rabbit digging incidents entries ##cion dishes ##oi introducing ##ati ##fied freshman slot jill tackles baroque backs ##iest lone sponsor destiny altogether convert ##aro consensus shapes demonstration basically feminist auction artifacts ##bing strongest twitter halifax 2019 allmusic mighty smallest precise alexandra viola ##los ##ille manuscripts ##illo dancers ari managers monuments blades barracks springfield maiden consolidated electron ##end berry airing wheat nobel inclusion blair payments geography bee cc eleanor react ##hurst afc manitoba ##yu su lineup fitness recreational investments airborne disappointment ##dis edmonton viewing ##row renovation ##cast infant bankruptcy roses aftermath pavilion ##yer carpenter withdrawal ladder ##hy discussing popped reliable agreements rochester ##abad curves bombers 220 rao reverend decreased choosing 107 stiff consulting naples crawford tracy ka ribbon cops ##lee crushed deciding unified teenager accepting flagship explorer poles sanchez inspection revived skilled induced exchanged flee locals tragedy swallow loading hanna demonstrate ##ela salvador flown contestants civilization ##ines wanna rhodes fletcher hector knocking considers ##ough nash mechanisms sensed mentally walt unclear ##eus renovated madame ##cks crews governmental ##hin undertaken monkey ##ben ##ato fatal armored copa caves governance grasp perception certification froze damp tugged wyoming ##rg ##ero newman ##lor nerves curiosity graph 115 ##ami withdraw tunnels dull meredith moss exhibits neighbors communicate accuracy explored raiders republicans secular kat superman penny criticised ##tch freed update conviction wade ham likewise delegation gotta doll promises technological myth nationality resolve convent ##mark sharon dig sip coordinator entrepreneur fold ##dine capability councillor synonym blown swan cursed 1815 jonas haired sofa canvas keeper rivalry ##hart rapper speedway swords postal maxwell estonia potter recurring ##nn ##ave errors ##oni cognitive 1834 ##² claws nadu roberto bce wrestler ellie ##ations infinite ink ##tia presumably finite staircase 108 noel patricia nacional ##cation chill eternal tu preventing prussia fossil limbs ##logist ernst frog perez rene ##ace pizza prussian ##ios ##vy molecules regulatory answering opinions sworn lengths supposedly hypothesis upward habitats seating ancestors drank yield hd synthesis researcher modest ##var mothers peered voluntary homeland ##the acclaim ##igan static valve luxembourg alto carroll fe receptor norton ambulance ##tian johnston catholics depicting jointly elephant gloria mentor badge ahmad distinguish remarked councils precisely allison advancing detection crowded ##10 cooperative ankle mercedes dagger surrendered pollution commit subway jeffrey lesson sculptures provider ##fication membrane timothy rectangular fiscal heating teammate basket particle anonymous deployment ##ple missiles courthouse proportion shoe sec ##ller complaints forbes blacks abandon remind sizes overwhelming autobiography natalie ##awa risks contestant countryside babies scorer invaded enclosed proceed hurling disorders ##cu reflecting continuously cruiser graduates freeway investigated ore deserved maid blocking phillip jorge shakes dove mann variables lacked burden accompanying que consistently organizing provisional complained endless ##rm tubes juice georges krishna mick labels thriller ##uch laps arcade sage snail ##table shannon fi laurence seoul vacation presenting hire churchill surprisingly prohibited savannah technically ##oli 170 ##lessly testimony suited speeds toys romans mlb flowering measurement talented kay settings charleston expectations shattered achieving triumph ceremonies portsmouth lanes mandatory loser stretching cologne realizes seventy cornell careers webb ##ulating americas budapest ava suspicion ##ison yo conrad ##hai sterling jessie rector ##az 1831 transform organize loans christine volcanic warrant slender summers subfamily newer danced dynamics rhine proceeds heinrich gastropod commands sings facilitate easter ra positioned responses expense fruits yanked imported 25th velvet vic primitive tribune baldwin neighbourhood donna rip hay pr ##uro 1814 espn welcomed ##aria qualifier glare highland timing ##cted shells eased geometry louder exciting slovakia ##sion ##iz ##lot savings prairie ##ques marching rafael tonnes ##lled curtain preceding shy heal greene worthy ##pot detachment bury sherman ##eck reinforced seeks bottles contracted duchess outfit walsh ##sc mickey ##ase geoffrey archer squeeze dawson eliminate invention ##enberg neal ##eth stance dealer coral maple retire polo simplified ##ht 1833 hid watts backwards jules ##oke genesis mt frames rebounds burma woodland moist santos whispers drained subspecies ##aa streaming ulster burnt correspondence maternal gerard denis stealing ##load genius duchy ##oria inaugurated momentum suits placement sovereign clause thames ##hara confederation reservation sketch yankees lets rotten charm hal verses ultra commercially dot salon citation adopt winnipeg mist allocated cairo ##boy jenkins interference objectives ##wind 1820 portfolio armoured sectors ##eh initiatives ##world integrity exercises robe tap ab gazed ##tones distracted rulers 111 favorable jerome tended cart factories ##eri diplomat valued gravel charitable ##try calvin exploring chang shepherd terrace pdf pupil ##ural reflects ups ##rch governors shelf depths ##nberg trailed crest tackle ##nian ##ats hatred ##kai clare makers ethiopia longtime detected embedded lacking slapped rely thomson anticipation iso morton successive agnes screenwriter straightened philippe playwright haunted licence iris intentions sutton 112 logical correctly ##weight branded licked tipped silva ricky narrator requests ##ents greeted supernatural cow ##wald lung refusing employer strait gaelic liner ##piece zoe sabha ##mba driveway harvest prints bates reluctantly threshold algebra ira wherever coupled 240 assumption picks ##air designers raids gentlemen ##ean roller blowing leipzig locks screw dressing strand ##lings scar dwarf depicts ##nu nods ##mine differ boris ##eur yuan flip ##gie mob invested questioning applying ##ture shout ##sel gameplay blamed illustrations bothered weakness rehabilitation ##of ##zes envelope rumors miners leicester subtle kerry ##ico ferguson ##fu premiership ne ##cat bengali prof catches remnants dana ##rily shouting presidents baltic ought ghosts dances sailors shirley fancy dominic ##bie madonna ##rick bark buttons gymnasium ashes liver toby oath providence doyle evangelical nixon cement carnegie embarked hatch surroundings guarantee needing pirate essence ##bee filter crane hammond projected immune percy twelfth ##ult regent doctoral damon mikhail ##ichi lu critically elect realised abortion acute screening mythology steadily ##fc frown nottingham kirk wa minneapolis ##rra module algeria mc nautical encounters surprising statues availability shirts pie alma brows munster mack soup crater tornado sanskrit cedar explosive bordered dixon planets stamp exam happily ##bble carriers kidnapped ##vis accommodation emigrated ##met knockout correspondent violation profits peaks lang specimen agenda ancestry pottery spelling equations obtaining ki linking 1825 debris asylum ##20 buddhism teddy ##ants gazette ##nger ##sse dental eligibility utc fathers averaged zimbabwe francesco coloured hissed translator lynch mandate humanities mackenzie uniforms lin ##iana ##gio asset mhz fitting samantha genera wei rim beloved shark riot entities expressions indo carmen slipping owing abbot neighbor sidney ##av rats recommendations encouraging squadrons anticipated commanders conquered ##oto donations diagnosed ##mond divide ##iva guessed decoration vernon auditorium revelation conversations ##kers ##power herzegovina dash alike protested lateral herman accredited mg ##gent freeman mel fiji crow crimson ##rine livestock ##pped humanitarian bored oz whip ##lene ##ali legitimate alter grinning spelled anxious oriental wesley ##nin ##hole carnival controller detect ##ssa bowed educator kosovo macedonia ##sin occupy mastering stephanie janeiro para unaware nurses noon 135 cam hopefully ranger combine sociology polar rica ##eer neill ##sman holocaust ##ip doubled lust 1828 109 decent cooling unveiled ##card 1829 nsw homer chapman meyer ##gin dive mae reagan expertise ##gled darwin brooke sided prosecution investigating comprised petroleum genres reluctant differently trilogy johns vegetables corpse highlighted lounge pension unsuccessfully elegant aided ivory beatles amelia cain dubai sunny immigrant babe click ##nder underwater pepper combining mumbled atlas horns accessed ballad physicians homeless gestured rpm freak louisville corporations patriots prizes rational warn modes decorative overnight din troubled phantom ##ort monarch sheer ##dorf generals guidelines organs addresses ##zon enhance curling parishes cord ##kie linux caesar deutsche bavaria ##bia coleman cyclone ##eria bacon petty ##yama ##old hampton diagnosis 1824 throws complexity rita disputed ##₃ pablo ##sch marketed trafficking ##ulus examine plague formats ##oh vault faithful ##bourne webster ##ox highlights ##ient ##ann phones vacuum sandwich modeling ##gated bolivia clergy qualities isabel ##nas ##ars wears screams reunited annoyed bra ##ancy ##rate differential transmitter tattoo container poker ##och excessive resides cowboys ##tum augustus trash providers statute retreated balcony reversed void storey preceded masses leap laughs neighborhoods wards schemes falcon santo battlefield pad ronnie thread lesbian venus ##dian beg sandstone daylight punched gwen analog stroked wwe acceptable measurements dec toxic ##kel adequate surgical economist parameters varsity ##sberg quantity ella ##chy ##rton countess generating precision diamonds expressway ga ##ı 1821 uruguay talents galleries expenses scanned colleague outlets ryder lucien ##ila paramount ##bon syracuse dim fangs gown sweep ##sie toyota missionaries websites ##nsis sentences adviser val trademark spells ##plane patience starter slim ##borg toe incredibly shoots elliot nobility ##wyn cowboy endorsed gardner tendency persuaded organisms emissions kazakhstan amused boring chips themed ##hand llc constantinople chasing systematic guatemala borrowed erin carey ##hard highlands struggles 1810 ##ifying ##ced wong exceptions develops enlarged kindergarten castro ##ern ##rina leigh zombie juvenile ##most consul ##nar sailor hyde clarence intensive pinned nasty useless jung clayton stuffed exceptional ix apostolic 230 transactions ##dge exempt swinging cove religions ##ash shields dairy bypass 190 pursuing bug joyce bombay chassis southampton chat interact redesignated ##pen nascar pray salmon rigid regained malaysian grim publicity constituted capturing toilet delegate purely tray drift loosely striker weakened trinidad mitch itv defines transmitted ming scarlet nodding fitzgerald fu narrowly sp tooth standings virtue ##₁ ##wara ##cting chateau gloves lid ##nel hurting conservatory ##pel sinclair reopened sympathy nigerian strode advocated optional chronic discharge ##rc suck compatible laurel stella shi fails wage dodge 128 informal sorts levi buddha villagers ##aka chronicles heavier summoned gateway 3000 eleventh jewelry translations accordingly seas ##ency fiber pyramid cubic dragging ##ista caring ##ops android contacted lunar ##dt kai lisbon patted 1826 sacramento theft madagascar subtropical disputes ta holidays piper willow mare cane itunes newfoundland benny companions dong raj observe roar charming plaque tibetan fossils enacted manning bubble tina tanzania ##eda ##hir funk swamp deputies cloak ufc scenario par scratch metals anthem guru engaging specially ##boat dialects nineteen cecil duet disability messenger unofficial ##lies defunct eds moonlight drainage surname puzzle honda switching conservatives mammals knox broadcaster sidewalk cope ##ried benson princes peterson ##sal bedford sharks eli wreck alberto gasp archaeology lgbt teaches securities madness compromise waving coordination davidson visions leased possibilities eighty jun fernandez enthusiasm assassin sponsorship reviewer kingdoms estonian laboratories ##fy ##nal applies verb celebrations ##zzo rowing lightweight sadness submit mvp balanced dude ##vas explicitly metric magnificent mound brett mohammad mistakes irregular ##hing ##ass sanders betrayed shipped surge ##enburg reporters termed georg pity verbal bulls abbreviated enabling appealed ##are ##atic sicily sting heel sweetheart bart spacecraft brutal monarchy ##tter aberdeen cameo diane ##ub survivor clyde ##aries complaint ##makers clarinet delicious chilean karnataka coordinates 1818 panties ##rst pretending ar dramatically kiev bella tends distances 113 catalog launching instances telecommunications portable lindsay vatican ##eim angles aliens marker stint screens bolton ##rne judy wool benedict plasma europa spark imaging filmmaker swiftly ##een contributor ##nor opted stamps apologize financing butter gideon sophisticated alignment avery chemicals yearly speculation prominence professionally ##ils immortal institutional inception wrists identifying tribunal derives gains ##wo papal preference linguistic vince operative brewery ##ont unemployment boyd ##ured ##outs albeit prophet 1813 bi ##rr ##face ##rad quarterly asteroid cleaned radius temper ##llen telugu jerk viscount menu ##ote glimpse ##aya yacht hawaiian baden ##rl laptop readily ##gu monetary offshore scots watches ##yang ##arian upgrade needle xbox lea encyclopedia flank fingertips ##pus delight teachings confirm roth beaches midway winters ##iah teasing daytime beverly gambling bonnie ##backs regulated clement hermann tricks knot ##shing ##uring ##vre detached ecological owed specialty byron inventor bats stays screened unesco midland trim affection ##ander ##rry jess thoroughly feedback ##uma chennai strained heartbeat wrapping overtime pleaded ##sworth mon leisure oclc ##tate ##ele feathers angelo thirds nuts surveys clever gill commentator ##dos darren rides gibraltar ##nc ##mu dissolution dedication shin meals saddle elvis reds chaired taller appreciation functioning niece favored advocacy robbie criminals suffolk yugoslav passport constable congressman hastings vera ##rov consecrated sparks ecclesiastical confined ##ovich muller floyd nora 1822 paved 1827 cumberland ned saga spiral ##flow appreciated yi collaborative treating similarities feminine finishes ##ib jade import ##nse ##hot champagne mice securing celebrities helsinki attributes ##gos cousins phases ache lucia gandhi submission vicar spear shine tasmania biting detention constitute tighter seasonal ##gus terrestrial matthews ##oka effectiveness parody philharmonic ##onic 1816 strangers encoded consortium guaranteed regards shifts tortured collision supervisor inform broader insight theaters armour emeritus blink incorporates mapping ##50 ##ein handball flexible ##nta substantially generous thief ##own carr loses 1793 prose ucla romeo generic metallic realization damages mk commissioners zach default ##ther helicopters lengthy stems spa partnered spectators rogue indication penalties teresa 1801 sen ##tric dalton ##wich irving photographic ##vey dell deaf peters excluded unsure ##vable patterson crawled ##zio resided whipped latvia slower ecole pipes employers maharashtra comparable va textile pageant ##gel alphabet binary irrigation chartered choked antoine offs waking supplement ##wen quantities demolition regain locate urdu folks alt 114 ##mc scary andreas whites ##ava classrooms mw aesthetic publishes valleys guides cubs johannes bryant conventions affecting ##itt drain awesome isolation prosecutor ambitious apology captive downs atmospheric lorenzo aisle beef foul ##onia kidding composite disturbed illusion natives ##ffer emi rockets riverside wartime painters adolf melted ##ail uncertainty simulation hawks progressed meantime builder spray breach unhappy regina russians ##urg determining ##tation tram 1806 ##quin aging ##12 1823 garion rented mister diaz terminated clip 1817 depend nervously disco owe defenders shiva notorious disbelief shiny worcester ##gation ##yr trailing undertook islander belarus limitations watershed fuller overlooking utilized raphael 1819 synthetic breakdown klein ##nate moaned memoir lamb practicing ##erly cellular arrows exotic ##graphy witches 117 charted rey hut hierarchy subdivision freshwater giuseppe aloud reyes qatar marty sideways utterly sexually jude prayers mccarthy softball blend damien ##gging ##metric wholly erupted lebanese negro revenues tasted comparative teamed transaction labeled maori sovereignty parkway trauma gran malay 121 advancement descendant 2020 buzz salvation inventory symbolic ##making antarctica mps ##gas ##bro mohammed myanmar holt submarines tones ##lman locker patriarch bangkok emerson remarks predators kin afghan confession norwich rental emerge advantages ##zel rca ##hold shortened storms aidan ##matic autonomy compliance ##quet dudley atp ##osis 1803 motto documentation summary professors spectacular christina archdiocese flashing innocence remake ##dell psychic reef scare employ rs sticks meg gus leans ##ude accompany bergen tomas ##iko doom wages pools ##nch ##bes breasts scholarly alison outline brittany breakthrough willis realistic ##cut ##boro competitor ##stan pike picnic icon designing commercials washing villain skiing micro costumes auburn halted executives ##hat logistics cycles vowel applicable barrett exclaimed eurovision eternity ramon ##umi ##lls modifications sweeping disgust ##uck torch aviv ensuring rude dusty sonic donovan outskirts cu pathway ##band ##gun ##lines disciplines acids cadet paired ##40 sketches ##sive marriages ##⁺ folding peers slovak implies admired ##beck 1880s leopold instinct attained weston megan horace ##ination dorsal ingredients evolutionary ##its complications deity lethal brushing levy deserted institutes posthumously delivering telescope coronation motivated rapids luc flicked pays volcano tanner weighed ##nica crowds frankie gifted addressing granddaughter winding ##rna constantine gomez ##front landscapes rudolf anthropology slate werewolf ##lio astronomy circa rouge dreaming sack knelt drowned naomi prolific tracked freezing herb ##dium agony randall twisting wendy deposit touches vein wheeler ##bbled ##bor batted retaining tire presently compare specification daemon nigel ##grave merry recommendation czechoslovakia sandra ng roma ##sts lambert inheritance sheikh winchester cries examining ##yle comeback cuisine nave ##iv ko retrieve tomatoes barker polished defining irene lantern personalities begging tract swore 1809 175 ##gic omaha brotherhood ##rley haiti ##ots exeter ##ete ##zia steele dumb pearson 210 surveyed elisabeth trends ##ef fritz ##rf premium bugs fraction calmly viking ##birds tug inserted unusually ##ield confronted distress crashing brent turks resign ##olo cambodia gabe sauce ##kal evelyn 116 extant clusters quarry teenagers luna ##lers ##ister affiliation drill ##ashi panthers scenic libya anita strengthen inscriptions ##cated lace sued judith riots ##uted mint ##eta preparations midst dub challenger ##vich mock cf displaced wicket breaths enables schmidt analyst ##lum ag highlight automotive axe josef newark sufficiently resembles 50th ##pal flushed mum traits ##ante commodore incomplete warming titular ceremonial ethical 118 celebrating eighteenth cao lima medalist mobility strips snakes ##city miniature zagreb barton escapes umbrella automated doubted differs cooled georgetown dresden cooked fade wyatt rna jacobs carlton abundant stereo boost madras inning ##hia spur ip malayalam begged osaka groan escaping charging dose vista ##aj bud papa communists advocates edged tri ##cent resemble peaking necklace fried montenegro saxony goose glances stuttgart curator recruit grocery sympathetic ##tting ##fort 127 lotus randolph ancestor ##rand succeeding jupiter 1798 macedonian ##heads hiking 1808 handing fischer ##itive garbage node ##pies prone singular papua inclined attractions italia pouring motioned grandma garnered jacksonville corp ego ringing aluminum ##hausen ordering ##foot drawer traders synagogue ##play ##kawa resistant wandering fragile fiona teased var hardcore soaked jubilee decisive exposition mercer poster valencia hale kuwait 1811 ##ises ##wr ##eed tavern gamma 122 johan ##uer airways amino gil ##ury vocational domains torres ##sp generator folklore outcomes ##keeper canberra shooter fl beams confrontation ##lling ##gram feb aligned forestry pipeline jax motorway conception decay ##tos coffin ##cott stalin 1805 escorted minded ##nam sitcom purchasing twilight veronica additions passive tensions straw 123 frequencies 1804 refugee cultivation ##iate christie clary bulletin crept disposal ##rich ##zong processor crescent ##rol bmw emphasized whale nazis aurora ##eng dwelling hauled sponsors toledo mega ideology theatres tessa cerambycidae saves turtle cone suspects kara rusty yelling greeks mozart shades cocked participant ##tro shire spit freeze necessity ##cos inmates nielsen councillors loaned uncommon omar peasants botanical offspring daniels formations jokes 1794 pioneers sigma licensing ##sus wheelchair polite 1807 liquor pratt trustee ##uta forewings balloon ##zz kilometre camping explicit casually shawn foolish teammates nm hassan carrie judged satisfy vanessa knives selective cnn flowed ##lice eclipse stressed eliza mathematician cease cultivated ##roy commissions browns ##ania destroyers sheridan meadow ##rius minerals ##cial downstream clash gram memoirs ventures baha seymour archie midlands edith fare flynn invite canceled tiles stabbed boulder incorporate amended camden facial mollusk unreleased descriptions yoga grabs 550 raises ramp shiver ##rose coined pioneering tunes qing warwick tops 119 melanie giles ##rous wandered ##inal annexed nov 30th unnamed ##ished organizational airplane normandy stoke whistle blessing violations chased holders shotgun ##ctic outlet reactor ##vik tires tearing shores fortified mascot constituencies nc columnist productive tibet ##rta lineage hooked oct tapes judging cody ##gger hansen kashmir triggered ##eva solved cliffs ##tree resisted anatomy protesters transparent implied ##iga injection mattress excluding ##mbo defenses helpless devotion ##elli growl liberals weber phenomena atoms plug ##iff mortality apprentice howe convincing aaa swimmer barber leone promptly sodium def nowadays arise ##oning gloucester corrected dignity norm erie ##ders elders evacuated sylvia compression ##yar hartford pose backpack reasoning accepts 24th wipe millimetres marcel ##oda dodgers albion 1790 overwhelmed aerospace oaks 1795 showcase acknowledge recovering nolan ashe hurts geology fashioned disappearance farewell swollen shrug marquis wimbledon 124 rue 1792 commemorate reduces experiencing inevitable calcutta intel ##court murderer sticking fisheries imagery bloom 280 brake ##inus gustav hesitation memorable po viral beans accidents tunisia antenna spilled consort treatments aye perimeter ##gard donation hostage migrated banker addiction apex lil trout ##ously conscience ##nova rams sands genome passionate troubles ##lets ##set amid ##ibility ##ret higgins exceed vikings ##vie payne ##zan muscular ##ste defendant sucking ##wal ibrahim fuselage claudia vfl europeans snails interval ##garh preparatory statewide tasked lacrosse viktor ##lation angola ##hra flint implications employs teens patrons stall weekends barriers scrambled nucleus tehran jenna parsons lifelong robots displacement 5000 ##bles precipitation ##gt knuckles clutched 1802 marrying ecology marx accusations declare scars kolkata mat meadows bermuda skeleton finalists vintage crawl coordinate affects subjected orchestral mistaken ##tc mirrors dipped relied 260 arches candle ##nick incorporating wildly fond basilica owl fringe rituals whispering stirred feud tertiary slick goat honorable whereby skip ricardo stripes parachute adjoining submerged synthesizer ##gren intend positively ninety phi beaver partition fellows alexis prohibition carlisle bizarre fraternity ##bre doubts icy cbc aquatic sneak sonny combines airports crude supervised spatial merge alfonso ##bic corrupt scan undergo ##ams disabilities colombian comparing dolphins perkins ##lish reprinted unanimous bounced hairs underworld midwest semester bucket paperback miniseries coventry demise ##leigh demonstrations sensor rotating yan ##hler arrange soils ##idge hyderabad labs ##dr brakes grandchildren ##nde negotiated rover ferrari continuation directorate augusta stevenson counterpart gore ##rda nursery rican ave collectively broadly pastoral repertoire asserted discovering nordic styled fiba cunningham harley middlesex survives tumor tempo zack aiming lok urgent ##rade ##nto devils ##ement contractor turin ##wl ##ool bliss repaired simmons moan astronomical cr negotiate lyric 1890s lara bred clad angus pbs ##ience engineered posed ##lk hernandez possessions elbows psychiatric strokes confluence electorate lifts campuses lava alps ##ep ##ution ##date physicist woody ##page ##ographic ##itis juliet reformation sparhawk 320 complement suppressed jewel ##½ floated ##kas continuity sadly ##ische inability melting scanning paula flour judaism safer vague ##lm solving curb ##stown financially gable bees expired miserable cassidy dominion 1789 cupped 145 robbery facto amos warden resume tallest marvin ing pounded usd declaring gasoline ##aux darkened 270 650 sophomore ##mere erection gossip televised risen dial ##eu pillars ##link passages profound ##tina arabian ashton silicon nail ##ead ##lated ##wer ##hardt fleming firearms ducked circuits blows waterloo titans ##lina atom fireplace cheshire financed activation algorithms ##zzi constituent catcher cherokee partnerships sexuality platoon tragic vivian guarded whiskey meditation poetic ##late ##nga ##ake porto listeners dominance kendra mona chandler factions 22nd salisbury attitudes derivative ##ido ##haus intake paced javier illustrator barrels bias cockpit burnett dreamed ensuing ##anda receptors someday hawkins mattered ##lal slavic 1799 jesuit cameroon wasted tai wax lowering victorious freaking outright hancock librarian sensing bald calcium myers tablet announcing barack shipyard pharmaceutical ##uan greenwich flush medley patches wolfgang pt speeches acquiring exams nikolai ##gg hayden kannada ##type reilly ##pt waitress abdomen devastated capped pseudonym pharmacy fulfill paraguay 1796 clicked ##trom archipelago syndicated ##hman lumber orgasm rejection clifford lorraine advent mafia rodney brock ##ght ##used ##elia cassette chamberlain despair mongolia sensors developmental upstream ##eg ##alis spanning 165 trombone basque seeded interred renewable rhys leapt revision molecule ##ages chord vicious nord shivered 23rd arlington debts corpus sunrise bays blackburn centimetres ##uded shuddered gm strangely gripping cartoons isabelle orbital ##ppa seals proving ##lton refusal strengthened bust assisting baghdad batsman portrayal mara pushes spears og ##cock reside nathaniel brennan 1776 confirmation caucus ##worthy markings yemen nobles ku lazy viewer catalan encompasses sawyer ##fall sparked substances patents braves arranger evacuation sergio persuade dover tolerance penguin cum jockey insufficient townships occupying declining plural processed projection puppet flanders introduces liability ##yon gymnastics antwerp taipei hobart candles jeep wes observers 126 chaplain bundle glorious ##hine hazel flung sol excavations dumped stares sh bangalore triangular icelandic intervals expressing turbine ##vers songwriting crafts ##igo jasmine ditch rite ##ways entertaining comply sorrow wrestlers basel emirates marian rivera helpful ##some caution downward networking ##atory ##tered darted genocide emergence replies specializing spokesman convenient unlocked fading augustine concentrations resemblance elijah investigator andhra ##uda promotes bean ##rrell fleeing wan simone announcer ##ame ##bby lydia weaver 132 residency modification ##fest stretches ##ast alternatively nat lowe lacks ##ented pam tile concealed inferior abdullah residences tissues vengeance ##ided moisture peculiar groove zip bologna jennings ninja oversaw zombies pumping batch livingston emerald installations 1797 peel nitrogen rama ##fying ##star schooling strands responding werner ##ost lime casa accurately targeting ##rod underway ##uru hemisphere lester ##yard occupies 2d griffith angrily reorganized ##owing courtney deposited ##dd ##30 estadio ##ifies dunn exiled ##ying checks ##combe ##о ##fly successes unexpectedly blu assessed ##flower ##ه observing sacked spiders kn ##tail mu nodes prosperity audrey divisional 155 broncos tangled adjust feeds erosion paolo surf directory snatched humid admiralty screwed gt reddish ##nese modules trench lamps bind leah bucks competes ##nz ##form transcription ##uc isles violently clutching pga cyclist inflation flats ragged unnecessary ##hian stubborn coordinated harriet baba disqualified 330 insect wolfe ##fies reinforcements rocked duel winked embraced bricks ##raj hiatus defeats pending brightly jealousy ##xton ##hm ##uki lena gdp colorful ##dley stein kidney ##shu underwear wanderers ##haw ##icus guardians m³ roared habits ##wise permits gp uranium punished disguise bundesliga elise dundee erotic partisan pi collectors float individually rendering behavioral bucharest ser hare valerie corporal nutrition proportional ##isa immense ##kis pavement ##zie ##eld sutherland crouched 1775 ##lp suzuki trades endurance operas crosby prayed priory rory socially ##urn gujarat ##pu walton cube pasha privilege lennon floods thorne waterfall nipple scouting approve ##lov minorities voter dwight extensions assure ballroom slap dripping privileges rejoined confessed demonstrating patriotic yell investor ##uth pagan slumped squares ##cle ##kins confront bert embarrassment ##aid aston urging sweater starr yuri brains williamson commuter mortar structured selfish exports ##jon cds ##him unfinished ##rre mortgage destinations ##nagar canoe solitary buchanan delays magistrate fk ##pling motivation ##lier ##vier recruiting assess ##mouth malik antique 1791 pius rahman reich tub zhou smashed airs galway xii conditioning honduras discharged dexter ##pf lionel 129 debates lemon tiffany volunteered dom dioxide procession devi sic tremendous advertisements colts transferring verdict hanover decommissioned utter relate pac racism ##top beacon limp similarity terra occurrence ant ##how becky capt updates armament richie pal ##graph halloween mayo ##ssen ##bone cara serena fcc dolls obligations ##dling violated lafayette jakarta exploitation ##ime infamous iconic ##lah ##park kitty moody reginald dread spill crystals olivier modeled bluff equilibrium separating notices ordnance extinction onset cosmic attachment sammy expose privy anchored ##bil abbott admits bending baritone emmanuel policeman vaughan winged climax dresses denny polytechnic mohamed burmese authentic nikki genetics grandparents homestead gaza postponed metacritic una ##sby ##bat unstable dissertation ##rial ##cian curls obscure uncovered bronx praying disappearing ##hoe prehistoric coke turret mutations nonprofit pits monaco ##ي ##usion prominently dispatched podium ##mir uci ##uation 133 fortifications birthplace kendall ##lby ##oll preacher rack goodman ##rman persistent ##ott countless jaime recorder lexington persecution jumps renewal wagons ##11 crushing ##holder decorations ##lake abundance wrath laundry £1 garde ##rp jeanne beetles peasant ##sl splitting caste sergei ##rer ##ema scripts ##ively rub satellites ##vor inscribed verlag scrapped gale packages chick potato slogan kathleen arabs ##culture counterparts reminiscent choral ##tead rand retains bushes dane accomplish courtesy closes ##oth slaughter hague krakow lawson tailed elias ginger ##ttes canopy betrayal rebuilding turf ##hof frowning allegiance brigades kicks rebuild polls alias nationalism td rowan audition bowie fortunately recognizes harp dillon horrified ##oro renault ##tics ropes ##α presumed rewarded infrared wiping accelerated illustration ##rid presses practitioners badminton ##iard detained ##tera recognizing relates misery ##sies ##tly reproduction piercing potatoes thornton esther manners hbo ##aan ours bullshit ernie perennial sensitivity illuminated rupert ##jin ##iss ##ear rfc nassau ##dock staggered socialism ##haven appointments nonsense prestige sharma haul ##tical solidarity gps ##ook ##rata igor pedestrian ##uit baxter tenants wires medication unlimited guiding impacts diabetes ##rama sasha pas clive extraction 131 continually constraints ##bilities sonata hunted sixteenth chu planting quote mayer pretended abs spat ##hua ceramic ##cci curtains pigs pitching ##dad latvian sore dayton ##sted ##qi patrols slice playground ##nted shone stool apparatus inadequate mates treason ##ija desires ##liga ##croft somalia laurent mir leonardo oracle grape obliged chevrolet thirteenth stunning enthusiastic ##ede accounted concludes currents basil ##kovic drought ##rica mai ##aire shove posting ##shed pilgrimage humorous packing fry pencil wines smells 144 marilyn aching newest clung bon neighbours sanctioned ##pie mug ##stock drowning ##mma hydraulic ##vil hiring reminder lilly investigators ##ncies sour ##eous compulsory packet ##rion ##graphic ##elle cannes ##inate depressed ##rit heroic importantly theresa ##tled conway saturn marginal rae ##xia corresponds royce pact jasper explosives packaging aluminium ##ttered denotes rhythmic spans assignments hereditary outlined originating sundays lad reissued greeting beatrice ##dic pillar marcos plots handbook alcoholic judiciary avant slides extract masculine blur ##eum ##force homage trembled owens hymn trey omega signaling socks accumulated reacted attic theo lining angie distraction primera talbot ##key 1200 ti creativity billed ##hey deacon eduardo identifies proposition dizzy gunner hogan ##yam ##pping ##hol ja ##chan jensen reconstructed ##berger clearance darius ##nier abe harlem plea dei circled emotionally notation fascist neville exceeded upwards viable ducks ##fo workforce racer limiting shri ##lson possesses 1600 kerr moths devastating laden disturbing locking ##cture gal fearing accreditation flavor aide 1870s mountainous ##baum melt ##ures motel texture servers soda ##mb herd ##nium erect puzzled hum peggy examinations gould testified geoff ren devised sacks ##law denial posters grunted cesar tutor ec gerry offerings byrne falcons combinations ct incoming pardon rocking 26th avengers flared mankind seller uttar loch nadia stroking exposing ##hd fertile ancestral instituted ##has noises prophecy taxation eminent vivid pol ##bol dart indirect multimedia notebook upside displaying adrenaline referenced geometric ##iving progression ##ddy blunt announce ##far implementing ##lav aggression liaison cooler cares headache plantations gorge dots impulse thickness ashamed averaging kathy obligation precursor 137 fowler symmetry thee 225 hears ##rai undergoing ads butcher bowler ##lip cigarettes subscription goodness ##ically browne ##hos ##tech kyoto donor ##erty damaging friction drifting expeditions hardened prostitution 152 fauna blankets claw tossing snarled butterflies recruits investigative coated healed 138 communal hai xiii academics boone psychologist restless lahore stephens mba brendan foreigners printer ##pc ached explode 27th deed scratched dared ##pole cardiac 1780 okinawa proto commando compelled oddly electrons ##base replica thanksgiving ##rist sheila deliberate stafford tidal representations hercules ou ##path ##iated kidnapping lenses ##tling deficit samoa mouths consuming computational maze granting smirk razor fixture ideals inviting aiden nominal ##vs issuing julio pitt ramsey docks ##oss exhaust ##owed bavarian draped anterior mating ethiopian explores noticing ##nton discarded convenience hoffman endowment beasts cartridge mormon paternal probe sleeves interfere lump deadline ##rail jenks bulldogs scrap alternating justified reproductive nam seize descending secretariat kirby coupe grouped smash panther sedan tapping ##18 lola cheer germanic unfortunate ##eter unrelated ##fan subordinate ##sdale suzanne advertisement ##ility horsepower ##lda cautiously discourse luigi ##mans ##fields noun prevalent mao schneider everett surround governorate kira ##avia westward ##take misty rails sustainability 134 unused ##rating packs toast unwilling regulate thy suffrage nile awe assam definitions travelers affordable ##rb conferred sells undefeated beneficial torso basal repeating remixes ##pass bahrain cables fang ##itated excavated numbering statutory ##rey deluxe ##lian forested ramirez derbyshire zeus slamming transfers astronomer banana lottery berg histories bamboo ##uchi resurrection posterior bowls vaguely ##thi thou preserving tensed offence ##inas meyrick callum ridden watt langdon tying lowland snorted daring truman ##hale ##girl aura overly filing weighing goa infections philanthropist saunders eponymous ##owski latitude perspectives reviewing mets commandant radial ##kha flashlight reliability koch vowels amazed ada elaine supper ##rth ##encies predator debated soviets cola ##boards ##nah compartment crooked arbitrary fourteenth ##ctive havana majors steelers clips profitable ambush exited packers ##tile nude cracks fungi ##е limb trousers josie shelby tens frederic ##ος definite smoothly constellation insult baton discs lingering ##nco conclusions lent staging becker grandpa shaky ##tron einstein obstacles sk adverse elle economically ##moto mccartney thor dismissal motions readings nostrils treatise ##pace squeezing evidently prolonged 1783 venezuelan je marguerite beirut takeover shareholders ##vent denise digit airplay norse ##bbling imaginary pills hubert blaze vacated eliminating ##ello vine mansfield ##tty retrospective barrow borne clutch bail forensic weaving ##nett ##witz desktop citadel promotions worrying dorset ieee subdivided ##iating manned expeditionary pickup synod chuckle 185 barney ##rz ##ffin functionality karachi litigation meanings uc lick turbo anders ##ffed execute curl oppose ankles typhoon ##د ##ache ##asia linguistics compassion pressures grazing perfection ##iting immunity monopoly muddy backgrounds 136 namibia francesca monitors attracting stunt tuition ##ии vegetable ##mates ##quent mgm jen complexes forts ##ond cellar bites seventeenth royals flemish failures mast charities ##cular peruvian capitals macmillan ipswich outward frigate postgraduate folds employing ##ouse concurrently fiery ##tai contingent nightmares monumental nicaragua ##kowski lizard mal fielding gig reject ##pad harding ##ipe coastline ##cin ##nos beethoven humphrey innovations ##tam ##nge norris doris solicitor huang obey 141 ##lc niagara ##tton shelves aug bourbon curry nightclub specifications hilton ##ndo centennial dispersed worm neglected briggs sm font kuala uneasy plc ##nstein ##bound ##aking ##burgh awaiting pronunciation ##bbed ##quest eh optimal zhu raped greens presided brenda worries ##life venetian marxist turnout ##lius refined braced sins grasped sunderland nickel speculated lowell cyrillic communism fundraising resembling colonists mutant freddie usc ##mos gratitude ##run mural ##lous chemist wi reminds 28th steals tess pietro ##ingen promoter ri microphone honoured rai sant ##qui feather ##nson burlington kurdish terrorists deborah sickness ##wed ##eet hazard irritated desperation veil clarity ##rik jewels xv ##gged ##ows ##cup berkshire unfair mysteries orchid winced exhaustion renovations stranded obe infinity ##nies adapt redevelopment thanked registry olga domingo noir tudor ole ##atus commenting behaviors ##ais crisp pauline probable stirling wigan ##bian paralympics panting surpassed ##rew luca barred pony famed ##sters cassandra waiter carolyn exported ##orted andres destructive deeds jonah castles vacancy suv ##glass 1788 orchard yep famine belarusian sprang ##forth skinny ##mis administrators rotterdam zambia zhao boiler discoveries ##ride ##physics lucius disappointing outreach spoon ##frame qualifications unanimously enjoys regency ##iidae stade realism veterinary rodgers dump alain chestnut castile censorship rumble gibbs ##itor communion reggae inactivated logs loads ##houses homosexual ##iano ale informs ##cas phrases plaster linebacker ambrose kaiser fascinated 850 limerick recruitment forge mastered ##nding leinster rooted threaten ##strom borneo ##hes suggestions scholarships propeller documentaries patronage coats constructing invest neurons comet entirety shouts identities annoying unchanged wary ##antly ##ogy neat oversight ##kos phillies replay constance ##kka incarnation humble skies minus ##acy smithsonian ##chel guerrilla jar cadets ##plate surplus audit ##aru cracking joanna louisa pacing ##lights intentionally ##iri diner nwa imprint australians tong unprecedented bunker naive specialists ark nichols railing leaked pedal ##uka shrub longing roofs v8 captains neural tuned ##ntal ##jet emission medina frantic codex definitive sid abolition intensified stocks enrique sustain genoa oxide ##written clues cha ##gers tributaries fragment venom ##rity ##ente ##sca muffled vain sire laos ##ingly ##hana hastily snapping surfaced sentiment motive ##oft contests approximate mesa luckily dinosaur exchanges propelled accord bourne relieve tow masks offended ##ues cynthia ##mmer rains bartender zinc reviewers lois ##sai legged arrogant rafe rosie comprise handicap blockade inlet lagoon copied drilling shelley petals ##inian mandarin obsolete ##inated onward arguably productivity cindy praising seldom busch discusses raleigh shortage ranged stanton encouragement firstly conceded overs temporal ##uke cbe ##bos woo certainty pumps ##pton stalked ##uli lizzie periodic thieves weaker ##night gases shoving chooses wc ##chemical prompting weights ##kill robust flanked sticky hu tuberculosis ##eb ##eal christchurch resembled wallet reese inappropriate pictured distract fixing fiddle giggled burger heirs hairy mechanic torque apache obsessed chiefly cheng logging ##tag extracted meaningful numb ##vsky gloucestershire reminding ##bay unite ##lit breeds diminished clown glove 1860s ##ن ##ug archibald focal freelance sliced depiction ##yk organism switches sights stray crawling ##ril lever leningrad interpretations loops anytime reel alicia delighted ##ech inhaled xiv suitcase bernie vega licenses northampton exclusion induction monasteries racecourse homosexuality ##right ##sfield ##rky dimitri michele alternatives ions commentators genuinely objected pork hospitality fencing stephan warships peripheral wit drunken wrinkled quentin spends departing chung numerical spokesperson ##zone johannesburg caliber killers ##udge assumes neatly demographic abigail bloc ##vel mounting ##lain bentley slightest xu recipients ##jk merlin ##writer seniors prisons blinking hindwings flickered kappa ##hel 80s strengthening appealing brewing gypsy mali lashes hulk unpleasant harassment bio treaties predict instrumentation pulp troupe boiling mantle ##ffe ins ##vn dividing handles verbs ##onal coconut senegal 340 thorough gum momentarily ##sto cocaine panicked destined ##turing teatro denying weary captained mans ##hawks ##code wakefield bollywood thankfully ##16 cyril ##wu amendments ##bahn consultation stud reflections kindness 1787 internally ##ovo tex mosaic distribute paddy seeming 143 ##hic piers ##15 ##mura ##verse popularly winger kang sentinel mccoy ##anza covenant ##bag verge fireworks suppress thrilled dominate ##jar swansea ##60 142 reconciliation ##ndi stiffened cue dorian ##uf damascus amor ida foremost ##aga porsche unseen dir ##had ##azi stony lexi melodies ##nko angular integer podcast ants inherent jaws justify persona ##olved josephine ##nr ##ressed customary flashes gala cyrus glaring backyard ariel physiology greenland html stir avon atletico finch methodology ked ##lent mas catholicism townsend branding quincy fits containers 1777 ashore aragon ##19 forearm poisoning ##sd adopting conquer grinding amnesty keller finances evaluate forged lankan instincts ##uto guam bosnian photographed workplace desirable protector ##dog allocation intently encourages willy ##sten bodyguard electro brighter ##ν bihar ##chev lasts opener amphibious sal verde arte ##cope captivity vocabulary yields ##tted agreeing desmond pioneered ##chus strap campaigned railroads ##ович emblem ##dre stormed 501 ##ulous marijuana northumberland ##gn ##nath bowen landmarks beaumont ##qua danube ##bler attorneys th ge flyers critique villains cass mutation acc ##0s colombo mckay motif sampling concluding syndicate ##rell neon stables ds warnings clint mourning wilkinson ##tated merrill leopard evenings exhaled emil sonia ezra discrete stove farrell fifteenth prescribed superhero ##rier worms helm wren ##duction ##hc expo ##rator hq unfamiliar antony prevents acceleration fiercely mari painfully calculations cheaper ign clifton irvine davenport mozambique ##np pierced ##evich wonders ##wig ##cate ##iling crusade ware ##uel enzymes reasonably mls ##coe mater ambition bunny eliot kernel ##fin asphalt headmaster torah aden lush pins waived ##care ##yas joao substrate enforce ##grad ##ules alvarez selections epidemic tempted ##bit bremen translates ensured waterfront 29th forrest manny malone kramer reigning cookies simpler absorption 205 engraved ##ffy evaluated 1778 haze 146 comforting crossover ##abe thorn ##rift ##imo ##pop suppression fatigue cutter ##tr 201 wurttemberg ##orf enforced hovering proprietary gb samurai syllable ascent lacey tick lars tractor merchandise rep bouncing defendants ##yre huntington ##ground ##oko standardized ##hor ##hima assassinated nu predecessors rainy liar assurance lyrical ##uga secondly flattened ios parameter undercover ##mity bordeaux punish ridges markers exodus inactive hesitate debbie nyc pledge savoy nagar offset organist ##tium hesse marin converting ##iver diagram propulsion pu validity reverted supportive ##dc ministries clans responds proclamation ##inae ##ø ##rea ein pleading patriot sf birch islanders strauss hates ##dh brandenburg concession rd ##ob 1900s killings textbook antiquity cinematography wharf embarrassing setup creed farmland inequality centred signatures fallon 370 ##ingham ##uts ceylon gazing directive laurie ##tern globally ##uated ##dent allah excavation threads ##cross 148 frantically icc utilize determines respiratory thoughtful receptions ##dicate merging chandra seine 147 builders builds diagnostic dev visibility goddamn analyses dhaka cho proves chancel concurrent curiously canadians pumped restoring 1850s turtles jaguar sinister spinal traction declan vows 1784 glowed capitalism swirling install universidad ##lder ##oat soloist ##genic ##oor coincidence beginnings nissan dip resorts caucasus combustion infectious ##eno pigeon serpent ##itating conclude masked salad jew ##gr surreal toni ##wc harmonica 151 ##gins ##etic ##coat fishermen intending bravery ##wave klaus titan wembley taiwanese ransom 40th incorrect hussein eyelids jp cooke dramas utilities ##etta ##print eisenhower principally granada lana ##rak openings concord ##bl bethany connie morality sega ##mons ##nard earnings ##kara ##cine wii communes ##rel coma composing softened severed grapes ##17 nguyen analyzed warlord hubbard heavenly behave slovenian ##hit ##ony hailed filmmakers trance caldwell skye unrest coward likelihood ##aging bern sci taliban honolulu propose ##wang 1700 browser imagining cobra contributes dukes instinctively conan violinist ##ores accessories gradual ##amp quotes sioux ##dating undertake intercepted sparkling compressed 139 fungus tombs haley imposing rests degradation lincolnshire retailers wetlands tulsa distributor dungeon nun greenhouse convey atlantis aft exits oman dresser lyons ##sti joking eddy judgement omitted digits ##cts ##game juniors ##rae cents stricken une ##ngo wizards weir breton nan technician fibers liking royalty ##cca 154 persia terribly magician ##rable ##unt vance cafeteria booker camille warmer ##static consume cavern gaps compass contemporaries foyer soothing graveyard maj plunged blush ##wear cascade demonstrates ordinance ##nov boyle ##lana rockefeller shaken banjo izzy ##ense breathless vines ##32 ##eman alterations chromosome dwellings feudal mole 153 catalonia relics tenant mandated ##fm fridge hats honesty patented raul heap cruisers accusing enlightenment infants wherein chatham contractors zen affinity hc osborne piston 156 traps maturity ##rana lagos ##zal peering ##nay attendant dealers protocols subset prospects biographical ##cre artery ##zers insignia nuns endured ##eration recommend schwartz serbs berger cromwell crossroads ##ctor enduring clasped grounded ##bine marseille twitched abel choke https catalyst moldova italians ##tist disastrous wee ##oured ##nti wwf nope ##piration ##asa expresses thumbs 167 ##nza coca 1781 cheating ##ption skipped sensory heidelberg spies satan dangers semifinal 202 bohemia whitish confusing shipbuilding relies surgeons landings ravi baku moor suffix alejandro ##yana litre upheld ##unk rajasthan ##rek coaster insists posture scenarios etienne favoured appoint transgender elephants poked greenwood defences fulfilled militant somali 1758 chalk potent ##ucci migrants wink assistants nos restriction activism niger ##ario colon shaun ##sat daphne ##erated swam congregations reprise considerations magnet playable xvi ##р overthrow tobias knob chavez coding ##mers propped katrina orient newcomer ##suke temperate ##pool farmhouse interrogation ##vd committing ##vert forthcoming strawberry joaquin macau ponds shocking siberia ##cellular chant contributors ##nant ##ologists sped absorb hail 1782 spared ##hore barbados karate opus originates saul ##xie evergreen leaped ##rock correlation exaggerated weekday unification bump tracing brig afb pathways utilizing ##ners mod mb disturbance kneeling ##stad ##guchi 100th pune ##thy decreasing 168 manipulation miriam academia ecosystem occupational rbi ##lem rift ##14 rotary stacked incorporation awakening generators guerrero racist ##omy cyber derivatives culminated allie annals panzer sainte wikipedia pops zu austro ##vate algerian politely nicholson mornings educate tastes thrill dartmouth ##gating db ##jee regan differing concentrating choreography divinity ##media pledged alexandre routing gregor madeline ##idal apocalypse ##hora gunfire culminating elves fined liang lam programmed tar guessing transparency gabrielle ##gna cancellation flexibility ##lining accession shea stronghold nets specializes ##rgan abused hasan sgt ling exceeding ##₄ admiration supermarket ##ark photographers specialised tilt resonance hmm perfume 380 sami threatens garland botany guarding boiled greet puppy russo supplier wilmington vibrant vijay ##bius paralympic grumbled paige faa licking margins hurricanes ##gong fest grenade ripping ##uz counseling weigh ##sian needles wiltshire edison costly ##not fulton tramway redesigned staffordshire cache gasping watkins sleepy candidacy ##group monkeys timeline throbbing ##bid ##sos berth uzbekistan vanderbilt bothering overturned ballots gem ##iger sunglasses subscribers hooker compelling ang exceptionally saloon stab ##rdi carla terrifying rom ##vision coil ##oids satisfying vendors 31st mackay deities overlooked ambient bahamas felipe olympia whirled botanist advertised tugging ##dden disciples morales unionist rites foley morse motives creepy ##₀ soo ##sz bargain highness frightening turnpike tory reorganization ##cer depict biographer ##walk unopposed manifesto ##gles institut emile accidental kapoor ##dam kilkenny cortex lively ##13 romanesque jain shan cannons ##ood ##ske petrol echoing amalgamated disappears cautious proposes sanctions trenton ##ر flotilla aus contempt tor canary cote theirs ##hun conceptual deleted fascinating paso blazing elf honourable hutchinson ##eiro ##outh ##zin surveyor tee amidst wooded reissue intro ##ono cobb shelters newsletter hanson brace encoding confiscated dem caravan marino scroll melodic cows imam ##adi ##aneous northward searches biodiversity cora 310 roaring ##bers connell theologian halo compose pathetic unmarried dynamo ##oot az calculation toulouse deserves humour nr forgiveness tam undergone martyr pamela myths whore counselor hicks 290 heavens battleship electromagnetic ##bbs stellar establishments presley hopped ##chin temptation 90s wills nas ##yuan nhs ##nya seminars ##yev adaptations gong asher lex indicator sikh tobago cites goin ##yte satirical ##gies characterised correspond bubbles lure participates ##vid eruption skate therapeutic 1785 canals wholesale defaulted sac 460 petit ##zzled virgil leak ravens 256 portraying ##yx ghetto creators dams portray vicente ##rington fae namesake bounty ##arium joachim ##ota ##iser aforementioned axle snout depended dismantled reuben 480 ##ibly gallagher ##lau ##pd earnest ##ieu ##iary inflicted objections ##llar asa gritted ##athy jericho ##sea ##was flick underside ceramics undead substituted 195 eastward undoubtedly wheeled chimney ##iche guinness cb ##ager siding ##bell traitor baptiste disguised inauguration 149 tipperary choreographer perched warmed stationary eco ##ike ##ntes bacterial ##aurus flores phosphate ##core attacker invaders alvin intersects a1 indirectly immigrated businessmen cornelius valves narrated pill sober ul nationale monastic applicants scenery ##jack 161 motifs constitutes cpu ##osh jurisdictions sd tuning irritation woven ##uddin fertility gao ##erie antagonist impatient glacial hides boarded denominations interception ##jas cookie nicola ##tee algebraic marquess bahn parole buyers bait turbines paperwork bestowed natasha renee oceans purchases 157 vaccine 215 ##tock fixtures playhouse integrate jai oswald intellectuals ##cky booked nests mortimer ##isi obsession sept ##gler ##sum 440 scrutiny simultaneous squinted ##shin collects oven shankar penned remarkably ##я slips luggage spectral 1786 collaborations louie consolidation ##ailed ##ivating 420 hoover blackpool harness ignition vest tails belmont mongol skinner ##nae visually mage derry ##tism ##unce stevie transitional ##rdy redskins drying prep prospective ##21 annoyance oversee ##loaded fills ##books ##iki announces fda scowled respects prasad mystic tucson ##vale revue springer bankrupt 1772 aristotle salvatore habsburg ##geny dal natal nut pod chewing darts moroccan walkover rosario lenin punjabi ##ße grossed scattering wired invasive hui polynomial corridors wakes gina portrays ##cratic arid retreating erich irwin sniper ##dha linen lindsey maneuver butch shutting socio bounce commemorative postseason jeremiah pines 275 mystical beads bp abbas furnace bidding consulted assaulted empirical rubble enclosure sob weakly cancel polly yielded ##emann curly prediction battered 70s vhs jacqueline render sails barked detailing grayson riga sloane raging ##yah herbs bravo ##athlon alloy giggle imminent suffers assumptions waltz ##itate accomplishments ##ited bathing remixed deception prefix ##emia deepest ##tier ##eis balkan frogs ##rong slab ##pate philosophers peterborough grains imports dickinson rwanda ##atics 1774 dirk lan tablets ##rove clone ##rice caretaker hostilities mclean ##gre regimental treasures norms impose tsar tango diplomacy variously complain 192 recognise arrests 1779 celestial pulitzer ##dus bing libretto ##moor adele splash ##rite expectation lds confronts ##izer spontaneous harmful wedge entrepreneurs buyer ##ope bilingual translate rugged conner circulated uae eaton ##gra ##zzle lingered lockheed vishnu reelection alonso ##oom joints yankee headline cooperate heinz laureate invading ##sford echoes scandinavian ##dham hugging vitamin salute micah hind trader ##sper radioactive ##ndra militants poisoned ratified remark campeonato deprived wander prop ##dong outlook ##tani ##rix ##eye chiang darcy ##oping mandolin spice statesman babylon 182 walled forgetting afro ##cap 158 giorgio buffer ##polis planetary ##gis overlap terminals kinda centenary ##bir arising manipulate elm ke 1770 ak ##tad chrysler mapped moose pomeranian quad macarthur assemblies shoreline recalls stratford ##rted noticeable ##evic imp ##rita ##sque accustomed supplying tents disgusted vogue sipped filters khz reno selecting luftwaffe mcmahon tyne masterpiece carriages collided dunes exercised flare remembers muzzle ##mobile heck ##rson burgess lunged middleton boycott bilateral ##sity hazardous lumpur multiplayer spotlight jackets goldman liege porcelain rag waterford benz attracts hopeful battling ottomans kensington baked hymns cheyenne lattice levine borrow polymer clashes michaels monitored commitments denounced ##25 ##von cavity ##oney hobby akin ##holders futures intricate cornish patty ##oned illegally dolphin ##lag barlow yellowish maddie apologized luton plagued ##puram nana ##rds sway fanny łodz ##rino psi suspicions hanged ##eding initiate charlton ##por nak competent 235 analytical annex wardrobe reservations ##rma sect 162 fairfax hedge piled buckingham uneven bauer simplicity snyder interpret accountability donors moderately byrd continents ##cite ##max disciple hr jamaican ping nominees ##uss mongolian diver attackers eagerly ideological pillows miracles apartheid revolver sulfur clinics moran 163 ##enko ile katy rhetoric ##icated chronology recycling ##hrer elongated mughal pascal profiles vibration databases domination ##fare ##rant matthias digest rehearsal polling weiss initiation reeves clinging flourished impress ngo ##hoff ##ume buckley symposium rhythms weed emphasize transforming ##taking ##gence ##yman accountant analyze flicker foil priesthood voluntarily decreases ##80 ##hya slater sv charting mcgill ##lde moreno ##iu besieged zur robes ##phic admitting api deported turmoil peyton earthquakes ##ares nationalists beau clair brethren interrupt welch curated galerie requesting 164 ##ested impending steward viper ##vina complaining beautifully brandy foam nl 1660 ##cake alessandro punches laced explanations ##lim attribute clit reggie discomfort ##cards smoothed whales ##cene adler countered duffy disciplinary widening recipe reliance conducts goats gradient preaching ##shaw matilda quasi striped meridian cannabis cordoba certificates ##agh ##tering graffiti hangs pilgrims repeats ##ych revive urine etat ##hawk fueled belts fuzzy susceptible ##hang mauritius salle sincere beers hooks ##cki arbitration entrusted advise sniffed seminar junk donnell processors principality strapped celia mendoza everton fortunes prejudice starving reassigned steamer ##lund tuck evenly foreman ##ffen dans 375 envisioned slit ##xy baseman liberia rosemary ##weed electrified periodically potassium stride contexts sperm slade mariners influx bianca subcommittee ##rane spilling icao estuary ##nock delivers iphone ##ulata isa mira bohemian dessert ##sbury welcoming proudly slowing ##chs musee ascension russ ##vian waits ##psy africans exploit ##morphic gov eccentric crab peck ##ull entrances formidable marketplace groom bolted metabolism patton robbins courier payload endure ##ifier andes refrigerator ##pr ornate ##uca ruthless illegitimate masonry strasbourg bikes adobe ##³ apples quintet willingly niche bakery corpses energetic ##cliffe ##sser ##ards 177 centimeters centro fuscous cretaceous rancho ##yde andrei telecom tottenham oasis ordination vulnerability presiding corey cp penguins sims ##pis malawi piss ##48 correction ##cked ##ffle ##ryn countdown detectives psychiatrist psychedelic dinosaurs blouse ##get choi vowed ##oz randomly ##pol 49ers scrub blanche bruins dusseldorf ##using unwanted ##ums 212 dominique elevations headlights om laguna ##oga 1750 famously ignorance shrewsbury ##aine ajax breuning che confederacy greco overhaul ##screen paz skirts disagreement cruelty jagged phoebe shifter hovered viruses ##wes mandy ##lined ##gc landlord squirrel dashed ##ι ornamental gag wally grange literal spurs undisclosed proceeding yin ##text billie orphan spanned humidity indy weighted presentations explosions lucian ##tary vaughn hindus ##anga ##hell psycho 171 daytona protects efficiently rematch sly tandem ##oya rebranded impaired hee metropolis peach godfrey diaspora ethnicity prosperous gleaming dar grossing playback ##rden stripe pistols ##tain births labelled ##cating 172 rudy alba ##onne aquarium hostility ##gb ##tase shudder sumatra hardest lakers consonant creeping demos homicide capsule zeke liberties expulsion pueblo ##comb trait transporting ##ddin ##neck ##yna depart gregg mold ledge hangar oldham playboy termination analysts gmbh romero ##itic insist cradle filthy brightness slash shootout deposed bordering ##truct isis microwave tumbled sheltered cathy werewolves messy andersen convex clapped clinched satire wasting edo vc rufus ##jak mont ##etti poznan ##keeping restructuring transverse ##rland azerbaijani slovene gestures roommate choking shear ##quist vanguard oblivious ##hiro disagreed baptism ##lich coliseum ##aceae salvage societe cory locke relocation relying versailles ahl swelling ##elo cheerful ##word ##edes gin sarajevo obstacle diverted ##nac messed thoroughbred fluttered utrecht chewed acquaintance assassins dispatch mirza ##wart nike salzburg swell yen ##gee idle ligue samson ##nds ##igh playful spawned ##cise tease ##case burgundy ##bot stirring skeptical interceptions marathi ##dies bedrooms aroused pinch ##lik preferences tattoos buster digitally projecting rust ##ital kitten priorities addison pseudo ##guard dusk icons sermon ##psis ##iba bt ##lift ##xt ju truce rink ##dah ##wy defects psychiatry offences calculate glucose ##iful ##rized ##unda francaise ##hari richest warwickshire carly 1763 purity redemption lending ##cious muse bruises cerebral aero carving ##name preface terminology invade monty ##int anarchist blurred ##iled rossi treats guts shu foothills ballads undertaking premise cecilia affiliates blasted conditional wilder minors drone rudolph buffy swallowing horton attested ##hop rutherford howell primetime livery penal ##bis minimize hydro wrecked wrought palazzo ##gling cans vernacular friedman nobleman shale walnut danielle ##ection ##tley sears ##kumar chords lend flipping streamed por dracula gallons sacrifices gamble orphanage ##iman mckenzie ##gible boxers daly ##balls ##ان 208 ##ific ##rative ##iq exploited slated ##uity circling hillary pinched goldberg provost campaigning lim piles ironically jong mohan successors usaf ##tem ##ught autobiographical haute preserves ##ending acquitted comparisons 203 hydroelectric gangs cypriot torpedoes rushes chrome derive bumps instability fiat pets ##mbe silas dye reckless settler ##itation info heats ##writing 176 canonical maltese fins mushroom stacy aspen avid ##kur ##loading vickers gaston hillside statutes wilde gail kung sabine comfortably motorcycles ##rgo 169 pneumonia fetch ##sonic axel faintly parallels ##oop mclaren spouse compton interdisciplinary miner ##eni 181 clamped ##chal ##llah separates versa ##mler scarborough labrador ##lity ##osing rutgers hurdles como 166 burt divers ##100 wichita cade coincided ##erson bruised mla ##pper vineyard ##ili ##brush notch mentioning jase hearted kits doe ##acle pomerania ##ady ronan seizure pavel problematic ##zaki domenico ##ulin catering penelope dependence parental emilio ministerial atkinson ##bolic clarkson chargers colby grill peeked arises summon ##aged fools ##grapher faculties qaeda ##vial garner refurbished ##hwa geelong disasters nudged bs shareholder lori algae reinstated rot ##ades ##nous invites stainless 183 inclusive ##itude diocesan til ##icz denomination ##xa benton floral registers ##ider ##erman ##kell absurd brunei guangzhou hitter retaliation ##uled ##eve blanc nh consistency contamination ##eres ##rner dire palermo broadcasters diaries inspire vols brewer tightening ky mixtape hormone ##tok stokes ##color ##dly ##ssi pg ##ometer ##lington sanitation ##tility intercontinental apps ##adt ¹⁄₂ cylinders economies favourable unison croix gertrude odyssey vanity dangling ##logists upgrades dice middleweight practitioner ##ight 206 henrik parlor orion angered lac python blurted ##rri sensual intends swings angled ##phs husky attain peerage precinct textiles cheltenham shuffled dai confess tasting bhutan ##riation tyrone segregation abrupt ruiz ##rish smirked blackwell confidential browning amounted ##put vase scarce fabulous raided staple guyana unemployed glider shay ##tow carmine troll intervene squash superstar ##uce cylindrical len roadway researched handy ##rium ##jana meta lao declares ##rring ##tadt ##elin ##kova willem shrubs napoleonic realms skater qi volkswagen ##ł tad hara archaeologist awkwardly eerie ##kind wiley ##heimer ##24 titus organizers cfl crusaders lama usb vent enraged thankful occupants maximilian ##gaard possessing textbooks ##oran collaborator quaker ##ulo avalanche mono silky straits isaiah mustang surged resolutions potomac descend cl kilograms plato strains saturdays ##olin bernstein ##ype holstein ponytail ##watch belize conversely heroine perpetual ##ylus charcoal piedmont glee negotiating backdrop prologue ##jah ##mmy pasadena climbs ramos sunni ##holm ##tner ##tri anand deficiency hertfordshire stout ##avi aperture orioles ##irs doncaster intrigued bombed coating otis ##mat cocktail ##jit ##eto amir arousal sar ##proof ##act ##ories dixie pots ##bow whereabouts 159 ##fted drains bullying cottages scripture coherent fore poe appetite ##uration sampled ##ators ##dp derrick rotor jays peacock installment ##rro advisors ##coming rodeo scotch ##mot ##db ##fen ##vant ensued rodrigo dictatorship martyrs twenties ##н towed incidence marta rainforest sai scaled ##cles oceanic qualifiers symphonic mcbride dislike generalized aubrey colonization ##iation ##lion ##ssing disliked lublin salesman ##ulates spherical whatsoever sweating avalon contention punt severity alderman atari ##dina ##grant ##rop scarf seville vertices annexation fairfield fascination inspiring launches palatinate regretted ##rca feral ##iom elk nap olsen reddy yong ##leader ##iae garment transports feng gracie outrage viceroy insides ##esis breakup grady organizer softer grimaced 222 murals galicia arranging vectors ##rsten bas ##sb ##cens sloan ##eka bitten ara fender nausea bumped kris banquet comrades detector persisted ##llan adjustment endowed cinemas ##shot sellers ##uman peek epa kindly neglect simpsons talon mausoleum runaway hangul lookout ##cic rewards coughed acquainted chloride ##ald quicker accordion neolithic ##qa artemis coefficient lenny pandora tx ##xed ecstasy litter segunda chairperson gemma hiss rumor vow nasal antioch compensate patiently transformers ##eded judo morrow penis posthumous philips bandits husbands denote flaming ##any ##phones langley yorker 1760 walters ##uo ##kle gubernatorial fatty samsung leroy outlaw ##nine unpublished poole jakob ##ᵢ ##ₙ crete distorted superiority ##dhi intercept crust mig claus crashes positioning 188 stallion 301 frontal armistice ##estinal elton aj encompassing camel commemorated malaria woodward calf cigar penetrate ##oso willard ##rno ##uche illustrate amusing convergence noteworthy ##lma ##rva journeys realise manfred ##sable 410 ##vocation hearings fiance ##posed educators provoked adjusting ##cturing modular stockton paterson vlad rejects electors selena maureen ##tres uber ##rce swirled ##num proportions nanny pawn naturalist parma apostles awoke ethel wen ##bey monsoon overview ##inating mccain rendition risky adorned ##ih equestrian germain nj conspicuous confirming ##yoshi shivering ##imeter milestone rumours flinched bounds smacked token ##bei lectured automobiles ##shore impacted ##iable nouns nero ##leaf ismail prostitute trams ##lace bridget sud stimulus impressions reins revolves ##oud ##gned giro honeymoon ##swell criterion ##sms ##uil libyan prefers ##osition 211 preview sucks accusation bursts metaphor diffusion tolerate faye betting cinematographer liturgical specials bitterly humboldt ##ckle flux rattled ##itzer archaeologists odor authorised marshes discretion ##ов alarmed archaic inverse ##leton explorers ##pine drummond tsunami woodlands ##minate ##tland booklet insanity owning insert crafted calculus ##tore receivers ##bt stung ##eca ##nched prevailing travellers eyeing lila graphs ##borne 178 julien ##won morale adaptive therapist erica cw libertarian bowman pitches vita ##ional crook ##ads ##entation caledonia mutiny ##sible 1840s automation ##ß flock ##pia ironic pathology ##imus remarried ##22 joker withstand energies ##att shropshire hostages madeleine tentatively conflicting mateo recipes euros ol mercenaries nico ##ndon albuquerque augmented mythical bel freud ##child cough ##lica 365 freddy lillian genetically nuremberg calder 209 bonn outdoors paste suns urgency vin restraint tyson ##cera ##selle barrage bethlehem kahn ##par mounts nippon barony happier ryu makeshift sheldon blushed castillo barking listener taped bethel fluent headlines pornography rum disclosure sighing mace doubling gunther manly ##plex rt interventions physiological forwards emerges ##tooth ##gny compliment rib recession visibly barge faults connector exquisite prefect ##rlin patio ##cured elevators brandt italics pena 173 wasp satin ea botswana graceful respectable ##jima ##rter ##oic franciscan generates ##dl alfredo disgusting ##olate ##iously sherwood warns cod promo cheryl sino ##ة ##escu twitch ##zhi brownish thom ortiz ##dron densely ##beat carmel reinforce ##bana 187 anastasia downhill vertex contaminated remembrance harmonic homework ##sol fiancee gears olds angelica loft ramsay quiz colliery sevens ##cape autism ##hil walkway ##boats ruben abnormal ounce khmer ##bbe zachary bedside morphology punching ##olar sparrow convinces ##35 hewitt queer remastered rods mabel solemn notified lyricist symmetric ##xide 174 encore passports wildcats ##uni baja ##pac mildly ##ease bleed commodity mounds glossy orchestras ##omo damian prelude ambitions ##vet awhile remotely ##aud asserts imply ##iques distinctly modelling remedy ##dded windshield dani xiao ##endra audible powerplant 1300 invalid elemental acquisitions ##hala immaculate libby plata smuggling ventilation denoted minh ##morphism 430 differed dion kelley lore mocking sabbath spikes hygiene drown runoff stylized tally liberated aux interpreter righteous aba siren reaper pearce millie ##cier ##yra gaius ##iso captures ##ttering dorm claudio ##sic benches knighted blackness ##ored discount fumble oxidation routed ##ς novak perpendicular spoiled fracture splits ##urt pads topology ##cats axes fortunate offenders protestants esteem 221 broadband convened frankly hound prototypes isil facilitated keel ##sher sahara awaited bubba orb prosecutors 186 hem 520 ##xing relaxing remnant romney sorted slalom stefano ulrich ##active exemption folder pauses foliage hitchcock epithet 204 criticisms ##aca ballistic brody hinduism chaotic youths equals ##pala pts thicker analogous capitalist improvised overseeing sinatra ascended beverage ##tl straightforward ##kon curran ##west bois 325 induce surveying emperors sax unpopular ##kk cartoonist fused ##mble unto ##yuki localities ##cko ##ln darlington slain academie lobbying sediment puzzles ##grass defiance dickens manifest tongues alumnus arbor coincide 184 appalachian mustafa examiner cabaret traumatic yves bracelet draining heroin magnum baths odessa consonants mitsubishi ##gua kellan vaudeville ##fr joked null straps probation ##ław ceded interfaces ##pas ##zawa blinding viet 224 rothschild museo 640 huddersfield ##vr tactic ##storm brackets dazed incorrectly ##vu reg glazed fearful manifold benefited irony ##sun stumbling ##rte willingness balkans mei wraps ##aba injected ##lea gu syed harmless ##hammer bray takeoff poppy timor cardboard astronaut purdue weeping southbound cursing stalls diagonal ##neer lamar bryce comte weekdays harrington ##uba negatively ##see lays grouping ##cken ##henko affirmed halle modernist ##lai hodges smelling aristocratic baptized dismiss justification oilers ##now coupling qin snack healer ##qing gardener layla battled formulated stephenson gravitational ##gill ##jun 1768 granny coordinating suites ##cd ##ioned monarchs ##cote ##hips sep blended apr barrister deposition fia mina policemen paranoid ##pressed churchyard covert crumpled creep abandoning tr transmit conceal barr understands readiness spire ##cology ##enia ##erry 610 startling unlock vida bowled slots ##nat ##islav spaced trusting admire rig ##ink slack ##70 mv 207 casualty ##wei classmates ##odes ##rar ##rked amherst furnished evolve foundry menace mead ##lein flu wesleyan ##kled monterey webber ##vos wil ##mith ##на bartholomew justices restrained ##cke amenities 191 mediated sewage trenches ml mainz ##thus 1800s ##cula ##inski caine bonding 213 converts spheres superseded marianne crypt sweaty ensign historia ##br spruce ##post ##ask forks thoughtfully yukon pamphlet ames ##uter karma ##yya bryn negotiation sighs incapable ##mbre ##ntial actresses taft ##mill luce prevailed ##amine 1773 motionless envoy testify investing sculpted instructors provence kali cullen horseback ##while goodwin ##jos gaa norte ##ldon modify wavelength abd 214 skinned sprinter forecast scheduling marries squared tentative ##chman boer ##isch bolts swap fisherman assyrian impatiently guthrie martins murdoch 194 tanya nicely dolly lacy med ##45 syn decks fashionable millionaire ##ust surfing ##ml ##ision heaved tammy consulate attendees routinely 197 fuse saxophonist backseat malaya ##lord scowl tau ##ishly 193 sighted steaming ##rks 303 911 ##holes ##hong ching ##wife bless conserved jurassic stacey unix zion chunk rigorous blaine 198 peabody slayer dismay brewers nz ##jer det ##glia glover postwar int penetration sylvester imitation vertically airlift heiress knoxville viva ##uin 390 macon ##rim ##fighter ##gonal janice ##orescence ##wari marius belongings leicestershire 196 blanco inverted preseason sanity sobbing ##due ##elt ##dled collingwood regeneration flickering shortest ##mount ##osi feminism ##lat sherlock cabinets fumbled northbound precedent snaps ##mme researching ##akes guillaume insights manipulated vapor neighbour sap gangster frey f1 stalking scarcely callie barnett tendencies audi doomed assessing slung panchayat ambiguous bartlett ##etto distributing violating wolverhampton ##hetic swami histoire ##urus liable pounder groin hussain larsen popping surprises ##atter vie curt ##station mute relocate musicals authorization richter ##sef immortality tna bombings ##press deteriorated yiddish ##acious robbed colchester cs pmid ao verified balancing apostle swayed recognizable oxfordshire retention nottinghamshire contender judd invitational shrimp uhf ##icient cleaner longitudinal tanker ##mur acronym broker koppen sundance suppliers ##gil 4000 clipped fuels petite ##anne landslide helene diversion populous landowners auspices melville quantitative ##xes ferries nicky ##llus doo haunting roche carver downed unavailable ##pathy approximation hiroshima ##hue garfield valle comparatively keyboardist traveler ##eit congestion calculating subsidiaries ##bate serb modernization fairies deepened ville averages ##lore inflammatory tonga ##itch co₂ squads ##hea gigantic serum enjoyment retailer verona 35th cis ##phobic magna technicians ##vati arithmetic ##sport levin ##dation amtrak chow sienna ##eyer backstage entrepreneurship ##otic learnt tao ##udy worcestershire formulation baggage hesitant bali sabotage ##kari barren enhancing murmur pl freshly putnam syntax aces medicines resentment bandwidth ##sier grins chili guido ##sei framing implying gareth lissa genevieve pertaining admissions geo thorpe proliferation sato bela analyzing parting ##gor awakened ##isman huddled secrecy ##kling hush gentry 540 dungeons ##ego coasts ##utz sacrificed ##chule landowner mutually prevalence programmer adolescent disrupted seaside gee trusts vamp georgie ##nesian ##iol schedules sindh ##market etched hm sparse bey beaux scratching gliding unidentified 216 collaborating gems jesuits oro accumulation shaping mbe anal ##xin 231 enthusiasts newscast ##egan janata dewey parkinson 179 ankara biennial towering dd inconsistent 950 ##chet thriving terminate cabins furiously eats advocating donkey marley muster phyllis leiden ##user grassland glittering iucn loneliness 217 memorandum armenians ##ddle popularized rhodesia 60s lame ##illon sans bikini header orbits ##xx ##finger ##ulator sharif spines biotechnology strolled naughty yates ##wire fremantle milo ##mour abducted removes ##atin humming wonderland ##chrome ##ester hume pivotal ##rates armand grams believers elector rte apron bis scraped ##yria endorsement initials ##llation eps dotted hints buzzing emigration nearer ##tom indicators ##ulu coarse neutron protectorate ##uze directional exploits pains loire 1830s proponents guggenheim rabbits ritchie 305 hectare inputs hutton ##raz verify ##ako boilers longitude ##lev skeletal yer emilia citrus compromised ##gau pokemon prescription paragraph eduard cadillac attire categorized kenyan weddings charley ##bourg entertain monmouth ##lles nutrients davey mesh incentive practised ecosystems kemp subdued overheard ##rya bodily maxim ##nius apprenticeship ursula ##fight lodged rug silesian unconstitutional patel inspected coyote unbeaten ##hak 34th disruption convict parcel ##cl ##nham collier implicated mallory ##iac ##lab susannah winkler ##rber shia phelps sediments graphical robotic ##sner adulthood mart smoked ##isto kathryn clarified ##aran divides convictions oppression pausing burying ##mt federico mathias eileen ##tana kite hunched ##acies 189 ##atz disadvantage liza kinetic greedy paradox yokohama dowager trunks ventured ##gement gupta vilnius olaf ##thest crimean hopper ##ej progressively arturo mouthed arrondissement ##fusion rubin simulcast oceania ##orum ##stra ##rred busiest intensely navigator cary ##vine ##hini ##bies fife rowe rowland posing insurgents shafts lawsuits activate conor inward culturally garlic 265 ##eering eclectic ##hui ##kee ##nl furrowed vargas meteorological rendezvous ##aus culinary commencement ##dition quota ##notes mommy salaries overlapping mule ##iology ##mology sums wentworth ##isk ##zione mainline subgroup ##illy hack plaintiff verdi bulb differentiation engagements multinational supplemented bertrand caller regis ##naire ##sler ##arts ##imated blossom propagation kilometer viaduct vineyards ##uate beckett optimization golfer songwriters seminal semitic thud volatile evolving ridley ##wley trivial distributions scandinavia jiang ##ject wrestled insistence ##dio emphasizes napkin ##ods adjunct rhyme ##ricted ##eti hopeless surrounds tremble 32nd smoky ##ntly oils medicinal padded steer wilkes 219 255 concessions hue uniquely blinded landon yahoo ##lane hendrix commemorating dex specify chicks ##ggio intercity 1400 morley ##torm highlighting ##oting pang oblique stalled ##liner flirting newborn 1769 bishopric shaved 232 currie ##ush dharma spartan ##ooped favorites smug novella sirens abusive creations espana ##lage paradigm semiconductor sheen ##rdo ##yen ##zak nrl renew ##pose ##tur adjutant marches norma ##enity ineffective weimar grunt ##gat lordship plotting expenditure infringement lbs refrain av mimi mistakenly postmaster 1771 ##bara ras motorsports tito 199 subjective ##zza bully stew ##kaya prescott 1a ##raphic ##zam bids styling paranormal reeve sneaking exploding katz akbar migrant syllables indefinitely ##ogical destroys replaces applause ##phine pest ##fide 218 articulated bertie ##thing ##cars ##ptic courtroom crowley aesthetics cummings tehsil hormones titanic dangerously ##ibe stadion jaenelle auguste ciudad ##chu mysore partisans ##sio lucan philipp ##aly debating henley interiors ##rano ##tious homecoming beyonce usher henrietta prepares weeds ##oman ely plucked ##pire ##dable luxurious ##aq artifact password pasture juno maddy minsk ##dder ##ologies ##rone assessments martian royalist 1765 examines ##mani ##rge nino 223 parry scooped relativity ##eli ##uting ##cao congregational noisy traverse ##agawa strikeouts nickelodeon obituary transylvania binds depictions polk trolley ##yed ##lard breeders ##under dryly hokkaido 1762 strengths stacks bonaparte connectivity neared prostitutes stamped anaheim gutierrez sinai ##zzling bram fresno madhya ##86 proton ##lena ##llum ##phon reelected wanda ##anus ##lb ample distinguishing ##yler grasping sermons tomato bland stimulation avenues ##eux spreads scarlett fern pentagon assert baird chesapeake ir calmed distortion fatalities ##olis correctional pricing ##astic ##gina prom dammit ying collaborate ##chia welterweight 33rd pointer substitution bonded umpire communicating multitude paddle ##obe federally intimacy ##insky betray ssr ##lett ##lean ##lves ##therapy airbus ##tery functioned ud bearer biomedical netflix ##hire ##nca condom brink ik ##nical macy ##bet flap gma experimented jelly lavender ##icles ##ulia munro ##mian ##tial rye ##rle 60th gigs hottest rotated predictions fuji bu ##erence ##omi barangay ##fulness ##sas clocks ##rwood ##liness cereal roe wight decker uttered babu onion xml forcibly ##df petra sarcasm hartley peeled storytelling ##42 ##xley ##ysis ##ffa fibre kiel auditor fig harald greenville ##berries geographically nell quartz ##athic cemeteries ##lr crossings nah holloway reptiles chun sichuan snowy 660 corrections ##ivo zheng ambassadors blacksmith fielded fluids hardcover turnover medications melvin academies ##erton ro roach absorbing spaniards colton ##founded outsider espionage kelsey 245 edible ##ulf dora establishes ##sham ##tries contracting ##tania cinematic costello nesting ##uron connolly duff ##nology mma ##mata fergus sexes gi optics spectator woodstock banning ##hee ##fle differentiate outfielder refinery 226 312 gerhard horde lair drastically ##udi landfall ##cheng motorsport odi ##achi predominant quay skins ##ental edna harshly complementary murdering ##aves wreckage ##90 ono outstretched lennox munitions galen reconcile 470 scalp bicycles gillespie questionable rosenberg guillermo hostel jarvis kabul volvo opium yd ##twined abuses decca outpost ##cino sensible neutrality ##64 ponce anchorage atkins turrets inadvertently disagree libre vodka reassuring weighs ##yal glide jumper ceilings repertory outs stain ##bial envy ##ucible smashing heightened policing hyun mixes lai prima ##ples celeste ##bina lucrative intervened kc manually ##rned stature staffed bun bastards nairobi priced ##auer thatcher ##kia tripped comune ##ogan ##pled brasil incentives emanuel hereford musica ##kim benedictine biennale ##lani eureka gardiner rb knocks sha ##ael ##elled ##onate efficacy ventura masonic sanford maize leverage ##feit capacities santana ##aur novelty vanilla ##cter ##tour benin ##oir ##rain neptune drafting tallinn ##cable humiliation ##boarding schleswig fabian bernardo liturgy spectacle sweeney pont routledge ##tment cosmos ut hilt sleek universally ##eville ##gawa typed ##dry favors allegheny glaciers ##rly recalling aziz ##log parasite requiem auf ##berto ##llin illumination ##breaker ##issa festivities bows govern vibe vp 333 sprawled larson pilgrim bwf leaping ##rts ##ssel alexei greyhound hoarse ##dler ##oration seneca ##cule gaping ##ulously ##pura cinnamon ##gens ##rricular craven fantasies houghton engined reigned dictator supervising ##oris bogota commentaries unnatural fingernails spirituality tighten ##tm canadiens protesting intentional cheers sparta ##ytic ##iere ##zine widen belgarath controllers dodd iaaf navarre ##ication defect squire steiner whisky ##mins 560 inevitably tome ##gold chew ##uid ##lid elastic ##aby streaked alliances jailed regal ##ined ##phy czechoslovak narration absently ##uld bluegrass guangdong quran criticizing hose hari ##liest ##owa skier streaks deploy ##lom raft bose dialed huff ##eira haifa simplest bursting endings ib sultanate ##titled franks whitman ensures sven ##ggs collaborators forster organising ui banished napier injustice teller layered thump ##otti roc battleships evidenced fugitive sadie robotics ##roud equatorial geologist ##iza yielding ##bron ##sr internationale mecca ##diment sbs skyline toad uploaded reflective undrafted lal leafs bayern ##dai lakshmi shortlisted ##stick ##wicz camouflage donate af christi lau ##acio disclosed nemesis 1761 assemble straining northamptonshire tal ##asi bernardino premature heidi 42nd coefficients galactic reproduce buzzed sensations zionist monsieur myrtle ##eme archery strangled musically viewpoint antiquities bei trailers seahawks cured pee preferring tasmanian lange sul ##mail ##working colder overland lucivar massey gatherings haitian ##smith disapproval flaws ##cco ##enbach 1766 npr ##icular boroughs creole forums techno 1755 dent abdominal streetcar ##eson ##stream procurement gemini predictable ##tya acheron christoph feeder fronts vendor bernhard jammu tumors slang ##uber goaltender twists curving manson vuelta mer peanut confessions pouch unpredictable allowance theodor vascular ##factory bala authenticity metabolic coughing nanjing ##cea pembroke ##bard splendid 36th ff hourly ##ahu elmer handel ##ivate awarding thrusting dl experimentation ##hesion ##46 caressed entertained steak ##rangle biologist orphans baroness oyster stepfather ##dridge mirage reefs speeding ##31 barons 1764 227 inhabit preached repealed ##tral honoring boogie captives administer johanna ##imate gel suspiciously 1767 sobs ##dington backbone hayward garry ##folding ##nesia maxi ##oof ##ppe ellison galileo ##stand crimea frenzy amour bumper matrices natalia baking garth palestinians ##grove smack conveyed ensembles gardening ##manship ##rup ##stituting 1640 harvesting topography jing shifters dormitory ##carriage ##lston ist skulls ##stadt dolores jewellery sarawak ##wai ##zier fences christy confinement tumbling credibility fir stench ##bria ##plication ##nged ##sam virtues ##belt marjorie pba ##eem ##made celebrates schooner agitated barley fulfilling anthropologist ##pro restrict novi regulating ##nent padres ##rani ##hesive loyola tabitha milky olson proprietor crambidae guarantees intercollegiate ljubljana hilda ##sko ignorant hooded ##lts sardinia ##lidae ##vation frontman privileged witchcraft ##gp jammed laude poking ##than bracket amazement yunnan ##erus maharaja linnaeus 264 commissioning milano peacefully ##logies akira rani regulator ##36 grasses ##rance luzon crows compiler gretchen seaman edouard tab buccaneers ellington hamlets whig socialists ##anto directorial easton mythological ##kr ##vary rhineland semantic taut dune inventions succeeds ##iter replication branched ##pired jul prosecuted kangaroo penetrated ##avian middlesbrough doses bleak madam predatory relentless ##vili reluctance ##vir hailey crore silvery 1759 monstrous swimmers transmissions hawthorn informing ##eral toilets caracas crouch kb ##sett 295 cartel hadley ##aling alexia yvonne ##biology cinderella eton superb blizzard stabbing industrialist maximus ##gm ##orus groves maud clade oversized comedic ##bella rosen nomadic fulham montane beverages galaxies redundant swarm ##rot ##folia ##llis buckinghamshire fen bearings bahadur ##rom gilles phased dynamite faber benoit vip ##ount ##wd booking fractured tailored anya spices westwood cairns auditions inflammation steamed ##rocity ##acion ##urne skyla thereof watford torment archdeacon transforms lulu demeanor fucked serge ##sor mckenna minas entertainer ##icide caress originate residue ##sty 1740 ##ilised ##org beech ##wana subsidies ##ghton emptied gladstone ru firefighters voodoo ##rcle het nightingale tamara edmond ingredient weaknesses silhouette 285 compatibility withdrawing hampson ##mona anguish giggling ##mber bookstore ##jiang southernmost tilting ##vance bai economical rf briefcase dreadful hinted projections shattering totaling ##rogate analogue indicted periodical fullback ##dman haynes ##tenberg ##ffs ##ishment 1745 thirst stumble penang vigorous ##ddling ##kor ##lium octave ##ove ##enstein ##inen ##ones siberian ##uti cbn repeal swaying ##vington khalid tanaka unicorn otago plastered lobe riddle ##rella perch ##ishing croydon filtered graeme tripoli ##ossa crocodile ##chers sufi mined ##tung inferno lsu ##phi swelled utilizes £2 cale periodicals styx hike informally coop lund ##tidae ala hen qui transformations disposed sheath chickens ##cade fitzroy sas silesia unacceptable odisha 1650 sabrina pe spokane ratios athena massage shen dilemma ##drum ##riz ##hul corona doubtful niall ##pha ##bino fines cite acknowledging bangor ballard bathurst ##resh huron mustered alzheimer garments kinase tyre warship ##cp flashback pulmonary braun cheat kamal cyclists constructions grenades ndp traveller excuses stomped signalling trimmed futsal mosques relevance ##wine wta ##23 ##vah ##lter hoc ##riding optimistic ##´s deco sim interacting rejecting moniker waterways ##ieri ##oku mayors gdansk outnumbered pearls ##ended ##hampton fairs totals dominating 262 notions stairway compiling pursed commodities grease yeast ##jong carthage griffiths residual amc contraction laird sapphire ##marine ##ivated amalgamation dissolve inclination lyle packaged altitudes suez canons graded lurched narrowing boasts guise wed enrico ##ovsky rower scarred bree cub iberian protagonists bargaining proposing trainers voyages vans fishes ##aea ##ivist ##verance encryption artworks kazan sabre cleopatra hepburn rotting supremacy mecklenburg ##brate burrows hazards outgoing flair organizes ##ctions scorpion ##usions boo 234 chevalier dunedin slapping ##34 ineligible pensions ##38 ##omic manufactures emails bismarck 238 weakening blackish ding mcgee quo ##rling northernmost xx manpower greed sampson clicking ##ange ##horpe ##inations ##roving torre ##eptive ##moral symbolism 38th asshole meritorious outfits splashed biographies sprung astros ##tale 302 737 filly raoul nw tokugawa linden clubhouse ##apa tracts romano ##pio putin tags ##note chained dickson gunshot moe gunn rashid ##tails zipper ##bas ##nea contrasted ##ply ##udes plum pharaoh ##pile aw comedies ingrid sandwiches subdivisions 1100 mariana nokia kamen hz delaney veto herring ##words possessive outlines ##roup siemens stairwell rc gallantry messiah palais yells 233 zeppelin ##dm bolivar ##cede smackdown mckinley ##mora ##yt muted geologic finely unitary avatar hamas maynard rees bog contrasting ##rut liv chico disposition pixel ##erate becca dmitry yeshiva narratives ##lva ##ulton mercenary sharpe tempered navigate stealth amassed keynes ##lini untouched ##rrie havoc lithium ##fighting abyss graf southward wolverine balloons implements ngos transitions ##icum ambushed concacaf dormant economists ##dim costing csi rana universite boulders verity ##llon collin mellon misses cypress fluorescent lifeless spence ##ulla crewe shepard pak revelations ##م jolly gibbons paw ##dro ##quel freeing ##test shack fries palatine ##51 ##hiko accompaniment cruising recycled ##aver erwin sorting synthesizers dyke realities sg strides enslaved wetland ##ghan competence gunpowder grassy maroon reactors objection ##oms carlson gearbox macintosh radios shelton ##sho clergyman prakash 254 mongols trophies oricon 228 stimuli twenty20 cantonese cortes mirrored ##saurus bhp cristina melancholy ##lating enjoyable nuevo ##wny downfall schumacher ##ind banging lausanne rumbled paramilitary reflex ax amplitude migratory ##gall ##ups midi barnard lastly sherry ##hp ##nall keystone ##kra carleton slippery ##53 coloring foe socket otter ##rgos mats ##tose consultants bafta bison topping ##km 490 primal abandonment transplant atoll hideous mort pained reproduced tae howling ##turn unlawful billionaire hotter poised lansing ##chang dinamo retro messing nfc domesday ##mina blitz timed ##athing ##kley ascending gesturing ##izations signaled tis chinatown mermaid savanna jameson ##aint catalina ##pet ##hers cochrane cy chatting ##kus alerted computation mused noelle majestic mohawk campo octagonal ##sant ##hend 241 aspiring ##mart comprehend iona paralyzed shimmering swindon rhone ##eley reputed configurations pitchfork agitation francais gillian lipstick ##ilo outsiders pontifical resisting bitterness sewer rockies ##edd ##ucher misleading 1756 exiting galloway ##nging risked ##heart 246 commemoration schultz ##rka integrating ##rsa poses shrieked ##weiler guineas gladys jerking owls goldsmith nightly penetrating ##unced lia ##33 ignited betsy ##aring ##thorpe follower vigorously ##rave coded kiran knit zoology tbilisi ##28 ##bered repository govt deciduous dino growling ##bba enhancement unleashed chanting pussy biochemistry ##eric kettle repression toxicity nrhp ##arth ##kko ##bush ernesto commended outspoken 242 mca parchment sms kristen ##aton bisexual raked glamour navajo a2 conditioned showcased ##hma spacious youthful ##esa usl appliances junta brest layne conglomerate enchanted chao loosened picasso circulating inspect montevideo ##centric ##kti piazza spurred ##aith bari freedoms poultry stamford lieu ##ect indigo sarcastic bahia stump attach dvds frankenstein lille approx scriptures pollen ##script nmi overseen ##ivism tides proponent newmarket inherit milling ##erland centralized ##rou distributors credentials drawers abbreviation ##lco ##xon downing uncomfortably ripe ##oes erase franchises ##ever populace ##bery ##khar decomposition pleas ##tet daryl sabah ##stle ##wide fearless genie lesions annette ##ogist oboe appendix nair dripped petitioned maclean mosquito parrot rpg hampered 1648 operatic reservoirs ##tham irrelevant jolt summarized ##fp medallion ##taff ##− clawed harlow narrower goddard marcia bodied fremont suarez altering tempest mussolini porn ##isms sweetly oversees walkers solitude grimly shrines hk ich supervisors hostess dietrich legitimacy brushes expressive ##yp dissipated ##rse localized systemic ##nikov gettysburg ##js ##uaries dialogues muttering 251 housekeeper sicilian discouraged ##frey beamed kaladin halftime kidnap ##amo ##llet 1754 synonymous depleted instituto insulin reprised ##opsis clashed ##ctric interrupting radcliffe insisting medici 1715 ejected playfully turbulent ##47 starvation ##rini shipment rebellious petersen verification merits ##rified cakes ##charged 1757 milford shortages spying fidelity ##aker emitted storylines harvested seismic ##iform cheung kilda theoretically barbie lynx ##rgy ##tius goblin mata poisonous ##nburg reactive residues obedience ##евич conjecture ##rac 401 hating sixties kicker moaning motown ##bha emancipation neoclassical ##hering consoles ebert professorship ##tures sustaining assaults obeyed affluent incurred tornadoes ##eber ##zow emphasizing highlanders cheated helmets ##ctus internship terence bony executions legislators berries peninsular tinged ##aco 1689 amplifier corvette ribbons lavish pennant ##lander worthless ##chfield ##forms mariano pyrenees expenditures ##icides chesterfield mandir tailor 39th sergey nestled willed aristocracy devotees goodnight raaf rumored weaponry remy appropriations harcourt burr riaa ##lence limitation unnoticed guo soaking swamps ##tica collapsing tatiana descriptive brigham psalm ##chment maddox ##lization patti caliph ##aja akron injuring serra ##ganj basins ##sari astonished launcher ##church hilary wilkins sewing ##sf stinging ##fia ##ncia underwood startup ##ition compilations vibrations embankment jurist ##nity bard juventus groundwater kern palaces helium boca cramped marissa soto ##worm jae princely ##ggy faso bazaar warmly ##voking 229 pairing ##lite ##grate ##nets wien freaked ulysses rebirth ##alia ##rent mummy guzman jimenez stilled ##nitz trajectory tha woken archival professions ##pts ##pta hilly shadowy shrink ##bolt norwood glued migrate stereotypes devoid ##pheus 625 evacuate horrors infancy gotham knowles optic downloaded sachs kingsley parramatta darryl mor ##onale shady commence confesses kan ##meter ##placed marlborough roundabout regents frigates io ##imating gothenburg revoked carvings clockwise convertible intruder ##sche banged ##ogo vicky bourgeois ##mony dupont footing ##gum pd ##real buckle yun penthouse sane 720 serviced stakeholders neumann bb ##eers comb ##gam catchment pinning rallies typing ##elles forefront freiburg sweetie giacomo widowed goodwill worshipped aspirations midday ##vat fishery ##trick bournemouth turk 243 hearth ethanol guadalajara murmurs sl ##uge afforded scripted ##hta wah ##jn coroner translucent 252 memorials puck progresses clumsy ##race 315 candace recounted ##27 ##slin ##uve filtering ##mac howl strata heron leveled ##ays dubious ##oja ##т ##wheel citations exhibiting ##laya ##mics ##pods turkic ##lberg injunction ##ennial ##mit antibodies ##44 organise ##rigues cardiovascular cushion inverness ##zquez dia cocoa sibling ##tman ##roid expanse feasible tunisian algiers ##relli rus bloomberg dso westphalia bro tacoma 281 downloads ##ours konrad duran ##hdi continuum jett compares legislator secession ##nable ##gues ##zuka translating reacher ##gley ##ła aleppo ##agi tc orchards trapping linguist versatile drumming postage calhoun superiors ##mx barefoot leary ##cis ignacio alfa kaplan ##rogen bratislava mori ##vot disturb haas 313 cartridges gilmore radiated salford tunic hades ##ulsive archeological delilah magistrates auditioned brewster charters empowerment blogs cappella dynasties iroquois whipping ##krishna raceway truths myra weaken judah mcgregor ##horse mic refueling 37th burnley bosses markus premio query ##gga dunbar ##economic darkest lyndon sealing commendation reappeared ##mun addicted ezio slaughtered satisfactory shuffle ##eves ##thic ##uj fortification warrington ##otto resurrected fargo mane ##utable ##lei ##space foreword ox ##aris ##vern abrams hua ##mento sakura ##alo uv sentimental ##skaya midfield ##eses sturdy scrolls macleod ##kyu entropy ##lance mitochondrial cicero excelled thinner convoys perceive ##oslav ##urable systematically grind burkina 287 ##tagram ops ##aman guantanamo ##cloth ##tite forcefully wavy ##jou pointless ##linger ##tze layton portico superficial clerical outlaws ##hism burials muir ##inn creditors hauling rattle ##leg calais monde archers reclaimed dwell wexford hellenic falsely remorse ##tek dough furnishings ##uttered gabon neurological novice ##igraphy contemplated pulpit nightstand saratoga ##istan documenting pulsing taluk ##firmed busted marital ##rien disagreements wasps ##yes hodge mcdonnell mimic fran pendant dhabi musa ##nington congratulations argent darrell concussion losers regrets thessaloniki reversal donaldson hardwood thence achilles ritter ##eran demonic jurgen prophets goethe eki classmate buff ##cking yank irrational ##inging perished seductive qur sourced ##crat ##typic mustard ravine barre horizontally characterization phylogenetic boise ##dit ##runner ##tower brutally intercourse seduce ##bbing fay ferris ogden amar nik unarmed ##inator evaluating kyrgyzstan sweetness ##lford ##oki mccormick meiji notoriety stimulate disrupt figuring instructional mcgrath ##zoo groundbreaking ##lto flinch khorasan agrarian bengals mixer radiating ##sov ingram pitchers nad tariff ##cript tata ##codes ##emi ##ungen appellate lehigh ##bled ##giri brawl duct texans ##ciation ##ropolis skipper speculative vomit doctrines stresses 253 davy graders whitehead jozef timely cumulative haryana paints appropriately boon cactus ##ales ##pid dow legions ##pit perceptions 1730 picturesque ##yse periphery rune wr ##aha celtics sentencing whoa ##erin confirms variance 425 moines mathews spade rave m1 fronted fx blending alleging reared ##gl 237 ##paper grassroots eroded ##free ##physical directs ordeal ##sław accelerate hacker rooftop ##inia lev buys cebu devote ##lce specialising ##ulsion choreographed repetition warehouses ##ryl paisley tuscany analogy sorcerer hash huts shards descends exclude nix chaplin gaga ito vane ##drich causeway misconduct limo orchestrated glands jana ##kot u2 ##mple ##sons branching contrasts scoop longed ##virus chattanooga ##75 syrup cornerstone ##tized ##mind ##iaceae careless precedence frescoes ##uet chilled consult modelled snatch peat ##thermal caucasian humane relaxation spins temperance ##lbert occupations lambda hybrids moons mp3 ##oese 247 rolf societal yerevan ness ##ssler befriended mechanized nominate trough boasted cues seater ##hom bends ##tangle conductors emptiness ##lmer eurasian adriatic tian ##cie anxiously lark propellers chichester jock ev 2a ##holding credible recounts tori loyalist abduction ##hoot ##redo nepali ##mite ventral tempting ##ango ##crats steered ##wice javelin dipping laborers prentice looming titanium ##ː badges emir tensor ##ntation egyptians rash denies hawthorne lombard showers wehrmacht dietary trojan ##reus welles executing horseshoe lifeboat ##lak elsa infirmary nearing roberta boyer mutter trillion joanne ##fine ##oked sinks vortex uruguayan clasp sirius ##block accelerator prohibit sunken byu chronological diplomats ochreous 510 symmetrical 1644 maia ##tology salts reigns atrocities ##ия hess bared issn ##vyn cater saturated ##cycle ##isse sable voyager dyer yusuf ##inge fountains wolff ##39 ##nni engraving rollins atheist ominous ##ault herr chariot martina strung ##fell ##farlane horrific sahib gazes saetan erased ptolemy ##olic flushing lauderdale analytic ##ices 530 navarro beak gorilla herrera broom guadalupe raiding sykes 311 bsc deliveries 1720 invasions carmichael tajikistan thematic ecumenical sentiments onstage ##rians ##brand ##sume catastrophic flanks molten ##arns waller aimee terminating ##icing alternately ##oche nehru printers outraged ##eving empires template banners repetitive za ##oise vegetarian ##tell guiana opt cavendish lucknow synthesized ##hani ##mada finalized ##ctable fictitious mayoral unreliable ##enham embracing peppers rbis ##chio ##neo inhibition slashed togo orderly embroidered safari salty 236 barron benito totaled ##dak pubs simulated caden devin tolkien momma welding sesame ##ept gottingen hardness 630 shaman temeraire 620 adequately pediatric ##kit ck assertion radicals composure cadence seafood beaufort lazarus mani warily cunning kurdistan 249 cantata ##kir ares ##41 ##clusive nape townland geared insulted flutter boating violate draper dumping malmo ##hh ##romatic firearm alta bono obscured ##clave exceeds panorama unbelievable ##train preschool ##essed disconnected installing rescuing secretaries accessibility ##castle ##drive ##ifice ##film bouts slug waterway mindanao ##buro ##ratic halves ##ل calming liter maternity adorable bragg electrification mcc ##dote roxy schizophrenia ##body munoz kaye whaling 239 mil tingling tolerant ##ago unconventional volcanoes ##finder deportivo ##llie robson kaufman neuroscience wai deportation masovian scraping converse ##bh hacking bulge ##oun administratively yao 580 amp mammoth booster claremont hooper nomenclature pursuits mclaughlin melinda ##sul catfish barclay substrates taxa zee originals kimberly packets padma ##ality borrowing ostensibly solvent ##bri ##genesis ##mist lukas shreveport veracruz ##ь ##lou ##wives cheney tt anatolia hobbs ##zyn cyclic radiant alistair greenish siena dat independents ##bation conform pieter hyper applicant bradshaw spores telangana vinci inexpensive nuclei 322 jang nme soho spd ##ign cradled receptionist pow ##43 ##rika fascism ##ifer experimenting ##ading ##iec ##region 345 jocelyn maris stair nocturnal toro constabulary elgin ##kker msc ##giving ##schen ##rase doherty doping sarcastically batter maneuvers ##cano ##apple ##gai ##git intrinsic ##nst ##stor 1753 showtime cafes gasps lviv ushered ##thed fours restart astonishment transmitting flyer shrugs ##sau intriguing cones dictated mushrooms medial ##kovsky ##elman escorting gaped ##26 godfather ##door ##sell djs recaptured timetable vila 1710 3a aerodrome mortals scientology ##orne angelina mag convection unpaid insertion intermittent lego ##nated endeavor kota pereira ##lz 304 bwv glamorgan insults agatha fey ##cend fleetwood mahogany protruding steamship zeta ##arty mcguire suspense ##sphere advising urges ##wala hurriedly meteor gilded inline arroyo stalker ##oge excitedly revered ##cure earle introductory ##break ##ilde mutants puff pulses reinforcement ##haling curses lizards stalk correlated ##fixed fallout macquarie ##unas bearded denton heaving 802 ##ocation winery assign dortmund ##lkirk everest invariant charismatic susie ##elling bled lesley telegram sumner bk ##ogen ##к wilcox needy colbert duval ##iferous ##mbled allotted attends imperative ##hita replacements hawker ##inda insurgency ##zee ##eke casts ##yla 680 ives transitioned ##pack ##powering authoritative baylor flex cringed plaintiffs woodrow ##skie drastic ape aroma unfolded commotion nt preoccupied theta routines lasers privatization wand domino ek clenching nsa strategically showered bile handkerchief pere storing christophe insulting 316 nakamura romani asiatic magdalena palma cruises stripping 405 konstantin soaring ##berman colloquially forerunner havilland incarcerated parasites sincerity ##utus disks plank saigon ##ining corbin homo ornaments powerhouse ##tlement chong fastened feasibility idf morphological usable ##nish ##zuki aqueduct jaguars keepers ##flies aleksandr faust assigns ewing bacterium hurled tricky hungarians integers wallis 321 yamaha ##isha hushed oblivion aviator evangelist friars ##eller monograph ode ##nary airplanes labourers charms ##nee 1661 hagen tnt rudder fiesta transcript dorothea ska inhibitor maccabi retorted raining encompassed clauses menacing 1642 lineman ##gist vamps ##ape ##dick gloom ##rera dealings easing seekers ##nut ##pment helens unmanned ##anu ##isson basics ##amy ##ckman adjustments 1688 brutality horne ##zell sui ##55 ##mable aggregator ##thal rhino ##drick ##vira counters zoom ##01 ##rting mn montenegrin packard ##unciation ##♭ ##kki reclaim scholastic thugs pulsed ##icia syriac quan saddam banda kobe blaming buddies dissent ##lusion ##usia corbett jaya delle erratic lexie ##hesis 435 amiga hermes ##pressing ##leen chapels gospels jamal ##uating compute revolving warp ##sso ##thes armory ##eras ##gol antrim loki ##kow ##asian ##good ##zano braid handwriting subdistrict funky pantheon ##iculate concurrency estimation improper juliana ##his newcomers johnstone staten communicated ##oco ##alle sausage stormy ##stered ##tters superfamily ##grade acidic collateral tabloid ##oped ##rza bladder austen ##ellant mcgraw ##hay hannibal mein aquino lucifer wo badger boar cher christensen greenberg interruption ##kken jem 244 mocked bottoms cambridgeshire ##lide sprawling ##bbly eastwood ghent synth ##buck advisers ##bah nominally hapoel qu daggers estranged fabricated towels vinnie wcw misunderstanding anglia nothin unmistakable ##dust ##lova chilly marquette truss ##edge ##erine reece ##lty ##chemist ##connected 272 308 41st bash raion waterfalls ##ump ##main labyrinth queue theorist ##istle bharatiya flexed soundtracks rooney leftist patrolling wharton plainly alleviate eastman schuster topographic engages immensely unbearable fairchild 1620 dona lurking parisian oliveira ia indictment hahn bangladeshi ##aster vivo ##uming ##ential antonia expects indoors kildare harlan ##logue ##ogenic ##sities forgiven ##wat childish tavi ##mide ##orra plausible grimm successively scooted ##bola ##dget ##rith spartans emery flatly azure epilogue ##wark flourish ##iny ##tracted ##overs ##oshi bestseller distressed receipt spitting hermit topological ##cot drilled subunit francs ##layer eel ##fk ##itas octopus footprint petitions ufo ##say ##foil interfering leaking palo ##metry thistle valiant ##pic narayan mcpherson ##fast gonzales ##ym ##enne dustin novgorod solos ##zman doin ##raph ##patient ##meyer soluble ashland cuffs carole pendleton whistling vassal ##river deviation revisited constituents rallied rotate loomed ##eil ##nting amateurs augsburg auschwitz crowns skeletons ##cona bonnet 257 dummy globalization simeon sleeper mandal differentiated ##crow ##mare milne bundled exasperated talmud owes segregated ##feng ##uary dentist piracy props ##rang devlin ##torium malicious paws ##laid dependency ##ergy ##fers ##enna 258 pistons rourke jed grammatical tres maha wig 512 ghostly jayne ##achal ##creen ##ilis ##lins ##rence designate ##with arrogance cambodian clones showdown throttle twain ##ception lobes metz nagoya 335 braking ##furt 385 roaming ##minster amin crippled ##37 ##llary indifferent hoffmann idols intimidating 1751 261 influenza memo onions 1748 bandage consciously ##landa ##rage clandestine observes swiped tangle ##ener ##jected ##trum ##bill ##lta hugs congresses josiah spirited ##dek humanist managerial filmmaking inmate rhymes debuting grimsby ur ##laze duplicate vigor ##tf republished bolshevik refurbishment antibiotics martini methane newscasts royale horizons levant iain visas ##ischen paler ##around manifestation snuck alf chop futile pedestal rehab ##kat bmg kerman res fairbanks jarrett abstraction saharan ##zek 1746 procedural clearer kincaid sash luciano ##ffey crunch helmut ##vara revolutionaries ##tute creamy leach ##mmon 1747 permitting nes plight wendell ##lese contra ts clancy ipa mach staples autopsy disturbances nueva karin pontiac ##uding proxy venerable haunt leto bergman expands ##helm wal ##pipe canning celine cords obesity ##enary intrusion planner ##phate reasoned sequencing 307 harrow ##chon ##dora marred mcintyre repay tarzan darting 248 harrisburg margarita repulsed ##hur ##lding belinda hamburger novo compliant runways bingham registrar skyscraper ic cuthbert improvisation livelihood ##corp ##elial admiring ##dened sporadic believer casablanca popcorn ##29 asha shovel ##bek ##dice coiled tangible ##dez casper elsie resin tenderness rectory ##ivision avail sonar ##mori boutique ##dier guerre bathed upbringing vaulted sandals blessings ##naut ##utnant 1680 306 foxes pia corrosion hesitantly confederates crystalline footprints shapiro tirana valentin drones 45th microscope shipments texted inquisition wry guernsey unauthorized resigning 760 ripple schubert stu reassure felony ##ardo brittle koreans ##havan ##ives dun implicit tyres ##aldi ##lth magnolia ##ehan ##puri ##poulos aggressively fei gr familiarity ##poo indicative ##trust fundamentally jimmie overrun 395 anchors moans ##opus britannia armagh ##ggle purposely seizing ##vao bewildered mundane avoidance cosmopolitan geometridae quartermaster caf 415 chatter engulfed gleam purge ##icate juliette jurisprudence guerra revisions ##bn casimir brew ##jm 1749 clapton cloudy conde hermitage 278 simulations torches vincenzo matteo ##rill hidalgo booming westbound accomplishment tentacles unaffected ##sius annabelle flopped sloping ##litz dreamer interceptor vu ##loh consecration copying messaging breaker climates hospitalized 1752 torino afternoons winfield witnessing ##teacher breakers choirs sawmill coldly ##ege sipping haste uninhabited conical bibliography pamphlets severn edict ##oca deux illnesses grips ##pl rehearsals sis thinkers tame ##keepers 1690 acacia reformer ##osed ##rys shuffling ##iring ##shima eastbound ionic rhea flees littered ##oum rocker vomiting groaning champ overwhelmingly civilizations paces sloop adoptive ##tish skaters ##vres aiding mango ##joy nikola shriek ##ignon pharmaceuticals ##mg tuna calvert gustavo stocked yearbook ##urai ##mana computed subsp riff hanoi kelvin hamid moors pastures summons jihad nectar ##ctors bayou untitled pleasing vastly republics intellect ##η ##ulio ##tou crumbling stylistic sb ##ی consolation frequented h₂o walden widows ##iens 404 ##ignment chunks improves 288 grit recited ##dev snarl sociological ##arte ##gul inquired ##held bruise clube consultancy homogeneous hornets multiplication pasta prick savior ##grin ##kou ##phile yoon ##gara grimes vanishing cheering reacting bn distillery ##quisite ##vity coe dockyard massif ##jord escorts voss ##valent byte chopped hawke illusions workings floats ##koto ##vac kv annapolis madden ##onus alvaro noctuidae ##cum ##scopic avenge steamboat forte illustrates erika ##trip 570 dew nationalities bran manifested thirsty diversified muscled reborn ##standing arson ##lessness ##dran ##logram ##boys ##kushima ##vious willoughby ##phobia 286 alsace dashboard yuki ##chai granville myspace publicized tricked ##gang adjective ##ater relic reorganisation enthusiastically indications saxe ##lassified consolidate iec padua helplessly ramps renaming regulars pedestrians accents convicts inaccurate lowers mana ##pati barrie bjp outta someplace berwick flanking invoked marrow sparsely excerpts clothed rei ##ginal wept ##straße ##vish alexa excel ##ptive membranes aquitaine creeks cutler sheppard implementations ns ##dur fragrance budge concordia magnesium marcelo ##antes gladly vibrating ##rral ##ggles montrose ##omba lew seamus 1630 cocky ##ament ##uen bjorn ##rrick fielder fluttering ##lase methyl kimberley mcdowell reductions barbed ##jic ##tonic aeronautical condensed distracting ##promising huffed ##cala ##sle claudius invincible missy pious balthazar ci ##lang butte combo orson ##dication myriad 1707 silenced ##fed ##rh coco netball yourselves ##oza clarify heller peg durban etudes offender roast blackmail curvature ##woods vile 309 illicit suriname ##linson overture 1685 bubbling gymnast tucking ##mming ##ouin maldives ##bala gurney ##dda ##eased ##oides backside pinto jars racehorse tending ##rdial baronetcy wiener duly ##rke barbarian cupping flawed ##thesis bertha pleistocene puddle swearing ##nob ##tically fleeting prostate amulet educating ##mined ##iti ##tler 75th jens respondents analytics cavaliers papacy raju ##iente ##ulum ##tip funnel 271 disneyland ##lley sociologist ##iam 2500 faulkner louvre menon ##dson 276 ##ower afterlife mannheim peptide referees comedians meaningless ##anger ##laise fabrics hurley renal sleeps ##bour ##icle breakout kristin roadside animator clover disdain unsafe redesign ##urity firth barnsley portage reset narrows 268 commandos expansive speechless tubular ##lux essendon eyelashes smashwords ##yad ##bang ##claim craved sprinted chet somme astor wrocław orton 266 bane ##erving ##uing mischief ##amps ##sund scaling terre ##xious impairment offenses undermine moi soy contiguous arcadia inuit seam ##tops macbeth rebelled ##icative ##iot 590 elaborated frs uniformed ##dberg 259 powerless priscilla stimulated 980 qc arboretum frustrating trieste bullock ##nified enriched glistening intern ##adia locus nouvelle ollie ike lash starboard ee tapestry headlined hove rigged ##vite pollock ##yme thrive clustered cas roi gleamed olympiad ##lino pressured regimes ##hosis ##lick ripley ##ophone kickoff gallon rockwell ##arable crusader glue revolutions scrambling 1714 grover ##jure englishman aztec 263 contemplating coven ipad preach triumphant tufts ##esian rotational ##phus 328 falkland ##brates strewn clarissa rejoin environmentally glint banded drenched moat albanians johor rr maestro malley nouveau shaded taxonomy v6 adhere bunk airfields ##ritan 1741 encompass remington tran ##erative amelie mazda friar morals passions ##zai breadth vis ##hae argus burnham caressing insider rudd ##imov ##mini ##rso italianate murderous textual wainwright armada bam weave timer ##taken ##nh fra ##crest ardent salazar taps tunis ##ntino allegro gland philanthropic ##chester implication ##optera esq judas noticeably wynn ##dara inched indexed crises villiers bandit royalties patterned cupboard interspersed accessory isla kendrick entourage stitches ##esthesia headwaters ##ior interlude distraught draught 1727 ##basket biased sy transient triad subgenus adapting kidd shortstop ##umatic dimly spiked mcleod reprint nellie pretoria windmill ##cek singled ##mps 273 reunite ##orous 747 bankers outlying ##omp ##ports ##tream apologies cosmetics patsy ##deh ##ocks ##yson bender nantes serene ##nad lucha mmm 323 ##cius ##gli cmll coinage nestor juarez ##rook smeared sprayed twitching sterile irina embodied juveniles enveloped miscellaneous cancers dq gulped luisa crested swat donegal ref ##anov ##acker hearst mercantile ##lika doorbell ua vicki ##alla ##som bilbao psychologists stryker sw horsemen turkmenistan wits ##national anson mathew screenings ##umb rihanna ##agne ##nessy aisles ##iani ##osphere hines kenton saskatoon tasha truncated ##champ ##itan mildred advises fredrik interpreting inhibitors ##athi spectroscopy ##hab ##kong karim panda ##oia ##nail ##vc conqueror kgb leukemia ##dity arrivals cheered pisa phosphorus shielded ##riated mammal unitarian urgently chopin sanitary ##mission spicy drugged hinges ##tort tipping trier impoverished westchester ##caster 267 epoch nonstop ##gman ##khov aromatic centrally cerro ##tively ##vio billions modulation sedimentary 283 facilitating outrageous goldstein ##eak ##kt ld maitland penultimate pollard ##dance fleets spaceship vertebrae ##nig alcoholism als recital ##bham ##ference ##omics m2 ##bm trois ##tropical ##в commemorates ##meric marge ##raction 1643 670 cosmetic ravaged ##ige catastrophe eng ##shida albrecht arterial bellamy decor harmon ##rde bulbs synchronized vito easiest shetland shielding wnba ##glers ##ssar ##riam brianna cumbria ##aceous ##rard cores thayer ##nsk brood hilltop luminous carts keynote larkin logos ##cta ##ا ##mund ##quay lilith tinted 277 wrestle mobilization ##uses sequential siam bloomfield takahashi 274 ##ieving presenters ringo blazed witty ##oven ##ignant devastation haydn harmed newt therese ##peed gershwin molina rabbis sudanese 001 innate restarted ##sack ##fus slices wb ##shah enroll hypothetical hysterical 1743 fabio indefinite warped ##hg exchanging 525 unsuitable ##sboro gallo 1603 bret cobalt homemade ##hunter mx operatives ##dhar terraces durable latch pens whorls ##ctuated ##eaux billing ligament succumbed ##gly regulators spawn ##brick ##stead filmfare rochelle ##nzo 1725 circumstance saber supplements ##nsky ##tson crowe wellesley carrot ##9th ##movable primate drury sincerely topical ##mad ##rao callahan kyiv smarter tits undo ##yeh announcements anthologies barrio nebula ##islaus ##shaft ##tyn bodyguards 2021 assassinate barns emmett scully ##mah ##yd ##eland ##tino ##itarian demoted gorman lashed prized adventist writ ##gui alla invertebrates ##ausen 1641 amman 1742 align healy redistribution ##gf ##rize insulation ##drop adherents hezbollah vitro ferns yanking 269 php registering uppsala cheerleading confines mischievous tully ##ross 49th docked roam stipulated pumpkin ##bry prompt ##ezer blindly shuddering craftsmen frail scented katharine scramble shaggy sponge helix zaragoza 279 ##52 43rd backlash fontaine seizures posse cowan nonfiction telenovela wwii hammered undone ##gpur encircled irs ##ivation artefacts oneself searing smallpox ##belle ##osaurus shandong breached upland blushing rankin infinitely psyche tolerated docking evicted ##col unmarked ##lving gnome lettering litres musique ##oint benevolent ##jal blackened ##anna mccall racers tingle ##ocene ##orestation introductions radically 292 ##hiff ##باد 1610 1739 munchen plead ##nka condo scissors ##sight ##tens apprehension ##cey ##yin hallmark watering formulas sequels ##llas aggravated bae commencing ##building enfield prohibits marne vedic civilized euclidean jagger beforehand blasts dumont ##arney ##nem 740 conversions hierarchical rios simulator ##dya ##lellan hedges oleg thrusts shadowed darby maximize 1744 gregorian ##nded ##routed sham unspecified ##hog emory factual ##smo ##tp fooled ##rger ortega wellness marlon ##oton ##urance casket keating ley enclave ##ayan char influencing jia ##chenko 412 ammonia erebidae incompatible violins cornered ##arat grooves astronauts columbian rampant fabrication kyushu mahmud vanish ##dern mesopotamia ##lete ict ##rgen caspian kenji pitted ##vered 999 grimace roanoke tchaikovsky twinned ##analysis ##awan xinjiang arias clemson kazakh sizable 1662 ##khand ##vard plunge tatum vittorio ##nden cholera ##dana ##oper bracing indifference projectile superliga ##chee realises upgrading 299 porte retribution ##vies nk stil ##resses ama bureaucracy blackberry bosch testosterone collapses greer ##pathic ioc fifties malls ##erved bao baskets adolescents siegfried ##osity ##tosis mantra detecting existent fledgling ##cchi dissatisfied gan telecommunication mingled sobbed 6000 controversies outdated taxis ##raus fright slams ##lham ##fect ##tten detectors fetal tanned ##uw fray goth olympian skipping mandates scratches sheng unspoken hyundai tracey hotspur restrictive ##buch americana mundo ##bari burroughs diva vulcan ##6th distinctions thumping ##ngen mikey sheds fide rescues springsteen vested valuation ##ece ##ely pinnacle rake sylvie ##edo almond quivering ##irus alteration faltered ##wad 51st hydra ticked ##kato recommends ##dicated antigua arjun stagecoach wilfred trickle pronouns ##pon aryan nighttime ##anian gall pea stitch ##hei leung milos ##dini eritrea nexus starved snowfall kant parasitic cot discus hana strikers appleton kitchens ##erina ##partisan ##itha ##vius disclose metis ##channel 1701 tesla ##vera fitch 1735 blooded ##tila decimal ##tang ##bai cyclones eun bottled peas pensacola basha bolivian crabs boil lanterns partridge roofed 1645 necks ##phila opined patting ##kla ##lland chuckles volta whereupon ##nche devout euroleague suicidal ##dee inherently involuntary knitting nasser ##hide puppets colourful courageous southend stills miraculous hodgson richer rochdale ethernet greta uniting prism umm ##haya ##itical ##utation deterioration pointe prowess ##ropriation lids scranton billings subcontinent ##koff ##scope brute kellogg psalms degraded ##vez stanisław ##ructured ferreira pun astonishing gunnar ##yat arya prc gottfried ##tight excursion ##ographer dina ##quil ##nare huffington illustrious wilbur gundam verandah ##zard naacp ##odle constructive fjord kade ##naud generosity thrilling baseline cayman frankish plastics accommodations zoological ##fting cedric qb motorized ##dome ##otted squealed tackled canucks budgets situ asthma dail gabled grasslands whimpered writhing judgments ##65 minnie pv ##carbon bananas grille domes monique odin maguire markham tierney ##estra ##chua libel poke speedy atrium laval notwithstanding ##edly fai kala ##sur robb ##sma listings luz supplementary tianjin ##acing enzo jd ric scanner croats transcribed ##49 arden cv ##hair ##raphy ##lver ##uy 357 seventies staggering alam horticultural hs regression timbers blasting ##ounded montagu manipulating ##cit catalytic 1550 troopers ##meo condemnation fitzpatrick ##oire ##roved inexperienced 1670 castes ##lative outing 314 dubois flicking quarrel ste learners 1625 iq whistled ##class 282 classify tariffs temperament 355 folly liszt ##yles immersed jordanian ceasefire apparel extras maru fished ##bio harta stockport assortment craftsman paralysis transmitters ##cola blindness ##wk fatally proficiency solemnly ##orno repairing amore groceries ultraviolet ##chase schoolhouse ##tua resurgence nailed ##otype ##× ruse saliva diagrams ##tructing albans rann thirties 1b antennas hilarious cougars paddington stats ##eger breakaway ipod reza authorship prohibiting scoffed ##etz ##ttle conscription defected trondheim ##fires ivanov keenan ##adan ##ciful ##fb ##slow locating ##ials ##tford cadiz basalt blankly interned rags rattling ##tick carpathian reassured sync bum guildford iss staunch ##onga astronomers sera sofie emergencies susquehanna ##heard duc mastery vh1 williamsburg bayer buckled craving ##khan ##rdes bloomington ##write alton barbecue ##bians justine ##hri ##ndt delightful smartphone newtown photon retrieval peugeot hissing ##monium ##orough flavors lighted relaunched tainted ##games ##lysis anarchy microscopic hopping adept evade evie ##beau inhibit sinn adjustable hurst intuition wilton cisco 44th lawful lowlands stockings thierry ##dalen ##hila ##nai fates prank tb maison lobbied provocative 1724 4a utopia ##qual carbonate gujarati purcell ##rford curtiss ##mei overgrown arenas mediation swallows ##rnik respectful turnbull ##hedron ##hope alyssa ozone ##ʻi ami gestapo johansson snooker canteen cuff declines empathy stigma ##ags ##iner ##raine taxpayers gui volga ##wright ##copic lifespan overcame tattooed enactment giggles ##ador ##camp barrington bribe obligatory orbiting peng ##enas elusive sucker ##vating cong hardship empowered anticipating estrada cryptic greasy detainees planck sudbury plaid dod marriott kayla ##ears ##vb ##zd mortally ##hein cognition radha 319 liechtenstein meade richly argyle harpsichord liberalism trumpets lauded tyrant salsa tiled lear promoters reused slicing trident ##chuk ##gami ##lka cantor checkpoint ##points gaul leger mammalian ##tov ##aar ##schaft doha frenchman nirvana ##vino delgado headlining ##eron ##iography jug tko 1649 naga intersections ##jia benfica nawab ##suka ashford gulp ##deck ##vill ##rug brentford frazier pleasures dunne potsdam shenzhen dentistry ##tec flanagan ##dorff ##hear chorale dinah prem quezon ##rogated relinquished sutra terri ##pani flaps ##rissa poly ##rnet homme aback ##eki linger womb ##kson ##lewood doorstep orthodoxy threaded westfield ##rval dioceses fridays subsided ##gata loyalists ##biotic ##ettes letterman lunatic prelate tenderly invariably souza thug winslow ##otide furlongs gogh jeopardy ##runa pegasus ##umble humiliated standalone tagged ##roller freshmen klan ##bright attaining initiating transatlantic logged viz ##uance 1723 combatants intervening stephane chieftain despised grazed 317 cdc galveston godzilla macro simulate ##planes parades ##esses 960 ##ductive ##unes equator overdose ##cans ##hosh ##lifting joshi epstein sonora treacherous aquatics manchu responsive ##sation supervisory ##christ ##llins ##ibar ##balance ##uso kimball karlsruhe mab ##emy ignores phonetic reuters spaghetti 820 almighty danzig rumbling tombstone designations lured outset ##felt supermarkets ##wt grupo kei kraft susanna ##blood comprehension genealogy ##aghan ##verted redding ##ythe 1722 bowing ##pore ##roi lest sharpened fulbright valkyrie sikhs ##unds swans bouquet merritt ##tage ##venting commuted redhead clerks leasing cesare dea hazy ##vances fledged greenfield servicemen ##gical armando blackout dt sagged downloadable intra potion pods ##4th ##mism xp attendants gambia stale ##ntine plump asteroids rediscovered buds flea hive ##neas 1737 classifications debuts ##eles olympus scala ##eurs ##gno ##mute hummed sigismund visuals wiggled await pilasters clench sulfate ##ances bellevue enigma trainee snort ##sw clouded denim ##rank ##rder churning hartman lodges riches sima ##missible accountable socrates regulates mueller ##cr 1702 avoids solids himalayas nutrient pup ##jevic squat fades nec ##lates ##pina ##rona ##ου privateer tequila ##gative ##mpton apt hornet immortals ##dou asturias cleansing dario ##rries ##anta etymology servicing zhejiang ##venor ##nx horned erasmus rayon relocating £10 ##bags escalated promenade stubble 2010s artisans axial liquids mora sho yoo ##tsky bundles oldies ##nally notification bastion ##ths sparkle ##lved 1728 leash pathogen highs ##hmi immature 880 gonzaga ignatius mansions monterrey sweets bryson ##loe polled regatta brightest pei rosy squid hatfield payroll addict meath cornerback heaviest lodging ##mage capcom rippled ##sily barnet mayhem ymca snuggled rousseau ##cute blanchard 284 fragmented leighton chromosomes risking ##md ##strel ##utter corinne coyotes cynical hiroshi yeomanry ##ractive ebook grading mandela plume agustin magdalene ##rkin bea femme trafford ##coll ##lun ##tance 52nd fourier upton ##mental camilla gust iihf islamabad longevity ##kala feldman netting ##rization endeavour foraging mfa orr ##open greyish contradiction graz ##ruff handicapped marlene tweed oaxaca spp campos miocene pri configured cooks pluto cozy pornographic ##entes 70th fairness glided jonny lynne rounding sired ##emon ##nist remade uncover ##mack complied lei newsweek ##jured ##parts ##enting ##pg 293 finer guerrillas athenian deng disused stepmother accuse gingerly seduction 521 confronting ##walker ##going gora nostalgia sabres virginity wrenched ##minated syndication wielding eyre ##56 ##gnon ##igny behaved taxpayer sweeps ##growth childless gallant ##ywood amplified geraldine scrape ##ffi babylonian fresco ##rdan ##kney ##position 1718 restricting tack fukuoka osborn selector partnering ##dlow 318 gnu kia tak whitley gables ##54 ##mania mri softness immersion ##bots ##evsky 1713 chilling insignificant pcs ##uis elites lina purported supplemental teaming ##americana ##dding ##inton proficient rouen ##nage ##rret niccolo selects ##bread fluffy 1621 gruff knotted mukherjee polgara thrash nicholls secluded smoothing thru corsica loaf whitaker inquiries ##rrier ##kam indochina 289 marlins myles peking ##tea extracts pastry superhuman connacht vogel ##ditional ##het ##udged ##lash gloss quarries refit teaser ##alic ##gaon 20s materialized sling camped pickering tung tracker pursuant ##cide cranes soc ##cini ##typical ##viere anhalt overboard workout chores fares orphaned stains ##logie fenton surpassing joyah triggers ##itte grandmaster ##lass ##lists clapping fraudulent ledger nagasaki ##cor ##nosis ##tsa eucalyptus tun ##icio ##rney ##tara dax heroism ina wrexham onboard unsigned ##dates moshe galley winnie droplets exiles praises watered noodles ##aia fein adi leland multicultural stink bingo comets erskine modernized canned constraint domestically chemotherapy featherweight stifled ##mum darkly irresistible refreshing hasty isolate ##oys kitchener planners ##wehr cages yarn implant toulon elects childbirth yue ##lind ##lone cn rightful sportsman junctions remodeled specifies ##rgh 291 ##oons complimented ##urgent lister ot ##logic bequeathed cheekbones fontana gabby ##dial amadeus corrugated maverick resented triangles ##hered ##usly nazareth tyrol 1675 assent poorer sectional aegean ##cous 296 nylon ghanaian ##egorical ##weig cushions forbid fusiliers obstruction somerville ##scia dime earrings elliptical leyte oder polymers timmy atm midtown piloted settles continual externally mayfield ##uh enrichment henson keane persians 1733 benji braden pep 324 ##efe contenders pepsi valet ##isches 298 ##asse ##earing goofy stroll ##amen authoritarian occurrences adversary ahmedabad tangent toppled dorchester 1672 modernism marxism islamist charlemagne exponential racks unicode brunette mbc pic skirmish ##bund ##lad ##powered ##yst hoisted messina shatter ##ctum jedi vantage ##music ##neil clemens mahmoud corrupted authentication lowry nils ##washed omnibus wounding jillian ##itors ##opped serialized narcotics handheld ##arm ##plicity intersecting stimulating ##onis crate fellowships hemingway casinos climatic fordham copeland drip beatty leaflets robber brothel madeira ##hedral sphinx ultrasound ##vana valor forbade leonid villas ##aldo duane marquez ##cytes disadvantaged forearms kawasaki reacts consular lax uncles uphold ##hopper concepcion dorsey lass ##izan arching passageway 1708 researches tia internationals ##graphs ##opers distinguishes javanese divert ##uven plotted ##listic ##rwin ##erik ##tify affirmative signifies validation ##bson kari felicity georgina zulu ##eros ##rained ##rath overcoming ##dot argyll ##rbin 1734 chiba ratification windy earls parapet ##marks hunan pristine astrid punta ##gart brodie ##kota ##oder malaga minerva rouse ##phonic bellowed pagoda portals reclamation ##gur ##odies ##⁄₄ parentheses quoting allergic palette showcases benefactor heartland nonlinear ##tness bladed cheerfully scans ##ety ##hone 1666 girlfriends pedersen hiram sous ##liche ##nator 1683 ##nery ##orio ##umen bobo primaries smiley ##cb unearthed uniformly fis metadata 1635 ind ##oted recoil ##titles ##tura ##ια 406 hilbert jamestown mcmillan tulane seychelles ##frid antics coli fated stucco ##grants 1654 bulky accolades arrays caledonian carnage optimism puebla ##tative ##cave enforcing rotherham seo dunlop aeronautics chimed incline zoning archduke hellenistic ##oses ##sions candi thong ##ople magnate rustic ##rsk projective slant ##offs danes hollis vocalists ##ammed congenital contend gesellschaft ##ocating ##pressive douglass quieter ##cm ##kshi howled salim spontaneously townsville buena southport ##bold kato 1638 faerie stiffly ##vus ##rled 297 flawless realising taboo ##7th bytes straightening 356 jena ##hid ##rmin cartwright berber bertram soloists 411 noses 417 coping fission hardin inca ##cen 1717 mobilized vhf ##raf biscuits curate ##85 ##anial 331 gaunt neighbourhoods 1540 ##abas blanca bypassed sockets behold coincidentally ##bane nara shave splinter terrific ##arion ##erian commonplace juris redwood waistband boxed caitlin fingerprints jennie naturalized ##ired balfour craters jody bungalow hugely quilt glitter pigeons undertaker bulging constrained goo ##sil ##akh assimilation reworked ##person persuasion ##pants felicia ##cliff ##ulent 1732 explodes ##dun ##inium ##zic lyman vulture hog overlook begs northwards ow spoil ##urer fatima favorably accumulate sargent sorority corresponded dispersal kochi toned ##imi ##lita internacional newfound ##agger ##lynn ##rigue booths peanuts ##eborg medicare muriel nur ##uram crates millennia pajamas worsened ##breakers jimi vanuatu yawned ##udeau carousel ##hony hurdle ##ccus ##mounted ##pod rv ##eche airship ambiguity compulsion recapture ##claiming arthritis ##osomal 1667 asserting ngc sniffing dade discontent glendale ported ##amina defamation rammed ##scent fling livingstone ##fleet 875 ##ppy apocalyptic comrade lcd ##lowe cessna eine persecuted subsistence demi hoop reliefs 710 coptic progressing stemmed perpetrators 1665 priestess ##nio dobson ebony rooster itf tortricidae ##bbon ##jian cleanup ##jean ##øy 1721 eighties taxonomic holiness ##hearted ##spar antilles showcasing stabilized ##nb gia mascara michelangelo dawned ##uria ##vinsky extinguished fitz grotesque £100 ##fera ##loid ##mous barges neue throbbed cipher johnnie ##a1 ##mpt outburst ##swick spearheaded administrations c1 heartbreak pixels pleasantly ##enay lombardy plush ##nsed bobbie ##hly reapers tremor xiang minogue substantive hitch barak ##wyl kwan ##encia 910 obscene elegance indus surfer bribery conserve ##hyllum ##masters horatio ##fat apes rebound psychotic ##pour iteration ##mium ##vani botanic horribly antiques dispose paxton ##hli ##wg timeless 1704 disregard engraver hounds ##bau ##version looted uno facilitates groans masjid rutland antibody disqualification decatur footballers quake slacks 48th rein scribe stabilize commits exemplary tho ##hort ##chison pantry traversed ##hiti disrepair identifiable vibrated baccalaureate ##nnis csa interviewing ##iensis ##raße greaves wealthiest 343 classed jogged £5 ##58 ##atal illuminating knicks respecting ##uno scrubbed ##iji ##dles kruger moods growls raider silvia chefs kam vr cree percival ##terol gunter counterattack defiant henan ze ##rasia ##riety equivalence submissions ##fra ##thor bautista mechanically ##heater cornice herbal templar ##mering outputs ruining ligand renumbered extravagant mika blockbuster eta insurrection ##ilia darkening ferocious pianos strife kinship ##aer melee ##anor ##iste ##may ##oue decidedly weep ##jad ##missive ##ppel 354 puget unease ##gnant 1629 hammering kassel ob wessex ##lga bromwich egan paranoia utilization ##atable ##idad contradictory provoke ##ols ##ouring ##tangled knesset ##very ##lette plumbing ##sden ##¹ greensboro occult sniff 338 zev beaming gamer haggard mahal ##olt ##pins mendes utmost briefing gunnery ##gut ##pher ##zh ##rok 1679 khalifa sonya ##boot principals urbana wiring ##liffe ##minating ##rrado dahl nyu skepticism np townspeople ithaca lobster somethin ##fur ##arina ##−1 freighter zimmerman biceps contractual ##herton amend hurrying subconscious ##anal 336 meng clermont spawning ##eia ##lub dignitaries impetus snacks spotting twigs ##bilis ##cz ##ouk libertadores nic skylar ##aina ##firm gustave asean ##anum dieter legislatures flirt bromley trolls umar ##bbies ##tyle blah parc bridgeport crank negligence ##nction 46th constantin molded bandages seriousness 00pm siegel carpets compartments upbeat statehood ##dner ##edging marko 730 platt ##hane paving ##iy 1738 abbess impatience limousine nbl ##talk 441 lucille mojo nightfall robbers ##nais karel brisk calves replicate ascribed telescopes ##olf intimidated ##reen ballast specialization ##sit aerodynamic caliphate rainer visionary ##arded epsilon ##aday ##onte aggregation auditory boosted reunification kathmandu loco robyn 402 acknowledges appointing humanoid newell redeveloped restraints ##tained barbarians chopper 1609 italiana ##lez ##lho investigates wrestlemania ##anies ##bib 690 ##falls creaked dragoons gravely minions stupidity volley ##harat ##week musik ##eries ##uously fungal massimo semantics malvern ##ahl ##pee discourage embryo imperialism 1910s profoundly ##ddled jiangsu sparkled stat ##holz sweatshirt tobin ##iction sneered ##cheon ##oit brit causal smyth ##neuve diffuse perrin silvio ##ipes ##recht detonated iqbal selma ##nism ##zumi roasted ##riders tay ##ados ##mament ##mut ##rud 840 completes nipples cfa flavour hirsch ##laus calderon sneakers moravian ##ksha 1622 rq 294 ##imeters bodo ##isance ##pre ##ronia anatomical excerpt ##lke dh kunst ##tablished ##scoe biomass panted unharmed gael housemates montpellier ##59 coa rodents tonic hickory singleton ##taro 451 1719 aldo breaststroke dempsey och rocco ##cuit merton dissemination midsummer serials ##idi haji polynomials ##rdon gs enoch prematurely shutter taunton £3 ##grating ##inates archangel harassed ##asco 326 archway dazzling ##ecin 1736 sumo wat ##kovich 1086 honneur ##ently ##nostic ##ttal ##idon 1605 403 1716 blogger rents ##gnan hires ##ikh ##dant howie ##rons handler retracted shocks 1632 arun duluth kepler trumpeter ##lary peeking seasoned trooper ##mara laszlo ##iciencies ##rti heterosexual ##inatory ##ssion indira jogging ##inga ##lism beit dissatisfaction malice ##ately nedra peeling ##rgeon 47th stadiums 475 vertigo ##ains iced restroom ##plify ##tub illustrating pear ##chner ##sibility inorganic rappers receipts watery ##kura lucinda ##oulos reintroduced ##8th ##tched gracefully saxons nutritional wastewater rained favourites bedrock fisted hallways likeness upscale ##lateral 1580 blinds prequel ##pps ##tama deter humiliating restraining tn vents 1659 laundering recess rosary tractors coulter federer ##ifiers ##plin persistence ##quitable geschichte pendulum quakers ##beam bassett pictorial buffet koln ##sitor drills reciprocal shooters ##57 ##cton ##tees converge pip dmitri donnelly yamamoto aqua azores demographics hypnotic spitfire suspend wryly roderick ##rran sebastien ##asurable mavericks ##fles ##200 himalayan prodigy ##iance transvaal demonstrators handcuffs dodged mcnamara sublime 1726 crazed ##efined ##till ivo pondered reconciled shrill sava ##duk bal cad heresy jaipur goran ##nished 341 lux shelly whitehall ##hre israelis peacekeeping ##wled 1703 demetrius ousted ##arians ##zos beale anwar backstroke raged shrinking cremated ##yck benign towing wadi darmstadt landfill parana soothe colleen sidewalks mayfair tumble hepatitis ferrer superstructure ##gingly ##urse ##wee anthropological translators ##mies closeness hooves ##pw mondays ##roll ##vita landscaping ##urized purification sock thorns thwarted jalan tiberius ##taka saline ##rito confidently khyber sculptors ##ij brahms hammersmith inspectors battista fivb fragmentation hackney ##uls arresting exercising antoinette bedfordshire ##zily dyed ##hema 1656 racetrack variability ##tique 1655 austrians deteriorating madman theorists aix lehman weathered 1731 decreed eruptions 1729 flaw quinlan sorbonne flutes nunez 1711 adored downwards fable rasped 1712 moritz mouthful renegade shivers stunts dysfunction restrain translit 327 pancakes ##avio ##cision ##tray 351 vial ##lden bain ##maid ##oxide chihuahua malacca vimes ##rba ##rnier 1664 donnie plaques ##ually 337 bangs floppy huntsville loretta nikolay ##otte eater handgun ubiquitous ##hett eras zodiac 1634 ##omorphic 1820s ##zog cochran ##bula ##lithic warring ##rada dalai excused blazers mcconnell reeling bot este ##abi geese hoax taxon ##bla guitarists ##icon condemning hunts inversion moffat taekwondo ##lvis 1624 stammered ##rest ##rzy sousa fundraiser marylebone navigable uptown cabbage daniela salman shitty whimper ##kian ##utive programmers protections rm ##rmi ##rued forceful ##enes fuss ##tao ##wash brat oppressive reykjavik spartak ticking ##inkles ##kiewicz adolph horst maui protege straighten cpc landau concourse clements resultant ##ando imaginative joo reactivated ##rem ##ffled ##uising consultative ##guide flop kaitlyn mergers parenting somber ##vron supervise vidhan ##imum courtship exemplified harmonies medallist refining ##rrow ##ка amara ##hum 780 goalscorer sited overshadowed rohan displeasure secretive multiplied osman ##orth engravings padre ##kali ##veda miniatures mis ##yala clap pali rook ##cana 1692 57th antennae astro oskar 1628 bulldog crotch hackett yucatan ##sure amplifiers brno ferrara migrating ##gree thanking turing ##eza mccann ting andersson onslaught gaines ganga incense standardization ##mation sentai scuba stuffing turquoise waivers alloys ##vitt regaining vaults ##clops ##gizing digger furry memorabilia probing ##iad payton rec deutschland filippo opaque seamen zenith afrikaans ##filtration disciplined inspirational ##merie banco confuse grafton tod ##dgets championed simi anomaly biplane ##ceptive electrode ##para 1697 cleavage crossbow swirl informant ##lars ##osta afi bonfire spec ##oux lakeside slump ##culus ##lais ##qvist ##rrigan 1016 facades borg inwardly cervical xl pointedly 050 stabilization ##odon chests 1699 hacked ctv orthogonal suzy ##lastic gaulle jacobite rearview ##cam ##erted ashby ##drik ##igate ##mise ##zbek affectionately canine disperse latham ##istles ##ivar spielberg ##orin ##idium ezekiel cid ##sg durga middletown ##cina customized frontiers harden ##etano ##zzy 1604 bolsheviks ##66 coloration yoko ##bedo briefs slabs debra liquidation plumage ##oin blossoms dementia subsidy 1611 proctor relational jerseys parochial ter ##ici esa peshawar cavalier loren cpi idiots shamrock 1646 dutton malabar mustache ##endez ##ocytes referencing terminates marche yarmouth ##sop acton mated seton subtly baptised beige extremes jolted kristina telecast ##actic safeguard waldo ##baldi ##bular endeavors sloppy subterranean ##ensburg ##itung delicately pigment tq ##scu 1626 ##ound collisions coveted herds ##personal ##meister ##nberger chopra ##ricting abnormalities defective galician lucie ##dilly alligator likened ##genase burundi clears complexion derelict deafening diablo fingered champaign dogg enlist isotope labeling mrna ##erre brilliance marvelous ##ayo 1652 crawley ether footed dwellers deserts hamish rubs warlock skimmed ##lizer 870 buick embark heraldic irregularities ##ajan kiara ##kulam ##ieg antigen kowalski ##lge oakley visitation ##mbit vt ##suit 1570 murderers ##miento ##rites chimneys ##sling condemn custer exchequer havre ##ghi fluctuations ##rations dfb hendricks vaccines ##tarian nietzsche biking juicy ##duced brooding scrolling selangor ##ragan 352 annum boomed seminole sugarcane ##dna departmental dismissing innsbruck arteries ashok batavia daze kun overtook ##rga ##tlan beheaded gaddafi holm electronically faulty galilee fractures kobayashi ##lized gunmen magma aramaic mala eastenders inference messengers bf ##qu 407 bathrooms ##vere 1658 flashbacks ideally misunderstood ##jali ##weather mendez ##grounds 505 uncanny ##iii 1709 friendships ##nbc sacrament accommodated reiterated logistical pebbles thumped ##escence administering decrees drafts ##flight ##cased ##tula futuristic picket intimidation winthrop ##fahan interfered 339 afar francoise morally uta cochin croft dwarfs ##bruck ##dents ##nami biker ##hner ##meral nano ##isen ##ometric ##pres ##ан brightened meek parcels securely gunners ##jhl ##zko agile hysteria ##lten ##rcus bukit champs chevy cuckoo leith sadler theologians welded ##section 1663 jj plurality xander ##rooms ##formed shredded temps intimately pau tormented ##lok ##stellar 1618 charred ems essen ##mmel alarms spraying ascot blooms twinkle ##abia ##apes internment obsidian ##chaft snoop ##dav ##ooping malibu ##tension quiver ##itia hays mcintosh travers walsall ##ffie 1623 beverley schwarz plunging structurally m3 rosenthal vikram ##tsk 770 ghz ##onda ##tiv chalmers groningen pew reckon unicef ##rvis 55th ##gni 1651 sulawesi avila cai metaphysical screwing turbulence ##mberg augusto samba 56th baffled momentary toxin ##urian ##wani aachen condoms dali steppe ##3d ##app ##oed ##year adolescence dauphin electrically inaccessible microscopy nikita ##ega atv ##cel ##enter ##oles ##oteric ##ы accountants punishments wrongly bribes adventurous clinch flinders southland ##hem ##kata gough ##ciency lads soared ##ה undergoes deformation outlawed rubbish ##arus ##mussen ##nidae ##rzburg arcs ##ingdon ##tituted 1695 wheelbase wheeling bombardier campground zebra ##lices ##oj ##bain lullaby ##ecure donetsk wylie grenada ##arding ##ης squinting eireann opposes ##andra maximal runes ##broken ##cuting ##iface ##ror ##rosis additive britney adultery triggering ##drome detrimental aarhus containment jc swapped vichy ##ioms madly ##oric ##rag brant ##ckey ##trix 1560 1612 broughton rustling ##stems ##uder asbestos mentoring ##nivorous finley leaps ##isan apical pry slits substitutes ##dict intuitive fantasia insistent unreasonable ##igen ##vna domed hannover margot ponder ##zziness impromptu jian lc rampage stemming ##eft andrey gerais whichever amnesia appropriated anzac clicks modifying ultimatum cambrian maids verve yellowstone ##mbs conservatoire ##scribe adherence dinners spectra imperfect mysteriously sidekick tatar tuba ##aks ##ifolia distrust ##athan ##zle c2 ronin zac ##pse celaena instrumentalist scents skopje ##mbling comical compensated vidal condor intersect jingle wavelengths ##urrent mcqueen ##izzly carp weasel 422 kanye militias postdoctoral eugen gunslinger ##ɛ faux hospice ##for appalled derivation dwarves ##elis dilapidated ##folk astoria philology ##lwyn ##otho ##saka inducing philanthropy ##bf ##itative geek markedly sql ##yce bessie indices rn ##flict 495 frowns resolving weightlifting tugs cleric contentious 1653 mania rms ##miya ##reate ##ruck ##tucket bien eels marek ##ayton ##cence discreet unofficially ##ife leaks ##bber 1705 332 dung compressor hillsborough pandit shillings distal ##skin 381 ##tat ##you nosed ##nir mangrove undeveloped ##idia textures ##inho ##500 ##rise ae irritating nay amazingly bancroft apologetic compassionate kata symphonies ##lovic airspace ##lch 930 gifford precautions fulfillment sevilla vulgar martinique ##urities looting piccolo tidy ##dermott quadrant armchair incomes mathematicians stampede nilsson ##inking ##scan foo quarterfinal ##ostal shang shouldered squirrels ##owe 344 vinegar ##bner ##rchy ##systems delaying ##trics ars dwyer rhapsody sponsoring ##gration bipolar cinder starters ##olio ##urst 421 signage ##nty aground figurative mons acquaintances duets erroneously soyuz elliptic recreated ##cultural ##quette ##ssed ##tma ##zcz moderator scares ##itaire ##stones ##udence juniper sighting ##just ##nsen britten calabria ry bop cramer forsyth stillness ##л airmen gathers unfit ##umber ##upt taunting ##rip seeker streamlined ##bution holster schumann tread vox ##gano ##onzo strive dil reforming covent newbury predicting ##orro decorate tre ##puted andover ie asahi dept dunkirk gills ##tori buren huskies ##stis ##stov abstracts bets loosen ##opa 1682 yearning ##glio ##sir berman effortlessly enamel napoli persist ##peration ##uez attache elisa b1 invitations ##kic accelerating reindeer boardwalk clutches nelly polka starbucks ##kei adamant huey lough unbroken adventurer embroidery inspecting stanza ##ducted naia taluka ##pone ##roids chases deprivation florian ##jing ##ppet earthly ##lib ##ssee colossal foreigner vet freaks patrice rosewood triassic upstate ##pkins dominates ata chants ks vo ##400 ##bley ##raya ##rmed 555 agra infiltrate ##ailing ##ilation ##tzer ##uppe ##werk binoculars enthusiast fujian squeak ##avs abolitionist almeida boredom hampstead marsden rations ##ands inflated 334 bonuses rosalie patna ##rco 329 detachments penitentiary 54th flourishing woolf ##dion ##etched papyrus ##lster ##nsor ##toy bobbed dismounted endelle inhuman motorola tbs wince wreath ##ticus hideout inspections sanjay disgrace infused pudding stalks ##urbed arsenic leases ##hyl ##rrard collarbone ##waite ##wil dowry ##bant ##edance genealogical nitrate salamanca scandals thyroid necessitated ##! ##" ### ##$ ##% ##& ##' ##( ##) ##* ##+ ##, ##- ##. ##/ ##: ##; ##< ##= ##> ##? ##@ ##[ ##\ ##] ##^ ##_ ##` ##{ ##| ##} ##~ ##¡ ##¢ ##£ ##¤ ##¥ ##¦ ##§ ##¨ ##© ##ª ##« ##¬ ##® ##± ##´ ##µ ##¶ ##· ##º ##» ##¼ ##¾ ##¿ ##æ ##ð ##÷ ##þ ##đ ##ħ ##ŋ ##œ ##ƒ ##ɐ ##ɑ ##ɒ ##ɔ ##ɕ ##ə ##ɡ ##ɣ ##ɨ ##ɪ ##ɫ ##ɬ ##ɯ ##ɲ ##ɴ ##ɹ ##ɾ ##ʀ ##ʁ ##ʂ ##ʃ ##ʉ ##ʊ ##ʋ ##ʌ ##ʎ ##ʐ ##ʑ ##ʒ ##ʔ ##ʰ ##ʲ ##ʳ ##ʷ ##ʸ ##ʻ ##ʼ ##ʾ ##ʿ ##ˈ ##ˡ ##ˢ ##ˣ ##ˤ ##β ##γ ##δ ##ε ##ζ ##θ ##κ ##λ ##μ ##ξ ##ο ##π ##ρ ##σ ##τ ##υ ##φ ##χ ##ψ ##ω ##б ##г ##д ##ж ##з ##м ##п ##с ##у ##ф ##х ##ц ##ч ##ш ##щ ##ъ ##э ##ю ##ђ ##є ##і ##ј ##љ ##њ ##ћ ##ӏ ##ա ##բ ##գ ##դ ##ե ##թ ##ի ##լ ##կ ##հ ##մ ##յ ##ն ##ո ##պ ##ս ##վ ##տ ##ր ##ւ ##ք ##־ ##א ##ב ##ג ##ד ##ו ##ז ##ח ##ט ##י ##ך ##כ ##ל ##ם ##מ ##ן ##נ ##ס ##ע ##ף ##פ ##ץ ##צ ##ק ##ר ##ש ##ת ##، ##ء ##ب ##ت ##ث ##ج ##ح ##خ ##ذ ##ز ##س ##ش ##ص ##ض ##ط ##ظ ##ع ##غ ##ـ ##ف ##ق ##ك ##و ##ى ##ٹ ##پ ##چ ##ک ##گ ##ں ##ھ ##ہ ##ے ##अ ##आ ##उ ##ए ##क ##ख ##ग ##च ##ज ##ट ##ड ##ण ##त ##थ ##द ##ध ##न ##प ##ब ##भ ##म ##य ##र ##ल ##व ##श ##ष ##स ##ह ##ा ##ि ##ी ##ो ##। ##॥ ##ং ##অ ##আ ##ই ##উ ##এ ##ও ##ক ##খ ##গ ##চ ##ছ ##জ ##ট ##ড ##ণ ##ত ##থ ##দ ##ধ ##ন ##প ##ব ##ভ ##ম ##য ##র ##ল ##শ ##ষ ##স ##হ ##া ##ি ##ী ##ে ##க ##ச ##ட ##த ##ந ##ன ##ப ##ம ##ய ##ர ##ல ##ள ##வ ##ா ##ி ##ு ##ே ##ை ##ನ ##ರ ##ಾ ##ක ##ය ##ර ##ල ##ව ##ා ##ก ##ง ##ต ##ท ##น ##พ ##ม ##ย ##ร ##ล ##ว ##ส ##อ ##า ##เ ##་ ##། ##ག ##ང ##ད ##ན ##པ ##བ ##མ ##འ ##ར ##ལ ##ས ##မ ##ა ##ბ ##გ ##დ ##ე ##ვ ##თ ##ი ##კ ##ლ ##მ ##ნ ##ო ##რ ##ს ##ტ ##უ ##ᄀ ##ᄂ ##ᄃ ##ᄅ ##ᄆ ##ᄇ ##ᄉ ##ᄊ ##ᄋ ##ᄌ ##ᄎ ##ᄏ ##ᄐ ##ᄑ ##ᄒ ##ᅡ ##ᅢ ##ᅥ ##ᅦ ##ᅧ ##ᅩ ##ᅪ ##ᅭ ##ᅮ ##ᅯ ##ᅲ ##ᅳ ##ᅴ ##ᅵ ##ᆨ ##ᆫ ##ᆯ ##ᆷ ##ᆸ ##ᆼ ##ᴬ ##ᴮ ##ᴰ ##ᴵ ##ᴺ ##ᵀ ##ᵃ ##ᵇ ##ᵈ ##ᵉ ##ᵍ ##ᵏ ##ᵐ ##ᵒ ##ᵖ ##ᵗ ##ᵘ ##ᵣ ##ᵤ ##ᵥ ##ᶜ ##ᶠ ##‐ ##‑ ##‒ ##– ##— ##― ##‖ ##‘ ##’ ##‚ ##“ ##” ##„ ##† ##‡ ##• ##… ##‰ ##′ ##″ ##› ##‿ ##⁄ ##⁰ ##ⁱ ##⁴ ##⁵ ##⁶ ##⁷ ##⁸ ##⁹ ##⁻ ##ⁿ ##₅ ##₆ ##₇ ##₈ ##₉ ##₊ ##₍ ##₎ ##ₐ ##ₑ ##ₒ ##ₓ ##ₕ ##ₖ ##ₗ ##ₘ ##ₚ ##ₛ ##ₜ ##₤ ##₩ ##€ ##₱ ##₹ ##ℓ ##№ ##ℝ ##™ ##⅓ ##⅔ ##← ##↑ ##→ ##↓ ##↔ ##↦ ##⇄ ##⇌ ##⇒ ##∂ ##∅ ##∆ ##∇ ##∈ ##∗ ##∘ ##√ ##∞ ##∧ ##∨ ##∩ ##∪ ##≈ ##≡ ##≤ ##≥ ##⊂ ##⊆ ##⊕ ##⊗ ##⋅ ##─ ##│ ##■ ##▪ ##● ##★ ##☆ ##☉ ##♠ ##♣ ##♥ ##♦ ##♯ ##⟨ ##⟩ ##ⱼ ##⺩ ##⺼ ##⽥ ##、 ##。 ##〈 ##〉 ##《 ##》 ##「 ##」 ##『 ##』 ##〜 ##あ ##い ##う ##え ##お ##か ##き ##く ##け ##こ ##さ ##し ##す ##せ ##そ ##た ##ち ##っ ##つ ##て ##と ##な ##に ##ぬ ##ね ##の ##は ##ひ ##ふ ##へ ##ほ ##ま ##み ##む ##め ##も ##や ##ゆ ##よ ##ら ##り ##る ##れ ##ろ ##を ##ん ##ァ ##ア ##ィ ##イ ##ウ ##ェ ##エ ##オ ##カ ##キ ##ク ##ケ ##コ ##サ ##シ ##ス ##セ ##タ ##チ ##ッ ##ツ ##テ ##ト ##ナ ##ニ ##ノ ##ハ ##ヒ ##フ ##ヘ ##ホ ##マ ##ミ ##ム ##メ ##モ ##ャ ##ュ ##ョ ##ラ ##リ ##ル ##レ ##ロ ##ワ ##ン ##・ ##ー ##一 ##三 ##上 ##下 ##不 ##世 ##中 ##主 ##久 ##之 ##也 ##事 ##二 ##五 ##井 ##京 ##人 ##亻 ##仁 ##介 ##代 ##仮 ##伊 ##会 ##佐 ##侍 ##保 ##信 ##健 ##元 ##光 ##八 ##公 ##内 ##出 ##分 ##前 ##劉 ##力 ##加 ##勝 ##北 ##区 ##十 ##千 ##南 ##博 ##原 ##口 ##古 ##史 ##司 ##合 ##吉 ##同 ##名 ##和 ##囗 ##四 ##国 ##國 ##土 ##地 ##坂 ##城 ##堂 ##場 ##士 ##夏 ##外 ##大 ##天 ##太 ##夫 ##奈 ##女 ##子 ##学 ##宀 ##宇 ##安 ##宗 ##定 ##宣 ##宮 ##家 ##宿 ##寺 ##將 ##小 ##尚 ##山 ##岡 ##島 ##崎 ##川 ##州 ##巿 ##帝 ##平 ##年 ##幸 ##广 ##弘 ##張 ##彳 ##後 ##御 ##德 ##心 ##忄 ##志 ##忠 ##愛 ##成 ##我 ##戦 ##戸 ##手 ##扌 ##政 ##文 ##新 ##方 ##日 ##明 ##星 ##春 ##昭 ##智 ##曲 ##書 ##月 ##有 ##朝 ##木 ##本 ##李 ##村 ##東 ##松 ##林 ##森 ##楊 ##樹 ##橋 ##歌 ##止 ##正 ##武 ##比 ##氏 ##民 ##水 ##氵 ##氷 ##永 ##江 ##沢 ##河 ##治 ##法 ##海 ##清 ##漢 ##瀬 ##火 ##版 ##犬 ##王 ##生 ##田 ##男 ##疒 ##発 ##白 ##的 ##皇 ##目 ##相 ##省 ##真 ##石 ##示 ##社 ##神 ##福 ##禾 ##秀 ##秋 ##空 ##立 ##章 ##竹 ##糹 ##美 ##義 ##耳 ##良 ##艹 ##花 ##英 ##華 ##葉 ##藤 ##行 ##街 ##西 ##見 ##訁 ##語 ##谷 ##貝 ##貴 ##車 ##軍 ##辶 ##道 ##郎 ##郡 ##部 ##都 ##里 ##野 ##金 ##鈴 ##镇 ##長 ##門 ##間 ##阝 ##阿 ##陳 ##陽 ##雄 ##青 ##面 ##風 ##食 ##香 ##馬 ##高 ##龍 ##龸 ##fi ##fl ##！ ##（ ##） ##， ##－ ##． ##／ ##： ##？ ##～ ================================================ FILE: data_utils/uppercase_vocab.txt ================================================ [PAD] [unused1] [unused2] [unused3] [unused4] [unused5] [unused6] [unused7] [unused8] [unused9] [unused10] [unused11] [unused12] [unused13] [unused14] [unused15] [unused16] [unused17] [unused18] [unused19] [unused20] [unused21] [unused22] [unused23] [unused24] [unused25] [unused26] [unused27] [unused28] [unused29] [unused30] [unused31] [unused32] [unused33] [unused34] [unused35] [unused36] [unused37] [unused38] [unused39] [unused40] [unused41] [unused42] [unused43] [unused44] [unused45] [unused46] [unused47] [unused48] [unused49] [unused50] [unused51] [unused52] [unused53] [unused54] [unused55] [unused56] [unused57] [unused58] [unused59] [unused60] [unused61] [unused62] [unused63] [unused64] [unused65] [unused66] [unused67] [unused68] [unused69] [unused70] [unused71] [unused72] [unused73] [unused74] [unused75] [unused76] [unused77] [unused78] [unused79] [unused80] [unused81] [unused82] [unused83] [unused84] [unused85] [unused86] [unused87] [unused88] [unused89] [unused90] [unused91] [unused92] [unused93] [unused94] [unused95] [unused96] [unused97] [unused98] [unused99] [UNK] [CLS] [SEP] [MASK] [unused100] [unused101] ! " # $ % & ' ( ) * + , - . / 0 1 2 3 4 5 6 7 8 9 : ; < = > ? @ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z [ \ ] ^ _ ` a b c d e f g h i j k l m n o p q r s t u v w x y z { | } ~ ¡ ¢ £ ¥ § ¨ © ª « ¬ ® ° ± ² ³ ´ µ ¶ · ¹ º » ¼ ½ ¾ ¿ À Á Â Ä Å Æ Ç È É Í Î Ñ Ó Ö × Ø Ú Ü Þ ß à á â ã ä å æ ç è é ê ë ì í î ï ð ñ ò ó ô õ ö ÷ ø ù ú û ü ý þ ÿ Ā ā ă ą Ć ć Č č ď Đ đ ē ė ę ě ğ ġ Ħ ħ ĩ Ī ī İ ı ļ Ľ ľ Ł ł ń ņ ň ŋ Ō ō ŏ ő Œ œ ř Ś ś Ş ş Š š Ţ ţ ť ũ ū ŭ ů ű ų ŵ ŷ ź Ż ż Ž ž Ə ƒ ơ ư ǎ ǐ ǒ ǔ ǫ Ș ș Ț ț ɐ ɑ ɔ ɕ ə ɛ ɡ ɣ ɨ ɪ ɲ ɾ ʀ ʁ ʂ ʃ ʊ ʋ ʌ ʐ ʑ ʒ ʔ ʰ ʲ ʳ ʷ ʻ ʼ ʾ ʿ ˈ ː ˡ ˢ ˣ ́ ̃ ̍ ̯ ͡ Α Β Γ Δ Ε Η Θ Ι Κ Λ Μ Ν Ο Π Σ Τ Φ Χ Ψ Ω ά έ ή ί α β γ δ ε ζ η θ ι κ λ μ ν ξ ο π ρ ς σ τ υ φ χ ψ ω ό ύ ώ І Ј А Б В Г Д Е Ж З И К Л М Н О П Р С Т У Ф Х Ц Ч Ш Э Ю Я а б в г д е ж з и й к л м н о п р с т у ф х ц ч ш щ ъ ы ь э ю я ё і ї ј њ ћ Ա Հ ա ե ի կ մ յ ն ո ս տ ր ւ ְ ִ ֵ ֶ ַ ָ ֹ ּ א ב ג ד ה ו ז ח ט י כ ל ם מ ן נ ס ע פ צ ק ר ש ת ، ء آ أ إ ئ ا ب ة ت ث ج ح خ د ذ ر ز س ش ص ض ط ظ ع غ ف ق ك ل م ن ه و ى ي َ ِ ٹ پ چ ک گ ہ ی ے ं आ क ग च ज ण त द ध न प ब भ म य र ल व श ष स ह ा ि ी ु े ो ् । ॥ আ ই এ ও ক খ গ চ ছ জ ট ত থ দ ধ ন প ব ম য র ল শ স হ ় া ি ী ু ে ো ্ য় க த ப ம ய ர ல வ ா ி ு ் ร ་ ག ང ད ན བ མ ར ལ ས ི ུ ེ ོ ა ე ი ლ ნ ო რ ს ᴬ ᴵ ᵀ ᵃ ᵇ ᵈ ᵉ ᵍ ᵏ ᵐ ᵒ ᵖ ᵗ ᵘ ᵢ ᵣ ᵤ ᵥ ᶜ ᶠ ḍ Ḥ ḥ Ḩ ḩ ḳ ṃ ṅ ṇ ṛ ṣ ṭ ạ ả ấ ầ ẩ ậ ắ ế ề ể ễ ệ ị ọ ố ồ ổ ộ ớ ờ ợ ụ ủ ứ ừ ử ữ ự ỳ ỹ ἀ ἐ ὁ ὐ ὰ ὶ ὸ ῆ ῖ ῦ ῶ ‐ ‑ ‒ – — ― ‖ ‘ ’ ‚ “ ” „ † ‡ • … ‰ ′ ″ ⁄ ⁰ ⁱ ⁴ ⁵ ⁶ ⁷ ⁸ ⁹ ⁺ ⁻ ⁿ ₀ ₁ ₂ ₃ ₄ ₅ ₆ ₇ ₈ ₉ ₊ ₍ ₎ ₐ ₑ ₒ ₓ ₕ ₖ ₘ ₙ ₚ ₛ ₜ ₤ € ₱ ₹ ℓ № ℝ ⅓ ← ↑ → ↔ ⇌ ⇒ ∂ ∈ − ∗ ∘ √ ∞ ∧ ∨ ∩ ∪ ≈ ≠ ≡ ≤ ≥ ⊂ ⊆ ⊕ ⋅ ─ │ ■ ● ★ ☆ ☉ ♠ ♣ ♥ ♦ ♭ ♯ ⟨ ⟩ ⱼ 、 。 《 》 「 」 『 』 〜 い う え お か き く け こ さ し す せ そ た ち つ て と な に の は ひ ま み む め も や ゆ よ ら り る れ ん ア ィ イ ウ エ オ カ ガ キ ク グ コ サ シ ジ ス ズ タ ダ ッ テ デ ト ド ナ ニ ハ バ パ フ ブ プ マ ミ ム ャ ュ ラ リ ル レ ロ ン ・ ー 一 三 上 下 中 事 二 井 京 人 亻 仁 佐 侍 光 公 力 北 十 南 原 口 史 司 吉 同 和 囗 国 國 土 城 士 大 天 太 夫 女 子 宀 安 宮 宿 小 尚 山 島 川 州 平 年 心 愛 戸 文 新 方 日 明 星 書 月 木 本 李 村 東 松 林 正 武 氏 水 氵 江 河 海 版 犬 王 生 田 白 皇 省 真 石 社 神 竹 美 義 花 藤 西 谷 車 辶 道 郎 郡 部 野 金 長 門 陽 青 食 馬 高 龍 龸 사 씨 의 이 한 fi fl ！ （ ） ， － ／ ： the of and to in was The is for as on with that ##s his by he at from it her He had an were you be In she are but which It not or have my him one this me has also up their first out who been they She into all would its ##ing time two ##a ##e said about when over more other can after back them then ##ed there like so only ##n could ##d ##i ##y what no ##o where This made than if You ##ly through we before ##r just some ##er years do New ##t down between new now will three most On around year used such being well during They know against under later did part known off while His re ... ##l people until way American didn University your both many get United became head There second As work any But still again born even eyes After including de took And long team season family see right same called name because film don 10 found much school ##es going won place away We day left John 000 hand since World these how make number each life area man four go No here very National ##m played released never began States album home last too held several May own ##on take end School ##h ll series What want use another city When 2010 side At may That came face June think game those high March early September ##al 2011 looked July state small thought went January October ##u based August ##us world good April York us 12 2012 2008 For 2009 group along few South little ##k following November something 2013 December set 2007 old 2006 2014 located ##an music County City former ##in room ve next All ##man got father house ##g body 15 20 18 started If 2015 town our line War large population named British company member five My single ##en age State moved February 11 Her should century government built come best show However within look men door without need wasn 2016 water One system knew every died League turned asked North St wanted building received song served though felt ##ia station band ##ers local public himself different death say ##1 30 ##2 2005 16 night behind children English members near saw together son 14 voice village 13 hands help ##3 due French London top told open published third 2017 play across During put final often include 25 ##le main having 2004 once ever let book led gave late front find club ##4 German included species College form opened mother women enough West must 2000 power really 17 making half ##6 order might ##is given million times days point full service With km major ##7 original become seen II north six ##te love ##0 national International ##5 24 So District lost run couldn career always ##9 2003 ##th country ##z House air tell south worked woman player ##A almost war River ##ic married continued Then James close black short ##8 ##na using history returned light car ##ra sure William things General ##ry 2002 better support 100 among From feet King anything 21 19 established district 2001 feel great ##ton level Cup These written games others already title story ##p law thing US record role however By students England white control least inside land ##C 22 give community hard ##ie non ##c produced George round period Park business various ##ne does present wife far taken per reached David able version working young live created joined East living appeared case High done 23 important President Award France position office looking total general class To production ##S football party brother keep mind free Street hair announced development either nothing moment Church followed wrote why India San election 1999 lead How ##ch ##rs words European course considered America arms Army political ##la 28 26 west east ground further church less site First Not Australia toward California ##ness described works An Council heart past military 27 ##or heard field human soon founded 1998 playing trying ##x ##ist ##ta television mouth although taking win fire Division ##ity Party Royal program Some Don Association According tried TV Paul outside daughter Best While someone match recorded Canada closed region Air above months elected ##da ##ian road ##ar brought move 1997 leave ##um Thomas 1996 am low Robert formed person services points Mr miles ##b stop rest doing needed international release floor start sound call killed real dark research finished language Michael professional change sent 50 upon 29 track hit event 2018 term example Germany similar return ##ism fact pulled stood says ran information yet result developed girl ##re God 1995 areas signed decided ##ment Company seemed ##el co turn race common video Charles Indian ##ation blood art red ##able added rather 1994 met director addition design average minutes ##ies ##ted available bed coming friend idea kind Union Road remained ##ting everything ##ma running care finally Chinese appointed 1992 Australian ##ley popular mean teams probably ##land usually project social Championship possible word Russian instead mi herself ##T Peter Hall Center seat style money 1993 else Department table Music current 31 features special events character Two square sold debut ##v process Although Since ##ka 40 Central currently education placed lot China quickly forward seven ##ling Europe arm performed Japanese 1991 Henry Now Dr ##ion week Group myself big UK Washington ten deep 1990 Club Japan space La directed smile episode hours whole ##de ##less Why wouldn designed strong training changed Society stage involved hadn towards leading police eight kept Institute study largest child eventually private modern Court throughout getting originally attack ##E talk Great longer songs alone ##ine wide dead walked shot ##ri Oh force ##st Art today friends Island Richard 1989 center construction believe size White ship completed ##B gone Just rock sat ##R radio below entire families league includes type lived official range hold featured Most ##ter president passed means ##f forces lips Mary Do guitar ##ce food wall Of spent Its performance hear ##P Western reported sister ##et morning ##M especially ##ive Minister itself post bit groups 1988 ##tion Black ##ng Well raised sometimes Canadian Paris Spanish replaced schools Academy leaving central female Christian Jack whose college onto provided ##D ##ville players actually stopped ##son Museum doesn ##ts books fight allowed ##ur beginning Records awarded parents coach ##os Red saying ##ck Smith Yes Lake ##L aircraft 1987 ##ble previous ft action Italian African happened vocals Act future court ##ge 1986 degree phone ##ro Is countries winning breath Love river matter Lord Other list self parts ##ate provide cut shows plan 1st interest ##ized Africa stated Sir fell owned earlier ended competition attention 1985 lower nearly bad older stay Saint ##se certain 1984 fingers blue try fourth Grand ##as king ##nt makes chest movement states moving data introduced model date section Los deal ##I skin entered middle success Texas ##w summer island ##N Republic length husband 1980 ##ey reason anyone forced via base 500 job covered Festival Roman successful rights cover Man writing Ireland ##F related goal takes buildings true weeks 1983 Because opening novel ISBN meet gold ##ous mid km² standing Football Chicago shook whom ##ki 1982 Day feeling scored boy higher Force leader heavy fall question sense army Second energy meeting themselves kill ##am board census ##ya ##ns mine meant market required battle campaign attended approximately Kingdom runs active ##ha contract clear previously health 1979 Arts complete Catholic couple units ##ll ##ty Committee shoulder sea systems listed ##O caught tournament ##G northern author Film Your ##men holding offered personal 1981 southern artist traditional studio 200 capital ##ful regular ask giving organization month news Are read managed helped studied student defeated natural industry Year noted decision Government quite ##id smiled 1972 Maybe tracks ##ke Mark al media engine hour Their relationship plays property structure 1976 ago Hill Martin 1978 ready Many Like Bay immediately generally Italy Greek practice caused division significant Joseph speed Let thinking completely 1974 primary mostly ##field ##K 1975 ##to Even writer ##led dropped magazine collection understand route highest particular films lines network Science loss carried direction green 1977 location producer according Women Queen neck thus independent view 1970 Angeles Soviet distance problem Board tour western income appearance access Mexico nodded street surface arrived believed Old 1968 1973 becoming whether 1945 figure singer stand Following issue window wrong pain everyone lives issues park slowly la act ##va bring Lee operations key comes fine cold famous Navy 1971 Me additional individual ##ner Zealand goals county contains Service minute 2nd reach talking particularly ##ham movie Director glass paper studies ##co railway standard Education 45 represented Chief Louis launched Star terms 60 1969 experience watched Another Press Tom staff starting subject break Virginia nine eye ##age evidence foot ##est companies Prince ##V gun create Big People guy Green simply numerous ##line increased twenty ##ga ##do 1967 award officer stone Before material Northern grew male plant Life legs step Al unit 35 except answer ##U report response Edward commercial edition trade science ##ca Irish Law shown rate failed ##ni remains changes mm limited larger Later cause waiting Time ##wood cost Bill manager activities likely allow operated retired ##ping 65 directly Who associated effect hell Florida straight hot Valley management girls expected eastern Mike chance cast centre chair hurt problems ##li walk programs Team characters Battle edge pay maybe corner majority medical Joe Summer ##io attempt Pacific command Radio ##by names municipality 1964 train economic Brown feature sex source agreed remember Three 1966 1965 Pennsylvania victory senior annual III Southern results Sam serving religious Jones appears ##der despite claimed Both musical matches fast security selected Young double complex hospital chief Times ##ve Championships filled Public Despite beautiful Research plans Province ##ally Wales ##ko artists metal nearby Spain ##il 32 houses supported piece ##no stared recording nature legal Russia ##ization remaining looks ##sh bridge closer cases scene marriage Little ##é uses Earth specific Frank theory Good discovered referred bass culture university presented Congress ##go metres continue 1960 isn Awards meaning cell composed separate Series forms Blue cross ##tor increase test computer slightly Where Jewish Town tree status 1944 variety responsible pretty initially ##way realized pass provides Captain Alexander recent score broke Scott drive financial showed Line stories ordered soldiers genus operation gaze sitting society Only hope actor follow Empire Yeah technology happy focus policy spread situation ##ford ##ba Mrs watch Can 1963 Commission touch earned troops Under 1962 individuals cannot 19th ##lin mile expression exactly suddenly weight dance stepped places appear difficult Railway anti numbers kilometres star ##ier department ice Britain removed Once ##lo Boston value ##ant mission trees Order sports join serve Major poor Poland mainly Theatre pushed Station ##it Lady federal silver ##ler foreign ##ard Eastern ##den box hall subsequently lies acquired 1942 ancient CD History Jean beyond ##ger El ##les growing championship native Parliament Williams watching direct overall offer Also 80 Secretary spoke Latin ability ##ated safe presence ##ial headed regional planned 1961 Johnson throat consists ##W extended Or bar walls Chris stations politician Olympics influence share fighting speak hundred Carolina die stars ##tic color Chapter ##ish fear sleep goes Francisco oil Bank sign physical ##berg Dutch seasons ##rd Games Governor sorry lack Centre memory baby smaller charge Did multiple ships shirt Assembly amount leaves 3rd Foundation conditions 1943 Rock Democratic Daniel ##at winner products ##ina store latter Professor civil prior host 1956 soft vote needs Each rules 1958 pressure letter normal proposed levels records 1959 paid intended Victoria purpose okay historical issued 1980s broadcast rule simple picked firm Sea 1941 Elizabeth 1940 serious featuring highly graduated mentioned choice 1948 replied percent Scotland ##hi females constructed 1957 settled Steve recognized cities crew glanced kiss competed flight knowledge editor More Conference ##H fifth elements ##ee ##tes function newspaper recently Miss cultural brown twice Office 1939 truth Creek 1946 households USA 1950 quality ##tt border seconds destroyed pre wait ahead build image 90 cars ##mi 33 promoted professor et bank medal text broken Middle revealed sides wing seems channel 1970s Ben loved effort officers Will ##ff 70 Israel Jim upper fully label Jr assistant powerful pair positive ##ary gives 1955 20th races remain kitchen primarily ##ti Sydney easy Tour whispered buried 300 News Polish 1952 Duke Columbia produce accepted 00 approach minor 1947 Special 44 Asian basis visit Fort Civil finish formerly beside leaned ##ite median rose coast effects supposed Cross ##hip Corps residents Jackson ##ir Bob basketball 36 Asia seem Bishop Book ##ber ring ##ze owner BBC ##ja transferred acting De appearances walking Le press grabbed 1954 officially 1953 ##pe risk taught review ##X lay ##well council Avenue seeing losing Ohio Super province ones travel ##sa projects equipment spot Berlin administrative heat potential shut capacity elections growth fought Republican mixed Andrew teacher turning strength shoulders beat wind 1949 Health follows camp suggested perhaps Alex mountain contact divided candidate fellow 34 Show necessary workers ball horse ways questions protect gas activity younger bottom founder Scottish screen treatment easily com ##house dedicated Master warm Night Georgia Long von ##me perfect website 1960s piano efforts ##ide Tony sort offers Development Simon executive ##nd save Over Senate 1951 1990s draw master Police ##ius renamed boys initial prominent damage Co ##ov ##za online begin occurred captured youth Top account tells Justice conducted forest ##town bought teeth Jersey ##di purchased agreement Michigan ##ure campus prison becomes product secret guess Route huge types drums 64 split defeat estate housing ##ot brothers Coast declared happen titled therefore sun commonly alongside Stadium library Home article steps telling slow assigned refused laughed wants Nick wearing Rome Open ##ah Hospital pointed Taylor lifted escape participated ##j drama parish Santa ##per organized mass pick Airport gets Library unable pull Live ##ging surrounding ##ries focused Adam facilities ##ning ##ny 38 ##ring notable era connected gained operating laid Regiment branch defined Christmas machine Four academic Iran adopted concept Men compared search traffic Max Maria greater ##ding widely ##burg serves 1938 37 Go hotel shared typically scale 1936 leg suffered yards pieces Ministry Wilson episodes empty 1918 safety continues yellow historic settlement 400 Come Corporation enemy content picture evening territory method trial solo driver Here ##ls entrance Prize spring whatever ##ent 75 ##ji reading Arthur ##cy Our clothes Prime Illinois Kong code ##ria sit Harry Federal chosen administration bodies begins stomach Though seats Hong density Sun leaders Field museum chart platform languages ##ron birth holds Gold ##un fish combined ##ps 4th 1937 largely captain trust Game van boat Oxford basic beneath Islands painting nice Toronto path males sources block conference parties murder clubs crowd calling About Business peace knows lake speaking stayed Brazil allowing Born unique thick Technology ##que receive des semi alive noticed format ##ped coffee digital ##ned handed guard tall faced setting plants partner claim reduced temple animals determined classes ##out estimated ##ad Olympic providing Massachusetts learned Inc Philadelphia Social carry 42 possibly hosted tonight respectively Today shape Mount roles designated brain etc Korea thoughts Brian Highway doors background drew models footballer tone turns 1935 quiet tower wood bus write software weapons flat marked 1920 newly tight Eric finger Journal FC Van rise critical Atlantic granted returning communities humans quick 39 48 ranked sight pop Swedish Stephen card analysis attacked ##wa Sunday identified Jason champion situated 1930 expanded tears ##nce reaching Davis protection Emperor positions nominated Bridge tax dress allows avoid leadership killing actress guest steel knowing electric cells disease grade unknown ##ium resulted Pakistan confirmed ##ged tongue covers ##Y roof entirely applied votes drink interview exchange Township reasons ##ised page calls dog agent nose teaching ##ds ##ists advanced wish Golden existing vehicle del 1919 develop attacks pressed Sports planning resulting facility Sarah notes 1933 Class Historic winter ##mo audience Community household Netherlands creation ##ize keeping 1914 claims dry guys opposite ##ak explained Ontario secondary difference Francis actions organizations yard animal Up Lewis titles Several 1934 Ryan 55 Supreme rolled 1917 distribution figures afraid rural yourself ##rt sets barely Instead passing awards 41 silence authority occupied environment windows engineering surprised flying crime reports Mountain powers driving succeeded reviews 1929 Head missing Song Jesus opportunity inspired ends albums conversation impact injury surprise billion learning heavily oldest union creating ##ky festival literature letters sexual ##tte apartment Final comedy nation orders ##sen contemporary Power drawn existence connection ##ating Post Junior remembered message Medal castle note engineer sounds Beach crossed ##dy ear scientific sales ##ai theme starts clearly ##ut trouble ##gan bag ##han BC sons 1928 silent versions daily Studies ending Rose guns 1932 headquarters reference obtained Squadron concert none du Among ##don prevent Member answered staring Between ##lla portion drug liked association performances Nations formation Castle lose learn scoring relatively quarter 47 Premier ##ors Sweden baseball attempted trip worth perform airport fields enter honor Medical rear commander officials condition supply materials 52 Anna volume threw Persian 43 interested Gallery achieved visited laws relief Area Matt singles Lieutenant Country fans Cambridge sky Miller effective tradition Port ##ana minister extra entitled System sites authorities acres committee racing 1931 desk trains ass weren Family farm ##ance industrial ##head iron 49 abandoned Out Holy chairman waited frequently display Light transport starring Patrick Engineering eat FM judge reaction centuries price ##tive Korean defense Get arrested 1927 send urban ##ss pilot Okay Media reality arts soul thirty ##be catch generation ##nes apart Anne drop See ##ving sixth trained Management magic cm height Fox Ian resources vampire principal Was haven ##au Walter Albert rich 1922 causing entry ##ell shortly 46 worry doctor composer rank Network bright showing regions 1924 wave carrying kissed finding missed Earl lying target vehicles Military controlled dinner ##board briefly lyrics motion duty strange attempts invited kg villages 5th Land ##mer Christ prepared twelve check thousand earth copies en transfer citizens Americans politics nor theatre Project ##bo clean rooms laugh ##ran application contained anyway containing Sciences 1925 rare speech exist 1950s falling passenger ##im stands 51 ##ol ##ow phase governor kids details methods Vice employed performing counter Jane heads Channel wine opposition aged 1912 Every 1926 highway ##ura 1921 aired 978 permanent Forest finds joint approved ##pur brief doubt acts brand wild closely Ford Kevin chose shall port sweet fun asking Be ##bury sought Dave Mexican mom Right Howard Moscow Charlie Stone ##mann admitted ##ver wooden 1923 Officer relations Hot combat publication chain shop inhabitants proved ideas address 1915 Memorial explain increasing conflict Anthony Melbourne narrow temperature slid 1916 worse selling documentary Ali Ray opposed vision dad extensive Infantry commissioned Doctor offices programming core respect storm ##pa ##ay ##om promotion der struck anymore shit Region receiving DVD alternative ##ue ride maximum 1910 ##ious Third Affairs cancer Executive ##op dream 18th Due ##ker ##worth economy IV Billboard identity subsequent statement skills ##back funding ##ons Round Foreign truck Please lights wondered ##ms frame yes Still districts fiction Colonel converted 150 grown accident critics fit Information architecture Point Five armed Billy poet functions consisted suit Turkish Band object desire ##ities sounded flow Norwegian articles Marie pulling thin singing Hunter Human Battalion Federation Kim origin represent dangerous weather fuel ex ##sing Last bedroom aid knees Alan angry assumed plane Something founding concerned global Fire di please Portuguese touched Roger nuclear Register Jeff fixed royal lie finals NFL Manchester towns handle shaped Chairman Dean launch understanding Children violence failure sector Brigade wrapped fired sharp tiny developing expansion Free institutions technical Nothing otherwise Main inch Saturday wore Senior attached cheek representing Kansas ##chi ##kin actual advantage Dan Austria ##dale hoped multi squad Norway streets 1913 Services hired grow pp wear painted Minnesota stuff Building 54 Philippines 1900 ##ties educational Khan Magazine ##port Cape signal Gordon sword Anderson cool engaged Commander images Upon tied Security cup rail Vietnam successfully ##red Muslim gain bringing Native hers occurs negative Philip Kelly Colorado category ##lan 600 Have supporting wet 56 stairs Grace observed ##ung funds restaurant 1911 Jews ##ments ##che Jake Back 53 asks journalist accept bands bronze helping ##ice decades mayor survived usual influenced Douglas Hey ##izing surrounded retirement Temple derived Pope registered producing ##ral structures Johnny contributed finishing buy specifically ##king patients Jordan internal regarding Samuel Clark ##q afternoon Finally scenes notice refers quietly threat Water Those Hamilton promise freedom Turkey breaking maintained device lap ultimately Champion Tim Bureau expressed investigation extremely capable qualified recognition items ##up Indiana adult rain greatest architect Morgan dressed equal Antonio collected drove occur Grant graduate anger Sri worried standards ##ore injured somewhere damn Singapore Jimmy pocket homes stock religion aware regarded Wisconsin ##tra passes fresh ##ea argued Ltd EP Diego importance Census incident Egypt Missouri domestic leads ceremony Early camera Father challenge Switzerland lands familiar hearing spend educated Tennessee Thank ##ram Thus concern putting inches map classical Allen crazy valley Space softly ##my pool worldwide climate experienced neighborhood scheduled neither fleet 1908 Girl ##J Part engines locations darkness Revolution establishment lawyer objects apparently Queensland Entertainment bill mark Television ##ong pale demand Hotel selection ##rn ##ino Labour Liberal burned Mom merged Arizona request ##lia ##light hole employees ##ical incorporated 95 independence Walker covering joining ##ica task papers backing sell biggest 6th strike establish ##ō gently 59 Orchestra Winter protein Juan locked dates Boy aren shooting Luke solid charged Prior resigned interior garden spoken improve wonder promote hidden ##med combination Hollywood Swiss consider ##ks Lincoln literary drawing Marine weapon Victor Trust Maryland properties ##ara exhibition understood hung Tell installed loud fashion affected junior landing flowers ##he Internet beach Heart tries Mayor programme 800 wins noise ##ster ##ory 58 contain fair delivered ##ul wedding Square advance behavior Program Oregon ##rk residence realize certainly hill Houston 57 indicated ##water wounded Village massive Moore thousands personnel dating opera poetry ##her causes feelings Frederick applications push approached foundation pleasure sale fly gotten northeast costs raise paintings ##ney views horses formal Arab hockey typical representative rising ##des clock stadium shifted Dad peak Fame vice disappeared users Way Naval prize hoping values evil Bell consisting ##ón Regional ##ics improved circle carefully broad ##ini Fine maintain operate offering mention Death stupid Through Princess attend interests ruled somewhat wings roads grounds ##ual Greece Champions facing hide voted require Dark Matthew credit sighed separated manner ##ile Boys 1905 committed impossible lip candidates 7th Bruce arranged Islamic courses criminal ##ened smell ##bed 08 consecutive ##ening proper purchase weak Prix 1906 aside introduction Look ##ku changing budget resistance factory Forces agency ##tone northwest user 1907 stating ##one sport Design environmental cards concluded Carl 250 accused ##ology Girls sick intelligence Margaret responsibility Guard ##tus 17th sq goods 1909 hate ##ek capture stores Gray comic Modern Silver Andy electronic wheel ##ied Deputy ##bs Czech zone choose constant reserve ##lle Tokyo spirit sub degrees flew pattern compete Dance ##ik secretary Imperial 99 reduce Hungarian confused ##rin Pierre describes regularly Rachel 85 landed passengers ##ise ##sis historian meters Youth ##ud participate ##cing arrival tired Mother ##gy jumped Kentucky faces feed Israeli Ocean ##Q ##án plus snow techniques plate sections falls jazz ##ris tank loan repeated opinion ##res unless rugby journal Lawrence moments shock distributed ##ded adjacent Argentina crossing uncle ##ric Detroit communication mental tomorrow session Emma Without ##gen Miami charges Administration hits coat protected Cole invasion priest 09 Gary enjoyed plot measure bound friendly throw musician ##lon ##ins Age knife damaged birds driven lit ears breathing Arabic Jan faster Jonathan ##gate Independent starred Harris teachers Alice sequence mph file translated decide determine Review documents sudden threatened ##ft bear distinct decade burning ##sky 1930s replace begun extension ##time 1904 equivalent accompanied Christopher Danish ##ye Besides ##more persons fallen Rural roughly saved willing ensure Belgium 05 musicians ##ang giant Six Retrieved worst purposes ##bly mountains seventh slipped brick 07 ##py somehow Carter Iraq cousin favor islands journey FIFA contrast planet vs calm ##ings concrete branches gray profit Russell ##ae ##ux ##ens philosophy businesses talked parking ##ming owners Place ##tle agricultural Kate 06 southeast draft Eddie earliest forget Dallas Commonwealth edited 66 inner ed operates 16th Harvard assistance ##si designs Take bathroom indicate CEO Command Louisiana 1902 Dublin Books 1901 tropical 1903 ##tors Places tie progress forming solution 62 letting ##ery studying ##jo duties Baseball taste Reserve ##ru Ann ##gh visible ##vi notably link NCAA southwest Never storage mobile writers favorite Pro pages truly count ##tta string kid 98 Ross row ##idae Kennedy ##tan Hockey hip waist grandfather listen ##ho feels busy 72 stream obvious cycle shaking Knight ##ren Carlos painter trail web linked 04 Palace existed ##ira responded closing End examples Marshall weekend jaw Denmark lady township medium chin Story option fifteen Moon represents makeup investment jump childhood Oklahoma roll normally Ten Operation Graham Seattle Atlanta paused promised rejected treated returns flag ##ita Hungary danger glad movements visual subjects credited soldier Norman ill translation José Quebec medicine warning theater praised municipal 01 commune churches acid folk 8th testing add survive Sound devices residential severe presidential Mississippi Austin Perhaps Charlotte hanging Montreal grin ##ten racial partnership shoot shift ##nie Les downtown Brothers Garden matters restored mirror forever winners rapidly poverty ##ible Until DC faith hundreds Real Ukraine Nelson balance Adams contest relative ethnic Edinburgh composition ##nts emergency ##van marine reputation Down pack 12th Communist Mountains pro stages measures ##ld ABC Li victims benefit Iowa Broadway gathered rating Defense classic ##ily ceiling ##ions snapped Everything constituency Franklin Thompson Stewart entering Judge forth ##sk wanting smiling moves tunnel premiered grass unusual Ukrainian bird Friday tail Portugal coal element Fred guards Senator collaboration beauty Wood chemical beer justice signs ##Z sees ##zi Puerto ##zed 96 smooth Bowl gift limit 97 heading Source wake requires Ed Constitution factor Lane factors adding Note cleared pictures pink ##ola Kent Local Singh moth Ty ##ture courts Seven temporary involving Vienna emerged fishing agree defensive stuck secure Tamil ##ick bottle 03 Player instruments Spring patient flesh contributions cry Malaysia 120 Global da Alabama Within ##work debuted expect Cleveland concerns retained horror 10th spending Peace Transport grand Crown instance institution acted Hills mounted Campbell shouldn 1898 ##ably chamber soil 88 Ethan sand cheeks ##gi marry 61 weekly classification DNA Elementary Roy definitely Soon Rights gate suggests aspects imagine golden beating Studios Warren differences significantly glance occasionally ##od clothing Assistant depth sending possibility mode prisoners requirements daughters dated Representatives prove guilty interesting smoke cricket 93 ##ates rescue Connecticut underground Opera 13th reign ##ski thanks leather equipped routes fan ##ans script Wright bishop Welsh jobs faculty eleven Railroad appearing anniversary Upper ##down anywhere Rugby Metropolitan Meanwhile Nicholas champions forehead mining drinking 76 Jerry membership Brazilian Wild Rio scheme Unlike strongly ##bility fill ##rian easier MP Hell ##sha Stanley banks Baron ##ique Robinson 67 Gabriel Austrian Wayne exposed ##wan Alfred 1899 manage mix visitors eating ##rate Sean commission Cemetery policies Camp parallel traveled guitarist 02 supplies couples poem blocks Rick Training Energy achieve appointment Wing Jamie 63 novels ##em 1890 songwriter Base Jay ##gar naval scared miss labor technique crisis Additionally backed destroy seriously tools tennis 91 god ##ington continuing steam obviously Bobby adapted fifty enjoy Jacob publishing column ##ular Baltimore Donald Liverpool 92 drugs movies ##ock Heritage ##je ##istic vocal strategy gene advice ##bi Ottoman riding ##side Agency Indonesia 11th laughing sleeping und muttered listening deck tip 77 ownership grey Claire deeply provincial popularity Cooper ##á Emily ##sed designer Murray describe Danny Around Parker ##dae 68 rates suffering considerable 78 nervous powered tons circumstances wished belonged Pittsburgh flows 9th ##use belt 81 useful 15th context List Dead Iron seek Season worn frequency legislation replacement memories Tournament Again Barry organisation copy Gulf waters meets struggle Oliver 1895 Susan protest kick Alliance components 1896 Tower Windows demanded regiment sentence Woman Logan Referee hosts debate knee Blood ##oo universities practices Ward ranking correct happening Vincent attracted classified ##stic processes immediate waste increasingly Helen ##po Lucas Phil organ 1897 tea suicide actors lb crash approval waves ##ered hated grip 700 amongst 69 74 hunting dying lasted illegal ##rum stare defeating ##gs shrugged °C Jon Count Orleans 94 affairs formally ##and ##ves criticized Disney Vol successor tests scholars palace Would celebrated rounds grant Schools Such commanded demon Romania ##all Karl 71 ##yn 84 Daily totally Medicine fruit Die upset Lower Conservative 14th Mitchell escaped shoes Morris ##tz queen harder prime Thanks indeed Sky authors rocks definition Nazi accounts printed experiences ##ters divisions Cathedral denied depending Express ##let 73 appeal loose colors filed ##isation gender ##ew throne forests Finland domain boats Baker squadron shore remove ##ification careful wound railroad 82 seeking agents ##ved Blues ##off customers ignored net ##ction hiding Originally declined ##ess franchise eliminated NBA merely pure appropriate visiting forty markets offensive coverage cave ##nia spell ##lar Benjamin ##ire Convention filmed Trade ##sy ##ct Having palm 1889 Evans intense plastic Julia document jeans vessel SR ##fully proposal Birmingham le ##ative assembly 89 fund lock 1893 AD meetings occupation modified Years odd aimed reform Mission Works shake cat exception convinced executed pushing dollars replacing soccer manufacturing ##ros expensive kicked minimum Josh coastal Chase ha Thailand publications deputy Sometimes Angel effectively ##illa criticism conduct Serbian landscape NY absence passage ##ula Blake Indians 1892 admit Trophy ##ball Next ##rated ##ians charts kW orchestra 79 heritage 1894 rough exists boundary Bible Legislative moon medieval ##over cutting print ##ett birthday ##hood destruction Julian injuries influential sisters raising statue colour dancing characteristics orange ##ok ##aries Ken colonial twin Larry surviving ##shi Barbara personality entertainment assault ##ering talent happens license 86 couch Century soundtrack shower swimming cash Staff bent 1885 bay lunch ##lus dozen vessels CBS greatly critic Test symbol panel shell output reaches 87 Front motor ocean ##era ##ala maintenance violent scent Limited Las Hope Theater Which survey Robin recordings compilation ##ward bomb insurance Authority sponsored satellite Jazz refer stronger blow whilst Wrestling suggest ##rie climbed ##els voices shopping 1891 Neil discovery ##vo ##ations burst Baby peaked Brooklyn knocked lift ##try false nations Hugh Catherine preserved distinguished terminal resolution ratio pants cited competitions completion DJ bone uniform schedule shouted 83 1920s rarely Basketball Taiwan artistic bare vampires arrest Utah Marcus assist gradually qualifying Victorian vast rival Warner Terry Economic ##cia losses boss versus audio runner apply surgery Play twisted comfortable ##cs Everyone guests ##lt Harrison UEFA lowered occasions ##lly ##cher chapter youngest eighth Culture ##room ##stone 1888 Songs Seth Digital involvement expedition relationships signing 1000 fault annually circuit afterwards meat creature ##ou cable Bush ##net Hispanic rapid gonna figured extent considering cried ##tin sigh dynasty ##ration cabinet Richmond stable ##zo 1864 Admiral Unit occasion shares badly longest ##ify Connor extreme wondering girlfriend Studio ##tions 1865 tribe exact muscles hat Luis Orthodox decisions amateur description ##lis hips kingdom ##ute Portland whereas Bachelor outer discussion partly Arkansas 1880 dreams perfectly Lloyd ##bridge asleep ##tti Greg permission trading pitch mill Stage liquid Keith ##tal wolf processing stick Jerusalem profile rushed spiritual argument Ice Guy till Delhi roots Section missions Glasgow penalty NBC encouraged identify keyboards ##zing ##ston disc plain informed Bernard thinks fled Justin ##day newspapers ##wick Ralph ##zer unlike Stars artillery ##ified recovered arrangement searching ##pers ##tory ##rus deaths Egyptian diameter ##í marketing corporate teach marks Turner staying hallway Sebastian chapel naked mistake possession 1887 dominated jacket creative Fellow Falls Defence suspended employment ##rry Hebrew Hudson Week Wars recognize Natural controversial Tommy thank Athletic benefits decline intention ##ets Lost Wall participation elevation supports parliament 1861 concentration Movement ##IS competing stops behalf ##mm limits funded discuss Collins departure obtain woods latest universe alcohol Laura rush blade funny Dennis forgotten Amy Symphony apparent graduating 1862 Rob Grey collections Mason emotions ##ugh literally Any counties 1863 nomination fighter habitat respond external Capital exit Video carbon sharing Bad opportunities Perry photo ##mus Orange posted remainder transportation portrayed Labor recommended percussion rated Grade rivers partially suspected strip adults button struggled intersection Canal ##ability poems claiming Madrid 1886 Together ##our Much Vancouver instrument instrumental 1870 mad angle Control Phoenix Leo Communications mail ##ette ##ev preferred adaptation alleged discussed deeper ##ane Yet Monday volumes thrown Zane ##logy displayed rolling dogs Along Todd ##ivity withdrew representation belief ##sia crown Late Short hardly grinned romantic Pete ##ken networks enemies Colin Eventually Side donated ##su steady grab guide Finnish Milan pregnant controversy reminded 1884 Stuart ##bach ##ade Race Belgian LP Production Zone lieutenant infantry Child confusion sang resident ##ez victim 1881 channels Ron businessman ##gle Dick colony pace producers ##ese agencies Craig Lucy Very centers Yorkshire photography ##ched Album championships Metro substantial Standard terrible directors contribution advertising emotional ##its layer segment sir folded Roberts ceased Hampshire ##ray detailed partners m² ##pt Beth genre commented generated remote aim Hans credits concerts periods breakfast gay shadow defence Too Had transition Afghanistan ##book eggs defend ##lli writes Systems bones mess seed scientists Shortly Romanian ##zy Freedom muscle hero parent agriculture checked Islam Bristol Freyja Arena cabin Germans electricity ranks viewed medals Wolf associate Madison Sorry fort Chile detail widespread attorney boyfriend ##nan Students Spencer ##ig bite Maine demolished Lisa erected Someone operational Commissioner NHL Coach Bar forcing Dream Rico cargo Murphy ##fish ##ase distant ##master ##ora Organization doorway Steven traded electrical frequent ##wn Branch Sure 1882 placing Manhattan attending attributed excellent pounds ruling principles component Mediterranean Vegas machines percentage infrastructure throwing affiliated Kings secured Caribbean Track Ted honour opponent Virgin Construction grave produces Challenge stretched paying murmured ##ata integrated waved Nathan ##ator transmission videos ##yan ##hu Nova descent AM Harold conservative Therefore venue competitive ##ui conclusion funeral confidence releases scholar ##sson Treaty stress mood ##sm Mac residing Action Fund ##ship animated fitted ##kar defending voting tend ##berry answers believes ##ci helps Aaron ##tis themes ##lay populations Players stroke Trinity electoral paint abroad charity keys Fair ##pes interrupted participants murdered Days supporters ##ab expert borders mate ##llo solar architectural tension ##bling Parish tape operator Cultural Clinton indicates publisher ordinary sugar arrive rifle acoustic ##uring assets ##shire SS sufficient options HMS Classic bars rebuilt governments Beijing reporter screamed Abbey crying mechanical instantly communications Political cemetery Cameron Stop representatives USS texts mathematics innings civilian Serbia ##hill practical patterns dust Faculty debt ##end ##cus junction suppose experimental Computer Food wrist abuse dealing bigger cap principle ##pin Muhammad Fleet Collection attempting dismissed ##burn regime Herbert ##ua shadows 1883 Eve Lanka 1878 Performance fictional ##lock Noah Run Voivodeship exercise broadcasting ##fer RAF Magic Bangladesh suitable ##low ##del styles toured Code identical links insisted 110 flash Model slave Derek Rev fairly Greater sole ##lands connecting zero bench ##ome switched Fall Owen yours Electric shocked convention ##bra climb memorial swept Racing decides belong ##nk parliamentary ##und ages proof ##dan delivery 1860 ##ów sad publicly leaning Archbishop dirt ##ose categories 1876 burn ##bing requested Guinea Historical rhythm relation ##heim ye pursue merchant ##mes lists continuous frowned colored tool gods involves Duncan photographs Cricket slight Gregory atmosphere wider Cook ##tar essential Being FA emperor wealthy nights ##bar licensed Hawaii viewers Language load nearest milk kilometers platforms ##ys territories Rogers sheet Rangers contested ##lation isolated assisted swallowed Small Contemporary Technical Edwards express Volume endemic ##ei tightly Whatever indigenous Colombia ##ulation hp characterized ##ida Nigeria Professional duo Soccer slaves Farm smart Attorney Attendance Common salt ##vin tribes nod sentenced bid sample Drive switch instant 21st Cuba drunk Alaska proud awareness hitting sessions Thai locally elsewhere Dragon gentle touching ##lee Springs Universal Latino spin 1871 Chart recalled Type pointing ##ii lowest ##ser grandmother Adelaide Jacques spotted Buffalo restoration Son Joan farmers Lily 1879 lucky ##dal luck eldest ##rant Market drummer deployed warned prince sing amazing sailed ##oon 1875 Primary traveling Masters Sara cattle Trail gang Further desert relocated ##tch ##ord Flight illness Munich ninth repair Singles ##lated Tyler tossed boots Work sized earning shoved magazines housed dam researchers Former spun premiere spaces organised wealth crimes devoted stones Urban automatic hop affect outstanding tanks mechanism Muslims Ms shots argue Jeremy connections Armenian increases rubbed 1867 retail gear Pan bonus jurisdiction weird concerning whisper ##gal Microsoft tenure hills www Gmina porch files reportedly venture Storm ##ence Nature killer panic fate Secret Wang scream drivers belongs Chamber clan monument mixing Peru bet Riley Friends Isaac submarine 1877 130 judges harm ranging affair prepare pupils householder Policy decorated Nation slammed activist implemented Room qualify Publishing establishing Baptist touring subsidiary ##nal legend 1872 laughter PC Athens settlers ties dual dear Draft strategic Ivan reveal closest dominant Ah ##ult Denver bond boundaries drafted tables ##TV eyed Edition ##ena 1868 belonging 1874 Industrial cream Ridge Hindu scholarship Ma opens initiated ##ith yelled compound random Throughout grades physics sank grows exclusively settle Saints brings Amsterdam Make Hart walks battery violin ##born explanation ##ware 1873 ##har provinces thrust exclusive sculpture shops ##fire VI constitution Barcelona monster Devon Jefferson Sullivan bow ##din desperate ##ć Julie ##mon ##ising terminus Jesse abilities golf ##ple ##via ##away Raymond measured jury firing revenue suburb Bulgarian 1866 ##cha timber Things ##weight Morning spots Alberta Data explains Kyle friendship raw tube demonstrated aboard immigrants reply breathe Manager ease ##ban ##dia Diocese ##vy ##ía pit ongoing ##lie Gilbert Costa 1940s Report voters cloud traditions ##MS gallery Jennifer swung Broadcasting Does diverse reveals arriving initiative ##ani Give Allied Pat Outstanding monastery blind Currently ##war bloody stopping focuses managing Florence Harvey creatures 900 breast internet Artillery purple ##mate alliance excited fee Brisbane lifetime Private ##aw ##nis ##gue ##ika phrase regulations reflected manufactured conventional pleased client ##ix ##ncy Pedro reduction ##con welcome jail comfort Iranian Norfolk Dakota ##tein evolution everywhere Initially sensitive Olivia Oscar implementation sits stolen demands slide grandson ##ich merger ##mic Spirit ##° ticket root difficulty Nevada ##als lined Dylan Original Call biological EU dramatic ##hn Operations treaty gap ##list Am Romanized moral Butler perspective Furthermore Manuel absolutely unsuccessful disaster dispute preparation tested discover ##ach shield squeezed brushed battalion Arnold ##ras superior treat clinical ##so Apple Syria Cincinnati package flights editions Leader minority wonderful hang Pop Philippine telephone bell honorary ##mar balls Democrat dirty thereafter collapsed Inside slip wrestling ##ín listened regard bowl None Sport completing trapped ##view copper Wallace Honor blame Peninsula ##ert ##oy Anglo bearing simultaneously honest ##ias Mix Got speaker voiced impressed prices error 1869 ##feld trials Nine Industry substitute Municipal departed slept ##ama Junction Socialist flower dropping comment fantasy ##ress arrangements travelled furniture fist relieved ##tics Leonard linear earn expand Soul Plan Leeds Sierra accessible innocent Winner Fighter Range winds vertical Pictures 101 charter cooperation prisoner interviews recognised sung manufacturer exposure submitted Mars leaf gauge screaming likes eligible ##ac gathering columns ##dra belly UN maps messages speakers ##ants garage unincorporated Number Watson sixteen lots beaten Could Municipality ##ano Horse talks Drake scores Venice genetic ##mal ##ère Cold Jose nurse traditionally ##bus Territory Key Nancy ##win thumb São index dependent carries controls Comics coalition physician referring Ruth Based restricted inherited internationally stretch THE plates margin Holland knock significance valuable Kenya carved emotion conservation municipalities overseas resumed Finance graduation blinked temperatures constantly productions scientist ghost cuts permitted ##ches firmly ##bert patrol ##yo Croatian attacking 1850 portrait promoting sink conversion ##kov locomotives Guide ##val nephew relevant Marc drum originated Chair visits dragged Price favour corridor properly respective Caroline reporting inaugural 1848 industries ##ching edges Christianity Maurice Trent Economics carrier Reed ##gon tribute Pradesh ##ale extend attitude Yale ##lu settlements glasses taxes targets ##ids quarters ##ological connect hence metre collapse underneath banned Future clients alternate explosion kinds Commons hungry dragon Chapel Buddhist lover depression pulls ##ges ##uk origins computers crosses kissing assume emphasis lighting ##ites personally crashed beam touchdown lane comparison ##mont Hitler ##las execution ##ene acre sum Pearl ray ##point essentially worker convicted tear Clay recovery Literature Unfortunately ##row partial Petersburg Bulgaria coaching evolved reception enters narrowed elevator therapy defended pairs ##lam breaks Bennett Uncle cylinder ##ison passion bases Actor cancelled battles extensively oxygen Ancient specialized negotiations ##rat acquisition convince interpretation ##00 photos aspect colleges Artist keeps ##wing Croatia ##ona Hughes Otto comments ##du Ph Sweet adventure describing Student Shakespeare scattered objective Aviation Phillips Fourth athletes ##hal ##tered Guitar intensity née dining curve Obama topics legislative Mill Cruz ##ars Members recipient Derby inspiration corresponding fed YouTube coins pressing intent Karen cinema Delta destination shorter Christians imagined canal Newcastle Shah Adrian super Males 160 liberal lord bat supplied Claude meal worship ##atic Han wire °F ##tha punishment thirteen fighters ##ibility 1859 Ball gardens ##ari Ottawa pole indicating Twenty Higher Bass Ivy farming ##urs certified Saudi plenty ##ces restaurants Representative Miles payment ##inger ##rit Confederate festivals references ##ić Mario PhD playoffs witness rice mask saving opponents enforcement automatically relegated ##oe radar whenever Financial imperial uncredited influences Abraham skull Guardian Haven Bengal impressive input mixture Warsaw altitude distinction 1857 collective Annie ##ean ##bal directions Flying ##nic faded ##ella contributing ##ó employee ##lum ##yl ruler oriented conductor focusing ##die Giants Mills mines Deep curled Jessica guitars Louise procedure Machine failing attendance Nepal Brad Liam tourist exhibited Sophie depicted Shaw Chuck ##can expecting challenges ##nda equally resignation ##logical Tigers loop pitched outdoor reviewed hopes True temporarily Borough torn jerked collect Berkeley Independence cotton retreat campaigns participating Intelligence Heaven ##ked situations borough Democrats Harbor ##len Liga serial circles fourteen ##lot seized filling departments finance absolute Roland Nate floors raced struggling deliver protests ##tel Exchange efficient experiments ##dar faint 3D binding Lions lightly skill proteins difficulties ##cal monthly camps flood loves Amanda Commerce ##oid ##lies elementary ##tre organic ##stein ##ph receives Tech enormous distinctive Joint experiment Circuit citizen ##hy shelter ideal practically formula addressed Foster Productions ##ax variable punk Voice fastest concentrated ##oma ##yer stored surrender vary Sergeant Wells ward Wait ##ven playoff reducing cavalry ##dle Venezuela tissue amounts sweat ##we Non ##nik beetle ##bu ##tu Jared Hunt ##₂ fat Sultan Living Circle Secondary Suddenly reverse ##min Travel ##bin Lebanon ##mas virus Wind dissolved enrolled holiday Keep helicopter Clarke constitutional technologies doubles instructions ##ace Azerbaijan ##ill occasional frozen trick wiped writings Shanghai preparing challenged mainstream summit 180 ##arian ##rating designation ##ada revenge filming tightened Miguel Montana reflect celebration bitch flashed signals rounded peoples ##tation renowned Google characteristic Campaign sliding ##rman usage Record Using woke solutions holes theories logo Protestant relaxed brow nickname Reading marble ##tro symptoms Overall capita ##ila outbreak revolution deemed Principal Hannah approaches inducted Wellington vulnerable Environmental Drama incumbent Dame 1854 travels samples accurate physically Sony Nashville ##sville ##lic ##og Producer Lucky tough Stanford resort repeatedly eyebrows Far choir commenced ##ep ##ridge rage swing sequel heir buses ad Grove ##late ##rick updated ##SA Delaware ##fa Athletics warmth Off excitement verse Protection Villa corruption intellectual Jenny ##lyn mystery prayer healthy ##ologist Bear lab Ernest Remix register basement Montgomery consistent tier 1855 Preston Brooks ##maker vocalist laboratory delayed wheels rope bachelor pitcher Block Nevertheless suspect efficiency Nebraska siege FBI planted ##AC Newton breeding ##ain eighteen Argentine encounter servant 1858 elder Shadow Episode fabric doctors survival removal chemistry volunteers Kane variant arrives Eagle Left ##fe Jo divorce ##ret yesterday Bryan handling diseases customer Sheriff Tiger Harper ##oi resting Linda Sheffield gasped sexy economics alien tale footage Liberty yeah fundamental Ground flames Actress photographer Maggie Additional joke custom Survey Abu silk consumption Ellis bread ##uous engagement puts Dog ##hr poured guilt CDP boxes hardware clenched ##cio stem arena extending ##com examination Steel encountered revised 140 picking Car hasn Minor pride Roosevelt boards ##mia blocked curious drag narrative brigade Prefecture mysterious namely connects Devil historians CHAPTER quit installation Golf empire elevated ##eo releasing Bond ##uri harsh ban ##BA contracts cloth presents stake chorus ##eau swear ##mp allies generations Motor meter pen warrior veteran ##EC comprehensive missile interaction instruction Renaissance rested Dale fix fluid les investigate loaded widow exhibit artificial select rushing tasks signature nowhere Engineer feared Prague bother extinct gates Bird climbing heels striking artwork hunt awake ##hin Formula thereby commitment imprisoned Beyond ##MA transformed Agriculture Low Movie radical complicated Yellow Auckland mansion tenth Trevor predecessor ##eer disbanded sucked circular witch gaining lean Behind illustrated rang celebrate bike consist framework ##cent Shane owns 350 comprises collaborated colleagues ##cast engage fewer ##ave 1856 observation diplomatic legislature improvements Interstate craft MTV martial administered jet approaching permanently attraction manuscript numbered Happy Andrea shallow Gothic Anti ##bad improvement trace preserve regardless rode dies achievement maintaining Hamburg spine ##air flowing encourage widened posts ##bound 125 Southeast Santiago ##bles impression receiver Single closure ##unt communist honors Northwest 105 ##ulated cared un hug magnetic seeds topic perceived prey prevented Marvel Eight Michel Transportation rings Gate ##gne Byzantine accommodate floating ##dor equation ministry ##ito ##gled Rules earthquake revealing Brother Celtic blew chairs Panama Leon attractive descendants Care Ambassador tours breathed threatening ##cho smiles Lt Beginning ##iness fake assists fame strings Mobile Liu parks http 1852 brush Aunt bullet consciousness ##sta ##ther consequences gather dug 1851 bridges Doug ##sion Artists ignore Carol brilliant radiation temples basin clouds ##cted Stevens spite soap consumer Damn Snow recruited ##craft Advanced tournaments Quinn undergraduate questioned Palmer Annual Others feeding Spider printing ##orn cameras functional Chester readers Alpha universal Faith Brandon François authored Ring el aims athletic possessed Vermont programmes ##uck bore Fisher statements shed saxophone neighboring pronounced barrel bags ##dge organisations pilots casualties Kenneth ##brook silently Malcolm span Essex anchor ##hl virtual lessons Henri Trump Page pile locomotive wounds uncomfortable sustained Diana Eagles ##pi 2000s documented ##bel Cassie delay kisses ##ines variation ##ag growled ##mark ##ways Leslie studios Friedrich aunt actively armor eaten historically Better purse honey ratings ##ée naturally 1840 peer Kenny Cardinal database Looking runners handsome Double PA ##boat ##sted protecting ##jan Diamond concepts interface ##aki Watch Article Columbus dialogue pause ##rio extends blanket pulse 1853 affiliate ladies Ronald counted kills demons ##zation Airlines Marco Cat companion mere Yugoslavia Forum Allan pioneer Competition Methodist patent nobody Stockholm ##ien regulation ##ois accomplished ##itive washed sake Vladimir crops prestigious humor Sally labour tributary trap altered examined Mumbai bombing Ash noble suspension ruins ##bank spare displays guided dimensional Iraqi ##hon sciences Franz relating fence followers Palestine invented proceeded Batman Bradley ##yard ##ova crystal Kerala ##ima shipping handled Want abolished Drew ##tter Powell Half ##table ##cker exhibitions Were assignment assured ##rine Indonesian Grammy acknowledged Kylie coaches structural clearing stationed Say Total Rail besides glow threats afford Tree Musical ##pp elite centered explore Engineers Stakes Hello tourism severely assessment ##tly crack politicians ##rrow sheets volunteer ##borough ##hold announcement recover contribute lungs ##ille mainland presentation Johann Writing 1849 ##bird Study Boulevard coached fail airline Congo Plus Syrian introduce ridge Casey manages ##fi searched Support succession progressive coup cultures ##lessly sensation Cork Elena Sofia Philosophy mini trunk academy Mass Liz practiced Reid ##ule satisfied experts Wilhelm Woods invitation Angels calendar joy Sr Dam packed ##uan bastard Workers broadcasts logic cooking backward ##ack Chen creates enzyme ##xi Davies aviation VII Conservation fucking Knights ##kan requiring hectares wars ate ##box Mind desired oak absorbed Really Vietnamese Paulo athlete ##car ##eth Talk Wu ##cks survivors Yang Joel Almost Holmes Armed Joshua priests discontinued ##sey blond Rolling suggesting CA clay exterior Scientific ##sive Giovanni Hi farther contents Winners animation neutral mall Notes layers professionals Armstrong Against Piano involve monitor angel parked bears seated feat beliefs ##kers Version suffer ##ceae guidance ##eur honored raid alarm Glen Ellen Jamaica trio enabled ##ils procedures ##hus moderate upstairs ##ses torture Georgian rebellion Fernando Nice ##are Aires Campus beast ##hing 1847 ##FA Isle ##logist Princeton cathedral Oakland Solomon ##tto Milwaukee upcoming midfielder Neither sacred Eyes appreciate Brunswick secrets Rice Somerset Chancellor Curtis ##gel Rich separation grid ##los ##bon urge ##ees ##ree freight towers psychology requirement dollar ##fall ##sman exile tomb Salt Stefan Buenos Revival Porter tender diesel chocolate Eugene Legion Laboratory sheep arched hospitals orbit Full ##hall drinks ripped ##RS tense Hank leagues ##nberg PlayStation fool Punjab relatives Comedy sur 1846 Tonight Sox ##if Rabbi org speaks institute defender painful wishes Weekly literacy portions snake item deals ##tum autumn sharply reforms thighs prototype ##ition argues disorder Physics terror provisions refugees predominantly independently march ##graphy Arabia Andrews Bus Money drops ##zar pistol matrix revolutionary ##ust Starting ##ptic Oak Monica ##ides servants ##hed archaeological divorced rocket enjoying fires ##nel assembled qualification retiring ##fied Distinguished handful infection Durham ##itz fortune renewed Chelsea ##sley curved gesture retain exhausted ##ifying Perth jumping Palestinian Simpson colonies steal ##chy corners Finn arguing Martha ##var Betty emerging Heights Hindi Manila pianist founders regret Napoleon elbow overhead bold praise humanity ##ori Revolutionary ##ere fur ##ole Ashley Official ##rm lovely Architecture ##sch Baronet virtually ##OS descended immigration ##das ##kes Holly Wednesday maintains theatrical Evan Gardens citing ##gia segments Bailey Ghost ##city governing graphics ##ined privately potentially transformation Crystal Cabinet sacrifice hesitated mud Apollo Desert bin victories Editor Railways Web Case tourists Brussels Franco compiled topped Gene engineers commentary egg escort nerve arch necessarily frustration Michelle democracy genes Facebook halfway ##ient 102 flipped Won ##mit NASA Lynn Provincial ambassador Inspector glared Change McDonald developments tucked noting Gibson circulation dubbed armies resource Headquarters ##iest Mia Albanian Oil Albums excuse intervention Grande Hugo integration civilians depends reserves Dee compositions identification restrictions quarterback Miranda Universe favourite ranges hint loyal Op entity Manual quoted dealt specialist Zhang download Westminster Rebecca streams Anglican variations Mine detective Films reserved ##oke ##key sailing ##gger expanding recall discovers particles behaviour Gavin blank permit Java Fraser Pass ##non ##TA panels statistics notion courage dare venues ##roy Box Newport travelling Thursday warriors Glenn criteria 360 mutual restore varied bitter Katherine ##lant ritual bits ##à Henderson trips Richardson Detective curse psychological Il midnight streak facts Dawn Indies Edmund roster Gen ##nation 1830 congregation shaft ##ically ##mination Indianapolis Sussex loving ##bit sounding horrible Continental Griffin advised magical millions ##date 1845 Safety lifting determination valid dialect Penn Know triple avoided dancer judgment sixty farmer lakes blast aggressive Abby tag chains inscription ##nn conducting Scout buying ##wich spreading ##OC array hurried Environment improving prompted fierce Taking Away tune pissed Bull catching ##ying eyebrow metropolitan terrain ##rel Lodge manufacturers creator ##etic happiness ports ##ners Relations fortress targeted ##ST allegedly blues ##osa Bosnia ##dom burial similarly stranger pursued symbols rebels reflection routine traced indoor eventual ##ska ##ão ##una MD ##phone oh grants Reynolds rid operators ##nus Joey vital siblings keyboard br removing societies drives solely princess lighter Various Cavalry believing SC underwent relay smelled syndrome welfare authorized seemingly Hard chicken ##rina Ages Bo democratic barn Eye shorts ##coming ##hand disappointed unexpected centres Exhibition Stories Site banking accidentally Agent conjunction André Chloe resist width Queens provision ##art Melissa Honorary Del prefer abruptly duration ##vis Glass enlisted ##ado discipline Sisters carriage ##ctor ##sburg Lancashire log fuck ##iz closet collecting holy rape trusted cleaning inhabited Rocky 104 editorial ##yu ##ju succeed strict Cuban ##iya Bronze outcome ##ifies ##set corps Hero barrier Kumar groaned Nina Burton enable stability Milton knots ##ination slavery ##borg curriculum trailer warfare Dante Edgar revival Copenhagen define advocate Garrett Luther overcome pipe 750 construct Scotia kings flooding ##hard Ferdinand Felix forgot Fish Kurt elaborate ##BC graphic gripped colonel Sophia Advisory Self ##uff ##lio monitoring seal senses rises peaceful journals 1837 checking legendary Ghana ##power ammunition Rosa Richards nineteenth ferry aggregate Troy inter ##wall Triple steep tent Cyprus 1844 ##woman commanding farms doi navy specified na cricketer transported Think comprising grateful solve ##core beings clerk grain vector discrimination ##TC Katie reasonable drawings veins consideration Monroe repeat breed dried witnessed ordained Current spirits remarkable consultant urged Remember anime singers phenomenon Rhode Carlo demanding findings manual varying Fellowship generate safely heated withdrawn ##ao headquartered ##zon ##lav ##ency Col Memphis imposed rivals Planet healing ##hs ensemble Warriors ##bone cult Frankfurt ##HL diversity Gerald intermediate ##izes reactions Sister ##ously ##lica quantum awkward mentions pursuit ##ography varies profession molecular consequence lectures cracked 103 slowed ##tsu cheese upgraded suite substance Kingston 1800 Idaho Theory ##een ain Carson Molly ##OR configuration Whitney reads audiences ##tie Geneva Outside ##nen ##had transit volleyball Randy Chad rubber motorcycle respected eager Level coin ##lets neighbouring ##wski confident ##cious poll uncertain punch thesis Tucker IATA Alec ##ographic ##law 1841 desperately 1812 Lithuania accent Cox lightning skirt ##load Burns Dynasty ##ug chapters Working dense Morocco ##kins casting Set activated oral Brien horn HIV dawn stumbled altar tore considerably Nicole interchange registration biography Hull Stan bulk consent Pierce ##ER Fifth marched terrorist ##piece ##itt Presidential Heather staged Plant relegation sporting joins ##ced Pakistani dynamic Heat ##lf ourselves Except Elliott nationally goddess investors Burke Jackie ##ā ##RA Tristan Associate Tuesday scope Near bunch ##abad ##ben sunlight ##aire manga Willie trucks boarding Lion lawsuit Learning Der pounding awful ##mine IT Legend romance Serie AC gut precious Robertson hometown realm Guards Tag batting ##vre halt conscious 1838 acquire collar ##gg ##ops Herald nationwide citizenship Aircraft decrease em Fiction Female corporation Located ##ip fights unconscious Tampa Poetry lobby Malta ##sar ##bie layout Tate reader stained ##bre ##rst ##ulate loudly Eva Cohen exploded Merit Maya ##rable Rovers ##IC Morrison Should vinyl ##mie onwards ##gie vicinity Wildlife probability Mar Barnes ##ook spinning Moses ##vie Surrey Planning conferences protective Plaza deny Canterbury manor Estate tilted comics IBM destroying server Dorothy ##horn Oslo lesser heaven Marshal scales strikes ##ath firms attract ##BS controlling Bradford southeastern Amazon Travis Janet governed 1842 Train Holden bleeding gifts rent 1839 palms ##ū judicial Ho Finals conflicts unlikely draws ##cies compensation adds elderly Anton lasting Nintendo codes ministers pot associations capabilities ##cht libraries ##sie chances performers runway ##af ##nder Mid Vocals ##uch ##eon interpreted priority Uganda ruined Mathematics cook AFL Lutheran AIDS Capitol chase axis Moreover María Saxon storyline ##ffed Tears Kid cent colours Sex ##long pm blonde Edwin CE diocese ##ents ##boy Inn ##ller Saskatchewan ##kh stepping Windsor ##oka ##eri Xavier Resources 1843 ##top ##rad ##lls Testament poorly 1836 drifted slope CIA remix Lords mature hosting diamond beds ##ncies luxury trigger ##lier preliminary hybrid journalists Enterprise proven expelled insects Beautiful lifestyle vanished ##ake ##ander matching surfaces Dominican Kids referendum Orlando Truth Sandy privacy Calgary Speaker sts Nobody shifting ##gers Roll Armenia Hand ##ES 106 ##ont Guild larvae Stock flame gravity enhanced Marion surely ##tering Tales algorithm Emmy darker VIII ##lash hamlet deliberately occurring choices Gage fees settling ridiculous ##ela Sons cop custody ##ID proclaimed Cardinals ##pm Metal Ana 1835 clue Cardiff riders observations MA sometime ##och performer intact Points allegations rotation Tennis tenor Directors ##ats Transit thigh Complex ##works twentieth Factory doctrine Daddy ##ished pretend Winston cigarette ##IA specimens hydrogen smoking mathematical arguments openly developer ##iro fists somebody ##san Standing Caleb intelligent Stay Interior echoed Valentine varieties Brady cluster Ever voyage ##of deposits ultimate Hayes horizontal proximity ##ás estates exploration NATO Classical ##most bills condemned 1832 hunger ##ato planes deserve offense sequences rendered acceptance ##ony manufacture Plymouth innovative predicted ##RC Fantasy ##une supporter absent Picture bassist rescued ##MC Ahmed Monte ##sts ##rius insane novelist ##és agrees Antarctic Lancaster Hopkins calculated startled ##star tribal Amendment ##hoe invisible patron deer Walk tracking Lyon tickets ##ED philosopher compounds chuckled ##wi pound loyalty Academic petition refuses marking Mercury northeastern dimensions scandal Canyon patch publish ##oning Peak minds ##boro Presbyterian Hardy theoretical magnitude bombs cage ##ders ##kai measuring explaining avoiding touchdowns Card theology ##ured Popular export suspicious Probably photograph Lou Parks Arms compact Apparently excess Banks lied stunned territorial Filipino spectrum learns wash imprisonment ugly ##rose Albany Erik sends ##hara ##rid consumed ##gling Belgrade Da opposing Magnus footsteps glowing delicate Alexandria Ludwig gorgeous Bros Index ##PA customs preservation bonds ##mond environments ##nto instructed parted adoption locality workshops goalkeeper ##rik ##uma Brighton Slovenia ##ulating ##tical towel hugged stripped Bears upright Wagner ##aux secretly Adventures nest Course Lauren Boeing Abdul Lakes 450 ##cu USSR caps Chan ##nna conceived Actually Belfast Lithuanian concentrate possess militia pine protagonist Helena ##PS ##band Belle Clara Reform currency pregnancy 1500 ##rim Isabella hull Name trend journalism diet ##mel Recording acclaimed Tang Jace steering vacant suggestion costume laser ##š ##ink ##pan ##vić integral achievements wise classroom unions southwestern ##uer Garcia toss Tara Large ##tate evident responsibilities populated satisfaction ##bia casual Ecuador ##ght arose ##ović Cornwall embrace refuse Heavyweight XI Eden activists ##uation biology ##shan fraud Fuck matched legacy Rivers missionary extraordinary Didn holder wickets crucial Writers Hurricane Iceland gross trumpet accordance hurry flooded doctorate Albania ##yi united deceased jealous grief flute portraits ##а pleasant Founded Face crowned Raja advisor Salem ##ec Achievement admission freely minimal Sudan developers estimate disabled ##lane downstairs Bruno ##pus pinyin ##ude lecture deadly underlying optical witnesses Combat Julius tapped variants ##like Colonial Critics Similarly mouse voltage sculptor Concert salary Frances ##ground hook premises Software instructor nominee ##ited fog slopes ##zu vegetation sail ##rch Body Apart atop View utility ribs cab migration ##wyn bounded 2019 pillow trails ##ub Halifax shade Rush ##lah ##dian Notre interviewed Alexandra Springfield Indeed rubbing dozens amusement legally ##lers Jill Cinema ignoring Choice ##ures pockets ##nell laying Blair tackles separately ##teen Criminal performs theorem Communication suburbs ##iel competitors rows ##hai Manitoba Eleanor interactions nominations assassination ##dis Edmonton diving ##dine essay ##tas AFC Edge directing imagination sunk implement Theodore trembling sealed ##rock Nobel ##ancy ##dorf ##chen genuine apartments Nicolas AA Bach Globe Store 220 ##10 Rochester ##ño alert 107 Beck ##nin Naples Basin Crawford fears Tracy ##hen disk ##pped seventeen Lead backup reconstruction ##lines terrified sleeve nicknamed popped ##making ##ern Holiday Gospel ibn ##ime convert divine resolved ##quet ski realizing ##RT Legislature reservoir Rain sinking rainfall elimination challenging tobacco ##outs Given smallest Commercial pin rebel comedian exchanged airing dish Salvador promising ##wl relax presenter toll aerial ##eh Fletcher brass disappear zones adjusted contacts ##lk sensed Walt mild toes flies shame considers wildlife Hanna Arsenal Ladies naming ##ishing anxiety discussions cute undertaken Cash strain Wyoming dishes precise Angela ##ided hostile twins 115 Built ##pel Online tactics Newman ##bourne unclear repairs embarrassed listing tugged Vale ##gin Meredith bout ##cle velocity tips froze evaluation demonstrate ##card criticised Nash lineup Rao monks bacteria lease ##lish frightened den revived finale ##rance flee Letters decreased ##oh Sounds wrap Sharon incidents renovated everybody stole Bath boxing 1815 withdraw backs interim react murders Rhodes Copa framed flown Estonia Heavy explored ##rra ##GA ##ali Istanbul 1834 ##rite ##aging ##ues Episcopal arc orientation Maxwell infected ##rot BCE Brook grasp Roberto Excellence 108 withdrawal Marines rider Lo ##sin ##run Subsequently garrison hurricane facade Prussia crushed enterprise ##mber Twitter Generation Physical Sugar editing communicate Ellie ##hurst Ernst wagon promotional conquest Parliamentary courtyard lawyers Superman email Prussian lately lecturer Singer Majesty Paradise sooner Heath slot curves convoy ##vian induced synonym breeze ##plane ##ox peered Coalition ##hia odds ##esh ##lina Tomorrow Nadu ##ico ##rah damp autonomous console Victory counts Luxembourg intimate Archived Carroll spy Zero habit Always faction teenager Johnston chaos ruin commerce blog ##shed ##the reliable Word Yu Norton parade Catholics damned ##iling surgeon ##tia Allison Jonas remarked ##ès idiot Making proposals Industries strategies artifacts batteries reward ##vers Agricultural distinguish lengths Jeffrey Progressive kicking Patricia ##gio ballot ##ios skilled ##gation Colt limestone ##AS peninsula ##itis LA hotels shapes Crime depicting northwestern HD silly Das ##² ##ws ##ash ##matic thermal Has forgive surrendered Palm Nacional drank haired Mercedes ##foot loading Timothy ##roll mechanisms traces digging discussing Natalie ##zhou Forbes landmark Anyway Manor conspiracy gym knocking viewing Formation Pink Beauty limbs Phillip sponsor Joy granite Harbour ##ero payments Ballet conviction ##dam Hood estimates lacked Mad Jorge ##wen refuge ##LA invaded Kat suburban ##fold investigated Ari complained creek Georges ##uts powder accepting deserved carpet Thunder molecules Legal cliff strictly enrollment ranch ##rg ##mba proportion renovation crop grabbing ##liga finest entries receptor helmet blown Listen flagship workshop resolve nails Shannon portal jointly shining Violet overwhelming upward Mick proceedings ##dies ##aring Laurence Churchill ##rice commit 170 inclusion Examples ##verse ##rma fury paths ##SC ankle nerves Chemistry rectangular sworn screenplay cake Mann Seoul Animal sizes Speed vol Population Southwest Hold continuously Qualified wishing Fighting Made disappointment Portsmouth Thirty ##beck Ahmad teammate MLB graph Charleston realizes ##dium exhibits preventing ##int fever rivalry Male mentally dull ##lor ##rich consistently ##igan Madame certificate suited Krishna accuracy Webb Budapest Rex 1831 Cornell OK surveillance ##gated habitats Adventure Conrad Superior Gay sofa aka boot Statistics Jessie Liberation ##lip ##rier brands saint Heinrich Christine bath Rhine ballet Jin consensus chess Arctic stack furious cheap toy ##yre ##face ##gging gastropod ##nne Romans membrane answering 25th architects sustainable ##yne Hon 1814 Baldwin dome ##awa ##zen celebrity enclosed ##uit ##mmer Electronic locals ##CE supervision mineral Chemical Slovakia alley hub ##az heroes Creative ##AM incredible politically ESPN yanked halls Aboriginal Greatest yield ##20 congressional robot Kiss welcomed MS speeds proceed Sherman eased Greene Walsh Geoffrey variables rocky ##print acclaim Reverend Wonder tonnes recurring Dawson continent finite AP continental ID facilitate essays Rafael Neal 1833 ancestors ##met ##gic Especially teenage frustrated Jules cock expense ##oli ##old blocking Notable prohibited ca dock organize ##wald Burma Gloria dimension aftermath choosing Mickey torpedo pub ##used manuscripts laps Ulster staircase sphere Insurance Contest lens risks investigations ERA glare ##play Graduate auction Chronicle ##tric ##50 Coming seating Wade seeks inland Thames Rather butterfly contracted positioned consumers contestants fragments Yankees Santos administrator hypothesis retire Denis agreements Winnipeg ##rill 1820 trophy crap shakes Jenkins ##rium ya twist labels Maritime ##lings ##iv 111 ##ensis Cairo Anything ##fort opinions crowded ##nian abandon ##iff drained imported ##rr tended ##rain Going introducing sculptures bankruptcy danced demonstration stance settings gazed abstract pet Calvin stiff strongest wrestler ##dre Republicans grace allocated cursed snail advancing Return errors Mall presenting eliminate Amateur Institution counting ##wind warehouse ##nde Ethiopia trailed hollow ##press Literary capability nursing preceding lamp Thomson Morton ##ctic Crew Close composers boom Clare missiles 112 hunter snap ##oni ##tail Us declaration ##cock rally huh lion straightened Philippe Sutton alpha valued maker navigation detected favorable perception Charter ##ña Ricky rebounds tunnels slapped Emergency supposedly ##act deployment socialist tubes anybody corn ##NA Seminary heating pump ##AA achieving souls ##ass Link ##ele ##smith greeted Bates Americas Elder cure contestant 240 fold Runner Uh licked Politics committees neighbors fairy Silva Leipzig tipped correctly exciting electronics foundations cottage governmental ##hat allied claws presidency cruel Agreement slender accompanying precisely ##pass driveway swim Stand crews ##mission rely everyday Wings demo ##hic recreational min nationality ##duction Easter ##hole canvas Kay Leicester talented Discovery shells ##ech Kerry Ferguson Leave ##place altogether adopt butt wolves ##nsis ##ania modest soprano Boris ##ught electron depicts hid cruise differ treasure ##nch Gun Mama Bengali trainer merchants innovation presumably Shirley bottles proceeds Fear invested Pirates particle Dominic blamed Fight Daisy ##pper ##graphic nods knight Doyle tales Carnegie Evil Inter Shore Nixon transform Savannah ##gas Baltic stretching worlds protocol Percy Toby Heroes brave dancers ##aria backwards responses Chi Gaelic Berry crush embarked promises Madonna researcher realised inaugurated Cherry Mikhail Nottingham reinforced subspecies rapper ##kie Dreams Re Damon Minneapolis monsters suspicion Tel surroundings afterward complaints OF sectors Algeria lanes Sabha objectives Donna bothered distracted deciding ##ives ##CA ##onia bishops Strange machinery Voiced synthesis reflects interference ##TS ##ury keen ##ign frown freestyle ton Dixon Sacred Ruby Prison ##ión 1825 outfit ##tain curiosity ##ight frames steadily emigrated horizon ##erly Doc philosophical Table UTC Marina ##DA secular ##eed Zimbabwe cops Mack sheriff Sanskrit Francesco catches questioning streaming Kill testimony hissed tackle countryside copyright ##IP Buddhism ##rator ladder ##ON Past rookie depths ##yama ##ister ##HS Samantha Dana Educational brows Hammond raids envelope ##sco ##hart ##ulus epic detection Streets Potter statistical für ni accounting ##pot employer Sidney Depression commands Tracks averaged lets Ram longtime suits branded chip Shield loans ought Said sip ##rome requests Vernon bordered veterans ##ament Marsh Herzegovina Pine ##igo mills anticipation reconnaissance ##ef expectations protested arrow guessed depot maternal weakness ##ap projected pour Carmen provider newer remind freed ##rily ##wal ##tones intentions Fiji timing Match managers Kosovo Herman Wesley Chang 135 semifinals shouting Indo Janeiro Chess Macedonia Buck ##onies rulers Mail ##vas ##sel MHz Programme Task commercially subtle propaganda spelled bowling basically Raven 1828 Colony 109 ##ingham ##wara anticipated 1829 ##iers graduates ##rton ##fication endangered ISO diagnosed ##tage exercises Battery bolt poison cartoon ##ción hood bowed heal Meyer Reagan ##wed subfamily ##gent momentum infant detect ##sse Chapman Darwin mechanics NSW Cancer Brooke Nuclear comprised hire sanctuary wingspan contrary remembering surprising Basic stealing OS hatred ##lled masters violation Rule ##nger assuming conquered louder robe Beatles legitimate ##vation massacre Rica unsuccessfully poets ##enberg careers doubled premier battalions Dubai Paper Louisville gestured dressing successive mumbled Vic referee pupil ##cated ##rre ceremonies picks ##IN diplomat alike geographical rays ##HA ##read harbour factories pastor playwright Ultimate nationalist uniforms obtaining kit Amber ##pling screenwriter ancestry ##cott Fields PR Coleman rat Bavaria squeeze highlighted Adult reflecting Mel 1824 bicycle organizing sided Previously Underground Prof athletics coupled mortal Hampton worthy immune Ava ##gun encouraging simplified ##ssa ##nte ##ann Providence entities Pablo Strong Housing ##ista ##ators kidnapped mosque Kirk whispers fruits shattered fossil Empress Johns Webster Thing refusing differently specimen Ha ##EN ##tina ##elle ##night Horn neighbourhood Bolivia ##rth genres Pre ##vich Amelia swallow Tribune Forever Psychology Use ##bers Gazette ash ##usa Monster ##cular delegation blowing Oblast retreated automobile ##ex profits shirts devil Treasury ##backs Drums Ronnie gameplay expertise Evening resides Caesar unity Crazy linking Vision donations Isabel valve Sue WWE logical availability fitting revolt ##mill Linux taxi Access pollution statues Augustus ##pen cello ##some lacking ##ati Gwen ##aka ##ovich 1821 Wow initiatives Uruguay Cain stroked examine ##ī mentor moist disorders buttons ##tica ##anna Species Lynch museums scorer Poor eligibility op unveiled cats Title wheat critically Syracuse ##osis marketed enhance Ryder ##NG ##ull ##rna embedded throws foods happily ##ami lesson formats punched ##rno expressions qualities ##sal Gods ##lity elect wives ##lling jungle Toyota reversed Grammar Cloud Agnes ##ules disputed verses Lucien threshold ##rea scanned ##bled ##dley ##lice Kazakhstan Gardner Freeman ##rz inspection Rita accommodation advances chill Elliot thriller Constantinople ##mos debris whoever 1810 Santo Carey remnants Guatemala ##irs carriers equations mandatory ##WA anxious measurement Summit Terminal Erin ##zes LLC ##uo glancing sin ##₃ Downtown flowering Euro Leigh Lance warn decent recommendations ##ote Quartet ##rrell Clarence colleague guarantee 230 Clayton Beast addresses prospect destroyer vegetables Leadership fatal prints 190 ##makers Hyde persuaded illustrations Southampton Joyce beats editors mount ##grave Malaysian Bombay endorsed ##sian ##bee applying Religion nautical bomber Na airfield gravel ##rew Cave bye dig decree burden Election Hawk Fe ##iled reunited ##tland liver Teams Put delegates Ella ##fect Cal invention Castro bored ##kawa ##ail Trinidad NASCAR pond develops ##pton expenses Zoe Released ##rf organs beta parameters Neill ##lene lateral Beat blades Either ##hale Mitch ##ET ##vous Rod burnt phones Rising ##front investigating ##dent Stephanie ##keeper screening ##uro Swan Sinclair modes bullets Nigerian melody ##ques Rifle ##12 128 ##jin charm Venus ##tian fusion advocated visitor pinned genera 3000 Ferry Solo quantity regained platinum shoots narrowly preceded update ##ichi equality unaware regiments ally ##tos transmitter locks Seeing outlets feast reopened ##ows struggles Buddy 1826 bark elegant amused Pretty themed schemes Lisbon Te patted terrorism Mystery ##croft ##imo Madagascar Journey dealer contacted ##quez ITV vacation Wong Sacramento organisms ##pts balcony coloured sheer defines MC abortion forbidden accredited Newfoundland tendency entrepreneur Benny Tanzania needing finalist mythology weakened gown sentences Guest websites Tibetan UFC voluntary annoyed Welcome honestly correspondence geometry Deutsche Biology Help ##aya Lines Hector ##ael reluctant ##ages wears inquiry ##dell Holocaust Tourism Wei volcanic ##mates Visual sorts neighborhoods Running apple shy Laws bend Northeast feminist Speedway Murder visa stuffed fangs transmitted fiscal Ain enlarged ##ndi Cecil Peterson Benson Bedford acceptable ##CC ##wer purely triangle foster Alberto educator Highland acute LGBT Tina Mi adventures Davidson Honda translator monk enacted summoned ##ional collector Genesis Un liner Di Statistical ##CS filter Knox Religious Stella Estonian Turn ##ots primitive parishes ##lles complexity autobiography rigid cannon pursuing exploring ##gram ##mme freshman caves Expedition Traditional iTunes certification cooling ##ort ##gna ##IT ##lman ##VA Motion explosive licence boxer shrine loosely Brigadier Savage Brett MVP heavier ##elli ##gged Buddha Easy spells fails incredibly Georg stern compatible Perfect applies cognitive excessive nightmare neighbor Sicily appealed static ##₁ Aberdeen ##leigh slipping bride ##guard Um Clyde 1818 ##gible Hal Frost Sanders interactive Hour ##vor hurting bull termed shelf capturing ##pace rolls 113 ##bor Chilean teaches ##rey exam shipped Twin borrowed ##lift Shit ##hot Lindsay Below Kiev Lin leased ##sto Eli Diane Val subtropical shoe Bolton Dragons ##rification Vatican ##pathy Crisis dramatically talents babies ##ores surname ##AP ##cology cubic opted Archer sweep tends Karnataka Judy stint Similar ##nut explicitly ##nga interact Mae portfolio clinic abbreviated Counties ##iko hearts ##ı providers screams Individual ##etti Monument ##iana accessed encounters gasp ##rge defunct Avery ##rne nobility useless Phase Vince senator ##FL 1813 surprisingly ##illo ##chin Boyd rumors equity Gone Hearts chassis overnight Trek wrists submit civic designers ##rity prominence decorative derives starter ##AF wisdom Powers reluctantly measurements doctoral Noel Gideon Baden Cologne lawn Hawaiian anthology ##rov Raiders embassy Sterling ##pal Telugu troubled ##FC ##bian fountain observe ore ##uru ##gence spelling Border grinning sketch Benedict Xbox dialects readily immigrant Constitutional aided nevertheless SE tragedy ##ager ##rden Flash ##MP Europa emissions ##ield panties Beverly Homer curtain ##oto toilet Isn Jerome Chiefs Hermann supernatural juice integrity Scots auto Patriots Strategic engaging prosecution cleaned Byron investments adequate vacuum laughs ##inus ##nge Usually Roth Cities Brand corpse ##ffy Gas rifles Plains sponsorship Levi tray owed della commanders ##ead tactical ##rion García harbor discharge ##hausen gentleman endless highways ##itarian pleaded ##eta archive Midnight exceptions instances Gibraltar cart ##NS Darren Bonnie ##yle ##iva OCLC bra Jess ##EA consulting Archives Chance distances commissioner ##AR LL sailors ##sters enthusiasm Lang ##zia Yugoslav confirm possibilities Suffolk ##eman banner 1822 Supporting fingertips civilization ##gos technically 1827 Hastings sidewalk strained monuments Floyd Chennai Elvis villagers Cumberland strode albeit Believe planets combining Mohammad container ##mouth ##tures verb BA Tank Midland screened Gang Democracy Helsinki screens thread charitable ##version swiftly ma rational combine ##SS ##antly dragging Cliff Tasmania quest professionally ##aj rap ##lion livestock ##hua informal specially lonely Matthews Dictionary 1816 Observatory correspondent constitute homeless waving appreciated Analysis Meeting dagger ##AL Gandhi flank Giant Choir ##not glimpse toe Writer teasing springs ##dt Glory healthcare regulated complaint math Publications makers ##hips cement Need apologize disputes finishes Partners boring ups gains 1793 Congressional clergy Folk ##made ##nza Waters stays encoded spider betrayed Applied inception ##urt ##zzo wards bells UCLA Worth bombers Mo trademark Piper ##vel incorporates 1801 ##cial dim Twelve ##word Appeals tighter spacecraft ##tine coordinates ##iac mistakes Zach laptop Teresa ##llar ##yr favored Nora sophisticated Irving hammer División corporations niece ##rley Patterson UNESCO trafficking Ming balanced plaque Latvia broader ##owed Save confined ##vable Dalton tide ##right ##ural ##num swords caring ##eg IX Acting paved ##moto launching Antoine substantially Pride Philharmonic grammar Indoor Ensemble enabling 114 resided Angelo publicity chaired crawled Maharashtra Telegraph lengthy preference differential anonymous Honey ##itation wage ##iki consecrated Bryant regulatory Carr ##én functioning watches ##ú shifts diagnosis Search app Peters ##SE ##cat Andreas honours temper counsel Urdu Anniversary maritime ##uka harmony ##unk essence Lorenzo choked Quarter indie ##oll loses ##prints amendment Adolf scenario similarities ##rade ##LC technological metric Russians thoroughly ##tead cruiser 1806 ##nier 1823 Teddy ##psy au progressed exceptional broadcaster partnered fitness irregular placement mothers unofficial Garion Johannes 1817 regain Solar publishes Gates Broken thirds conversations dive Raj contributor quantities Worcester governance ##flow generating pretending Belarus ##voy radius skating Marathon 1819 affection undertook ##wright los ##bro locate PS excluded recreation tortured jewelry moaned ##logue ##cut Complete ##rop 117 ##II plantation whipped slower crater ##drome Volunteer attributes celebrations regards Publishers oath utilized Robbie Giuseppe fiber indication melted archives Damien storey affecting identifying dances alumni comparable upgrade rented sprint ##kle Marty ##lous treating railways Lebanese erupted occupy sympathy Jude Darling Qatar drainage McCarthy heel Klein computing wireless flip Du Bella ##ast ##ssen narrator mist sings alignment 121 2020 securing ##rail Progress missionaries brutal mercy ##shing Hip ##ache ##olo switching ##here Malay ##ob constituted Mohammed Often standings surge teachings ink detached systematic Trial Myanmar ##wo offs Reyes decoration translations wherever reviewer speculation Bangkok terminated ##ester beard RCA Aidan Associated Emerson Charity 1803 generous Dudley ATP ##haven prizes toxic gloves ##iles ##dos Turning myth Parade ##building Hits ##eva teamed Above Duchess Holt ##oth Sub Ace atomic inform Ship depend Jun ##bes Norwich globe Baroque Christina Cotton Tunnel kidding Concerto Brittany tasted phases stems angles ##TE ##nam ##40 charted Alison intensive Willis glory ##lit Bergen est taller ##dicate labeled ##ido commentator Warrior Viscount shortened aisle Aria Spike spectators goodbye overlooking mammals ##lude wholly Barrett ##gus accompany seventy employ ##mb ambitious beloved basket ##mma ##lding halted descendant pad exclaimed cloak ##pet Strait Bang Aviv sadness ##ffer Donovan 1880s agenda swinging ##quin jerk Boat ##rist nervously Silence Echo shout implies ##iser ##cking Shiva Weston damages ##tist effectiveness Horace cycling Rey ache Photography PDF Dear leans Lea ##vision booth attained disbelief ##eus ##ution Hop pension toys Eurovision faithful ##heads Andre owe default Atlas Megan highlights lovers Constantine Sixth masses ##garh emerge Auto Slovak ##oa ##vert Superintendent flicked inventor Chambers Frankie Romeo pottery companions Rudolf ##liers diary Unless tap alter Randall ##ddle ##eal limitations ##boards utterly knelt guaranteed Cowboys Islander horns ##ike Wendy sexually Smart breasts ##cian compromise Duchy AT Galaxy analog Style ##aking weighed Nigel optional Czechoslovakia practicing Ham ##0s feedback batted uprising operative applicable criminals classrooms Somehow ##ode ##OM Naomi Winchester ##pping Bart Regina competitor Recorded Yuan Vera lust Confederation ##test suck 1809 Lambert 175 Friend ##ppa Slowly ##⁺ Wake Dec ##aneous chambers Color Gus ##site Alternative ##world Exeter Omaha celebrities striker 210 dwarf meals Oriental Pearson financing revenues underwater Steele screw Feeling Mt acids badge swore theaters Moving admired lung knot penalties 116 fork ##cribed Afghan outskirts Cambodia oval wool fossils Ned Countess Darkness delicious ##nica Evelyn Recordings guidelines ##CP Sandra meantime Antarctica modeling granddaughter ##rial Roma Seventh Sunshine Gabe ##nton Shop Turks prolific soup parody ##nta Judith disciplines resign Companies Libya Jets inserted Mile retrieve filmmaker ##rand realistic unhappy ##30 sandstone ##nas ##lent ##ush ##rous Brent trash Rescue ##unted Autumn disgust flexible infinite sideways ##oss ##vik trailing disturbed 50th Newark posthumously ##rol Schmidt Josef ##eous determining menu Pole Anita Luc peaks 118 Yard warrant generic deserted Walking stamp tracked ##berger paired surveyed sued Rainbow ##isk Carpenter submarines realization touches sweeping Fritz module Whether resembles ##form ##lop unsure hunters Zagreb unemployment Senators Georgetown ##onic Barker foul commercials Dresden Words collision Carlton Fashion doubted ##ril precision MIT Jacobs mob Monk retaining gotta ##rod remake Fast chips ##pled sufficiently ##lights delivering ##enburg Dancing Barton Officers metals ##lake religions ##ré motivated differs dorsal ##birds ##rts Priest polished ##aling Saxony Wyatt knockout ##hor Lopez RNA ##link metallic ##kas daylight Montenegro ##lining wrapping resemble Jam Viking uncertainty angels enables ##fy Stuttgart tricks tattoo 127 wicked asset breach ##yman MW breaths Jung im 1798 noon vowel ##qua calmly seasonal chat ingredients cooled Randolph ensuring ##ib ##idal flashing 1808 Macedonian Cool councils ##lick advantages Immediately Madras ##cked Pain fancy chronic Malayalam begged ##nese Inner feathers ##vey Names dedication Sing pan Fischer nurses Sharp inning stamps Meg ##ello edged motioned Jacksonville ##ffle ##dic ##US divide garnered Ranking chasing modifications ##oc clever midst flushed ##DP void ##sby ambulance beaches groan isolation strengthen prevention ##ffs Scouts reformed geographic squadrons Fiona Kai Consequently ##uss overtime ##yas Fr ##BL Papua Mixed glances Haiti Sporting sandy confronted René Tanner 1811 ##IM advisory trim ##ibe González gambling Jupiter ##ility ##owski ##nar 122 apology teased Pool feminine wicket eagle shiny ##lator blend peaking nasty nodding fraction tech Noble Kuwait brushing Italia Canberra duet Johan 1805 Written cameo Stalin pig cord ##zio Surely SA owing holidays 123 Ranger lighthouse ##ige miners 1804 ##ë ##gren ##ried crashing ##atory wartime highlight inclined Torres Tax ##zel ##oud Own ##corn Divine EMI Relief Northwestern ethics BMW click plasma Christie coordinator Shepherd washing cooked ##dio ##eat Cerambycidae algebra Engine costumes Vampire vault submission virtue assumption ##rell Toledo ##oting ##rva crept emphasized ##lton ##ood Greeks surgical crest Patrol Beta Tessa ##GS pizza traits rats Iris spray ##GC Lightning binary escapes ##take Clary crowds ##zong hauled maid ##fen Manning ##yang Nielsen aesthetic sympathetic affiliation soaked Mozart personalities begging ##iga clip Raphael yearly Lima abundant ##lm 1794 strips Initiative reporters ##vsky consolidated ##itated Civic rankings mandate symbolic ##ively 1807 rental duck nave complications ##nor Irene Nazis haunted scholarly Pratt Gran Embassy Wave pity genius bats canton Tropical marker ##cos escorted Climate ##posed appreciation freezing puzzle Internal pools Shawn pathway Daniels Fitzgerald extant olive Vanessa marriages cocked ##dging prone chemicals doll drawer ##HF Stark Property ##tai flowed Sheridan ##uated Less Omar remarks catalogue Seymour wreck Carrie ##bby Mercer displaced sovereignty rip Flynn Archie Quarterfinals Hassan ##ards vein Osaka pouring wages Romance ##cript ##phere 550 ##eil ##stown Documentary ancestor CNN Panthers publishers Rise ##mu biting Bright String succeeding 119 loaned Warwick Sheikh Von Afterwards Jax Camden helicopters Hence Laurel ##ddy transaction Corp clause ##owing ##kel Investment cups Lucia Moss Giles chef López decisive 30th distress linguistic surveys Ready maiden Touch frontier incorporate exotic mollusk Leopold Ride ##wain ##ndo teammates tones drift ordering Feb Penny Normandy Present Flag pipes ##rro delight motto Tibet leap Eliza Produced teenagers sitcom Try Hansen Cody wandered terrestrial frog scare resisted employers coined ##DS resistant Fly captive dissolution judged associates defining ##court Hale ##mbo raises clusters twelfth ##metric Roads ##itude satisfy Android Reds Gloucester Category Valencia Daemon stabbed Luna Churches Canton ##eller Attack Kashmir annexed grabs asteroid Hartford recommendation Rodriguez handing stressed frequencies delegate Bones Erie Weber Hands Acts millimetres 24th Fat Howe casually ##SL convent 1790 IF ##sity 1795 yelling ##ises drain addressing amino Marcel Sylvia Paramount Gerard Volleyball butter 124 Albion ##GB triggered 1792 folding accepts ##ße preparations Wimbledon dose ##grass escaping ##tling import charging ##dation 280 Nolan ##fried Calcutta ##pool Cove examining minded heartbeat twisting domains bush Tunisia Purple Leone ##code evacuated battlefield tiger Electrical ##ared chased ##cre cultivated Jet solved shrug ringing Impact ##iant kilometre ##log commemorate migrated singular designing promptly Higgins ##own ##aves freshwater Marketing Payne beg locker pray implied AAA corrected Trans Europeans Ashe acknowledge Introduction ##writer ##llen Munster auxiliary growl Hours Poems ##AT reduces Plain plague canceled detention polite necklace Gustav ##gu ##lance En Angola ##bb dwelling ##hea 5000 Qing Dodgers rim ##ored ##haus spilled Elisabeth Viktor backpack 1802 amended ##worthy Phantom ##ctive keeper ##loom Vikings ##gua employs Tehran specialty ##bate Marx Mirror Jenna rides needle prayers clarinet forewings ##walk Midlands convincing advocacy Cao Birds cycles Clement Gil bubble Maximum humanitarian Tan cries ##SI Parsons Trio offshore Innovation clutched 260 ##mund ##duct Prairie relied Falcon ##ste Kolkata Gill Swift Negro Zoo valleys ##OL Opening beams MPs outline Bermuda Personal exceed productive ##MT republic forum ##sty tornado Known dipped Edith folks mathematician watershed Ricardo synthetic ##dication deity ##₄ gaming subjected suspects Foot swollen Motors ##tty ##ý aloud ceremonial es nuts intend Carlisle tasked hesitation sponsors unified inmates ##ctions ##stan tiles jokes whereby outcomes Lights scary Stoke Portrait Blind sergeant violations cultivation fuselage Mister Alfonso candy sticks teen agony Enough invite Perkins Appeal mapping undergo Glacier Melanie affects incomplete ##dd Colombian ##nate CBC purchasing bypass Drug Electronics Frontier Coventry ##aan autonomy scrambled Recent bounced cow experiencing Rouge cuisine Elite disability Ji inheritance wildly Into ##wig confrontation Wheeler shiver Performing aligned consequently Alexis Sin woodland executives Stevenson Ferrari inevitable ##cist ##dha ##base Corner comeback León ##eck ##urus MacDonald pioneering breakdown landscapes Veterans Rican Theological stirred participant Credit Hyderabad snails Claudia ##ocene compliance ##MI Flags Middlesex storms winding asserted er ##ault ##kal waking ##rates abbey Augusta tooth trustees Commodore ##uded Cunningham NC Witch marching Sword Same spiral Harley ##ahan Zack Audio 1890s ##fit Simmons Kara Veronica negotiated Speaking FIBA Conservatory formations constituencies explicit facial eleventh ##ilt villain ##dog ##case ##hol armored tin hairs ##umi ##rai mattress Angus cease verbal Recreation savings Aurora peers Monastery Airways drowned additions downstream sticking Shi mice skiing ##CD Raw Riverside warming hooked boost memorable posed treatments 320 ##dai celebrating blink helpless circa Flowers PM uncommon Oct Hawks overwhelmed Sparhawk repaired Mercy pose counterpart compare survives ##½ ##eum coordinate Lil grandchildren notorious Yi Judaism Juliet accusations 1789 floated marathon roar fortified reunion 145 Nov Paula ##fare ##toria tearing Cedar disappearance Si gifted scar 270 PBS Technologies Marvin 650 roller cupped negotiate ##erman passport tram miracle styled ##tier necessity Des rehabilitation Lara USD psychic wipe ##lem mistaken ##lov charming Rider pageant dynamics Cassidy ##icus defenses ##tadt ##vant aging ##inal declare mistress supervised ##alis ##rest Ashton submerged sack Dodge grocery ramp Teacher lineage imagery arrange inscriptions Organisation Siege combines pounded Fleming legends columnist Apostolic prose insight Arabian expired ##uses ##nos Alone elbows ##asis ##adi ##combe Step Waterloo Alternate interval Sonny plains Goals incorporating recruit adjoining Cheshire excluding marrying ducked Cherokee par ##inate hiking Coal ##bow natives ribbon Allies con descriptions positively ##lal defendant 22nd Vivian ##beat Weather possessions Date sweetheart inability Salisbury adviser ideology Nordic ##eu Cubs IP Administrative ##nick facto liberation Burnett Javier fashioned Electoral Turin theft unanimous Per 1799 Clan Hawkins Teachers ##wes Cameroon Parkway ##gment demolition atoms nucleus ##thi recovering ##yte ##vice lifts Must deposit Hancock Semi darkened Declaration moan muscular Myers attractions sauce simulation ##weed Alps barriers ##baum Barack galleries Min holders Greenwich donation Everybody Wolfgang sandwich Kendra Collegiate casino Slavic ensuing Porto ##grapher Jesuit suppressed tires Ibrahim protesters Ibn Amos 1796 phenomena Hayden Paraguay Squad Reilly complement aluminum ##eers doubts decay demise Practice patience fireplace transparent monarchy ##person Rodney mattered rotating Clifford disposal Standards paced ##llie arise tallest tug documentation node freeway Nikolai ##cite clicked imaging Lorraine Tactical Different Regular Holding 165 Pilot guarded ##polis Classics Mongolia Brock monarch cellular receptors Mini Chandler financed financially Lives erection Fuller unnamed Kannada cc passive plateau ##arity freak ##rde retrieved transactions ##sus 23rd swimmer beef fulfill Arlington offspring reasoning Rhys saves pseudonym centimetres shivered shuddered ##ME Feel ##otic professors Blackburn ##eng ##life ##haw interred lodge fragile Della guardian ##bbled catalog clad observer tract declaring ##headed Lok dean Isabelle 1776 irrigation spectacular shuttle mastering ##aro Nathaniel Retired ##lves Brennan ##kha dick ##dated ##hler Rookie leapt televised weekends Baghdad Yemen ##fo factions ion Lab mortality passionate Hammer encompasses confluence demonstrations Ki derivative soils ##unch Ranch Universities conventions outright aiming hierarchy reside illusion graves rituals 126 Antwerp Dover ##ema campuses Hobart lifelong aliens ##vity Memory coordination alphabet ##mina Titans pushes Flanders ##holder Normal excellence capped profound Taipei portrayal sparked scratch se ##eas ##hir Mackenzie ##cation Neo Shin ##lined magnificent poster batsman ##rgent persuade ##ement Icelandic miserable collegiate Feature geography ##mura Comic Circus processor barracks Tale ##11 Bulls ##rap strengthened ##bell injection miniature broadly Letter fare hostage traders ##nium ##mere Fortune Rivera Lu triumph Browns Bangalore cooperative Basel announcing Sawyer ##him ##cco ##kara darted ##AD ##nova sucking ##position perimeter flung Holdings ##NP Basque sketches Augustine Silk Elijah analyst armour riots acquiring ghosts ##ems 132 Pioneer Colleges Simone Economy Author semester Soldier il ##unting ##bid freaking Vista tumor ##bat murderer ##eda unreleased ##grove ##sser ##té edit statute sovereign ##gawa Killer stares Fury comply ##lord ##nant barrels Andhra Maple generator mascot unusually eds ##ante ##runner rod ##tles Historically Jennings dumped Established resemblance ##lium ##cise ##body ##voke Lydia ##hou ##iring nonetheless 1797 corrupt patrons physicist sneak Livingston Citizens Architects Werner trends Melody eighty markings brakes ##titled oversaw processed mock Midwest intervals ##EF stretches werewolf ##MG Pack controller ##dition Honours cane Griffith vague repertoire Courtney orgasm Abdullah dominance occupies Ya introduces Lester instinct collaborative Indigenous refusal ##rank outlet debts spear 155 ##keeping ##ulu Catalan ##osh tensions ##OT bred crude Dunn abdomen accurately ##fu ##lough accidents Row Audrey rude Getting promotes replies Paolo merge ##nock trans Evangelical automated Canon ##wear ##ggy ##gma Broncos foolish icy Voices knives Aside dreamed generals molecule AG rejection insufficient ##nagar deposited sacked Landing arches helpful devotion intake Flower PGA dragons evolutionary ##mail 330 GM tissues ##tree arcade composite lid Across implications lacks theological assessed concentrations Den ##mans ##ulous Fu homeland ##stream Harriet ecclesiastical troop ecological winked ##xed eighteenth Casino specializing ##sworth unlocked supreme devastated snatched trauma GDP Nord saddle Wes convenient competes ##nu ##iss Marian subway ##rri successes umbrella ##far ##ually Dundee ##cence spark ##rix ##я Quality Geological cockpit rpm Cam Bucharest riot ##PM Leah ##dad ##pose Ka m³ Bundesliga Wolfe grim textile quartet expressing fantastic destroyers eternal picnic ##oro contractor 1775 spanning declining ##cating Lowe Sutherland Emirates downward nineteen violently scout viral melting enterprises ##cer Crosby Jubilee antenna urgent Rory ##uin ##sure wandering ##gler ##vent Suzuki Lifetime Dirty occupying ##quent Disc Guru mound Lennon Humanities listeners Walton uh Braves Bologna ##bis ##gra Dwight crawl flags memoir Thorne Archdiocese dairy ##uz ##tery roared adjust patches inn Knowing ##bbed ##zan scan Papa precipitation angrily passages postal Phi embraced blacks economist triangular Sen shooter punished Millennium Swimming confessed Aston defeats Era cousins Williamson ##rer daytime dumb ##rek underway specification Buchanan prayed concealed activation ##issa canon awesome Starr plural summers ##fields Slam unnecessary 1791 resume trilogy compression ##rough selective dignity Yan ##xton immense ##yun lone seeded hiatus lightweight summary Yo approve Galway rejoined Elise garbage burns speeches 129 Honduras ##liness inventory jersey FK assure slumped Lionel Suite ##sbury Lena continuation ##AN brightly ##nti GT Knowledge ##park ##lius lethal ##tribution ##sions Certificate Mara ##lby algorithms Jade blows pirates fleeing wheelchair Stein sophomore Alt Territorial diploma snakes ##olic ##tham Tiffany Pius flush urging Hanover Reich ##olate Unity Pike collectively Theme ballad kindergarten rocked zoo ##page whip Rodríguez strokes checks Becky Stern upstream ##uta Silent volunteered Sigma ##ingen ##tract ##ede Gujarat screwed entertaining ##action ##ryn defenders innocence lesbian que Richie nodes Lie juvenile Jakarta safer confront Bert breakthrough gospel Cable ##zie institutional Archive brake liquor feeds ##iate chancellor Encyclopedia Animation scanning teens ##mother Core Rear Wine ##flower reactor Ave cardinal sodium strands Olivier crouched Vaughan Sammy Image scars Emmanuel flour bias nipple revelation ##ucci Denny ##ssy Form Runners admits Rama violated Burmese feud underwear Mohamed Named swift statewide Door Recently comparing Hundred ##idge ##nity ##rds Rally Reginald Auburn solving waitress Treasurer ##ilization Halloween Ministers Boss Shut ##listic Rahman demonstrating ##pies Gaza Yuri installations Math schooling ##bble Bronx exiled gasoline 133 bundle humid FCC proportional relate VFL ##dez continuity ##cene syndicated atmospheric arrows Wanderers reinforcements Willow Lexington Rotten ##yon discovering Serena portable ##lysis targeting £1 Goodman Steam sensors detachment Malik ##erie attitudes Goes Kendall Read Sleep beans Nikki modification Jeanne knuckles Eleven ##iously Gross Jaime dioxide moisture Stones UCI displacement Metacritic Jury lace rendering elephant Sergei ##quire GP Abbott ##type projection Mouse Bishops whispering Kathleen Rams ##jar whites ##oran assess dispatched ##hire kin ##mir Nursing advocates tremendous sweater assisting ##bil Farmer prominently reddish Hague cyclone ##SD Sage Lawson Sanctuary discharged retains ##ube shotgun wilderness Reformed similarity Entry Watts Bahá Quest Looks visions Reservoir Arabs curls Blu dripping accomplish Verlag drill sensor Dillon physicians smashed ##dir painters Renault straw fading Directorate lounge commissions Brain ##graph neo ##urg plug coordinated ##houses Critical lamps illustrator Returning erosion Crow ##ciation blessing Thought Wife medalist synthesizer Pam Thornton Esther HBO fond Associates ##raz pirate permits Wide tire ##PC Ernie Nassau transferring RFC ##ntly um spit AS ##mps Mining polar villa anchored ##zzi embarrassment relates ##ă Rupert counterparts 131 Baxter ##18 Igor recognizes Clive ##hane ##eries ##ibly occurrence ##scope fin colorful Rapids banker tile ##rative ##dus delays destinations ##llis Pond Dane grandparents rewarded socially motorway ##hof ##lying ##human modeled Dayton Forward conscience Sharma whistle Mayer Sasha ##pical circuits Zhou ##ça Latvian finalists predators Lafayette closes obligations Resolution ##vier Trustees reminiscent ##hos Highlands Protected asylum evacuation ##acy Chevrolet confession Somalia emergence separating ##rica alright calcium Laurent Welfare Leonardo ashes dental Deal minerals ##lump ##mount accounted staggered slogan photographic builder ##imes ##raft tragic 144 SEC Hit tailed ##ples ##rring ##rson ethical wrestlers concludes lunar ##ept nitrogen Aid cyclist quarterfinals ##ه harvest ##hem Pasha IL ##mis continually ##forth Intel bucket ##ended witches pretended dresses viewer peculiar lowering volcano Marilyn Qualifier clung ##sher Cut modules Bowie ##lded onset transcription residences ##pie ##itor scrapped ##bic Monaco Mayo eternity Strike uncovered skeleton ##wicz Isles bug Promoted ##rush Mechanical XII ##ivo gripping stubborn velvet TD decommissioned operas spatial unstable Congressman wasted ##aga ##ume advertisements ##nya obliged Cannes Conway bricks ##gnant ##mity ##uise jumps Clear ##cine ##sche chord utter Su podium spokesman Royce assassin confirmation licensing liberty ##rata Geographic individually detained ##ffe Saturn crushing airplane bushes knights ##PD Lilly hurts unexpectedly Conservatives pumping Forty candle Pérez peasants supplement Sundays ##ggs ##rries risen enthusiastic corresponds pending ##IF Owens floods Painter inflation presumed inscribed Chamberlain bizarre 1200 liability reacted tub Legacy ##eds ##pted shone ##litz ##NC Tiny genome bays Eduardo robbery stall hatch Depot Variety Flora reprinted trembled outlined CR Theresa spans ##plication Jensen ##eering posting ##rky pays ##ost Marcos fortifications inferior ##ential Devi despair Talbot ##chus updates ego Booth Darius tops ##lau Scene ##DC Harlem Trey Generally candles ##α Neville Admiralty ##hong iconic victorious 1600 Rowan abundance miniseries clutching sanctioned ##words obscure ##ision ##rle ##EM disappearing Resort Obviously ##eb exceeded 1870s Adults ##cts Cry Kerr ragged selfish ##lson circled pillars galaxy ##asco ##mental rebuild caution Resistance Start bind splitting Baba Hogan ps partnerships slam Peggy courthouse ##OD organizational packages Angie ##nds possesses ##rp Expressway Gould Terror Him Geoff nobles ##ope shark ##nh identifies ##oor testified Playing ##ump ##isa stool Idol ##pice ##tana Byrne Gerry grunted 26th observing habits privilege immortal wagons ##thy dot Bring ##lian ##witz newest ##uga constraints Screen Issue ##RNA ##vil reminder ##gles addiction piercing stunning var ##rita Signal accumulated ##wide float devastating viable cartoons Uttar flared ##encies Theology patents ##bahn privileges ##ava ##CO 137 ##oped ##NT orchestral medication 225 erect Nadia École fried Sales scripts ##rease airs Cage inadequate structured countless Avengers Kathy disguise mirrors Investigation reservation ##nson Legends humorous Mona decorations attachment Via motivation Browne strangers ##ński Shadows Twins ##pressed Alma Nominated ##ott Sergio canopy 152 Semifinals devised ##irk upwards Traffic Goddess Move beetles 138 spat ##anne holdings ##SP tangled Whilst Fowler anthem ##ING ##ogy snarled moonlight songwriting tolerance Worlds exams ##pia notices sensitivity poetic Stephens Boone insect reconstructed Fresh 27th balloon ##ables Brendan mug ##gee 1780 apex exports slides Lahore hiring Shell electorate sexuality poker nonprofit ##imate cone ##uce Okinawa superintendent ##HC referenced turret Sprint Citizen equilibrium Stafford curb Driver Valerie ##rona aching impacts ##bol observers Downs Shri ##uth airports ##uda assignments curtains solitary icon patrols substances Jasper mountainous Published ached ##ingly announce dove damaging ##tism Primera Dexter limiting batch ##uli undergoing refugee Ye admiral pavement ##WR ##reed pipeline desires Ramsey Sheila thickness Brotherhood Tea instituted Belt Break plots ##ais masculine ##where Theo ##aged ##mined Experience scratched Ethiopian Teaching ##nov Aiden Abe Samoa conditioning ##mous Otherwise fade Jenks ##encing Nat ##lain Anyone ##kis smirk Riding ##nny Bavarian blessed potatoes Hook ##wise likewise hardened Merry amid persecution ##sten Elections Hoffman Pitt ##vering distraction exploitation infamous quote averaging healed Rhythm Germanic Mormon illuminated guides ##ische interfere ##ilized rector perennial ##ival Everett courtesy ##nham Kirby Mk ##vic Medieval ##tale Luigi limp ##diction Alive greeting shove ##force ##fly Jasmine Bend Capt Suzanne ditch 134 ##nning Host fathers rebuilding Vocal wires ##manship tan Factor fixture ##LS Māori Plate pyramid ##umble slap Schneider yell ##ulture ##tional Goodbye sore ##pher depressed ##dox pitching Find Lotus ##wang strand Teen debates prevalent ##bilities exposing hears billed ##rse reorganized compelled disturbing displaying ##tock Clinical emotionally ##iah Derbyshire grouped ##quel Bahrain Journalism IN persistent blankets Crane camping Direct proving Lola ##dding Corporate birthplace ##boats ##ender Figure dared Assam precursor ##nched Tribe Restoration slate Meyrick hunted stroking Earlier Kind polls appeals monetary ##reate Kira Langdon explores GPS extensions squares Results draped announcer merit ##ennial ##tral ##roved ##cion robots supervisor snorted ##group Cannon procession monkey freeze sleeves Nile verdict ropes firearms extraction tensed EC Saunders ##tches diamonds Marriage ##amble curling Amazing ##haling unrelated ##roads Daughter cum discarded kidney cliffs forested Candy ##lap authentic tablet notation ##nburg Bulldogs Callum Meet mouths coated ##xe Truman combinations ##mation Steelers Fan Than paternal ##father ##uti Rebellion inviting Fun theatres ##ي ##rom curator ##cision networking Oz drought ##ssel granting MBA Shelby Elaine jealousy Kyoto shores signaling tenants debated Intermediate Wise ##hes ##pu Havana duke vicious exited servers Nonetheless Reports explode ##beth Nationals offerings Oval conferred eponymous folklore ##NR Shire planting 1783 Zeus accelerated Constable consuming troubles McCartney texture bust Immigration excavated hopefully ##cession ##coe ##name ##ully lining Einstein Venezuelan reissued minorities Beatrice crystals ##nies circus lava Beirut extinction ##shu Becker ##uke issuing Zurich extract ##esta ##rred regulate progression hut alcoholic plea AB Norse Hubert Mansfield ashamed ##put Bombardment stripes electrons Denise horrified Nor arranger Hay Koch ##ddling ##iner Birthday Josie deliberate explorer ##jiang ##signed Arrow wiping satellites baritone mobility ##rals Dorset turbine Coffee 185 ##lder Cara Colts pits Crossing coral ##birth Tai zombie smoothly ##hp mates ##ady Marguerite ##tary puzzled tapes overly Sonic Prayer Thinking ##uf IEEE obligation ##cliffe Basil redesignated ##mmy nostrils Barney XIII ##phones vacated unused Berg ##roid Towards viola 136 Event subdivided rabbit recruiting ##nery Namibia ##16 ##ilation recruits Famous Francesca ##hari Goa ##lat Karachi haul biblical ##cible MGM ##rta horsepower profitable Grandma importantly Martinez incoming ##kill beneficial nominal praying ##isch gable nail noises ##ttle Polytechnic rub ##cope Thor audition erotic ##ending ##iano Ultimately armoured ##mum presently pedestrian ##tled Ipswich offence ##ffin ##borne Flemish ##hman echo ##cting auditorium gentlemen winged ##tched Nicaragua Unknown prosperity exhaust pie Peruvian compartment heights disabilities ##pole Harding Humphrey postponed moths Mathematical Mets posters axe ##nett Nights Typically chuckle councillors alternating 141 Norris ##ately ##etus deficit dreaming cooler oppose Beethoven ##esis Marquis flashlight headache investor responding appointments ##shore Elias ideals shades torch lingering ##real pier fertile Diploma currents Snake ##horse ##15 Briggs ##ota ##hima ##romatic Coastal Kuala ankles Rae slice Hilton locking Approximately Workshop Niagara strangely ##scence functionality advertisement Rapid Anders ho Soviets packing basal Sunderland Permanent ##fting rack tying Lowell ##ncing Wizard mighty tertiary pencil dismissal torso grasped ##yev Sand gossip ##nae Beer implementing ##19 ##riya Fork Bee ##eria Win ##cid sailor pressures ##oping speculated Freddie originating ##DF ##SR ##outh 28th melt Brenda lump Burlington USC marginal ##bine Dogs swamp cu Ex uranium metro spill Pietro seize Chorus partition ##dock ##media engineered ##oria conclusions subdivision ##uid Illustrated Leading ##hora Berkshire definite ##books ##cin ##suke noun winced Doris dissertation Wilderness ##quest braced arbitrary kidnapping Kurdish ##but clearance excavations wanna Allmusic insult presided yacht ##SM Honour Tin attracting explosives Gore Bride ##ience Packers Devils Observer ##course Loser ##erry ##hardt ##mble Cyrillic undefeated ##stra subordinate ##ame Wigan compulsory Pauline Cruise Opposition ##ods Period dispersed expose ##60 ##has Certain Clerk Wolves ##hibition apparatus allegiance orbital justified thanked ##ević Biblical Carolyn Graves ##tton Hercules backgrounds replica 1788 aquatic Mega Stirling obstacles filing Founder vowels Deborah Rotterdam surpassed Belarusian ##ologists Zambia Ren Olga Alpine bi councillor Oaks Animals eliminating digit Managing ##GE laundry ##rdo presses slamming Tudor thief posterior ##bas Rodgers smells ##ining Hole SUV trombone numbering representations Domingo Paralympics cartridge ##rash Combined shelves Kraków revision ##frame Sánchez ##tracted ##bler Alain townships sic trousers Gibbs anterior symmetry vaguely Castile IRA resembling Penguin ##ulent infections ##stant raped ##pressive worrying brains bending JR Evidence Venetian complexes Jonah 850 exported Ambrose Gap philanthropist ##atus Marxist weighing ##KO ##nath Soldiers chiefs reject repeating shaky Zürich preserving ##xin cigarettes ##break mortar ##fin Already reproduction socks Waiting amazed ##aca dash ##path Airborne ##harf ##get descending OBE Sant Tess Lucius enjoys ##ttered ##ivation ##ete Leinster Phillies execute geological unfinished Courts SP Beaver Duck motions Platinum friction ##aud ##bet Parts Stade entirety sprang Smithsonian coffin prolonged Borneo ##vise unanimously ##uchi Cars Cassandra Australians ##CT ##rgen Louisa spur Constance ##lities Patent racism tempo ##ssion ##chard ##nology ##claim Million Nichols ##dah Numerous ing Pure plantations donor ##EP ##rip convenience ##plate dots indirect ##written Dong failures adapt wizard unfortunately ##gion practitioners economically Enrique unchanged kingdoms refined definitions lazy worries railing ##nay Kaiser ##lug cracks sells ninety ##WC Directed denotes developmental papal unfortunate disappointing sixteenth Jen ##urier NWA drifting Horror ##chemical behaviors bury surfaced foreigners slick AND ##rene ##ditions ##teral scrap kicks comprise buddy ##anda Mental ##ype Dom wines Limerick Luca Rand ##won Tomatoes homage geometric ##nted telescope Shelley poles ##fan shareholders Autonomous cope intensified Genoa Reformation grazing ##tern Zhao provisional ##bies Con ##riel Cynthia Raleigh vivid threaten Length subscription roses Müller ##isms robin ##tial Laos Stanton nationalism ##clave ##ND ##17 ##zz staging Busch Cindy relieve ##spective packs neglected CBE alpine Evolution uneasy coastline Destiny Barber Julio ##tted informs unprecedented Pavilion ##bei ##ference betrayal awaiting leaked V8 puppet adverse Bourne Sunset collectors ##glass ##sque copied Demon conceded resembled Rafe Levy prosecutor ##ject flora manned deaf Mosque reminds Lizzie Products Funny cassette congress ##rong Rover tossing prompting chooses Satellite cautiously Reese ##UT Huang Gloucestershire giggled Kitty ##å Pleasant Aye ##ond judging 1860s intentionally Hurling aggression ##xy transfers employing ##fies ##oda Archibald Blessed Ski flavor Rosie ##burgh sunset Scholarship WC surround ranged ##jay Degree Houses squeezing limb premium Leningrad steals ##inated ##ssie madness vacancy hydraulic Northampton ##prise Marks Boxing ##fying academics ##lich ##TY CDs ##lma hardcore monitors paperback cables Dimitri upside advent Ra ##clusive Aug Christchurch objected stalked Simple colonists ##laid CT discusses fellowship Carnival cares Miracle pastoral rooted shortage borne Quentin meditation tapping Novel ##ades Alicia Burn famed residency Fernández Johannesburg Zhu offended Mao outward ##inas XV denial noticing ##ís quarry ##hound ##amo Bernie Bentley Joanna mortgage ##rdi ##sumption lenses extracted depiction ##RE Networks Broad Revenue flickered virgin flanked ##о Enterprises probable Liberals Falcons drowning phrases loads assumes inhaled awe logs slightest spiders waterfall ##pate rocking shrub ##uil roofs ##gard prehistoric wary ##rak TO clips sustain treason microphone voter Lamb psychologist wrinkled ##ères mating Carrier 340 ##lbert sensing ##rino destiny distract weaker UC Nearly neurons spends Apache ##rem genuinely wells ##lanted stereo ##girl Lois Leaving consul fungi Pier Cyril 80s Jungle ##tani illustration Split ##hana Abigail ##patrick 1787 diminished Selected packaging ##EG Martínez communal Manufacturing sentiment 143 unwilling praising Citation pills ##iti ##rax muffled neatly workforce Yep leisure Tu ##nding Wakefield ancestral ##uki destructive seas Passion showcase ##ceptive heroic 142 exhaustion Customs ##aker Scholar sliced ##inian Direction ##OW Swansea aluminium ##eep ceramic McCoy Career Sector chartered Damascus pictured Interest stiffened Plateau obsolete ##tant irritated inappropriate overs ##nko bail Talent Sur ours ##nah barred legged sociology Bud dictionary ##luk Cover obey ##oring annoying ##dong apprentice Cyrus Role ##GP ##uns ##bag Greenland Porsche Rocket ##32 organism ##ntary reliability ##vocation ##й Found ##hine motors promoter unfair ##oms ##note distribute eminent rails appealing chiefly meaningful Stephan ##rehension Consumer psychiatric bowler saints ##iful ##н 1777 Pol Dorian Townsend hastily ##jima Quincy Sol fascinated Scarlet alto Avon certainty ##eding Keys ##chu Chu ##VE ions tributaries Thanksgiving ##fusion astronomer oxide pavilion Supply Casa Bollywood sadly mutations Keller ##wave nationals ##rgo ##ym predict Catholicism Vega ##eration ##ums Mali tuned Lankan Plans radial Bosnian Lexi ##14 ##ü sacks unpleasant Empty handles ##taking Bon switches intently tuition antique ##jk fraternity notebook Desmond ##sei prostitution ##how deed ##OP 501 Somewhere Rocks ##mons campaigned frigate gases suppress ##hang Merlin Northumberland dominate expeditions thunder ##ups ##rical Cap thorough Ariel ##kind renewable constructing pacing terrorists Bowen documentaries westward ##lass ##nage Merchant ##ued Beaumont Din ##hian Danube peasant Garrison encourages gratitude reminding stormed ##ouse pronunciation ##ailed Weekend suggestions ##ffing ##DI Active Colombo ##logists Merrill ##cens Archaeological Medina captained ##yk duel cracking Wilkinson Guam pickup renovations ##ël ##izer delighted ##iri Weaver ##ctional tens ##hab Clint ##usion ##each petals Farrell ##sable caste ##will Ezra ##qi ##standing thrilled ambush exhaled ##SU Resource blur forearm specifications contingent cafe ##iology Antony fundraising grape ##rgy turnout ##udi Clifton laboratories Irvine ##opus ##lid Monthly Bihar statutory Roses Emil ##rig lumber optimal ##DR pumps plaster Mozambique ##aco nightclub propelled ##hun ked surplus wax ##urai pioneered Sunny imprint Forget Eliot approximate patronage ##bek ##ely ##mbe Partnership curl snapping 29th Patriarch ##jord seldom ##ature astronomy Bremen XIV airborne 205 1778 recognizing stranded arrogant bombardment destined ensured 146 robust Davenport Interactive Offensive Fi prevents probe propeller sorrow Blade mounting automotive ##dged wallet 201 lashes Forrest ##ift Cell Younger shouts ##cki folds ##chet Epic yields homosexual tunes ##minate ##text Manny chemist hindwings ##urn pilgrimage ##sfield ##riff MLS ##rive Huntington translates Path slim ##ndra ##oz climax commuter desperation ##reet denying ##rious daring seminary polo ##clamation Teatro Torah Cats identities Poles photographed fiery popularly ##cross winters Hesse ##vio Nurse Senegal Salon prescribed justify ##gues ##и ##orted HQ ##hiro evaluated momentarily ##unts Debbie ##licity ##TP Mighty Rabbit ##chal Events Savoy ##ht Brandenburg Bordeaux ##laus Release ##IE ##kowski 1900s SK Strauss ##aly Sonia Updated synagogue McKay flattened 370 clutch contests toast evaluate pope heirs jam tutor reverted ##ading nonsense hesitate Lars Ceylon Laurie ##guchi accordingly customary 148 Ethics Multiple instincts IGN ##ä bullshit ##hit ##par desirable ##ducing ##yam alias ashore licenses ##lification misery 147 Cola assassinated fiercely ##aft las goat substrate lords Cass Bridges ICC lasts sights reproductive ##asi Ivory Clean fixing ##lace seeming aide 1850s harassment ##FF ##LE reasonably ##coat ##cano NYC 1784 Fifty immunity Canadians Cheng comforting meanwhile ##tera ##blin breeds glowed ##vour Aden ##verted ##aded ##oral neat enforced poisoning ##ews ##hone enforce predecessors survivor Month unfamiliar pierced waived dump responds Mai Declan angular Doesn interpretations ##yar invest Dhaka policeman Congregation Eighth painfully ##este ##vior Württemberg ##cles blockade encouragement ##fie Caucasus Malone Universidad utilize Nissan inherent 151 agreeing syllable determines Protocol conclude ##gara 40th Xu Taiwanese ##ather boiler printer Lacey titular Klaus Fallon Wembley fox Chandra Governorate obsessed ##Ps micro ##25 Cooke gymnasium weaving Shall Hussein glaring softball Reader Dominion Trouble varsity Cooperation Chaos Kang Kramer Eisenhower proves Connie consortium governors Bethany opener Normally Willy linebacker Regent Used AllMusic Twilight ##shaw Companion Tribunal simpler ##gam Experimental Slovenian cellar deadline trout Hubbard ads idol ##hetto Granada clues salmon 1700 Omega Caldwell softened Bills Honolulu ##gn Terrace suitcase ##IL frantic ##oons Abbot Sitting Fortress Riders sickness enzymes trustee Bern forged ##13 ##ruff ##rl ##versity inspector champagne ##held ##FI hereditary Taliban handball ##wine Sioux ##dicated honoured 139 ##tude Skye meanings ##rkin cardiac analyzed vegetable ##FS Royals dial freelance ##fest partisan petroleum ridden Lincolnshire panting ##comb presidents Haley ##chs contributes Jew discoveries panicked Woody eyelids Fate Tulsa mg whiskey zombies Wii ##udge investigators ##bull centred ##screen Bone Lana ##oise forts ##ske Conan Lyons ##writing SH ##ride rhythmic 154 ##llah pioneers ##bright captivity Sanchez Oman ##mith Flint Platform ##ioned emission packet Persia ##formed takeover tempted Vance Few Toni receptions ##ن exchanges Camille whale Chronicles ##rent ##ushing ##rift Alto Genus ##asing onward foremost longing Rockefeller containers ##cribe intercepted ##olt pleading Bye bee ##umbling 153 undertake Izzy cheaper Ultra validity ##pse Sa hovering ##pert vintage engraved ##rise farmland ##ever ##ifier Atlantis propose Catalonia plunged ##edly demonstrates gig ##cover 156 Osborne cowboy herd investigator loops Burning rests Instrumental embarrassing focal install readings swirling Chatham parameter ##zin ##holders Mandarin Moody converting Escape warnings ##chester incarnation ##ophone adopting ##lins Cromwell ##laws Axis Verde Kappa Schwartz Serbs caliber Wanna Chung ##ality nursery principally Bulletin likelihood logging ##erty Boyle supportive twitched ##usive builds Marseille omitted motif Lands ##lusion ##ssed Barrow Airfield Harmony WWF endured merging convey branding examinations 167 Italians ##dh dude 1781 ##teau crawling thoughtful clasped concluding brewery Moldova Wan Towers Heidelberg 202 ##ict Lagos imposing ##eval ##serve Bacon frowning thirteenth conception calculations ##ович ##mile ##ivated mutation strap ##lund demographic nude perfection stocks ##renched ##dit Alejandro bites fragment ##hack ##rchy GB Surgery Berger punish boiling consume Elle Sid Dome relies Crescent treasurer Bloody 1758 upheld Guess Restaurant signatures font millennium mural stakes Abel hailed insists Alumni Breton ##jun digits ##FM ##thal Talking motive reigning babe masks ##ø Shaun potato sour whitish Somali ##derman ##rab ##wy chancel telecommunications Noise messenger tidal grinding ##ogenic Rebel constituent peripheral recruitment ##ograph ##tler pumped Ravi poked ##gley Olive diabetes discs liking sting fits stir Mari Sega creativity weights Macau mandated Bohemia disastrous Katrina Baku Rajasthan waiter ##psis Siberia verbs ##truction patented 1782 ##ndon Relegated Hunters Greenwood Shock accusing skipped Sessions markers subset monumental Viola comparative Alright Barbados setup Session standardized ##ík ##sket appoint AFB Nationalist ##WS Troop leaped Treasure goodness weary originates 100th compassion expresses recommend 168 composing seventeenth Tex Atlético bald Finding Presidency Sharks favoured inactive ##lter suffix princes brighter ##ctus classics defendants culminated terribly Strategy evenings ##ção ##iver ##urance absorb ##rner Territories RBI soothing Martín concurrently ##tr Nicholson fibers swam ##oney Allie Algerian Dartmouth Mafia ##bos ##tts Councillor vocabulary ##bla ##lé intending ##dler Guerrero sunshine pedal ##TO administrators periodic scholarships Loop Madeline exaggerated ##ressed Regan ##cellular Explorer ##oids Alexandre vows Reporter Unable Average absorption ##bedience Fortunately Auxiliary Grandpa ##HP ##ovo potent temporal adrenaline ##udo confusing guiding Dry qualifications joking wherein heavyweight ##ices nightmares pharmaceutical Commanding ##aled ##ove Gregor ##UP censorship degradation glorious Austro ##rench 380 Miriam sped ##orous offset ##KA fined specialists Pune João ##dina propped fungus ##ς frantically Gabrielle Hare committing ##plied Ask Wilmington stunt numb warmer preacher earnings ##lating integer ##ija federation homosexuality ##cademia epidemic grumbled shoving Milk Satan Tobias innovations ##dington geology memoirs ##IR spared culminating Daphne Focus severed stricken Paige Mans flats Russo communes litigation strengthening ##powered Staffordshire Wiltshire Painting Watkins ##د specializes Select ##rane ##aver Fulton playable ##VN openings sampling ##coon ##21 Allah travelers allocation ##arily Loch ##hm commentators fulfilled ##troke Emeritus Vanderbilt Vijay pledged ##tative diagram drilling ##MD ##plain Edison productivity 31st ##rying ##ption ##gano ##oration ##bara posture bothering platoon politely ##inating redevelopment Job ##vale stark incorrect Mansion renewal threatens Bahamas fridge ##tata Uzbekistan ##edia Sainte ##mio gaps neural ##storm overturned Preservation shields ##ngo ##physics ah gradual killings ##anza consultation premiership Felipe coincidence ##ène ##any Handbook ##loaded Edit Guns arguably ##ş compressed depict seller ##qui Kilkenny ##kling Olympia librarian ##acles dramas JP Kit Maj ##lists proprietary ##nged ##ettes ##tok exceeding Lock induction numerical ##vist Straight foyer imaginary ##pop violinist Carla bouncing ##ashi abolition ##uction restoring scenic ##č Doom overthrow para ##vid ##ughty Concord HC cocaine deputies ##aul visibility ##wart Kapoor Hutchinson ##agan flashes kn decreasing ##ronology quotes vain satisfying ##iam ##linger 310 Hanson fauna ##zawa ##rrel Trenton ##VB Employment vocational Exactly bartender butterflies tow ##chers ##ocks pigs merchandise ##game ##pine Shea ##gration Connell Josephine monopoly ##dled Cobb warships cancellation someday stove ##Cs candidacy superhero unrest Toulouse admiration undergone whirled Reconnaissance costly ##ships 290 Cafe amber Tory ##mpt definitive ##dress proposes redesigned acceleration ##asa ##raphy Presley exits Languages ##cel Mode spokesperson ##tius Ban forthcoming grounded ACC compelling logistics retailers abused ##gating soda ##yland ##lution Landmark XVI blush ##tem hurling dread Tobago Foley ##uad scenarios ##mentation ##rks Score fatigue hairy correspond ##iard defences confiscated ##rudence 1785 Formerly Shot advertised 460 Text ridges Promise Dev exclusion NHS tuberculosis rockets ##offs sparkling 256 disappears mankind ##hore HP ##omo taxation Multi DS Virgil ##ams Dell stacked guessing Jump Nope cheer hates ballots overlooked analyses Prevention maturity dos ##cards ##lect Mare ##yssa Petty ##wning differing iOS ##ior Joachim Sentinel ##nstein 90s Pamela 480 Asher ##lary Vicente landings portray ##rda ##xley Virtual ##uary finances Jain Somebody Tri behave Michele ##ider dwellings FAA Gallagher ##lide Monkey 195 aforementioned ##rism ##bey ##kim ##puted Mesa hopped unopposed recipients Reality Been gritted 149 playground pillar ##rone Guinness ##tad Théâtre depended Tipperary Reuben frightening wooded Target globally ##uted Morales Baptiste drunken Institut characterised ##chemistry Strip discrete Premiership ##zzling gazing Outer ##quisition Sikh Booker ##yal contemporaries Jericho ##chan ##physical ##witch Militia ##rez ##zard dangers ##utter ##₀ Programs darling participates railroads ##ienne behavioral bureau ##rook 161 Hicks ##rises Comes inflicted bees kindness norm ##ković generators ##pard ##omy ##ili methodology Alvin façade latitude ##plified DE Morse ##mered educate intersects ##MF ##cz ##vated AL ##graded ##fill constitutes artery feudal avant cautious ##ogue immigrated ##chenko Saul Clinic Fang choke Cornelius flexibility temperate pins ##erson oddly inequality 157 Natasha Sal ##uter 215 aft blinking ##ntino northward Exposition cookies Wedding impulse Overseas terrifying ##ough Mortimer ##see 440 https og imagining ##cars Nicola exceptionally threads ##cup Oswald Provisional dismantled deserves 1786 Fairy discourse Counsel departing Arc guarding ##orse 420 alterations vibrant Em squinted terrace rowing Led accessories SF Sgt cheating Atomic ##raj Blackpool ##iary boarded substituted bestowed lime kernel ##jah Belmont shaken sticky retrospective Louie migrants weigh sunglasses thumbs ##hoff excavation ##nks Extra Polo motives Drum infrared tastes berth verge ##stand programmed warmed Shankar Titan chromosome cafeteria dividing pepper CPU Stevie satirical Nagar scowled Died backyard ##gata ##reath ##bir Governors portraying ##yah Revenge ##acing 1772 margins Bahn OH lowland ##razed catcher replay ##yoshi Seriously ##licit Aristotle ##ald Habsburg weekday Secretariat CO ##dly ##joy ##stad litre ultra ##cke Mongol Tucson correlation compose traps Groups Hai Salvatore ##dea cents ##eese concession clash Trip Panzer Moroccan cruisers torque Ba grossed ##arate restriction concentrating FDA ##Leod ##ones Scholars ##esi throbbing specialised ##heses Chicken ##fia ##ificant Erich Residence ##trate manipulation namesake ##tom Hoover cue Lindsey Lonely 275 ##HT combustion subscribers Punjabi respects Jeremiah penned ##gor ##rilla suppression ##tration Crimson piston Derry crimson lyrical oversee portrays CF Districts Lenin Cora searches clans VHS ##hel Jacqueline Redskins Clubs desktop indirectly alternatives marijuana suffrage ##smos Irwin ##liff Process ##hawks Sloane ##bson Sonata yielded Flores ##ares armament adaptations integrate neighbours shelters ##tour Skinner ##jet ##tations 1774 Peterborough ##elles ripping Liang Dickinson charities Rwanda monasteries crossover racist barked guerrilla ##ivate Grayson ##iques ##vious ##got Rolls denominations atom affinity ##delity Wish ##inted ##inae interrogation ##cey ##erina ##lifting 192 Sands 1779 mast Likewise ##hyl ##oft contempt ##por assaulted fills establishments Mal consulted ##omi ##sight greet ##roma ##egan Pulitzer ##rried ##dius ##ractical ##voked Hasan CB ##zzy Romanesque Panic wheeled recorder ##tters ##warm ##gly botanist Balkan Lockheed Polly farewell suffers purchases Eaton ##80 Quick commenting Saga beasts hides motifs ##icks Alonso Springer Wikipedia circulated encoding jurisdictions snout UAE Integrated unmarried Heinz ##lein ##figured deleted ##tley Zen Cycling Fuel Scandinavian ##rants Conner reef Marino curiously lingered Gina manners activism Mines Expo Micah promotions Server booked derivatives eastward detailing reelection ##chase 182 Campeonato Po 158 Peel winger ##itch canyon ##pit LDS A1 ##shin Giorgio pathetic ##rga ##mist Aren ##lag confronts motel textbook shine turbines 1770 Darcy ##cot Southeastern ##lessness Banner recognise stray Kitchen paperwork realism Chrysler filmmakers fishermen ##hetic variously Vishnu fiddle Eddy Origin ##tec ##ulin Flames Rs bankrupt Extreme Pomeranian ##emption ratified ##iu jockey Stratford ##ivating ##oire Babylon pardon AI affordable deities disturbance Trying ##sai Ida Papers advancement 70s archbishop Luftwaffe announces tugging ##lphin ##sistence ##eel ##ishes ambition aura ##fled ##lected ##vue Prasad boiled clarity Violin investigative routing Yankee ##uckle McMahon bugs eruption ##rooms Minutes relics ##ckle ##nse sipped valves weakly ##ital Middleton collided ##quer bamboo insignia Tyne exercised Ninth echoing polynomial considerations lunged ##bius objections complain disguised plaza ##VC institutes Judicial ascent imminent Waterford hello Lumpur Niger Goldman vendors Kensington Wren browser ##bner ##tri ##mize ##pis ##lea Cheyenne Bold Settlement Hollow Paralympic axle ##toire ##actic impose perched utilizing slips Benz Michaels manipulate Chiang ##mian Dolphins prohibition attacker ecology Estadio ##SB ##uild attracts recalls glacier lad ##rima Barlow kHz melodic ##aby ##iracy assumptions Cornish ##aru DOS Maddie ##mers lyric Luton nm ##tron Reno Fin YOU Broadcast Finch sensory ##bent Jeep ##uman additionally Buildings businessmen treaties 235 Stranger gateway Charlton accomplishments Diary apologized zinc histories supplier ##tting 162 asphalt Treatment Abbas ##pating ##yres Bloom sedan soloist ##cum antagonist denounced Fairfax ##aving ##enko noticeable Budget Buckingham Snyder retreating Jai spoon invading giggle woven gunfire arrests ##vered ##come respiratory violet ##aws Byrd shocking tenant Jamaican Ottomans Seal theirs ##isse ##48 cooperate peering ##nius 163 Composer organist Mongolian Bauer Spy collects prophecy congregations ##moor Brick calculation fixtures exempt ##dden Ada Thousand ##lue tracing ##achi bodyguard vicar supplying Łódź interception monitored ##heart Paso overlap annoyance ##dice yellowish stables elders illegally honesty ##oar skinny spinal ##puram Bourbon ##cor flourished Medium ##stics ##aba Follow ##ckey stationary ##scription dresser scrutiny Buckley Clearly ##SF Lyrics ##heimer drying Oracle internally rains ##last Enemy ##oes McLean Ole phosphate Rosario Rifles ##mium battered Pepper Presidents conquer Château castles ##aldo ##ulf Depending Lesser Boom trades Peyton 164 emphasize accustomed SM Ai Classification ##mins ##35 ##rons leak piled deeds lush ##self beginnings breathless 1660 McGill ##ago ##chaft ##gies humour Bomb securities Might ##zone ##eves Matthias Movies Levine vengeance ##ads Challenger Misty Traditionally constellation ##rass deepest workplace ##oof ##vina impatient ##ML Mughal Alessandro scenery Slater postseason troupe ##ń Volunteers Facility militants Reggie sanctions Expeditionary Nam countered interpret Basilica coding expectation Duffy def Tong wakes Bowling Vehicle Adler salad intricate stronghold medley ##uries ##bur joints ##rac ##yx ##IO Ordnance Welch distributor Ark cavern trench Weiss Mauritius decreases docks eagerly irritation Matilda biographer Visiting ##marked ##iter ##ear ##gong Moreno attendant Bury instrumentation theologian clit nuns symphony translate 375 loser ##user ##VR ##meter ##orious harmful ##yuki Commissioners Mendoza sniffed Hulk ##dded ##ulator ##nz Donnell ##eka deported Met SD Aerospace ##cultural ##odes Fantastic cavity remark emblem fearing ##iance ICAO Liberia stab ##yd Pac Gymnasium IS Everton ##vanna mantle ##ief Ramon ##genic Shooting Smoke Random Africans MB tavern bargain voluntarily Ion Peoples Rusty attackers Patton sins ##cake Hat moderately ##hala ##alia requesting mechanic ##eae Seine Robbins ##ulum susceptible Bravo Slade Strasbourg rubble entrusted Creation ##amp smoothed ##uintet evenly reviewers skip Sculpture 177 Rough ##rrie Reeves ##cede Administrator garde minus carriages grenade Ninja fuscous ##kley Punk contributors Aragon Tottenham ##cca ##sir VA laced dealers ##sonic crisp harmonica Artistic Butch Andes Farmers corridors unseen ##tium Countries Lone envisioned Katy ##lang ##cc Quarterly ##neck consort ##aceae bidding Corey concurrent ##acts ##gum Highness ##lient ##rators arising ##unta pathways 49ers bolted complaining ecosystem libretto Ser narrated 212 Soft influx ##dder incorporation plagued tents ##ddled 1750 Risk citation Tomas hostilities seals Bruins Dominique attic competent ##UR ##cci hugging Breuning bacterial Shrewsbury vowed eh elongated hangs render centimeters ##ficient Mu turtle besieged ##gaard grapes bravery collaborations deprived ##amine ##using ##gins arid ##uve coats hanged ##sting Pa prefix ##ranged Exit Chain Flood Materials suspicions ##ö hovered Hidden ##state Malawi ##24 Mandy norms fascinating airlines delivers ##rust Cretaceous spanned pillows ##onomy jar ##kka regent fireworks morality discomfort lure uneven ##jack Lucian 171 archaeology ##til mornings Billie Marquess impending spilling tombs ##volved Celia Coke underside ##bation Vaughn Daytona Godfrey Pascal Alien ##sign 172 ##lage iPhone Gonna genocide ##rber oven endure dashed simultaneous ##phism Wally ##rō ants predator reissue ##aper Speech funk Rudy claw Hindus Numbers Bing lantern ##aurus scattering poisoned ##active Andrei algebraic baseman ##ritz Gregg ##cola selections ##putation lick Laguna ##IX Sumatra Warning turf buyers Burgess Oldham exploit worm initiate strapped tuning filters haze ##е ##ledge ##ydro ##culture amendments Promotion ##union Clair ##uria petty shutting ##eveloped Phoebe Zeke conducts grains clashes ##latter illegitimate willingly Deer Lakers Reference chaplain commitments interrupt salvation Panther Qualifying Assessment cancel efficiently attorneys Dynamo impress accession clinging randomly reviewing Romero Cathy charting clapped rebranded Azerbaijani coma indicator punches ##tons Sami monastic prospects Pastor ##rville electrified ##CI ##utical tumbled Chef muzzle selecting UP Wheel protocols ##tat Extended beautifully nests ##stal Andersen ##anu ##³ ##rini kneeling ##reis ##xia anatomy dusty Safe turmoil Bianca ##elo analyze ##ر ##eran podcast Slovene Locke Rue ##retta ##uni Person Prophet crooked disagreed Versailles Sarajevo Utrecht ##ogen chewing ##ception ##iidae Missile attribute majors Arch intellectuals ##andra ideological Cory Salzburg ##fair Lot electromagnetic Distribution ##oper ##pered Russ Terra repeats fluttered Riga ##ific ##gt cows Hair labelled protects Gale Personnel Düsseldorf Moran rematch ##OE Slow forgiveness ##ssi proudly Macmillan insist undoubtedly Québec Violence ##yuan ##aine mourning linen accidental ##iol ##arium grossing lattice maneuver ##marine prestige petrol gradient invasive militant Galerie widening ##aman ##quist disagreement ##ales creepy remembers buzz ##erial Exempt Dirk mon Addison ##inen deposed ##agon fifteenth Hang ornate slab ##lades Fountain contractors das Warwickshire 1763 ##rc Carly Essays Indy Ligue greenhouse slit ##sea chewed wink ##azi Playhouse ##kon Gram Ko Samson creators revive ##rians spawned seminars Craft Tall diverted assistants computational enclosure ##acity Coca ##eve databases Drop ##loading ##hage Greco Privy entrances pork prospective Memories robes ##market transporting ##lik Rudolph Horton visually ##uay ##nja Centro Tor Howell ##rsey admitting postgraduate herbs ##att Chin Rutherford ##bot ##etta Seasons explanations ##bery Friedman heap ##ryl ##sberg jaws ##agh Choi Killing Fanny ##suming ##hawk hopeful ##aid Monty gum remarkably Secrets disco harp advise ##avia Marathi ##cycle Truck abbot sincere urine ##mology masked bathing ##tun Fellows ##TM ##gnetic owl ##jon hymn ##leton 208 hostility ##cée baked Bottom ##AB shudder ##ater ##von ##hee reorganization Cycle ##phs Lex ##style ##rms Translation ##erick ##imeter ##ière attested Hillary ##DM gal wander Salle ##laming Perez Pit ##LP USAF contexts Disease blazing aroused razor walled Danielle Mont Funk royalty thee 203 donors ##erton famously processors reassigned welcoming Goldberg ##quities undisclosed Orient Patty vaccine refrigerator Cypriot consonant ##waters 176 sober ##lement Racecourse ##uate Luckily Selection conceptual vines Breaking wa lions oversight sheltered Dancer ponds borrow ##BB ##pulsion Daly ##eek fertility spontaneous Worldwide gasping ##tino 169 ABS Vickers ambient energetic prisons ##eson Stacy ##roach GmbH Afro Marin farmhouse pinched ##cursion ##sp Sabine ##pire 181 nak swelling humble perfume ##balls Rai cannons ##taker Married Maltese canals interceptions hats lever slowing ##ppy Nike Silas Scarborough skirts 166 inauguration Shuttle alloy beads belts Compton Cause battling critique surf Dock roommate ##ulet invade Garland ##slow nutrition persona ##zam Wichita acquaintance coincided ##cate Dracula clamped ##gau overhaul ##broken ##rrier melodies ventures Paz convex Roots ##holding Tribute transgender ##ò chimney ##riad Ajax Thereafter messed nowadays pH ##100 ##alog Pomerania ##yra Rossi glove ##TL Races ##asily tablets Jase ##ttes diner ##rns Hu Mohan anytime weighted remixes Dove cherry imports ##urity GA ##TT ##iated ##sford Clarkson evidently rugged Dust siding ##ometer acquitted choral ##mite infants Domenico gallons Atkinson gestures slated ##xa Archaeology unwanted ##ibes ##duced premise Colby Geelong disqualified ##pf ##voking simplicity Walkover Qaeda Warden ##bourg ##ān Invasion Babe harness 183 ##tated maze Burt bedrooms ##nsley Horizon ##oast minimize peeked MLA Trains tractor nudged ##iform Growth Benton separates ##about ##kari buffer anthropology brigades foil ##wu Domain licking whore ##rage ##sham Initial Courthouse Rutgers dams villains supermarket ##brush Brunei Palermo arises Passenger outreach ##gill Labrador McLaren ##uy Lori ##fires Heads magistrate ¹⁄₂ Weapons ##wai ##roke projecting ##ulates bordering McKenzie Pavel midway Guangzhou streamed racer ##lished eccentric spectral 206 ##mism Wilde Grange preparatory lent ##tam starving Gertrude ##cea ##ricted Breakfast Mira blurted derive ##lair blunt sob Cheltenham Henrik reinstated intends ##istan unite ##ector playful sparks mapped Cadet luggage prosperous ##ein salon ##utes Biological ##rland Tyrone buyer ##lose amounted Saw smirked Ronan Reviews Adele trait ##proof Bhutan Ginger ##junct digitally stirring ##isted coconut Hamlet Dinner Scale pledge ##RP Wrong Goal Panel therapeutic elevations infectious priesthood ##inda Guyana diagnostic ##mbre Blackwell sails ##arm literal periodically gleaming Robot Rector ##abulous ##tres Reaching Romantic CP Wonderful ##tur ornamental ##nges traitor ##zilla genetics mentioning ##eim resonance Areas Shopping ##nard Gail Solid ##rito ##mara Willem Chip Matches Volkswagen obstacle Organ invites Coral attain ##anus ##dates Midway shuffled Cecilia dessert Gateway Ch Napoleonic Petroleum jets goose striped bowls vibration Sims nickel Thirteen problematic intervene ##grading ##unds Mum semifinal Radical ##izations refurbished ##sation ##harine Maximilian cites Advocate Potomac surged preserves Curry angled ordination ##pad Cade ##DE ##sko researched torpedoes Resident wetlands hay applicants depart Bernstein ##pic ##ario ##rae favourable ##wari ##р metabolism nobleman Defaulted calculate ignition Celebrity Belize sulfur Flat Sc USB flicker Hertfordshire Sept CFL Pasadena Saturdays Titus ##nir Canary Computing Isaiah ##mler formidable pulp orchid Called Solutions kilograms steamer ##hil Doncaster successors Stokes Holstein ##sius sperm API Rogue instability Acoustic ##rag 159 undercover Wouldn ##pra ##medical Eliminated honorable ##chel denomination abrupt Buffy blouse fi Regardless Subsequent ##rdes Lover ##tford bacon ##emia carving ##cripts Massacre Ramos Latter ##ulp ballroom ##gement richest bruises Rest Wiley ##aster explosions ##lastic Edo ##LD Mir choking disgusted faintly Barracks blasted headlights Tours ensued presentations ##cale wrought ##oat ##coa Quaker ##sdale recipe ##gny corpses ##liance comfortably ##wat Landscape niche catalyst ##leader Securities messy ##RL Rodrigo backdrop ##opping treats Emilio Anand bilateral meadow VC socialism ##grad clinics ##itating ##ppe ##ymphonic seniors Advisor Armoured Method Alley ##orio Sad fueled raided Axel NH rushes Dixie Otis wrecked ##22 capitalism café ##bbe ##pion ##forcing Aubrey Lublin Whenever Sears Scheme ##lana Meadows treatise ##RI ##ustic sacrifices sustainability Biography mystical Wanted multiplayer Applications disliked ##tisfied impaired empirical forgetting Fairfield Sunni blurred Growing Avalon coil Camera Skin bruised terminals ##fted ##roving Commando ##hya ##sper reservations needles dangling ##rsch ##rsten ##spect ##mbs yoga regretted Bliss Orion Rufus glucose Olsen autobiographical ##dened 222 humidity Shan ##ifiable supper ##rou flare ##MO campaigning descend socio declares Mounted Gracie Arte endurance ##ety Copper costa airplay ##MB Proceedings dislike grimaced occupants births glacial oblivious cans installment muddy ##ł captains pneumonia Quiet Sloan Excuse ##nine Geography gymnastics multimedia drains Anthology Gear cylindrical Fry undertaking ##pler ##tility Nan ##recht Dub philosophers piss Atari ##pha Galicia México ##nking Continuing bump graveyard persisted Shrine ##erapy defects Advance Bomber ##oil ##ffling cheerful ##lix scrub ##eto awkwardly collaborator fencing ##alo prophet Croix coughed ##lication roadway slaughter elephants ##erated Simpsons vulnerability ivory Birth lizard scarce cylinders fortunes ##NL Hate Priory ##lai McBride ##copy Lenny liaison Triangle coronation sampled savage amidst Grady whatsoever instinctively Reconstruction insides seizure Drawing ##rlin Antioch Gao Díaz 1760 Sparks ##tien ##bidae rehearsal ##bbs botanical ##hers compensate wholesale Seville shareholder prediction astronomical Reddy hardest circling whereabouts termination Rep Assistance Dramatic Herb ##ghter climbs 188 Poole 301 ##pable wit ##istice Walters relying Jakob ##redo proceeding Langley affiliates ou ##allo ##holm Samsung ##ishi Missing Xi vertices Claus foam restless ##uating ##sso ##ttering Philips delta bombed Catalogue coaster Ling Willard satire 410 Composition Net Orioles ##ldon fins Palatinate Woodward tease tilt brightness ##70 ##bbling ##loss ##dhi ##uilt Whoever ##yers hitter Elton Extension ace Affair restructuring ##loping Paterson hi ##rya spouse Shay Himself piles preaching ##gical bikes Brave expulsion Mirza stride Trees commemorated famine masonry Selena Watt Banking Rancho Stockton dip tattoos Vlad acquainted Flyers ruthless fourteenth illustrate ##akes EPA ##rows ##uiz bumped Designed Leaders mastered Manfred swirled McCain ##rout Artemis rabbi flinched upgrades penetrate shipyard transforming caretaker ##eiro Maureen tightening ##founded RAM ##icular ##mper ##rung Fifteen exploited consistency interstate ##ynn Bridget contamination Mistress ##rup coating ##FP ##jective Libyan 211 Gemma dependence shrubs ##ggled Germain retaliation traction ##PP Dangerous terminology psychiatrist ##garten hurdles Natal wasting Weir revolves stripe ##reased preferences ##entation ##lde ##áil ##otherapy Flame ##ologies viruses Label Pandora veil ##ogical Coliseum Cottage creeping Jong lectured ##çaise shoreline ##fference ##hra Shade Clock Faye bilingual Humboldt Operating ##fter ##was algae towed amphibious Parma impacted smacked Piedmont Monsters ##omb Moor ##lberg sinister Postal 178 Drummond Sign textbooks hazardous Brass Rosemary Pick Sit Architect transverse Centennial confess polling ##aia Julien ##mand consolidation Ethel ##ulse severity Yorker choreographer 1840s ##ltry softer versa ##geny ##quila ##jō Caledonia Friendship Visa rogue ##zzle bait feather incidence Foods Ships ##uto ##stead arousal ##rote Hazel ##bolic Swing ##ej ##cule ##jana ##metry ##uity Valuable ##ₙ Shropshire ##nect 365 Ones realise Café Albuquerque ##grown ##stadt 209 ##ᵢ prefers withstand Lillian MacArthur Hara ##fulness domination ##VO ##school Freddy ethnicity ##while adorned hormone Calder Domestic Freud Shields ##phus ##rgan BP Segunda Mustang ##GI Bonn patiently remarried ##umbria Crete Elephant Nuremberg tolerate Tyson ##evich Programming ##lander Bethlehem segregation Constituency quarterly blushed photographers Sheldon porcelain Blanche goddamn lively ##fused bumps ##eli curated coherent provoked ##vet Madeleine ##isco rainy Bethel accusation ponytail gag ##lington quicker scroll ##vate Bow Gender Ira crashes ACT Maintenance ##aton ##ieu bitterly strains rattled vectors ##arina ##ishly 173 parole ##nx amusing Gonzalez ##erative Caucus sensual Penelope coefficient Mateo ##mani proposition Duty lacrosse proportions Plato profiles Botswana Brandt reins mandolin encompassing ##gens Kahn prop summon ##MR ##yrian ##zaki Falling conditional thy ##bao ##ych radioactive ##nics Newspaper ##people ##nded Gaming sunny ##look Sherwood crafted NJ awoke 187 timeline giants possessing ##ycle Cheryl ng Ruiz polymer potassium Ramsay relocation ##leen Sociology ##bana Franciscan propulsion denote ##erjee registers headline Tests emerges Articles Mint livery breakup kits Rap Browning Bunny ##mington ##watch Anastasia Zachary arranging biographical Erica Nippon ##membrance Carmel ##sport ##xes Paddy ##holes Issues Spears compliment ##stro ##graphs Castillo ##MU ##space Corporal ##nent 174 Gentlemen ##ilize ##vage convinces Carmine Crash ##hashi Files Doctors brownish sweating goats ##conductor rendition ##bt NL ##spiration generates ##cans obsession ##noy Danger Diaz heats Realm priorities ##phon 1300 initiation pagan bursts archipelago chloride Screenplay Hewitt Khmer bang judgement negotiating ##ait Mabel densely Boulder knob 430 Alfredo ##kt pitches ##ées ##ان Macdonald ##llum imply ##mot Smile spherical ##tura Derrick Kelley Nico cortex launches differed parallels Navigation ##child ##rming canoe forestry reinforce ##mote confirming tasting scaled ##resh ##eting Understanding prevailing Pearce CW earnest Gaius asserts denoted landmarks Chargers warns ##flies Judges jagged ##dain tails Historian Millie ##sler 221 ##uard absurd Dion ##ially makeshift Specifically ignorance Eat ##ieri comparisons forensic 186 Giro skeptical disciplinary battleship ##45 Libby 520 Odyssey ledge ##post Eternal Missionary deficiency settler wonders ##gai raging ##cis Romney Ulrich annexation boxers sect 204 ARIA dei Hitchcock te Varsity ##fic CC lending ##nial ##tag ##rdy ##obe Defensive ##dson ##pore stellar Lam Trials contention Sung ##uminous Poe superiority ##plicate 325 bitten conspicuous ##olly Lila Pub Petit distorted ISIL distinctly ##family Cowboy mutant ##cats ##week Changes Sinatra epithet neglect Innocent gamma thrill reggae ##adia ##ational ##due landlord ##leaf visibly ##ì Darlington Gomez ##iting scarf ##lade Hinduism Fever scouts ##roi convened ##oki 184 Lao boycott unemployed ##lore ##ß ##hammer Curran disciples odor ##ygiene Lighthouse Played whales discretion Yves ##ceived pauses coincide ##nji dizzy ##scopic routed Guardians Kellan carnival nasal 224 ##awed Mitsubishi 640 Cast silky Projects joked Huddersfield Rothschild zu ##olar Divisions mildly ##eni ##lge Appalachian Sahara pinch ##roon wardrobe ##dham ##etal Bubba ##lini ##rumbling Communities Poznań unification Beau Kris SV Rowing Minh reconciliation ##saki ##sor taped ##reck certificates gubernatorial rainbow ##uing litter ##lique ##oted Butterfly benefited Images induce Balkans Velvet ##90 ##xon Bowman ##breaker penis ##nitz ##oint ##otive crust ##pps organizers Outdoor nominees ##rika TX ##ucks Protestants ##imation appetite Baja awaited ##points windshield ##igh ##zled Brody Buster stylized Bryce ##sz Dollar vest mold ounce ok receivers ##uza Purdue Harrington Hodges captures ##ggio Reservation ##ssin ##tman cosmic straightforward flipping remixed ##athed Gómez Lim motorcycles economies owning Dani ##rosis myths sire kindly 1768 Bean graphs ##mee ##RO ##geon puppy Stephenson notified ##jer Watching ##rama Sino urgency Islanders ##mash Plata fumble ##chev ##stance ##rack ##she facilitated swings akin enduring payload ##phine Deputies murals ##tooth 610 Jays eyeing ##quito transparency ##cote Timor negatively ##isan battled ##fected thankful Rage hospitality incorrectly 207 entrepreneurs ##cula ##wley hedge ##cratic Corpus Odessa Whereas ##ln fetch happier Amherst bullying graceful Height Bartholomew willingness qualifier 191 Syed Wesleyan Layla ##rrence Webber ##hum Rat ##cket ##herence Monterey contaminated Beside Mustafa Nana 213 ##pruce Reason ##spense spike ##gé AU disciple charcoal ##lean formulated Diesel Mariners accreditation glossy 1800s ##ih Mainz unison Marianne shear overseeing vernacular bowled ##lett unpopular ##ckoned ##monia Gaston ##TI ##oters Cups ##bones ##ports Museo minors 1773 Dickens ##EL ##NBC Presents ambitions axes Río Yukon bedside Ribbon Units faults conceal ##lani prevailed 214 Goodwin Jaguar crumpled Cullen Wireless ceded remotely Bin mocking straps ceramics ##avi ##uding ##ader Taft twenties ##aked Problem quasi Lamar ##ntes ##avan Barr ##eral hooks sa ##ône 194 ##ross Nero Caine trance Homeland benches Guthrie dismiss ##lex César foliage ##oot ##alty Assyrian Ahead Murdoch dictatorship wraps ##ntal Corridor Mackay respectable jewels understands ##pathic Bryn ##tep ON capsule intrigued Sleeping communists ##chayat ##current ##vez doubling booklet ##uche Creed ##NU spies ##sef adjusting 197 Imam heaved Tanya canonical restraint senators stainless ##gnate Matter cache restrained conflicting stung ##ool Sustainable antiquity 193 heavens inclusive ##ador fluent 303 911 archaeologist superseded ##plex Tammy inspire ##passing ##lub Lama Mixing ##activated ##yote parlor tactic 198 Stefano prostitute recycling sorted banana Stacey Musée aristocratic cough ##rting authorised gangs runoff thoughtfully ##nish Fisheries Provence detector hum ##zhen pill ##árez Map Leaves Peabody skater vent ##color 390 cerebral hostages mare Jurassic swell ##isans Knoxville Naked Malaya scowl Cobra ##anga Sexual ##dron ##iae 196 ##drick Ravens Blaine ##throp Ismail symmetric ##lossom Leicestershire Sylvester glazed ##tended Radar fused Families Blacks Sale Zion foothills microwave slain Collingwood ##pants ##dling killers routinely Janice hearings ##chanted ##ltration continents ##iving ##yster ##shot ##yna injected Guillaume ##ibi kinda Confederacy Barnett disasters incapable ##grating rhythms betting draining ##hak Callie Glover ##iliated Sherlock hearted punching Wolverhampton Leaf Pi builders furnished knighted Photo ##zle Touring fumbled pads ##ий Bartlett Gunner eerie Marius Bonus pots ##hino ##pta Bray Frey Ortiz stalls belongings Subway fascination metaphor Bat Boer Colchester sway ##gro rhetoric ##dheim Fool PMID admire ##hsil Strand TNA ##roth Nottinghamshire ##mat ##yler Oxfordshire ##nacle ##roner BS ##nces stimulus transports Sabbath ##postle Richter 4000 ##grim ##shima ##lette deteriorated analogous ##ratic UHF energies inspiring Yiddish Activities ##quential ##boe Melville ##ilton Judd consonants labs smuggling ##fari avid ##uc truce undead ##raith Mostly bracelet Connection Hussain awhile ##UC ##vention liable genetically ##phic Important Wildcats daddy transmit ##cas conserved Yesterday ##lite Nicky Guys Wilder Lay skinned Communists Garfield Nearby organizer Loss crafts walkway Chocolate Sundance Synod ##enham modify swayed Surface analysts brackets drone parachute smelling Andrés filthy frogs vertically ##OK localities marries AHL 35th ##pian Palazzo cube dismay relocate ##на Hear ##digo ##oxide prefecture converts hangar ##oya ##ucking Spectrum deepened spoiled Keeping ##phobic Verona outrage Improvement ##UI masterpiece slung Calling chant Haute mediated manipulated affirmed ##hesis Hangul skies ##llan Worcestershire ##kos mosaic ##bage ##wned Putnam folder ##LM guts noteworthy ##rada AJ sculpted ##iselle ##rang recognizable ##pent dolls lobbying impatiently Se staple Serb tandem Hiroshima thieves ##ynx faculties Norte ##alle ##trusion chords ##ylon Gareth ##lops ##escu FIA Levin auspices groin Hui nun Listed Honourable Larsen rigorous ##erer Tonga ##pment ##rave ##track ##aa ##enary 540 clone sediment esteem sighted cruelty ##boa inverse violating Amtrak Status amalgamated vertex AR harmless Amir mounts Coronation counseling Audi CO₂ splits ##eyer Humans Salmon ##have ##rado ##čić 216 takeoff classmates psychedelic ##gni Gypsy 231 Anger GAA ME ##nist ##tals Lissa Odd baptized Fiat fringe ##hren 179 elevators perspectives ##TF ##ngle Question frontal 950 thicker Molecular ##nological Sixteen Baton Hearing commemorative dorm Architectural purity ##erse risky Georgie relaxing ##ugs downed ##rar Slim ##phy IUCN ##thorpe Parkinson 217 Marley Shipping sweaty Jesuits Sindh Janata implying Armenians intercept Ankara commissioners ascended sniper Grass Walls salvage Dewey generalized learnt PT ##fighter ##tech DR ##itrus ##zza mercenaries slots ##burst ##finger ##nsky Princes Rhodesia ##munication ##strom Fremantle homework ins ##Os ##hao ##uffed Thorpe Xiao exquisite firstly liberated technician Oilers Phyllis herb sharks MBE ##stock Product banjo ##morandum ##than Visitors unavailable unpublished oxidation Vogue ##copic ##etics Yates ##ppard Leiden Trading cottages Principles ##Millan ##wife ##hiva Vicar nouns strolled ##eorological ##eton ##science precedent Armand Guido rewards ##ilis ##tise clipped chick ##endra averages tentatively 1830s ##vos Certainly 305 Société Commandant ##crats ##dified ##nka marsh angered ventilation Hutton Ritchie ##having Eclipse flick motionless Amor Fest Loire lays ##icit ##sband Guggenheim Luck disrupted ##ncia Disco ##vigator criticisms grins ##lons ##vial ##ody salute Coaches junk saxophonist ##eology Uprising Diet ##marks chronicles robbed ##iet ##ahi Bohemian magician wavelength Kenyan augmented fashionable ##ogies Luce F1 Monmouth ##jos ##loop enjoyment exemption Centers ##visor Soundtrack blinding practitioner solidarity sacrificed ##oso ##cture ##riated blended Abd Copyright ##nob 34th ##reak Claudio hectare rotor testify ##ends ##iably ##sume landowner ##cess ##ckman Eduard Silesian backseat mutually ##abe Mallory bounds Collective Poet Winkler pertaining scraped Phelps crane flickering Proto bubbles popularized removes ##86 Cadillac Warfare audible rites shivering ##sist ##nst ##biotic Mon fascist Bali Kathryn ambiguous furiously morale patio Sang inconsistent topology Greens monkeys Köppen 189 Toy vow ##ías bombings ##culus improvised lodged subsidiaries garment startling practised Hume Thorn categorized Till Eileen wedge ##64 Federico patriotic unlock ##oshi badminton Compared Vilnius ##KE Crimean Kemp decks spaced resolutions sighs ##mind Imagine Cartoon huddled policemen forwards ##rouch equals ##nter inspected Charley MG ##rte pamphlet Arturo dans scarcely ##ulton ##rvin parental unconstitutional watts Susannah Dare ##sitive Rowland Valle invalid ##ué Detachment acronym Yokohama verified ##lsson groove Liza clarified compromised 265 ##rgon ##orf hesitant Fruit Application Mathias icons ##cell Qin interventions ##uron punt remnant ##rien Ames manifold spines floral ##zable comrades Fallen orbits Annals hobby Auditorium implicated researching Pueblo Ta terminate ##pella Rings approximation fuzzy ##ús thriving ##ket Conor alarmed etched Cary ##rdon Ally ##rington Pay mint ##hasa ##unity ##dman ##itate Oceania furrowed trams ##aq Wentworth ventured choreography prototypes Patel mouthed trenches ##licing ##yya Lies deception ##erve ##vations Bertrand earthquakes ##tography Southwestern ##aja token Gupta ##yō Beckett initials ironic Tsar subdued shootout sobbing liar Scandinavia Souls ch therapist trader Regulation Kali busiest ##pation 32nd Telephone Vargas ##moky ##nose ##uge Favorite abducted bonding 219 255 correction mat drown fl unbeaten Pocket Summers Quite rods Percussion ##ndy buzzing cadet Wilkes attire directory utilities naive populous Hendrix ##actor disadvantage 1400 Landon Underworld ##ense Occasionally mercury Davey Morley spa wrestled ##vender eclipse Sienna supplemented thou Stream liturgical ##gall ##berries ##piration 1769 Bucks abandoning ##jutant ##nac 232 venom ##31 Roche dotted Currie Córdoba Milo Sharif divides justification prejudice fortunate ##vide ##ābād Rowe inflammatory ##eld avenue Sources ##rimal Messenger Blanco advocating formulation ##pute emphasizes nut Armored ##ented nutrients ##tment insistence Martins landowners ##RB comparatively headlines snaps ##qing Celebration ##mad republican ##NE Trace ##500 1771 proclamation NRL Rubin Buzz Weimar ##AG 199 posthumous ##ental ##deacon Distance intensely overheard Arcade diagonal hazard Giving weekdays ##ù Verdi actresses ##hare Pulling ##erries ##pores catering shortest ##ctors ##cure ##restle ##reta ##runch ##brecht ##uddin Moments senate Feng Prescott ##thest 218 divisional Bertie sparse surrounds coupling gravitational werewolves ##lax Rankings ##mated ##tries Shia ##mart ##23 ##vocative interfaces morphology newscast ##bide inputs solicitor Olaf cabinets puzzles ##tains Unified ##firmed WA solemn ##opy Tito Jaenelle Neolithic horseback ##ires pharmacy prevalence ##lint Swami ##bush ##tudes Philipp mythical divers Scouting aperture progressively ##bay ##nio bounce Floor ##elf Lucan adulthood helm Bluff Passage Salvation lemon napkin scheduling ##gets Elements Mina Novak stalled ##llister Infrastructure ##nky ##tania ##uished Katz Norma sucks trusting 1765 boilers Accordingly ##hered 223 Crowley ##fight ##ulo Henrietta ##hani pounder surprises ##chor ##glia Dukes ##cracy ##zier ##fs Patriot silicon ##VP simulcast telegraph Mysore cardboard Len ##QL Auguste accordion analytical specify ineffective hunched abnormal Transylvania ##dn ##tending Emilia glittering Maddy ##wana 1762 External Lecture endorsement Hernández Anaheim Ware offences ##phorus Plantation popping Bonaparte disgusting neared ##notes Identity heroin nicely ##raverse apron congestion ##PR padded ##fts invaders ##came freshly Halle endowed fracture ROM ##max sediments diffusion dryly ##tara Tam Draw Spin Talon Anthropology ##lify nausea ##shirt insert Fresno capitalist indefinitely apples Gift scooped 60s Cooperative mistakenly ##lover murmur ##iger Equipment abusive orphanage ##9th ##lterweight ##unda Baird ant saloon 33rd Chesapeake ##chair ##sound ##tend chaotic pornography brace ##aret heiress SSR resentment Arbor headmaster ##uren unlimited ##with ##jn Bram Ely Pokémon pivotal ##guous Database Marta Shine stumbling ##ovsky ##skin Henley Polk functioned ##layer ##pas ##udd ##MX blackness cadets feral Damian ##actions 2D ##yla Apocalypse ##aic inactivated ##china ##kovic ##bres destroys nap Macy sums Madhya Wisdom rejects ##amel 60th Cho bandwidth ##sons ##obbing ##orama Mutual shafts ##estone ##rsen accord replaces waterfront ##gonal ##rida convictions ##ays calmed suppliers Cummings GMA fearful Scientist Sinai examines experimented Netflix Enforcement Scarlett ##lasia Healthcare ##onte Dude inverted ##36 ##regation ##lidae Munro ##angay Airbus overlapping Drivers lawsuits bodily ##udder Wanda Effects Fathers ##finery ##islav Ridley observatory pod ##utrition Electricity landslide ##mable ##zoic ##imator ##uration Estates sleepy Nickelodeon steaming irony schedules snack spikes Hmm ##nesia ##bella ##hibit Greenville plucked Harald ##ono Gamma infringement roaring deposition ##pol ##orum 660 seminal passports engagements Akbar rotated ##bina ##gart Hartley ##lown ##truct uttered traumatic Dex ##ôme Holloway MV apartheid ##nee Counter Colton OR 245 Spaniards Regency Schedule scratching squads verify ##alk keyboardist rotten Forestry aids commemorating ##yed ##érie Sting ##elly Dai ##fers ##berley ##ducted Melvin cannabis glider ##enbach ##rban Costello Skating cartoonist AN audit ##pectator distributing 226 312 interpreter header Alternatively ##ases smug ##kumar cabins remastered Connolly Kelsey LED tentative Check Sichuan shaved ##42 Gerhard Harvest inward ##rque Hopefully hem ##34 Typical binds wrath Woodstock forcibly Fergus ##charged ##tured prepares amenities penetration ##ghan coarse ##oned enthusiasts ##av ##twined fielded ##cky Kiel ##obia 470 beers tremble youths attendees ##cademies ##sex Macon communism dir ##abi Lennox Wen differentiate jewel ##SO activate assert laden unto Gillespie Guillermo accumulation ##GM NGO Rosenberg calculating drastically ##omorphic peeled Liège insurgents outdoors ##enia Aspen Sep awakened ##eye Consul Maiden insanity ##brian furnace Colours distributions longitudinal syllables ##scent Martian accountant Atkins husbands sewage zur collaborate highlighting ##rites ##PI colonization nearer ##XT dunes positioning Ku multitude luxurious Volvo linguistics plotting squared ##inder outstretched ##uds Fuji ji ##feit ##ahu ##loat ##gado ##luster ##oku América ##iza Residents vine Pieces DD Vampires ##ová smoked harshly spreads ##turn ##zhi betray electors ##settled Considering exploits stamped Dusty enraged Nairobi ##38 intervened ##luck orchestras ##lda Hereford Jarvis calf ##itzer ##CH salesman Lovers cigar Angelica doomed heroine ##tible Sanford offenders ##ulously articulated ##oam Emanuel Gardiner Edna Shu gigantic ##stable Tallinn coasts Maker ale stalking ##oga ##smus lucrative southbound ##changing Reg ##lants Schleswig discount grouping physiological ##OH ##sun Galen assurance reconcile rib scarlet Thatcher anarchist ##oom Turnpike ##ceding cocktail Sweeney Allegheny concessions oppression reassuring ##poli ##ticus ##TR ##VI ##uca ##zione directional strikeouts Beneath Couldn Kabul ##national hydroelectric ##jit Desire ##riot enhancing northbound ##PO Ok Routledge volatile Bernardo Python 333 ample chestnut automobiles ##innamon ##care ##hering BWF salaries Turbo acquisitions ##stituting strengths pilgrims Ponce Pig Actors Beard sanitation ##RD ##mett Telecommunications worms ##idas Juno Larson Ventura Northeastern weighs Houghton collaborating lottery ##rano Wonderland gigs ##lmer ##zano ##edd ##nife mixtape predominant tripped ##ruly Alexei investing Belgarath Brasil hiss ##crat ##xham Côte 560 kilometer ##cological analyzing ##As engined listener ##cakes negotiation ##hisky Santana ##lemma IAAF Seneca skeletal Covenant Steiner ##lev ##uen Neptune retention ##upon Closing Czechoslovak chalk Navarre NZ ##IG ##hop ##oly ##quatorial ##sad Brewery Conflict Them renew turrets disagree Petra Slave ##reole adjustment ##dela ##regard ##sner framing stature ##rca ##sies ##46 ##mata Logic inadvertently naturalist spheres towering heightened Dodd rink ##fle Keyboards bulb diver ul ##tsk Exodus Deacon España Canadiens oblique thud reigned rug Whitman Dash ##iens Haifa pets ##arland manually dart ##bial Sven textiles subgroup Napier graffiti revolver humming Babu protector typed Provinces Sparta Wills subjective ##rella temptation ##liest FL Sadie manifest Guangdong Transfer entertain eve recipes ##33 Benedictine retailer ##dence establishes ##cluded ##rked Ursula ##ltz ##lars ##rena qualifiers ##curement colt depictions ##oit Spiritual differentiation staffed transitional ##lew 1761 fatalities ##oan Bayern Northamptonshire Weeks ##CU Fife capacities hoarse ##latt ##ة evidenced ##HD ##ographer assessing evolve hints 42nd streaked ##lve Yahoo ##estive ##rned ##zas baggage Elected secrecy ##champ Character Pen Decca cape Bernardino vapor Dolly counselor ##isers Benin ##khar ##CR notch ##thus ##racy bounty lend grassland ##chtenstein ##dating pseudo golfer simplest ##ceive Lucivar Triumph dinosaur dinosaurs ##šić Seahawks ##nco resorts reelected 1766 reproduce universally ##OA ER tendencies Consolidated Massey Tasmanian reckless ##icz ##ricks 1755 questionable Audience ##lates preseason Quran trivial Haitian Freeway dialed Appointed Heard ecosystems ##bula hormones Carbon Rd ##arney ##working Christoph presiding pu ##athy Morrow Dar ensures posing remedy EA disclosed ##hui ##rten rumours surveying ##ficiency Aziz Jewel Plays ##smatic Bernhard Christi ##eanut ##friend jailed ##dr govern neighbour butler Acheron murdering oils mac Editorial detectives bolts ##ulon Guitars malaria 36th Pembroke Opened ##hium harmonic serum ##sio Franks fingernails ##gli culturally evolving scalp VP deploy uploaded mater ##evo Jammu Spa ##icker flirting ##cursions Heidi Majority sprawled ##alytic Zheng bunker ##lena ST ##tile Jiang ceilings ##ently ##ols Recovery dire ##good Manson Honestly Montréal 1764 227 quota Lakshmi incentive Accounting ##cilla Eureka Reaper buzzed ##uh courtroom dub ##mberg KC Gong Theodor Académie NPR criticizing protesting ##pired ##yric abuses fisheries ##minated 1767 yd Gemini Subcommittee ##fuse Duff Wasn Wight cleaner ##tite planetary Survivor Zionist mounds ##rary landfall disruption yielding ##yana bids unidentified Garry Ellison Elmer Fishing Hayward demos modelling ##anche ##stick caressed entertained ##hesion piers Crimea ##mass WHO boulder trunks 1640 Biennale Palestinians Pursuit ##udes Dora contender ##dridge Nanjing ##ezer ##former ##ibel Whole proliferation ##tide ##weiler fuels predictions ##ente ##onium Filming absorbing Ramón strangled conveyed inhabit prostitutes recession bonded clinched ##eak ##iji ##edar Pleasure Rite Christy Therapy sarcasm ##collegiate hilt probation Sarawak coefficients underworld biodiversity SBS groom brewing dungeon ##claiming Hari turnover ##ntina ##omer ##opped orthodox styling ##tars ##ulata priced Marjorie ##eley ##abar Yong ##tically Crambidae Hernandez ##ego ##rricular ##ark ##lamour ##llin ##augh ##tens Advancement Loyola ##4th ##hh goin marshes Sardinia ##ša Ljubljana Singing suspiciously ##hesive Félix Regarding flap stimulation ##raught Apr Yin gaping tighten skier ##itas ##lad ##rani 264 Ashes Olson Problems Tabitha ##rading balancing sunrise ##ease ##iture ##ritic Fringe ##iciency Inspired Linnaeus PBA disapproval ##kles ##rka ##tails ##urger Disaster Laboratories apps paradise Aero Came sneaking Gee Beacon ODI commodity Ellington graphical Gretchen spire ##skaya ##trine RTÉ efficacy plc tribunal ##ytic downhill flu medications ##kaya widen Sunrise ##nous distinguishing pawn ##BO ##irn ##ssing ##ν Easton ##vila Rhineland ##aque defect ##saurus Goose Ju ##classified Middlesbrough shaping preached 1759 ##erland Ein Hailey musicals ##altered Galileo Hilda Fighters Lac ##ometric 295 Leafs Milano ##lta ##VD ##ivist penetrated Mask Orchard plaintiff ##icorn Yvonne ##fred outfielder peek Collier Caracas repealed Bois dell restrict Dolores Hadley peacefully ##LL condom Granny Orders sabotage ##toon ##rings compass marshal gears brigadier dye Yunnan communicating donate emerald vitamin administer Fulham ##classical ##llas Buckinghamshire Held layered disclosure Akira programmer shrimp Crusade ##ximal Luzon bakery ##cute Garth Citadel uniquely Curling info mum Para ##ști sleek ##ione hey Lantern mesh ##lacing ##lizzard ##gade prosecuted Alba Gilles greedy twists ##ogged Viper ##kata Appearances Skyla hymns ##pelled curving predictable Grave Watford ##dford ##liptic ##vary Westwood fluids Models statutes ##ynamite 1740 ##culate Framework Johanna ##gression Vuelta imp ##otion ##raga ##thouse Ciudad festivities ##love Beyoncé italics ##vance DB ##haman outs Singers ##ueva ##urning ##51 ##ntiary ##mobile 285 Mimi emeritus nesting Keeper Ways ##onal ##oux Edmond MMA ##bark ##oop Hampson ##ñez ##rets Gladstone wreckage Pont Playboy reluctance ##ná apprenticeship preferring Value originate ##wei ##olio Alexia ##rog Parachute jammed stud Eton vols ##ganized 1745 straining creep indicators ##mán humiliation hinted alma tanker ##egation Haynes Penang amazement branched rumble ##ddington archaeologists paranoid expenditure Absolutely Musicians banished ##fining baptism Joker Persons hemisphere ##tieth ##ück flock ##xing lbs Kung crab ##dak ##tinent Regulations barrage parcel ##ós Tanaka ##rsa Natalia Voyage flaws stepfather ##aven ##eological Botanical Minsk ##ckers Cinderella Feast Loving Previous Shark ##took barrister collaborators ##nnes Croydon Graeme Juniors ##7th ##formation ##ulos ##ák £2 ##hwa ##rove ##ș Whig demeanor Otago ##TH ##ooster Faber instructors ##ahl ##bha emptied ##schen saga ##lora exploding ##rges Crusaders ##caster ##uations streaks CBN bows insights ka 1650 diversion LSU Wingspan ##liva Response sanity Producers imitation ##fine Lange Spokane splash weed Siberian magnet ##rocodile capitals ##rgus swelled Rani Bells Silesia arithmetic rumor ##hampton favors Weird marketplace ##orm tsunami unpredictable ##citation ##ferno Tradition postwar stench succeeds ##roup Anya Users oversized totaling pouch ##nat Tripoli leverage satin ##cline Bathurst Lund Niall thereof ##quid Bangor barge Animated ##53 ##alan Ballard utilizes Done ballistic NDP gatherings ##elin ##vening Rockets Sabrina Tamara Tribal WTA ##citing blinded flux Khalid Una prescription ##jee Parents ##otics ##food Silicon cured electro perpendicular intimacy ##rified Lots ##ceiving ##powder incentives McKenna ##arma ##ounced ##rinkled Alzheimer ##tarian 262 Seas ##cam Novi ##hout ##morphic ##hazar ##hul ##nington Huron Bahadur Pirate pursed Griffiths indicted swap refrain ##mulating Lal stomped ##Pad ##mamoto Reef disposed plastered weeping ##rato Minas hourly tumors ##ruising Lyle ##yper ##sol Odisha credibility ##Dowell Braun Graphic lurched muster ##nex ##ührer ##connected ##iek ##ruba Carthage Peck maple bursting ##lava Enrico rite ##jak Moment ##skar Styx poking Spartan ##urney Hepburn Mart Titanic newsletter waits Mecklenburg agitated eats ##dious Chow matrices Maud ##sexual sermon 234 ##sible ##lung Qi cemeteries mined sprinter ##ckett coward ##gable ##hell ##thin ##FB Contact ##hay rainforest 238 Hemisphere boasts ##nders ##verance ##kat Convent Dunedin Lecturer lyricist ##bject Iberian comune ##pphire chunk ##boo thrusting fore informing pistols echoes Tier battleships substitution ##belt moniker ##charya ##lland Thoroughbred 38th ##01 ##tah parting tongues Cale ##seau Unionist modular celebrates preview steamed Bismarck 302 737 vamp ##finity ##nbridge weaknesses husky ##berman absently ##icide Craven tailored Tokugawa VIP syntax Kazan captives doses filtered overview Cleopatra Conversely stallion Burger Suez Raoul th ##reaves Dickson Nell Rate anal colder ##sław Arm Semitic ##green reflective 1100 episcopal journeys ##ours ##pository ##dering residue Gunn ##27 ##ntial ##crates ##zig Astros Renee Emerald ##vili connectivity undrafted Sampson treasures ##kura ##theon ##vern Destroyer ##iable ##ener Frederic briefcase confinement Bree ##WD Athena 233 Padres Thom speeding ##hali Dental ducks Putin ##rcle ##lou Asylum ##usk dusk pasture Institutes ONE jack ##named diplomacy Intercontinental Leagues Towns comedic premature ##edic ##mona ##ories trimmed Charge Cream guarantees Dmitry splashed Philosophical tramway ##cape Maynard predatory redundant ##gratory ##wry sobs Burgundy edible outfits Handel dazed dangerously idle Operational organizes ##sional blackish broker weddings ##halt Becca McGee ##gman protagonists ##pelling Keynes aux stumble ##ordination Nokia reel sexes ##woods ##pheric ##quished ##voc ##oir ##pathian ##ptus ##sma ##tating ##ê fulfilling sheath ##ayne Mei Ordinary Collin Sharpe grasses interdisciplinary ##OX Background ##ignment Assault transforms Hamas Serge ratios ##sik swaying ##rcia Rosen ##gant ##versible cinematographer curly penny Kamal Mellon Sailor Spence phased Brewers amassed Societies ##ropriations ##buted mythological ##SN ##byss ##ired Sovereign preface Parry ##ife altitudes crossings ##28 Crewe southernmost taut McKinley ##owa ##tore 254 ##ckney compiling Shelton ##hiko 228 Poll Shepard Labs Pace Carlson grasping ##ов Delaney Winning robotic intentional shattering ##boarding ##git ##grade Editions Reserves ignorant proposing ##hanna cutter Mongols NW ##eux Codex Cristina Daughters Rees forecast ##hita NGOs Stations Beaux Erwin ##jected ##EX ##trom Schumacher ##hrill ##rophe Maharaja Oricon ##sul ##dynamic ##fighting Ce Ingrid rumbled Prospect stairwell Barnard applause complementary ##uba grunt ##mented Bloc Carleton loft noisy ##hey 490 contrasted ##inator ##rief ##centric ##fica Cantonese Blanc Lausanne License artifact ##ddin rot Amongst Prakash RF ##topia milestone ##vard Winters Mead churchyard Lulu estuary ##ind Cha Infinity Meadow subsidies ##valent CONCACAF Ching medicinal navigate Carver Twice abdominal regulating RB toilets Brewer weakening ambushed ##aut ##vignon Lansing unacceptable reliance stabbing ##mpo ##naire Interview ##ested ##imed bearings ##lts Rashid ##iation authenticity vigorous ##frey ##uel biologist NFC ##rmaid ##wash Makes ##aunt ##steries withdrawing ##qa Buccaneers bleed inclination stain ##ilo ##ppel Torre privileged cereal trailers alumnus neon Cochrane Mariana caress ##47 ##ients experimentation Window convict signaled ##YP rower Pharmacy interacting 241 Strings dominating kinase Dinamo Wire pains sensations ##suse Twenty20 ##39 spotlight ##hend elemental ##pura Jameson Swindon honoring pained ##ediatric ##lux Psychological assemblies ingredient Martial Penguins beverage Monitor mysteries ##ION emigration mused ##sique crore AMC Funding Chinatown Establishment Finalist enjoyable 1756 ##mada ##rams NO newborn CS comprehend Invisible Siemens ##acon 246 contraction ##volving ##moration ##rok montane ##ntation Galloway ##llow Verity directorial pearl Leaning ##rase Fernandez swallowing Automatic Madness haunting paddle ##UE ##rrows ##vies ##zuki ##bolt ##iber Fender emails paste ##lancing hind homestead hopeless ##dles Rockies garlic fatty shrieked ##ismic Gillian Inquiry Schultz XML ##cius ##uld Domesday grenades northernmost ##igi Tbilisi optimistic ##poon Refuge stacks Bose smash surreal Nah Straits Conquest ##roo ##weet ##kell Gladys CH ##lim ##vitation Doctorate NRHP knocks Bey Romano ##pile 242 Diamonds strides eclectic Betsy clade ##hady ##leashed dissolve moss Suburban silvery ##bria tally turtles ##uctive finely industrialist ##nary Ernesto oz pact loneliness ##hov Tomb multinational risked Layne USL ne ##quiries Ad Message Kamen Kristen reefs implements ##itative educators garments gunshot ##essed ##rve Montevideo vigorously Stamford assemble packaged ##same état Viva paragraph ##eter ##wire Stick Navajo MCA ##pressing ensembles ABA ##zor ##llus Partner raked ##BI Iona thump Celeste Kiran ##iscovered ##rith inflammation ##arel Features loosened ##yclic Deluxe Speak economical Frankenstein Picasso showcased ##zad ##eira ##planes ##linear ##overs monsoon prosecutors slack Horses ##urers Angry coughing ##truder Questions ##tō ##zak challenger clocks ##ieving Newmarket ##acle cursing stimuli ##mming ##qualified slapping ##vasive narration ##kini Advertising CSI alliances mixes ##yes covert amalgamation reproduced ##ardt ##gis 1648 id Annette Boots Champagne Brest Daryl ##emon ##jou ##llers Mean adaptive technicians ##pair ##usal Yoga fronts leaping Jul harvesting keel ##44 petitioned ##lved yells Endowment proponent ##spur ##tised ##zal Homes Includes ##ifer ##oodoo ##rvette awarding mirrored ransom Flute outlook ##ganj DVDs Sufi frontman Goddard barren ##astic Suicide hillside Harlow Lau notions Amnesty Homestead ##irt GE hooded umpire mustered Catch Masonic ##erd Dynamics Equity Oro Charts Mussolini populace muted accompaniment ##lour ##ndes ignited ##iferous ##laced ##atch anguish registry ##tub ##hards ##neer 251 Hooker uncomfortably ##6th ##ivers Catalina MiG giggling 1754 Dietrich Kaladin pricing ##quence Sabah ##lving ##nical Gettysburg Vita Telecom Worst Palais Pentagon ##brand ##chichte Graf unnatural 1715 bio ##26 Radcliffe ##utt chatting spices ##aus untouched ##eper Doll turkey Syndicate ##rlene ##JP ##roots Como clashed modernization 1757 fantasies ##iating dissipated Sicilian inspect sensible reputed ##final Milford poised RC metabolic Tobacco Mecca optimization ##heat lobe rabbits NAS geologist ##liner Kilda carpenter nationalists ##brae summarized ##venge Designer misleading beamed ##meyer Matrix excuses ##aines ##biology 401 Moose drafting Sai ##ggle Comprehensive dripped skate ##WI ##enan ##ruk narrower outgoing ##enter ##nounce overseen ##structure travellers banging scarred ##thing ##arra Ebert Sometime ##nated BAFTA Hurricanes configurations ##MLL immortality ##heus gothic ##mpest clergyman viewpoint Maxim Instituto emitted quantitative 1689 Consortium ##rsk Meat Tao swimmers Shaking Terence mainline ##linity Quantum ##rogate Nair banquet 39th reprised lagoon subdivisions synonymous incurred password sprung ##vere Credits Petersen Faces ##vu statesman Zombie gesturing ##going Sergey dormant possessive totals southward Ángel ##odies HM Mariano Ramirez Wicked impressions ##Net ##cap ##ème Transformers Poker RIAA Redesignated ##chuk Harcourt Peña spacious tinged alternatively narrowing Brigham authorization Membership Zeppelin ##amed Handball steer ##orium ##rnal ##rops Committees endings ##MM ##yung ejected grams ##relli Birch Hilary Stadion orphan clawed ##kner Motown Wilkins ballads outspoken ##ancipation ##bankment ##cheng Advances harvested novelty ineligible oversees ##´s obeyed inevitably Kingdoms burying Fabian relevance Tatiana ##MCA sarcastic ##onda Akron 229 sandwiches Adobe Maddox ##azar Hunting ##onized Smiling ##tology Juventus Leroy Poets attach lo ##rly ##film Structure ##igate olds projections SMS outnumbered ##tase judiciary paramilitary playfully ##rsing ##tras Chico Vin informally abandonment ##russ Baroness injuring octagonal deciduous ##nea ##olm Hz Norwood poses Marissa alerted willed ##KS Dino ##ddler ##vani Barbie Thankfully 625 bicycles shimmering ##tinuum ##wolf Chesterfield ##idy ##urgency Knowles sweetly Ventures ##ponents ##valence Darryl Powerplant RAAF ##pec Kingsley Parramatta penetrating spectacle ##inia Marlborough residual compatibility hike Underwood depleted ministries ##odus ##ropriation rotting Faso ##inn Happiness Lille Suns cookie rift warmly ##lvin Bugs Gotham Gothenburg Properties ##seller ##ubi Created MAC Noelle Requiem Ulysses ##ails franchises ##icious ##rwick celestial kinetic 720 STS transmissions amplitude forums freeing reptiles tumbling ##continent ##rising ##tropy physiology ##uster Loves bodied neutrality Neumann assessments Vicky ##hom hampered ##uku Custom timed ##eville ##xious elastic ##section rig stilled shipment 243 artworks boulders Bournemouth ##hly ##LF ##linary rumored ##bino ##drum Chun Freiburg ##dges Equality 252 Guadalajara ##sors ##taire Roach cramped ##ultural Logistics Punch fines Lai caravan ##55 lame Collector pausing 315 migrant hawk signalling ##erham ##oughs Demons surfing Rana insisting Wien adolescent ##jong ##rera ##umba Regis brushes ##iman residues storytelling Consider contrasting regeneration ##elling ##hlete afforded reactors costing ##biotics ##gat ##евич chanting secondly confesses ##ikos ##uang ##ronological ##− Giacomo ##eca vaudeville weeds rejecting revoked affluent fullback progresses geologic proprietor replication gliding recounted ##bah ##igma Flow ii newcomer ##lasp ##miya Candace fractured interiors confidential Inverness footing ##robe Coordinator Westphalia jumper ##chism dormitory ##gno 281 acknowledging leveled ##éra Algiers migrate Frog Rare ##iovascular ##urous DSO nomadic ##iera woken lifeless ##graphical ##ifications Dot Sachs crow nmi Tacoma Weight mushroom RS conditioned ##zine Tunisian altering ##mizing Handicap Patti Monsieur clicking gorge interrupting ##powerment drawers Serra ##icides Specialist ##itte connector worshipped ##ask consoles tags ##iler glued ##zac fences Bratislava honeymoon 313 A2 disposition Gentleman Gilmore glaciers ##scribed Calhoun convergence Aleppo shortages ##43 ##orax ##worm ##codes ##rmal neutron ##ossa Bloomberg Salford periodicals ##ryan Slayer ##ynasties credentials ##tista surveyor File stinging unnoticed Medici ecstasy espionage Jett Leary circulating bargaining concerto serviced 37th HK ##fueling Delilah Marcia graded ##join Kaplan feasible ##nale ##yt Burnley dreadful ministerial Brewster Judah ##ngled ##rrey recycled Iroquois backstage parchment ##numbered Kern Motorsports Organizations ##mini Seems Warrington Dunbar Ezio ##eor paralyzed Ara yeast ##olis cheated reappeared banged ##ymph ##dick Lyndon glide Mat ##natch Hotels Household parasite irrelevant youthful ##smic ##tero ##anti 2d Ignacio squash ##nets shale ##اد Abrams ##oese assaults ##dier ##otte Swamp 287 Spurs ##economic Fargo auditioned ##mé Haas une abbreviation Turkic ##tisfaction favorites specials ##lial Enlightenment Burkina ##vir Comparative Lacrosse elves ##lerical ##pear Borders controllers ##villa excelled ##acher ##varo camouflage perpetual ##ffles devoid schooner ##bered ##oris Gibbons Lia discouraged sue ##gnition Excellent Layton noir smack ##ivable ##evity ##lone Myra weaken weaponry ##azza Shake backbone Certified clown occupational caller enslaved soaking Wexford perceive shortlisted ##pid feminism Bari Indie ##avelin ##ldo Hellenic Hundreds Savings comedies Honors Mohawk Told coded Incorporated hideous trusts hose Calais Forster Gabon Internationale AK Colour ##UM ##heist McGregor localized ##tronomy Darrell ##iara squirrel freaked ##eking ##manned ##ungen radiated ##dua commence Donaldson ##iddle MR SAS Tavern Teenage admissions Instruments ##ilizer Konrad contemplated ##ductor Jing Reacher recalling Dhabi emphasizing illumination ##tony legitimacy Goethe Ritter McDonnell Polar Seconds aspiring derby tunic ##rmed outlines Changing distortion ##cter Mechanics ##urly ##vana Egg Wolverine Stupid centralized knit ##Ms Saratoga Ogden storylines ##vres lavish beverages ##grarian Kyrgyzstan forcefully superb Elm Thessaloniki follower Plants slang trajectory Nowadays Bengals Ingram perch coloring carvings doubtful ##aph ##gratulations ##41 Curse 253 nightstand Campo Meiji decomposition ##giri McCormick Yours ##amon ##bang Texans injunction organise periodical ##peculative oceans ##aley Success Lehigh ##guin 1730 Davy allowance obituary ##tov treasury ##wayne euros readiness systematically ##stered ##igor ##xen ##cliff ##lya Send ##umatic Celtics Judiciary 425 propagation rebellious ##ims ##lut Dal ##ayman ##cloth Boise pairing Waltz torment Hatch aspirations diaspora ##hame Rank 237 Including Muir chained toxicity Université ##aroo Mathews meadows ##bio Editing Khorasan ##them ##ahn ##bari ##umes evacuate ##sium gram kidnap pinning ##diation ##orms beacon organising McGrath ##ogist Qur Tango ##ceptor ##rud ##cend ##cie ##jas ##sided Tuscany Venture creations exhibiting ##rcerer ##tten Butcher Divinity Pet Whitehead falsely perished handy Moines cyclists synthesizers Mortal notoriety ##ronic Dialogue expressive uk Nightingale grimly vineyards Driving relentless compiler ##district ##tuated Hades medicines objection Answer Soap Chattanooga ##gogue Haryana Parties Turtle ##ferred explorers stakeholders ##aar ##rbonne tempered conjecture ##tee ##hur Reeve bumper stew ##church ##generate ##ilitating ##chanized ##elier ##enne translucent ##lows Publisher evangelical inherit ##rted 247 SmackDown bitterness lesions ##worked mosques wed ##lashes Ng Rebels booking ##nail Incident Sailing yo confirms Chaplin baths ##kled modernist pulsing Cicero slaughtered boasted ##losure zipper ##hales aristocracy halftime jolt unlawful Marching sustaining Yerevan bracket ram Markus ##zef butcher massage ##quisite Leisure Pizza collapsing ##lante commentaries scripted ##disciplinary ##sused eroded alleging vase Chichester Peacock commencement dice hotter poisonous executions ##occo frost fielding vendor Counts Troops maize Divisional analogue shadowy Nuevo Ville radiating worthless Adriatic Buy blaze brutally horizontally longed ##matical federally Rolf Root exclude rag agitation Lounge astonished ##wirl Impossible transformations ##IVE ##ceded ##slav downloaded fucked Egyptians Welles ##ffington U2 befriended radios ##jid archaic compares ##ccelerator ##imated ##tosis Hung Scientists Thousands geographically ##LR Macintosh fluorescent ##ipur Wehrmacht ##BR ##firmary Chao ##ague Boyer ##grounds ##hism ##mento ##taining infancy ##cton 510 Boca ##loy 1644 ben dong stresses Sweat expressway graders ochreous nets Lawn thirst Uruguayan satisfactory ##tracts baroque rusty ##ław Shen Gdańsk chickens ##graving Hodge Papal SAT bearer ##ogo ##rger merits Calendar Highest Skills ##ortex Roberta paradigm recounts frigates swamps unitary ##oker balloons Hawthorne Muse spurred advisors reclaimed stimulate fibre pat repeal ##dgson ##iar ##rana anthropologist descends flinch reared ##chang ##eric ##lithic commissioning ##cumenical ##lume ##rchen Wolff ##tsky Eurasian Nepali Nightmare ZIP playback ##latz ##vington Warm ##75 Martina Rollins Saetan Variations sorting ##م 530 Joaquin Ptolemy thinner ##iator ##pticism Cebu Highlanders Linden Vanguard ##SV ##mor ##ulge ISSN cartridges repression Étienne 311 Lauderdale commodities null ##rb 1720 gearbox ##reator Ang Forgotten dubious ##rls ##dicative ##phate Groove Herrera ##çais Collections Maximus ##published Fell Qualification filtering ##tized Roe hazards ##37 ##lative ##tröm Guadalupe Tajikistan Preliminary fronted glands ##paper ##iche ##iding Cairns rallies Location seduce ##mple BYU ##itic ##FT Carmichael Prentice songwriters forefront Physicians ##rille ##zee Preparatory ##cherous UV ##dized Navarro misses ##nney Inland resisting ##sect Hurt ##lino galaxies ##raze Institutions devote ##lamp ##ciating baron ##bracing Hess operatic ##CL ##ος Chevalier Guiana ##lattered Fed ##cuted ##smo Skull denies 236 Waller ##mah Sakura mole nominate sermons ##bering widowed ##röm Cavendish ##struction Nehru Revelation doom Gala baking Nr Yourself banning Individuals Sykes orchestrated 630 Phone steered 620 specialising starvation ##AV ##alet ##upation seductive ##jects ##zure Tolkien Benito Wizards Submarine dictator Duo Caden approx basins ##nc shrink ##icles ##sponsible 249 mit outpost ##bayashi ##rouse ##tl Jana Lombard RBIs finalized humanities ##function Honorable tomato ##iot Pie tee ##pect Beaufort Ferris bucks ##graduate ##ocytes Directory anxiously ##nating flanks ##Ds virtues ##believable Grades criterion manufactures sourced ##balt ##dance ##tano Ying ##BF ##sett adequately blacksmith totaled trapping expanse Historia Worker Sense ascending housekeeper ##oos Crafts Resurrection ##verty encryption ##aris ##vat ##pox ##runk ##iability gazes spying ##ths helmets wired ##zophrenia Cheung WR downloads stereotypes 239 Lucknow bleak Bragg hauling ##haft prohibit ##ermined ##castle barony ##hta Typhoon antibodies ##ascism Hawthorn Kurdistan Minority Gorge Herr appliances disrupt Drugs Lazarus ##ilia ##ryo ##tany Gotta Masovian Roxy choreographed ##rissa turbulent ##listed Anatomy exiting ##det ##isław 580 Kaufman sage ##apa Symposium ##rolls Kaye ##ptera ##rocław jerking ##menclature Guo M1 resurrected trophies ##lard Gathering nestled serpent Dow reservoirs Claremont arbitration chronicle eki ##arded ##zers ##mmoth Congregational Astronomical NE RA Robson Scotch modelled slashed ##imus exceeds ##roper ##utile Laughing vascular superficial ##arians Barclay Caucasian classmate sibling Kimberly Shreveport ##ilde ##liche Cheney Deportivo Veracruz berries ##lase Bed MI Anatolia Mindanao broadband ##olia ##arte ##wab darts ##immer ##uze believers ordinance violate ##wheel ##ynth Alongside Coupe Hobbs arrondissement earl townland ##dote ##lihood ##sla Ghosts midfield pulmonary ##eno cues ##gol ##zda 322 Siena Sultanate Bradshaw Pieter ##thical Raceway bared competence ##ssent Bet ##urer ##ła Alistair Göttingen appropriately forge ##osterone ##ugen DL 345 convoys inventions ##resses ##cturnal Fay Integration slash ##roats Widow barking ##fant 1A Hooper ##cona ##runched unreliable ##emont ##esign ##stabulary ##stop Journalists bony ##iba ##trata ##ège horrific ##bish Jocelyn ##rmon ##apon ##cier trainers ##ulatory 1753 BR corpus synthesized ##bidden ##rafford Elgin ##entry Doherty clockwise ##played spins ##ample ##bley Cope constructions seater warlord Voyager documenting fairies ##viator Lviv jewellery suites ##gold Maia NME ##eavor ##kus Eugène furnishings ##risto MCC Metropolis Older Telangana ##mpus amplifier supervising 1710 buffalo cushion terminating ##powering steak Quickly contracting dem sarcastically Elsa ##hein bastards narratives Takes 304 composure typing variance ##ifice Softball ##rations McLaughlin gaped shrines ##hogany Glamorgan ##icle ##nai ##ntin Fleetwood Woodland ##uxe fictitious shrugs ##iper BWV conform ##uckled Launch ##ductory ##mized Tad ##stituted ##free Bel Chávez messing quartz ##iculate ##folia ##lynn ushered ##29 ##ailing dictated Pony ##opsis precinct 802 Plastic ##ughter ##uno ##porated Denton Matters SPD hating ##rogen Essential Deck Dortmund obscured ##maging Earle ##bred ##ittle ##ropolis saturated ##fiction ##ression Pereira Vinci mute warehouses ##ún biographies ##icking sealing ##dered executing pendant ##wives murmurs ##oko substrates symmetrical Susie ##mare Yusuf analogy ##urage Lesley limitation ##rby ##ío disagreements ##mise embroidered nape unarmed Sumner Stores dwell Wilcox creditors ##rivatization ##shes ##amia directs recaptured scouting McGuire cradle ##onnell Sato insulin mercenary tolerant Macquarie transitions cradled ##berto ##ivism ##yotes FF Ke Reach ##dbury 680 ##bill ##oja ##sui prairie ##ogan reactive ##icient ##rits Cyclone Sirius Survival Pak ##coach ##trar halves Agatha Opus contrasts ##jection ominous ##iden Baylor Woodrow duct fortification intercourse ##rois Colbert envy ##isi Afterward geared ##flections accelerate ##lenching Witness ##rrer Angelina Material assertion misconduct Nix cringed tingling ##eti ##gned Everest disturb sturdy ##keepers ##vied Profile heavenly ##kova ##victed translating ##sses 316 Invitational Mention martyr ##uristic Barron hardness Nakamura 405 Genevieve reflections ##falls jurist ##LT Pyramid ##yme Shoot heck linguist ##tower Ives superiors ##leo Achilles ##phological Christophe Padma precedence grassy Oral resurrection ##itting clumsy ##lten ##rue huts ##stars Equal ##queduct Devin Gaga diocesan ##plating ##upe ##graphers Patch Scream hail moaning tracts ##hdi Examination outsider ##ergic ##oter Archipelago Havilland greenish tilting Aleksandr Konstantin warship ##emann ##gelist ##ought billionaire ##blivion 321 Hungarians transplant ##jured ##fters Corbin autism pitchers Garner thence Scientology transitioned integrating repetitive ##dant Rene vomit ##burne 1661 Researchers Wallis insulted wavy ##wati Ewing excitedly ##kor frescoes injustice ##achal ##lumber ##úl novella ##sca Liv ##enstein ##river monstrous topping downfall looming sinks trillion ##pont Effect ##phi ##urley Sites catchment ##H1 Hopper ##raiser 1642 Maccabi lance ##chia ##sboro NSA branching retorted tensor Immaculate drumming feeder ##mony Dyer homicide Temeraire fishes protruding skins orchards ##nso inlet ventral ##finder Asiatic Sul 1688 Melinda assigns paranormal gardening Tau calming ##inge ##crow regimental Nik fastened correlated ##gene ##rieve Sick ##minster ##politan hardwood hurled ##ssler Cinematography rhyme Montenegrin Packard debating ##itution Helens Trick Museums defiance encompassed ##EE ##TU ##nees ##uben ##ünster ##nosis 435 Hagen cinemas Corbett commended ##fines ##oman bosses ripe scraping ##loc filly Saddam pointless Faust Orléans Syriac ##♭ longitude ##ropic Alfa bliss gangster ##ckling SL blending ##eptide ##nner bends escorting ##bloid ##quis burials ##sle ##è Ambulance insults ##gth Antrim unfolded ##missible splendid Cure warily Saigon Waste astonishment boroughs ##VS ##dalgo ##reshing ##usage rue marital versatile unpaid allotted bacterium ##coil ##cue Dorothea IDF ##location ##yke RPG ##tropical devotees liter ##pree Johnstone astronaut attends pollen periphery doctrines meta showered ##tyn GO Huh laude 244 Amar Christensen Ping Pontifical Austen raiding realities ##dric urges ##dek Cambridgeshire ##otype Cascade Greenberg Pact ##cognition ##aran ##urion Riot mimic Eastwood ##imating reversal ##blast ##henian Pitchfork ##sunderstanding Staten WCW lieu ##bard ##sang experimenting Aquino ##lums TNT Hannibal catastrophic ##lsive 272 308 ##otypic 41st Highways aggregator ##fluenza Featured Reece dispatch simulated ##BE Communion Vinnie hardcover inexpensive til ##adores groundwater kicker blogs frenzy ##wala dealings erase Anglia ##umour Hapoel Marquette ##raphic ##tives consult atrocities concussion ##érard Decree ethanol ##aen Rooney ##chemist ##hoot 1620 menacing Schuster ##bearable laborers sultan Juliana erased onstage ##ync Eastman ##tick hushed ##yrinth Lexie Wharton Lev ##PL Testing Bangladeshi ##bba ##usions communicated integers internship societal ##odles Loki ET Ghent broadcasters Unix ##auer Kildare Yamaha ##quencing ##zman chilled ##rapped ##uant Duval sentiments Oliveira packets Horne ##rient Harlan Mirage invariant ##anger ##tensive flexed sweetness ##wson alleviate insulting limo Hahn ##llars ##hesia ##lapping buys ##oaming mocked pursuits scooted ##conscious ##ilian Ballad jackets ##kra hilly ##cane Scenic McGraw silhouette whipping ##roduced ##wark ##chess ##rump Lemon calculus demonic ##latine Bharatiya Govt Que Trilogy Ducks Suit stairway ##ceipt Isa regulator Automobile flatly ##buster ##lank Spartans topography Tavi usable Chartered Fairchild ##sance ##vyn Digest nuclei typhoon ##llon Alvarez DJs Grimm authoritative firearm ##chschule Origins lair unmistakable ##xial ##cribing Mouth ##genesis ##shū ##gaon ##ulter Jaya Neck ##UN ##oing ##static relativity ##mott ##utive ##esan ##uveau BT salts ##roa Dustin preoccupied Novgorod ##asus Magnum tempting ##histling ##ilated Musa ##ghty Ashland pubs routines ##etto Soto 257 Featuring Augsburg ##alaya Bit loomed expects ##abby ##ooby Auschwitz Pendleton vodka ##sent rescuing systemic ##inet ##leg Yun applicant revered ##nacht ##ndas Muller characterization ##patient ##roft Carole ##asperated Amiga disconnected gel ##cologist Patriotic rallied assign veterinary installing ##cedural 258 Jang Parisian incarcerated stalk ##iment Jamal McPherson Palma ##oken ##viation 512 Rourke irrational ##rippled Devlin erratic ##NI ##payers Ni engages Portal aesthetics ##rrogance Milne assassins ##rots 335 385 Cambodian Females fellows si ##block ##otes Jayne Toro flutter ##eera Burr ##lanche relaxation ##fra Fitzroy ##undy 1751 261 comb conglomerate ribbons veto ##Es casts ##ege 1748 Ares spears spirituality comet ##nado ##yeh Veterinary aquarium yer Councils ##oked ##ynamic Malmö remorse auditions drilled Hoffmann Moe Nagoya Yacht ##hakti ##race ##rrick Talmud coordinating ##EI ##bul ##his ##itors ##ligent ##uerra Narayan goaltender taxa ##asures Det ##mage Infinite Maid bean intriguing ##cription gasps socket ##mentary ##reus sewing transmitting ##different ##furbishment ##traction Grimsby sprawling Shipyard ##destine ##hropic ##icked trolley ##agi ##lesh Josiah invasions Content firefighters intro Lucifer subunit Sahib Myrtle inhibitor maneuvers ##teca Wrath slippery ##versing Shoes ##dial ##illiers ##luded ##mmal ##pack handkerchief ##edestal ##stones Fusion cumulative ##mell ##cacia ##rudge ##utz foe storing swiped ##meister ##orra batter strung ##venting ##kker Doo Taste immensely Fairbanks Jarrett Boogie 1746 mage Kick legislators medial ##ilon ##logies ##ranton Hybrid ##uters Tide deportation Metz ##secration ##virus UFO ##fell ##orage ##raction ##rrigan 1747 fabricated ##BM ##GR ##rter muttering theorist ##tamine BMG Kincaid solvent ##azed Thin adorable Wendell ta ##viour pulses ##pologies counters exposition sewer Luciano Clancy ##angelo ##riars Showtime observes frankly ##oppy Bergman lobes timetable ##bri ##uest FX ##dust ##genus Glad Helmut Meridian ##besity ##ontaine Revue miracles ##titis PP bluff syrup 307 Messiah ##erne interfering picturesque unconventional dipping hurriedly Kerman 248 Ethnic Toward acidic Harrisburg ##65 intimidating ##aal Jed Pontiac munitions ##nchen growling mausoleum ##ération ##wami Cy aerospace caucus Doing ##around ##miring Cuthbert ##poradic ##rovisation ##wth evaluating ##scraper Belinda owes ##sitic ##thermal ##fast economists ##lishing ##uerre ##ân credible ##koto Fourteen cones ##ebrates bookstore towels ##phony Appearance newscasts ##olin Karin Bingham ##elves 1680 306 disks ##lston ##secutor Levant ##vout Micro snuck ##ogel ##racker Exploration drastic ##kening Elsie endowment ##utnant Blaze ##rrosion leaking 45th ##rug ##uernsey 760 Shapiro cakes ##ehan ##mei ##ité ##kla repetition successively Friendly Île Koreans Au Tirana flourish Spirits Yao reasoned ##leam Consort cater marred ordeal supremacy ##ritable Paisley euro healer portico wetland ##kman restart ##habilitation ##zuka ##Script emptiness communion ##CF ##inhabited ##wamy Casablanca pulsed ##rrible ##safe 395 Dual Terrorism ##urge ##found ##gnolia Courage patriarch segregated intrinsic ##liography ##phe PD convection ##icidal Dharma Jimmie texted constituents twitch ##calated ##mitage ##ringing 415 milling ##geons Armagh Geometridae evergreen needy reflex template ##pina Schubert ##bruck ##icted ##scher ##wildered 1749 Joanne clearer ##narl 278 Print automation consciously flashback occupations ##ests Casimir differentiated policing repay ##aks ##gnesium Evaluation commotion ##CM ##smopolitan Clapton mitochondrial Kobe 1752 Ignoring Vincenzo Wet bandage ##rassed ##unate Maris ##eted ##hetical figuring ##eit ##nap leopard strategically ##reer Fen Iain ##ggins ##pipe Matteo McIntyre ##chord ##feng Romani asshole flopped reassure Founding Styles Torino patrolling ##erging ##ibrating ##ructural sincerity ##ät ##teacher Juliette ##cé ##hog ##idated ##span Winfield ##fender ##nast ##pliant 1690 Bai Je Saharan expands Bolshevik rotate ##root Britannia Severn ##cini ##gering ##say sly Steps insertion rooftop Piece cuffs plausible ##zai Provost semantic ##data ##vade ##cimal IPA indictment Libraries flaming highlands liberties ##pio Elders aggressively ##pecific Decision pigeon nominally descriptive adjustments equestrian heaving ##mour ##dives ##fty ##yton intermittent ##naming ##sets Calvert Casper Tarzan ##kot Ramírez ##IB ##erus Gustavo Roller vaulted ##solation ##formatics ##tip Hunger colloquially handwriting hearth launcher ##idian ##ilities ##lind ##locating Magdalena Soo clubhouse ##kushima ##ruit Bogotá Organic Worship ##Vs ##wold upbringing ##kick groundbreaking ##urable ##ván repulsed ##dira ##ditional ##ici melancholy ##bodied ##cchi 404 concurrency H₂O bouts ##gami 288 Leto troll ##lak advising bundled ##nden lipstick littered ##leading ##mogeneous Experiment Nikola grove ##ogram Mace ##jure cheat Annabelle Tori lurking Emery Walden ##riz paints Markets brutality overrun ##agu ##sat din ostensibly Fielding flees ##eron Pound ornaments tornadoes ##nikov ##organisation ##reen ##Works ##ldred ##olten ##stillery soluble Mata Grimes Léon ##NF coldly permitting ##inga ##reaked Agents hostess ##dl Dyke Kota avail orderly ##saur ##sities Arroyo ##ceps ##egro Hawke Noctuidae html seminar ##ggles ##wasaki Clube recited ##sace Ascension Fitness dough ##ixel Nationale ##solidate pulpit vassal 570 Annapolis bladder phylogenetic ##iname convertible ##ppan Comet paler ##definite Spot ##dices frequented Apostles slalom ##ivision ##mana ##runcated Trojan ##agger ##iq ##league Concept Controller ##barian ##curate ##spersed ##tring engulfed inquired ##hmann 286 ##dict ##osy ##raw MacKenzie su ##ienced ##iggs ##quitaine bisexual ##noon runways subsp ##! ##" ### ##$ ##% ##& ##' ##( ##) ##* ##+ ##, ##- ##. ##/ ##: ##; ##< ##= ##> ##? ##@ ##[ ##\ ##] ##^ ##_ ##` ##{ ##| ##} ##~ ##¡ ##¢ ##£ ##¥ ##§ ##¨ ##© ##ª ##« ##¬ ##® ##± ##´ ##µ ##¶ ##· ##¹ ##º ##» ##¼ ##¾ ##¿ ##À ##Á ## ##Ä ##Å ##Æ ##Ç ##È ##É ##Í ##Î ##Ñ ##Ó ##Ö ##× ##Ø ##Ú ##Ü ##Þ ##â ##ã ##æ ##ç ##î ##ï ##ð ##ñ ##ô ##õ ##÷ ##û ##þ ##ÿ ##Ā ##ą ##Ć ##Č ##ď ##Đ ##đ ##ē ##ė ##ę ##ě ##ğ ##ġ ##Ħ ##ħ ##ĩ ##Ī ##İ ##ļ ##Ľ ##ľ ##Ł ##ņ ##ň ##ŋ ##Ō ##ŏ ##ő ##Œ ##œ ##ř ##Ś ##ś ##Ş ##Š ##Ţ ##ţ ##ť ##ũ ##ŭ ##ů ##ű ##ų ##ŵ ##ŷ ##ź ##Ż ##ż ##Ž ##ž ##Ə ##ƒ ##ơ ##ư ##ǎ ##ǐ ##ǒ ##ǔ ##ǫ ##Ș ##Ț ##ț ##ɐ ##ɑ ##ɔ ##ɕ ##ə ##ɛ ##ɡ ##ɣ ##ɨ ##ɪ ##ɲ ##ɾ ##ʀ ##ʁ ##ʂ ##ʃ ##ʊ ##ʋ ##ʌ ##ʐ ##ʑ ##ʒ ##ʔ ##ʰ ##ʲ ##ʳ ##ʷ ##ʻ ##ʼ ##ʾ ##ʿ ##ˈ ##ː ##ˡ ##ˢ ##ˣ ##́ ##̃ ##̍ ##̯ ##͡ ##Α ##Β ##Γ ##Δ ##Ε ##Η ##Θ ##Ι ##Κ ##Λ ##Μ ##Ν ##Ο ##Π ##Σ ##Τ ##Φ ##Χ ##Ψ ##Ω ##ά ##έ ##ή ##ί ##β ##γ ##δ ##ε ##ζ ##η ##θ ##ι ##κ ##λ ##μ ##ξ ##ο ##π ##ρ ##σ ##τ ##υ ##φ ##χ ##ψ ##ω ##ό ##ύ ##ώ ##І ##Ј ##А ##Б ##В ##Г ##Д ##Е ##Ж ##З ##И ##К ##Л ##М ##Н ##О ##П ##Р ##С ##Т ##У ##Ф ##Х ##Ц ##Ч ##Ш ##Э ##Ю ##Я ##б ##в ##г ##д ##ж ##з ##к ##л ##м ##п ##с ##т ##у ##ф ##х ##ц ##ч ##ш ##щ ##ъ ##ы ##ь ##э ##ю ##ё ##і ##ї ##ј ##њ ##ћ ##Ա ##Հ ##ա ##ե ##ի ##կ ##մ ##յ ##ն ##ո ##ս ##տ ##ր ##ւ ##ְ ##ִ ##ֵ ##ֶ ##ַ ##ָ ##ֹ ##ּ ##א ##ב ##ג ##ד ##ה ##ו ##ז ##ח ##ט ##י ##כ ##ל ##ם ##מ ##ן ##נ ##ס ##ע ##פ ##צ ##ק ##ר ##ש ##ת ##، ##ء ##آ ##أ ##إ ##ئ ##ا ##ب ##ت ##ث ##ج ##ح ##خ ##ذ ##ز ##س ##ش ##ص ##ض ##ط ##ظ ##ع ##غ ##ف ##ق ##ك ##ل ##و ##ى ##َ ##ِ ##ٹ ##پ ##چ ##ک ##گ ##ہ ##ی ##ے ##ं ##आ ##क ##ग ##च ##ज ##ण ##त ##द ##ध ##न ##प ##ब ##भ ##म ##य ##र ##ल ##व ##श ##ष ##स ##ह ##ा ##ि ##ी ##ु ##े ##ो ##् ##। ##॥ ##আ ##ই ##এ ##ও ##ক ##খ ##গ ##চ ##ছ ##জ ##ট ##ত ##থ ##দ ##ধ ##ন ##প ##ব ##ম ##য ##র ##ল ##শ ##স ##হ ##় ##া ##ি ##ী ##ু ##ে ##ো ##্ ##য় ##க ##த ##ப ##ம ##ய ##ர ##ல ##வ ##ா ##ி ##ு ##் ##ร ##་ ##ག ##ང ##ད ##ན ##བ ##མ ##ར ##ལ ##ས ##ི ##ུ ##ེ ##ོ ##ა ##ე ##ი ##ლ ##ნ ##ო ##რ ##ს ##ᴬ ##ᴵ ##ᵀ ##ᵃ ##ᵇ ##ᵈ ##ᵉ ##ᵍ ##ᵏ ##ᵐ ##ᵒ ##ᵖ ##ᵗ ##ᵘ ##ᵣ ##ᵤ ##ᵥ ##ᶜ ##ᶠ ##ḍ ##Ḥ ##ḥ ##Ḩ ##ḩ ##ḳ ##ṃ ##ṅ ##ṇ ##ṛ ##ṣ ##ṭ ##ạ ##ả ##ấ ##ầ ##ẩ ##ậ ##ắ ##ế ##ề ##ể ##ễ ##ệ ##ị ##ọ ##ố ##ồ ##ổ ##ộ ##ớ ##ờ ##ợ ##ụ ##ủ ##ứ ##ừ ##ử ##ữ ##ự ##ỳ ##ỹ ##ἀ ##ἐ ##ὁ ##ὐ ##ὰ ##ὶ ##ὸ ##ῆ ##ῖ ##ῦ ##ῶ ##‐ ##‑ ##‒ ##– ##— ##― ##‖ ##‘ ##’ ##‚ ##“ ##” ##„ ##† ##‡ ##• ##… ##‰ ##′ ##″ ##⁄ ##⁰ ##ⁱ ##⁴ ##⁵ ##⁶ ##⁷ ##⁸ ##⁹ ##⁻ ##ⁿ ##₅ ##₆ ##₇ ##₈ ##₉ ##₊ ##₍ ##₎ ##ₐ ##ₑ ##ₒ ##ₓ ##ₕ ##ₖ ##ₘ ##ₚ ##ₛ ##ₜ ##₤ ##€ ##₱ ##₹ ##ℓ ##№ ##ℝ ##⅓ ##← ##↑ ##→ ##↔ ##⇌ ##⇒ ##∂ ##∈ ##∗ ##∘ ##√ ##∞ ##∧ ##∨ ##∩ ##∪ ##≈ ##≠ ##≡ ##≤ ##≥ ##⊂ ##⊆ ##⊕ ##⋅ ##─ ##│ ##■ ##● ##★ ##☆ ##☉ ##♠ ##♣ ##♥ ##♦ ##♯ ##⟨ ##⟩ ##ⱼ ##、 ##。 ##《 ##》 ##「 ##」 ##『 ##』 ##〜 ##い ##う ##え ##お ##か ##き ##く ##け ##こ ##さ ##し ##す ##せ ##そ ##た ##ち ##つ ##て ##と ##な ##に ##の ##は ##ひ ##ま ##み ##む ##め ##も ##や ##ゆ ##よ ##ら ##り ##る ##れ ##ん ##ア ##ィ ##イ ##ウ ##エ ##オ ##カ ##ガ ##キ ##ク ##グ ##コ ##サ ##シ ##ジ ##ス ##ズ ##タ ##ダ ##ッ ##テ ##デ ##ト ##ド ##ナ ##ニ ##ハ ##バ ##パ ##フ ##ブ ##プ ##マ ##ミ ##ム ##ャ ##ュ ##ラ ##リ ##ル ##レ ##ロ ##ン ##・ ##ー ##一 ##三 ##上 ##下 ##中 ##事 ##二 ##井 ##京 ##人 ##亻 ##仁 ##佐 ##侍 ##光 ##公 ##力 ##北 ##十 ##南 ##原 ##口 ##史 ##司 ##吉 ##同 ##和 ##囗 ##国 ##國 ##土 ##城 ##士 ##大 ##天 ##太 ##夫 ##女 ##子 ##宀 ##安 ##宮 ##宿 ##小 ##尚 ##山 ##島 ##川 ##州 ##平 ##年 ##心 ##愛 ##戸 ##文 ##新 ##方 ##日 ##明 ##星 ##書 ##月 ##木 ##本 ##李 ##村 ##東 ##松 ##林 ##正 ##武 ##氏 ##水 ##氵 ##江 ##河 ##海 ##版 ##犬 ##王 ##生 ##田 ##白 ##皇 ##省 ##真 ##石 ##社 ##神 ##竹 ##美 ##義 ##花 ##藤 ##西 ##谷 ##車 ##辶 ##道 ##郎 ##郡 ##部 ##野 ##金 ##長 ##門 ##陽 ##青 ##食 ##馬 ##高 ##龍 ##龸 ##사 ##씨 ##의 ##이 ##한 ##fi ##fl ##！ ##（ ##） ##， ##－ ##／ ##： ================================================ FILE: func_builders/input_fn_builder.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # author: xiaoy li # description: # # import tensorflow as tf def file_based_input_fn_builder(input_file, num_window=None, window_size=None, max_num_mention=None, is_training=False, drop_remainder=True): """Creates an `input_fn` closure to be passed to TPUEstimator.""" name_to_features = { 'sentence_map': tf.FixedLenFeature([num_window * window_size], tf.int64), 'text_len': tf.FixedLenFeature([num_window], tf.int64), 'subtoken_map': tf.FixedLenFeature([num_window * window_size], tf.int64), 'speaker_ids': tf.FixedLenFeature([num_window * window_size], tf.int64), 'flattened_input_ids': tf.FixedLenFeature([num_window * window_size], tf.int64), 'flattened_input_mask': tf.FixedLenFeature([num_window * window_size], tf.int64), 'span_starts': tf.FixedLenFeature([max_num_mention], tf.int64), 'span_ends': tf.FixedLenFeature([max_num_mention], tf.int64), 'cluster_ids': tf.FixedLenFeature([max_num_mention], tf.int64), } def _decode_record(record, name_to_features): """Decodes a record to a TensorFlow example.""" example = tf.io.parse_single_example(record, name_to_features) # tf.Example only supports tf.int64, but the TPU only supports tf.int32. # So cast all int64 to int32. for name in list(example.keys()): t = example[name] if t.dtype == tf.int64: t = tf.to_int32(t) example[name] = t return example def input_fn_from_tfrecord(params): """The actual input function.""" batch_size = params["batch_size"] # For training, we want a lot of parallel reading and shuffling. # For eval, we want no shuffling and parallel reading doesn't matter. d = tf.data.TFRecordDataset(input_file) if is_training: d = d.repeat() d = d.shuffle(buffer_size=100) d = d.apply( tf.contrib.data.map_and_batch( lambda record: _decode_record(record, name_to_features), batch_size=batch_size, drop_remainder=drop_remainder)) return d return input_fn_from_tfrecord ================================================ FILE: func_builders/model_fn_builder.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # author: xiaoy li # description: # # import tensorflow as tf from utils import util from utils.radam import RAdam def model_fn_builder(config, model_sign="mention_proposal"): def mention_proposal_model_fn(features, labels, mode, params): """The `model_fn` for TPUEstimator.""" input_ids = features["flattened_input_ids"] input_mask = features["flattened_input_mask"] text_len = features["text_len"] speaker_ids = features["speaker_ids"] gold_starts = features["span_starts"] gold_ends = features["span_ends"] cluster_ids = features["cluster_ids"] sentence_map = features["sentence_map"] is_training = (mode == tf.estimator.ModeKeys.TRAIN) model = util.get_model(config, model_sign="mention_proposal") if config.use_tpu: def tpu_scaffold(): return tf.train.Scaffold() scaffold_fn = tpu_scaffold else: scaffold_fn = None if mode == tf.estimator.ModeKeys.TRAIN: tf.logging.info("****************************** tf.estimator.ModeKeys.TRAIN ******************************") tf.logging.info("********* Features *********") for name in sorted(features.keys()): tf.logging.info(" name = %s, shape = %s" % (name, features[name].shape)) instance = (input_ids, input_mask, sentence_map, text_len, speaker_ids, gold_starts, gold_ends, cluster_ids) total_loss, start_scores, end_scores, span_scores = model.get_mention_proposal_and_loss(instance, is_training) gold_start_sequence_labels, gold_end_sequence_labels, gold_span_sequence_labels = model.get_gold_mention_sequence_labels_from_pad_index(gold_starts, gold_ends, text_len) if config.use_tpu: optimizer = tf.train.AdamOptimizer(learning_rate=config.learning_rate, beta1=0.9, beta2=0.999, epsilon=1e-08) optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer) train_op = optimizer.minimize(total_loss, tf.train.get_global_step()) output_spec = tf.contrib.tpu.TPUEstimatorSpec( mode=mode, loss=total_loss, train_op=train_op, scaffold_fn=scaffold_fn) else: optimizer = RAdam(learning_rate=config.learning_rate, epsilon=1e-8, beta1=0.9, beta2=0.999) train_op = optimizer.minimize(total_loss, tf.train.get_global_step()) train_logging_hook = tf.train.LoggingTensorHook({"loss": total_loss}, every_n_iter=1) output_spec = tf.contrib.tpu.TPUEstimatorSpec( mode=mode, loss=total_loss, train_op=train_op, scaffold_fn=scaffold_fn, training_hooks=[train_logging_hook]) elif mode == tf.estimator.ModeKeys.EVAL: tf.logging.info("****************************** tf.estimator.ModeKeys.EVAL ******************************") instance = (input_ids, input_mask, sentence_map, text_len, speaker_ids, gold_starts, gold_ends, cluster_ids) total_loss, start_scores, end_scores, span_scores = model.get_mention_proposal_and_loss(instance, is_training) total_loss, start_scores, end_scores, span_scores = model.get_mention_proposal_and_loss(instance, is_training) gold_start_sequence_labels, gold_end_sequence_labels, gold_span_sequence_labels = model.get_gold_mention_sequence_labels_from_pad_index(gold_starts, gold_ends, text_len) def metric_fn(start_scores, end_scores, span_scores, gold_span_label): start_scores = tf.reshape(start_scores, [-1, config.window_size]) end_scores = tf.reshape(end_scores, [-1, config.window_size]) start_scores = tf.tile(tf.expand_dims(start_scores, 2), [1, 1, config.window_size]) end_scores = tf.tile(tf.expand_dims(end_scores, 2), [1, 1, config.window_size]) sce_span_scores = (start_scores + end_scores + span_scores)/ 3 pred_span_label = tf.cast(tf.reshape(tf.math.greater_equal(sce_span_scores, config.mention_threshold), [-1]), tf.bool) gold_span_label = tf.cast(tf.reshape(gold_span_sequence_labels, [-1]), tf.bool) return {"precision": tf.compat.v1.metrics.precision(gold_span_label, pred_span_label), "recall": tf.compat.v1.metrics.recall(gold_span_label, pred_span_label)} eval_metrics = (metric_fn, [start_scores, end_scores, span_scores]) output_spec = tf.contrib.tpu.TPUEstimatorSpec( mode=tf.estimator.ModeKeys.EVAL, loss=total_loss, eval_metrics=eval_metrics, scaffold_fn=scaffold_fn) elif mode == tf.estimator.ModeKeys.PREDICT: tf.logging.info("****************************** tf.estimator.ModeKeys.PREDICT ******************************") instance = (input_ids, input_mask, sentence_map, text_len, speaker_ids, gold_starts, gold_ends, cluster_ids) total_loss, start_scores, end_scores, span_scores = model.get_mention_proposal_and_loss(instance, is_training) gold_start_sequence_labels, gold_end_sequence_labels, gold_span_sequence_labels = model.get_gold_mention_sequence_labels_from_pad_index(gold_starts, gold_ends, text_len) predictions = { "total_loss": total_loss, "start_scores": start_scores, "start_gold": gold_starts, "end_gold": gold_ends, "end_scores": end_scores, "span_scores": span_scores } output_spec = tf.contrib.tpu.TPUEstimatorSpec( mode=tf.estimator.ModeKeys.PREDICT, predictions=predictions, scaffold_fn=scaffold_fn) else: raise ValueError("Please check the the mode ! ") return output_spec def corefqa_model_fn(features, labels, mode, params): """The `model_fn` for TPUEstimator.""" input_ids = features["flattened_input_ids"] input_mask = features["flattened_input_mask"] text_len = features["text_len"] speaker_ids = features["speaker_ids"] gold_starts = features["span_starts"] gold_ends = features["span_ends"] cluster_ids = features["cluster_ids"] sentence_map = features["sentence_map"] is_training = (mode == tf.estimator.ModeKeys.TRAIN) model = util.get_model(config, model_sign="corefqa") if config.use_tpu: tf.logging.info("****************************** Training on TPU ******************************") def tpu_scaffold(): return tf.train.Scaffold() scaffold_fn = tpu_scaffold else: scaffold_fn = None if mode == tf.estimator.ModeKeys.TRAIN: tf.logging.info("****************************** tf.estimator.ModeKeys.TRAIN ******************************") tf.logging.info("********* Features *********") for name in sorted(features.keys()): tf.logging.info(" name = %s, shape = %s" % (name, features[name].shape)) instance = (input_ids, input_mask, sentence_map, text_len, speaker_ids, gold_starts, gold_ends, cluster_ids) total_loss, (topk_mention_start_indices, topk_mention_end_indices), (forward_topc_mention_start_indices, forward_topc_mention_end_indices), top_mention_span_linking_scores = model.get_coreference_resolution_and_loss(instance, is_training, use_tpu=config.use_tpu) if config.use_tpu: optimizer = tf.train.AdamOptimizer(learning_rate=config.learning_rate, beta1=0.9, beta2=0.999, epsilon=1e-08) optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer) train_op = optimizer.minimize(total_loss, tf.train.get_global_step()) output_spec = tf.contrib.tpu.TPUEstimatorSpec( mode=tf.estimator.ModeKeys.TRAIN, loss=total_loss, train_op=train_op, scaffold_fn=scaffold_fn) else: optimizer = RAdam(learning_rate=config.learning_rate, epsilon=1e-8, beta1=0.9, beta2=0.999) train_op = optimizer.minimize(total_loss, tf.train.get_global_step()) training_logging_hook = tf.train.LoggingTensorHook({"loss": total_loss}, every_n_iter=1) output_spec = tf.contrib.tpu.TPUEstimatorSpec( mode=tf.estimator.ModeKeys.TRAIN, loss=total_loss, train_op=train_op, scaffold_fn=scaffold_fn, training_hooks=[training_logging_hook]) elif mode == tf.estimator.ModeKeys.EVAL: tf.logging.info("****************************** tf.estimator.ModeKeys.EVAL ******************************") tf.logging.info("@@@@@ MERELY support tf.estimator.ModeKeys.PREDICT ! @@@@@") tf.logging.info("@@@@@ YOU can EVAL your checkpoints after the training process. @@@@@") tf.logging.info("****************************** tf.estimator.ModeKeys.EVAL ******************************") elif mode == tf.estimator.ModeKeys.PREDICT : tf.logging.info("****************************** tf.estimator.ModeKeys.PREDICT ******************************") instance = (input_ids, input_mask, sentence_map, text_len, speaker_ids, gold_starts, gold_ends, cluster_ids) total_loss, (topk_mention_start_indices, topk_mention_end_indices), (forward_topc_mention_start_indices, forward_topc_mention_end_indices), top_mention_span_linking_scores = model.get_coreference_resolution_and_loss(instance, True, use_tpu=config.use_tpu) top_antecedent = tf.math.argmax(top_mention_span_linking_scores, axis=-1) predictions = { "total_loss": total_loss, "topk_span_starts": topk_mention_start_indices, "topk_span_ends": topk_mention_end_indices, "top_antecedent_scores": top_mention_span_linking_scores, "top_antecedent": top_antecedent, "cluster_ids" : cluster_ids, "gold_starts": gold_starts, "gold_ends": gold_ends} output_spec = tf.contrib.tpu.TPUEstimatorSpec(mode=tf.estimator.ModeKeys.PREDICT, predictions=predictions, scaffold_fn=scaffold_fn) else: raise ValueError("Please check the the mode ! ") return output_spec if model_sign == "mention_proposal": return mention_proposal_model_fn elif model_sign == "corefqa": return corefqa_model_fn else: raise ValueError("Please check the model sign! Only support [mention_proposal] and [corefqa] .") ================================================ FILE: logs/corefqa_log.txt ================================================ /home/xiaoyli1110/venv/lib/python3.5/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`. from ._conv import register_converters as _register_converters /home/xiaoyli1110/xiaoya/Coref-tf W0713 13:36:28.637916 139854454523648 module_wrapper.py:139] From /home/xiaoyli1110/xiaoya/Coref-tf/run/train_corefqa.py:308: The name tf.app.run is deprecated. Please use tf.compat.v1.app.run instead. loading experiments_tpu.conf ... W0713 13:36:28.752999 139854454523648 module_wrapper.py:139] From /home/xiaoyli1110/xiaoya/Coref-tf/utils/util.py:41: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead. I0716 13:36:28.753216 139854454523648 util.py:41] %*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*% I0716 13:36:28.753291 139854454523648 util.py:42] %*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*% I0716 13:36:28.753346 139854454523648 util.py:43] %%%%%%%% Configs are showed as follows : %%%%%%%% I0716 13:36:28.753432 139854454523648 util.py:45] max_top_antecedents : 60 I0716 13:36:28.753505 139854454523648 util.py:45] max_training_sentences : 3 I0716 13:36:28.753576 139854454523648 util.py:45] top_span_ratio : 0.3 I0716 13:36:28.753644 139854454523648 util.py:45] max_num_speakers : 20 I0716 13:36:28.753710 139854454523648 util.py:45] max_segment_len : 384 I0716 13:36:28.753773 139854454523648 util.py:45] max_cluster_num : 30 I0716 13:36:28.753848 139854454523648 util.py:45] tpu : True I0716 13:36:28.753910 139854454523648 util.py:45] max_query_len : 150 I0716 13:36:28.753972 139854454523648 util.py:45] max_context_len : 150 I0716 13:36:28.754034 139854454523648 util.py:45] max_qa_len : 300 I0716 13:36:28.754097 139854454523648 util.py:45] hidden_size : 1024 I0716 13:36:28.754161 139854454523648 util.py:45] max_candidate_mentions : 60 I0716 13:36:28.754229 139854454523648 util.py:45] learning_rate : 8e-06 I0716 13:36:28.754301 139854454523648 util.py:45] num_docs : 5604 I0716 13:36:28.754365 139854454523648 util.py:45] start_ratio : 0.8 I0716 13:36:28.754428 139854454523648 util.py:45] end_ratio : 0.8 I0716 13:36:28.754492 139854454523648 util.py:45] mention_ratio : 1.0 I0716 13:36:28.754556 139854454523648 util.py:45] corefqa_loss_ratio : 0.9 I0716 13:36:28.754620 139854454523648 util.py:45] score_ratio : 0.5 I0716 13:36:28.754682 139854454523648 util.py:45] run : estimator I0716 13:36:28.754746 139854454523648 util.py:45] threshold : 0.5 I0716 13:36:28.754814 139854454523648 util.py:45] dropout_rate : 0.3 I0716 13:36:28.754945 139854454523648 util.py:45] ffnn_size : 1024 I0716 13:36:28.755008 139854454523648 util.py:45] ffnn_depth : 1 I0716 13:36:28.755071 139854454523648 util.py:45] num_epochs : 8 I0716 13:36:28.755135 139854454523648 util.py:45] max_span_width : 30 I0716 13:36:28.755199 139854454523648 util.py:45] use_segment_distance : True I0716 13:36:28.755261 139854454523648 util.py:45] model_heads : True I0716 13:36:28.755324 139854454523648 util.py:45] coref_depth : 2 I0716 13:36:28.755383 139854454523648 util.py:45] corefqa_only_concate : False I0716 13:36:28.755445 139854454523648 util.py:45] train_path : gs://xiaoy-data-europe/overlap_384_3/train.128.english.tfrecord I0716 13:36:28.755507 139854454523648 util.py:45] eval_path : test.english.jsonlines I0716 13:36:28.755571 139854454523648 util.py:45] conll_eval_path : gs://corefqa-europe/spanbert_large_overlap_384_3_out put_2e-5/test.english.v4_gold_conll I0716 13:36:28.755634 139854454523648 util.py:45] single_example : False I0716 13:36:28.755702 139854454523648 util.py:45] genres : ['bc', 'bn', 'mz', 'nw', 'pt', 'tc', 'wb'] I0716 13:36:28.755765 139854454523648 util.py:45] log_root : gs://corefqa-europe/spanbert_large_overlap_384_3_output_2e-5 I0716 13:36:28.755842 139854454523648 util.py:45] save_checkpoints_steps : 1000 I0716 13:36:28.755906 139854454523648 util.py:45] dev_path : gs://xiaoy-data-europe/overlap_384_3/dev.256.english.tfrecord I0716 13:36:28.755968 139854454523648 util.py:45] test_path : gs://xiaoy-data-europe/overlap_384_3/test.256.english.tfrecord I0716 13:36:28.756030 139854454523648 util.py:45] bert_config_file : gs://xiaoy-data-europe/spanbert_large_tf/bert_config.json I0716 13:36:28.756093 139854454523648 util.py:45] vocab_file : gs://xiaoy-data-europe/spanbert_large_tf/vocab.txt I0716 13:36:28.756155 139854454523648 util.py:45] tf_checkpoint : gs://xiaoy-data-europe/spanbert_large_tf/bert_model.ckpt I0716 13:36:28.756217 139854454523648 util.py:45] init_checkpoint : gs://xiaoy-data-europe/spanbert_large_tf/bert_model.ckpt I0716 13:36:28.756279 139854454523648 util.py:45] eval_checkpoint : gs://corefqa-europe/spanbert_large_overlap_384_3_out put_1e-5_0.3_8/model.ckpt-20 I0716 13:36:28.756341 139854454523648 util.py:45] output_path : gs://corefqa-europe/spanbert_large_overlap_384_3_output_ 1e-5_0.3_8 I0716 13:36:28.756391 139854454523648 util.py:47] %*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*%%*% I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-500 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.6219, recall: 0.5093, f1: 0.56 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON TEST SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [TEST EVAL] ***** : precision: 0.6413, recall: 0.5428, f1: 0.588 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-1000 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.6227, recall: 0.5841, f1: 0.6028 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON TEST SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [TEST EVAL] ***** : precision: 0.6744, recall: 0.6126, f1: 0.6149 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-1500 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.6906, recall: 0.5288, f1: 0.599 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON TEST SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [TEST EVAL] ***** : precision: 0.6966, recall: 0.506, f1: 0.5862 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-2000 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.6712, recall: 0.5717, f1: 0.6174 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON TEST SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [TEST EVAL] ***** : precision: 0.6696, recall: 0.541, f1: 0.5985 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-2500 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.8091, recall: 0.5967, f1: 0.6868 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON TEST SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [TEST EVAL] ***** : precision: 0.8018, recall: 0.5692, f1: 0.6657 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-3000 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.6675, recall: 0.6382, f1: 0.6525 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-3500 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.7155, recall: 0.7515, f1: 0.7331 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON TEST SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [TEST EVAL] ***** : precision: 0.7239, recall: 0.7711, f1: 0.7468 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-4000 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.7762, recall: 0.5819, f1: 0.6651 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-4500 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.6661, recall: 0.6236, f1: 0.6442 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-5000 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.7814, recall: 0.7246, f1: 0.7519 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON TEST SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [TEST EVAL] ***** : precision: 0.7155, recall: 0.7515, f1: 0.7331 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-5500 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.8042, recall: 0.7417, f1: 0.7717 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON TEST SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [TEST EVAL] ***** : precision: 0.8439, recall: 0.6328, f1: 0.7232 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-6000 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.7942, recall: 0.7217, f1: 0.7417 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-6500 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.7816, recall: 0.7831, f1: 0.7823 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON TEST SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [TEST EVAL] ***** : precision: 0.7893, recall: 0.8075, f1: 0.7983 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-7000 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.8103, recall: 0.814, f1: 0.8121 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON TEST SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [TEST EVAL] ***** : precision: 0.8022, recall: 0.7838, f1: 0.7929 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-7500 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.7935, recall: 0.8292, f1: 0.8109 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-8000 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.8997, recall: 0.7292, f1: 0.8056 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-8500 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.8737, recall: 0.7444, f1: 0.8039 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-9000 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.8107, recall: 0.814, f1: 0.8124 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-10500 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.8439, recall: 0.7952, f1: 0.8188 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON TEST SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [TEST EVAL] ***** : precision: 0.8228, recall: 0.8093, f1: 0.8107 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-11000 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.8386, recall: 0.8147, f1: 0.8273 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON TEST SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [TEST EVAL] ***** : precision: 0.8239, recall: 0.8104, f1: 0.8143 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-12000 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.8398, recall: 0.8201, f1: 0.8369 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-12500 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.8666, recall: 0.7766, f1: 0.8192 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-13000 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.8397, recall: 0.7892, f1: 0.8056 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-13500 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.8357, recall: 0.8169, f1: 0.8262 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON TEST SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [TEST EVAL] ***** : precision: 0.8327, recall: 0.8288, f1: 0.8322 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-14000 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.8269, recall: 0.8108, f1: 0.8201 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-14500 I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:17:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.8344, recall: 0.8104, f1: 0.8215 I0716 14:46:19.214575 139854454523648 tpu_estimator.py:2307] ***** Current ckpt path is ***** I0716 14:46:19.214575 139854454523648 tpu_estimator.py:2307] gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-15000 I0716 14:46:19.214575 139854454523648 tpu_estimator.py:2307] ***** EVAL ON DEV SET ***** I0716 14:46:19.214575 139854454523648 tpu_estimator.py:2307] ***** [DEV EVAL] ***** : precision: 0.8287, recall: 0.8177, f1: 0.8223 I0716 14:46:19.214575 139854454523648 tpu_estimator.py:2307] ************************* I0716 14:46:19.214575 139854454523648 tpu_estimator.py:2307] - @@@@@ the path to the BEST DEV result is : gs://corefqa-europe-europe/spanbert_large_overlap_384_3_output_8e6_0.2_8/model.ckpt-13500 I0716 14:46:19.214575 139854454523648 tpu_estimator.py:2307] - @@@@@ BEST DEV F1 : 0.8262, Precision : 0.8357, Recall : 0.8169 I0716 14:46:19.214575 139854454523648 tpu_estimator.py:2307] - @@@@@ TEST when DEV best F1 : 0.8322, Precision : 0.8327, Recall : 0.8288 I0716 14:46:19.214575 139854454523648 tpu_estimator.py:2307] - @@@@@ mention_proposal_only_concate False ================================================ FILE: models/corefqa.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import os import sys repo_path = "/".join(os.path.realpath(__file__).split("/")[:-2]) if repo_path not in sys.path: sys.path.insert(0, repo_path) import tensorflow as tf from bert import modeling class CorefQAModel(object): def __init__(self, config): self.config = config self.dropout = None self.pad_idx = 0 self.mention_start_idx = 37 self.mention_end_idx = 42 self.bert_config = modeling.BertConfig.from_json_file(config.bert_config_file) self.bert_config.hidden_dropout_prob = config.dropout_rate self.cls_in_vocab = 101 self.sep_in_vocab = 102 def get_coreference_resolution_and_loss(self, instance, is_training, use_tpu=False): self.use_tpu = use_tpu self.dropout = self.get_dropout(self.config.dropout_rate, is_training) flat_window_input_ids, flat_window_input_mask, flat_doc_sentence_map, window_text_len, speaker_ids, gold_starts, gold_ends, gold_cluster_ids = instance # flat_input_ids: (num_window, window_size) # flat_doc_overlap_input_mask: (num_window, window_size) # flat_sentence_map: (num_window, window_size) # text_len: dynamic length and is padded to fix length # gold_start: (max_num_mention), mention start index in the original (NON-OVERLAP) document. Pad with -1 to the fix length max_num_mention. # gold_end: (max_num_mention), mention end index in the original (NON-OVERLAP) document. Pad with -1 to the fix length max_num_mention. # cluster_ids/speaker_ids is not used in the mention proposal model. flat_window_input_ids = tf.math.maximum(flat_window_input_ids, tf.zeros_like(flat_window_input_ids, tf.int32)) # (num_window * window_size) flat_doc_overlap_input_mask = tf.where(tf.math.greater_equal(flat_window_input_mask, 0), x=tf.ones_like(flat_window_input_mask, tf.int32), y=tf.zeros_like(flat_window_input_mask, tf.int32)) # (num_window * window_size) # flat_doc_overlap_input_mask = tf.math.maximum(flat_doc_overlap_input_mask, tf.zeros_like(flat_doc_overlap_input_mask, tf.int32)) flat_doc_sentence_map = tf.math.maximum(flat_doc_sentence_map, tf.zeros_like(flat_doc_sentence_map, tf.int32)) # (num_window * window_size) gold_start_end_mask = tf.cast(tf.math.greater_equal(gold_starts, tf.zeros_like(gold_starts, tf.int32)), tf.bool) # (max_num_mention) gold_start_index_labels = self.boolean_mask_1d(gold_starts, gold_start_end_mask, name_scope="gold_starts", use_tpu=self.use_tpu) # (num_of_mention) gold_end_index_labels = self.boolean_mask_1d(gold_ends, gold_start_end_mask, name_scope="gold_ends", use_tpu=self.use_tpu) # (num_of_mention) gold_cluster_mask = tf.cast(tf.math.greater_equal(gold_cluster_ids, tf.zeros_like(gold_cluster_ids, tf.int32)), tf.bool) # (max_num_cluster) gold_cluster_ids = self.boolean_mask_1d(gold_cluster_ids, gold_cluster_mask, name_scope="gold_cluster", use_tpu=self.use_tpu) window_text_len = tf.math.maximum(window_text_len, tf.zeros_like(window_text_len, tf.int32)) # (num_of_non_empty_window) num_subtoken_in_doc = tf.math.reduce_sum(window_text_len) # the value should be num_subtoken_in_doc #################### #################### ## mention proposal stage starts mention_input_ids = tf.reshape(flat_window_input_ids, [-1, self.config.window_size]) # (num_window, window_size) # each row of mention_input_ids is a subdocument mention_input_mask = tf.ones_like(mention_input_ids, tf.int32) # (num_window, window_size) mention_model = modeling.BertModel(config=self.bert_config, is_training=is_training, input_ids=mention_input_ids, input_mask=mention_input_mask, use_one_hot_embeddings=False, scope='bert') mention_doc_overlap_window_embs = mention_model.get_sequence_output() # (num_window, window_size, hidden_size) # get BERT embeddings for mention_input_ids doc_overlap_input_mask = tf.reshape(flat_doc_overlap_input_mask, [self.config.num_window, self.config.window_size]) # (num_window, window_size) mention_doc_flat_embs = self.transform_overlap_sliding_windows_to_original_document(mention_doc_overlap_window_embs, doc_overlap_input_mask) mention_doc_flat_embs = tf.reshape(mention_doc_flat_embs, [-1, self.config.hidden_size]) # (num_subtoken_in_doc, hidden_size) candidate_mention_starts = tf.tile(tf.expand_dims(tf.range(num_subtoken_in_doc), 1), [1, self.config.max_span_width]) # (num_subtoken_in_doc, max_span_width) # getting all eligible mentions in each subdocument # the number if eligible mentions of each subdocument is config.max_span_width * num_subtoken_in_doc candidate_mention_ends = tf.math.add(candidate_mention_starts, tf.expand_dims(tf.range(self.config.max_span_width), 0)) # (num_subtoken_in_doc, max_span_width) candidate_mention_sentence_start_idx = tf.gather(tf.reshape(flat_doc_sentence_map, [-1]), candidate_mention_starts) # (num_subtoken_in_doc, max_span_width) candidate_mention_sentence_end_idx = tf.gather(tf.reshape(flat_doc_sentence_map, [-1]), candidate_mention_ends) # (num_subtoken_in_doc, max_span_width) candidate_mention_mask = tf.logical_and(candidate_mention_ends < num_subtoken_in_doc, tf.equal(candidate_mention_sentence_start_idx, candidate_mention_sentence_end_idx)) candidate_mention_mask = tf.reshape(candidate_mention_mask, [-1]) candidate_mention_starts = self.boolean_mask_1d(tf.reshape(candidate_mention_starts, [-1]), candidate_mention_mask, name_scope="candidate_mention_starts", use_tpu=self.use_tpu) candidate_mention_ends = self.boolean_mask_1d(tf.reshape(candidate_mention_ends, [-1]), candidate_mention_mask, name_scope="candidate_mention_ends", use_tpu=self.use_tpu) # num_candidate_mention_in_doc is smaller than num_subtoken_in_doc candidate_cluster_idx_labels = self.get_candidate_cluster_labels(candidate_mention_starts, candidate_mention_ends, gold_start_index_labels, gold_end_index_labels, gold_cluster_ids) candidate_mention_span_embs, candidate_mention_start_embs, candidate_mention_end_embs = self.get_candidate_span_embedding( mention_doc_flat_embs, candidate_mention_starts, candidate_mention_ends) # candidate_mention_span_embs -> (num_candidate_mention_in_doc, 2 * hidden_size) # candidate_mention_start_embs -> (num_candidate_mention_in_doc, hidden_size) # candidate_mention_end_embs -> (num_candidate_mention_in_doc, hidden_size) gold_label_candidate_mention_spans, gold_label_candidate_mention_starts, gold_label_candidate_mention_ends = self.get_candidate_mention_gold_sequence_label( candidate_mention_starts, candidate_mention_ends, gold_start_index_labels, gold_end_index_labels, num_subtoken_in_doc) # gold_label_candidate_mention_spans -> (num_candidate_mention_in_doc) # gold_label_candidate_mention_starts -> (num_candidate_mention_in_doc) # gold_label_candidate_mention_ends -> (num_candidate_mention_in_doc) mention_proposal_loss, candidate_mention_start_prob, candidate_mention_end_prob, candidate_mention_span_prob, candidate_mention_span_scores = self.get_mention_score_and_loss( candidate_mention_span_embs, candidate_mention_start_embs, candidate_mention_end_embs, gold_label_candidate_mention_spans=gold_label_candidate_mention_spans, gold_label_candidate_mention_starts=gold_label_candidate_mention_starts, gold_label_candidate_mention_ends=gold_label_candidate_mention_ends, expect_length_of_labels=num_subtoken_in_doc) # mention_proposal_loss -> a scalar # candidate_mention_start_prob, candidate_mention_end_prob, candidate_mention_span_prob, -> (num_candidate_mention_in_doc) self.k = tf.minimum(self.config.max_candidate_mentions, tf.to_int32(tf.floor(tf.to_float(num_subtoken_in_doc) * self.config.top_span_ratio))) # self.k is a hyper-parameter. We want to select the top self.k mentions from the config.max_span_width * num_subtoken_in_doc mentions. candidate_mention_span_scores = tf.reshape(candidate_mention_span_scores, [-1]) topk_mention_span_scores, topk_mention_span_indices = tf.nn.top_k(candidate_mention_span_scores, self.k, sorted=False) topk_mention_span_indices = tf.reshape(topk_mention_span_indices, [-1]) # topk_mention_span_scores -> (k,) # topk_mention_span_indices -> (k,) topk_mention_start_indices = tf.gather(candidate_mention_starts, topk_mention_span_indices) # (k,) topk_mention_end_indices = tf.gather(candidate_mention_ends, topk_mention_span_indices) # (k,) topk_mention_span_cluster_ids = tf.gather(candidate_cluster_idx_labels, topk_mention_span_indices) # (k,) topk_mention_span_scores = tf.gather(candidate_mention_span_scores, topk_mention_span_indices) # (k,) ## mention proposal stage ends ########### ########### ###### mention linking stage starts ## foward QA score computation starts ## for a given proposed mention i, we first compute the score of a span j being the correferent answer to i, denoted by s(j|i) i0 = tf.constant(0) forward_qa_input_ids = tf.zeros((1, self.config.num_window, self.config.window_size + self.config.max_query_len + 2), dtype=tf.int32) # (1, num_window, max_query_len + window_size + 2) forward_qa_input_mask = tf.zeros((1, self.config.num_window, self.config.window_size + self.config.max_query_len + 2), dtype=tf.int32) # (1, num_window, max_query_len + window_size + 2) forward_qa_input_token_type_mask = tf.zeros((1, self.config.num_window, self.config.window_size + self.config.max_query_len + 2), dtype=tf.int32) # (1, num_window, max_query_len + window_size + 2) # prepare for non-overlap input token ids doc_overlap_input_mask = tf.reshape(flat_doc_overlap_input_mask, [self.config.num_window, self.config.window_size]) nonoverlap_doc_input_ids = self.transform_overlap_sliding_windows_to_original_document(flat_window_input_ids, doc_overlap_input_mask) # (num_subtoken_in_doc) overlap_window_input_ids = tf.reshape(flat_window_input_ids, [self.config.num_window, self.config.window_size]) # (num_window, window_size) @tf.function def forward_qa_mention_linking(i, batch_qa_input_ids, batch_qa_input_mask, batch_qa_input_token_type_mask): tmp_mention_start_idx = tf.gather(topk_mention_start_indices, i) tmp_mention_end_idx = tf.gather(topk_mention_end_indices, i) query_input_token_ids, mention_start_idx_in_sent, mention_end_idx_in_sent = self.get_query_token_ids( nonoverlap_doc_input_ids, flat_doc_sentence_map, tmp_mention_start_idx, tmp_mention_end_idx) query_pad_token_ids = tf.zeros([self.config.max_query_len - self.get_shape(query_input_token_ids, 0)], dtype=tf.int32) pad_query_input_token_ids = tf.concat([query_input_token_ids, query_pad_token_ids], axis=0) # (max_query_len,) pad_query_input_token_mask = tf.ones_like(pad_query_input_token_ids, tf.int32) # (max_query_len) pad_query_input_token_type_mask = tf.zeros_like(pad_query_input_token_ids, tf.int32) # (max_query_len) expand_pad_query_input_token_ids = tf.tile(tf.expand_dims(pad_query_input_token_ids, 0), [self.config.num_window, 1]) # (num_window, max_query_len) expand_pad_query_input_token_mask = tf.tile(tf.expand_dims(pad_query_input_token_mask, 0), [self.config.num_window, 1]) # (num_window, max_query_len) expand_pad_query_input_token_type_mask = tf.tile(tf.expand_dims(pad_query_input_token_type_mask, 0), [self.config.num_window, 1]) # (num_window, max_query_len) sep_tokens = tf.cast(tf.fill([self.config.num_window, 1], self.sep_in_vocab), tf.int32) # (num_window, 1) cls_tokens = tf.cast(tf.fill([self.config.num_window, 1], self.cls_in_vocab), tf.int32) # (num_window, 1) query_context_input_token_ids = tf.concat([cls_tokens, expand_pad_query_input_token_ids, sep_tokens, overlap_window_input_ids], axis=1) # (1, num_window, max_query_len + window_size + 2) query_context_input_token_mask = tf.concat([tf.ones_like(cls_tokens, tf.int32), expand_pad_query_input_token_mask, tf.ones_like(sep_tokens, tf.int32), tf.ones_like(overlap_window_input_ids, tf.int32)], axis=1) # (1, num_window, max_query_len + window_size + 2) query_context_input_token_type_mask = tf.concat([tf.zeros_like(cls_tokens, tf.int32), expand_pad_query_input_token_type_mask, tf.zeros_like(sep_tokens, tf.int32), tf.ones_like(overlap_window_input_ids, tf.int32)], axis=1) # (1, num_window, max_query_len + window_size + 2) query_context_input_token_ids = tf.reshape(query_context_input_token_ids, [1, self.config.num_window, self.config.max_query_len+self.config.window_size+2]) query_context_input_token_mask = tf.reshape(query_context_input_token_mask, [1, self.config.num_window, self.config.max_query_len+self.config.window_size+2]) query_context_input_token_type_mask = tf.reshape(query_context_input_token_type_mask, [1, self.config.num_window, self.config.max_query_len+self.config.window_size+2]) return [tf.math.add(i, 1), tf.concat([batch_qa_input_ids, query_context_input_token_ids], 0), tf.concat([batch_qa_input_mask, query_context_input_token_mask], 0), tf.concat([batch_qa_input_token_type_mask, query_context_input_token_type_mask], 0)] _, stack_forward_qa_input_ids, stack_forward_qa_input_mask, stack_forward_qa_input_type_mask = tf.while_loop( cond=lambda i, o1, o2, o3 : i < self.k, body=forward_qa_mention_linking, loop_vars=[i0, forward_qa_input_ids, forward_qa_input_mask, forward_qa_input_token_type_mask], shape_invariants=[i0.get_shape(), tf.TensorShape([None, None, None]), tf.TensorShape([None, None, None]), tf.TensorShape([None, None, None])]) # stack_forward_qa_input_ids, stack_forward_qa_input_mask, stack_forward_qa_input_type_mask -> (k, num_window, max_query_len + window_size + 2) batch_forward_qa_input_ids = tf.reshape(stack_forward_qa_input_ids, [-1, self.config.max_query_len+self.config.window_size+2]) # (k * num_window, max_query_len + window_size + 2) batch_forward_qa_input_mask = tf.reshape(stack_forward_qa_input_mask, [-1, self.config.max_query_len+self.config.window_size+2]) # (k * num_window, max_query_len + window_size + 2) batch_forward_qa_input_type_mask = tf.reshape(stack_forward_qa_input_type_mask, [-1, self.config.max_query_len+self.config.window_size+2]) # (k * num_window, max_query_len + window_size + 2) forward_qa_linking_model = modeling.BertModel(config=self.bert_config, is_training=is_training, input_ids=batch_forward_qa_input_ids, input_mask=batch_forward_qa_input_mask, token_type_ids=batch_forward_qa_input_type_mask, use_one_hot_embeddings=False, scope="bert") forward_qa_overlap_window_embs = forward_qa_linking_model.get_sequence_output() # (k * num_window, max_query_len + window_size + 2, hidden_size) forward_context_overlap_window_embs = self.transform_overlap_sliding_windows_to_original_document(forward_qa_overlap_window_embs, batch_forward_qa_input_type_mask) forward_context_overlap_window_embs = tf.reshape(forward_context_overlap_window_embs, [self.k*self.config.num_window, self.config.window_size]) # forward_context_overlap_window_embs -> (k*num_window, window_size, hidden_size) expand_doc_overlap_input_mask = tf.tile(tf.expand_dims(doc_overlap_input_mask, 0), [self.k, 1, 1]) # (k, num_window, window_size) expand_doc_overlap_input_mask = tf.reshape(expand_doc_overlap_input_mask, [-1, self.config.window_size]) # (k * num_window, window_size) forward_context_flat_doc_embs = self.transform_overlap_sliding_windows_to_original_document(forward_context_overlap_window_embs, expand_doc_overlap_input_mask) # (k * num_subtoken_in_doc, hidden_size) forward_context_flat_doc_embs = tf.reshape(forward_context_flat_doc_embs, [self.k, -1, self.config.hidden_size]) # (k, num_subtoken_in_doc, hidden_size) num_candidate_mention = self.get_shape(candidate_mention_span_embs, 0) # (num_candidate_mention_in_doc) forward_qa_mention_pos_offset = tf.cast(tf.tile(tf.reshape(tf.range(0, num_candidate_mention) * num_subtoken_in_doc, [1, -1]), [self.k, 1]), tf.int32) # (k, num_candidate_mention_in_doc) forward_qa_mention_starts = tf.tile(tf.expand_dims(candidate_mention_starts, 0), [self.k, 1]) + forward_qa_mention_pos_offset # (k, num_candidate_mention_in_doc) forward_qa_mention_ends = tf.tile(tf.expand_dims(candidate_mention_ends, 0), [self.k, 1]) + forward_qa_mention_pos_offset # (k, num_candidate_mention_in_doc) forward_qa_mention_span_embs, forward_qa_mention_start_embs, forward_qa_mention_end_embs = self.get_candidate_span_embedding(tf.reshape(forward_context_flat_doc_embs, [-1, self.config.hidden_size]), tf.reshape(forward_qa_mention_starts, [-1]), tf.reshape(forward_qa_mention_ends, [-1])) # forward_qa_mention_span_embs -> (k * num_candidate_mention_in_doc, hidden_size*2) # forward_qa_mention_start_embs -> (k * num_candidate_mention_in_doc, hidden_size) self.c = tf.to_int32(tf.minimum(self.config.max_top_antecedents, self.k)) forward_qa_mention_span_scores, forward_qa_mention_start_scores, forward_qa_mention_end_scores = self.get_mention_score_and_loss(forward_qa_mention_span_embs, forward_qa_mention_start_embs, forward_qa_mention_end_embs, name_scope="forward_qa") # forward_qa_mention_span_prob, forward_qa_mention_start_prob, forward_qa_mention_end_prob -> (k * num_candidate_mention_in_doc) # computes the s(j|i) for all eligible span j in the document if self.config.sec_qa_mention_score: forward_qa_mention_span_scores = (forward_qa_mention_span_scores + forward_qa_mention_start_scores + forward_qa_mention_end_scores)/3.0 else: forward_qa_mention_span_scores = forward_qa_mention_span_scores forward_candidate_mention_span_scores = tf.reshape(forward_qa_mention_span_scores, [self.k, -1]) # (k, num_candidate_mention_in_doc) forward_topc_mention_span_scores, local_forward_topc_mention_span_indices = tf.nn.top_k(forward_candidate_mention_span_scores, self.c, sorted=False) # (k, c) # for each i, we only maintain the top self.c spans based on s(j|i) local_flat_forward_topc_mention_span_indices = tf.reshape(local_forward_topc_mention_span_indices, [-1]) # (k * c) # topk_mention_start_indices forward_topc_mention_start_indices = tf.gather(candidate_mention_starts, local_flat_forward_topc_mention_span_indices) # (k, c) forward_topc_mention_end_indices = tf.gather(candidate_mention_ends, local_flat_forward_topc_mention_span_indices) # (k, c) forward_topc_mention_span_scores_in_mention_proposal = tf.gather(candidate_mention_span_scores, local_flat_forward_topc_mention_span_indices) # (k, c) forward_topc_span_cluster_ids = tf.gather(candidate_cluster_idx_labels, local_flat_forward_topc_mention_span_indices) ## foward QA score computation ends ## backward QA score computation begins ## we need to compute the score of backward score, i.e., the span i is the correferent answer for j, denoted by s(i|j) i0 = tf.constant(0) backward_qa_input_ids = tf.reshape(tf.zeros((1, self.config.max_query_len + self.config.max_context_len + 2), dtype=tf.int32), [1, -1]) # (1, max_query_len + max_context_len + 2) backward_qa_input_mask = tf.reshape(tf.zeros((1, self.config.max_query_len + self.config.max_context_len + 2), dtype=tf.int32), [1, -1]) # (1, max_query_len + max_context_len + 2) backward_qa_input_token_type_mask = tf.reshape(tf.zeros((1, self.config.max_query_len + self.config.max_context_len + 2), dtype=tf.int32), [1, -1]) # (1, max_query_len + max_context_len + 2) backward_qa_mention_start_in_context = tf.convert_to_tensor(tf.constant([0]), dtype=tf.int32) backward_qa_mention_end_in_context = tf.convert_to_tensor(tf.constant([0]), dtype=tf.int32) @tf.function def backward_qa_mention_linking(i, batch_qa_input_ids, batch_qa_input_mask, batch_qa_input_token_type_mask, batch_qa_mention_start_in_context, batch_qa_mention_end_in_context): tmp_query_mention_start_idx = tf.gather(forward_topc_mention_start_indices, i) tmp_query_mention_end_idx = tf.gather(forward_topc_mention_end_indices, i) tmp_index_for_topk_mention = tf.floor_div(i, self.k) tmp_context_mention_start_idx = tf.gather(topk_mention_start_indices, tmp_index_for_topk_mention) tmp_context_mention_end_idx = tf.gather(topk_mention_end_indices, tmp_index_for_topk_mention) query_input_token_ids, mention_start_idx_in_query, mention_end_idx_in_query = self.get_query_token_ids( nonoverlap_doc_input_ids, flat_doc_sentence_map, tmp_query_mention_start_idx, tmp_query_mention_end_idx) context_input_token_ids, mention_start_idx_in_context, mention_end_idx_in_context = self.get_query_token_ids( nonoverlap_doc_input_ids, flat_doc_sentence_map, tmp_context_mention_start_idx, tmp_context_mention_end_idx) query_pad_token_ids = tf.zeros([self.config.max_query_len - self.get_shape(query_input_token_ids, 0)], dtype=tf.int32) context_pad_token_ids = tf.zeros([self.config.max_context_len - self.get_shape(context_input_token_ids, 0)], dtype=tf.int32) pad_query_input_token_ids = tf.concat([query_input_token_ids, query_pad_token_ids], axis=0) # (max_query_len) pad_query_input_token_mask = tf.ones_like(pad_query_input_token_ids, tf.int32) pad_query_input_token_type_mask = tf.zeros_like(pad_query_input_token_ids, tf.int32) pad_context_input_token_ids = tf.concat([context_input_token_ids, context_pad_token_ids], axis=0) # (max_context_len) pad_context_input_token_mask = tf.ones_like(pad_context_input_token_ids, tf.int32) pad_context_input_token_type_mask = tf.ones_like(pad_context_input_token_ids, tf.int32) sep_tokens = tf.cast(tf.fill([1], self.sep_in_vocab), tf.int32) # (num_window, 1) cls_tokens = tf.cast(tf.fill([1], self.cls_in_vocab), tf.int32) # (num_window, 1) query_context_input_token_ids = tf.concat([cls_tokens, pad_query_input_token_ids, sep_tokens, pad_context_input_token_ids], axis=0) query_context_input_token_mask = tf.concat([tf.ones_like(cls_tokens, tf.int32), pad_query_input_token_mask, tf.zeros_like(sep_tokens, tf.int32), pad_context_input_token_mask], axis=0) query_context_input_token_type_mask = tf.concat([tf.zeros_like(cls_tokens, tf.int32), pad_query_input_token_type_mask, tf.zeros_like(sep_tokens, tf.int32), pad_context_input_token_type_mask], axis=0) query_context_input_token_ids = tf.reshape(query_context_input_token_ids, [1, self.config.max_query_len + self.config.max_context_len + 2]) query_context_input_token_mask = tf.reshape(query_context_input_token_mask, [1, self.config.max_query_len + self.config.max_context_len + 2]) query_context_input_token_type_mask = tf.reshape(query_context_input_token_type_mask, [1, self.config.max_query_len + self.config.max_context_len + 2]) mention_start_idx_in_context = tf.reshape(mention_start_idx_in_context, [-1]) mention_end_idx_in_context = tf.reshape(mention_end_idx_in_context, [-1]) return [tf.math.add(i, 1), tf.concat([batch_qa_input_ids, query_context_input_token_ids], 0), tf.concat([batch_qa_input_mask, query_context_input_token_mask], 0), tf.concat([batch_qa_input_token_type_mask, query_context_input_token_type_mask], 0), tf.concat([backward_qa_mention_start_in_context, mention_start_idx_in_context], 0), tf.concat([backward_qa_mention_end_in_context, mention_end_idx_in_context], 0)] _, stack_backward_qa_input_ids, stack_backward_qa_input_mask, stack_backward_qa_input_type_mask, stack_backward_mention_start_in_context, stack_backward_mention_end_in_context = tf.while_loop( cond = lambda i, o1, o2, o3, o4, o5: i < self.k * self.c, body=backward_qa_mention_linking, loop_vars=[i0, backward_qa_input_ids, backward_qa_input_mask, backward_qa_input_token_type_mask, backward_qa_mention_start_in_context, backward_qa_mention_end_in_context], shape_invariants=[i0.get_shape(), tf.TensorShape([None, None]), tf.TensorShape([None, None]), tf.TensorShape([None, None]), tf.TensorShape([None]), tf.TensorShape([None])]) # stack_backward_qa_input_ids, stack_backward_qa_input_mask, stack_backward_qa_input_type_mask -> (k*c, max_query_len + max_context_len + 2) # stack_backward_mention_start_in_context, stack_backward_mention_end_in_context -> (k*c,) batch_backward_qa_input_ids = tf.reshape(stack_backward_qa_input_ids, [-1, self.config.max_query_len+self.config.max_context_len+2]) batch_backward_qa_input_mask = tf.reshape(stack_backward_qa_input_mask, [-1, self.config.max_query_len+self.config.max_context_len+2]) batch_backward_qa_input_type_mask = tf.reshape(stack_backward_qa_input_type_mask, [-1, self.config.max_query_len+self.config.max_context_len+2]) backward_qa_linking_model = modeling.BertModel(config=self.bert_config, is_training=is_training, input_ids=batch_backward_qa_input_ids, input_mask=batch_backward_qa_input_mask, token_type_ids=batch_backward_qa_input_type_mask, use_one_hot_embeddings=False, scope="bert") backward_query_context_embs = backward_qa_linking_model.get_sequence_output() # (k*c, max_query_len + max_context_len + 2, hidden_size) backward_query_context_embs = tf.reshape(backward_query_context_embs, [self.k*self.c, -1, self.config.hidden_size]) flat_batch_backward_qa_input_type_mask = tf.reshape(batch_backward_qa_input_type_mask, [self.k*self.c, -1]) backward_context_flat_embs = self.transform_overlap_sliding_windows_to_original_document(backward_query_context_embs, flat_batch_backward_qa_input_type_mask) # (k*c, max_context_len, hidden_size) batch_backward_mention_start_in_context = tf.reshape(stack_backward_mention_start_in_context, [-1]) + tf.range(0, self.c*self.k) * (self.config.max_query_len+self.config.max_context_len) batch_backward_mention_end_in_context = tf.reshape(stack_backward_mention_end_in_context, [-1]) + tf.range(0, self.c*self.k) * (self.config.max_query_len+self.config.max_context_len) backward_qa_mention_span_embs, backward_qa_mention_start_embs, backward_qa_mention_end_embs = self.get_candidate_span_embedding(tf.reshape(backward_context_flat_embs, [-1, self.config.hidden_size]), tf.reshape(batch_backward_mention_start_in_context, [-1]), tf.reshape(batch_backward_mention_end_in_context, [-1])) # backward_qa_mention_span_embs -> (k*c, 2*hidden_size) # backward_qa_mention_start_embs, backward_qa_mention_end_embs -> (k*c, hidden_size) backward_qa_mention_span_scores, backward_qa_mention_start_scores, backward_qa_mention_end_scores = self.get_mention_score_and_loss(backward_qa_mention_span_embs, backward_qa_mention_start_embs, backward_qa_mention_end_embs, name_scope="backward_qa") # backward_qa_mention_span_prob -> (k*c) # backward_qa_mention_start_prob, backward_qa_mention_end_prob -> (k*c) if self.config.sec_qa_mention_score: backward_qa_mention_span_scores = (backward_qa_mention_span_scores + backward_qa_mention_start_scores + backward_qa_mention_end_scores)/3.0 else: backward_qa_mention_span_scores = backward_qa_mention_span_scores ############# ############# backward QA computation ends forward_topc_mention_span_scores = tf.reshape(forward_topc_mention_span_scores, [-1]) expand_forward_topc_mention_span_scores = tf.tile(tf.expand_dims(forward_topc_mention_span_scores, 0), [self.k, 1]) # forward_topc_mention_span_scores -> (c); expand_forward_topc_mention_span_scores -> (c, k) expand_forward_topc_mention_span_scores_in_mention_proposal = tf.tile(tf.expand_dims(forward_topc_mention_span_scores_in_mention_proposal, 0), [self.k, 1]) expand_topk_mention_span_scores = tf.tile(tf.expand_dims(topk_mention_span_scores, 1), [1, self.c]) # (k, c) backward_qa_mention_span_scores = tf.reshape(backward_qa_mention_span_scores, [self.k, self.c]) # (k, c) mention_span_linking_scores = (expand_forward_topc_mention_span_scores + backward_qa_mention_span_scores ) / 2.0 mention_span_linking_scores = mention_span_linking_scores+ expand_forward_topc_mention_span_scores_in_mention_proposal + expand_topk_mention_span_scores mention_span_linking_scores = tf.reshape(mention_span_linking_scores, [self.k, self.c]) # (k, c) dummy_scores = tf.zeros([self.k, 1]) # (k, 1) top_mention_span_linking_scores = tf.concat([dummy_scores, mention_span_linking_scores], axis=1) # (k, c) forward_topc_span_cluster_ids = tf.reshape(forward_topc_span_cluster_ids, [self.k, self.c]) # (k, c) same_cluster_indicator = tf.equal(forward_topc_span_cluster_ids, tf.expand_dims(topk_mention_span_cluster_ids, 1)) non_dummy_indicator = tf.expand_dims(topk_mention_span_cluster_ids > 0, 1) pairwise_labels = tf.logical_and(same_cluster_indicator, non_dummy_indicator) dummy_labels = tf.logical_not(tf.reduce_any(pairwise_labels, 1, keepdims=True)) top_mention_span_linking_labels = tf.concat([dummy_labels, pairwise_labels], 1) linking_loss = self.marginal_likelihood_loss(top_mention_span_linking_scores, top_mention_span_linking_labels) total_loss = mention_proposal_loss + linking_loss return total_loss, (topk_mention_start_indices, topk_mention_end_indices), (forward_topc_mention_start_indices, forward_topc_mention_end_indices), top_mention_span_linking_scores def marginal_likelihood_loss(self, antecedent_scores, antecedent_labels): """ Desc: marginal likelihood of gold antecedent spans form coreference cluster Args: antecedent_scores: [k, c+1] the predicted scores by the model antecedent_labels: [k, c+1] the gold-truth cluster labels Returns: a scalar of loss """ gold_scores = tf.math.add(antecedent_scores, tf.log(tf.to_float(antecedent_labels))) marginalized_gold_scores = tf.math.reduce_logsumexp(gold_scores, [1]) # [k] log_norm = tf.math.reduce_logsumexp(antecedent_scores, [1]) # [k] loss = log_norm - marginalized_gold_scores # [k] return tf.math.reduce_sum(loss) def get_query_token_ids(self, nonoverlap_doc_input_ids, sentence_map, mention_start_idx, mention_end_idx, paddding=True): """ Desc: construct question based on the selected mention. """ nonoverlap_doc_input_ids = tf.reshape(nonoverlap_doc_input_ids, [-1]) sentence_idx_for_mention = tf.gather(sentence_map, mention_start_idx) sentence_mask_for_mention = tf.math.equal(sentence_map, sentence_idx_for_mention) query_token_input_ids = self.boolean_mask_1d(nonoverlap_doc_input_ids, sentence_mask_for_mention, name_scope="query_mention", use_tpu=self.use_tpu) sentence_start = tf.where(tf.equal(nonoverlap_doc_input_ids, tf.gather(query_token_input_ids, tf.constant(0)))) mention_start_in_sent = mention_start_idx - tf.cast(sentence_start, tf.int32) mention_end_in_sent = mention_end_idx - tf.cast(sentence_start, tf.int32) return query_token_input_ids, mention_start_in_sent, mention_end_in_sent def get_mention_score_and_loss(self, candidate_mention_span_embs, candidate_mention_start_embs, candidate_mention_end_embs, gold_label_candidate_mention_spans=None, gold_label_candidate_mention_starts=None, gold_label_candidate_mention_ends=None, expect_length_of_labels=None, name_scope="mention"): candidate_mention_span_logits = self.ffnn(candidate_mention_span_embs, self.config.hidden_size*2, 1, dropout=self.dropout, name_scope="{}_span".format(name_scope)) candidate_mention_start_logits = self.ffnn(candidate_mention_start_embs, self.config.hidden_size, 1, dropout=self.dropout, name_scope="{}_start".format(name_scope)) candidate_mention_end_logits = self.ffnn(candidate_mention_end_embs, self.config.hidden_size, 1, dropout=self.dropout, name_scope="{}_end".format(name_scope)) if gold_label_candidate_mention_spans is None or gold_label_candidate_mention_starts is None or gold_label_candidate_mention_ends is None: candidate_mention_span_scores = tf.math.log(tf.sigmoid(candidate_mention_span_logits)) candidate_mention_start_scores = tf.math.log(tf.sigmoid(candidate_mention_start_logits)) candidate_mention_end_scores = tf.math.log(tf.sigmoid(candidate_mention_end_logits)) return candidate_mention_span_scores, candidate_mention_start_scores, candidate_mention_end_scores start_loss, candidate_mention_start_probability = self.compute_mention_score_and_loss(candidate_mention_start_logits, gold_label_candidate_mention_starts) end_loss, candidate_mention_end_probability = self.compute_mention_score_and_loss(candidate_mention_end_logits, gold_label_candidate_mention_ends) span_loss, candidate_mention_span_probability = self.compute_mention_score_and_loss(candidate_mention_span_logits, gold_label_candidate_mention_spans) total_loss = start_loss + end_loss + span_loss candidate_mention_span_scores = (tf.math.log(candidate_mention_start_probability) + tf.math.log(candidate_mention_end_probability) + tf.math.log(candidate_mention_span_probability)) / 3.0 return total_loss, candidate_mention_start_probability, candidate_mention_end_probability, candidate_mention_span_probability, candidate_mention_span_scores def compute_mention_score_and_loss(self, pred_sequence_logits, gold_sequence_labels, loss_mask=None): """ Desc: compute the unifrom start/end loss and probabilities. Args: pred_sequence_logits: (input_shape, 1) gold_sequence_labels: (input_shape, 1) loss_mask: [optional] if is not None, it should be (input_shape). should be tf.int32 0/1 tensor. FOR start/end score and loss, input_shape should be num_subtoken_in_doc. FOR span score and loss, input_shape should be num_subtoken_in_doc * num_subtoken_in_doc. """ pred_sequence_probabilities = tf.cast(tf.reshape(tf.sigmoid(pred_sequence_logits), [-1]),tf.float32) # (input_shape) expand_pred_sequence_scores = tf.stack([(1 - pred_sequence_probabilities), pred_sequence_probabilities], axis=-1) # (input_shape, 2) expand_gold_sequence_labels = tf.cast(tf.one_hot(tf.reshape(gold_sequence_labels, [-1]), 2, axis=-1), tf.float32) # (input_shape, 2) loss = tf.keras.losses.binary_crossentropy(expand_gold_sequence_labels, expand_pred_sequence_scores) # loss -> shape is (input_shape) if loss_mask is not None: loss = tf.multiply(loss, tf.cast(loss_mask, tf.float32)) total_loss = tf.reduce_mean(loss) # total_loss -> a scalar return total_loss, pred_sequence_probabilities def get_candidate_span_embedding(self, doc_sequence_embeddings, candidate_span_starts, candidate_span_ends): doc_sequence_embeddings = tf.reshape(doc_sequence_embeddings, [-1, self.config.hidden_size]) span_start_embedding = tf.gather(doc_sequence_embeddings, candidate_span_starts) span_end_embedding = tf.gather(doc_sequence_embeddings, candidate_span_ends) span_embedding = tf.concat([span_start_embedding, span_end_embedding], 1) return span_embedding, span_start_embedding, span_end_embedding def get_candidate_mention_gold_sequence_label(self, candidate_mention_starts, candidate_mention_ends, gold_start_index_labels, gold_end_index_labels, expect_length_of_labels): gold_start_sequence_label = self.scatter_gold_index_to_label_sequence(gold_start_index_labels, expect_length_of_labels) gold_end_sequence_label = self.scatter_gold_index_to_label_sequence(gold_end_index_labels, expect_length_of_labels) gold_label_candidate_mention_starts = tf.gather(gold_start_sequence_label, candidate_mention_starts) gold_label_candidate_mention_ends = tf.gather(gold_end_sequence_label, candidate_mention_ends) gold_mention_sparse_label = tf.reshape(tf.stack([gold_start_index_labels, gold_end_index_labels], axis=1), [2, -1]) gold_span_value = tf.reshape(tf.ones_like(gold_start_index_labels, tf.int32), [-1]) gold_span_shape = tf.Variable([expect_length_of_labels, expect_length_of_labels], shape=(2, )) gold_span_label = tf.cast(tf.scatter_nd(gold_mention_sparse_label, gold_span_value, gold_span_shape), tf.int32) candidate_mention_spans = tf.stack([candidate_mention_starts, candidate_mention_ends], axis=1) gold_label_candidate_mention_spans = tf.gather_nd(gold_span_label, tf.expand_dims(candidate_mention_spans, 1)) return gold_label_candidate_mention_spans, gold_label_candidate_mention_starts, gold_label_candidate_mention_ends def scatter_gold_index_to_label_sequence(self, gold_index_labels, expect_length_of_labels): """ Desc: transform the mention start/end position index tf.int32 Tensor to a tf.int32 Tensor with 1/0 labels for the input subtoken sequences. 1 denotes this subtoken is the start/end for a mention. Args: gold_index_labels: a tf.int32 Tensor with mention start/end position index in the original document. expect_length_of_labels: the number of subtokens in the original document. """ gold_labels_pos = tf.reshape(gold_index_labels, [-1, 1]) # (num_of_mention, 1) gold_value = tf.reshape(tf.ones_like(gold_index_labels), [-1]) # (num_of_mention) label_shape = tf.Variable(expect_length_of_labels) label_shape = tf.reshape(label_shape, [1]) # [1] gold_label_sequence = tf.cast(tf.scatter_nd(gold_labels_pos, gold_value, label_shape), tf.int32) # (num_subtoken_in_doc) return gold_label_sequence def scatter_span_sequence_labels(self, gold_start_index_labels, gold_end_index_labels, expect_length_of_labels): """ Desc: transform the mention (start, end) position pairs to a span matrix gold_span_sequence_labels. matrix[i][j]: whether the subtokens between the position $i$ to $j$ can be a mention. if matrix[i][j] == 0: from $i$ to $j$ is not a mention. if matrix[i][j] == 1: from $i$ to $j$ is a mention. Args: gold_start_index_labels: a tf.int32 Tensor with mention start position index in the original document. gold_end_index_labels: a tf.int32 Tensor with mention end position index in the original document. expect_length_of_labels: a scalar, should be the same with num_subtoken_in_doc """ gold_span_index_labels = tf.stack([gold_start_index_labels, gold_end_index_labels], axis=1) # (num_of_mention, 2) gold_span_value = tf.reshape(tf.ones_like(gold_start_index_labels, tf.int32), [-1]) # (num_of_mention) gold_span_label_shape = tf.Variable([expect_length_of_labels, expect_length_of_labels]) gold_span_label_shape = tf.reshape(gold_span_label_shape, [-1]) gold_span_sequence_labels = tf.cast(tf.scatter_nd(gold_span_index_labels, gold_span_value, gold_span_label_shape), tf.int32) # (num_subtoken_in_doc, num_subtoken_in_doc) return gold_span_sequence_labels def get_candidate_cluster_labels(self, candidate_mention_starts, candidate_mention_ends, gold_mention_starts, gold_mention_ends, gold_cluster_ids): same_mention_start = tf.equal(gold_mention_starts, candidate_mention_starts) same_mention_end = tf.equal(gold_mention_ends, candidate_mention_ends) # same_mention_start = tf.equal(tf.expand_dims(gold_mention_starts, 1), tf.expand_dims(candidate_mention_starts, 0)) # same_mention_end = tf.equal(tf.expand_dims(gold_mention_ends, 1), tf.expand_dims(candidate_mention_ends, 0)) same_mention_span = tf.logical_and(same_mention_start, same_mention_end) candidate_cluster_idx_labels = tf.matmul(tf.expand_dims(gold_cluster_ids, 0), tf.to_int32(same_mention_span)) # [1, num_candidates] candidate_cluster_idx_labels = tf.squeeze(candidate_cluster_idx_labels, 0) # [num_candidates] return candidate_cluster_idx_labels def transform_overlap_sliding_windows_to_original_document(self, overlap_window_inputs, overlap_window_mask): """ Desc: hidden_size should be equal to embeddding_size. Args: doc_overlap_window_embs: (num_window, window_size, hidden_size). the output of (num_window, window_size) input_ids forward into BERT model. doc_overlap_input_mask: (num_window, window_size). A tf.int32 Tensor contains 0/1. 0 represents token in this position should be neglected. 1 represents token in this position should be reserved. """ ones_input_mask = tf.ones_like(overlap_window_mask, tf.int32) # (num_window, window_size) cumsum_input_mask = tf.math.cumsum(ones_input_mask, axis=1) # (num_window, window_size) # do not equal to cumsum_input_mask -> (num_window, window_size) # offset_input_mask = tf.tile(tf.expand_dims(tf.range(self.config.num_window) * self.config.window_size, 1), [1, self.config.window_size]) # (num_window, window_size) row_cumsum_input_mask = self.get_shape(cumsum_input_mask, 0) col_cumsum_input_mask = self.get_shape(cumsum_input_mask, 1) offset_input_mask = tf.tile(tf.expand_dims(tf.range(row_cumsum_input_mask) * col_cumsum_input_mask, 1), [1, col_cumsum_input_mask]) offset_cumsum_input_mask = offset_input_mask + cumsum_input_mask # (num_window, window_size) global_input_mask = tf.math.multiply(ones_input_mask, offset_cumsum_input_mask) # (num_window, window_size) global_input_mask = tf.reshape(global_input_mask, [-1]) # (num_window * window_size) global_input_mask_index = self.boolean_mask_1d(global_input_mask, tf.math.greater(global_input_mask, tf.zeros_like(global_input_mask, tf.int32))) # (num_subtoken_in_doc) overlap_window_inputs = tf.reshape(overlap_window_inputs, [self.config.num_window * self.config.window_size, -1]) # (num_window * window_size, hidden_size) original_doc_inputs = tf.gather(overlap_window_inputs, global_input_mask_index) # (num_subtoken_in_doc, hidden_size) return original_doc_inputs def ffnn(self, inputs, hidden_size, output_size, dropout=None, name_scope="fully-conntected-neural-network", hidden_initializer=tf.truncated_normal_initializer(stddev=0.02)): """ Desc: fully-connected neural network. transform non-linearly the [input] tensor with [hidden_size] to a fix [output_size] size. Args: hidden_size: should be the size of last dimension of [inputs]. """ with tf.variable_scope(name_scope, reuse=tf.AUTO_REUSE): hidden_weights = tf.get_variable("hidden_weights", [hidden_size, output_size], initializer=hidden_initializer) hidden_bias = tf.get_variable("hidden_bias", [output_size], initializer=tf.zeros_initializer()) outputs = tf.nn.relu(tf.nn.xw_plus_b(inputs, hidden_weights, hidden_bias)) if dropout is not None: outputs = tf.nn.dropout(outputs, dropout) return outputs def boolean_mask_1d(self, itemtensor, boolmask_indicator, name_scope="boolean_mask1d", use_tpu=False): """ Desc: the same functionality of tf.boolean_mask. The tf.boolean_mask operation is not available on the cloud TPU. Args: itemtensor : a Tensor contains [tf.int32, tf.float32] numbers. Should be 1-Rank. boolmask_indicator : a tf.bool Tensor. Should be 1-Rank. scope : name scope for the operation. use_tpu : if False, return tf.boolean_mask. """ with tf.name_scope(name_scope): boolmask_sum = tf.reduce_sum(tf.cast(boolmask_indicator, tf.int32)) selected_positions = tf.cast(boolmask_indicator, dtype=tf.float32) indexed_positions = tf.cast(tf.multiply(tf.cumsum(selected_positions), selected_positions),dtype=tf.int32) one_hot_selector = tf.one_hot(indexed_positions - 1, boolmask_sum, dtype=tf.float32) sampled_indices = tf.cast(tf.tensordot(tf.cast(tf.range(tf.shape(boolmask_indicator)[0]), dtype=tf.float32), one_hot_selector,axes=[0, 0]),dtype=tf.int32) sampled_indices = tf.reshape(sampled_indices, [-1]) mask_itemtensor = tf.gather(itemtensor, sampled_indices) return mask_itemtensor def get_dropout(self, dropout_rate, is_training): return 1 - (tf.to_float(is_training) * dropout_rate) def get_shape(self, x, dim): """ Desc: return the size of input x in DIM. """ return x.get_shape()[dim].value or tf.shape(x)[dim] def evaluate(self, top_span_starts, top_span_ends, predicted_antecedents, gold_clusters, gold_starts, gold_ends): """ Desc: expected cluster ids is : [[[21, 25], [18, 18]], [[63, 65], [46, 48], [27, 29]], [[88, 88], [89, 89]]] Args: top_span_starts: top_span_ends: predicted_antecedents: Returns: predicted_clusters: gold_clusters: mention_to_predicted: mention_to_gold: """ # predicted_antecedents = np.argmax(predicted_antecedents, axis=-1) top_span_starts, top_span_ends, predicted_antecedents = top_span_starts.tolist(), top_span_ends.tolist(), predicted_antecedents.tolist() gold_clusters, gold_starts, gold_ends = gold_clusters.tolist()[0], gold_starts.tolist()[0], gold_ends.tolist()[0] def transform_gold_labels(gold_clusters, gold_starts, gold_ends): gold_clusters_idx = [tmp for tmp in gold_clusters if tmp >= 0] gold_starts = [tmp for tmp in gold_starts if tmp >= 0] gold_ends = [tmp for tmp in gold_ends if tmp >= 0] gold_clusters_dict = {} gold_cluster_lst = [] for idx, (tmp_start, tmp_end) in enumerate(zip(gold_starts, gold_ends)): tmp_cluster_idx = gold_clusters_idx[idx] if tmp_cluster_idx not in gold_clusters_dict.keys(): gold_cluster_lst.append(tmp_cluster_idx) gold_clusters_dict[tmp_cluster_idx] = [[tmp_start, tmp_end]] else: gold_clusters_dict[tmp_cluster_idx].append([tmp_start, tmp_end]) gold_cluster = [gold_clusters_dict[tmp_idx] for tmp_idx in gold_cluster_lst] return gold_cluster, gold_starts, gold_ends gold_clusters, gold_starts, gold_ends = transform_gold_labels(gold_clusters, gold_starts, gold_ends) gold_clusters = [tuple(tuple(m) for m in gc) for gc in gold_clusters] mention_to_gold = {} for gc in gold_clusters: for mention in gc: mention_to_gold[mention] = gc predicted_clusters, mention_to_predicted = self.get_predicted_clusters(top_span_starts, top_span_ends, predicted_antecedents) return predicted_clusters, gold_clusters, mention_to_predicted, mention_to_gold ================================================ FILE: models/mention_proposal.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # author: xiaoy li # description: # the mention proposal model for pre-training the Span-BERT model. import os import sys repo_path = "/".join(os.path.realpath(__file__).split("/")[:-2]) if repo_path not in sys.path: sys.path.insert(0, repo_path) import tensorflow as tf from bert import modeling class MentionProposalModel(object): def __init__(self, config): self.config = config self.bert_config = modeling.BertConfig.from_json_file(config.bert_config_file) self.bert_config.hidden_dropout_prob = config.dropout_rate def get_mention_proposal_and_loss(self, instance, is_training, use_tpu=False): """ Desc: forward function for training mention proposal module. Args: instance: a tuple of train/dev/test data instance. e.g., (flat_input_ids, flat_doc_overlap_input_mask, flat_sentence_map, text_len, speaker_ids, gold_starts, gold_ends, cluster_ids) is_training: True/False is in the training process. """ self.use_tpu = use_tpu self.dropout = self.get_dropout(self.config.dropout_rate, is_training) flat_input_ids, flat_doc_overlap_input_mask, flat_sentence_map, text_len, speaker_ids, gold_starts, gold_ends, cluster_ids = instance # flat_input_ids: (num_window, window_size) # flat_doc_overlap_input_mask: (num_window, window_size) # flat_sentence_map: (num_window, window_size) # text_len: dynamic length and is padded to fix length # gold_start: (max_num_mention), mention start index in the original (NON-OVERLAP) document. Pad with -1 to the fix length max_num_mention. # gold_end: (max_num_mention), mention end index in the original (NON-OVERLAP) document. Pad with -1 to the fix length max_num_mention. # cluster_ids/speaker_ids is not used in the mention proposal model. flat_input_ids = tf.math.maximum(flat_input_ids, tf.zeros_like(flat_input_ids, tf.int32)) # (num_window * window_size) flat_doc_overlap_input_mask = tf.where(tf.math.greater_equal(flat_doc_overlap_input_mask, 0), x=tf.ones_like(flat_doc_overlap_input_mask, tf.int32), y=tf.zeros_like(flat_doc_overlap_input_mask, tf.int32)) # (num_window * window_size) # flat_doc_overlap_input_mask = tf.math.maximum(flat_doc_overlap_input_mask, tf.zeros_like(flat_doc_overlap_input_mask, tf.int32)) flat_sentence_map = tf.math.maximum(flat_sentence_map, tf.zeros_like(flat_sentence_map, tf.int32)) # (num_window * window_size) gold_start_end_mask = tf.cast(tf.math.greater_equal(gold_starts, tf.zeros_like(gold_starts, tf.int32)), tf.bool) # (max_num_mention) gold_start_index_labels = self.boolean_mask_1d(gold_starts, gold_start_end_mask, name_scope="gold_starts", use_tpu=self.use_tpu) # (num_of_mention) gold_end_index_labels = self.boolean_mask_1d(gold_ends, gold_start_end_mask, name_scope="gold_ends", use_tpu=self.use_tpu) # (num_of_mention) text_len = tf.math.maximum(text_len, tf.zeros_like(text_len, tf.int32)) # (num_of_non_empty_window) num_subtoken_in_doc = tf.math.reduce_sum(text_len) # the value should be num_subtoken_in_doc input_ids = tf.reshape(flat_input_ids, [-1, self.config.window_size]) # (num_window, window_size) input_mask = tf.ones_like(input_ids, tf.int32) # (num_window, window_size) model = modeling.BertModel(config=self.bert_config, is_training=is_training, input_ids=input_ids, input_mask=input_mask, use_one_hot_embeddings=False, scope='bert') doc_overlap_window_embs = model.get_sequence_output() # (num_window, window_size, hidden_size) doc_overlap_input_mask = tf.reshape(flat_doc_overlap_input_mask, [self.config.num_window, self.config.window_size]) # (num_window, window_size) doc_flat_embs = self.transform_overlap_windows_to_original_doc(doc_overlap_window_embs, doc_overlap_input_mask) doc_flat_embs = tf.reshape(doc_flat_embs, [-1, self.config.hidden_size]) # (num_subtoken_in_doc, hidden_size) expand_start_embs = tf.tile(tf.expand_dims(doc_flat_embs, 1), [1, num_subtoken_in_doc, 1]) # (num_subtoken_in_doc, num_subtoken_in_doc, hidden_size) expand_end_embs = tf.tile(tf.expand_dims(doc_flat_embs, 0), [num_subtoken_in_doc, 1, 1]) # (num_subtoken_in_doc, num_subtoken_in_doc, hidden_size) expand_mention_span_embs = tf.concat([expand_start_embs, expand_end_embs], axis=-1) # (num_subtoken_in_doc, num_subtoken_in_doc, 2*hidden_size) expand_mention_span_embs = tf.reshape(expand_mention_span_embs, [-1, self.config.hidden_size*2]) span_sequence_logits = self.ffnn(expand_mention_span_embs, self.config.hidden_size*2, 1, dropout=self.dropout, name_scope="mention_span") # (num_subtoken_in_doc * num_subtoken_in_doc) if self.config.start_end_share: start_end_sequence_logits = self.ffnn(doc_flat_embs, self.config.hidden_size, 2, dropout=self.dropout, name_scope="mention_start_end") # (num_subtoken_in_doc, 2) start_sequence_logits, end_sequence_logits = tf.split(start_end_sequence_logits, axis=1) # start_sequence_logits -> (num_subtoken_in_doc, 1) # end_sequence_logits -> (num_subtoken_in_doc, 1) else: start_sequence_logits = self.ffnn(doc_flat_embs, self.config.hidden_size, 1, dropout=self.dropout, name_scope="mention_start") # (num_subtoken_in_doc) end_sequence_logits = self.ffnn(doc_flat_embs, self.config.hidden_size, 1, dropout=self.dropout, name_scope="mention_end") # (num_subtoken_in_doc) gold_start_sequence_labels = self.scatter_gold_index_to_label_sequence(gold_start_index_labels, num_subtoken_in_doc) # (num_subtoken_in_doc) gold_end_sequence_labels = self.scatter_gold_index_to_label_sequence(gold_end_index_labels, num_subtoken_in_doc) # (num_subtoken_in_doc) start_loss, start_sequence_probabilities = self.compute_score_and_loss(start_sequence_logits, gold_start_sequence_labels) end_loss, end_sequence_probabilities = self.compute_score_and_loss(end_sequence_logits, gold_end_sequence_labels) # *_loss -> a scalar # *_sequence_scores -> (num_subtoken_in_doc) gold_span_sequence_labels = self.scatter_span_sequence_labels(gold_start_index_labels, gold_end_index_labels, num_subtoken_in_doc) # (num_subtoken_in_doc * num_subtoken_in_doc) span_loss, span_sequence_probabilities = self.compute_score_and_loss(span_sequence_logits, gold_span_sequence_labels) # span_loss -> a scalar # span_sequence_probabilities -> (num_subtoken_in_doc * num_subtoken_in_doc) total_loss = self.config.loss_start_ratio * start_loss + self.config.loss_end_ratio * end_loss + self.config.loss_span_ratio * span_loss return total_loss, start_sequence_probabilities, end_sequence_probabilities, span_sequence_probabilities def get_gold_mention_sequence_labels_from_pad_index(self, pad_gold_start_index_labels, pad_gold_end_index_labels, pad_text_len): """ Desc: the original gold labels is padded to the fixed length and only contains the position index of gold mentions. return the gold sequence of labels for evaluation. Args: pad_gold_start_index_labels: a tf.int32 tensor with a fixed length (self.config.max_num_mention). every element in the tensor is the start position index for the mentions. pad_gold_end_index_labels: a tf.int32 tensor with a fixed length (self.config.max_num_mention). every element in the tensor is the end position index of the mentions. pad_text_len: a tf.int32 tensor with a fixed length (self.config.num_window). every positive element in the tensor indicates that the number of subtokens in the window. Returns: gold_start_sequence_labels: a tf.int32 tensor with the shape of (num_subtoken_in_doc). if the element in the tensor equals to 0, this subtoken is not a start for a mention. if the elemtn in the tensor equals to 1, this subtoken is a start for a mention. gold_end_sequence_labels: a tf.int32 tensor with the shape of (num_subtoken_in_doc). if the element in the tensor equals to 0, this subtoken is not a end for a mention. if the elemtn in the tensor equals to 1, this subtoken is a end for a mention. gold_span_sequence_labels: a tf.int32 tensor with the shape of (num_subtoken_in_doc, num_subtoken_in_doc)/ if the element[i][j] equals to 0, this subtokens from $i$ to $j$ is not a mention. if the element[i][j] equals to 1, this subtokens from $i$ to $j$ is a mention. """ text_len = tf.math.maximum(pad_text_len, tf.zeros_like(pad_text_len, tf.int32)) # (num_of_non_empty_window) num_subtoken_in_doc = tf.math.reduce_sum(text_len) # the value should be num_subtoken_in_doc gold_start_end_mask = tf.cast(tf.math.greater_equal(pad_gold_start_index_labels, tf.zeros_like(pad_gold_start_index_labels, tf.int32)), tf.bool) # (max_num_mention) gold_start_index_labels = self.boolean_mask_1d(pad_gold_start_index_labels, gold_start_end_mask, name_scope="gold_starts", use_tpu=self.use_tpu) # (num_of_mention) gold_end_index_labels = self.boolean_mask_1d(pad_gold_end_index_labels, gold_start_end_mask, name_scope="gold_ends", use_tpu=self.use_tpu) # (num_of_mention) gold_start_sequence_labels = self.scatter_gold_index_to_label_sequence(gold_start_index_labels, num_subtoken_in_doc) # (num_subtoken_in_doc) gold_end_sequence_labels = self.scatter_gold_index_to_label_sequence(gold_end_index_labels, num_subtoken_in_doc) # (num_subtoken_in_doc) gold_span_sequence_labels = self.scatter_span_sequence_labels(gold_start_index_labels, gold_end_index_labels, num_subtoken_in_doc) # (num_subtoken_in_doc, num_subtoken_in_doc) return gold_start_sequence_labels, gold_end_sequence_labels, gold_span_sequence_labels def scatter_gold_index_to_label_sequence(self, gold_index_labels, expect_length_of_labels): """ Desc: transform the mention start/end position index tf.int32 Tensor to a tf.int32 Tensor with 1/0 labels for the input subtoken sequences. 1 denotes this subtoken is the start/end for a mention. Args: gold_index_labels: a tf.int32 Tensor with mention start/end position index in the original document. expect_length_of_labels: the number of subtokens in the original document. """ gold_labels_pos = tf.reshape(gold_index_labels, [-1, 1]) # (num_of_mention, 1) gold_value = tf.reshape(tf.ones_like(gold_index_labels), [-1]) # (num_of_mention) label_shape = tf.Variable(expect_length_of_labels) label_shape = tf.reshape(label_shape, [1]) # [1] gold_sequence_labels = tf.cast(tf.scatter_nd(gold_labels_pos, gold_value, label_shape), tf.int32) # (num_subtoken_in_doc) return gold_sequence_labels def scatter_span_sequence_labels(self, gold_start_index_labels, gold_end_index_labels, expect_length_of_labels): """ Desc: transform the mention (start, end) position pairs to a span matrix gold_span_sequence_labels. matrix[i][j]: whether the subtokens between the position $i$ to $j$ can be a mention. if matrix[i][j] == 0: from $i$ to $j$ is not a mention. if matrix[i][j] == 1: from $i$ to $j$ is a mention. Args: gold_start_index_labels: a tf.int32 Tensor with mention start position index in the original document. gold_end_index_labels: a tf.int32 Tensor with mention end position index in the original document. expect_length_of_labels: a scalar, should be the same with num_subtoken_in_doc """ gold_span_index_labels = tf.stack([gold_start_index_labels, gold_end_index_labels], axis=1) # (num_of_mention, 2) gold_span_value = tf.reshape(tf.ones_like(gold_start_index_labels, tf.int32), [-1]) # (num_of_mention) gold_span_label_shape = tf.Variable([expect_length_of_labels, expect_length_of_labels]) gold_span_label_shape = tf.reshape(gold_span_label_shape, [-1]) gold_span_sequence_labels = tf.cast(tf.scatter_nd(gold_span_index_labels, gold_span_value, gold_span_label_shape), tf.int32) # (num_subtoken_in_doc, num_subtoken_in_doc) return gold_span_sequence_labels def compute_score_and_loss(self, pred_sequence_logits, gold_sequence_labels, loss_mask=None): """ Desc: compute the unifrom start/end loss and probabilities. Args: pred_sequence_logits: (input_shape, 1) gold_sequence_labels: (input_shape, 1) loss_mask: [optional] if is not None, it should be (input_shape). should be tf.int32 0/1 tensor. FOR start/end score and loss, input_shape should be num_subtoken_in_doc. FOR span score and loss, input_shape should be num_subtoken_in_doc * num_subtoken_in_doc. """ pred_sequence_probabilities = tf.cast(tf.reshape(tf.sigmoid(pred_sequence_logits), [-1]),tf.float32) # (input_shape) expand_pred_sequence_scores = tf.stack([(1 - pred_sequence_probabilities), pred_sequence_probabilities], axis=-1) # (input_shape, 2) expand_gold_sequence_labels = tf.cast(tf.one_hot(tf.reshape(gold_sequence_labels, [-1]), 2, axis=-1), tf.float32) # (input_shape, 2) loss = tf.keras.losses.binary_crossentropy(expand_gold_sequence_labels, expand_pred_sequence_scores) # loss -> shape is (input_shape) if loss_mask is not None: loss = tf.multiply(loss, tf.cast(loss_mask, tf.float32)) total_loss = tf.reduce_mean(loss) # total_loss -> a scalar return total_loss, pred_sequence_probabilities def transform_overlap_windows_to_original_doc(self, doc_overlap_window_embs, doc_overlap_input_mask): """ Desc: hidden_size should be equal to embeddding_size. Args: doc_overlap_window_embs: (num_window, window_size, hidden_size). the output of (num_window, window_size) input_ids forward into BERT model. doc_overlap_input_mask: (num_window, window_size). A tf.int32 Tensor contains 0/1. 0 represents token in this position should be neglected. 1 represents token in this position should be reserved. """ ones_input_mask = tf.ones_like(doc_overlap_input_mask, tf.int32) # (num_window, window_size) cumsum_input_mask = tf.math.cumsum(ones_input_mask, axis=1) # (num_window, window_size) offset_input_mask = tf.tile(tf.expand_dims(tf.range(self.config.num_window) * self.config.window_size, 1), [1, self.config.window_size]) # (num_window, window_size) offset_cumsum_input_mask = offset_input_mask + cumsum_input_mask # (num_window, window_size) global_input_mask = tf.math.multiply(ones_input_mask, offset_cumsum_input_mask) # (num_window, window_size) global_input_mask = tf.reshape(global_input_mask, [-1]) # (num_window * window_size) global_input_mask_index = self.boolean_mask_1d(global_input_mask, tf.math.greater(global_input_mask, tf.zeros_like(global_input_mask, tf.int32))) # (num_subtoken_in_doc) doc_overlap_window_embs = tf.reshape(doc_overlap_window_embs, [-1, self.config.hidden_size]) # (num_window * window_size, hidden_size) original_doc_embs = tf.gather(doc_overlap_window_embs, global_input_mask_index) # (num_subtoken_in_doc, hidden_size) return original_doc_embs def ffnn(self, inputs, hidden_size, output_size, dropout=None, name_scope="fully-conntected-neural-network", hidden_initializer=tf.truncated_normal_initializer(stddev=0.02)): """ Desc: fully-connected neural network. transform non-linearly the [input] tensor with [hidden_size] to a fix [output_size] size. Args: hidden_size: should be the size of last dimension of [inputs]. """ with tf.variable_scope(name_scope, reuse=tf.AUTO_REUSE): hidden_weights = tf.get_variable("hidden_weights", [hidden_size, output_size], initializer=hidden_initializer) hidden_bias = tf.get_variable("hidden_bias", [output_size], initializer=tf.zeros_initializer()) outputs = tf.nn.relu(tf.nn.xw_plus_b(inputs, hidden_weights, hidden_bias)) if dropout is not None: outputs = tf.nn.dropout(outputs, dropout) return outputs def get_dropout(self, dropout_rate, is_training): return 1 - (tf.to_float(is_training) * dropout_rate) def get_shape(self, x, dim): """ Desc: return the size of input x in DIM. """ return x.get_shape()[dim].value or tf.shape(x)[dim] def boolean_mask_1d(self, itemtensor, boolmask_indicator, name_scope="boolean_mask1d", use_tpu=False): """ Desc: the same functionality of tf.boolean_mask. The tf.boolean_mask operation is not available on the cloud TPU. Args: itemtensor : a Tensor contains [tf.int32, tf.float32] numbers. Should be 1-Rank. boolmask_indicator : a tf.bool Tensor. Should be 1-Rank. scope : name scope for the operation. use_tpu : if False, return tf.boolean_mask. """ with tf.name_scope(name_scope): if not use_tpu: return tf.boolean_mask(itemtensor, boolmask_indicator) boolmask_sum = tf.reduce_sum(tf.cast(boolmask_indicator, tf.int32)) selected_positions = tf.cast(boolmask_indicator, dtype=tf.float32) indexed_positions = tf.cast(tf.multiply(tf.cumsum(selected_positions), selected_positions),dtype=tf.int32) one_hot_selector = tf.one_hot(indexed_positions - 1, boolmask_sum, dtype=tf.float32) sampled_indices = tf.cast(tf.tensordot(tf.cast(tf.range(tf.shape(boolmask_indicator)[0]), dtype=tf.float32), one_hot_selector,axes=[0, 0]),dtype=tf.int32) sampled_indices = tf.reshape(sampled_indices, [-1]) mask_itemtensor = tf.gather(itemtensor, sampled_indices) return mask_itemtensor ================================================ FILE: requirements.txt ================================================ pyhocon tensorboard==1.15.0 tensorflow-estimator==1.15.1 tensorflow-gpu==1.15.0 pyyaml==5.2 scikit-learn==0.19.1 scipy torch==1.2.0 ================================================ FILE: run/build_dataset_to_tfrecord.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # author: xiaoy li # description: # generate tfrecord for train/dev/test set for the model. # TODO (xiaoya): need to add help description for args import os import sys import re import json import argparse import numpy as np import tensorflow as tf from collections import OrderedDict REPO_PATH = "/".join(os.path.realpath(__file__).split("/")[:-2]) if REPO_PATH not in sys.path: sys.path.insert(0, REPO_PATH) from data_utils import conll from bert.tokenization import FullTokenizer SPEAKER_START = '[unused19]' SPEAKER_END = '[unused73]' subtoken_maps = {} gold = {} """ Desc: a single training/test example for the squad dataset. suppose origin input_tokens are : ['[unused19]', 'speaker', '#', '1', '[unused73]', '-', '-', 'basically', ',', 'it', 'was', 'unanimously', 'agreed', 'upon', 'by', 'the', 'various', 'relevant', 'parties', '.', 'To', 'express', 'its', 'determination', ',', 'the', 'Chinese', 'securities', 'regulatory', 'department', 'compares', 'this', 'stock', 'reform', 'to', 'a', 'die', 'that', 'has', 'been', 'cast', '.', 'It', 'takes', 'time', 'to', 'prove', 'whether', 'the', 'stock', 'reform', 'can', 'really', 'meet', 'expectations', ',', 'and', 'whether', 'any', 'de', '##viation', '##s', 'that', 'arise', 'during', 'the', 'stock', 'reform', 'can', 'be', 'promptly', 'corrected', '.', '[unused19]', 'Xu', '_', 'l', '##i', '[unused73]', 'Dear', 'viewers', ',', 'the', 'China', 'News', 'program', 'will', 'end', 'here', '.', 'This', 'is', 'Xu', 'Li', '.', 'Thank', 'you', 'everyone', 'for', 'watching', '.', 'Coming', 'up', 'is', 'the', 'Focus', 'Today', 'program', 'hosted', 'by', 'Wang', 'Shi', '##lin', '.', 'Good', '-', 'bye', ',', 'dear', 'viewers', '.'] IF sliding window size is 50. Args: doc_idx: a string: cctv/bn/0001 sentence_map: e.g. [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7] subtoken_map: e.g. [0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 53, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 97, 98, 99, 99, 99, 100, 101, 102, 103] flattened_window_input_ids: [num-window, window-size] e.g. before bert_tokenizer convert subtokens into ids: [['[CLS]', '[unused19]', 'speaker', '#', '1', '[unused73]', '-', '-', 'basically', ',', 'it', 'was', 'unanimously', 'agreed', 'upon', 'by', 'the', 'various', 'relevant', 'parties', '.', 'To', 'express', 'its', 'determination', ',', 'the', 'Chinese', 'securities', 'regulatory', 'department', 'compares', 'this', 'stock', 'reform', 'to', 'a', 'die', 'that', 'has', 'been', 'cast', '.', 'It', 'takes', 'time', 'to', 'prove', 'whether', '[SEP]'], ['[CLS]', ',', 'the', 'Chinese', 'securities', 'regulatory', 'department', 'compares', 'this', 'stock', 'reform', 'to', 'a', 'die', 'that', 'has', 'been', 'cast', '.', 'It', 'takes', 'time', 'to', 'prove', 'whether', 'the', 'stock', 'reform', 'can', 'really', 'meet', 'expectations', ',', 'and', 'whether', 'any', 'de', '##viation', '##s', 'that', 'arise', 'during', 'the', 'stock', 'reform', 'can', 'be', 'promptly', 'corrected', '[SEP]'], ['[CLS]', 'the', 'stock', 'reform', 'can', 'really', 'meet', 'expectations', ',', 'and', 'whether', 'any', 'de', '##viation', '##s', 'that', 'arise', 'during', 'the', 'stock', 'reform', 'can', 'be', 'promptly', 'corrected', '.', '[unused19]', 'Xu', '_', 'l', '##i', '[unused73]', 'Dear', 'viewers', ',', 'the', 'China', 'News', 'program', 'will', 'end', 'here', '.', 'This', 'is', 'Xu', 'Li', '.', 'Thank', '[SEP]'], ['[CLS]', '.', '[unused19]', 'Xu', '_', 'l', '##i', '[unused73]', 'Dear', 'viewers', ',', 'the', 'China', 'News', 'program', 'will', 'end', 'here', '.', 'This', 'is', 'Xu', 'Li', '.', 'Thank', 'you', 'everyone', 'for', 'watching', '.', 'Coming', 'up', 'is', 'the', 'Focus', 'Today', 'program', 'hosted', 'by', 'Wang', 'Shi', '##lin', '.', 'Good', '-', 'bye', ',', 'dear', 'viewers', '[SEP]'], ['[CLS]', 'you', 'everyone', 'for', 'watching', '.', 'Coming', 'up', 'is', 'the', 'Focus', 'Today', 'program', 'hosted', 'by', 'Wang', 'Shi', '##lin', '.', 'Good', '-', 'bye', ',', 'dear', 'viewers', '.', '[SEP]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']] flattened_window_masked_ids: [num-window, window-size] e.g.: before bert_tokenizer ids: [[-3, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -3], [-3, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -3], [-3, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, -1, -1, -1, -1, -1, -1, 68, 69, 70, 71, 72, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -3], [-3, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -3], [-3, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, -3, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4]] span_start: e.g.: mention start indices in the original document [17, 20, 26, 43, 60, 85, 86] span_end: e.g.: mention end indices in the original document cluster_ids: e.g.: cluster ids for the (span_start, span_end) pairs [1, 1, 2, 2, 2, 3, 3] check the mention in the subword list: 1. ['its'] 1. ['the', 'Chinese', 'securities', 'regulatory', 'department'] 2. ['this', 'stock', 'reform'] 2. ['the', 'stock', 'reform'] 2. ['the', 'stock', 'reform'] 3. ['you'] 3. ['everyone'] """ def prepare_train_dataset(input_file, output_data_dir, output_filename, window_size, num_window, tokenizer=None, vocab_file=None, language="english", max_doc_length=None, genres=None, max_num_mention=10, max_num_cluster=30, demo=False, lowercase=False): if vocab_file is None: if not lowercase: vocab_file = os.path.join(REPO_PATH, "data_utils", "uppercase_vocab.txt") else: vocab_file = os.path.join(REPO_PATH, "data_utils", "lowercase_vocab.txt") if tokenizer is None: tokenizer = FullTokenizer(vocab_file=vocab_file, do_lower_case=lowercase) writer = tf.python_io.TFRecordWriter(os.path.join(output_data_dir, "{}.{}.tfrecord".format(output_filename, language))) doc_map = {} documents = read_conll_file(input_file) for doc_idx, document in enumerate(documents): doc_info = parse_document(document, language) tokenized_document = tokenize_document(genres, doc_info, tokenizer) doc_key = tokenized_document['doc_key'] token_windows, mask_windows, text_len = convert_to_sliding_window(tokenized_document, window_size) input_id_windows = [tokenizer.convert_tokens_to_ids(tokens) for tokens in token_windows] span_start, span_end, mention_span, cluster_ids = flatten_clusters(tokenized_document['clusters']) tmp_speaker_ids = tokenized_document["speakers"] tmp_speaker_ids = [[0]*130]* num_window instance = (input_id_windows, mask_windows, text_len, tmp_speaker_ids, tokenized_document["genre"], span_start, span_end, cluster_ids, tokenized_document['sentence_map']) write_instance_to_example_file(writer, instance, doc_key, window_size=window_size, num_window=num_window, max_num_mention=max_num_mention, max_num_cluster=max_num_cluster) doc_map[doc_idx] = doc_key if demo and doc_idx > 3: break with open(os.path.join(output_data_dir, "{}.{}.map".format(output_filename, language)), 'w') as fo: json.dump(doc_map, fo, indent=2) def write_instance_to_example_file(writer, instance, doc_key, window_size=64, num_window=5, max_num_mention=20, max_num_cluster=30, pad_idx=-1): input_ids, input_mask, text_len, speaker_ids, genre, gold_starts, gold_ends, cluster_ids, sentence_map = instance input_id_windows = input_ids mask_windows = input_mask flattened_input_ids = [i for j in input_id_windows for i in j] flattened_input_mask = [i for j in mask_windows for i in j] cluster_ids = [int(tmp) for tmp in cluster_ids] max_sequence_len = int(num_window) max_seg_len = int(window_size) sentence_map = clip_or_pad(sentence_map, max_sequence_len*max_seg_len, pad_idx=pad_idx) text_len = clip_or_pad(text_len, max_sequence_len, pad_idx=pad_idx) tmp_subtoken_maps = clip_or_pad(subtoken_maps[doc_key], max_sequence_len*max_seg_len, pad_idx=pad_idx) tmp_speaker_ids = clip_or_pad(speaker_ids[0], max_sequence_len*max_seg_len, pad_idx=pad_idx) flattened_input_ids = clip_or_pad(flattened_input_ids, max_sequence_len*max_seg_len, pad_idx=pad_idx) flattened_input_mask = clip_or_pad(flattened_input_mask, max_sequence_len*max_seg_len, pad_idx=pad_idx) gold_starts = clip_or_pad(gold_starts, max_num_mention, pad_idx=pad_idx) gold_ends = clip_or_pad(gold_ends, max_num_mention, pad_idx=pad_idx) cluster_ids = clip_or_pad(cluster_ids, max_num_cluster, pad_idx=pad_idx) features = OrderedDict() features['sentence_map'] = create_int_feature(sentence_map) features['text_len'] = create_int_feature(text_len) features['subtoken_map'] = create_int_feature(tmp_subtoken_maps) features['speaker_ids'] = create_int_feature(tmp_speaker_ids) features['flattened_input_ids'] = create_int_feature(flattened_input_ids) features['flattened_input_mask'] = create_int_feature(flattened_input_mask) features['span_starts'] = create_int_feature(gold_starts) features['span_ends'] = create_int_feature(gold_ends) features['cluster_ids'] = create_int_feature(cluster_ids) tf_example = tf.train.Example(features=tf.train.Features(feature=features)) writer.write(tf_example.SerializeToString()) def create_int_feature(values): feature = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) return feature def clip_or_pad(var, max_var_len, pad_idx=-1): if len(var) >= max_var_len: return var[:max_var_len] else: pad_var = (max_var_len - len(var)) * [pad_idx] var = list(var) + list(pad_var) return var def flatten_clusters(clusters): span_starts = [] span_ends = [] cluster_ids = [] mention_span = [] for cluster_id, cluster in enumerate(clusters): for start, end in cluster: span_starts.append(start) span_ends.append(end) mention_span.append((start, end)) cluster_ids.append(cluster_id + 1) return span_starts, span_ends, mention_span, cluster_ids def read_conll_file(conll_file_path): documents = [] with open(conll_file_path, "r", encoding="utf-8") as fi: for line in fi: begin_document_match = re.match(conll.BEGIN_DOCUMENT_REGEX, line) if begin_document_match: doc_key = conll.get_doc_key(begin_document_match.group(1), begin_document_match.group(2)) documents.append((doc_key, [])) elif line.startswith("#end document"): continue else: documents[-1][1].append(line.strip()) return documents def parse_document(document, language): """ get basic information from one document annotation. :param document: :param language: english, chinese or arabic :return: """ doc_key = document[0] sentences = [[]] speakers = [] coreferences = [] word_idx = -1 last_speaker = '' for line_id, line in enumerate(document[1]): row = line.split() sentence_end = len(row) == 0 if not sentence_end: assert len(row) >= 12 word_idx += 1 word = normalize_word(row[3], language) sentences[-1].append(word) speaker = row[9] if speaker != last_speaker: speakers.append((word_idx, speaker)) last_speaker = speaker coreferences.append(row[-1]) else: sentences.append([]) clusters = coreference_annotations_to_clusters(coreferences) doc_info = {'doc_key': doc_key, 'sentences': sentences[: -1], 'speakers': speakers, 'clusters': clusters} return doc_info def normalize_word(word, language): if language == "arabic": word = word[:word.find("#")] if word == "/." or word == "/?": return word[1:] else: return word def coreference_annotations_to_clusters(annotations): """ convert coreference information to clusters :param annotations: :return: """ clusters = OrderedDict() coref_stack = OrderedDict() for word_idx, annotation in enumerate(annotations): if annotation == '-': continue for ann in annotation.split('|'): cluster_id = int(ann.replace('(', '').replace(')', '')) if ann[0] == '(' and ann[-1] == ')': if cluster_id not in clusters.keys(): clusters[cluster_id] = [(word_idx, word_idx)] else: clusters[cluster_id].append((word_idx, word_idx)) elif ann[0] == '(': if cluster_id not in coref_stack.keys(): coref_stack[cluster_id] = [word_idx] else: coref_stack[cluster_id].append(word_idx) elif ann[-1] == ')': span_start = coref_stack[cluster_id].pop() if cluster_id not in clusters.keys(): clusters[cluster_id] = [(span_start, word_idx)] else: clusters[cluster_id].append((span_start, word_idx)) else: raise NotImplementedError assert all([len(starts) == 0 for starts in coref_stack.values()]) return list(clusters.values()) def checkout_clusters(doc_info): words = [i for j in doc_info['sentences'] for i in j] clusters = [[' '.join(words[start: end + 1]) for start, end in cluster] for cluster in doc_info['clusters']] print(clusters) def tokenize_document(genres, doc_info, tokenizer): """ tokenize into sub tokens :param doc_info: :param tokenizer: max_doc_length: pad to max_doc_length :return: """ genres = {g: i for i, g in enumerate(genres)} sub_tokens = [] # all sub tokens of a document sentence_map = [] # collected tokenized tokens -> sentence id subtoken_map = [] # collected tokenized tokens -> original token id word_idx = -1 for sentence_id, sentence in enumerate(doc_info['sentences']): for token in sentence: word_idx += 1 word_tokens = tokenizer.tokenize(token) sub_tokens.extend(word_tokens) sentence_map.extend([sentence_id] * len(word_tokens)) subtoken_map.extend([word_idx] * len(word_tokens)) subtoken_maps[doc_info['doc_key']] = subtoken_map genre = genres.get(doc_info['doc_key'][:2], 0) speakers = {subtoken_map.index(word_index): tokenizer.tokenize(speaker) for word_index, speaker in doc_info['speakers']} clusters = [[(subtoken_map.index(start), len(subtoken_map) - 1 - subtoken_map[::-1].index(end)) for start, end in cluster] for cluster in doc_info['clusters']] tokenized_document = {'sub_tokens': sub_tokens, 'sentence_map': sentence_map, 'subtoken_map': subtoken_map, 'speakers': speakers, 'clusters': clusters, 'doc_key': doc_info['doc_key'], "genre": genre} return tokenized_document def convert_to_sliding_window(tokenized_document, sliding_window_size): """ construct sliding windows, allocate tokens and masks into each window :param tokenized_document: :param sliding_window_size: :return: """ expanded_tokens, expanded_masks = expand_with_speakers(tokenized_document) sliding_windows = construct_sliding_windows(len(expanded_tokens), sliding_window_size - 2) token_windows = [] # expanded tokens to sliding window mask_windows = [] # expanded masks to sliding window text_len = [] for window_start, window_end, window_mask in sliding_windows: original_tokens = expanded_tokens[window_start: window_end] original_masks = expanded_masks[window_start: window_end] window_masks = [-2 if w == 0 else o for w, o in zip(window_mask, original_masks)] one_window_token = ['[CLS]'] + original_tokens + ['[SEP]'] + ['[PAD]'] * ( sliding_window_size - 2 - len(original_tokens)) one_window_mask = [-3] + window_masks + [-3] + [-4] * (sliding_window_size - 2 - len(original_tokens)) token_calculate = [tmp for tmp in one_window_mask if tmp >= 0] text_len.append(len(token_calculate)) assert len(one_window_token) == sliding_window_size assert len(one_window_mask) == sliding_window_size token_windows.append(one_window_token) mask_windows.append(one_window_mask) assert len(tokenized_document['sentence_map']) == sum([i >= 0 for j in mask_windows for i in j]) text_len = np.array(text_len) return token_windows, mask_windows, text_len def expand_with_speakers(tokenized_document): """ add speaker name information :param tokenized_document: tokenized document information :return: """ expanded_tokens = [] expanded_masks = [] for token_idx, token in enumerate(tokenized_document['sub_tokens']): if token_idx in tokenized_document['speakers']: speaker = [SPEAKER_START] + tokenized_document['speakers'][token_idx] + [SPEAKER_END] expanded_tokens.extend(speaker) expanded_masks.extend([-1] * len(speaker)) expanded_tokens.append(token) expanded_masks.append(token_idx) return expanded_tokens, expanded_masks def construct_sliding_windows(sequence_length, sliding_window_size): """ construct sliding windows for BERT processing :param sequence_length: e.g. 9 :param sliding_window_size: e.g. 4 :return: [(0, 4, [1, 1, 1, 0]), (2, 6, [0, 1, 1, 0]), (4, 8, [0, 1, 1, 0]), (6, 9, [0, 1, 1])] """ sliding_windows = [] stride = int(sliding_window_size / 2) start_index = 0 end_index = 0 while end_index < sequence_length: end_index = min(start_index + sliding_window_size, sequence_length) left_value = 1 if start_index == 0 else 0 right_value = 1 if end_index == sequence_length else 0 mask = [left_value] * int(sliding_window_size / 4) + [1] * int(sliding_window_size / 2) \ + [right_value] * (sliding_window_size - int(sliding_window_size / 2) - int(sliding_window_size / 4)) mask = mask[: end_index - start_index] sliding_windows.append((start_index, end_index, mask)) start_index += stride assert sum([sum(window[2]) for window in sliding_windows]) == sequence_length return sliding_windows def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("--source_files_dir", default="/home/lixiaoya/data", type=str, required=True) parser.add_argument("--target_output_dir", default="/home/lixiaoya/tfrecord_data", type=str, required=True) parser.add_argument("--num_window", default=5, type=int, required=True) parser.add_argument("--window_size", default=64, type=int, required=True) parser.add_argument("--max_num_mention", default=30, type=int) parser.add_argument("--max_num_cluster", default=20, type=int) parser.add_argument("--vocab_file", default="/home/lixiaoya/spanbert_large_cased/vocab.txt", type=str) parser.add_argument("--language", default="english", type=str) parser.add_argument("--max_doc_length", default=600, type=int) parser.add_argument("--lowercase", help="DO or NOT lowercase the datasets.", action="store_true") parser.add_argument("--demo", help="Wether to generate a small dataset for testing the code.", action="store_true") parser.add_argument('--genres', default=["bc","bn","mz","nw","pt","tc","wb"]) parser.add_argument("--seed", default=2333, type=int) args = parser.parse_args() os.makedirs(args.target_output_dir, exist_ok=True) np.random.seed(args.seed) tf.set_random_seed(args.seed) return args def main(): args_config = parse_args() print("*"*60) print("***** ***** show configs ***** ***** ") print("window_size : {}".format(str(args_config.window_size))) print("num_window : {}".format(str(args_config.num_window))) print("*"*60) for data_sign in ["train", "dev", "test"]: source_data_file = os.path.join(args_config.source_files_dir, "{}.{}.v4_gold_conll".format(data_sign, args_config.language)) output_filename = "{}.overlap.corefqa".format(data_sign) if args_config.demo: if args_config.lowercase: output_filename="demo.lowercase.{}.overlap.corefqa".format(data_sign) else: output_filename="demo.{}.overlap.corefqa".format(data_sign) print("$"*60) print("generate {}/{}".format(args_config.target_output_dir, output_filename)) prepare_train_dataset(source_data_file, args_config.target_output_dir, output_filename, args_config.window_size, args_config.num_window, vocab_file=args_config.vocab_file, language=args_config.language, max_doc_length=args_config.max_doc_length, genres=args_config.genres, max_num_mention=args_config.max_num_mention, max_num_cluster=args_config.max_num_cluster, demo=args_config.demo, lowercase=args_config.lowercase) if __name__ == "__main__": main() # please refer ${REPO_PATH}/scripts/data/generate_tfrecord_dataset.sh # # for generate tfrecord datasets # # python3 build_dataset_to_tfrecord.py \ # --source_files_dir /xiaoya/data \ # --target_output_dir /xiaoya/corefqa_data/overlap_64_2 \ # --num_window 2 \ # --window_size 64 \ # --max_num_mention 50 \ # --max_num_cluster 30 \ # --vocab_file /xiaoya/pretrain_ckpt/cased_L-12_H-768_A-12/vocab.txt \ # --language english \ # --max_doc_length 600 # ================================================ FILE: run/run_corefqa.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- """ this file contains training and testing the CorefQA model. """ import os import math import logging import random import numpy as np import tensorflow as tf from utils import util from utils import metrics from data_utils.config_utils import ModelConfig from func_builders.model_fn_builder import model_fn_builder from func_builders.input_fn_builder import file_based_input_fn_builder tf.app.flags.DEFINE_string('f', '', 'kernel') flags = tf.app.flags flags.DEFINE_string("output_dir", "data", "The output directory.") flags.DEFINE_string("bert_config_file", "/home/uncased_L-2_H-128_A-2/config.json", "The config json file corresponding to the pre-trained BERT model.") flags.DEFINE_string("init_checkpoint", "/home/uncased_L-2_H-128_A-2/bert_model.ckpt", "Initial checkpoint (usually from a pre-trained BERT model).") flags.DEFINE_string("vocab_file", "/home/uncased_L-2_H-128_A-2/vocab.txt", "The vocabulary file that the BERT model was trained on.") flags.DEFINE_string("logfile_path", "/home/lixiaoya/spanbert_large_mention_proposal.log", "the path to the exported log file.") flags.DEFINE_integer("num_epochs", 20, "Total number of training epochs to perform.") flags.DEFINE_integer("keep_checkpoint_max", 30, "How many checkpoint models keep at most.") flags.DEFINE_integer("save_checkpoints_steps", 500, "Save checkpoint every X updates steps.") flags.DEFINE_string("train_file", "/home/lixiaoya/train.english.tfrecord", "TFRecord file for training. E.g., train.english.tfrecord") flags.DEFINE_string("dev_file", "/home/lixiaoya/dev.english.tfrecord", "TFRecord file for validating. E.g., dev.english.tfrecord") flags.DEFINE_string("test_file", "/home/lixiaoya/test.english.tfrecord", "TFRecord file for testing. E.g., test.english.tfrecord") flags.DEFINE_bool("do_train", True, "Whether to train a model.") flags.DEFINE_bool("do_eval", False, "Whether to do evaluation: evaluation is done on a set of trained checkpoints, the model will select the best one on the dev set, and report result on the test set") flags.DEFINE_bool("do_predict", False, "Whether to test (only) one trained model.") flags.DEFINE_string("eval_checkpoint", "/home/lixiaoya/mention_proposal_output_dir/bert_model.ckpt", "[Optional] The saved checkpoint for evaluation (usually after the training process).") flags.DEFINE_integer("iterations_per_loop", 1000, "How many steps to make in each estimator call.") flags.DEFINE_float("learning_rate", 3e-5, "The initial learning rate for Adam.") flags.DEFINE_float("dropout_rate", 0.3, "Dropout rate for the training process.") flags.DEFINE_float("mention_threshold", 0.5, "The threshold for determining whether the span is a mention.") flags.DEFINE_integer("hidden_size", 128, "The size of hidden layers for the pre-trained model.") flags.DEFINE_integer("num_docs", 5604, "[Optional] The number of documents in the training files. Only need to change when conduct experiments on the small test sets.") flags.DEFINE_integer("window_size", 384, "The number of sliding window size. Each document is split into a set of subdocuments with length set to window_size.") flags.DEFINE_integer("num_window", 5, "The max number of windows for one document. This is used for fitting a document into fix shape for TF computation. \ If a document is longer than num_window*window_size, the exceeding part will be abandoned. This only affects training and does not affect test, since the all \ docs in the test set is shorter than num_window*window_size") flags.DEFINE_integer("max_num_mention", 30, "The max number of mentions in one document.") flags.DEFINE_bool("start_end_share", False, "Whether only to use [start, end] embedding to calculate the start/end scores.") flags.DEFINE_integer("max_span_width", 5, "The max length of a mention.") flags.DEFINE_integer("max_candidate_mentions", 30, "The number of candidate mentions.") flags.DEFINE_float("top_span_ratio", 0.2, "The ratio of.") flags.DEFINE_integer("max_top_antecedents", 30, "The number of top_antecedents candidate mentions.") flags.DEFINE_integer("max_query_len", 150, ".") flags.DEFINE_integer("max_context_len", 150, ".") flags.DEFINE_bool("sec_qa_mention_score", False, "Whether to use TPU or GPU/CPU.") flags.DEFINE_bool("use_tpu", False, "Whether to use TPU or GPU/CPU.") flags.DEFINE_string("tpu_name", None, "The Cloud TPU to use for training. This should be either the name used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 url.") flags.DEFINE_string("tpu_zone", None, "[Optional] GCE zone where the Cloud TPU is located in. If not specified, we will attempt to automatically detect the GCE project from metadata.") flags.DEFINE_string("gcp_project", None, "[Optional] Project name for the Cloud TPU-enabled project. If not specified, we will attempt to automatically detect the GCE project from metadata.") flags.DEFINE_string("master", None, "[Optional] TensorFlow master URL.") flags.DEFINE_integer("num_tpu_cores", 1, "[Optional] Only used if `use_tpu` is True. Total number of TPU cores to use.") flags.DEFINE_integer("seed", 2333, "[Optional] Random seed for initialization.") FLAGS = tf.flags.FLAGS format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s' logging.basicConfig(format=format, filename=FLAGS.logfile_path, level=logging.INFO) logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) def main(_): tf.logging.set_verbosity(tf.logging.INFO) num_train_steps = FLAGS.num_docs * FLAGS.num_epochs keep_chceckpoint_max = max(math.ceil(num_train_steps / FLAGS.save_checkpoints_steps), FLAGS.keep_checkpoint_max) if not FLAGS.do_train and not FLAGS.do_eval and not FLAGS.do_predict: raise ValueError("At least one of `do_train`, `do_eval` or `do_predict' must be True.") tf.gfile.MakeDirs(FLAGS.output_dir) tpu_cluster_resolver = None if FLAGS.use_tpu and FLAGS.tpu_name: tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver( FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) tf.config.experimental_connect_to_cluster(tpu_cluster_resolver) tf.tpu.experimental.initialize_tpu_system(tpu_cluster_resolver) is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 run_config = tf.contrib.tpu.RunConfig( cluster=tpu_cluster_resolver, master=FLAGS.master, model_dir=FLAGS.output_dir, evaluation_master=FLAGS.master, keep_checkpoint_max = keep_chceckpoint_max, save_checkpoints_steps=FLAGS.save_checkpoints_steps, session_config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True), tpu_config=tf.contrib.tpu.TPUConfig( iterations_per_loop=FLAGS.iterations_per_loop, num_shards=FLAGS.num_tpu_cores, per_host_input_for_training=is_per_host)) model_config = ModelConfig(FLAGS, FLAGS.output_dir) model_config.logging_configs() model_fn = model_fn_builder(model_config, model_sign="corefqa") estimator = tf.contrib.tpu.TPUEstimator( use_tpu=FLAGS.use_tpu, eval_on_tpu=FLAGS.use_tpu, warm_start_from=tf.estimator.WarmStartSettings(FLAGS.init_checkpoint, vars_to_warm_start="bert*"), model_fn=model_fn, config=run_config, train_batch_size=1, eval_batch_size=1, predict_batch_size=1) if FLAGS.do_train: estimator.train(input_fn=file_based_input_fn_builder(FLAGS.train_file, num_window=FLAGS.num_window, window_size=FLAGS.window_size, max_num_mention=FLAGS.max_num_mention, is_training=True, drop_remainder=True), max_steps=num_train_steps) if FLAGS.do_eval: best_dev_f1, best_dev_prec, best_dev_rec, test_f1_when_dev_best, test_prec_when_dev_best, test_rec_when_dev_best = 0, 0, 0, 0, 0, 0 best_ckpt_path = "" checkpoints_iterator = [os.path.join(FLAGS.eval_dir, "model.ckpt-{}".format(str(int(ckpt_idx)))) for ckpt_idx in range(0, num_train_steps+1, FLAGS.save_checkpoints_steps)] model = util.get_model(model_config, model_sign="corefqa") for checkpoint_path in checkpoints_iterator[1:]: dev_coref_evaluator = metrics.CorefEvaluator() for result in estimator.predict(file_based_input_fn_builder(FLAGS.dev_file, num_window=FLAGS.num_window, window_size=FLAGS.window_size, max_num_mention=FLAGS.max_num_mention, is_training=False, drop_remainder=False), steps=698, checkpoint_path=checkpoint_path, yield_single_examples=False): predicted_clusters, gold_clusters, mention_to_predicted, mention_to_gold = model.evaluate(result["topk_span_starts"], result["topk_span_ends"], result["top_antecedent"], result["cluster_ids"], result["gold_starts"], result["gold_ends"]) dev_coref_evaluator.update(predicted_clusters, gold_clusters, mention_to_predicted, mention_to_gold) dev_prec, dev_rec, dev_f1 = dev_coref_evaluator.get_prf() tf.logging.info("***** Current ckpt path is ***** : {}".format(checkpoint_path)) tf.logging.info("***** EVAL ON DEV SET *****") tf.logging.info("***** [DEV EVAL] ***** : precision: {:.4f}, recall: {:.4f}, f1: {:.4f}".format(dev_prec, dev_rec, dev_f1)) if dev_f1 > best_dev_f1: best_ckpt_path = checkpoint_path best_dev_f1 = dev_f1 best_dev_prec = dev_prec best_dev_rec = dev_rec test_coref_evaluator = metrics.CorefEvaluator() for result in estimator.predict(file_based_input_fn_builder(FLAGS.test_file, num_window=FLAGS.num_window, window_size=FLAGS.window_size, max_num_mention=FLAGS.max_num_mention, is_training=False, drop_remainder=False),steps=698, checkpoint_path=checkpoint_path, yield_single_examples=False): predicted_clusters, gold_clusters, mention_to_predicted, mention_to_gold = model.evaluate(result["topk_span_starts"], result["topk_span_ends"], result["top_antecedent"], result["cluster_ids"], result["gold_starts"], result["gold_ends"]) test_coref_evaluator.update(predicted_clusters, gold_clusters, mention_to_predicted, mention_to_gold) test_pre, test_rec, test_f1 = test_coref_evaluator.get_prf() test_f1_when_dev_best, test_prec_when_dev_best, test_rec_when_dev_best = test_f1, test_pre, test_rec tf.logging.info("***** EVAL ON TEST SET *****") tf.logging.info("***** [TEST EVAL] ***** : precision: {:.4f}, recall: {:.4f}, f1: {:.4f}".format(test_pre, test_rec, test_f1)) tf.logging.info("*"*20) tf.logging.info("- @@@@@ the path to the BEST DEV result is : {}".format(best_ckpt_path)) tf.logging.info("- @@@@@ BEST DEV F1 : {:.4f}, Precision : {:.4f}, Recall : {:.4f},".format(best_dev_f1, best_dev_prec, best_dev_rec)) tf.logging.info("- @@@@@ TEST when DEV best F1 : {:.4f}, Precision : {:.4f}, Recall : {:.4f},".format(test_f1_when_dev_best, test_prec_when_dev_best, test_rec_when_dev_best)) if FLAGS.do_predict: coref_evaluator = metrics.CorefEvaluator() model = util.get_model(model_config, model_sign="corefqa") for result in estimator.predict(file_based_input_fn_builder(FLAGS.test_file, num_window=FLAGS.num_window, window_size=FLAGS.window_size, max_num_mention=FLAGS.max_num_mention, is_training=False, drop_remainder=False),steps=698, checkpoint_path=checkpoint_path, yield_single_examples=False): predicted_clusters, gold_clusters, mention_to_predicted, mention_to_gold = model.evaluate(result["topk_span_starts"], result["topk_span_ends"], result["top_antecedent"], result["cluster_ids"], result["gold_starts"], result["gold_ends"]) coref_evaluator.update(predicted_clusters, gold_clusters, mention_to_predicted, mention_to_gold) p, r, f = coref_evaluator.get_prf() tf.logging.info("Average precision: {:.4f}, Average recall: {:.4f}, Average F1 {:.4f}".format(p, r, f)) if __name__ == '__main__': # set the random seed. random.seed(FLAGS.seed) np.random.seed(FLAGS.seed) tf.set_random_seed(FLAGS.seed) # start train/evaluate the model. tf.app.run() ================================================ FILE: run/run_mention_proposal.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ this file contains pre-training and testing the mention proposal model """ import os import math import random import logging import numpy as np import tensorflow as tf from data_utils.config_utils import ModelConfig from func_builders.model_fn_builder import model_fn_builder from func_builders.input_fn_builder import file_based_input_fn_builder from utils.metrics import mention_proposal_prediction tf.app.flags.DEFINE_string('f', '', 'kernel') flags = tf.app.flags flags.DEFINE_string("output_dir", "data", "The output directory of the model training.") flags.DEFINE_string("bert_config_file", "/home/uncased_L-2_H-128_A-2/config.json", "The config json file corresponding to the pre-trained BERT model.") flags.DEFINE_string("init_checkpoint", "/home/uncased_L-2_H-128_A-2/bert_model.ckpt", "Initial checkpoint (usually from a pre-trained BERT model).") flags.DEFINE_string("vocab_file", "/home/uncased_L-2_H-128_A-2/vocab.txt", "The vocabulary file that the BERT model was trained on.") flags.DEFINE_string("logfile_path", "/home/lixiaoya/spanbert_large_mention_proposal.log", "the path to the exported log file.") flags.DEFINE_integer("num_epochs", 20, "Total number of training epochs to perform.") flags.DEFINE_integer("keep_checkpoint_max", 30, "How many checkpoint models keep at most.") flags.DEFINE_integer("save_checkpoints_steps", 500, "Save checkpoint every X updates steps.") flags.DEFINE_string("train_file", "/home/lixiaoya/train.english.tfrecord", "TFRecord file for training. E.g., train.english.tfrecord") flags.DEFINE_string("dev_file", "/home/lixiaoya/dev.english.tfrecord", "TFRecord file for validating. E.g., dev.english.tfrecord") flags.DEFINE_string("test_file", "/home/lixiaoya/test.english.tfrecord", "TFRecord file for testing. E.g., test.english.tfrecord") flags.DEFINE_bool("do_train", True, "Whether to train a model.") flags.DEFINE_bool("do_eval", False, "whether to do evaluation: evaluation is done on a set of trained checkpoints, the checkpoint with the best score on the dev set will be selected.") flags.DEFINE_bool("do_predict", False, "Whether to test (only) one trained model.") flags.DEFINE_string("eval_checkpoint", "/home/lixiaoya/mention_proposal_output_dir/bert_model.ckpt", "[Optional] The saved checkpoint for evaluation (usually after the training process).") flags.DEFINE_integer("iterations_per_loop", 1000, "How many steps to make in each estimator call.") flags.DEFINE_float("learning_rate", 3e-5, "The initial learning rate for Adam.") flags.DEFINE_float("dropout_rate", 0.3, "Dropout rate for the training process.") flags.DEFINE_float("mention_threshold", 0.5, "The threshold for determining whether the span is a mention.") flags.DEFINE_integer("hidden_size", 128, "The size of hidden layers for the pre-trained model.") flags.DEFINE_integer("num_docs", 5604, "[Optional] The number of documents in the training files. Only need to change when conduct experiments on the small test sets.") flags.DEFINE_integer("window_size", 384, "The number of sliding window size. Each document is split into a set of subdocuments with length set to window_size.") flags.DEFINE_integer("num_window", 5, "The max number of windows for one document. This is used for fitting a document into fix shape for TF computation. \ If a document is longer than num_window*window_size, the exceeding part will be abandoned. This only affects training and does not affect test, since the all \ docs in the test set is shorter than num_window*window_size") flags.DEFINE_integer("max_num_mention", 30, "The max number of mentions in one document.") flags.DEFINE_bool("start_end_share", False, "Whether only to use [start, end] embedding to calculate the start/end scores.") flags.DEFINE_float("loss_start_ratio", 0.3, "As described in the paper, the loss for a span being a mention is -loss_start_ratio* log p(the start of the given span is a start).") flags.DEFINE_float("loss_end_ratio", 0.3, "As described in the paper, the loss for a span being a mention is -loss_end_ratio* log p(the end of the given span is a end).") flags.DEFINE_float("loss_span_ratio", 0.4, "As described in the paper, the loss for a span being a mention is -loss_span_ratio* log p(the start and the end forms a span).") flags.DEFINE_bool("use_tpu", False, "Whether to use TPU or GPU/CPU.") flags.DEFINE_string("tpu_name", None, "The Cloud TPU to use for training. This should be either the name used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 url.") flags.DEFINE_string("tpu_zone", None, "[Optional] GCE zone where the Cloud TPU is located in. If not specified, we will attempt to automatically detect the GCE project from metadata.") flags.DEFINE_string("gcp_project", None, "[Optional] Project name for the Cloud TPU-enabled project. If not specified, we will attempt to automatically detect the GCE project from metadata.") flags.DEFINE_string("master", None, "[Optional] TensorFlow master URL.") flags.DEFINE_integer("num_tpu_cores", 1, "[Optional] Only used if `use_tpu` is True. Total number of TPU cores to use.") flags.DEFINE_integer("seed", 2333, "[Optional] Random seed for initialization.") FLAGS = tf.flags.FLAGS format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s' logging.basicConfig(format=format, filename=FLAGS.logfile_path, level=logging.INFO) logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) def main(_): tf.logging.set_verbosity(tf.logging.INFO) num_train_steps = FLAGS.num_docs * FLAGS.num_epochs # num_train_steps = 100 keep_chceckpoint_max = max(math.ceil(num_train_steps / FLAGS.save_checkpoints_steps), FLAGS.keep_checkpoint_max) if not FLAGS.do_train and not FLAGS.do_eval and not FLAGS.do_predict: raise ValueError("At least one of `do_train`, `do_eval` or `do_predict' must be True.") tf.gfile.MakeDirs(FLAGS.output_dir) tpu_cluster_resolver = None if FLAGS.use_tpu and FLAGS.tpu_name: tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver( FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) tf.config.experimental_connect_to_cluster(tpu_cluster_resolver) tf.tpu.experimental.initialize_tpu_system(tpu_cluster_resolver) is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 run_config = tf.contrib.tpu.RunConfig( cluster=tpu_cluster_resolver, master=FLAGS.master, # evaluation_master=FLAGS.master, model_dir=FLAGS.output_dir, keep_checkpoint_max = keep_chceckpoint_max, save_checkpoints_steps=FLAGS.save_checkpoints_steps, # session_config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True), tpu_config=tf.contrib.tpu.TPUConfig( iterations_per_loop=FLAGS.iterations_per_loop, num_shards=FLAGS.num_tpu_cores, per_host_input_for_training=is_per_host)) model_config = ModelConfig(FLAGS, FLAGS.output_dir) model_config.logging_configs() model_fn = model_fn_builder(model_config, model_sign="mention_proposal") estimator = tf.contrib.tpu.TPUEstimator( use_tpu=FLAGS.use_tpu, # eval_on_tpu=FLAGS.use_tpu, warm_start_from=tf.estimator.WarmStartSettings(FLAGS.init_checkpoint, vars_to_warm_start="bert*"), model_fn=model_fn, config=run_config, train_batch_size=1, predict_batch_size=1) if FLAGS.do_train: estimator.train(input_fn=file_based_input_fn_builder(model_config.train_file, num_window=model_config.num_window, window_size=model_config.window_size, max_num_mention=model_config.max_num_mention, is_training=True, drop_remainder=True), max_steps=num_train_steps) if FLAGS.do_eval: # doing evaluation on a set of trained checkpoints, the checkpoint with the best score on the dev set will be selected. best_dev_f1, best_dev_prec, best_dev_rec, test_f1_when_dev_best, test_prec_when_dev_best, test_rec_when_dev_best = 0, 0, 0, 0, 0, 0 best_ckpt_path = "" checkpoints_iterator = [os.path.join(FLAGS.eval_dir, "model.ckpt-{}".format(str(int(ckpt_idx)))) for ckpt_idx in range(0, num_train_steps, FLAGS.save_checkpoints_steps)] for checkpoint_path in checkpoints_iterator[1:]: eval_dev_result = estimator.evaluate(input_fn=file_based_input_fn_builder(FLAGS.dev_file, num_window=FLAGS.num_window, window_size=FLAGS.window_size, max_num_mention=FLAGS.max_num_mention, is_training=False, drop_remainder=False), steps=698, checkpoint_path=checkpoint_path) dev_f1 = 2*eval_dev_result["precision"] * eval_dev_result["recall"] / (eval_dev_result["precision"] + eval_dev_result["recall"]+1e-10) tf.logging.info("***** Current ckpt path is ***** : {}".format(checkpoint_path)) tf.logging.info("***** EVAL ON DEV SET *****") tf.logging.info("***** [DEV EVAL] ***** : precision: {:.4f}, recall: {:.4f}, f1: {:.4f}".format(eval_dev_result["precision"], eval_dev_result["recall"], dev_f1)) if dev_f1 > best_dev_f1: best_dev_f1, best_dev_prec, best_dev_rec = dev_f1, eval_dev_result["precision"], eval_dev_result["recall"] best_ckpt_path = checkpoint_path eval_test_result = estimator.evaluate(input_fn=file_based_input_fn_builder(FLAGS.test_file, num_window=FLAGS.num_window, window_size=FLAGS.window_size, max_num_mention=FLAGS.max_num_mention, is_training=False, drop_remainder=False),steps=698, checkpoint_path=checkpoint_path) test_f1 = 2*eval_test_result["precision"] * eval_test_result["recall"] / (eval_test_result["precision"] + eval_test_result["recall"]+1e-10) test_f1_when_dev_best, test_prec_when_dev_best, test_rec_when_dev_best = test_f1, eval_test_result["precision"], eval_test_result["recall"] tf.logging.info("***** EVAL ON TEST SET *****") tf.logging.info("***** [TEST EVAL] ***** : precision: {:.4f}, recall: {:.4f}, f1: {:.4f}".format(eval_test_result["precision"], eval_test_result["recall"], test_f1)) tf.logging.info("*"*20) tf.logging.info("- @@@@@ the path to the BEST DEV result is : {}".format(best_ckpt_path)) tf.logging.info("- @@@@@ BEST DEV F1 : {:.4f}, Precision : {:.4f}, Recall : {:.4f},".format(best_dev_f1, best_dev_prec, best_dev_rec)) tf.logging.info("- @@@@@ TEST when DEV best F1 : {:.4f}, Precision : {:.4f}, Recall : {:.4f},".format(test_f1_when_dev_best, test_prec_when_dev_best, test_rec_when_dev_best)) tf.logging.info("- @@@@@ mention_proposal_only_concate {}".format(FLAGS.mention_proposal_only_concate)) if FLAGS.do_predict: tp, fp, fn = 0, 0, 0 epsilon = 1e-10 for doc_output in estimator.predict(file_based_input_fn_builder(FLAGS.test_file, num_window=FLAGS.num_window, window_size=FLAGS.window_size, max_num_mention=FLAGS.max_num_mention, is_training=False, drop_remainder=False), checkpoint_path=FLAGS.eval_checkpoint, yield_single_examples=False): # iterate over each doc for evaluation pred_span_label, gold_span_label = mention_proposal_prediction(FLAGS, doc_output) tem_tp = np.logical_and(pred_span_label, gold_span_label).sum() tem_fp = np.logical_and(pred_span_label, np.logical_not(gold_span_label)).sum() tem_fn = np.logical_and(np.logical_not(pred_span_label), gold_span_label).sum() tp += tem_tp fp += tem_fp fn += tem_fn p = tp / (tp+fp+epsilon) r = tp / (tp+fn+epsilon) f = 2*p*r/(p+r+epsilon) tf.logging.info("Average precision: {:.4f}, Average recall: {:.4f}, Average F1 {:.4f}".format(p, r, f)) if __name__ == '__main__': # set the random seed. random.seed(FLAGS.seed) np.random.seed(FLAGS.seed) tf.set_random_seed(FLAGS.seed) # start train/evaluate the model. tf.app.run() ================================================ FILE: run/run_squad.py ================================================ # coding=utf-8 # Copyright 2018 The Google AI Language Team Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Run BERT on SQuAD 1.1 and SQuAD 2.0.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function import collections import json import math import os import random import six import tensorflow as tf from bert import modeling from bert import optimization from bert import tokenization flags = tf.flags FLAGS = flags.FLAGS ## Required parameters flags.DEFINE_string( "bert_config_file", None, "The config json file corresponding to the pre-trained BERT model. " "This specifies the model architecture.") flags.DEFINE_string("vocab_file", None, "The vocabulary file that the BERT model was trained on.") flags.DEFINE_string( "output_dir", None, "The output directory where the model checkpoints will be written.") ## Other parameters flags.DEFINE_string("train_file", None, "SQuAD json for training. E.g., train-v1.1.json") flags.DEFINE_string( "predict_file", None, "SQuAD json for predictions. E.g., dev-v1.1.json or test-v1.1.json") flags.DEFINE_string( "init_checkpoint", None, "Initial checkpoint (usually from a pre-trained BERT model).") flags.DEFINE_bool( "do_lower_case", True, "Whether to lower case the input text. Should be True for uncased " "models and False for cased models.") flags.DEFINE_integer( "max_seq_length", 384, "The maximum total input sequence length after WordPiece tokenization. " "Sequences longer than this will be truncated, and sequences shorter " "than this will be padded.") flags.DEFINE_integer( "doc_stride", 128, "When splitting up a long document into chunks, how much stride to " "take between chunks.") flags.DEFINE_integer( "max_query_length", 64, "The maximum number of tokens for the question. Questions longer than " "this will be truncated to this length.") flags.DEFINE_bool("do_train", False, "Whether to run training.") flags.DEFINE_bool("do_predict", False, "Whether to run eval on the dev set.") flags.DEFINE_integer("train_batch_size", 32, "Total batch size for training.") flags.DEFINE_integer("predict_batch_size", 8, "Total batch size for predictions.") flags.DEFINE_float("learning_rate", 5e-5, "The initial learning rate for Adam.") flags.DEFINE_float("num_train_epochs", 3.0, "Total number of training epochs to perform.") flags.DEFINE_float( "warmup_proportion", 0.1, "Proportion of training to perform linear learning rate warmup for. " "E.g., 0.1 = 10% of training.") flags.DEFINE_integer("save_checkpoints_steps", 1000, "How often to save the model checkpoint.") flags.DEFINE_integer("iterations_per_loop", 1000, "How many steps to make in each estimator call.") flags.DEFINE_integer( "n_best_size", 20, "The total number of n-best predictions to generate in the " "nbest_predictions.json output file.") flags.DEFINE_integer( "max_answer_length", 30, "The maximum length of an answer that can be generated. This is needed " "because the start and end predictions are not conditioned on one another.") flags.DEFINE_bool("use_tpu", False, "Whether to use TPU or GPU/CPU.") tf.flags.DEFINE_string( "tpu_name", None, "The Cloud TPU to use for training. This should be either the name " "used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 " "url.") tf.flags.DEFINE_string( "tpu_zone", None, "[Optional] GCE zone where the Cloud TPU is located in. If not " "specified, we will attempt to automatically detect the GCE project from " "metadata.") tf.flags.DEFINE_string( "gcp_project", None, "[Optional] Project name for the Cloud TPU-enabled project. If not " "specified, we will attempt to automatically detect the GCE project from " "metadata.") tf.flags.DEFINE_string("master", None, "[Optional] TensorFlow master URL.") flags.DEFINE_integer( "num_tpu_cores", 8, "Only used if `use_tpu` is True. Total number of TPU cores to use.") flags.DEFINE_bool( "verbose_logging", False, "If true, all of the warnings related to data processing will be printed. " "A number of warnings are expected for a normal SQuAD evaluation.") flags.DEFINE_bool( "version_2_with_negative", False, "If true, the SQuAD examples contain some that do not have an answer.") flags.DEFINE_float( "null_score_diff_threshold", 0.0, "If null_score - best_non_null is greater than the threshold predict null.") class SquadExample(object): """A single training/test example for simple sequence classification. For examples without an answer, the start and end position are -1. """ def __init__(self, qas_id, question_text, doc_tokens, orig_answer_text=None, start_position=None, end_position=None, is_impossible=False): self.qas_id = qas_id self.question_text = question_text self.doc_tokens = doc_tokens self.orig_answer_text = orig_answer_text self.start_position = start_position self.end_position = end_position self.is_impossible = is_impossible def __str__(self): return self.__repr__() def __repr__(self): s = "" s += "qas_id: %s" % (tokenization.printable_text(self.qas_id)) s += ", question_text: %s" % ( tokenization.printable_text(self.question_text)) s += ", doc_tokens: [%s]" % (" ".join(self.doc_tokens)) if self.start_position: s += ", start_position: %d" % (self.start_position) if self.start_position: s += ", end_position: %d" % (self.end_position) if self.start_position: s += ", is_impossible: %r" % (self.is_impossible) return s class InputFeatures(object): """A single set of features of data.""" def __init__(self, unique_id, example_index, doc_span_index, tokens, token_to_orig_map, token_is_max_context, input_ids, input_mask, segment_ids, start_position=None, end_position=None, is_impossible=None): self.unique_id = unique_id self.example_index = example_index self.doc_span_index = doc_span_index self.tokens = tokens self.token_to_orig_map = token_to_orig_map self.token_is_max_context = token_is_max_context self.input_ids = input_ids self.input_mask = input_mask self.segment_ids = segment_ids self.start_position = start_position self.end_position = end_position self.is_impossible = is_impossible def read_squad_examples(input_file, is_training): """Read a SQuAD json file into a list of SquadExample.""" with tf.gfile.Open(input_file, "r") as reader: input_data = json.load(reader)["data"] def is_whitespace(c): if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F: return True return False examples = [] for entry in input_data: for paragraph in entry["paragraphs"]: paragraph_text = paragraph["context"] doc_tokens = [] char_to_word_offset = [] prev_is_whitespace = True for c in paragraph_text: if is_whitespace(c): prev_is_whitespace = True else: if prev_is_whitespace: doc_tokens.append(c) else: doc_tokens[-1] += c prev_is_whitespace = False char_to_word_offset.append(len(doc_tokens) - 1) for qa in paragraph["qas"]: qas_id = qa["id"] question_text = qa["question"] start_position = None end_position = None orig_answer_text = None is_impossible = False if is_training: if FLAGS.version_2_with_negative: is_impossible = qa["is_impossible"] if (len(qa["answers"]) != 1) and (not is_impossible): raise ValueError( "For training, each question should have exactly 1 answer.") if not is_impossible: answer = qa["answers"][0] orig_answer_text = answer["text"] answer_offset = answer["answer_start"] answer_length = len(orig_answer_text) start_position = char_to_word_offset[answer_offset] end_position = char_to_word_offset[answer_offset + answer_length - 1] # Only add answers where the text can be exactly recovered from the # document. If this CAN'T happen it's likely due to weird Unicode # stuff so we will just skip the example. # # Note that this means for training mode, every example is NOT # guaranteed to be preserved. actual_text = " ".join( doc_tokens[start_position:(end_position + 1)]) cleaned_answer_text = " ".join( tokenization.whitespace_tokenize(orig_answer_text)) if actual_text.find(cleaned_answer_text) == -1: tf.logging.warning("Could not find answer: '%s' vs. '%s'", actual_text, cleaned_answer_text) continue else: start_position = -1 end_position = -1 orig_answer_text = "" example = SquadExample( qas_id=qas_id, question_text=question_text, doc_tokens=doc_tokens, orig_answer_text=orig_answer_text, start_position=start_position, end_position=end_position, is_impossible=is_impossible) examples.append(example) return examples def convert_examples_to_features(examples, tokenizer, max_seq_length, doc_stride, max_query_length, is_training, output_fn): """Loads a data file into a list of `InputBatch`s.""" unique_id = 1000000000 for (example_index, example) in enumerate(examples): query_tokens = tokenizer.tokenize(example.question_text) if len(query_tokens) > max_query_length: query_tokens = query_tokens[0:max_query_length] tok_to_orig_index = [] orig_to_tok_index = [] all_doc_tokens = [] for (i, token) in enumerate(example.doc_tokens): orig_to_tok_index.append(len(all_doc_tokens)) sub_tokens = tokenizer.tokenize(token) for sub_token in sub_tokens: tok_to_orig_index.append(i) all_doc_tokens.append(sub_token) tok_start_position = None tok_end_position = None if is_training and example.is_impossible: tok_start_position = -1 tok_end_position = -1 if is_training and not example.is_impossible: tok_start_position = orig_to_tok_index[example.start_position] if example.end_position < len(example.doc_tokens) - 1: tok_end_position = orig_to_tok_index[example.end_position + 1] - 1 else: tok_end_position = len(all_doc_tokens) - 1 (tok_start_position, tok_end_position) = _improve_answer_span( all_doc_tokens, tok_start_position, tok_end_position, tokenizer, example.orig_answer_text) # The -3 accounts for [CLS], [SEP] and [SEP] max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 # We can have documents that are longer than the maximum sequence length. # To deal with this we do a sliding window approach, where we take chunks # of the up to our max length with a stride of `doc_stride`. _DocSpan = collections.namedtuple( # pylint: disable=invalid-name "DocSpan", ["start", "length"]) doc_spans = [] start_offset = 0 while start_offset < len(all_doc_tokens): length = len(all_doc_tokens) - start_offset if length > max_tokens_for_doc: length = max_tokens_for_doc doc_spans.append(_DocSpan(start=start_offset, length=length)) if start_offset + length == len(all_doc_tokens): break start_offset += min(length, doc_stride) for (doc_span_index, doc_span) in enumerate(doc_spans): tokens = [] token_to_orig_map = {} token_is_max_context = {} segment_ids = [] tokens.append("[CLS]") segment_ids.append(0) for token in query_tokens: tokens.append(token) segment_ids.append(0) tokens.append("[SEP]") segment_ids.append(0) for i in range(doc_span.length): split_token_index = doc_span.start + i token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index] is_max_context = _check_is_max_context(doc_spans, doc_span_index, split_token_index) token_is_max_context[len(tokens)] = is_max_context tokens.append(all_doc_tokens[split_token_index]) segment_ids.append(1) tokens.append("[SEP]") segment_ids.append(1) input_ids = tokenizer.convert_tokens_to_ids(tokens) # The mask has 1 for real tokens and 0 for padding tokens. Only real # tokens are attended to. input_mask = [1] * len(input_ids) # Zero-pad up to the sequence length. while len(input_ids) < max_seq_length: input_ids.append(0) input_mask.append(0) segment_ids.append(0) assert len(input_ids) == max_seq_length assert len(input_mask) == max_seq_length assert len(segment_ids) == max_seq_length start_position = None end_position = None if is_training and not example.is_impossible: # For training, if our document chunk does not contain an annotation # we throw it out, since there is nothing to predict. doc_start = doc_span.start doc_end = doc_span.start + doc_span.length - 1 out_of_span = False if not (tok_start_position >= doc_start and tok_end_position <= doc_end): out_of_span = True if out_of_span: start_position = 0 end_position = 0 else: doc_offset = len(query_tokens) + 2 start_position = tok_start_position - doc_start + doc_offset end_position = tok_end_position - doc_start + doc_offset if is_training and example.is_impossible: start_position = 0 end_position = 0 if example_index < 20: tf.logging.info("*** Example ***") tf.logging.info("unique_id: %s" % (unique_id)) tf.logging.info("example_index: %s" % (example_index)) tf.logging.info("doc_span_index: %s" % (doc_span_index)) tf.logging.info("tokens: %s" % " ".join( [tokenization.printable_text(x) for x in tokens])) tf.logging.info("token_to_orig_map: %s" % " ".join( ["%d:%d" % (x, y) for (x, y) in six.iteritems(token_to_orig_map)])) tf.logging.info("token_is_max_context: %s" % " ".join([ "%d:%s" % (x, y) for (x, y) in six.iteritems(token_is_max_context) ])) tf.logging.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) tf.logging.info( "input_mask: %s" % " ".join([str(x) for x in input_mask])) tf.logging.info( "segment_ids: %s" % " ".join([str(x) for x in segment_ids])) if is_training and example.is_impossible: tf.logging.info("impossible example") if is_training and not example.is_impossible: answer_text = " ".join(tokens[start_position:(end_position + 1)]) tf.logging.info("start_position: %d" % (start_position)) tf.logging.info("end_position: %d" % (end_position)) tf.logging.info( "answer: %s" % (tokenization.printable_text(answer_text))) feature = InputFeatures( unique_id=unique_id, example_index=example_index, doc_span_index=doc_span_index, tokens=tokens, token_to_orig_map=token_to_orig_map, token_is_max_context=token_is_max_context, input_ids=input_ids, input_mask=input_mask, segment_ids=segment_ids, start_position=start_position, end_position=end_position, is_impossible=example.is_impossible) # Run callback output_fn(feature) unique_id += 1 def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer, orig_answer_text): """Returns tokenized answer spans that better match the annotated answer.""" # The SQuAD annotations are character based. We first project them to # whitespace-tokenized words. But then after WordPiece tokenization, we can # often find a "better match". For example: # # Question: What year was John Smith born? # Context: The leader was John Smith (1895-1943). # Answer: 1895 # # The original whitespace-tokenized answer will be "(1895-1943).". However # after tokenization, our tokens will be "( 1895 - 1943 ) .". So we can match # the exact answer, 1895. # # However, this is not always possible. Consider the following: # # Question: What country is the top exporter of electornics? # Context: The Japanese electronics industry is the lagest in the world. # Answer: Japan # # In this case, the annotator chose "Japan" as a character sub-span of # the word "Japanese". Since our WordPiece tokenizer does not split # "Japanese", we just use "Japanese" as the annotation. This is fairly rare # in SQuAD, but does happen. tok_answer_text = " ".join(tokenizer.tokenize(orig_answer_text)) for new_start in range(input_start, input_end + 1): for new_end in range(input_end, new_start - 1, -1): text_span = " ".join(doc_tokens[new_start:(new_end + 1)]) if text_span == tok_answer_text: return (new_start, new_end) return (input_start, input_end) def _check_is_max_context(doc_spans, cur_span_index, position): """Check if this is the 'max context' doc span for the token.""" # Because of the sliding window approach taken to scoring documents, a single # token can appear in multiple documents. E.g. # Doc: the man went to the store and bought a gallon of milk # Span A: the man went to the # Span B: to the store and bought # Span C: and bought a gallon of # ... # # Now the word 'bought' will have two scores from spans B and C. We only # want to consider the score with "maximum context", which we define as # the *minimum* of its left and right context (the *sum* of left and # right context will always be the same, of course). # # In the example the maximum context for 'bought' would be span C since # it has 1 left context and 3 right context, while span B has 4 left context # and 0 right context. best_score = None best_span_index = None for (span_index, doc_span) in enumerate(doc_spans): end = doc_span.start + doc_span.length - 1 if position < doc_span.start: continue if position > end: continue num_left_context = position - doc_span.start num_right_context = end - position score = min(num_left_context, num_right_context) + 0.01 * doc_span.length if best_score is None or score > best_score: best_score = score best_span_index = span_index return cur_span_index == best_span_index def create_model(bert_config, is_training, input_ids, input_mask, segment_ids, use_one_hot_embeddings): """Creates a classification model.""" model = modeling.BertModel( config=bert_config, is_training=is_training, input_ids=input_ids, input_mask=input_mask, token_type_ids=segment_ids, use_one_hot_embeddings=use_one_hot_embeddings) final_hidden = model.get_sequence_output() final_hidden_shape = modeling.get_shape_list(final_hidden, expected_rank=3) batch_size = final_hidden_shape[0] seq_length = final_hidden_shape[1] hidden_size = final_hidden_shape[2] output_weights = tf.get_variable( "cls/squad/output_weights", [2, hidden_size], initializer=tf.truncated_normal_initializer(stddev=0.02)) output_bias = tf.get_variable( "cls/squad/output_bias", [2], initializer=tf.zeros_initializer()) final_hidden_matrix = tf.reshape(final_hidden, [batch_size * seq_length, hidden_size]) logits = tf.matmul(final_hidden_matrix, output_weights, transpose_b=True) logits = tf.nn.bias_add(logits, output_bias) logits = tf.reshape(logits, [batch_size, seq_length, 2]) logits = tf.transpose(logits, [2, 0, 1]) unstacked_logits = tf.unstack(logits, axis=0) (start_logits, end_logits) = (unstacked_logits[0], unstacked_logits[1]) return (start_logits, end_logits) def model_fn_builder(bert_config, init_checkpoint, learning_rate, num_train_steps, num_warmup_steps, use_tpu, use_one_hot_embeddings): """Returns `model_fn` closure for TPUEstimator.""" def model_fn(features, labels, mode, params): # pylint: disable=unused-argument """The `model_fn` for TPUEstimator.""" tf.logging.info("*** Features ***") for name in sorted(features.keys()): tf.logging.info(" name = %s, shape = %s" % (name, features[name].shape)) unique_ids = features["unique_ids"] input_ids = features["input_ids"] input_mask = features["input_mask"] segment_ids = features["segment_ids"] is_training = (mode == tf.estimator.ModeKeys.TRAIN) (start_logits, end_logits) = create_model( bert_config=bert_config, is_training=is_training, input_ids=input_ids, input_mask=input_mask, segment_ids=segment_ids, use_one_hot_embeddings=use_one_hot_embeddings) tvars = tf.trainable_variables() initialized_variable_names = {} scaffold_fn = None if init_checkpoint: (assignment_map, initialized_variable_names ) = modeling.get_assignment_map_from_checkpoint(tvars, init_checkpoint) if use_tpu: def tpu_scaffold(): tf.train.init_from_checkpoint(init_checkpoint, assignment_map) return tf.train.Scaffold() scaffold_fn = tpu_scaffold else: tf.train.init_from_checkpoint(init_checkpoint, assignment_map) tf.logging.info("**** Trainable Variables ****") for var in tvars: init_string = "" if var.name in initialized_variable_names: init_string = ", *INIT_FROM_CKPT*" tf.logging.info(" name = %s, shape = %s%s", var.name, var.shape, init_string) output_spec = None if mode == tf.estimator.ModeKeys.TRAIN: seq_length = modeling.get_shape_list(input_ids)[1] def compute_loss(logits, positions): one_hot_positions = tf.one_hot( positions, depth=seq_length, dtype=tf.float32) log_probs = tf.nn.log_softmax(logits, axis=-1) loss = -tf.reduce_mean( tf.reduce_sum(one_hot_positions * log_probs, axis=-1)) return loss start_positions = features["start_positions"] end_positions = features["end_positions"] start_loss = compute_loss(start_logits, start_positions) end_loss = compute_loss(end_logits, end_positions) total_loss = (start_loss + end_loss) / 2.0 train_op = optimization.create_optimizer( total_loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu) output_spec = tf.contrib.tpu.TPUEstimatorSpec( mode=mode, loss=total_loss, train_op=train_op, scaffold_fn=scaffold_fn) elif mode == tf.estimator.ModeKeys.PREDICT: predictions = { "unique_ids": unique_ids, "start_logits": start_logits, "end_logits": end_logits, } output_spec = tf.contrib.tpu.TPUEstimatorSpec( mode=mode, predictions=predictions, scaffold_fn=scaffold_fn) else: raise ValueError( "Only TRAIN and PREDICT modes are supported: %s" % (mode)) return output_spec return model_fn def input_fn_builder(input_file, seq_length, is_training, drop_remainder): """Creates an `input_fn` closure to be passed to TPUEstimator.""" name_to_features = { "unique_ids": tf.FixedLenFeature([], tf.int64), "input_ids": tf.FixedLenFeature([seq_length], tf.int64), "input_mask": tf.FixedLenFeature([seq_length], tf.int64), "segment_ids": tf.FixedLenFeature([seq_length], tf.int64), } if is_training: name_to_features["start_positions"] = tf.FixedLenFeature([], tf.int64) name_to_features["end_positions"] = tf.FixedLenFeature([], tf.int64) def _decode_record(record, name_to_features): """Decodes a record to a TensorFlow example.""" example = tf.parse_single_example(record, name_to_features) # tf.Example only supports tf.int64, but the TPU only supports tf.int32. # So cast all int64 to int32. for name in list(example.keys()): t = example[name] if t.dtype == tf.int64: t = tf.to_int32(t) example[name] = t return example def input_fn(params): """The actual input function.""" batch_size = params["batch_size"] # For training, we want a lot of parallel reading and shuffling. # For eval, we want no shuffling and parallel reading doesn't matter. d = tf.data.TFRecordDataset(input_file) if is_training: d = d.repeat() d = d.shuffle(buffer_size=100) d = d.apply( tf.contrib.data.map_and_batch( lambda record: _decode_record(record, name_to_features), batch_size=batch_size, drop_remainder=drop_remainder)) return d return input_fn RawResult = collections.namedtuple("RawResult", ["unique_id", "start_logits", "end_logits"]) def write_predictions(all_examples, all_features, all_results, n_best_size, max_answer_length, do_lower_case, output_prediction_file, output_nbest_file, output_null_log_odds_file): """Write final predictions to the json file and log-odds of null if needed.""" tf.logging.info("Writing predictions to: %s" % (output_prediction_file)) tf.logging.info("Writing nbest to: %s" % (output_nbest_file)) example_index_to_features = collections.defaultdict(list) for feature in all_features: example_index_to_features[feature.example_index].append(feature) unique_id_to_result = {} for result in all_results: unique_id_to_result[result.unique_id] = result _PrelimPrediction = collections.namedtuple( # pylint: disable=invalid-name "PrelimPrediction", ["feature_index", "start_index", "end_index", "start_logit", "end_logit"]) all_predictions = collections.OrderedDict() all_nbest_json = collections.OrderedDict() scores_diff_json = collections.OrderedDict() for (example_index, example) in enumerate(all_examples): features = example_index_to_features[example_index] prelim_predictions = [] # keep track of the minimum score of null start+end of position 0 score_null = 1000000 # large and positive min_null_feature_index = 0 # the paragraph slice with min mull score null_start_logit = 0 # the start logit at the slice with min null score null_end_logit = 0 # the end logit at the slice with min null score for (feature_index, feature) in enumerate(features): result = unique_id_to_result[feature.unique_id] start_indexes = _get_best_indexes(result.start_logits, n_best_size) end_indexes = _get_best_indexes(result.end_logits, n_best_size) # if we could have irrelevant answers, get the min score of irrelevant if FLAGS.version_2_with_negative: feature_null_score = result.start_logits[0] + result.end_logits[0] if feature_null_score < score_null: score_null = feature_null_score min_null_feature_index = feature_index null_start_logit = result.start_logits[0] null_end_logit = result.end_logits[0] for start_index in start_indexes: for end_index in end_indexes: # We could hypothetically create invalid predictions, e.g., predict # that the start of the span is in the question. We throw out all # invalid predictions. if start_index >= len(feature.tokens): continue if end_index >= len(feature.tokens): continue if start_index not in feature.token_to_orig_map: continue if end_index not in feature.token_to_orig_map: continue if not feature.token_is_max_context.get(start_index, False): continue if end_index < start_index: continue length = end_index - start_index + 1 if length > max_answer_length: continue prelim_predictions.append( _PrelimPrediction( feature_index=feature_index, start_index=start_index, end_index=end_index, start_logit=result.start_logits[start_index], end_logit=result.end_logits[end_index])) if FLAGS.version_2_with_negative: prelim_predictions.append( _PrelimPrediction( feature_index=min_null_feature_index, start_index=0, end_index=0, start_logit=null_start_logit, end_logit=null_end_logit)) prelim_predictions = sorted( prelim_predictions, key=lambda x: (x.start_logit + x.end_logit), reverse=True) _NbestPrediction = collections.namedtuple( # pylint: disable=invalid-name "NbestPrediction", ["text", "start_logit", "end_logit"]) seen_predictions = {} nbest = [] for pred in prelim_predictions: if len(nbest) >= n_best_size: break feature = features[pred.feature_index] if pred.start_index > 0: # this is a non-null prediction tok_tokens = feature.tokens[pred.start_index:(pred.end_index + 1)] orig_doc_start = feature.token_to_orig_map[pred.start_index] orig_doc_end = feature.token_to_orig_map[pred.end_index] orig_tokens = example.doc_tokens[orig_doc_start:(orig_doc_end + 1)] tok_text = " ".join(tok_tokens) # De-tokenize WordPieces that have been split off. tok_text = tok_text.replace(" ##", "") tok_text = tok_text.replace("##", "") # Clean whitespace tok_text = tok_text.strip() tok_text = " ".join(tok_text.split()) orig_text = " ".join(orig_tokens) final_text = get_final_text(tok_text, orig_text, do_lower_case) if final_text in seen_predictions: continue seen_predictions[final_text] = True else: final_text = "" seen_predictions[final_text] = True nbest.append( _NbestPrediction( text=final_text, start_logit=pred.start_logit, end_logit=pred.end_logit)) # if we didn't inlude the empty option in the n-best, inlcude it if FLAGS.version_2_with_negative: if "" not in seen_predictions: nbest.append( _NbestPrediction( text="", start_logit=null_start_logit, end_logit=null_end_logit)) # In very rare edge cases we could have no valid predictions. So we # just create a nonce prediction in this case to avoid failure. if not nbest: nbest.append( _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0)) assert len(nbest) >= 1 total_scores = [] best_non_null_entry = None for entry in nbest: total_scores.append(entry.start_logit + entry.end_logit) if not best_non_null_entry: if entry.text: best_non_null_entry = entry probs = _compute_softmax(total_scores) nbest_json = [] for (i, entry) in enumerate(nbest): output = collections.OrderedDict() output["text"] = entry.text output["probability"] = probs[i] output["start_logit"] = entry.start_logit output["end_logit"] = entry.end_logit nbest_json.append(output) assert len(nbest_json) >= 1 if not FLAGS.version_2_with_negative: all_predictions[example.qas_id] = nbest_json[0]["text"] else: # predict "" iff the null score - the score of best non-null > threshold score_diff = score_null - best_non_null_entry.start_logit - ( best_non_null_entry.end_logit) scores_diff_json[example.qas_id] = score_diff if score_diff > FLAGS.null_score_diff_threshold: all_predictions[example.qas_id] = "" else: all_predictions[example.qas_id] = best_non_null_entry.text all_nbest_json[example.qas_id] = nbest_json with tf.gfile.GFile(output_prediction_file, "w") as writer: writer.write(json.dumps(all_predictions, indent=4) + "\n") with tf.gfile.GFile(output_nbest_file, "w") as writer: writer.write(json.dumps(all_nbest_json, indent=4) + "\n") if FLAGS.version_2_with_negative: with tf.gfile.GFile(output_null_log_odds_file, "w") as writer: writer.write(json.dumps(scores_diff_json, indent=4) + "\n") def get_final_text(pred_text, orig_text, do_lower_case): """Project the tokenized prediction back to the original text.""" # When we created the data, we kept track of the alignment between original # (whitespace tokenized) tokens and our WordPiece tokenized tokens. So # now `orig_text` contains the span of our original text corresponding to the # span that we predicted. # # However, `orig_text` may contain extra characters that we don't want in # our prediction. # # For example, let's say: # pred_text = steve smith # orig_text = Steve Smith's # # We don't want to return `orig_text` because it contains the extra "'s". # # We don't want to return `pred_text` because it's already been normalized # (the SQuAD eval script also does punctuation stripping/lower casing but # our tokenizer does additional normalization like stripping accent # characters). # # What we really want to return is "Steve Smith". # # Therefore, we have to apply a semi-complicated alignment heruistic between # `pred_text` and `orig_text` to get a character-to-charcter alignment. This # can fail in certain cases in which case we just return `orig_text`. def _strip_spaces(text): ns_chars = [] ns_to_s_map = collections.OrderedDict() for (i, c) in enumerate(text): if c == " ": continue ns_to_s_map[len(ns_chars)] = i ns_chars.append(c) ns_text = "".join(ns_chars) return (ns_text, ns_to_s_map) # We first tokenize `orig_text`, strip whitespace from the result # and `pred_text`, and check if they are the same length. If they are # NOT the same length, the heuristic has failed. If they are the same # length, we assume the characters are one-to-one aligned. tokenizer = tokenization.BasicTokenizer(do_lower_case=do_lower_case) tok_text = " ".join(tokenizer.tokenize(orig_text)) start_position = tok_text.find(pred_text) if start_position == -1: if FLAGS.verbose_logging: tf.logging.info( "Unable to find text: '%s' in '%s'" % (pred_text, orig_text)) return orig_text end_position = start_position + len(pred_text) - 1 (orig_ns_text, orig_ns_to_s_map) = _strip_spaces(orig_text) (tok_ns_text, tok_ns_to_s_map) = _strip_spaces(tok_text) if len(orig_ns_text) != len(tok_ns_text): if FLAGS.verbose_logging: tf.logging.info("Length not equal after stripping spaces: '%s' vs '%s'", orig_ns_text, tok_ns_text) return orig_text # We then project the characters in `pred_text` back to `orig_text` using # the character-to-character alignment. tok_s_to_ns_map = {} for (i, tok_index) in six.iteritems(tok_ns_to_s_map): tok_s_to_ns_map[tok_index] = i orig_start_position = None if start_position in tok_s_to_ns_map: ns_start_position = tok_s_to_ns_map[start_position] if ns_start_position in orig_ns_to_s_map: orig_start_position = orig_ns_to_s_map[ns_start_position] if orig_start_position is None: if FLAGS.verbose_logging: tf.logging.info("Couldn't map start position") return orig_text orig_end_position = None if end_position in tok_s_to_ns_map: ns_end_position = tok_s_to_ns_map[end_position] if ns_end_position in orig_ns_to_s_map: orig_end_position = orig_ns_to_s_map[ns_end_position] if orig_end_position is None: if FLAGS.verbose_logging: tf.logging.info("Couldn't map end position") return orig_text output_text = orig_text[orig_start_position:(orig_end_position + 1)] return output_text def _get_best_indexes(logits, n_best_size): """Get the n-best logits from a list.""" index_and_score = sorted(enumerate(logits), key=lambda x: x[1], reverse=True) best_indexes = [] for i in range(len(index_and_score)): if i >= n_best_size: break best_indexes.append(index_and_score[i][0]) return best_indexes def _compute_softmax(scores): """Compute softmax probability over raw logits.""" if not scores: return [] max_score = None for score in scores: if max_score is None or score > max_score: max_score = score exp_scores = [] total_sum = 0.0 for score in scores: x = math.exp(score - max_score) exp_scores.append(x) total_sum += x probs = [] for score in exp_scores: probs.append(score / total_sum) return probs class FeatureWriter(object): """Writes InputFeature to TF example file.""" def __init__(self, filename, is_training): self.filename = filename self.is_training = is_training self.num_features = 0 self._writer = tf.python_io.TFRecordWriter(filename) def process_feature(self, feature): """Write a InputFeature to the TFRecordWriter as a tf.train.Example.""" self.num_features += 1 def create_int_feature(values): feature = tf.train.Feature( int64_list=tf.train.Int64List(value=list(values))) return feature features = collections.OrderedDict() features["unique_ids"] = create_int_feature([feature.unique_id]) features["input_ids"] = create_int_feature(feature.input_ids) features["input_mask"] = create_int_feature(feature.input_mask) features["segment_ids"] = create_int_feature(feature.segment_ids) if self.is_training: features["start_positions"] = create_int_feature([feature.start_position]) features["end_positions"] = create_int_feature([feature.end_position]) impossible = 0 if feature.is_impossible: impossible = 1 features["is_impossible"] = create_int_feature([impossible]) tf_example = tf.train.Example(features=tf.train.Features(feature=features)) self._writer.write(tf_example.SerializeToString()) def close(self): self._writer.close() def validate_flags_or_throw(bert_config): """Validate the input FLAGS or throw an exception.""" tokenization.validate_case_matches_checkpoint(FLAGS.do_lower_case, FLAGS.init_checkpoint) if not FLAGS.do_train and not FLAGS.do_predict: raise ValueError("At least one of `do_train` or `do_predict` must be True.") if FLAGS.do_train: if not FLAGS.train_file: raise ValueError( "If `do_train` is True, then `train_file` must be specified.") if FLAGS.do_predict: if not FLAGS.predict_file: raise ValueError( "If `do_predict` is True, then `predict_file` must be specified.") if FLAGS.max_seq_length > bert_config.max_position_embeddings: raise ValueError( "Cannot use sequence length %d because the BERT model " "was only trained up to sequence length %d" % (FLAGS.max_seq_length, bert_config.max_position_embeddings)) if FLAGS.max_seq_length <= FLAGS.max_query_length + 3: raise ValueError( "The max_seq_length (%d) must be greater than max_query_length " "(%d) + 3" % (FLAGS.max_seq_length, FLAGS.max_query_length)) def main(_): tf.logging.set_verbosity(tf.logging.INFO) bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file) validate_flags_or_throw(bert_config) tf.gfile.MakeDirs(FLAGS.output_dir) tokenizer = tokenization.FullTokenizer( vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) tpu_cluster_resolver = None if FLAGS.use_tpu and FLAGS.tpu_name: tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 run_config = tf.contrib.tpu.RunConfig( cluster=tpu_cluster_resolver, master=FLAGS.master, model_dir=FLAGS.output_dir, save_checkpoints_steps=FLAGS.save_checkpoints_steps, tpu_config=tf.contrib.tpu.TPUConfig( iterations_per_loop=FLAGS.iterations_per_loop, num_shards=FLAGS.num_tpu_cores, per_host_input_for_training=is_per_host)) train_examples = None num_train_steps = None num_warmup_steps = None if FLAGS.do_train: train_examples = read_squad_examples( input_file=FLAGS.train_file, is_training=True) num_train_steps = int( len(train_examples) / FLAGS.train_batch_size * FLAGS.num_train_epochs) num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion) # Pre-shuffle the input to avoid having to make a very large shuffle # buffer in in the `input_fn`. rng = random.Random(12345) rng.shuffle(train_examples) model_fn = model_fn_builder( bert_config=bert_config, init_checkpoint=FLAGS.init_checkpoint, learning_rate=FLAGS.learning_rate, num_train_steps=num_train_steps, num_warmup_steps=num_warmup_steps, use_tpu=FLAGS.use_tpu, use_one_hot_embeddings=FLAGS.use_tpu) # If TPU is not available, this will fall back to normal Estimator on CPU # or GPU. estimator = tf.contrib.tpu.TPUEstimator( use_tpu=FLAGS.use_tpu, model_fn=model_fn, config=run_config, train_batch_size=FLAGS.train_batch_size, predict_batch_size=FLAGS.predict_batch_size) if FLAGS.do_train: # We write to a temporary file to avoid storing very large constant tensors # in memory. train_writer = FeatureWriter( filename=os.path.join(FLAGS.output_dir, "train.tf_record"), is_training=True) convert_examples_to_features( examples=train_examples, tokenizer=tokenizer, max_seq_length=FLAGS.max_seq_length, doc_stride=FLAGS.doc_stride, max_query_length=FLAGS.max_query_length, is_training=True, output_fn=train_writer.process_feature) train_writer.close() tf.logging.info("***** Running training *****") tf.logging.info(" Num orig examples = %d", len(train_examples)) tf.logging.info(" Num split examples = %d", train_writer.num_features) tf.logging.info(" Batch size = %d", FLAGS.train_batch_size) tf.logging.info(" Num steps = %d", num_train_steps) del train_examples train_input_fn = input_fn_builder( input_file=train_writer.filename, seq_length=FLAGS.max_seq_length, is_training=True, drop_remainder=True) estimator.train(input_fn=train_input_fn, max_steps=num_train_steps) if FLAGS.do_predict: eval_examples = read_squad_examples( input_file=FLAGS.predict_file, is_training=False) eval_writer = FeatureWriter( filename=os.path.join(FLAGS.output_dir, "eval.tf_record"), is_training=False) eval_features = [] def append_feature(feature): eval_features.append(feature) eval_writer.process_feature(feature) convert_examples_to_features( examples=eval_examples, tokenizer=tokenizer, max_seq_length=FLAGS.max_seq_length, doc_stride=FLAGS.doc_stride, max_query_length=FLAGS.max_query_length, is_training=False, output_fn=append_feature) eval_writer.close() tf.logging.info("***** Running predictions *****") tf.logging.info(" Num orig examples = %d", len(eval_examples)) tf.logging.info(" Num split examples = %d", len(eval_features)) tf.logging.info(" Batch size = %d", FLAGS.predict_batch_size) all_results = [] predict_input_fn = input_fn_builder( input_file=eval_writer.filename, seq_length=FLAGS.max_seq_length, is_training=False, drop_remainder=False) # If running eval on the TPU, you will need to specify the number of # steps. all_results = [] for result in estimator.predict( predict_input_fn, yield_single_examples=True): if len(all_results) % 1000 == 0: tf.logging.info("Processing example: %d" % (len(all_results))) unique_id = int(result["unique_ids"]) start_logits = [float(x) for x in result["start_logits"].flat] end_logits = [float(x) for x in result["end_logits"].flat] all_results.append( RawResult( unique_id=unique_id, start_logits=start_logits, end_logits=end_logits)) output_prediction_file = os.path.join(FLAGS.output_dir, "predictions.json") output_nbest_file = os.path.join(FLAGS.output_dir, "nbest_predictions.json") output_null_log_odds_file = os.path.join(FLAGS.output_dir, "null_odds.json") write_predictions(eval_examples, eval_features, all_results, FLAGS.n_best_size, FLAGS.max_answer_length, FLAGS.do_lower_case, output_prediction_file, output_nbest_file, output_null_log_odds_file) if __name__ == "__main__": flags.mark_flag_as_required("vocab_file") flags.mark_flag_as_required("bert_config_file") flags.mark_flag_as_required("output_dir") tf.app.run() ================================================ FILE: run/transform_spanbert_pytorch_to_tf.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # author: xiaoy li # description: # transform pytorch .bin models to tensorflow ckpt import os import sys import shutil import torch import argparse import random import numpy as np import tensorflow as tf REPO_PATH = "/".join(os.path.realpath(__file__).split("/")[:-2]) if REPO_PATH not in sys.path: sys.path.insert(0, REPO_PATH) from bert import modeling from utils.load_pytorch_to_tf import load_from_pytorch_checkpoint def load_models(bert_config_path, ): bert_config = modeling.BertConfig.from_json_file(bert_config_path) input_ids = tf.ones((8, 128), tf.int32) model = modeling.BertModel( config=bert_config, is_training=False, input_ids=input_ids, use_one_hot_embeddings=False, scope="bert") return model, bert_config def copy_checkpoint(source, target): for ext in (".index", ".data-00000-of-00001"): shutil.copyfile(source + ext, target + ext) def main(bert_config_path, bert_ckpt_path, pytorch_init_checkpoint, output_tf_dir): with tf.Session() as session: model, bert_config = load_models(bert_config_path) tvars = tf.trainable_variables() assignment_map, initialized_variable_names = modeling.get_assignment_map_from_checkpoint(tvars, bert_ckpt_path) session.run(tf.global_variables_initializer()) init_from_checkpoint = load_from_pytorch_checkpoint init_from_checkpoint(pytorch_init_checkpoint, assignment_map) for var in tvars: init_string = "" if var.name in initialized_variable_names: init_string = ", *INIT_FROM_CKPT*" print("name = %s, shape = %s%s" % (var.name, var.shape, init_string)) saver = tf.train.Saver() saver.save(session, os.path.join(output_tf_dir, "model"), global_step=100) copy_checkpoint(os.path.join(output_tf_dir, "model-{}".format(str(100))), os.path.join(output_tf_dir, "bert_model.ckpt")) print("=*="*30) print("save models : {}".format(output_tf_dir)) print("=*="*30) def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("--spanbert_config_path", default="/home/lixiaoya/spanbert_base_cased/config.json", type=str) parser.add_argument("--bert_tf_ckpt_path", default="/home/lixiaoya/cased_L-12_H-768_A-12/bert_model.ckpt", type=str) parser.add_argument("--spanbert_pytorch_bin_path", default="/home/lixiaoya/spanbert_base_cased/pytorch_model.bin", type=str) parser.add_argument("--output_spanbert_tf_dir", default="/home/lixiaoya/tf_spanbert_base_case", type=str) parser.add_argument("--seed", default=2333, type=int) args = parser.parse_args() random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) tf.set_random_seed(args.seed) torch.cuda.manual_seed_all(args.seed) os.makedirs(args.output_spanbert_tf_dir, exist_ok=True) try: shutil(args.spanbert_config_path, args.output_spanbert_tf_dir) except: print("#=#"*30) print("copy spanbert_config from {} to {}".format(args.spanbert_config_path, args.output_spanbert_tf_dir)) return args if __name__ == "__main__": args_config = parse_args() main(args_config.spanbert_config_path, args_config.bert_tf_ckpt_path, args_config.spanbert_pytorch_bin_path, args_config.output_spanbert_tf_dir) # # Please refer to scripts/data/transform_ckpt_pytorch_to_tf.sh # # for spanbert large # # python3 transform_spanbert_pytorch_to_tf.py \ # --spanbert_config_path /xiaoya/pretrain_ckpt/spanbert_large_cased/config.json \ # --bert_tf_ckpt_path /xiaoya/pretrain_ckpt/cased_L-24_H-1024_A-16/bert_model.ckpt \ # --spanbert_pytorch_bin_path /xiaoya/pretrain_ckpt/spanbert_large_cased/pytorch_model.bin \ # --output_spanbert_tf_dir /xiaoya/pretrain_ckpt/tf_spanbert_large_cased # for spanbert base # # python3 transform_spanbert_pytorch_to_tf.py \ # --spanbert_config_path /xiaoya/pretrain_ckpt/spanbert_base_cased/config.json \ # --bert_tf_ckpt_path /xiaoya/pretrain_ckpt/cased_L-12_H-768_A-12/bert_model.ckpt \ # --spanbert_pytorch_bin_path /xiaoya/pretrain_ckpt/spanbert_base_cased/pytorch_model.bin \ # --output_spanbert_tf_dir /xiaoya/pretrain_ckpt/tf_spanbert_base_cased ================================================ FILE: scripts/data/download_pretrained_mlm.sh ================================================ #!/usr/bin/env bash # -*- coding: utf-8 -*- # Author: xiaoy li # description: # download pretrained model ckpt BERT_PRETRAIN_CKPT=$1 MODEL_NAME=$2 if [[ $MODEL_NAME == "bert_base" ]]; then mkdir -p $BERT_PRETRAIN_CKPT echo "DownLoad BERT Cased Base" wget https://storage.googleapis.com/bert_models/2018_10_18/cased_L-12_H-768_A-12.zip -P $BERT_PRETRAIN_CKPT unzip $BERT_PRETRAIN_CKPT/cased_L-12_H-768_A-12.zip -d $BERT_PRETRAIN_CKPT rm $BERT_PRETRAIN_CKPT/cased_L-12_H-768_A-12.zip elif [[ $MODEL_NAME == "bert_large" ]]; then echo "DownLoad BERT Cased Large" wget https://storage.googleapis.com/bert_models/2018_10_18/cased_L-24_H-1024_A-16.zip -P $BERT_PRETRAIN_CKPT unzip $BERT_PRETRAIN_CKPT/cased_L-24_H-1024_A-16.zip -d $BERT_PRETRAIN_CKPT rm $BERT_PRETRAIN_CKPT/cased_L-24_H-1024_A-16.zip elif [[ $MODEL_NAME == "spanbert_base" ]]; then echo "DownLoad Span-BERT Cased Base" wget https://dl.fbaipublicfiles.com/fairseq/models/spanbert_hf_base.tar.gz -P $BERT_PRETRAIN_CKPT tar -zxvf $BERT_PRETRAIN_CKPT/spanbert_hf_base.tar.gz -C $BERT_PRETRAIN_CKPT rm $BERT_PRETRAIN_CKPT/spanbert_hf_base.tar.gz elif [[ $MODEL_NAME == "spanbert_large" ]]; then echo "DownLoad Span-BERT Cased Large" wget https://dl.fbaipublicfiles.com/fairseq/models/spanbert_hf.tar.gz -P $BERT_PRETRAIN_CKPT tar -zxvf $BERT_PRETRAIN_CKPT/spanbert_hf.tar.gz -C $BERT_PRETRAIN_CKPT rm $BERT_PRETRAIN_CKPT/spanbert_hf.tar.gz elif [[ $MODEL_NAME == "bert_tiny" ]]; then each "DownLoad BERT Uncased Tiny; Helps to debug on GPU." wget https://storage.googleapis.com/bert_models/2020_02_20/uncased_L-2_H-128_A-2.zip -P $BERT_PRETRAIN_CKPT tar -zxvf $BERT_PRETRAIN_CKPT/uncased_L-2_H-128_A-2.zip -C $BERT_PRETRAIN_CKPT rm $BERT_PRETRAIN_CKPT/uncased_L-2_H-128_A-2.zip else echo 'Unknown argment 2 (Model Sign)' fi ================================================ FILE: scripts/data/generate_tfrecord_dataset.sh ================================================ #!/usr/bin/env bash # -*- coding: utf-8 -*- # author: xiaoy li # description: # generate train/dev/test tfrecord files for training the model. # example: # bash generate_tfrecord_dataset.sh /path-to-conll-coreference-resolution-dataset /path-to-save-tfrecord-for-training /cased_L-12_H-768_A-12/vocab.txt REPO_PATH=/home/lixiaoya/coref-tf export PYTHONPATH=$REPO_PATH source_dir=$1 target_dir=$2 vocab_file=$3 mkdir -p ${target_dir} python3 ${REPO_PATH}/run/build_dataset_to_tfrecord.py \ --source_files_dir $source_dir \ --target_output_dir $target_dir \ --num_window 2 \ --window_size 64 \ --max_num_mention 50 \ --max_num_cluster 40 \ --vocab_file $vocab_file \ --language english \ --max_doc_length 600 ================================================ FILE: scripts/data/preprocess_ontonotes_annfiles.sh ================================================ #!/usr/bin/env bash # author: xiaoy li # description: # generate annotated CONLL-2012 coreference resolution datasets from the official released OntoNotes 5.0 dataset. # ###################################### # NOTICE: ###################################### # the scripts only work with python 2. # if you want to run with python 3, please refer to https://github.com/huggingface/neuralcoref/blob/master/neuralcoref/train/training.md#get-the-data # Thanks to their amazing job ! # # Reference: # https://github.com/huggingface/neuralcoref/blob/master/neuralcoref/train/training.md#get-the-data # https://github.com/mandarjoshi90/coref # path_to_ontonotes5.0_directory=$1 path_to_save_processed_data_directory=$2 language=$3 dlx() { wget -P $path_to_save_processed_data_directory $1/$2 tar -xvzf $path_to_save_processed_data_directory/$2 -C $path_to_save_processed_data_directory rm $path_to_save_processed_data_directory/$2 } conll_url=http://conll.cemantix.org/2012/download dlx $conll_url conll-2012-train.v4.tar.gz dlx $conll_url conll-2012-development.v4.tar.gz dlx $conll_url/test conll-2012-test-key.tar.gz dlx $conll_url/test conll-2012-test-official.v9.tar.gz dlx $conll_url conll-2012-scripts.v3.tar.gz dlx http://conll.cemantix.org/download reference-coreference-scorers.v8.01.tar.gz bash $path_to_save_processed_data_directory/conll-2012/v3/scripts/skeleton2conll.sh -D $path_to_ontonotes5.0_directory/data/files/data $path_to_save_processed_data_directory/conll-2012 function compile_partition() { rm -f $2.$5.$3$4 cat $path_to_save_processed_data_directory/conll-2012/$3/data/$1/data/$5/annotations/*/*/*/*.$3$4 >> $path_to_save_processed_data_directory/$2.$5.$3$4 } function compile_language() { compile_partition development dev v4 _gold_conll $1 compile_partition train train v4 _gold_conll $1 compile_partition test test v4 _gold_conll $1 } compile_language $language ================================================ FILE: scripts/data/transform_ckpt_pytorch_to_tf.sh ================================================ #!/usr/bin/env bash # -*- coding: utf-8 -*- # author: xiaoy li # description: # transform trained spanbert language model from pytorch(.bin) to tensorflow(.ckpt). # PLEASE NOTICE: the same scale(Base/Large) BERT(TF) Models are also necessary. REPO_PATH=/home/lixiaoya/coref-tf export PYTHONPATH=${REPO_PATH} MODEL_NAME=$1 PATH_TO_SPANBERT_PYTORCH_DIR=$2 PATH_TO_SAME_SCALE_BERT_TF_DIR=$3 PATH_TO_SAVE_SPANBERT_TF_DIR=$4 if [[ $MODEL_NAME == "spanbert_base" ]]; then # spanbert large echo "Transform SpanBERT Cased Base from Pytorch To TF" python3 ${REPO_PATH}/run/transform_spanbert_pytorch_to_tf.py \ --spanbert_config_path $PATH_TO_SPANBERT_PYTORCH_DIR/config.json \ --bert_tf_ckpt_path $PATH_TO_SAME_SCALE_BERT_TF_DIR/bert_model.ckpt \ --spanbert_pytorch_bin_path $PATH_TO_SPANBERT_PYTORCH_DIR/pytorch_model.bin \ --output_spanbert_tf_dir $PATH_TO_SAVE_SPANBERT_TF_DIR elif [[ $MODEL_NAME == "spanbert_large" ]]; then # spanbert base echo "Transform SpanBERT Cased Large from Pytorch To TF" python3 ${REPO_PATH}/run/transform_spanbert_pytorch_to_tf.py \ --spanbert_config_path $PATH_TO_SPANBERT_PYTORCH_DIR/config.json \ --bert_tf_ckpt_path $PATH_TO_SAME_SCALE_BERT_TF_DIR/bert_model.ckpt \ --spanbert_pytorch_bin_path $PATH_TO_SPANBERT_PYTORCH_DIR/pytorch_model.bin \ --output_spanbert_tf_dir $PATH_TO_SAVE_SPANBERT_TF_DIR else echo 'Unknown argment 1 (Model Sign)' fi ================================================ FILE: scripts/models/corefqa_gpu.sh ================================================ #!/usr/bin/env bash # -*- coding: utf-8 -*- # author: xiaoy li # description: # train and evaluate the middle checkpoints on dev and test sets. REPO_PATH=/home/lixiaoya/xiaoy_tf export PYTHONPATH="$PYTHONPATH:$REPO_PATH" output_dir=/xiaoya/mention_proposal_output bert_dir=/xiaoya/pretrain_ckpt/uncased_L-2_H-128_A-2 data_dir=/xiaoya/corefqa_data/final_overlap_64_2 rm -rf ${output_dir} mkdir -p ${output_dir} CUDA_VISIBLE_DEVICES=3 python3 ${REPO_PATH}/run/run_corefqa.py \ --output_dir=${output_dir} \ --bert_config_file=${bert_dir}/bert_config_nodropout.json \ --init_checkpoint=${bert_dir}/bert_model.ckpt \ --vocab_file=${bert_dir}/vocab.txt \ --logfile_path=${output_dir}/train.log \ --num_epochs=20 \ --keep_checkpoint_max=50 \ --save_checkpoints_steps=500 \ --train_file=${data_dir}/train.64.english.tfrecord \ --dev_file=${data_dir}/dev.64.english.tfrecord \ --test_file=${data_dir}/test.64.english.tfrecord \ --do_train=True \ --do_eval=False \ --do_predict=False \ --learning_rate=1e-5 \ --dropout_rate=0.0 \ --mention_threshold=0.5 \ --hidden_size=128 \ --num_docs=5604 \ --window_size=64 \ --num_window=2 \ --max_num_mention=20 \ --start_end_share=False \ --max_span_width=20 \ --max_candidate_mentions=50 \ --top_span_ratio=0.2 \ --max_top_antecedents=30 \ --max_query_len=150 \ --max_context_len=150 \ --sec_qa_mention_score=False \ --use_tpu=False \ --seed=2333 ================================================ FILE: scripts/models/corefqa_tpu.sh ================================================ #!/usr/bin/env bash # -*- coding: utf-8 -*- # author: xiaoy li # description: # clean code and add comments REPO_PATH=/home/xiaoyli1110/xiaoya/Coref-tf export PYTHONPATH="$PYTHONPATH:$REPO_PATH" export TPU_NAME=tensorflow-tpu export TPU_ZONE=europe-west4-a export GCP_PROJECT=xiaoyli-20-04-274510 output_dir=gs://europe_mention_proposal/output_bertlarge bert_dir=gs://europe_pretrain_mlm/uncased_L-2_H-128_A-2 data_dir=gs://europe_corefqa_data/final_overlap_64_2 python3 ${REPO_PATH}/run/run_corefqa.py \ --output_dir=${output_dir} \ --bert_config_file=${bert_dir}/bert_config_nodropout.json \ --init_checkpoint=${bert_dir}/bert_model.ckpt \ --vocab_file=${bert_dir}/vocab.txt \ --logfile_path=${output_dir}/train.log \ --num_epochs=20 \ --keep_checkpoint_max=50 \ --save_checkpoints_steps=500 \ --train_file=${data_dir}/train.64.english.tfrecord \ --dev_file=${data_dir}/dev.64.english.tfrecord \ --test_file=${data_dir}/test.64.english.tfrecord \ --do_train=True \ --do_eval=False \ --do_predict=False \ --learning_rate=1e-5 \ --dropout_rate=0.0 \ --mention_threshold=0.5 \ --hidden_size=128 \ --num_docs=5604 \ --window_size=64 \ --num_window=2 \ --max_num_mention=20 \ --start_end_share=False \ --max_span_width=20 \ --max_candidate_mentions=50 \ --top_span_ratio=0.2 \ --max_top_antecedents=30 \ --max_query_len=150 \ --max_context_len=150 \ --sec_qa_mention_score=False \ --use_tpu=True \ --tpu_name=$TPU_NAME \ --tpu_zone=$TPU_ZONE \ --gcp_project=$GCP_PROJECT \ --num_tpu_cores=1 \ --seed=2333 ================================================ FILE: scripts/models/mention_gpu.sh ================================================ #!/usr/bin/env bash # -*- coding: utf-8 -*- # author: xiaoy li # description: # clean code and add comments REPO_PATH=/home/lixiaoya/xiaoy_tf export PYTHONPATH="$PYTHONPATH:$REPO_PATH" output_dir=/xiaoya/mention_proposal_output bert_dir=/xiaoya/pretrain_ckpt/uncased_L-2_H-128_A-2 data_dir=/xiaoya/corefqa_data/final_overlap_64_2 rm -rf ${output_dir} mkdir -p ${output_dir} CUDA_VISIBLE_DEVICES=3 python3 ${REPO_PATH}/run/run_mention_proposal.py \ --output_dir=${output_dir} \ --bert_config_file=${bert_dir}/bert_config_nodropout.json \ --init_checkpoint=${bert_dir}/bert_model.ckpt \ --vocab_file=${bert_dir}/vocab.txt \ --logfile_path=${output_dir}/train.log \ --num_epochs=20 \ --keep_checkpoint_max=50 \ --save_checkpoints_steps=500 \ --train_file=${data_dir}/train.64.english.tfrecord \ --dev_file=${data_dir}/dev.64.english.tfrecord \ --test_file=${data_dir}/test.64.english.tfrecord \ --do_train=True \ --do_eval=False \ --do_predict=False \ --learning_rate=1e-5 \ --dropout_rate=0.0 \ --mention_threshold=0.5 \ --hidden_size=128 \ --num_docs=5604 \ --window_size=64 \ --num_window=2 \ --max_num_mention=20 \ --start_end_share=False \ --loss_start_ratio=0.3 \ --loss_end_ratio=0.3 \ --loss_span_ratio=0.3 \ --use_tpu=False \ --seed=2333 ================================================ FILE: scripts/models/mention_tpu.sh ================================================ #!/usr/bin/env bash # -*- coding: utf-8 -*- # author: xiaoy li # description: # clean code and add comments REPO_PATH=/home/xiaoyli1110/xiaoya/Coref-tf export PYTHONPATH="$PYTHONPATH:$REPO_PATH" export TPU_NAME=tensorflow-tpu export TPU_ZONE=europe-west4-a export GCP_PROJECT=xiaoyli-20-04-274510 output_dir=gs://europe_mention_proposal/output_bertlarge bert_dir=gs://europe_pretrain_mlm/uncased_L-2_H-128_A-2 data_dir=gs://europe_corefqa_data/final_overlap_64_2 python3 ${REPO_PATH}/run/run_mention_proposal.py \ --output_dir=${output_dir} \ --bert_config_file=${bert_dir}/bert_config_nodropout.json \ --init_checkpoint=${bert_dir}/bert_model.ckpt \ --vocab_file=${bert_dir}/vocab.txt \ --logfile_path=${output_dir}/train.log \ --num_epochs=20 \ --keep_checkpoint_max=50 \ --save_checkpoints_steps=500 \ --train_file=${data_dir}/train.64.english.tfrecord \ --dev_file=${data_dir}/dev.64.english.tfrecord \ --test_file=${data_dir}/test.64.english.tfrecord \ --do_train=True \ --do_eval=False \ --do_predict=False \ --learning_rate=1e-5 \ --dropout_rate=0.0 \ --mention_threshold=0.5 \ --hidden_size=128 \ --num_docs=5604 \ --window_size=64 \ --num_window=2 \ --max_num_mention=20 \ --start_end_share=False \ --loss_start_ratio=0.3 \ --loss_end_ratio=0.3 \ --loss_span_ratio=0.3 \ --use_tpu=True \ --tpu_name=$TPU_NAME \ --tpu_zone=$TPU_ZONE \ --gcp_project=$GCP_PROJECT \ --num_tpu_cores=1 \ --seed=2333 ================================================ FILE: scripts/models/quoref_tpu.sh ================================================ #!/usr/bin/env bash # -*- coding: utf-8 -*- # author: xiaoy li # description: # finetune the spanbert model on squad 2.0 for data augment. REPO_PATH=/home/shannon/coref-tf export TPU_NAME=tf-tpu export PYTHONPATH="$PYTHONPATH:$REPO_PATH" QUOREF_DIR=gs://qa_tasks/quoref BERT_DIR=gs://corefqa_output_squad/panbert_large_squad2_2e-5 OUTPUT_DIR=gs://corefqa_output_quoref/spanbert_large_squad2_best_quoref_3e-5 python3 ${REPO_PATH}/run_quoref.py \ --vocab_file=$BERT_DIR/vocab.txt \ --bert_config_file=$BERT_DIR/bert_config.json \ --init_checkpoint=$BERT_DIR/best_bert_model.ckpt \ --do_train=True \ --train_file=$QUOREF_DIR/quoref-train-v0.1.json \ --do_predict=True \ --predict_file=$QUOREF_DIR/quoref-dev-v0.1.json \ --train_batch_size=8 \ --learning_rate=3e-5 \ --num_train_epochs=5 \ --max_seq_length=384 \ --do_lower_case=False \ --doc_stride=128 \ --output_dir=${OUTPUT_DIR} \ --use_tpu=True \ --tpu_name=$TPU_NAME ================================================ FILE: scripts/models/squad_tpu.sh ================================================ #!/usr/bin/env bash # -*- coding: utf-8 -*- # author: xiaoy li # description: # finetune the spanbert model on squad 2.0 for data augment. REPO_PATH=/home/shannon/coref-tf export TPU_NAME=tf-tpu export PYTHONPATH="$PYTHONPATH:$REPO_PATH" SQUAD_DIR=gs://qa_tasks/squad2 BERT_DIR=gs://pretrained_mlm_checkpoint/spanbert_large_tf OUTPUT_DIR=gs://corefqa_output_squad/spanbert_large_squad2_2e-5 python3 ${REPO_PATH}/run/run_squad.py \ --vocab_file=$BERT_DIR/vocab.txt \ --bert_config_file=$BERT_DIR/bert_config.json \ --init_checkpoint=$BERT_DIR/bert_model.ckpt \ --do_train=True \ --train_file=$SQUAD_DIR/train-v2.0.json \ --do_predict=True \ --predict_file=$SQUAD_DIR/dev-v2.0.json \ --train_batch_size=8 \ --learning_rate=2e-5 \ --num_train_epochs=4.0 \ --max_seq_length=384 \ --do_lower_case=False \ --doc_stride=128 \ --output_dir=${OUTPUT_DIR} \ --use_tpu=True \ --tpu_name=$TPU_NAME \ --version_2_with_negative=True ================================================ FILE: tests/bitwise_and.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # author: xiaoy li # description: # import tensorflow as tf if __name__ == "__main__": sess = tf.compat.v1.InteractiveSession() lhs = tf.constant([0, 5, 3, 14], dtype=tf.int32) rhs = tf.constant([5, 0, 7, 11], dtype=tf.int32) res = tf.bitwise.bitwise_and(lhs, rhs) res.eval() # array([ 0, 0, 3, 10], dtype=int32) sess.close() ================================================ FILE: tests/construct_label.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # desc: # construct labels import tensorflow as tf if __name__ == "__main__": sess = tf.compat.v1.InteractiveSession() gold_starts = tf.constant([1, 2, 3, 4]) gold_ends = tf.constant([2, 3, 4, 5]) num_word = 10 gold_mention_sparse_label = tf.stack([gold_starts, gold_ends], axis=1) gold_mention_sparse_label.eval() gold_span_value = tf.reshape(tf.ones_like(gold_starts, tf.int32), [-1]) gold_span_shape = tf.constant([num_word, num_word]) gold_span_label = tf.cast(tf.scatter_nd(gold_mention_sparse_label, gold_span_value, gold_span_shape), tf.int32) gold_span_label.eval() candidate_start = tf.constant([1, 4, 5]) candidate_end = tf.constant([2, 5, 5]) candidate_span = tf.stack([candidate_start, candidate_end], axis=1) gold_span_label = tf.gather_nd(gold_span_label, tf.expand_dims(candidate_span, 1)) gold_span_label.eval() ================================================ FILE: tests/cumsum.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ input_a: array([[0, 1, 0, 1, 1], [0, 1, 1, 1, 1], [0, 1, 1, 0, 1], [1, 1, 1, 1, 1], [0, 1, 1, 1, 0]], dtype=int32) cum_input_b: array([[1, 2, 3, 4, 5], [1, 2, 3, 4, 5], [1, 2, 3, 4, 5], [1, 2, 3, 4, 5], [1, 2, 3, 4, 5]], dtype=int32) input_c: array([[0, 2, 0, 4, 5], [0, 2, 3, 4, 5], [0, 2, 3, 0, 5], [1, 2, 3, 4, 5], [0, 2, 3, 4, 0]], dtype=int32) input_c: array([[ 1, 2, 3, 4, 5], [129, 130, 131, 132, 133], [257, 258, 259, 260, 261], [385, 386, 387, 388, 389], [513, 514, 515, 516, 517]], dtype=int32) input_d: array([[ 0, 2, 0, 4, 5], [ 0, 130, 131, 132, 133], [ 0, 258, 259, 0, 261], [385, 386, 387, 388, 389], [ 0, 514, 515, 516, 0]], dtype=int32) flat_input_d: array([ 0, 2, 0, 4, 5, 0, 130, 131, 132, 133, 0, 258, 259, 0, 261, 385, 386, 387, 388, 389, 0, 514, 515, 516, 0], dtype=int32) boolean_mask: array([False, True, False, True, True, False, True, True, True, True, False, True, True, False, True, True, True, True, True, True, False, True, True, True, False]) input_f: array([ 2, 4, 5, 130, 131, 132, 133, 258, 259, 261, 385, 386, 387, 388, 389, 514, 515, 516], dtype=int32) """ import tensorflow as tf if __name__ == "__main__": sess = tf.compat.v1.InteractiveSession() input_a = tf.constant([ [0, 1, 0, 1, 1], [0, 1, 1, 1, 1], [0, 1, 1, 0, 1], [1, 1, 1, 1, 1], [0, 1, 1, 1, 0]]) ones_input_b = tf.ones_like(input_a, tf.int32) cum_input_b = tf.math.cumsum(ones_input_b, axis=1) cum_input_b.eval() # input_c = tf.math.multiply(cum_input_b, input_a) # input_c.eval() seq_len = 128 offset = tf.tile(tf.expand_dims(tf.range(5) * 128, 1), [1, 5]) offset.eval() input_e = offset + cum_input_b input_e.eval() input_d = tf.math.multiply(input_e, input_a) input_d.eval() flat_input_d = tf.reshape(input_d, [-1]) flat_input_d.eval() boolean_mask = tf.math.greater(flat_input_d, tf.zeros_like(flat_input_d, tf.int32)) boolean_mask.eval() input_f = tf.boolean_mask(flat_input_d, boolean_mask) input_f.eval() sess.close() ================================================ FILE: tests/gather.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # author: xiaoy li import tensorflow as tf if __name__ == "__main__": sess = tf.compat.v1.InteractiveSession() lhs = tf.zeros((4, 3)) slice_lhs = tf.gather(lhs, 1) # slice_lhs_nd = tf.gather_nd(lhs, 1) slice_lhs = tf.gather(lhs, [1, 2]) slice_lhs.eval() # slice_lhs_nd.eval() # array([ 0, 0, 3, 10], dtype=int32) sess.close() ================================================ FILE: tests/model_fn.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # author: xiaoy li # description: # test config in model fn builder def model_fn(config): def mention_proposal_fn(): print("the number of document is : ") print(config.document_number) print(config.number_window_size) return mention_proposal_fn class Config: number_window_size = 2 document_number = 3 if __name__ == "__main__": config = Config() get_model_fn = model_fn(config) get_model_fn() ================================================ FILE: tests/tile_repeat.py ================================================ #!/usr/bin/env python3 import numpy as np import tensorflow as tf a_np = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9], [2, 3, 4], [5, 6, 7], [8, 9, 10]]) print(a_np.shape) # exit() def shape(x, dim): return x.get_shape()[dim].value or tf.shape(x)[dim] if __name__ == "__main__": original_array = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9], [2, 3, 4], [5, 6, 7], [8, 9, 10]]) sess = tf.compat.v1.InteractiveSession() start_scores = tf.convert_to_tensor(tf.constant([[1, 2, 3], [4, 5, 6], [7, 8, 9], [2, 3, 4], [5, 6, 7], [8, 9, 10]])) print(tf.shape(start_scores)) # exit() expand_scores = tf.tile(tf.expand_dims(start_scores, 2), [1, 1, 3]) # (6, 3, 3) print(expand_scores.eval()) print(shape(expand_scores, 0)) print(shape(expand_scores, 1)) print(shape(expand_scores, 2)) print("=*="*20) # tf.convert_to_tensor(data_np, np.float32) # ndarray_scores = tf.make_ndarray(expand_scores) # ndarray_scores = tf.convert_to_tensor(expand_scores, np.int32) # print(ndarray_scores) # exit() ndarray_scores = np.array([[[1, 1, 1], [ 2 , 2 , 2], [ 3 , 3 , 3]], [[ 4 , 4 , 4],[ 5 , 5 , 5],[ 6 , 6 , 6]], [[ 7 , 7 , 7],[ 8 , 8 , 8],[ 9 , 9 , 9]], [[ 2 , 2 , 2], [ 3 , 3 , 3], [ 4 , 4 , 4]], [[ 5 , 5 , 5], [ 6 , 6 , 6], [ 7 , 7 , 7]], [[ 8 , 8 , 8], [ 9 , 9 , 9], [10 , 10 ,10]]]) print("$="*20) print("test_a is : {}".format(str(ndarray_scores[2, 2, 2]))) print("test_b is : {}".format(str(original_array[2, 2]))) print("^-"*20) print("test_a is : {}".format(str(ndarray_scores[2, 1, 1]))) print("test_b is : {}".format(str(original_array[2, 1]))) print("^-"*20) print("test_a is : {}".format(str(ndarray_scores[2, 0, 0]))) print("test_b is : {}".format(str(original_array[2, 0]))) sess.close() # span_scores[k][i][j] = start_scores[k][i] + end_scores[k][j] # start_scores[k][i][j] = start_scores[k][i] # end_scores[k][i][j] = end_scores[k][j] # [[1, 2, 3], [4, 5, 6], [7, 8, 9], [2, 3, 4], [5, 6, 7], [8, 9, 10]] """ [[[1, 1, 1], [ 2 , 2 , 2], [ 3 , 3 , 3]], [[ 4 , 4 , 4],[ 5 , 5 , 5],[ 6 , 6 , 6]], [[ 7 , 7 , 7],[ 8 , 8 , 8],[ 9 , 9 , 9]], [[ 2 , 2 , 2], [ 3 , 3 , 3], [ 4 , 4 , 4]], [[ 5 , 5 , 5], [ 6 , 6 , 6], [ 7 , 7 , 7]], [[ 8 , 8 , 8], [ 9 , 9 , 9], [10 , 10 ,10]]] """ ================================================ FILE: tests/tpu_operation.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # author: xiaoy li # descripiton: # test math operations in tpu import tensorflow as tf from tensorflow.contrib import tpu from tensorflow.contrib.cluster_resolver import TPUClusterResolver TPU_NAME = "tensorflow-tpu" TPU_ZONE = "us-central1-f" GCP_PROJECT = "xiaoyli-20-04-274510" if __name__ == "__main__": tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(TPU_NAME, zone=TPU_ZONE, project=GCP_PROJECT) # tpu_cluster_resolver = TPUClusterResolver(tpu=['tensorflow-tpu']).get_master() tf.config.experimental_connect_to_cluster(tpu_cluster_resolver) tf.tpu.experimental.initialize_tpu_system(tpu_cluster_resolver) scores = tf.constant([1.0, 2.3, 3.2, 4.3, 1.5, 1.8, 98, 2.9]) k = 2 def test_top_k(): top_scores, top_index = tf.nn.top_k(scores, k) return top_scores, top_index test_op = test_top_k # with tf.compat.v1.InteractiveSession(tpu_cluster_resolver) as sess: with tf.compat.v1.Session(tpu_cluster_resolver) as sess: sess.run(tpu.initialize_system()) scores = tf.constant([1.0, 2.3, 3.2, 4.3, 1.5, 1.8, 98, 2.9]) k = 2 print("ALL Devices: ", tf.config.experimental_list_devices()) top_scores, top_index = tf.nn.top_k(scores, k) print(top_scores.eval()) print(top_index.eval()) sess.run(tpu.shutdown_system()) ================================================ FILE: utils/load_pytorch_to_tf.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # author: xiaoy li # description: # transform pretrained model checkpoints from [pytorch *.bin] to [tensorflow *.ckpt] # Reference: # https://github.com/mandarjoshi90/coref/blob/master/pytorch_to_tf.py import numpy as np import torch import tensorflow as tf from tensorflow.python.framework import ops from tensorflow.python.ops import variable_scope as vs tensors_to_transpose = ( "dense/kernel", "attention/self/query", "attention/self/key", "attention/self/value" ) var_map = ( ('layer.', 'layer_'), ('word_embeddings.weight', 'word_embeddings'), ('position_embeddings.weight', 'position_embeddings'), ('token_type_embeddings.weight', 'token_type_embeddings'), ('.', '/'), ('LayerNorm/weight', 'LayerNorm/gamma'), ('LayerNorm/bias', 'LayerNorm/beta'), ('weight', 'kernel') ) def to_tf_var_name(name: str): for patt, repl in iter(var_map): name = name.replace(patt, repl) return '{}'.format(name) def my_convert_keys(model): converted = {} for k_pt, v in model.items(): k_tf = to_tf_var_name(k_pt) converted[k_tf] = v return converted def load_from_pytorch_checkpoint(checkpoint, assignment_map): pytorch_model = torch.load(checkpoint, map_location='cpu') pt_model_with_tf_keys = my_convert_keys(pytorch_model) for _, name in assignment_map.items(): store_vars = vs._get_default_variable_store()._vars var = store_vars.get(name, None) assert var is not None if name not in pt_model_with_tf_keys: print('WARNING:', name, 'not found in original model.') continue array = pt_model_with_tf_keys[name].cpu().numpy() if any([x in name for x in tensors_to_transpose]): array = array.transpose() assert tuple(var.get_shape().as_list()) == tuple(array.shape) init_value = ops.convert_to_tensor(array, dtype=np.float32) var._initial_value = init_value var._initializer_op = var.assign(init_value) def print_vars(pytorch_ckpt, tf_ckpt): tf_vars = tf.train.list_variables(tf_ckpt) tf_vars = {k: v for (k, v) in tf_vars} pytorch_model = torch.load(pytorch_ckpt) pt_model_with_tf_keys = my_convert_keys(pytorch_model) only_pytorch, only_tf, common = [], [], [] tf_only = set(tf_vars.keys()) for k, v in pt_model_with_tf_keys.items(): if k in tf_vars: common.append(k) tf_only.remove(k) else: only_pytorch.append(k) print('-------------------') print('Common', len(common)) for k in common: array = pt_model_with_tf_keys[k].cpu().numpy() if any([x in k for x in tensors_to_transpose]): array = array.transpose() tf_shape = tuple(tf_vars[k]) pt_shape = tuple(array.shape) if tf_shape != pt_shape: print(k, tf_shape, pt_shape) print('-------------------') print('Pytorch only', len(only_pytorch)) for k in only_pytorch: print(k, pt_model_with_tf_keys[k].size()) print('-------------------') print('TF only', len(tf_only)) for k in tf_only: print(k, tf_vars[k]) ================================================ FILE: utils/metrics.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import numpy as np from collections import Counter from scipy.optimize import linear_sum_assignment def mention_proposal_prediction(config, current_doc_result, concat_only=True): """ current_doc_result: "total_loss": total_loss, "start_scores": start_scores, "start_gold": gold_starts, "end_gold": gold_ends, "end_scores": end_scores, "span_scores": span_scores, "span_gold": span_mention """ span_scores = current_doc_result["span_scores"] span_gold = current_doc_result["span_gold"] if concat_only: scores = span_scores else: start_scores = current_doc_result["start_scores"], end_scores = current_doc_result["end_scores"] # start_scores = tf.tile(tf.expand_dims(start_scores, 2), [1, 1, config["max_segment_len"]]) start_scores = np.tile(np.expand_dims(start_scores, axis=2), (1, 1, config["max_segment_len"])) end_scores = np.tile(np.expand_dims(end_scores, axis=2), (1, 1, config["max_segment_len"])) start_scores = np.reshape(start_scores, [-1, config["max_segment_len"], config["max_segment_len"]]) end_scores = np.reshape(end_scores, [-1, config["max_segment_len"], config["max_segment_len"]]) # end_scores -> max_training_sent, max_segment_len scores = (start_scores + end_scores + span_scores)/3 pred_span_label = scores >= 0.5 pred_span_label = np.reshape(pred_span_label, [-1]) gold_span_label = np.reshape(span_gold, [-1]) return pred_span_label, gold_span_label def f1(p_num, p_den, r_num, r_den, beta=1): p = 0 if p_den == 0 else p_num / float(p_den) r = 0 if r_den == 0 else r_num / float(r_den) return 0 if p + r == 0 else (1 + beta * beta) * p * r / (beta * beta * p + r) class CorefEvaluator(object): def __init__(self): self.evaluators = [Evaluator(m) for m in (muc, b_cubed, ceafe)] def update(self, predicted, gold, mention_to_predicted, mention_to_gold): for e in self.evaluators: e.update(predicted, gold, mention_to_predicted, mention_to_gold) def get_f1(self): return sum(e.get_f1() for e in self.evaluators) / len(self.evaluators) def get_recall(self): return sum(e.get_recall() for e in self.evaluators) / len(self.evaluators) def get_precision(self): return sum(e.get_precision() for e in self.evaluators) / len(self.evaluators) def get_prf(self): return self.get_precision(), self.get_recall(), self.get_f1() class Evaluator(object): def __init__(self, metric, beta=1): self.p_num = 0 self.p_den = 0 self.r_num = 0 self.r_den = 0 self.metric = metric self.beta = beta def update(self, predicted, gold, mention_to_predicted, mention_to_gold): if self.metric == ceafe: pn, pd, rn, rd = self.metric(predicted, gold) else: pn, pd = self.metric(predicted, mention_to_gold) rn, rd = self.metric(gold, mention_to_predicted) self.p_num += pn self.p_den += pd self.r_num += rn self.r_den += rd def get_f1(self): return f1(self.p_num, self.p_den, self.r_num, self.r_den, beta=self.beta) def get_recall(self): return 0 if self.r_num == 0 else self.r_num / float(self.r_den) def get_precision(self): return 0 if self.p_num == 0 else self.p_num / float(self.p_den) def get_prf(self): return self.get_precision(), self.get_recall(), self.get_f1() def get_counts(self): return self.p_num, self.p_den, self.r_num, self.r_den def evaluate_documents(documents, metric, beta=1): evaluator = Evaluator(metric, beta=beta) for document in documents: evaluator.update(document) return evaluator.get_precision(), evaluator.get_recall(), evaluator.get_f1() def b_cubed(clusters, mention_to_gold): num, dem = 0, 0 for c in clusters: if len(c) == 1: continue gold_counts = Counter() correct = 0 for m in c: if m in mention_to_gold: gold_counts[tuple(mention_to_gold[m])] += 1 for c2, count in gold_counts.items(): if len(c2) != 1: correct += count * count num += correct / float(len(c)) dem += len(c) return num, dem def muc(clusters, mention_to_gold): tp, p = 0, 0 for c in clusters: p += len(c) - 1 tp += len(c) linked = set() for m in c: if m in mention_to_gold: linked.add(mention_to_gold[m]) else: tp -= 1 tp -= len(linked) return tp, p def phi4(c1, c2): return 2 * len([m for m in c1 if m in c2]) / float(len(c1) + len(c2)) def ceafe(clusters, gold_clusters): clusters = [c for c in clusters if len(c) != 1] scores = np.zeros((len(gold_clusters), len(clusters))) for i in range(len(gold_clusters)): for j in range(len(clusters)): scores[i, j] = phi4(gold_clusters[i], clusters[j]) row_ind, col_ind = linear_sum_assignment(-scores) similarity = sum(scores[row_ind, col_ind]) return similarity, len(clusters), similarity, len(gold_clusters) def lea(clusters, mention_to_gold): num, dem = 0, 0 for c in clusters: if len(c) == 1: continue common_links = 0 all_links = len(c) * (len(c) - 1) / 2.0 for i, m in enumerate(c): if m in mention_to_gold: for m2 in c[i + 1:]: if m2 in mention_to_gold and mention_to_gold[m] == mention_to_gold[m2]: common_links += 1 num += len(c) * common_links / float(all_links) dem += len(c) return num, dem ================================================ FILE: utils/radam.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import tensorflow as tf from tensorflow.python import ( ops, math_ops, state_ops, control_flow_ops, resource_variable_ops) from tensorflow.python.training.optimizer import Optimizer __all__ = ['RAdam'] class RAdam(Optimizer): """Rectified Adam (RAdam) optimizer. According to the paper [On The Variance Of The Adaptive Learning Rate And Beyond](https://arxiv.org/pdf/1908.03265v1.pdf). """ def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8, amsgrad=False, use_locking=False, name='RAdam'): r"""Construct a new Rectified Adam optimizer. Args: learning_rate: A Tensor or a floating point value. The learning rate. beta1: A float value or a constant float tensor. The exponential decay rate for the 1st moment estimates. beta2: A float value or a constant float tensor. The exponential decay rate for the 2nd moment estimates. epsilon: A small constant for numerical stability. This epsilon is "epsilon hat" in the Kingma and Ba paper (in the formula just before Section 2.1), not the epsilon in Algorithm 1 of the paper. amsgrad: boolean. Whether to apply AMSGrad variant of this algorithm from the paper "On the Convergence of Adam and beyond". use_locking: If `True` use locks for update operations. name: Optional name for the operations created when applying gradients. Defaults to "Adam". @compatibility(eager) When eager execution is enabled, `learning_rate`, `beta1`, `beta2`, and `epsilon` can each be a callable that takes no arguments and returns the actual value to use. This can be useful for changing these values across different invocations of optimizer functions. @end_compatibility """ super(RAdam, self).__init__(use_locking, name) self._lr = learning_rate self._beta1 = beta1 self._beta2 = beta2 self._epsilon = epsilon self._amsgrad = amsgrad def _get_beta_accumulators(self): with ops.init_scope(): graph = ops.get_default_graph() return (self._get_non_slot_variable("beta1_power", graph=graph), self._get_non_slot_variable("beta2_power", graph=graph), ) def _get_niter(self): with ops.init_scope(): graph = ops.get_default_graph() return self._get_non_slot_variable("niter", graph=graph) def _create_slots(self, var_list): first_var = min(var_list, key=lambda x: x.name) self._create_non_slot_variable( initial_value=self._beta1, name="beta1_power", colocate_with=first_var) self._create_non_slot_variable( initial_value=self._beta2, name="beta2_power", colocate_with=first_var) self._create_non_slot_variable( initial_value=1, name="niter", colocate_with=first_var) for var in var_list: self._zeros_slot(var, 'm', self._name) self._zeros_slot(var, 'v', self._name) if self._amsgrad: for var in var_list: self._zeros_slot(var, 'vhat', self._name) def _prepare(self): learning_rate = self._call_if_callable(self._lr) beta1 = self._call_if_callable(self._beta1) beta2 = self._call_if_callable(self._beta2) epsilon = self._call_if_callable(self._epsilon) self._lr_t = ops.convert_to_tensor(learning_rate, name="learning_rate") self._beta1_t = ops.convert_to_tensor(beta1, name="beta1") self._beta2_t = ops.convert_to_tensor(beta2, name="beta2") self._epsilon_t = ops.convert_to_tensor(epsilon, name="epsilon") def _apply_dense_shared(self, grad, var): var_dtype = var.dtype.base_dtype beta1_power, beta2_power = self._get_beta_accumulators() beta1_power = math_ops.cast(beta1_power, var_dtype) beta2_power = math_ops.cast(beta2_power, var_dtype) niter = self._get_niter() niter = math_ops.cast(niter, var_dtype) lr_t = math_ops.cast(self._lr_t, var_dtype) beta1_t = math_ops.cast(self._beta1_t, var_dtype) beta2_t = math_ops.cast(self._beta2_t, var_dtype) epsilon_t = math_ops.cast(self._epsilon_t, var_dtype) sma_inf = 2.0 / (1.0 - beta2_t) - 1.0 sma_t = sma_inf - 2.0 * niter * beta2_power / (1.0 - beta2_power) m = self.get_slot(var, 'm') m_t = state_ops.assign(m, beta1_t * m + (1.0 - beta1_t) * grad, use_locking=self._use_locking) m_corr_t = m_t / (1.0 - beta1_power) v = self.get_slot(var, 'v') v_t = state_ops.assign(v, beta2_t * v + (1.0 - beta2_t) * math_ops.square(grad), use_locking=self._use_locking) if self._amsgrad: vhat = self.get_slot(var, 'vhat') vhat_t = state_ops.assign(vhat, math_ops.maximum(vhat, v_t), use_locking=self._use_locking) v_corr_t = math_ops.sqrt(vhat_t / (1.0 - beta2_power) + epsilon_t) else: v_corr_t = math_ops.sqrt(v_t / (1.0 - beta2_power) + epsilon_t) r_t = math_ops.sqrt((sma_t - 4.0) / (sma_inf - 4.0) * (sma_t - 2.0) / (sma_inf - 2.0) * sma_inf / sma_t) var_t = tf.where(sma_t > 5.0, r_t * m_corr_t / v_corr_t, m_corr_t) var_update = state_ops.assign_sub(var, lr_t * var_t, use_locking=self._use_locking) updates = [var_update, m_t, v_t] if self._amsgrad: updates.append(vhat_t) return control_flow_ops.group(*updates) def _apply_dense(self, grad, var): return self._apply_dense_shared(grad, var) def _resource_apply_dense(self, grad, var): return self._apply_dense_shared(grad, var.handle) def _apply_sparse_shared(self, grad, var, indices, scatter_add): var_dtype = var.dtype.base_dtype beta1_power, beta2_power = self._get_beta_accumulators() beta1_power = math_ops.cast(beta1_power, var_dtype) beta2_power = math_ops.cast(beta2_power, var_dtype) niter = self._get_niter() niter = math_ops.cast(niter, var_dtype) lr_t = math_ops.cast(self._lr_t, var_dtype) beta1_t = math_ops.cast(self._beta1_t, var_dtype) beta2_t = math_ops.cast(self._beta2_t, var_dtype) epsilon_t = math_ops.cast(self._epsilon_t, var_dtype) sma_inf = 2.0 / (1.0 - beta2_t) - 1.0 sma_t = sma_inf - 2.0 * niter * beta2_power / (1.0 - beta2_power) m = self.get_slot(var, 'm') m_t = state_ops.assign(m, beta1_t * m, use_locking=self._use_locking) with ops.control_dependencies([m_t]): m_t = scatter_add(m, indices, grad * (1 - beta1_t)) m_corr_t = m_t / (1.0 - beta1_power) v = self.get_slot(var, 'v') v_t = state_ops.assign(v, beta2_t * v, use_locking=self._use_locking) with ops.control_dependencies([v_t]): v_t = scatter_add(v, indices, (1.0 - beta2_t) * math_ops.square(grad)) if self._amsgrad: vhat = self.get_slot(var, 'vhat') vhat_t = state_ops.assign(vhat, math_ops.maximum(vhat, v_t), use_locking=self._use_locking) v_corr_t = math_ops.sqrt(vhat_t / (1.0 - beta2_power) + epsilon_t) else: v_corr_t = math_ops.sqrt(v_t / (1.0 - beta2_power) + epsilon_t) r_t = math_ops.sqrt((sma_t - 4.0) / (sma_inf - 4.0) * (sma_t - 2.0) / (sma_inf - 2.0) * sma_inf / sma_t) var_t = tf.where(sma_t > 5.0, r_t * m_corr_t / v_corr_t, m_corr_t) var_update = state_ops.assign_sub(var, lr_t * var_t, use_locking=self._use_locking) updates = [var_update, m_t, v_t] if self._amsgrad: updates.append(vhat_t) return control_flow_ops.group(*updates) def _apply_sparse(self, grad, var): return self._apply_sparse_shared( grad.values, var, grad.indices, lambda x, i, v: state_ops.scatter_add( # pylint: disable=g-long-lambda x, i, v, use_locking=self._use_locking)) def _resource_apply_sparse(self, grad, var, indices): return self._apply_sparse_shared(grad, var, indices, self._resource_scatter_add) def _resource_scatter_add(self, x, i, v): with ops.control_dependencies( [resource_variable_ops.resource_scatter_add(x.handle, i, v)]): return x.value() def _finish(self, update_ops, name_scope): # Update the power accumulators. with ops.control_dependencies(update_ops): beta1_power, beta2_power = self._get_beta_accumulators() niter = self._get_niter() with ops.colocate_with(beta1_power): update_beta1 = beta1_power.assign( beta1_power * self._beta1_t, use_locking=self._use_locking) update_beta2 = beta2_power.assign( beta2_power * self._beta2_t, use_locking=self._use_locking) update_niter = niter.assign( niter + 1, use_locking=self._use_locking) return control_flow_ops.group( *update_ops + [update_beta1, update_beta2, update_niter], name=name_scope) ================================================ FILE: utils/util.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import codecs import collections import errno import os import shutil import pyhocon import tensorflow as tf from models import corefqa from models import mention_proposal repo_path = "/".join(os.path.realpath(__file__).split("/")[:-2]) def get_model(config, model_sign="corefqa"): if model_sign == "corefqa": return corefqa.CorefQAModel(config) else: return mention_proposal.MentionProposalModel(config) def initialize_from_env(eval_test=False, config_params="train_spanbert_base", config_file="experiments_tinybert.conf", use_tpu=False, print_info=False): if not use_tpu: print("loading experiments.conf ... ") config = pyhocon.ConfigFactory.parse_file(os.path.join(repo_path, config_file)) else: print("loading experiments_tpu.conf ... ") config = pyhocon.ConfigFactory.parse_file(os.path.join(repo_path, config_file)) config = config[config_params] if print_info: tf.logging.info("%*%"*20) tf.logging.info("%*%"*20) tf.logging.info("%%%%%%%% Configs are showed as follows : %%%%%%%%") for tmp_key, tmp_value in config.items(): tf.logging.info(str(tmp_key) + " : " + str(tmp_value)) tf.logging.info("%*%"*20) tf.logging.info("%*%"*20) config["log_dir"] = mkdirs(os.path.join(config["log_root"], config_params)) if print_info: tf.logging.info(pyhocon.HOCONConverter.convert(config, "hocon")) return config def copy_checkpoint(source, target): for ext in (".index", ".data-00000-of-00001"): shutil.copyfile(source + ext, target + ext) def make_summary(value_dict): return tf.Summary(value=[tf.Summary.Value(tag=k, simple_value=v) for k, v in value_dict.items()]) def flatten(l): return [item for sublist in l for item in sublist] def set_gpus(*gpus): # pass os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(str(g) for g in gpus) print("Setting CUDA_VISIBLE_DEVICES to: {}".format(os.environ["CUDA_VISIBLE_DEVICES"])) def mkdirs(path): try: os.makedirs(path) except OSError as exception: if exception.errno != errno.EEXIST: raise return path def load_char_dict(char_vocab_path): vocab = [u""] with codecs.open(char_vocab_path, encoding="utf-8") as f: vocab.extend(l.strip() for l in f.readlines()) char_dict = collections.defaultdict(int) char_dict.update({c: i for i, c in enumerate(vocab)}) return char_dict def maybe_divide(x, y): return 0 if y == 0 else x / float(y) def shape(x, dim): return x.get_shape()[dim].value or tf.shape(x)[dim] def ffnn(inputs, num_hidden_layers, hidden_size, output_size, dropout, output_weights_initializer=tf.truncated_normal_initializer(stddev=0.02), hidden_initializer=tf.truncated_normal_initializer(stddev=0.02)): if len(inputs.get_shape()) > 3: raise ValueError("FFNN with rank {} not supported".format(len(inputs.get_shape()))) current_inputs = inputs hidden_weights = tf.get_variable("hidden_weights", [hidden_size, output_size], initializer=hidden_initializer) hidden_bias = tf.get_variable("hidden_bias", [output_size], initializer=tf.zeros_initializer()) current_outputs = tf.nn.relu(tf.nn.xw_plus_b(current_inputs, hidden_weights, hidden_bias)) return current_outputs def batch_gather(emb, indices): batch_size = shape(emb, 0) seqlen = shape(emb, 1) if len(emb.get_shape()) > 2: emb_size = shape(emb, 2) else: emb_size = 1 flattened_emb = tf.reshape(emb, [batch_size * seqlen, emb_size]) # [batch_size * seqlen, emb] offset = tf.expand_dims(tf.range(batch_size) * seqlen, 1) # [batch_size, 1] gathered = tf.gather(flattened_emb, indices + offset) # [batch_size, num_indices, emb] if len(emb.get_shape()) == 2: gathered = tf.squeeze(gathered, 2) # [batch_size, num_indices] return gathered