Showing preview only (5,826K chars total). Download the full file or copy to clipboard to get everything.
Repository: hanxiao/daanet
Branch: master
Commit: 362aa1b69ec7
Files: 49
Total size: 5.5 MB
Directory structure:
gitextract_as6ks4uh/
├── .gitignore
├── README.md
├── api.py
├── app.py
├── base/
│ ├── base_io.py
│ └── base_model.py
├── daanet/
│ ├── base.py
│ └── basic.py
├── dataio_utils/
│ ├── __init__.py
│ ├── flow_io.py
│ ├── full_load_io.py
│ └── helper.py
├── default.yaml
├── gpu_env.py
├── grid.yaml
├── grid_search.py
├── model_utils/
│ ├── __init__.py
│ └── helper.py
├── nlp/
│ ├── __init__.py
│ ├── encode_blocks.py
│ ├── match_blocks.py
│ ├── nn.py
│ └── seq2seq/
│ ├── __init__.py
│ ├── common.py
│ ├── pointer_generator.py
│ └── rnn.py
├── sample_data/
│ ├── sample.charembed.txt
│ ├── sample.embed.txt
│ ├── sample.json
│ └── sample.out.json
└── utils/
├── __init__.py
├── eval_4/
│ ├── __init__.py
│ ├── bleu_metric/
│ │ ├── __init__.py
│ │ ├── bleu.py
│ │ └── bleu_scorer.py
│ ├── eval.py
│ ├── exact_f1/
│ │ ├── __init__.py
│ │ └── exact_f1.py
│ ├── meteor/
│ │ ├── __init__.py
│ │ ├── meteor-1.5.jar
│ │ ├── meter.py
│ │ ├── reference
│ │ ├── res.txt
│ │ └── test
│ └── rouge_metric/
│ ├── __init__.py
│ └── rouge.py
├── helper.py
├── predictor.py
└── vocab.py
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
models/
preprocessed/
raw/
*.pyc
vocab.search
local
.idea/
add_copyright.py
copyright
.DS_Store
data/vocab
data/results/
_job*.yaml
toy*.py
tmp.json
summary.json
data/*
.*.yaml
================================================
FILE: README.md
================================================
# DAANet: Dual Ask-Answer Network for Machine Reading Comprehension
[](https://opensource.org/licenses/MIT) [](https://opensource.org/licenses/MIT)
## Requirements
- Python >= 3.6
- Tensorflow >= 1.6 (self-compiled TF-gpu is recommended!)
- gputil >= 1.3.0
- GPU
## Usage
For dual learning run:
```
python grid_search.py daanet
```
For QA-only model (corresponds to `mono` in the experiment) run:
```
python grid_search.py monoqa
```
For QG-only model (corresponds to `mono` in the experiment) , run :
```
python grid_search.py monoqg
```
All hyperparameters used in the paper are stored in `default.yaml`:
You can change the data path and save dir in `grid.yaml`
## Evaluation
Evaluation on the dev set is automatically done after each epoch.
To do evaluation manually,
```bash
python app.py evaluate ~/save/models/DDMM-HHMMSS/default.yaml
```
, where `~/save/models/DDMM-HHMMSS/default.yaml` is the saved yaml config of model `DDMM-HHMMSS`. It is created during the training procedure. It automatically loads the parameters from the best epoch (or fallback to the last epoch) to the model.
## Continuous Training
```bash
python app.py train ~/save/models/DDMM-HHMMSS/default.yaml
```
It will load the best (or last) model so far and conducts incremental training.
## Generated Samples
Selected outputs from DAANET and mono. Yellow text is question-related; green text is answer-related.
## Attention Matrix
Question-Context and Answer Context attention matrices
================================================
FILE: api.py
================================================
import logging
from tensorflow.python.framework.errors_impl import NotFoundError, InvalidArgumentError
from gpu_env import ModeKeys, APP_NAME
from utils.helper import build_model
logger = logging.getLogger(APP_NAME)
def train(args):
# check run_mode
if 'run_mode' in args:
args.set_hparam('run_mode', ModeKeys.TRAIN.value)
model = build_model(args)
try:
model.restore(use_ema=False, use_partial_loader=False)
model.reset() # for continous training, we reset some layers to random if necessary
except (NotFoundError, InvalidArgumentError) as e:
logger.debug(e)
logger.info('no available model, will train from scratch!')
model.train()
def evaluate(args):
model = build_model(args)
model.restore()
return model.evaluate()
def demo(args):
args.is_serving = True # set it to true to ignore data set loading
model = build_model(args)
model.restore()
sample_context = ''
sample_questions = ['What was Maria Curie the first female recipient of?',
'What year was Casimir Pulaski born in Warsaw?',
'Who was one of the most famous people born in Warsaw?',
'Who was Frédéric Chopin?',
'How old was Chopin when he moved to Warsaw with his family?']
sample_answers = ['Nobel Prize',
'1745',
'Maria Skłodowska-Curie',
'Famous musicians',
'seven months old']
for q, g in zip(sample_questions, sample_answers):
a = model.predict(sample_context, q) # real work is here!
logger.info('QUESTION: %s' % q)
logger.info('ANSWER: %s <- GOLD: %s' % (a, g))
================================================
FILE: app.py
================================================
import sys
from gpu_env import DEVICE_ID, MODEL_ID, CONFIG_SET
from utils.helper import set_logger, parse_args, get_args_cli
def run():
set_logger(model_id='%s:%s' % (DEVICE_ID, MODEL_ID))
followup_args = get_args_cli(sys.argv[3:]) if len(sys.argv) > 3 else None
args = parse_args(sys.argv[2] if len(sys.argv) > 2 else None, MODEL_ID, CONFIG_SET, followup_args)
getattr(__import__('api'), sys.argv[1])(args)
if __name__ == '__main__':
run()
================================================
FILE: base/base_io.py
================================================
import logging
from typing import List
from dataio_utils.helper import build_vocab
from gpu_env import APP_NAME, ModeKeys
class BaseDataIO:
def __init__(self, args):
self.args = args
self.logger = logging.getLogger(APP_NAME)
self.vocab = build_vocab(args.word_embedding_files)
self.pad_id = self.vocab.get_id(self.vocab.pad_token)
self.unk_id = self.vocab.get_id(self.vocab.unk_token)
self.start_token_id = self.vocab.get_id(self.vocab.start_token)
self.stop_token_id = self.vocab.get_id(self.vocab.stop_token)
self.datasets = {}
def next_batch(self, batch_size: int, mode: ModeKeys):
raise NotImplementedError
def load_data(self, file_paths: List[str], mode: ModeKeys):
raise NotImplementedError
def make_mini_batch(self, data):
raise NotImplementedError
================================================
FILE: base/base_model.py
================================================
import importlib
import json
import logging
import os
from collections import defaultdict
from math import ceil
import numpy as np
import tensorflow as tf
from ruamel.yaml import YAML
from gpu_env import ModeKeys, APP_NAME, SummaryType
from model_utils.helper import mblock, partial_restore, sample_element_from_var
class BaseModel:
def __init__(self, args):
self.logger = logging.getLogger(APP_NAME)
self.args = args
self.train_op = None
self.ema = None
self.is_var_ema = False
self.fetch_nodes = defaultdict(lambda: defaultdict(int))
self.monitored_non_vars = []
self.sess = None
self.loss = None
self._loss = {} # other auxiliary loss
self.embed_loaded = False
dataio = importlib.import_module(args.package_dataio)
self.data_io = dataio.DataIO(args)
self.vocab_size = self.data_io.vocab.size()
self.pretrain_vocab_size = self.data_io.vocab.pretraind_size()
self.initial_tokens_size = self.data_io.vocab.initial_tokens_size()
self.vocab_dim = self.data_io.vocab.embed_dim
try:
self.char_vocab_size = self.data_io.char_vocab.size()
self.char_vocab_dim = self.data_io.char_vocab.embed_dim
except:
self.char_vocab_size, self.char_vocab_dim = None, None
self._build_graph()
if self.args.run_mode == ModeKeys.TRAIN.value:
self._init_train_op()
self._init_tensorboard()
self._set_fetches()
self.init_session()
self.write_num_pars()
if self.args.run_mode == ModeKeys.TRAIN.value:
self.is_graph_valid()
def _build_graph(self):
raise NotImplementedError
def _set_learning_rate(self):
self.global_step = tf.get_variable('global_step', shape=[], dtype=tf.int32,
initializer=tf.constant_initializer(0),
trainable=False)
if self.args.learning_rate_strategy == 'FIXED':
self.lr = tf.minimum(self.args.learning_rate,
self.args.learning_rate / tf.log(999.) * tf.log(
tf.cast(self.global_step, tf.float32) + 1))
elif self.args.learning_rate_strategy == 'HALF_COSINE_MAX':
# from snapshot paper
t_m = tf.constant(ceil(self.args.learning_rate_reset_epoch * self.args.num_total_samples /
self.args.batch_size), dtype=tf.int32)
self.lr = (self.args.learning_rate / 2.0) * (
tf.cos(tf.constant(3.1415, tf.float32) *
tf.cast(tf.mod(self.global_step, t_m), tf.float32)
/ tf.cast(t_m, tf.float32)) + 1.0)
elif self.args.learning_rate_strategy == 'HALF_COSINE_ZERO':
# from snapshot paper
t_m = tf.constant(ceil(self.args.learning_rate_reset_epoch * self.args.num_total_samples /
self.args.batch_size), dtype=tf.int32)
self.lr = (self.args.learning_rate / 2.0) * (1.0 -
tf.cos(tf.constant(3.1415, tf.float32) *
tf.cast(tf.mod(self.global_step, t_m), tf.float32)
/ tf.cast(t_m, tf.float32)))
elif self.args.learning_rate_strategy == 'COSINE_ZERO':
t_m = tf.constant(ceil(self.args.learning_rate_reset_epoch * self.args.num_total_samples /
self.args.batch_size), dtype=tf.int32)
self.lr = (self.args.learning_rate / 2.0) * (1.0 -
tf.cos(tf.constant(2 * 3.1415, tf.float32) *
tf.cast(tf.mod(self.global_step, t_m), tf.float32)
/ tf.cast(t_m, tf.float32)))
elif self.args.learning_rate_strategy == 'COSINE_MAX':
t_m = tf.constant(ceil(self.args.learning_rate_reset_epoch * self.args.num_total_samples /
self.args.batch_size), dtype=tf.int32)
self.lr = (self.args.learning_rate / 2.0) * (1.0 +
tf.cos(tf.constant(2 * 3.1415, tf.float32) *
tf.cast(tf.mod(self.global_step, t_m), tf.float32)
/ tf.cast(t_m, tf.float32)))
elif self.args.learning_rate_strategy == 'COSINE_ZERO_DECAY':
t_m = tf.constant(ceil(self.args.learning_rate_reset_epoch * self.args.num_total_samples /
self.args.batch_size), dtype=tf.int32)
self.lr = (self.args.learning_rate /
tf.ceil(tf.cast(self.global_step, tf.float32) / tf.cast(t_m, tf.float32)) + 1) \
* (1.0 - tf.cos(tf.constant(2 * 3.1415, tf.float32) *
tf.cast(tf.mod(self.global_step, t_m), tf.float32)
/ tf.cast(t_m, tf.float32)))
elif self.args.learning_rate_strategy in ['CYCLE_LINEAR', 'CYCLE_SIN']:
self.lr = tf.get_variable('lr', shape=[], dtype=tf.float32,
initializer=tf.constant_initializer(self.args.learning_rate),
trainable=False)
else:
raise NotImplementedError
def _set_fetches(self):
self.add_fetch('loss', self.loss, [ModeKeys.TRAIN])
self.add_fetch('task_loss', self._loss, [ModeKeys.TRAIN])
self.add_fetch('_train_op', self.train_op, ModeKeys.TRAIN)
self.add_fetch('global_step', self.global_step, ModeKeys.TRAIN)
self.add_fetch('learning_rate', self.lr, ModeKeys.TRAIN)
if self.args.enable_tensorboard:
self.add_fetch('merged_summary', tf.summary.merge_all(), [ModeKeys.TRAIN])
def add_tfboard(self, name, value, mode):
if self.args.enable_tensorboard:
if isinstance(mode, list):
for m in mode:
self.add_tfboard(name, value, m)
elif mode == SummaryType.SCALAR:
tf.summary.scalar(name, value)
elif mode == SummaryType.HISTOGRAM:
tf.summary.histogram(name, value)
elif mode == SummaryType.SAMPLED:
self.monitored_non_vars.append(value)
else:
raise NotImplementedError
def add_fetch(self, name, value, mode):
if isinstance(mode, list):
for m in mode:
self.fetch_nodes[m][name] = value
elif isinstance(mode, ModeKeys):
self.fetch_nodes[mode][name] = value
else:
raise AttributeError('mode must be a list of ModeKeys or a ModeKeys!')
def get_fetch(self, name, mode):
return self.fetch_nodes[mode][name]
def init_session(self):
# session info
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.intra_op_parallelism_threads = 10
config.inter_op_parallelism_threads = 10
self.sess = tf.Session(config=config)
# initialize the model
self.sess.run(tf.global_variables_initializer())
self.saver = tf.train.Saver(max_to_keep=self.args.saver_max_to_keep)
self.tb_writer = tf.summary.FileWriter(self.args.summary_dir, self.sess.graph) if \
self.args.enable_tensorboard else None
def write_num_pars(self):
get_num_pars = lambda x: sum(list(map(np.prod, self.sess.run([tf.shape(v) for v in x]))))
total_num_pars = get_num_pars(tf.trainable_variables())
total_trained = 0
group_by_scope = defaultdict(list)
for v in tf.trainable_variables():
vscope = v.name.split('/')[0]
group_by_scope[vscope].append(v)
for k, v in group_by_scope.items():
n = get_num_pars(v)
if k in self.args.fixed_layers:
self.logger.info('%s%20s : %d' % ('|F|', k, n))
else:
self.logger.info('%s%20s : %d' % ('|V|', k, n))
total_trained += n
self.logger.info('trainable parameters: %d total: %d' % (total_trained, total_num_pars))
if 'num_parameters' not in self.args:
self.args.add_hparam('num_parameters', int(total_num_pars))
def is_graph_valid(self):
for v in [self.sess, self.saver, self.loss,
self.train_op, self.args, self.logger,
self.fetch_nodes, self.data_io]:
assert v is not None, '%s must be initialized' % v
self.logger.info('graph passed sanity check!')
def batch2feed_dict(self, batch, mode):
# add task-specific learning rate to batch
batch.update({'ph_dropout_keep_prob': self.args.dropout_keep_params if mode == ModeKeys.TRAIN else 1.0})
success_binds = []
ignored_binds = []
feed_dict = {}
allv = vars(self)
for k, v in batch.items():
if k in allv and isinstance(allv[k], tf.Tensor):
feed_dict[allv[k]] = v
success_binds.append(k)
else:
ignored_binds.append(k)
# self.logger.info('success bindings: %s' % success_binds)
# self.logger.warning('ignored bindings: %s' % ignored_binds)
return feed_dict
def run_sess_op(self, batch, mode):
feed_dict = self.batch2feed_dict(batch, mode)
return self.sess.run(self.fetch_nodes[mode], feed_dict)
def is_effective_epoch(self, metric, last_metric, best_metric):
is_effective = 0
for k, v in metric.items():
if v >= best_metric[k]:
is_effective += 1
if self.args.early_stop_metric in metric:
is_key_metric_effective = (metric[self.args.early_stop_metric] > best_metric[self.args.early_stop_metric])
else:
is_key_metric_effective = False
self.logger.info('%d/%d effective metrics! effective %s: %s' % (is_effective, len(last_metric),
self.args.early_stop_metric,
is_key_metric_effective))
return is_effective >= (len(best_metric) / 2) or is_key_metric_effective
def load_embedding(self):
raise NotImplementedError
def get_tfboard_vars(self):
raise NotImplementedError
def train(self, mode=ModeKeys.TRAIN):
"""
code example:
self.load_embedding()
loss_logger = xxxLoger()
for j in range(self.args.epoch_last, self.args.epoch_last + self.args.epoch_total + 1):
self.logger.info('start train epoch %d ...' % j)
try:
while True:
batch = self.data_io.next_batch(self.args.batch_size, mode)
fetches = self.run_sess_op(batch, mode)
loss_logger.record(fetches)
except EOFError:
self.logger.info('epoch %d is done!' % j)
metrics = self.evaluate(epoch=j)
cur_metric = metrics[self.args.metric_early_stop]
if not loss_logger.is_overfitted(cur_metric):
self.save(epoch=j)
else:
self.logger.info('early stop due to overfitting: %s %.4f -> %.4f' % (
self.args.metric_early_stop, loss_logger._last_metric, cur_metric))
break
"""
raise NotImplementedError
def predict(self, inputs, mode=ModeKeys.EVAL):
"""
inputs : dict
code example:
batch = self.data_io.single2batch(context, question)
fetches = self.run_sess_op(batch, mode)
s_id, e_id, raw = fetches['start_pos'][0], fetches['end_pos'][0], batch['raw'][0]
return ' '.join(raw['context'][s_id: (e_id + 1)])
"""
raise NotImplementedError
def evaluate(self, epoch=-1, mode=ModeKeys.EVAL):
"""
code example:
if epoch < 0:
epoch = self.args.epoch_best if self.args.epoch_best > 0 else self.args.epoch_last
preds = {}
try:
while True:
batch = self.data_io.next_batch(self.args.batch_size, mode)
fetches = self.run_sess_op(batch, mode)
# process fetches result
except EOFError:
self.logger.info('evaluation at epoch %d is done!' % epoch)
metric = self.save_eval_preds(preds, epoch=epoch)
return metric
"""
raise NotImplementedError
def save_eval_preds(self, preds, epoch=-1, mode=ModeKeys.EVAL):
"""
save eval result to files
code example:
result_file = get_filename(self.args, mode)
with open(result_file, 'w', encoding='utf8') as fp:
json.dump(preds, fp, ensure_ascii=False, sort_keys=True)
self.logger.info('prediction saved to %s' % result_file)
opt = namedtuple('OPT', 'id epoch pred ref out_file '
'na_prob_file na_prob_thresh out_image_dir verbose')(
self.args.model_id, epoch, result_file, self.args.dev_files[0],
self.args.out_metric_file, None, 1.0, None, True)
# evaluate predict result
# metrics = do_evaluation(opt)
# log metrics
for k, v in metrics.items():
if not k.startswith('_'):
if isinstance(v, int):
self.logger.info('%-20s: %d' % (k, v))
elif isinstance(v, float):
self.logger.info('%-20s: %.4f' % (k, v))
else:
self.logger.info('%-20s: %s' % (k, v))
self.logger.info('prediction metric is added to %s' % self.args.out_metric_file)
# save to loss file
if not os.path.isfile(self.args.loss_csv_file):
with open(self.args.loss_csv_file, 'w') as fp:
fp.write('epoch %s\n' % (' '.join(k for k in metrics.keys() if not k.startswith('_'))))
with open(self.args.loss_csv_file, 'a') as fp:
fp.write('%d ' % epoch)
for k, v in metrics.items():
if not k.startswith('_'):
if isinstance(v, int):
fp.write('%d ' % v)
elif isinstance(v, float):
fp.write('%.4f ' % v)
else:
fp.write('%s ' % v)
fp.write('\n')
return metrics
"""
raise NotImplementedError
def save_for_serving(self, epoch):
raise NotImplementedError
def save(self, epoch):
self.save_model(epoch)
self.args.epoch_last = epoch
self.save_args()
def save_model(self, epoch, save='save'):
self.saver.save(self.sess, os.path.join(self.args.save_dir, 'epoch%d' % epoch))
# tf.train.write_graph(self.sess.graph.as_graph_def(), FLAGS.save, "graph.pb")
if self.args.save_for_serving:
self.save_for_serving(epoch)
self.logger.info('model %sd in %s, at epoch %d' % (save, self.args.save_dir, epoch))
def save_args(self):
with open(os.path.join(self.args.save_dir, 'default.yaml'), 'w') as fp:
YAML().dump(json.loads(self.args.to_json()), fp)
def restore(self, epoch=-1, use_ema=True, use_partial_loader=False):
if epoch < 0:
epoch = self.args.epoch_best if self.args.epoch_best > 0 else self.args.epoch_last
model_file = os.path.join(self.args.save_dir, 'epoch%d' % epoch)
if use_partial_loader:
partial_restore(self.sess, model_file)
self.logger.info('partial restore variables without EMA!')
else:
if (self.ema is None) or (not use_ema):
self.saver.restore(self.sess, model_file)
self.is_var_ema = False
self.logger.info('restore variables without EMA!')
else:
variables_to_restore = self.ema.variables_to_restore()
saver = tf.train.Saver(variables_to_restore)
saver.restore(self.sess, model_file)
self.is_var_ema = True
self.logger.info('EMA variables are restored!')
self.logger.info('model restored from %s, at epoch %d' % (self.args.save_dir, epoch))
@mblock('Train_Op')
def _init_train_op(self):
"""
Selects the training algorithm and creates a train operation with it
"""
self._set_learning_rate()
all_params = [v for v in tf.trainable_variables() if v.name.split('/')[0] not in self.args.fixed_layers]
if not all_params:
self.train_op = tf.no_op()
self.ema = tf.train.ExponentialMovingAverage(decay=self.args.ema_decay)
self.logger.warning('No training variables! perform no_op while training')
return
with tf.name_scope('Regularization_Layer'):
if self.args.weight_decay > 0:
# ref. https://stats.stackexchange.com/questions/29130/
# difference-between-neural-net-weight-decay-and-learning-rate
with tf.variable_scope('l2_loss'):
l2_loss = tf.add_n([tf.nn.l2_loss(v) for v in all_params])
self.loss += self.args.weight_decay * l2_loss
if self.args.optim == 'ADAM':
optimizer = tf.train.AdamOptimizer(learning_rate=self.lr, beta1=0.8, beta2=0.999, epsilon=1e-7)
elif self.args.optim == 'SGD':
optimizer = tf.train.GradientDescentOptimizer(learning_rate=self.lr)
else:
optimizer = {
'RMSP': tf.train.RMSPropOptimizer,
'ADAGRAD': tf.train.AdagradOptimizer,
}[self.args.optim](learning_rate=self.args.learning_rate, epsilon=1e-8)
if self.args.gradient_clip:
# Calculate and clip gradients
gradients = tf.gradients(self.loss, all_params)
clipped_gradients, _ = tf.clip_by_global_norm(gradients, self.args.gradient_max_norm)
train_op = optimizer.apply_gradients(zip(clipped_gradients, all_params),
global_step=self.global_step)
else:
train_op = optimizer.minimize(self.loss, global_step=self.global_step)
if self.args.ema_decay > 0:
self.ema = tf.train.ExponentialMovingAverage(decay=self.args.ema_decay)
with tf.control_dependencies([train_op]):
train_op_ema = self.ema.apply(all_params)
self.train_op = train_op_ema
self.logger.info('EMA is added to training op!')
else:
self.train_op = train_op
self.all_trainable_vars = all_params
def _init_tensorboard(self):
if self.args.enable_tensorboard:
with tf.name_scope('Basic'):
self.add_tfboard('learning_rate', self.lr, SummaryType.SCALAR)
self.add_tfboard('loss', self.loss, SummaryType.SCALAR)
for k, v in self._loss.items():
self.add_tfboard('auxiliary_loss/%s' % k, v, SummaryType.SCALAR)
with tf.name_scope('Sampled_Vars'):
sampled = sample_element_from_var(self.all_trainable_vars)
for k, v in sampled.items():
self.add_tfboard(k, v, SummaryType.SCALAR)
with tf.name_scope('Sampled_NonVars'):
sampled = sample_element_from_var(self.monitored_non_vars)
for k, v in sampled.items():
self.add_tfboard(k, v, SummaryType.SCALAR)
other_vars = self.get_tfboard_vars()
if other_vars is not None:
for kk, vv in other_vars.items():
for k in vv:
self.add_tfboard('/'.join([k[1].name.split('/')[0], k[0]]), k[1], kk)
def reset(self):
reset_params = [v for v in tf.trainable_variables() if v.name.split('/')[0] in
self.args.reset_restored_layers]
if reset_params:
total_reset_num_par = sum(list(map(np.prod, self.sess.run([tf.shape(v) for v in reset_params]))))
self.logger.info('resetting %d parameters from %s layers' %
(total_reset_num_par, ','.join(s for s in
self.args.reset_restored_layers)))
self.sess.run(tf.variables_initializer(reset_params))
================================================
FILE: daanet/base.py
================================================
import json
import os
import tensorflow as tf
from base import base_model
from gpu_env import ModeKeys
from model_utils.helper import LossCounter, get_filename
from utils.eval_4.eval import compute_bleu_rouge
from utils.helper import build_model
# model controller
class RCBase(base_model.BaseModel):
def __init__(self, args):
super().__init__(args)
def train(self, mode=ModeKeys.TRAIN):
self.load_embedding()
loss_logger = LossCounter(self.fetch_nodes[mode]['task_loss'].keys(),
log_interval=self.args.log_interval,
batch_size=self.args.batch_size,
tb_writer=self.tb_writer)
pre_metric = {self.args.metric_early_stop: 0.0}
for j in range(self.args.epoch_last + 1, self.args.epoch_last + self.args.epoch_total + 1):
self.logger.info('start train epoch %d ...' % j)
try:
while True:
batch = self.data_io.next_batch(self.args.batch_size, mode)
fetches = self.run_sess_op(batch, mode)
loss_logger.record(fetches)
except EOFError:
self.save(j)
metric = self.restore_evaluate(self.args)
# if metric[self.args.metric_early_stop] < pre_metric[self.args.metric_early_stop]:
# self.logger.info('early stop in epoch %s' % j)
# break
pre_metric = metric
self.logger.info('epoch %d is done!' % j)
def restore_evaluate(self, args):
args.set_hparam('run_mode', ModeKeys.EVAL.value)
args.set_hparam('dropout_keep_prob', 1.0)
graph = tf.Graph()
with graph.as_default():
model = build_model(args, False)
model.restore()
return model.evaluate()
def evaluate(self, epoch=-1, mode=ModeKeys.EVAL):
if epoch < 0:
epoch = self.args.epoch_best if self.args.epoch_best > 0 else self.args.epoch_last
a_pred_dict = {}
a_ref_dict = {}
q_pred_dict = {}
q_ref_dict = {}
try:
while True:
batch = self.data_io.next_batch(self.args.batch_size, mode)
fetches = self.run_sess_op(batch, mode)
batch_a_pred_dict, batch_a_ref_dict, batch_q_pred_dict, batch_q_ref_dict = self.parse_result(batch,
fetches)
a_pred_dict.update(batch_a_pred_dict)
a_ref_dict.update(batch_a_ref_dict)
q_pred_dict.update(batch_q_pred_dict)
q_ref_dict.update(batch_q_ref_dict)
except EOFError:
qa_metric = compute_bleu_rouge(a_pred_dict, a_ref_dict)
qg_metric = compute_bleu_rouge(q_pred_dict, q_ref_dict)
qa_metric['type'] = 'qa'
qg_metric['type'] = 'qg'
self.save_metrics(qa_metric, epoch=epoch)
self.save_metrics(qg_metric, epoch=epoch)
self.save_eval_preds({"pred_dict": a_pred_dict, "ref_dict": a_ref_dict, 'type': "qa"}, epoch=epoch)
self.save_eval_preds({"pred_dict": q_pred_dict, "ref_dict": q_ref_dict, 'type': "qg"}, epoch=epoch)
self.logger.info('evaluation at epoch %d is done!' % epoch)
# self.logger.info(metric)
return qa_metric
def predict(self, inputs, mode=ModeKeys.EVAL):
raise NotImplementedError
def save_eval_preds(self, preds, epoch=-1, mode=ModeKeys.EVAL):
result_file = get_filename(self.args, mode)
with open(result_file, 'w', encoding='utf8') as fp:
json.dump(preds['pred_dict'], fp, ensure_ascii=False, sort_keys=True)
self.logger.info('sample preds')
sample_count = 20
p_type = preds['type']
for qid, pred in preds['pred_dict'].items():
if sample_count < 0:
break
ans = preds['ref_dict'][qid]
if p_type == 'qa':
self.logger.info("qid=%s" % qid)
self.logger.info("answer=%s" % ans)
self.logger.info("pred_answer=%s" % pred)
else:
self.logger.info("qid=%s" % qid)
self.logger.info("question=%s" % ans)
self.logger.info("pred_question=%s" % pred)
sample_count -= 1
def save_metrics(self, metrics, epoch=-1):
# log metrics
for k, v in metrics.items():
if not k.startswith('_'):
if isinstance(v, int):
self.logger.info('%-20s: %d' % (k, v))
elif isinstance(v, float):
self.logger.info('%-20s: %.4f' % (k, v))
else:
self.logger.info('%-20s: %s' % (k, v))
self.logger.info('prediction metric is added to %s' % self.args.out_metric_file)
# save to loss file
if not os.path.isfile(self.args.loss_csv_file):
with open(self.args.loss_csv_file, 'w') as fp:
fp.write('epoch %s\n' % (' '.join(k for k in metrics.keys() if not k.startswith('_'))))
with open(self.args.loss_csv_file, 'a') as fp:
fp.write('%d ' % epoch)
for k, v in metrics.items():
if not k.startswith('_'):
if isinstance(v, int):
fp.write('%d ' % v)
elif isinstance(v, float):
fp.write('%.4f ' % v)
else:
fp.write('%s ' % v)
fp.write('\n')
def parse_result(self, batch, fetches):
def get_pred_and_ture(logits, true_tokens, oovs):
pred = []
for tid in logits:
if tid != self.data_io.stop_token_id:
pred.append(self.data_io.vocab.get_token_with_oovs(tid, oovs))
else:
break
pred = " ".join(pred)
true = " ".join(true_tokens)
return pred, true
a_pred_dict = {}
a_ref_dict = {}
q_pred_dict = {}
q_ref_dict = {}
for qid, ans, questions, a_logits, q_logits, oov_tokens in zip(batch['qid'], batch['answer_tokens'],
batch['question_tokens'],
fetches['answer_decoder_logits'],
fetches['question_decoder_logits'],
batch['oovs']):
a_pred, a_true = get_pred_and_ture(a_logits, ans, oov_tokens)
a_pred_dict[qid] = [a_pred]
a_ref_dict[qid] = [a_true]
q_pred, q_true = get_pred_and_ture(q_logits, questions, oov_tokens)
q_pred_dict[qid] = [q_pred]
q_ref_dict[qid] = [q_true]
return a_pred_dict, a_ref_dict, q_pred_dict, q_ref_dict
================================================
FILE: daanet/basic.py
================================================
"""Sequence-to-Sequence with attention model.
"""
import tensorflow as tf
from tensorflow.python.layers import core as layers_core
from gpu_env import ModeKeys, SummaryType
from model_utils.helper import mblock
from nlp.encode_blocks import LSTM_encode, CNN_encode
from nlp.match_blocks import dot_attention, Transformer_match
from nlp.nn import get_var, highway_network, linear_logit
from nlp.seq2seq.pointer_generator import PointerGeneratorDecoder, \
PointerGeneratorBahdanauAttention, PointerGeneratorAttentionWrapper
from nlp.seq2seq.rnn import multi_rnn_cell
from .base import RCBase
# import numpy as np
# import seq2seq_lib
class RCCore(RCBase):
def __init__(self, args):
super().__init__(args)
def _build_graph(self):
self._placeholders()
self._shortcuts()
self.loss = -1
self._masks()
self._embed()
self._encode()
self._decode()
if self.args.run_mode == ModeKeys.TRAIN.value:
self._model_loss()
# if self.args.run_mode != 'decode':
@mblock('Input_Layer')
def _placeholders(self):
# passage token input
self.ph_passage = tf.placeholder(tf.int32, [None, None], name="passage")
self.ph_passage_chars = tf.placeholder(tf.int32, [None, None, None], name="passage_chars")
# question token input
self.ph_question = tf.placeholder(tf.int32, [None, None], name="question")
self.ph_question_chars = tf.placeholder(tf.int32, [None, None, None], name="question_chars")
# answer
self.ph_answer = tf.placeholder(tf.int32, [None, None], name="answer") # answer token input
self.ph_answer_chars = tf.placeholder(tf.int32, [None, None, None], name="answer_chars")
# length
self.ph_passage_length = tf.placeholder(tf.int32, [None], name="passage_length")
self.ph_question_length = tf.placeholder(tf.int32, [None], name="question_length")
self.ph_answer_length = tf.placeholder(tf.int32, [None], name="answer_length")
# max number of oov words in this batch
self.ph_max_oov_length = tf.placeholder(
tf.int32,
shape=[],
name='source_oov_words')
# input tokens using source oov words and vocab
self.ph_passage_extend_tokens = tf.placeholder(
tf.int32,
shape=[None, None],
name='source_extend_tokens')
self.ph_q_decode_input = tf.placeholder(
tf.int32, [None, None],
name="question_decode_input")
self.ph_q_decode_target = tf.placeholder(
tf.int32, [None, None],
name="question_decode_target")
self.ph_q_decode_length = tf.placeholder(
tf.int32, [None],
name="question_decode_length")
self.ph_a_decode_input = tf.placeholder(
tf.int32, [None, None],
name="answer_decode_input"
)
self.ph_a_decode_target = tf.placeholder(
tf.int32, [None, None],
name="answer_decode_target"
)
self.ph_a_decode_length = tf.placeholder(
tf.int32, [None],
name="answer_decode_length"
)
self.ph_a_start_label = tf.placeholder(
tf.int32, [None],
name="answer_start_id")
self.ph_a_end_label = tf.placeholder(tf.int32, [None], name="answer_end_id")
if self.args.use_answer_masks:
self.ph_answer_masks = tf.placeholder(
tf.int32, [None, None, None],
name="answer_masks")
self.ph_dropout_keep_prob = tf.placeholder(
tf.float32,
name='dropout_keep_prob')
self.ph_word_emb = tf.placeholder(tf.float32,
[self.pretrain_vocab_size, self.vocab_dim],
name='word_embed_mat')
self.ph_char_emb = tf.placeholder(tf.float32,
[self.char_vocab_size, self.char_vocab_dim],
name='char_embed_mat')
self.ph_tokenid_2_charsid = tf.placeholder(tf.int32,
[self.vocab_size, self.args.max_token_len],
name='ph_tokenid_2_charids')
self.ph_is_train = tf.placeholder(tf.bool, [])
def _shortcuts(self):
self.batch_size = tf.shape(self.ph_passage)[0]
self.max_p_len = tf.shape(self.ph_passage)[1]
self.max_q_len = tf.shape(self.ph_question)[1]
self.max_a_len = tf.shape(self.ph_answer)[1]
self.max_q_char_len = tf.shape(self.ph_question_chars)[2]
self.max_p_char_len = tf.shape(self.ph_passage_chars)[2]
self.max_a_char_len = tf.shape(self.ph_answer_chars)[2]
@mblock('Input_Layer/Mask')
def _masks(self):
self.a_mask = tf.sequence_mask(
self.ph_answer_length, tf.shape(self.ph_answer)[1],
dtype=tf.float32,
name='answer_mask')
self.p_mask = tf.sequence_mask(
self.ph_passage_length, tf.shape(self.ph_passage)[1],
dtype=tf.float32,
name='passage_mask')
self.q_mask = tf.sequence_mask(
self.ph_question_length, tf.shape(self.ph_question)[1],
dtype=tf.float32,
name='question_mask')
self.decode_q_mask = tf.sequence_mask(
self.ph_q_decode_length, tf.shape(self.ph_q_decode_target)[1],
dtype=tf.float32,
name='decode_question_mask')
self.decode_a_mask = tf.sequence_mask(
self.ph_a_decode_length, tf.shape(self.ph_a_decode_target)[1],
dtype=tf.float32,
name='decode_answer_mask'
)
@mblock('Embedding')
def _embed(self):
self.pretrained_word_embeddings = get_var(
'pretrained_word_embeddings',
shape=[self.pretrain_vocab_size, self.vocab_dim],
trainable=self.args.embed_trainable)
self.init_tokens_embeddings = tf.get_variable(name="init_tokens_embeddings",
shape=[self.initial_tokens_size - 1, self.vocab_dim],
initializer=tf.random_normal_initializer())
self.pad_tokens_embeddings = tf.get_variable(name="pad_tokens_embeddings",
shape=[1, self.vocab_dim],
initializer=tf.zeros_initializer(), trainable=False)
self.pretrain_word_embed_init = self.pretrained_word_embeddings.assign(self.ph_word_emb)
# "unk, start end, pad" in the end of embeddings
self.word_embeddings = tf.concat(
[self.pretrained_word_embeddings, self.init_tokens_embeddings, self.pad_tokens_embeddings], axis=0,
name='word_embeddings')
self.word_embeddings = tf.nn.dropout(self.word_embeddings, self.ph_dropout_keep_prob)
self.char_emb = get_var('char_embeddings', shape=[self.char_vocab_size, self.args.char_embed_size],
trainable=True)
self.tokenid_2_charsid_map = tf.get_variable('tokenid_2_charsid_map', dtype=tf.int32,
shape=[self.vocab_size, self.args.max_token_len],
trainable=False, initializer=tf.zeros_initializer())
self.tokenid_2_charsid_map_init = self.tokenid_2_charsid_map.assign(self.ph_tokenid_2_charsid)
with tf.variable_scope("embedding", reuse=tf.AUTO_REUSE) as scope:
self.embedding_scope = scope
def emb_ff(ids):
"""
:param ids: shape of ids is [batch] or [batch,L]
:return: embedding [batch, D] or [batch, L, D]
"""
num_of_dim = ids.get_shape().ndims
if num_of_dim == 1:
ids = tf.reshape(ids, [self.batch_size, 1])
condition = tf.less(ids, self.vocab_size)
ids = tf.where(condition, ids, tf.ones_like(ids) * self.data_io.unk_id)
max_axis1_len = tf.shape(ids)[-1]
char_ids = tf.nn.embedding_lookup(self.tokenid_2_charsid_map, ids) # B,L,max_token_len
max_axis2_len = tf.shape(char_ids)[-1]
token_emb = tf.nn.embedding_lookup(self.word_embeddings, ids)
char_emb = tf.reshape(tf.nn.embedding_lookup(self.char_emb, char_ids), # B,L,max_token_len,D_char
[self.batch_size * max_axis1_len, max_axis2_len, self.args.char_embed_size])
char_emb = CNN_encode(char_emb, filter_size=self.args.embed_filter_size,
num_filters=self.args.char_embed_size, scope=scope, reuse=tf.AUTO_REUSE)
char_emb = tf.reshape(tf.reduce_max(char_emb, axis=1),
[self.batch_size, max_axis1_len, self.args.char_embed_size])
concat_emb = tf.concat([token_emb, char_emb], axis=-1)
concat_emb = linear_logit(concat_emb, self.args.embedding_output_dim, scope=scope, reuse=tf.AUTO_REUSE)
highway_out = highway_network(concat_emb, self.args.highway_layer_num, scope=scope, reuse=tf.AUTO_REUSE)
if num_of_dim == 1:
return tf.squeeze(highway_out, axis=1) # B,D
else:
return highway_out # B,L,D
self.embedding_func = emb_ff
self.p_emb = self.embedding_func(self.ph_passage)
self.q_emb = self.embedding_func(self.ph_question)
self.a_emb = self.embedding_func(self.ph_answer)
@mblock('Encoding')
def _encode(self):
with tf.variable_scope('Passage_Encoder'):
self.p_encodes_rnn = LSTM_encode(self.p_emb, num_layers=self.args.encode_num_layers,
num_units=self.args.encode_num_units, direction=self.args.encode_direction,
scope='p_encode')
all_p_encodes = [self.p_encodes_rnn]
if self.args.self_attention_encode:
self.p_encodes_trans = Transformer_match(self.p_emb, self.p_emb, self.p_mask, self.p_mask,
self.args.self_attention_num_units,
scope='Passage_Encoder_trans')
all_p_encodes.append(self.p_encodes_trans)
if self.args.highway_encode:
self.p_encodes_highway = highway_network(self.p_emb, 1, num_units=self.args.highway_num_units,
scope='Passage_Encoder')
all_p_encodes.append(self.p_encodes_highway)
self.p_encodes = tf.concat(all_p_encodes, -1) * tf.expand_dims(self.p_mask, -1)
with tf.variable_scope("Question_Encoder") as q_encode_scope:
self.q_encodes_rnn = LSTM_encode(self.q_emb, num_layers=self.args.encode_num_layers,
num_units=self.args.encode_num_units, direction=self.args.encode_direction,
scope='q_encode')
all_q_encodes = [self.q_encodes_rnn]
if self.args.self_attention_encode:
self.q_encodes_trans = Transformer_match(self.q_emb, self.q_emb, self.q_mask, self.q_mask,
self.args.self_attention_num_units,
scope=q_encode_scope, layer_norm_scope='2', causality=True)
all_q_encodes.append(self.q_encodes_trans)
if self.args.highway_encode:
self.q_encodes_highway = highway_network(self.q_emb, num_layers=1,
num_units=self.args.highway_num_units, scope=q_encode_scope)
all_q_encodes.append(self.q_encodes_highway)
self.q_encodes = tf.concat(all_q_encodes, -1) * tf.expand_dims(self.q_mask, -1)
def question_encoder_f(inputs):
all_e = []
if self.args.share_transformer_encode:
fake_q_mask = tf.ones(shape=tf.shape(inputs)[:2], dtype=tf.float32)
all_e.append(
Transformer_match(inputs, inputs, fake_q_mask, fake_q_mask, self.args.self_attention_num_units,
scope=q_encode_scope, reuse=True, layer_norm_scope='2', causality=True))
if self.args.share_highway_encode:
all_e.append(highway_network(inputs, 1, num_units=self.args.highway_num_units, scope=q_encode_scope,
reuse=True))
all_e = tf.concat(all_e, axis=-1)
return all_e[:, -1, :] # B,D
self.question_encoder_func = question_encoder_f
with tf.variable_scope("Answer_Encoder") as a_encode_scope:
self.a_encodes_rnn = LSTM_encode(self.a_emb, num_layers=self.args.encode_num_layers,
num_units=self.args.encode_num_units, direction=self.args.encode_direction,
scope='a_encode')
all_a_encodes = [self.a_encodes_rnn]
if self.args.self_attention_encode:
self.a_encodes_trans = Transformer_match(self.a_emb, self.a_emb, self.a_mask, self.a_mask,
self.args.self_attention_num_units,
scope=a_encode_scope, layer_norm_scope='2', causality=True)
all_a_encodes.append(self.a_encodes_trans)
if self.args.highway_encode:
self.a_encodes_highway = highway_network(self.a_emb, 1, num_units=self.args.highway_num_units,
scope=a_encode_scope)
all_a_encodes.append(self.a_encodes_highway)
self.a_encodes = tf.concat(all_a_encodes, -1) * tf.expand_dims(self.a_mask, -1)
def answer_encoder_f(inputs):
all_e = []
if self.args.share_transformer_encode:
fake_q_mask = tf.ones(shape=tf.shape(inputs)[:2], dtype=tf.float32)
all_e.append(
Transformer_match(inputs, inputs, fake_q_mask, fake_q_mask, self.args.self_attention_num_units,
scope=a_encode_scope, reuse=True, layer_norm_scope='2',
causality=True))
if self.args.share_highway_encode:
all_e.append(highway_network(inputs, 1, num_units=self.args.highway_num_units, scope=a_encode_scope,
reuse=True))
enc_res = tf.concat(all_e, -1)
return enc_res[:, -1, :] # B,D
self.answer_encoder_func = answer_encoder_f
self.encode_dim = self.args.encode_num_units * 2
with tf.variable_scope("Question_Passage_Attention"):
self.qp_att = dot_attention(self.q_encodes, self.p_encodes,
mask=self.p_mask,
hidden_size=self.encode_dim,
keep_prob=self.args.dropout_keep_prob,
is_train=self.ph_is_train,
scope="question_attention") # B, LQ, D
with tf.variable_scope("Answer_Passage_Attention"):
self.ap_att = dot_attention(self.a_encodes, self.p_encodes,
mask=self.p_mask,
hidden_size=self.encode_dim,
keep_prob=self.args.dropout_keep_prob,
is_train=self.ph_is_train,
scope="question_attention") # B, LQ, D
with tf.variable_scope("Question_Passage_Encode"):
self.question_encoder_cell = multi_rnn_cell(
"LSTM", self.args.decoder_num_units,
is_train=self.args.run_mode == ModeKeys.TRAIN.value,
keep_prob=self.args.dropout_keep_prob,
num_layers=self.args.lstm_num_layers)
_, self.qp_encoder_state = tf.nn.dynamic_rnn(
cell=self.question_encoder_cell,
inputs=self.qp_att,
sequence_length=self.ph_question_length,
dtype=tf.float32)
self.qp_encoder_outputs = dot_attention(
self.p_encodes, self.qp_att,
mask=self.q_mask,
hidden_size=self.encode_dim,
keep_prob=self.args.dropout_keep_prob,
is_train=self.ph_is_train,
scope="passage_attention")
with tf.variable_scope("Answer_Passage_Encode"):
self.answer_encoder_cell = multi_rnn_cell(
"LSTM", self.args.decoder_num_units,
is_train=self.args.run_mode == ModeKeys.TRAIN.value,
keep_prob=self.args.dropout_keep_prob,
num_layers=self.args.lstm_num_layers)
_, self.ap_encoder_state = tf.nn.dynamic_rnn(
cell=self.answer_encoder_cell,
inputs=self.ap_att,
sequence_length=self.ph_answer_length,
dtype=tf.float32)
self.ap_encoder_outputs = dot_attention(
self.p_encodes, self.ap_att,
mask=self.a_mask,
hidden_size=self.encode_dim,
keep_prob=self.args.dropout_keep_prob,
is_train=self.ph_is_train,
scope="passage_attention")
self.encode_dim = self.args.final_projection_num
self.add_tfboard('passage_encode', self.p_encodes,
SummaryType.HISTOGRAM)
self.add_tfboard('question_encode', self.q_encodes,
SummaryType.HISTOGRAM)
self.add_tfboard('qp_encoder_state', self.qp_encoder_state,
SummaryType.HISTOGRAM)
self.add_tfboard('qp_encoder_outputs', self.qp_encoder_outputs,
SummaryType.HISTOGRAM)
self.add_tfboard('ap_encoder_state', self.ap_encoder_state,
SummaryType.HISTOGRAM)
self.add_tfboard('ap_encoder_outputs', self.ap_encoder_outputs,
SummaryType.HISTOGRAM)
@mblock('Decoder')
def _decode(self):
vsize = self.vocab_size
with tf.variable_scope("decoder_output"):
projection_layer = layers_core.Dense(units=vsize, use_bias=False) # use_bias
answer_decoder_cell, answer_initial_state = self.get_decode_cell_state(self.qp_encoder_outputs,
self.qp_encoder_state,
encoder_func=self.answer_encoder_func,
scope="answer_decoder_cell_state")
question_decoder_cell, question_initial_state = self.get_decode_cell_state(self.ap_encoder_outputs,
self.ap_encoder_state,
encoder_func=self.question_encoder_func,
scope="question_decoder_cell_state")
if self.args.run_mode == ModeKeys.TRAIN.value:
# answer decoder
answer_training_decoder = self.get_training_decoder(self.ph_a_decode_input, self.ph_a_decode_length,
answer_decoder_cell, answer_initial_state,
projection_layer,
embedding_func=self.embedding_func,
scope="answer_training_decoder")
question_training_decoder = self.get_training_decoder(self.ph_q_decode_input, self.ph_q_decode_length,
question_decoder_cell, question_initial_state,
projection_layer,
embedding_func=self.embedding_func,
scope="question_training_decoder")
# Training decoding
# answer
self.answer_decoder_outputs, self.answer_decoder_state, _ = tf.contrib.seq2seq.dynamic_decode(
decoder=answer_training_decoder,
impute_finished=True,
scope="answer_decoder")
self.answer_decoder_logits = self.answer_decoder_outputs.rnn_output
self.add_fetch('answer_decoder_logits', self.answer_decoder_logits, [ModeKeys.TRAIN])
# question
self.question_decoder_outputs, self.question_decoder_state, _ = tf.contrib.seq2seq.dynamic_decode(
decoder=question_training_decoder,
impute_finished=True,
scope="question_decoder")
self.question_decoder_logits = self.question_decoder_outputs.rnn_output
self.add_fetch('question_decoder_logits', self.question_decoder_logits, [ModeKeys.TRAIN])
else:
answer_inference_decoder = self.get_inference_decoder(answer_decoder_cell, answer_initial_state,
projection_layer,
embedding_func=self.embedding_func,
scope="answer_inference_decoder")
question_inference_decoder = self.get_inference_decoder(question_decoder_cell, question_initial_state,
projection_layer,
embedding_func=self.embedding_func,
scope="question_inference_decoder")
# Inference Decoding
# Answer
self.answer_decoder_outputs, _, _ = tf.contrib.seq2seq.dynamic_decode(
decoder=answer_inference_decoder,
maximum_iterations=100,
impute_finished=False,
scope="answer_decoder")
self.answer_decoder_logits = self.answer_decoder_outputs.sample_id # B, L
self.add_fetch('answer_decoder_logits', self.answer_decoder_logits, [ModeKeys.DECODE, ModeKeys.EVAL])
# Question
self.question_decoder_outputs, _, _ = tf.contrib.seq2seq.dynamic_decode(
decoder=question_inference_decoder,
maximum_iterations=100,
impute_finished=False,
scope="question_decoder")
self.question_decoder_logits = self.question_decoder_outputs.sample_id # B, L
self.add_fetch('question_decoder_logits', self.question_decoder_logits,
[ModeKeys.DECODE, ModeKeys.EVAL])
@mblock('Loss')
def _model_loss(self):
qa_loss = self._loss_calc_helper(self.ph_a_decode_length, self.ph_a_decode_target, self.decode_a_mask,
self.answer_decoder_logits, self.answer_decoder_state)
qg_loss = self._loss_calc_helper(self.ph_q_decode_length, self.ph_q_decode_target, self.decode_q_mask,
self.question_decoder_logits, self.question_decoder_state)
if self.args.task_name == 'qa':
self.loss = qa_loss
elif self.args.task_name == 'qg':
self.loss = qg_loss
else:
self.loss = qa_loss + qg_loss
self._loss['loss'] = self.loss
self._loss['qa_loss'] = qa_loss
self._loss['qg_loss'] = qg_loss
def load_embedding(self):
if self.args.embed_use_pretrained and not self.embed_loaded:
self.sess.run([self.pretrain_word_embed_init, self.tokenid_2_charsid_map_init],
feed_dict={
self.ph_word_emb: self.data_io.vocab.pretrained_embeddings,
self.ph_tokenid_2_charsid: self.data_io.tokenid2charsid,
})
self.embed_loaded = True
def get_tfboard_vars(self):
return {
SummaryType.HISTOGRAM: [
('answer_decoder_logits', self.answer_decoder_logits),
('question_decoder_logits', self.question_decoder_logits),
]
}
def get_training_decoder(self, decoder_inputs, decoder_length, decoder_cell, train_initial_state, projection_layer,
embedding_func, scope='training_decoder', reuse=False):
with tf.variable_scope(scope, reuse=reuse):
decoder_embedding_inputs = embedding_func(decoder_inputs)
training_helper = tf.contrib.seq2seq.TrainingHelper(decoder_embedding_inputs, decoder_length)
training_decoder = PointerGeneratorDecoder(
source_extend_tokens=self.ph_passage_extend_tokens,
source_oov_words=self.ph_max_oov_length,
coverage=self.args.use_coverage,
cell=decoder_cell,
helper=training_helper,
initial_state=train_initial_state,
output_layer=projection_layer)
return training_decoder
def get_inference_decoder(self, decoder_cell, train_initial_state, projection_layer, embedding_func,
scope='inference_decoder', reuse=False):
with tf.variable_scope(scope, reuse=reuse):
start_tokens = tf.tile(
tf.constant([self.data_io.start_token_id],
dtype=tf.int32), [self.batch_size])
# using greedying decoder right now
helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(
embedding=embedding_func,
start_tokens=start_tokens,
end_token=self.data_io.stop_token_id)
inference_decoder = PointerGeneratorDecoder(
source_extend_tokens=self.ph_passage_extend_tokens,
source_oov_words=self.ph_max_oov_length,
coverage=self.args.use_coverage,
cell=decoder_cell,
helper=helper,
initial_state=train_initial_state,
output_layer=projection_layer)
return inference_decoder
def get_decode_cell_state(self, encoder_output, encoder_state, encoder_func=None, scope='decode_cell_state',
reuse=False):
with tf.variable_scope(scope, reuse=reuse):
_cell = multi_rnn_cell(
"LSTM", self.args.decoder_num_units,
is_train=self.args.run_mode == ModeKeys.TRAIN.value,
keep_prob=self.args.dropout_keep_prob,
num_layers=self.args.lstm_num_layers)
enc_lengths = self.ph_passage_length
attention_mechanism = PointerGeneratorBahdanauAttention(
self.encode_dim, encoder_output,
memory_sequence_length=enc_lengths,
coverage=self.args.use_coverage)
decoder_cell = PointerGeneratorAttentionWrapper(
cell=_cell,
encoder_func=encoder_func,
attention_mechanism=attention_mechanism,
attention_layer_size=self.args.decoder_num_units,
alignment_history=True,
coverage=self.args.use_coverage)
initial_state = decoder_cell.zero_state(self.batch_size, tf.float32)
initial_state = initial_state.clone(cell_state=encoder_state)
return decoder_cell, initial_state
def _loss_calc_helper(self, decode_length, decode_target, decode_mask, decoder_logits, decoder_state):
max_dec_len = tf.reduce_max(
decode_length,
name="max_dec_len")
# targets: [batch_size x max_dec_len]
# this is important, because we may have padded endings
targets = tf.slice(decode_target, [
0, 0
], [-1, max_dec_len], 'targets')
i1, i2 = tf.meshgrid(tf.range(self.batch_size),
tf.range(max_dec_len),
indexing="ij")
indices = tf.stack((i1, i2, targets), axis=2)
probs = tf.gather_nd(decoder_logits, indices)
# To prevent padding tokens got 0 prob, and get inf when calculating log(p), we set the lower bound of prob
# I spent a lot of time here to debug the nan losses, inf * 0 = nan
probs = tf.where(tf.less_equal(probs, 0),
tf.ones_like(probs) * 1e-10, probs)
crossent = -tf.log(probs)
loss = tf.reduce_sum(
crossent * decode_mask) / tf.to_float(self.batch_size)
if self.args.use_coverage:
# we got all the alignments from last state
# shape is: batch * atten_len * max_len
alignment_history = tf.transpose(decoder_state.alignment_history.stack(), [1, 2, 0])
coverage_loss = tf.minimum(alignment_history, tf.cumsum(
alignment_history,
axis=2,
exclusive=True))
coverage_loss = self.args.coverage_loss_weight * \
tf.reduce_sum(coverage_loss / tf.to_float(self.batch_size))
loss += coverage_loss
return loss
================================================
FILE: dataio_utils/__init__.py
================================================
================================================
FILE: dataio_utils/flow_io.py
================================================
import json
from typing import List
import tensorflow as tf
from base import base_io
from gpu_env import ModeKeys
# dataio controller
class FlowDataIO(base_io.BaseDataIO):
def __init__(self, args):
super().__init__(args)
if args.is_serving:
self.logger.info('model is serving request, ignoring train & dev sets!')
else:
self.datasets = {
ModeKeys.TRAIN: self.load_data(self.args.train_files, ModeKeys.TRAIN),
ModeKeys.EVAL: self.load_data(self.args.dev_files, ModeKeys.EVAL),
}
if 'test_files' in self.args:
self.datasets[ModeKeys.TEST] = self.load_data(self.args.test_files, ModeKeys.TEST)
self.data_node = {}
def make_node(self, mode: ModeKeys):
for k, v in self.datasets.items():
if k == mode:
self.data_node[k] = v.make_one_shot_iterator
def next_batch(self, batch_size: int, mode: ModeKeys):
return self.data_node[mode]().get_next()
def load_data(self, file_paths: List[str], mode: ModeKeys):
dataset = tf.data.TextLineDataset(file_paths) \
.shuffle(5000) \
.filter(lambda x: tf.py_func(self._filter_invalid_seq, [x], tf.bool)) \
.map(lambda x: tf.py_func(self.make_sample, [x], tf.string)) \
.batch(self.args.batch_size) \
.map(lambda x: tf.py_func(self.make_mini_batch, [x], tf.string)) \
.prefetch(self.args.batch_size * 5)
self.logger.info('loading data for %s' % mode.name)
return dataset
def _dump_to_json(self, sample):
try:
r = json.dumps(sample).encode()
except Exception as e:
print(e)
r = json.dumps({}).encode()
return r
def _load_from_json(self, batch):
return [json.loads(str(d, encoding='utf8')) for d in batch]
def _filter_invalid_seq(self, line):
raise NotImplementedError
def make_sample(self, line, mode=ModeKeys.TRAIN):
raise NotImplementedError
def make_mini_batch(self, data, mode=ModeKeys.TRAIN):
raise NotImplementedError
def single2batch(self, context):
raise NotImplementedError
================================================
FILE: dataio_utils/full_load_io.py
================================================
import random
from base import base_io
from gpu_env import ModeKeys
class DataIO(base_io.BaseDataIO):
def __init__(self, args):
super().__init__(args)
if args.is_serving:
self.logger.info('model is serving request, ignoring train & dev sets!')
else:
self.datasets = {
ModeKeys.TRAIN: self.load_data(self.args.train_files, ModeKeys.TRAIN),
ModeKeys.EVAL: self.load_data(self.args.dev_files, ModeKeys.EVAL),
}
self.data_pointer = {k: 0 for k in self.datasets.keys()}
self.post_process_fn = {
ModeKeys.TRAIN: self.post_process_train,
ModeKeys.EVAL: self.post_process_eval,
}
self.reset_pointer(ModeKeys.TRAIN, shuffle=True)
def reset_pointer(self, mode, shuffle=False):
self.data_pointer[mode] = 0
if shuffle:
random.shuffle(self.datasets[mode])
self.logger.info('%s data is shuffled' % mode.name)
def next_batch(self, batch_size: int, mode: ModeKeys):
batch = []
dataset = self.datasets[mode]
start_pointer = self.data_pointer[mode]
batch_data = dataset[start_pointer: (start_pointer + batch_size)]
if len(batch_data) == 0:
self.reset_pointer(mode, shuffle=(mode == ModeKeys.TRAIN))
raise EOFError('%s data is exhausted' % mode.name)
for sample in batch_data:
batch.append(self.post_process_fn[mode](sample))
start_pointer += 1
self.data_pointer[mode] = start_pointer
return self.make_mini_batch(batch)
def post_process_train(self, sample):
"""
# this is important! otherwise you always overwrite the samples
new_sample = copy.deepcopy(sample)
# process new sample
# for example, shuffle dropout.
return new_sample
"""
raise NotImplementedError
def post_process_eval(self, sample):
return sample
================================================
FILE: dataio_utils/helper.py
================================================
import copy
import json
import random
import re
def _tokenize(x):
tokens = [v for v in re.findall(r"\w+|[^\w]", x, re.UNICODE) if len(v)] # fix last hanging space
token_shifts = []
char_token_map = []
c, j = 0, 0
for v in tokens:
if v.strip():
token_shifts.append(j)
j += 1
else:
token_shifts.append(-1)
char_token_map += [token_shifts[-1]] * len(v)
# remove empty word and extra space in tokens
tokens = [v.strip() for v in tokens if v.strip()]
assert len(tokens) == max(char_token_map) + 1, \
'num tokens must equal to the max char_token_map, but %d vs %d' % (len(tokens), max(char_token_map))
assert len(char_token_map) == len(x), \
'length of char_token_map must equal to original string, but %d vs %d' % (len(char_token_map), len(x))
return tokens, char_token_map
def _char_token_start_end(char_start, answer_text, char_token_map, full_tokens=None):
# to get the tokens use [start: (end+1)]
start_id = char_token_map[char_start]
end_id = char_token_map[char_start + len(answer_text) - 1]
if full_tokens:
ans = ' '.join(full_tokens[start_id: (end_id + 1)])
ans_gold = ' '.join(_tokenize(answer_text)[0])
assert ans == ans_gold, 'answers are not identical "%s" vs "%s"' % (ans, ans_gold)
return start_id, end_id
def _dump_to_json(sample):
return json.dumps(sample).encode()
def _load_from_json(batch):
return [json.loads(d) for d in batch]
def _parse_line(line):
return json.loads(line.strip())
def _do_padding(token_ids, token_lengths, pad_id):
pad_len = max(token_lengths)
return [(ids + [pad_id] * (pad_len - len(ids)))[: pad_len] for ids in token_ids]
def _do_char_padding(char_ids, token_lengths, pad_id, char_pad_id):
pad_token_len = max(token_lengths)
pad_char_len = max(len(xx) for x in char_ids for xx in x)
pad_empty_token = [char_pad_id] * pad_char_len
return [[(ids + [pad_id] * (pad_char_len - len(ids)))[: pad_char_len] for ids in x] +
[pad_empty_token] * (pad_token_len - len(x)) for x in char_ids]
def _dropout_word(x, unk_id, dropout_keep_prob):
return [v if random.random() < dropout_keep_prob else unk_id for v in x]
def _fast_copy(x, ignore_keys):
y = {}
for k, v in x.items():
if k in ignore_keys:
y[k] = v
else:
y[k] = copy.deepcopy(v)
return y
def build_vocab(embd_files):
from utils.vocab import Vocab
if embd_files[0].endswith('pickle'):
return Vocab.load_from_pickle(embd_files[0])
return Vocab(embd_files, lower=True)
================================================
FILE: default.yaml
================================================
# DO NOT CHANGE THIS FILE!!! This file contains all the default values
# If you need to run a new experiment, please create a separate YAML file!
# Example can be found in examples directory
path:
gpu100: &gpu100
model_dir: './save/'
summary_dir: './summaries/'
out_metric_file: './sample_data/sample.out.json'
default: &default
optim: ADAM
learning_rate: 0.001
weight_decay: 0
dropout_keep_params: 1.0
dropout_keep_word: 1.0
batch_size: 32
epoch_last: 0
epoch_best: -1
epoch_early_stop: 2
metric_early_stop: f1
gradient_clip: true
gradient_max_norm: 5.0
reset_restored_layers: [none]
fixed_layers: [none]
ema_decay: 0
suffix_output: ''
suffix_model_id: ''
log_interval: 100
save_for_serving: false
saver_max_to_keep: 2
learning_rate_reset_epoch: 2.0
learning_rate_strategy: FIXED
enable_tensorboard: true
package_dataio: dataio_utils.full_load_io
package_rccore: daanet.basic
run_mode: 1 # Train
lstm_num_layers: 1
encode_direction: bidirectional
use_answer_masks: true
use_oovs: true
encode_num_layers: 3
encode_num_units: 150
dropout_keep_prob: 1.0
decoder_num_units: 150
use_coverage: True
flip_ratio: 0.7
cell: bi-gru
coverage_loss_weight: 1.0
embed_use_pretrained: true
epoch_total: 100
max_token_len: 16
embed_filter_size: 3
char_embed_size: 200
embedding_output_dim: 300
encode_tcn_layers: 3
embed_trainable: false
highway_layer_num: 2
self_attention_encode: true
highway_encode: true
highway_num_units: 16
share_transformer_encode: true
share_highway_encode: true
final_projection_num: 512
self_attention_num_units: 16
default_gpu:
<<: *gpu100
<<: *default
================================================
FILE: gpu_env.py
================================================
import os
from datetime import datetime
from enum import Enum
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
IGNORE_PATTERNS = ('data', '*.pyc', 'CVS', '.git', 'tmp', '.svn', '__pycache__', '.gitignore', '.*.yaml')
MODEL_ID = datetime.now().strftime("%m%d-%H%M%S") + (
os.environ['suffix_model_id'] if 'suffix_model_id' in os.environ else '')
APP_NAME = 'mrc'
class SummaryType(Enum):
SCALAR = 1
HISTOGRAM = 2
SAMPLED = 3
class ModeKeys(Enum):
TRAIN = 1
EVAL = 2
INFER = 3
INTERACT = 4
DECODE = 5
TEST = 6
try:
import GPUtil
# GPUtil.showUtilization()
DEVICE_ID_LIST = GPUtil.getFirstAvailable(order='random', maxMemory=0.1, maxLoad=0.1)
DEVICE_ID = DEVICE_ID_LIST[0] # grab first element from list
os.environ["CUDA_VISIBLE_DEVICES"] = str(DEVICE_ID)
CONFIG_SET = 'default_gpu'
except FileNotFoundError:
print('no gpu found!')
DEVICE_ID = 'x'
CONFIG_SET = 'default'
except RuntimeError:
print('all gpus are occupied!')
DEVICE_ID = '?'
CONFIG_SET = 'default_gpu'
print('use config: %s' % CONFIG_SET)
================================================
FILE: grid.yaml
================================================
common:
retry_delay: 300 # seconds, interval between checking available GPU
wait_until_next: 60 # seconds, time delay between two jobs
entrypoint: train
model_params: &mp
train_files:
- ['./sample_data/sample.json']
dev_files:
- ['./sample_data/sample.json']
test_files:
- ['./sample_data/sample.json']
word_embedding_files:
- ['./sample_data/sample.embed.txt']
char_embedding_files:
- ['./sample_data/sample.charembed.txt']
result_dir: ['./sample_data/result']
out_metric_file: ['./sample_data/sample.out.json']
daanet:
<<: *mp
task_name: ['qag']
monoqa:
<<: *mp
task_name: ['qa']
monoqg:
<<: *mp
task_name: ['qg']
================================================
FILE: grid_search.py
================================================
import itertools
import os
import sys
from ruamel.yaml import YAML
from utils.helper import set_logger, fill_gpu_jobs, get_tmp_yaml
def run():
logger = set_logger()
with open('grid.yaml') as fp:
settings = YAML().load(fp)
test_set = sys.argv[1:] if len(sys.argv) > 1 else settings['common']['config']
all_args = [settings[t] for t in test_set]
entrypoint = settings['common']['entrypoint']
with open('default.yaml') as fp:
settings_default = YAML().load(fp)
os.environ['suffix_model_id'] = settings_default['default']['suffix_model_id']
cmd = ' '.join(['python app.py', entrypoint, '%s'])
all_jobs = []
for all_arg in all_args:
k, v = zip(*[(k, v) for k, v in all_arg.items()])
all_jobs += [{kk: pp for kk, pp in zip(k, p)} for p in itertools.product(*v)]
while all_jobs:
all_jobs = fill_gpu_jobs(all_jobs, logger,
job_parser=lambda x: cmd % get_tmp_yaml(x,
(os.environ['suffix_model_id'] if
os.environ['suffix_model_id'] else
'+'.join(test_set)) + '-'),
wait_until_next=settings['common']['wait_until_next'],
retry_delay=settings['common']['retry_delay'],
do_shuffle=True)
logger.info('all jobs are done!')
if __name__ == '__main__':
run()
================================================
FILE: model_utils/__init__.py
================================================
================================================
FILE: model_utils/helper.py
================================================
import json
import logging
import os
import time
import tensorflow as tf
from gpu_env import APP_NAME, ModeKeys
def sample_element_from_var(all_var):
result = {}
for v in all_var:
try:
v_rank = len(v.get_shape())
v_ele1, v_ele2 = v, v
for j in range(v_rank):
v_ele1, v_ele2 = v_ele1[0], v_ele2[-1]
result['sampled/1_%s' + v.name], result['sampled/2_%s' + v.name] = v_ele1, v_ele2
except:
pass
return result
def partial_restore(session, save_file):
reader = tf.train.NewCheckpointReader(save_file)
saved_shapes = reader.get_variable_to_shape_map()
var_names = sorted([(var.name, var.name.split(':')[0]) for var in tf.global_variables()
if var.name.split(':')[0] in saved_shapes])
restore_vars = []
name2var = dict(zip(map(lambda x: x.name.split(':')[0], tf.global_variables()), tf.global_variables()))
with tf.variable_scope('', reuse=True):
for var_name, saved_var_name in var_names:
curr_var = name2var[saved_var_name]
var_shape = curr_var.get_shape().as_list()
if var_shape == saved_shapes[saved_var_name]:
restore_vars.append(curr_var)
saver = tf.train.Saver(restore_vars)
saver.restore(session, save_file)
def mblock(scope_name, device_name=None, reuse=None):
def f2(f):
def f2_v(self, *args, **kwargs):
start_t = time.time()
if device_name:
with tf.device(device_name), tf.variable_scope(scope_name, reuse=reuse):
f(self, *args, **kwargs)
else:
with tf.variable_scope(scope_name, reuse=reuse):
f(self, *args, **kwargs)
self.logger.info('%s is build in %.4f secs' % (scope_name, time.time() - start_t))
return f2_v
return f2
def get_filename(args, mode: ModeKeys):
return os.path.join(args.result_dir,
'-'.join(v for v in [args.model_id, mode.name, args.suffix_output] if
v.strip()) + '.json')
def write_dev_json(f, pred_answers):
with open(f, 'w', encoding='utf8') as fp:
for p in pred_answers:
fp.write(json.dumps(p, ensure_ascii=False) + '\n')
class LossCounter:
def __init__(self, task_names, log_interval, batch_size, tb_writer):
self._task_names = task_names
self._log_interval = log_interval
self._start_t = time.time()
self._num_step = 1
self._batch_size = batch_size
self._n_steps_loss = 0
self._n_batch_task_loss = {k: 0.0 for k in self._task_names}
self._reset_step_loss()
self._tb_writer = tb_writer
self._logger = logging.getLogger(APP_NAME)
self._last_metric = 0
def _reset_step_loss(self):
self._last_n_steps_loss = self._n_steps_loss / self._log_interval
self._n_steps_loss = 0
self._n_batch_task_loss = {k: 0.0 for k in self._task_names}
def record(self, fetches):
self._num_step += 1
self._n_steps_loss += fetches['loss']
for k, v in fetches['task_loss'].items():
self._n_batch_task_loss[k] += v
if self._trigger():
self.show_status()
self._reset_step_loss()
if self._tb_writer:
self._tb_writer.add_summary(fetches['merged_summary'], self._num_step)
def is_overfitted(self, metric):
if metric - self._last_metric < 1e-6:
return True
else:
self._last_metric = metric
return False
def _trigger(self):
return (self._log_interval > 0) and (self._num_step % self._log_interval == 0)
def show_status(self):
cur_loss = self._n_steps_loss / self._log_interval
self._logger.info('%-4d->%-4d L: %.3f -> %.3f %d/s %s' % (
self._num_step - self._log_interval + 1, self._num_step,
self._last_n_steps_loss, cur_loss,
round(self._num_step * self._batch_size / (time.time() - self._start_t)),
self._get_multitask_loss_str(self._n_batch_task_loss, normalizer=self._log_interval)
))
@staticmethod
def _get_multitask_loss_str(loss_dict, normalizer=1.0, show_key=True, show_value=True):
if show_key and not show_value:
to_str = lambda x, y: '%s' % x
elif show_key and show_value:
to_str = lambda x, y: '%s: %.3f' % (x, y)
elif not show_key and show_value:
to_str = lambda x, y: '%.3f' % y
else:
to_str = lambda x, y: ''
return ' '.join([to_str(k, v / normalizer) for k, v in loss_dict.items()])
================================================
FILE: nlp/__init__.py
================================================
================================================
FILE: nlp/encode_blocks.py
================================================
import tensorflow as tf
from nlp.nn import initializer, regularizer, spatial_dropout, get_lstm_init_state, layer_norm
def LSTM_encode(seqs, scope='lstm_encode_block', reuse=None, **kwargs):
with tf.variable_scope(scope, reuse=reuse):
batch_size = tf.shape(seqs)[0]
_seqs = tf.transpose(seqs, [1, 0, 2]) # to T, B, D
lstm = tf.contrib.cudnn_rnn.CudnnLSTM(**kwargs)
init_state = get_lstm_init_state(batch_size, **kwargs)
output = lstm(_seqs, init_state)[0] # 2nd return is state, ignore
return tf.transpose(output, [1, 0, 2]) # back to B, T, D
def TCN_encode(seqs, num_layers, normalize_output=True, scope='tcn_encode_block', reuse=None,
layer_norm_scope='layer_norm', **kwargs):
with tf.variable_scope(scope, reuse=reuse):
outputs = [seqs]
for i in range(num_layers):
dilation_size = 2 ** i
out = Res_DualCNN_encode(outputs[-1], dilation=dilation_size, scope='res_biconv_%d' % i, **kwargs)
outputs.append(out)
result = outputs[-1]
if normalize_output:
result = layer_norm(result, scope=layer_norm_scope, reuse=reuse)
return result
def Res_DualCNN_encode(seqs, use_spatial_dropout=True, scope='res_biconv_block', reuse=None, **kwargs):
input_dim = seqs.get_shape().as_list()[-1]
with tf.variable_scope(scope, reuse=reuse):
out1 = CNN_encode(seqs, scope='first_conv1d', **kwargs)
if use_spatial_dropout:
out1 = spatial_dropout(out1)
out2 = CNN_encode(out1, scope='second_conv1d', **kwargs)
if use_spatial_dropout:
out2 = CNN_encode(out2)
output_dim = out2.get_shape().as_list()[-1]
if input_dim != output_dim:
res_x = tf.layers.conv1d(seqs,
filters=output_dim,
kernel_size=1,
activation=None,
name='res_1x1conv')
else:
res_x = seqs
return tf.nn.relu(out2 + res_x)
def CNN_encode(seqs, filter_size=3, dilation=1,
num_filters=None, direction='forward', act_fn=tf.nn.relu,
scope=None,
reuse=None, **kwargs):
input_dim = seqs.get_shape().as_list()[-1]
num_filters = num_filters if num_filters else input_dim
# add causality: shift the whole seq to the right
padding = (filter_size - 1) * dilation
if direction == 'forward':
pad_seqs = tf.pad(seqs, [[0, 0], [padding, 0], [0, 0]])
padding_scheme = 'VALID'
elif direction == 'backward':
pad_seqs = tf.pad(seqs, [[0, 0], [0, padding], [0, 0]])
padding_scheme = 'VALID'
elif direction == 'none':
pad_seqs = seqs # no padding, must set to SAME so that we have same length
padding_scheme = 'SAME'
else:
raise NotImplementedError
with tf.variable_scope(scope or 'causal_conv_%s_%s' % (filter_size, direction), reuse=reuse):
return tf.layers.conv1d(
pad_seqs,
num_filters,
filter_size,
activation=act_fn,
padding=padding_scheme,
dilation_rate=dilation,
kernel_initializer=initializer,
bias_initializer=tf.zeros_initializer(),
kernel_regularizer=regularizer)
================================================
FILE: nlp/match_blocks.py
================================================
import tensorflow as tf
from nlp.nn import linear_logit, layer_norm
from nlp.seq2seq.common import dropout, softmax_mask
def Attention_match(context, query, context_mask, query_mask,
num_units=None,
scope='attention_match_block', reuse=None, **kwargs):
with tf.variable_scope(scope, reuse=reuse):
if num_units is None:
num_units = context.get_shape().as_list()[-1]
score = tf.matmul(context, query, transpose_b=True)
else:
score = tf.matmul(linear_logit(context, num_units, scope='context2hidden'),
linear_logit(query, num_units, scope='query2hidden'),
transpose_b=True)
mask = tf.matmul(tf.expand_dims(context_mask, -1), tf.expand_dims(query_mask, -1), transpose_b=True)
paddings = tf.ones_like(mask) * (-2 ** 32 + 1)
masked_score = tf.where(tf.equal(mask, 0), paddings, score / (num_units ** 0.5)) # B, Lc, Lq
query2context_score = tf.reduce_sum(masked_score, axis=2, keepdims=True) # B, Lc, 1
match_score = tf.nn.softmax(query2context_score, axis=1) # (B, Lc, 1)
return context * match_score
def Transformer_match(context,
query,
context_mask,
query_mask,
num_units=None,
num_heads=8,
dropout_keep_rate=1.0,
causality=True,
scope='MultiHead_Attention_Block',
reuse=None,
residual=False,
normalize_output=True,
**kwargs):
"""Applies multihead attention.
Args:
context: A 3d tensor with shape of [N, T_q, C_q].
query: A 3d tensor with shape of [N, T_k, C_k].
num_units: A scalar. Attention size.
dropout_rate: A floating point number.
is_training: Boolean. Controller of mechanism for dropout.
causality: Boolean. If true, units that reference the future are masked.
num_heads: An int. Number of heads.
scope: Optional scope for `variable_scope`.
reuse: Boolean, whether to reuse the weights of a previous layer
by the same name.
Returns
A 3d tensor with shape of (N, T_q, C)
"""
if num_units is None or residual:
num_units = context.get_shape().as_list()[-1]
with tf.variable_scope(scope, reuse=reuse):
# Set the fall back option for num_units
# Linear projections
Q = tf.layers.dense(context, num_units, activation=tf.nn.relu) # (N, T_q, C)
K = tf.layers.dense(query, num_units, activation=tf.nn.relu) # (N, T_k, C)
V = tf.layers.dense(query, num_units, activation=tf.nn.relu) # (N, T_k, C)
# Split and concat
Q_ = tf.concat(tf.split(Q, num_heads, axis=2), axis=0) # (h*N, T_q, C/h)
K_ = tf.concat(tf.split(K, num_heads, axis=2), axis=0) # (h*N, T_k, C/h)
V_ = tf.concat(tf.split(V, num_heads, axis=2), axis=0) # (h*N, T_k, C/h)
# Multiplication
outputs = tf.matmul(Q_, tf.transpose(K_, [0, 2, 1])) # (h*N, T_q, T_k)
# Scale
outputs = outputs / (K_.get_shape().as_list()[-1] ** 0.5)
# Key Masking, aka query
mask1 = tf.tile(query_mask, [num_heads, 1]) # (h*N, T_k)
mask1 = tf.tile(tf.expand_dims(mask1, 1), [1, tf.shape(context)[1], 1]) # (h*N, T_q, T_k)
paddings = tf.ones_like(outputs) * (-2 ** 32 + 1)
outputs = tf.where(tf.equal(mask1, 0), paddings, outputs) # (h*N, T_q, T_k)
# Causality = Future blinding
if causality:
diag_vals = tf.ones_like(outputs[0, :, :]) # (T_q, T_k)
tril = tf.contrib.linalg.LinearOperatorLowerTriangular(diag_vals).to_dense() # (T_q, T_k)
masks = tf.tile(tf.expand_dims(tril, 0), [tf.shape(outputs)[0], 1, 1]) # (h*N, T_q, T_k)
paddings = tf.ones_like(masks) * (-2 ** 32 + 1)
outputs = tf.where(tf.equal(masks, 0), paddings, outputs) # (h*N, T_q, T_k)
# Activation
outputs = tf.nn.softmax(outputs) # (h*N, T_q, T_k)
# Query Masking aka, context
mask2 = tf.tile(context_mask, [num_heads, 1]) # (h*N, T_q)
mask2 = tf.tile(tf.expand_dims(mask2, -1), [1, 1, tf.shape(query)[1]]) # (h*N, T_q, T_k)
outputs *= mask2 # (h*N, T_q, T_k)
# Dropouts
outputs = tf.nn.dropout(outputs, keep_prob=dropout_keep_rate)
# Weighted sum
outputs = tf.matmul(outputs, V_) # ( h*N, T_q, C/h)
# Restore shape
outputs = tf.concat(tf.split(outputs, num_heads, axis=0), axis=2) # (N, T_q, C)
if residual:
# Residual connection
outputs += context
if normalize_output:
# Normalize
outputs = layer_norm(outputs) # (N, T_q, C)
return outputs
def BiDaf_match(a, b, a_mask, b_mask, residual, scope=None, reuse=None, **kwargs):
# context: [batch, l, d]
# question: [batch, l2, d]
with tf.variable_scope(scope, reuse=reuse):
n_a = tf.tile(tf.expand_dims(a, 2), [1, 1, tf.shape(b)[1], 1])
n_b = tf.tile(tf.expand_dims(b, 1), [1, tf.shape(a)[1], 1, 1])
n_ab = n_a * n_b
n_abab = tf.concat([n_ab, n_a, n_b], -1)
kernel = tf.squeeze(tf.layers.dense(n_abab, units=1), -1)
a_mask = tf.expand_dims(a_mask, -1)
b_mask = tf.expand_dims(b_mask, -1)
kernel_mask = tf.matmul(a_mask, b_mask, transpose_b=True)
kernel += (kernel_mask - 1) * 1e5
con_query = tf.matmul(tf.nn.softmax(kernel, 1), b)
con_query = con_query * a_mask
query_con = tf.matmul(tf.transpose(
tf.reduce_max(kernel, 2, keepdims=True), [0, 2, 1]), a * a_mask)
query_con = tf.tile(query_con, [1, tf.shape(a)[1], 1])
if residual:
return tf.concat([a * query_con, a * con_query, a, query_con], 2)
else:
return tf.concat([a * query_con, a * con_query, a, query_con], 2)
def dot_attention(inputs, memory, mask, hidden_size, keep_prob=1.0, is_train=None, scope=None):
with tf.variable_scope(scope or 'dot_attention'):
d_inputs = dropout(inputs, keep_prob=keep_prob, is_train=is_train)
d_memory = dropout(memory, keep_prob=keep_prob, is_train=is_train)
JX = tf.shape(inputs)[1]
with tf.variable_scope("attention"):
inputs_ = tf.nn.relu(
tf.layers.dense(d_inputs, hidden_size, use_bias=False, name="inputs"))
memory_ = tf.nn.relu(
tf.layers.dense(d_memory, hidden_size, use_bias=False, name="memory"))
outputs = tf.matmul(inputs_, tf.transpose(
memory_, [0, 2, 1])) / (hidden_size ** 0.5)
mask = tf.tile(tf.expand_dims(mask, axis=1), [1, JX, 1])
logits = tf.nn.softmax(softmax_mask(outputs, mask))
outputs = tf.matmul(logits, memory)
res = tf.concat([inputs, outputs], axis=2)
with tf.variable_scope("gate"):
dim = res.get_shape().as_list()[-1]
d_res = dropout(res, keep_prob=keep_prob, is_train=is_train)
gate = tf.nn.sigmoid(tf.layers.dense(d_res, dim, use_bias=False))
return res * gate
================================================
FILE: nlp/nn.py
================================================
import tensorflow as tf
initializer = tf.contrib.layers.variance_scaling_initializer(factor=1.0,
mode='FAN_AVG',
uniform=True,
dtype=tf.float32)
initializer_relu = tf.contrib.layers.variance_scaling_initializer(factor=2.0,
mode='FAN_IN',
uniform=False,
dtype=tf.float32)
regularizer = tf.contrib.layers.l2_regularizer(scale=3e-7)
def minus_mask(x, mask, offset=1e30):
"""
masking by subtract a very large number
:param x: sequence data in the shape of [B, L, D]
:param mask: 0-1 mask in the shape of [B, L]
:param offset: very large negative number
:return: masked x
"""
return x - tf.expand_dims(1.0 - mask, axis=-1) * offset
def mul_mask(x, mask):
"""
masking by multiply zero
:param x: sequence data in the shape of [B, L, D]
:param mask: 0-1 mask in the shape of [B, L]
:return: masked x
"""
return x * tf.expand_dims(mask, axis=-1)
def masked_reduce_mean(x, mask):
return tf.reduce_sum(mul_mask(x, mask), axis=1) / tf.reduce_sum(mask, axis=1, keepdims=True)
def masked_reduce_max(x, mask):
return tf.reduce_max(minus_mask(x, mask), axis=1)
def weighted_sparse_softmax_cross_entropy(labels, preds, weights):
"""
computing sparse softmax cross entropy by weighting differently on classes
:param labels: sparse label in the shape of [B], size of label is L
:param preds: logit in the shape of [B, L]
:param weights: weight in the shape of [L]
:return: weighted sparse softmax cross entropy in the shape of [B]
"""
return tf.losses.sparse_softmax_cross_entropy(labels,
logits=preds,
weights=get_bounded_class_weight(labels, weights))
def get_bounded_class_weight(labels, weights, ub=None):
if weights is None:
return 1.0
else:
w = tf.gather(weights, labels)
w = w / tf.reduce_min(w)
w = tf.clip_by_value(1.0 + tf.log1p(w),
clip_value_min=1.0,
clip_value_max=ub if ub is not None else tf.cast(tf.shape(weights)[0], tf.float32) / 2.0)
return w
def weighted_smooth_softmax_cross_entropy(labels, num_labels, preds, weights,
epsilon=0.1):
"""
computing smoothed softmax cross entropy by weighting differently on classes
:param epsilon: smoothing factor
:param num_labels: maximum number of labels
:param labels: sparse label in the shape of [B], size of label is L
:param preds: logit in the shape of [B, L]
:param weights: weight in the shape of [L]
:return: weighted sparse softmax cross entropy in the shape of [B]
"""
return tf.losses.softmax_cross_entropy(tf.one_hot(labels, num_labels),
logits=preds,
label_smoothing=epsilon,
weights=get_bounded_class_weight(labels, weights))
def get_var(name, shape, dtype=tf.float32,
initializer_fn=initializer,
regularizer_fn=regularizer, **kwargs):
return tf.get_variable(name, shape,
initializer=initializer_fn,
dtype=dtype,
regularizer=regularizer_fn, **kwargs)
def layer_norm(inputs,
epsilon=1e-8,
scope=None,
reuse=None):
"""Applies layer normalization.
Args:
inputs: A tensor with 2 or more dimensions, where the first dimension has
`batch_size`.
epsilon: A floating number. A very small number for preventing ZeroDivision Error.
scope: Optional scope for `variable_scope`.
reuse: Boolean, whether to reuse the weights of a previous layer
by the same name.
Returns:
A tensor with the same shape and data dtype as `inputs`.
"""
with tf.variable_scope(scope or 'Layer_Normalize', reuse=reuse):
inputs_shape = inputs.get_shape()
params_shape = inputs_shape[-1:]
mean, variance = tf.nn.moments(inputs, [-1], keep_dims=True)
beta = tf.get_variable("beta", shape=params_shape, initializer=tf.constant_initializer(0.0))
gamma = tf.get_variable("gama", shape=params_shape, initializer=tf.constant_initializer(1.0))
normalized = (inputs - mean) / ((variance + epsilon) ** .5)
outputs = gamma * normalized + beta
return outputs
def linear_logit(x, units, act_fn=None, dropout_keep=1., use_layer_norm=False, scope=None, reuse=None, **kwargs):
with tf.variable_scope(scope or 'linear_logit', reuse=reuse):
logit = tf.layers.dense(x, units=units, activation=act_fn,
kernel_initializer=initializer,
kernel_regularizer=regularizer)
# do dropout
logit = tf.nn.dropout(logit, keep_prob=dropout_keep)
if use_layer_norm:
logit = tf.contrib.layers.layer_norm(logit)
return logit
def bilinear_logit(x, units, act_fn=None,
first_units=256,
first_act_fn=tf.nn.relu, scope=None, **kwargs):
with tf.variable_scope(scope or 'bilinear_logit'):
first = linear_logit(x, first_units, act_fn=first_act_fn, scope='first', **kwargs)
return linear_logit(first, units, scope='second', act_fn=act_fn, **kwargs)
def label_smoothing(inputs, epsilon=0.1):
"""Applies label smoothing. See https://arxiv.org/abs/1512.00567.
Args:
inputs: A 3d tensor with shape of [N, T, V], where V is the number of vocabulary.
epsilon: Smoothing rate.
For example,
```
import tensorflow as tf
inputs = tf.convert_to_tensor([[[0, 0, 1],
[0, 1, 0],
[1, 0, 0]],
[[1, 0, 0],
[1, 0, 0],
[0, 1, 0]]], tf.float32)
outputs = label_smoothing(inputs)
with tf.Session() as sess:
print(sess.run([outputs]))
>>
[array([[[ 0.03333334, 0.03333334, 0.93333334],
[ 0.03333334, 0.93333334, 0.03333334],
[ 0.93333334, 0.03333334, 0.03333334]],
[[ 0.93333334, 0.03333334, 0.03333334],
[ 0.93333334, 0.03333334, 0.03333334],
[ 0.03333334, 0.93333334, 0.03333334]]], dtype=float32)]
```
"""
K = inputs.get_shape().as_list()[-1] # number of channels
return ((1 - epsilon) * inputs) + (epsilon / K)
def normalize_by_axis(x, axis, smooth_factor=1e-5):
x += smooth_factor
return x / tf.reduce_sum(x, axis, keepdims=True) # num A x num B
def get_cross_correlated_mat(num_out_A, num_out_B, learn_cooc='FIXED', cooc_AB=None, scope=None, reuse=None):
with tf.variable_scope(scope or 'CrossCorrlated_Mat', reuse=reuse):
if learn_cooc == 'FIXED' and cooc_AB is not None:
pB_given_A = normalize_by_axis(cooc_AB, 1)
pA_given_B = normalize_by_axis(cooc_AB, 0)
elif learn_cooc == 'JOINT':
share_cooc = tf.nn.relu(get_var('cooc_ab', shape=[num_out_A, num_out_B]))
pB_given_A = normalize_by_axis(share_cooc, 1)
pA_given_B = normalize_by_axis(share_cooc, 0)
elif learn_cooc == 'DISJOINT':
cooc1 = tf.nn.relu(get_var('pb_given_a', shape=[num_out_A, num_out_B]))
cooc2 = tf.nn.relu(get_var('pa_given_b', shape=[num_out_A, num_out_B]))
pB_given_A = normalize_by_axis(cooc1, 1)
pA_given_B = normalize_by_axis(cooc2, 0)
else:
raise NotImplementedError
return pA_given_B, pB_given_A
def get_self_correlated_mat(num_out_A, scope=None, reuse=None):
with tf.variable_scope(scope or 'Self_Correlated_mat', reuse=reuse):
cooc1 = get_var('pa_corr', shape=[num_out_A, num_out_A],
initializer_fn=tf.contrib.layers.variance_scaling_initializer(factor=0.1,
mode='FAN_AVG',
uniform=True,
dtype=tf.float32),
regularizer_fn=tf.contrib.layers.l2_regularizer(scale=3e-4))
return tf.matmul(cooc1, cooc1, transpose_b=True) + tf.eye(num_out_A)
def gate_filter(x, scope=None, reuse=None):
with tf.variable_scope(scope or 'Gate', reuse=reuse):
threshold = get_var('threshold', shape=[])
gate = tf.cast(tf.greater(x, threshold), tf.float32)
return x * gate
from tensorflow.python.ops import array_ops
def focal_loss2(onehot_labels, prediction_tensor, alpha=0.25, gamma=2, ):
y_ = tf.cast(onehot_labels, dtype=tf.float32)
sigmoid_p = tf.nn.sigmoid(prediction_tensor)
zeros = array_ops.zeros_like(sigmoid_p, dtype=sigmoid_p.dtype)
# For poitive prediction, only need consider front part loss, back part is 0;
# target_tensor > zeros <=> z=1, so poitive coefficient = z - p.
pos_p_sub = array_ops.where(y_ > zeros, y_ - sigmoid_p, zeros)
# For negative prediction, only need consider back part loss, front part is 0;
# target_tensor > zeros <=> z=1, so negative coefficient = 0.
neg_p_sub = array_ops.where(y_ > zeros, zeros, sigmoid_p)
# per_entry_cross_ent = - alpha * (pos_p_sub ** gamma) * tf.log(tf.clip_by_value(sigmoid_p, 1e-8, 1.0)) \
# - (1 - alpha) * (neg_p_sub ** gamma) * tf.log(tf.clip_by_value(1.0 - sigmoid_p, 1e-8, 1.0))
per_entry_cross_ent = - (pos_p_sub ** gamma) * tf.log(tf.clip_by_value(sigmoid_p, 1e-8, 1.0)) - (
neg_p_sub ** gamma) * tf.log(tf.clip_by_value(1.0 - sigmoid_p, 1e-8, 1.0))
dn, dp = tf.dynamic_partition(per_entry_cross_ent, tf.cast(y_, tf.int32), 2)
return tf.reduce_sum(dp), tf.reduce_sum(dn), tf.reduce_sum(per_entry_cross_ent)
def focal_loss(prediction_tensor, target_tensor, weights=None, alpha=0.25, gamma=2):
r"""Compute focal loss for predictions.
Multi-labels Focal loss formula:
FL = -alpha * (z-p)^gamma * log(p) -(1-alpha) * p^gamma * log(1-p)
,which alpha = 0.25, gamma = 2, p = sigmoid(x), z = target_tensor.
Args:
prediction_tensor: A float tensor of shape [batch_size, num_anchors,
num_classes] representing the predicted logits for each class
target_tensor: A float tensor of shape [batch_size, num_anchors,
num_classes] representing one-hot encoded classification targets
weights: A float tensor of shape [batch_size, num_anchors]
alpha: A scalar tensor for focal loss alpha hyper-parameter
gamma: A scalar tensor for focal loss gamma hyper-parameter
Returns:
loss: A (scalar) tensor representing the value of the loss function
"""
sigmoid_p = tf.nn.sigmoid(prediction_tensor)
zeros = array_ops.zeros_like(sigmoid_p, dtype=sigmoid_p.dtype)
# For poitive prediction, only need consider front part loss, back part is 0;
# target_tensor > zeros <=> z=1, so poitive coefficient = z - p.
pos_p_sub = array_ops.where(target_tensor > zeros, target_tensor - sigmoid_p, zeros)
# For negative prediction, only need consider back part loss, front part is 0;
# target_tensor > zeros <=> z=1, so negative coefficient = 0.
neg_p_sub = array_ops.where(target_tensor > zeros, zeros, sigmoid_p)
per_entry_cross_ent = - alpha * (pos_p_sub ** gamma) * tf.log(tf.clip_by_value(sigmoid_p, 1e-8, 1.0)) \
- (1 - alpha) * (neg_p_sub ** gamma) * tf.log(tf.clip_by_value(1.0 - sigmoid_p, 1e-8, 1.0))
return tf.reduce_sum(per_entry_cross_ent)
def spatial_dropout(x, scope=None, reuse=None):
input_dim = x.get_shape().as_list()[-1]
with tf.variable_scope(scope or 'spatial_dropout', reuse=reuse):
d = tf.random_uniform(shape=[1], minval=0, maxval=input_dim, dtype=tf.int32)
f = tf.one_hot(d, on_value=0., off_value=1., depth=input_dim)
g = x * f # do dropout
g *= (1. + 1. / input_dim) # do rescale
return g
def get_last_output(output, seq_length, scope=None, reuse=None):
"""Get the last value of the returned output of an RNN.
http://disq.us/p/1gjkgdr
output: [batch x number of steps x ... ] Output of the dynamic lstm.
sequence_length: [batch] Length of each of the sequence.
"""
with tf.variable_scope(scope or 'gather_nd', reuse=reuse):
rng = tf.range(0, tf.shape(seq_length)[0])
indexes = tf.stack([rng, seq_length - 1], 1)
return tf.gather_nd(output, indexes)
def get_lstm_init_state(batch_size, num_layers, num_units, direction, scope=None, reuse=None, **kwargs):
with tf.variable_scope(scope or 'lstm_init_state', reuse=reuse):
num_dir = 2 if direction.startswith('bi') else 1
c = get_var('lstm_init_c', shape=[num_layers * num_dir, num_units])
c = tf.tile(tf.expand_dims(c, axis=1), [1, batch_size, 1])
h = get_var('lstm_init_h', shape=[num_layers * num_dir, num_units])
h = tf.tile(tf.expand_dims(h, axis=1), [1, batch_size, 1])
return c, h
def highway_layer(x, scope=None, reuse=None, **kwargs):
with tf.variable_scope(scope or "highway_layer", reuse=reuse):
d = x.get_shape()[-1]
trans = linear_logit(x, d, scope='trans', reuse=reuse)
trans = tf.nn.relu(trans)
gate = linear_logit(x, d, scope='gate', reuse=reuse)
gate = tf.nn.sigmoid(gate)
out = gate * trans + (1 - gate) * x
return out
def highway_network(x, num_layers, scope=None, reuse=None, **kwargs):
with tf.variable_scope(scope or "highway_network", reuse=reuse):
prev = x
cur = None
for layer_idx in range(num_layers):
cur = highway_layer(prev, scope="layer_{}".format(layer_idx), reuse=reuse)
prev = cur
return cur
================================================
FILE: nlp/seq2seq/__init__.py
================================================
================================================
FILE: nlp/seq2seq/common.py
================================================
# coding=utf-8
import math
import time
import tensorflow as tf
from tensorflow.python.ops.image_ops_impl import ResizeMethod
from tensorflow.python.ops.rnn_cell_impl import LSTMStateTuple
INF = 1e30
def initializer(): return tf.contrib.layers.variance_scaling_initializer(factor=1.0,
mode='FAN_AVG',
uniform=True,
dtype=tf.float32)
def rand_uniform_initializer(
mag): return tf.random_uniform_initializer(-mag, mag, seed=314159)
def truc_norm_initializer(
std): return tf.truncated_normal_initalizer(stddev=std)
def initializer_relu(): return tf.contrib.layers.variance_scaling_initializer(factor=2.0,
mode='FAN_IN',
uniform=False,
dtype=tf.float32)
regularizer = tf.contrib.layers.l2_regularizer(scale=3e-7)
def get_var(name, shape, dtype=tf.float32,
initializer_fn=initializer,
regularizer_fn=regularizer, **kwargs):
return tf.get_variable(name, shape,
initializer=initializer_fn,
dtype=dtype,
regularizer=regularizer_fn, **kwargs)
def softmax_mask(val, mask):
return -INF * (1 - tf.cast(mask, tf.float32)) + val
def dropout(args, keep_prob, is_train, mode="recurrent"):
if keep_prob < 1.0:
noise_shape = None
scale = 1.0
shape = tf.shape(args)
if mode == "embedding":
noise_shape = [shape[0], 1]
scale = keep_prob
if mode == "recurrent" and len(args.get_shape().as_list()) == 3:
noise_shape = [shape[0], 1, shape[-1]]
args = tf.cond(is_train, lambda: tf.nn.dropout(
args, keep_prob, noise_shape=noise_shape) * scale, lambda: args)
return args
def dense(inputs, hidden_size, use_bias=True, scope=None):
with tf.variable_scope(scope or "dense"):
shape = tf.shape(inputs)
dim = inputs.get_shape().as_list()[-1]
out_shape = [shape[idx] for idx in range(
len(inputs.get_shape().as_list()) - 1)] + [hidden_size]
flat_inputs = tf.reshape(inputs, [-1, dim])
W = tf.get_variable("W", [dim, hidden_size])
res = tf.matmul(flat_inputs, W)
if use_bias:
bias = tf.get_variable(
"bias", [hidden_size], initializer=tf.constant_initializer(0.))
res = tf.nn.bias_add(res, bias)
res = tf.reshape(res, out_shape)
return res
def layer_norm(x, filters=None, epsilon=1e-6, scope=None, reuse=None):
"""Layer normalize the tensor x, averaging over the last dimension."""
if filters is None:
filters = x.get_shape()[-1]
with tf.variable_scope(scope, default_name="layer_norm", values=[x], reuse=reuse):
scale = tf.get_variable(
"layer_norm_scale", [filters], regularizer=regularizer, initializer=tf.ones_initializer())
bias = tf.get_variable(
"layer_norm_bias", [filters], regularizer=regularizer, initializer=tf.zeros_initializer())
result = layer_norm_compute_python(x, epsilon, scale, bias)
return result
def layer_norm_compute_python(x, epsilon, scale, bias):
"""Layer norm raw computation."""
mean = tf.reduce_mean(x, axis=[-1], keepdims=True)
variance = tf.reduce_mean(tf.square(x - mean), axis=[-1], keepdims=True)
norm_x = (x - mean) * tf.rsqrt(variance + epsilon)
return norm_x * scale + bias
def get_scope_name():
return tf.get_variable_scope().name.split('/')[0]
def make_var(name, shape, trainable=True):
return tf.get_variable(name, shape,
initializer=initializer(),
dtype=tf.float32,
trainable=trainable,
regularizer=regularizer)
def mblock(scope_name, device_name=None, reuse=None):
def f2(f):
def f2_v(self, *args, **kwargs):
start_t = time.time()
if device_name:
with tf.device(device_name), tf.variable_scope(scope_name, reuse=reuse):
f(self, *args, **kwargs)
else:
with tf.variable_scope(scope_name, reuse=reuse):
f(self, *args, **kwargs)
self.logger.info('%s is build in %.4f secs' %
(scope_name, time.time() - start_t))
return f2_v
return f2
def get_init_state(args, name, q_type, shape):
hinit_embed = make_var('hinit_ebd_' + name, shape)
cinit_embed = make_var('cinit_ebd_' + name, shape)
h_init = tf.expand_dims(
tf.nn.embedding_lookup(hinit_embed, q_type), axis=0)
c_init = tf.expand_dims(
tf.nn.embedding_lookup(cinit_embed, q_type), axis=0)
cell_init_state = {
'lstm': lambda: LSTMStateTuple(c_init, h_init),
'sru': lambda: h_init,
'gru': lambda: h_init,
'rnn': lambda: h_init}[args.cell.replace('bi-', '')]()
return cell_init_state
def highway(x, size=None, activation=tf.nn.relu,
num_layers=2, scope="highway", dropout=0.0, reuse=None):
with tf.variable_scope(scope, reuse):
if size is None:
size = x.shape.as_list()[-1]
else:
x = conv(x, size, name="input_projection", reuse=reuse)
for i in range(num_layers):
T = conv(x, size, bias=True, activation=tf.sigmoid,
name="gate_%d" % i, reuse=reuse)
H = conv(x, size, bias=True, activation=activation,
name="activation_%d" % i, reuse=reuse)
H = tf.nn.dropout(H, 1.0 - dropout)
x = H * T + x * (1.0 - T)
return x
def conv(inputs, output_size, bias=None, activation=None, kernel_size=1, name="conv", reuse=None):
with tf.variable_scope(name, reuse=reuse):
shapes = inputs.shape.as_list()
if len(shapes) > 4:
raise NotImplementedError
elif len(shapes) == 4:
filter_shape = [1, kernel_size, shapes[-1], output_size]
bias_shape = [1, 1, 1, output_size]
strides = [1, 1, 1, 1]
else:
filter_shape = [kernel_size, shapes[-1], output_size]
bias_shape = [1, 1, output_size]
strides = 1
conv_func = tf.nn.conv1d if len(shapes) == 3 else tf.nn.conv2d
kernel_ = tf.get_variable("kernel_",
filter_shape,
dtype=tf.float32,
regularizer=regularizer,
initializer=initializer_relu() if activation is not None else initializer())
outputs = conv_func(inputs, kernel_, strides, "VALID")
if bias:
outputs += tf.get_variable("bias_",
bias_shape,
regularizer=regularizer,
initializer=tf.zeros_initializer())
if activation is not None:
return activation(outputs)
else:
return outputs
def sparse_nll_loss(probs, labels, epsilon=1e-9, scope=None):
"""
negative log likelihood loss
"""
with tf.name_scope(scope, "log_loss"):
labels = tf.one_hot(labels, tf.shape(
probs)[1], axis=1, dtype=tf.float32)
losses = - tf.reduce_sum(labels * tf.log(probs + epsilon), 1)
return losses
def normalize_distribution(p, eps=1e-9):
p += eps
norm = tf.reduce_sum(p, axis=1)
return tf.cast(p, tf.float32) / tf.reshape(norm, (-1, 1))
def kl_divergence(p, q, eps=1e-9):
p = normalize_distribution(p, eps)
q = normalize_distribution(q, eps)
return tf.reduce_sum(p * tf.log(p / q), axis=1)
def get_kl_loss(start_label, start_probs, bandwidth=1.0):
a = tf.reshape(tf.range(tf.shape(start_probs)[1]), (1, -1))
b = tf.reshape(start_label, (-1, 1))
start_true_probs = tf.exp(-tf.cast(tf.squared_difference(a,
b), tf.float32) / bandwidth)
return sym_kl_divergence(start_true_probs, start_probs)
def sym_kl_divergence(p, q, eps=1e-9):
return (kl_divergence(p, q, eps) + kl_divergence(q, p, eps)) / 2.0
def get_conv_feature(x, out_dim, window_len, upsampling=False):
a = tf.layers.conv1d(x, out_dim, window_len, strides=max(
int(math.floor(window_len / 2)), 1))
if upsampling:
return upsampling_a2b(a, x, out_dim)
else:
return a
def upsampling_a2b(a, b, D_a):
return tf.squeeze(tf.image.resize_images(tf.expand_dims(a, axis=-1), [tf.shape(b)[1], D_a],
method=ResizeMethod.NEAREST_NEIGHBOR), axis=-1)
================================================
FILE: nlp/seq2seq/pointer_generator.py
================================================
import tensorflow as tf
from tensorflow.contrib.seq2seq.python.ops.attention_wrapper import _compute_attention
from tensorflow.python.framework import ops
from tensorflow.python.ops import math_ops
# from tensorflow.contrib.rnn.python.ops.core_rnn_cell import _linear
from tensorflow.python.ops.rnn_cell_impl import _zero_state_tensors
from tensorflow.python.util import nest
class PointerGeneratorGreedyEmbeddingHelper(tf.contrib.seq2seq.GreedyEmbeddingHelper):
def __init__(self, embeddings, start_tokens, end_token, unk_token):
super(PointerGeneratorGreedyEmbeddingHelper, self).__init__(embeddings, start_tokens, end_token)
self.vocab_size = tf.shape(embeddings)[-1]
self.unk_token = unk_token
def sample(self, time, outputs, state, name=None):
"""sample for PointerGeneratorGreedyEmbeddingHelper."""
del time, state # unused by sample_fn
# Outputs are logits, use argmax to get the most probable id
if not isinstance(outputs, ops.Tensor):
raise TypeError("Expected outputs to be a single Tensor, got: %s" %
type(outputs))
sample_ids = tf.argmax(outputs, axis=-1, output_type=tf.int32)
return sample_ids
def next_inputs(self, time, outputs, state, sample_ids, name=None):
"""next_inputs_fn for GreedyEmbeddingHelper."""
del time, outputs # unused by next_inputs_fn
finished = tf.equal(sample_ids, self._end_token)
all_finished = tf.reduce_all(finished)
# since we have OOV words, we need change these words to UNK
condition = tf.less(sample_ids, self.vocab_size)
sample_ids = tf.where(condition, sample_ids, tf.ones_like(sample_ids) * self.unk_token)
next_inputs = tf.cond(
all_finished,
# If we're finished, the next_inputs value doesn't matter
lambda: self._start_inputs,
lambda: self._embedding_fn(sample_ids))
return (finished, next_inputs, state)
class PointerGeneratorDecoder(tf.contrib.seq2seq.BasicDecoder):
"""Pointer Generator sampling decoder."""
def __init__(self, source_extend_tokens, source_oov_words, coverage, cell, helper, initial_state,
output_layer=None, multi_rnn=False):
self.source_oov_words = source_oov_words
self.source_extend_tokens = source_extend_tokens
self.coverage = coverage
self.multi_rnn = multi_rnn
self.history_inputs = None
super(PointerGeneratorDecoder, self).__init__(cell, helper, initial_state, output_layer)
@property
def output_size(self):
# Return the cell output and the id
return tf.contrib.seq2seq.BasicDecoderOutput(
rnn_output=self._rnn_output_size() + self.source_oov_words,
sample_id=self._helper.sample_ids_shape)
@property
def output_dtype(self):
# Assume the dtype of the cell is the output_size structure
# containing the input_state's first component's dtype.
# Return that structure and the sample_ids_dtype from the helper.
dtype = nest.flatten(self._initial_state)[0].dtype
return tf.contrib.seq2seq.BasicDecoderOutput(
nest.map_structure(lambda _: dtype, self._rnn_output_size() + self.source_oov_words),
self._helper.sample_ids_dtype)
def step(self, time, inputs, state, name=None):
"""Perform a decoding step.
Args:
time: scalar `int32` tensor.
inputs: A (structure of) input tensors.
state: A (structure of) state tensors and TensorArrays.
name: Name scope for any created operations.
Returns:
`(outputs, next_state, next_inputs, finished)`.
"""
with ops.name_scope(name, "PGDecoderStep", (time, inputs, state)):
if self.history_inputs is None:
self.history_inputs = tf.expand_dims(inputs, axis=1) # B,1,D
else:
t_his = tf.expand_dims(inputs, axis=1)
self.history_inputs = tf.concat([self.history_inputs, t_his], axis=1)
self._cell.set_history(self.history_inputs)
cell_outputs, cell_state = self._cell(inputs, state)
# attention = cell_state.attention
# att_cell_state = cell_state.cell_state
# alignments = cell_state.alignments
attention = cell_state.attention
att_cell_state = cell_state.cell_state[-1]
alignments = cell_state.alignments
with tf.variable_scope('calculate_pgen'):
p_gen = tf.layers.dense(tf.concat([attention, inputs, att_cell_state.c, att_cell_state.h], 1), 1,
use_bias=True)
# p_gen = _linear([attention, inputs, att_cell_state], 1, True)
p_gen = tf.sigmoid(p_gen)
if self._output_layer is not None:
cell_outputs = tf.layers.dense(cell_outputs, 1024, activation=tf.nn.tanh)
cell_outputs = self._output_layer(cell_outputs)
vocab_dist = tf.nn.softmax(cell_outputs) * p_gen
# z = tf.reduce_sum(alignments,axis=1)
# z = tf.reduce_sum(tf.cast(tf.less_equal(alignments, 0),tf.int32))
alignments = alignments * (1 - p_gen)
# x = tf.reduce_sum(tf.cast(tf.less_equal((1-p_gen), 0),tf.int32))
# y = tf.reduce_sum(tf.cast(tf.less_equal(alignments[3], 0),tf.int32))
# this is only for debug
# alignments2 = tf.Print(alignments2,[tf.shape(inputs),x,y,alignments[2][9:12]],message="zeros in vocab dist and alignments")
# since we have OOV words, we need expand the vocab dist
vocab_size = tf.shape(vocab_dist)[-1]
extended_vsize = vocab_size + self.source_oov_words
batch_size = tf.shape(vocab_dist)[0]
extra_zeros = tf.zeros((batch_size, self.source_oov_words))
# batch * extend vocab size
vocab_dists_extended = tf.concat(axis=-1, values=[vocab_dist, extra_zeros])
# vocab_dists_extended = tf.Print(vocab_dists_extended,[tf.shape(vocab_dists_extended),self.source_oov_words],message='vocab_dists_extended size')
batch_nums = tf.range(0, limit=batch_size) # shape (batch_size)
batch_nums = tf.expand_dims(batch_nums, 1) # shape (batch_size, 1)
attn_len = tf.shape(self.source_extend_tokens)[1] # number of states we attend over
batch_nums = tf.tile(batch_nums, [1, attn_len]) # shape (batch_size, attn_len)
indices = tf.stack((batch_nums, self.source_extend_tokens), axis=2) # shape (batch_size, enc_t, 2)
shape = [batch_size, extended_vsize]
attn_dists_projected = tf.scatter_nd(indices, alignments, shape)
final_dists = attn_dists_projected + vocab_dists_extended
# final_dists = tf.Print(final_dists,[tf.reduce_sum(tf.cast(tf.less_equal(final_dists[0],0),tf.int32))],message='final dist')
# note: sample_ids will contains OOV words
sample_ids = self._helper.sample(
time=time, outputs=final_dists, state=cell_state)
(finished, next_inputs, next_state) = self._helper.next_inputs(
time=time,
outputs=cell_outputs,
state=cell_state,
sample_ids=sample_ids)
all_finished = math_ops.reduce_all(finished)
if all_finished is True:
self.history_inputs = None
outputs = tf.contrib.seq2seq.BasicDecoderOutput(final_dists, sample_ids)
return (outputs, next_state, next_inputs, finished)
class PointerGeneratorAttentionWrapper(tf.contrib.seq2seq.AttentionWrapper):
def __init__(self, cell,
attention_mechanism,
encoder_func=None, # return [B,D]
attention_layer_size=None,
alignment_history=False,
cell_input_fn=None,
output_attention=True,
initial_cell_state=None,
coverage=False,
name=None,
multi_rnn=False):
super(PointerGeneratorAttentionWrapper, self).__init__(
cell,
attention_mechanism,
attention_layer_size,
alignment_history,
cell_input_fn,
output_attention,
initial_cell_state,
name)
self.coverage = coverage
self.multi_rnn = multi_rnn
self.encoder_func = encoder_func
self.history_inputs = None
def zero_state(self, batch_size, dtype):
"""Return an initial (zero) state tuple for this `AttentionWrapper`.
**NOTE** Please see the initializer documentation for details of how
to call `zero_state` if using an `AttentionWrapper` with a
`BeamSearchDecoder`.
Args:
batch_size: `0D` integer tensor: the batch size.
dtype: The internal state data type.
Returns:
An `AttentionWrapperState` tuple containing zeroed out tensors and,
possibly, empty `TensorArray` objects.
Raises:
ValueError: (or, possibly at runtime, InvalidArgument), if
`batch_size` does not match the output size of the encoder passed
to the wrapper object at initialization time.
"""
with ops.name_scope(type(self).__name__ + "ZeroState", values=[batch_size]):
if self._initial_cell_state is not None:
cell_state = self._initial_cell_state
else:
cell_state = self._cell.zero_state(batch_size, dtype)
error_message = (
"When calling zero_state of AttentionWrapper %s: " % self._base_name +
"Non-matching batch sizes between the memory "
"(encoder output) and the requested batch size. Are you using "
"the BeamSearchDecoder? If so, make sure your encoder output has "
"been tiled to beam_width via tf.contrib.seq2seq.tile_batch, and "
"the batch_size= argument passed to zero_state is "
"batch_size * beam_width.")
with tf.control_dependencies(
self._batch_size_checks(batch_size, error_message)):
cell_state = nest.map_structure(
lambda s: tf.identity(s, name="checked_cell_state"),
cell_state)
return tf.contrib.seq2seq.AttentionWrapperState(
cell_state=cell_state,
time=tf.zeros([], dtype=tf.int32),
attention=_zero_state_tensors(self._attention_layer_size, batch_size,
dtype),
alignments=self._item_or_tuple(
attention_mechanism.initial_alignments(batch_size, dtype)
for attention_mechanism in self._attention_mechanisms),
attention_state=self._item_or_tuple(
attention_mechanism.initial_state(batch_size, dtype)
for attention_mechanism in self._attention_mechanisms),
# since we need to read the alignment history several times, so we need set clear_after_read to False
alignment_history=self._item_or_tuple(
tf.TensorArray(dtype=dtype, size=0, clear_after_read=False, dynamic_size=True)
if self._alignment_history else ()
for _ in self._attention_mechanisms))
def set_history(self, history_inputs):
self.history_inputs = history_inputs
def call(self, inputs, state):
"""Perform a step of attention-wrapped RNN.
- Step 1: Mix the `inputs` and previous step's `attention` output via
`cell_input_fn`.
- Step 2: Call the wrapped `cell` with this input and its previous state.
- Step 3: Score the cell's output with `attention_mechanism`.
- Step 4: Calculate the alignments by passing the score through the
`normalizer`.
- Step 5: Calculate the context vector as the inner product between the
alignments and the attention_mechanism's values (memory).
- Step 6: Calculate the attention output by concatenating the cell output
and context through the attention layer (a linear layer with
`attention_layer_size` outputs).
Args:
inputs: (Possibly nested tuple of) Tensor, the input at this time step.
state: An instance of `AttentionWrapperState` containing
tensors from the previous time step.
Returns:
A tuple `(attention_or_cell_output, next_state)`, where:
- `attention_or_cell_output` depending on `output_attention`.
- `next_state` is an instance of `AttentionWrapperState`
containing the state calculated at this time step.
Raises:
TypeError: If `state` is not an instance of `AttentionWrapperState`.
"""
if not isinstance(state, tf.contrib.seq2seq.AttentionWrapperState):
raise TypeError("Expected state to be instance of AttentionWrapperState. "
"Received type %s instead." % type(state))
# Step 1: Calculate the true inputs to the cell based on the
# previous attention value.
cell_inputs = self._cell_input_fn(inputs, state.attention)
cell_state = state.cell_state
# lstm
cell_output, next_cell_state = self._cell(cell_inputs, cell_state)
# other encoder
if self.encoder_func:
encoder_output = self.encoder_func(self.history_inputs)
cell_output = tf.concat([cell_output, encoder_output], axis=-1) # B,d1+d2
cell_batch_size = (
cell_output.shape[0].value or tf.shape(cell_output)[0])
error_message = (
"When applying AttentionWrapper %s: " % self.name +
"Non-matching batch sizes between the memory "
"(encoder output) and the query (decoder output). Are you using "
"the BeamSearchDecoder? You may need to tile your memory input via "
"the tf.contrib.seq2seq.tile_batch function with argument "
"multiple=beam_width.")
with tf.control_dependencies(self._batch_size_checks(cell_batch_size, error_message)):
cell_output = tf.identity(
cell_output, name="checked_cell_output")
if self._is_multi:
# previous_alignments = state.alignments
previous_attention_state = state.attention_state
previous_alignment_history = state.alignment_history
else:
# previous_alignments = [state.alignments]
previous_attention_state = [state.attention_state]
previous_alignment_history = [state.alignment_history]
all_alignments = []
all_attentions = []
all_histories = []
all_attention_states = []
for i, attention_mechanism in enumerate(self._attention_mechanisms):
# if self.coverage:
# # if we use coverage mode, previous alignments is coverage vector
# # alignment history stack has shape: decoder time * batch * atten_len
# # convert it to coverage vector
# previous_alignments[i] = tf.cond(
# previous_alignment_history[i].size()>0,
# lambda: tf.reduce_sum(tf.transpose(previous_alignment_history[i].stack(),[1,2,0]),axis=2),
# lambda: tf.zeros_like(previous_alignments[i]))
# attention, alignments = _compute_attention(
# attention_mechanism, cell_output, previous_alignments[i],
# self._attention_layers[i] if self._attention_layers else None)
# alignment_history = previous_alignment_history[i].write(
# state.time, alignments) if self._alignment_history else ()
# all_alignments.append(alignments)
# all_histories.append(alignment_history)
# all_attentions.append(attention)
attention, alignments, next_attention_state = _compute_attention(
attention_mechanism, cell_output, previous_attention_state[i],
self._attention_layers[i] if self._attention_layers else None)
alignment_history = previous_alignment_history[i].write(
state.time, alignments) if self._alignment_history else ()
all_attention_states.append(next_attention_state)
all_alignments.append(alignments)
all_histories.append(alignment_history)
all_attentions.append(attention)
attention = tf.concat(all_attentions, 1)
next_state = tf.contrib.seq2seq.AttentionWrapperState(
time=state.time + 1,
cell_state=next_cell_state,
attention=attention,
attention_state=self._item_or_tuple(all_attention_states),
alignments=self._item_or_tuple(all_alignments),
alignment_history=self._item_or_tuple(all_histories))
if self._output_attention:
return attention, next_state
else:
return cell_output, next_state
def _pg_bahdanau_score(processed_query, keys, coverage, coverage_vector):
"""Implements Bahdanau-style (additive) scoring function.
Args:
processed_query: Tensor, shape `[batch_size, num_units]` to compare to keys.
keys: Processed memory, shape `[batch_size, max_time, num_units]`.
coverage: Whether to use coverage mode.
coverage_vector: only used when coverage is true
Returns:
A `[batch_size, max_time]` tensor of unnormalized score values.
"""
dtype = processed_query.dtype
# Get the number of hidden units from the trailing dimension of keys
num_units = keys.shape[2].value or tf.shape(keys)[2]
# Reshape from [batch_size, ...] to [batch_size, 1, ...] for broadcasting.
wei = tf.layers.dense(processed_query, units=1)
processed_query = tf.expand_dims(processed_query, 1)
v = tf.get_variable(
"attention_v", [num_units], dtype=dtype)
b = tf.get_variable(
"attention_b", [num_units], dtype=dtype,
initializer=tf.zeros_initializer())
simi_score = tf.matmul(keys, processed_query, transpose_b=True)
simi_score = tf.squeeze(simi_score, 2) * wei
if coverage:
w_c = tf.get_variable(
"coverage_w", [num_units], dtype=dtype)
# debug
# coverage_vector = tf.Print(coverage_vector,[coverage_vector],message="score")
coverage_vector = tf.expand_dims(coverage_vector, -1)
return tf.reduce_sum(v * tf.tanh(keys + processed_query + coverage_vector * w_c + b), [2]) + simi_score
else:
return tf.reduce_sum(v * tf.tanh(keys + processed_query + b), [2]) + simi_score
class PointerGeneratorBahdanauAttention(tf.contrib.seq2seq.BahdanauAttention):
def __init__(self,
num_units,
memory,
memory_sequence_length=None,
normalize=False,
coverage=False,
probability_fn=None,
score_mask_value=float("-inf"),
name="PointerGeneratorBahdanauAttention"):
"""Construct the Attention mechanism.
Args:
num_units: The depth of the query mechanism.
memory: The memory to query; usually the output of an RNN encoder. This
tensor should be shaped `[batch_size, max_time, ...]`.
memory_sequence_length (optional): Sequence lengths for the batch entries
in memory. If provided, the memory tensor rows are masked with zeros
for values past the respective sequence lengths.
normalize: Python boolean. Whether to normalize the energy term.
probability_fn: (optional) A `callable`. Converts the score to
probabilities. The default is @{tf.nn.softmax}. Other options include
@{tf.contrib.seq2seq.hardmax} and @{tf.contrib.sparsemax.sparsemax}.
Its signature should be: `probabilities = probability_fn(score)`.
score_mask_value: (optional): The mask value for score before passing into
`probability_fn`. The default is -inf. Only used if
`memory_sequence_length` is not None.
name: Name to use when creating ops.
coverage: whether use coverage mode
"""
super(PointerGeneratorBahdanauAttention, self).__init__(
num_units=num_units,
memory=memory,
memory_sequence_length=memory_sequence_length,
normalize=normalize,
probability_fn=probability_fn,
score_mask_value=score_mask_value,
name=name)
self.coverage = coverage
def __call__(self, query, state):
"""Score the query based on the keys and values.
Args:
query: Tensor of dtype matching `self.values` and shape
`[batch_size, query_depth]`.
previous_alignments: Tensor of dtype matching `self.values` and shape
`[batch_size, alignments_size]`
(`alignments_size` is memory's `max_time`).
Returns:
alignments: Tensor of dtype matching `self.values` and shape
`[batch_size, alignments_size]` (`alignments_size` is memory's
`max_time`).
"""
with tf.variable_scope(None, "pointer_generator_bahdanau_attention", [query]):
processed_query = self.query_layer(query) if self.query_layer else query
# score = _pg_bahdanau_score(processed_query, self._keys, self.coverage, previous_alignments)
score = _pg_bahdanau_score(processed_query, self._keys, self.coverage, state)
alignments = self._probability_fn(score, state)
next_state = alignments
return alignments, next_state
================================================
FILE: nlp/seq2seq/rnn.py
================================================
# coding=utf-8
import tensorflow as tf
import tensorflow.contrib as tc
import gpu_env
from .common import dropout, dense, get_var
def single_rnn_cell(cell_name, num_units, is_train=None, keep_prob=0.75):
"""
Get a single rnn cell
"""
cell_name = cell_name.upper()
if cell_name == "GRU":
cell = tf.contrib.rnn.GRUCell(num_units)
elif cell_name == "LSTM":
cell = tf.contrib.rnn.LSTMCell(num_units)
else:
cell = tf.contrib.rnn.BasicRNNCell(num_units)
# dropout wrapper
if is_train and keep_prob < 1.0:
cell = tf.contrib.rnn.DropoutWrapper(
cell=cell,
input_keep_prob=keep_prob,
output_keep_prob=keep_prob)
return cell
def multi_rnn_cell(cell_name, num_units, is_train=None, keep_prob=1.0, num_layers=3):
cell_name = cell_name.upper()
if cell_name == "GRU":
cells = [tf.contrib.rnn.GRUCell(num_units) for _ in range(num_layers)]
elif cell_name == "LSTM":
cells = [tf.contrib.rnn.LayerNormBasicLSTMCell(num_units) for _ in range(num_layers)]
else:
cells = [tf.contrib.rnn.BasicRNNCell(num_units) for _ in range(num_layers)]
cell = tf.contrib.rnn.MultiRNNCell(cells)
# dropout wrapper
if is_train and keep_prob < 1.0:
cell = tf.contrib.rnn.DropoutWrapper(
cell=cell,
input_keep_prob=keep_prob,
output_keep_prob=keep_prob)
return cell
def get_lstm_init_state(batch_size, num_layers, num_units, direction, scope=None, reuse=None, **kwargs):
with tf.variable_scope(scope or 'lstm_init_state', reuse=reuse):
num_dir = 2 if direction.startswith('bi') else 1
c = get_var('lstm_init_c', shape=[num_layers * num_dir, num_units])
c = tf.tile(tf.expand_dims(c, axis=1), [1, batch_size, 1])
h = get_var('lstm_init_h', shape=[num_layers * num_dir, num_units])
h = tf.tile(tf.expand_dims(h, axis=1), [1, batch_size, 1])
return c, h
def LSTM_encode(seqs, scope=None, reuse=None, **kwargs):
with tf.variable_scope(scope or 'lstm_encode_block', reuse=reuse):
batch_size = tf.shape(seqs)[0]
# to T, B, D
_seqs = tf.transpose(seqs, [1, 0, 2])
lstm = tf.contrib.cudnn_rnn.CudnnLSTM(**kwargs)
init_state = get_lstm_init_state(batch_size, **kwargs)
output, state = lstm(_seqs, init_state)
return tf.transpose(output, [1, 0, 2]), state
def custom_dynamic_rnn(cell, inputs, inputs_len, initial_state=None):
"""
Implements a dynamic rnn that can store scores in the pointer network,
the reason why we implements this is that the raw_rnn or dynamic_rnn function in Tensorflow
seem to require the hidden unit and memory unit has the same dimension, and we cannot
store the scores directly in the hidden unit.
Args:
cell: RNN cell
inputs: the input sequence to rnn
inputs_len: valid length
initial_state: initial_state of the cell
Returns:
outputs and state
"""
batch_size = tf.shape(inputs)[0]
max_time = tf.shape(inputs)[1]
inputs_ta = tf.TensorArray(dtype=gpu_env.DTYPE_F, size=max_time)
inputs_ta = inputs_ta.unstack(tf.transpose(inputs, [1, 0, 2]))
emit_ta = tf.TensorArray(dtype=gpu_env.DTYPE_F, dynamic_size=True, size=0)
t0 = tf.constant(0, dtype=tf.int32)
if initial_state is not None:
s0 = initial_state
else:
s0 = cell.zero_state(batch_size, dtype=gpu_env.DTYPE_F)
f0 = tf.zeros([batch_size], dtype=tf.bool)
def loop_fn(time, prev_s, emit_ta, finished):
"""
the loop function of rnn
"""
cur_x = inputs_ta.read(time)
scores, cur_state = cell(cur_x, prev_s)
# copy through
scores = tf.where(finished, tf.zeros_like(scores), scores)
if isinstance(cell, tc.rnn.LSTMCell):
cur_c, cur_h = cur_state
prev_c, prev_h = prev_s
cur_state = tc.rnn.LSTMStateTuple(tf.where(finished, prev_c, cur_c),
tf.where(finished, prev_h, cur_h))
else:
cur_state = tf.where(finished, prev_s, cur_state)
emit_ta = emit_ta.write(time, scores)
finished = tf.greater_equal(time + 1, inputs_len)
return [time + 1, cur_state, emit_ta, finished]
_, state, emit_ta, _ = tf.while_loop(
cond=lambda _1, _2, _3, finished: tf.logical_not(
tf.reduce_all(finished)),
body=loop_fn,
loop_vars=(t0, s0, emit_ta, f0),
parallel_iterations=32,
swap_memory=False)
outputs = tf.transpose(emit_ta.stack(), [1, 0, 2])
return outputs, state
def reduce_state(fw_state, bw_state, hidden_size, act_fn=tf.nn.relu, scope=None):
# concatennation of fw and bw cell
with tf.variable_scope(scope or "reduce_final_state"):
_c = tf.concat([fw_state.c, bw_state.c], axis=1)
_h = tf.concat([fw_state.h, bw_state.h], axis=1)
c = act_fn(dense(_c, hidden_size, use_bias=True, scope="reduce_c"))
h = act_fn(dense(_h, hidden_size, use_bias=True, scope="reduce_h"))
return tc.rnn.LSTMStateTuple(c, h)
class CudaRNN:
def __init__(self, num_layers, num_units, cell_type):
self.num_layers = num_layers
self.num_units = num_units
if cell_type.endswith('gru'):
self.grus = [(tf.contrib.cudnn_rnn.CudnnGRU(1, num_units),
tf.contrib.cudnn_rnn.CudnnGRU(1, num_units)) for _ in range(num_layers)]
elif cell_type.endswith('lstm'):
self.grus = [(tf.contrib.cudnn_rnn.CudnnLSTM(1, num_units),
tf.contrib.cudnn_rnn.CudnnLSTM(1, num_units)) for _ in range(num_layers)]
else:
raise NotImplementedError
def __call__(self, inputs, seq_len, keep_prob=1.0,
is_train=None, init_states=None, concat_layers=True):
outputs = [tf.transpose(inputs, [1, 0, 2])]
batch_size = tf.shape(inputs)[0]
if not init_states:
init_states = []
for layer in range(self.num_layers):
init_fw = tf.tile(tf.Variable(
tf.zeros([1, 1, self.num_units])), [1, batch_size, 1])
init_bw = tf.tile(tf.Variable(
tf.zeros([1, 1, self.num_units])), [1, batch_size, 1])
init_states.append((init_fw, init_bw))
dropout_mask = []
for layer in range(self.num_layers):
input_size_ = inputs.get_shape().as_list(
)[-1] if layer == 0 else 2 * self.num_units
mask_fw = dropout(tf.ones([1, batch_size, input_size_], dtype=tf.float32),
keep_prob=keep_prob, is_train=is_train, mode=None)
mask_bw = dropout(tf.ones([1, batch_size, input_size_], dtype=tf.float32),
keep_prob=keep_prob, is_train=is_train, mode=None)
dropout_mask.append((mask_fw, mask_bw))
for layer in range(self.num_layers):
gru_fw, gru_bw = self.grus[layer]
init_fw, init_bw = init_states[layer]
mask_fw, mask_bw = dropout_mask[layer]
with tf.variable_scope("fw_{}".format(layer)):
out_fw, _ = gru_fw(
outputs[-1] * mask_fw, initial_state=(init_fw,))
with tf.variable_scope("bw_{}".format(layer)):
inputs_bw = tf.reverse_sequence(
outputs[-1] * mask_bw, seq_lengths=seq_len, seq_dim=0, batch_dim=1)
out_bw, _ = gru_bw(inputs_bw, initial_state=(init_bw,))
out_bw = tf.reverse_sequence(
out_bw, seq_lengths=seq_len, seq_dim=0, batch_dim=1)
outputs.append(tf.concat([out_fw, out_bw], axis=2))
if concat_layers:
res = tf.concat(outputs[1:], axis=2)
else:
res = outputs[-1]
res = tf.transpose(res, [1, 0, 2])
return res
class native_gru:
def __init__(self, num_layers, num_units, batch_size, input_size, keep_prob=1.0, is_train=None, scope="native_gru"):
self.num_layers = num_layers
self.grus = []
self.inits = []
self.dropout_mask = []
self.scope = scope
for layer in range(num_layers):
input_size_ = input_size if layer == 0 else 2 * num_units
gru_fw = tf.contrib.rnn.GRUCell(num_units)
gru_bw = tf.contrib.rnn.GRUCell(num_units)
init_fw = tf.tile(tf.Variable(
tf.zeros([1, num_units])), [batch_size, 1])
init_bw = tf.tile(tf.Variable(
tf.zeros([1, num_units])), [batch_size, 1])
mask_fw = dropout(tf.ones([batch_size, 1, input_size_], dtype=tf.float32),
keep_prob=keep_prob, is_train=is_train, mode=None)
mask_bw = dropout(tf.ones([batch_size, 1, input_size_], dtype=tf.float32),
keep_prob=keep_prob, is_train=is_train, mode=None)
self.grus.append((gru_fw, gru_bw,))
self.inits.append((init_fw, init_bw,))
self.dropout_mask.append((mask_fw, mask_bw,))
def __call__(self, inputs, seq_len, keep_prob=1.0, is_train=None, concat_layers=True):
outputs = [inputs]
with tf.variable_scope(self.scope):
for layer in range(self.num_layers):
gru_fw, gru_bw = self.grus[layer]
init_fw, init_bw = self.inits[layer]
mask_fw, mask_bw = self.dropout_mask[layer]
with tf.variable_scope("fw_{}".format(layer)):
out_fw, _ = tf.nn.dynamic_rnn(
gru_fw, outputs[-1] * mask_fw, seq_len, initial_state=init_fw, dtype=tf.float32)
with tf.variable_scope("bw_{}".format(layer)):
inputs_bw = tf.reverse_sequence(
outputs[-1] * mask_bw, seq_lengths=seq_len, seq_dim=1, batch_dim=0)
out_bw, _ = tf.nn.dynamic_rnn(
gru_bw, inputs_bw, seq_len, initial_state=init_bw, dtype=tf.float32)
out_bw = tf.reverse_sequence(
out_bw, seq_lengths=seq_len, seq_dim=1, batch_dim=0)
outputs.append(tf.concat([out_fw, out_bw], axis=2))
if concat_layers:
res = tf.concat(outputs[1:], axis=2)
else:
res = outputs[-1]
return res
================================================
FILE: sample_data/sample.charembed.txt
================================================
Y 0.273228 -0.422742 0.154817 0.260701 -0.0980259 -0.211214 -0.231156 0.098635 -0.131988 -1.42048 0.296819 0.22757 -0.252008 0.0940139 0.410244 -0.00936539 -0.148364 -1.22507 -0.0715888 0.190617 -0.0474986 -0.00811667 0.151874 -0.207459 0.185884 -0.241934 0.0646508 -0.0804561 -0.350576 -0.167038 -0.176406 -0.251458 0.0530961 -0.0582113 0.0256001 -0.0259313 -0.116083 -0.180181 -0.20367 0.353221 -0.04055 -0.050299 -0.0643419 0.0721474 0.407083 -0.0401198 0.278609 -0.278877 0.0806762 0.0194924 0.0634396 0.203195 -0.236354 0.23058 0.244124 -0.128595 0.318992 -0.346135 0.144452 0.111959 0.0203324 -0.0920697 -0.02784 -0.244591 -0.235814 -0.0857435 0.0164883 0.232046 0.227745 0.327455 -0.234145 0.124084 -0.195631 0.143333 0.161519 -0.0406187 -0.0500657 0.122136 0.356215 0.118454 -0.052973 0.16521 -0.0978034 0.0218739 -0.043792 -0.00808348 0.134504 -0.390658 -0.185923 0.184677 -0.022055 0.258343 0.0428807 0.17899 0.31459 0.411182 -0.110507 0.0558556 -0.337112 0.303737 0.152914 0.0217701 0.0915914 0.257085 -0.0430308 -0.324902 0.355415 -0.383755 0.0627576 -0.190388 0.289252 0.503951 -0.233088 0.188738 -0.48517 -0.0186042 0.0823368 0.0816004 -0.413855 0.246505 -0.0231189 -0.245208 -0.271153 -0.108787 0.316908 -0.206091 -0.0120366 0.0809189 -0.200749 0.424878 -0.236625 -0.214531 -0.0475734 -0.336846 0.0342122 0.0583956 0.0113792 0.0769178 -0.251193 0.12541 0.687179 0.0137081 -0.597457 -0.0952685 0.107251 -0.123227 -0.0669034 0.242318 -0.0730358 0.0597045 -0.215383 0.0245305 -0.150468 -0.0867538 0.0959263 0.0703843 -0.121586 -0.164161 -0.284556 -0.315421 0.142587 -0.0225364 0.125262 0.244728 0.296504 0.408041 0.359443 0.101347 -0.271349 0.0857058 0.368655 0.120178 0.234716 -0.457739 0.0461327 -0.129045 -0.0542913 -0.00240597 -0.0724887 0.26331 0.0371053 -0.248734 0.291676 0.189193 0.350622 0.113569 0.270612 -0.159546 0.0740794 -0.0348403 -0.0285623 0.190564 0.189016 -0.383509 -0.194866 -0.343181 -0.267861 0.137215 0.25375 -0.124499 -0.212442 0.20516 -0.148211 -0.0995658 -0.180864 -0.190831 -0.102183 -0.13119 0.057651 -0.033103 0.230872 0.309492 0.223463 0.119195 0.0291206 -0.095621 0.0995805 -0.273427 0.635907 -0.138081 0.172763 -0.116124 0.396234 -0.18321 -0.0668051 -0.0485386 -0.124531 0.311041 0.00670644 -0.54513 -0.00816306 0.239874 -0.207607 0.033632 -0.107991 0.264811 -0.0704034 -0.0477904 -0.107728 -0.0193663 0.0852454 -0.203177 -0.028578 -0.213157 -0.127573 0.0607251 0.160836 0.0020042 0.498921 0.00023542 -0.230862 0.022167 0.208157 0.0619848 -0.420923 -0.28115 0.147011 0.352333 0.109307 -0.0120332 -0.0595355 -0.483835 -0.0979524 -0.199016 -0.0328546 0.409816 -0.176899 0.277985 -0.608111 -0.0534234 -0.373298 0.0209329 0.0348306 0.422153 -0.319657 -0.044642 -0.155511 0.137939 0.0625065 -0.352055 0.093155 0.123266 -0.130472 0.283717 0.366981 0.364897 -0.089326 0.421994 -0.188589 0.118493 0.052634 0.0584408 -0.058704 0.367522 0.0951249 -0.0683492 0.369782 0.425945 0.193593 0.117144
F 0.181323 -0.367586 -0.148974 0.375991 -0.215051 0.178336 -0.138499 0.142261 -0.067079 -1.40225 0.19917 -0.140868 -0.0680895 0.391228 0.0658926 -0.0469452 0.336706 -1.16456 0.21112 0.459673 -0.267089 0.330568 -0.0239733 -0.194562 -0.388915 0.0892586 0.0414792 0.00536835 -0.101822 -0.29286 0.294356 0.240755 0.0416797 0.562502 -0.16025 -0.181532 0.0268602 0.0196838 0.195483 0.209183 0.184133 0.0814405 0.0631857 -0.0645846 -0.145243 -0.0570184 -0.0978382 -0.0452697 -0.0667391 0.233832 0.105349 -0.192496 -0.0853219 -0.0780681 0.0412666 0.0694489 0.0284339 0.0949413 -0.199618 0.0314374 0.0775029 -0.0654383 0.25072 -0.160872 -0.227864 0.0563717 0.0509413 0.0423587 0.351777 -0.0516549 0.13023 0.323176 -0.24339 0.132728 -0.0799921 0.0907703 0.0294809 0.429692 -0.00796785 -0.0233581 -0.257795 -0.156829 0.315156 -0.0785808 0.113077 0.396254 0.210869 -0.178943 -0.195334 -0.284097 0.0736572 0.0215486 0.338907 0.00705367 -0.0072909 0.062359 -0.42757 -0.141137 0.0990627 0.0823325 0.246951 -0.100012 -0.30618 0.304015 -0.0565938 -0.433008 0.020035 0.00220028 -0.13709 -0.200128 -0.224856 0.505688 0.0603656 -0.0348 0.00544377 -0.0338275 -0.0553999 -0.418488 -0.17251 -0.0431284 -0.121547 0.0476458 -0.0646573 -0.243455 0.180263 0.043294 0.394464 -0.353441 -0.712364 -0.0278541 0.168713 0.26239 0.00867887 0.163803 -0.0363926 -0.0600287 0.165583 -0.194106 0.371136 -0.148312 0.8247 -0.163434 -0.135129 0.221028 -0.069187 0.0859742 0.145038 0.0886666 0.0423489 0.553174 -0.183502 0.13021 -0.541537 -0.110732 -0.0768015 -0.174304 0.026846 -0.0253052 0.249364 0.0387014 -0.244885 -0.205078 -0.243753 0.0392924 0.345237 -0.0180184 0.0705166 -0.0957096 -0.452082 0.0708159 -0.0662071 0.103197 0.36611 -0.284553 0.216056 -0.0898405 0.0951325 -0.193126 -0.282438 0.0958877 -0.423061 -0.399675 0.215833 0.217166 0.231153 0.0102401 0.0780364 -0.572914 -0.292117 -0.24198 0.248412 0.248546 0.15198 -0.224715 0.130453 -0.11615 0.518509 0.0482156 0.280805 -0.117515 -0.456057 0.248823 0.0977158 -0.400019 -0.436797 -0.198814 0.105937 0.143762 -0.102061 -0.175226 0.084777 0.28369 0.186698 0.388156 -0.489864 -0.0778559 0.332956 -0.298626 0.322938 0.0231202 0.258133 -0.228118 0.148529 -0.231907 -0.178156 -0.163201 -0.0771549 -0.0369046 0.169023 -0.317798 -0.0379572 -0.194294 -0.336058 -0.0766741 0.15639 0.224286 0.319952 0.391513 0.210015 -0.235164 -0.216288 0.0571171 -0.134543 -0.0566089 -0.356347 0.441687 0.677745 0.143942 0.0901494 0.384575 -0.0668695 -0.0237228 0.523496 -0.0480187 -0.0799666 -0.57392 0.231489 0.162003 0.240839 0.232023 -0.00315335 -0.274903 -0.267854 -0.491405 0.236686 0.291687 -0.199964 0.0247361 -0.168292 0.114465 0.0396258 0.0245444 0.114152 -0.0422695 0.326608 -0.164494 0.130011 0.118381 0.0291718 -0.343782 -0.409372 -0.0870623 -0.359361 -0.0439144 0.142527 -0.00225151 -0.153862 0.139819 -0.180387 -0.0523852 0.330726 0.066681 -0.213377 -0.0994795 -0.137158 -0.238802 0.186167 0.234342 -0.149586 0.144857
d 0.310747 -0.44868 -0.0329828 0.0726109 -0.245013 -0.0477348 -0.204277 -0.0935042 -0.301948 -1.05966 0.423052 -0.157565 -0.393087 0.168164 0.133573 -0.0629206 0.11583 -0.88163 0.11133 0.261091 -0.286368 -0.00900965 -0.0332385 0.0755033 -0.140738 0.195873 0.0576766 -0.00203578 -0.242814 0.00970358 0.0706169 -0.260157 -0.16393 0.392391 -0.0624398 0.045898 -0.0151932 -0.00439554 -0.0986426 -0.0058753 0.0323047 -0.12148 -0.10748 -0.152248 -0.226612 -0.0348708 0.0772131 -0.0294754 -0.127496 0.185589 -0.0987717 0.0694208 -0.211201 -0.0753198 0.188667 -0.0786474 -0.0466422 0.140784 -0.0468092 0.374348 0.0122764 0.213842 -0.138321 -0.243883 -0.131388 0.278086 0.161967 -0.00486763 0.19295 -0.263554 -0.173287 0.160025 0.156363 0.037995 -0.200558 -0.14354 -0.0869376 0.165327 -0.377298 -0.132603 0.00307677 0.341914 -0.0570274 0.166826 -0.192985 0.273916 0.507052 -0.189697 -0.13428 -0.146905 0.0406364 -0.100123 0.0472507 0.216457 0.0512523 0.247197 -0.163923 0.260139 0.134187 -0.261138 -0.0528301 0.0284441 -0.301843 -0.288677 0.0512213 -0.574883 -0.062616 -0.251912 -0.11505 -0.086121 -0.216858 0.0838613 -0.276335 -0.0951659 -0.0529009 -0.115543 0.238387 0.00126458 0.049965 0.11539 0.122164 -0.244289 0.494465 0.13329 -0.167354 0.105681 -0.262817 -0.0957617 -0.147051 -0.0364248 -0.226408 0.104683 -0.258925 -0.0095446 -0.0213353 0.172413 -0.0528476 -0.0330454 -0.00210296 0.0925915 0.581611 0.0425508 -0.0867017 -0.0848503 0.0698587 0.0204551 -0.0069625 -0.128983 -0.0588751 0.0557412 0.0379992 -0.167348 -0.0223765 -0.114001 -0.163836 0.0621159 0.13734 0.0617552 0.0785929 -0.0641132 0.0420851 -0.304912 -0.0686793 0.403008 0.380253 0.0177989 -0.198949 -0.137942 -0.0863217 -0.0250705 0.392232 0.206115 0.0373908 -0.0193032 0.430078 -0.0716589 0.00566746 -0.131178 0.00525156 0.0970093 0.0181935 0.15051 0.237209 -0.0234499 0.0768707 0.158005 0.154618 0.0297203 0.0253173 0.152486 -0.151936 0.154896 0.0714721 0.00312185 -0.145397 0.059032 0.250173 0.0826228 -0.000925642 -0.296564 -0.147557 -0.00521982 -0.152476 -0.408938 0.170446 0.0609772 -0.131439 -0.0170924 0.0668658 -0.114273 -0.334692 0.148019 0.0227256 -0.335542 -0.273106 -0.357082 -0.0608098 0.211511 0.446749 -0.260934 0.0222667 -0.193566 -0.0395365 -0.0770541 0.0701472 0.167402 -0.0462758 -0.00632992 -0.0372712 -0.20775 0.10972 -0.197092 -0.220778 -0.043986 0.246131 0.09289 -0.108045 0.134835 0.272614 -0.431581 -0.0452758 0.346961 -0.0589501 0.087473 -0.0784508 0.192649 -0.126677 0.181287 -0.0660731 -0.0672873 -0.116361 0.0135331 0.196174 0.0302408 -0.18344 -0.375504 0.109311 -0.207996 0.202513 -0.0492957 -0.0817318 -0.386245 0.212933 -0.318107 -0.0189547 0.347626 0.0495122 0.364924 -0.37391 0.0917515 0.0386651 -0.25786 0.126652 -0.0128122 -0.230514 -0.342933 -0.319793 0.0929755 0.065635 -0.151007 -0.298741 0.0166219 -0.0575868 -0.00364547 0.118965 0.189292 -0.124889 -0.139221 -0.189635 -0.128717 0.253115 -0.076136 0.279359 -0.099269 -0.152493 0.149547 0.0782452 0.120672 0.000842545 0.0811712
N 0.291162 -0.434375 -0.159023 0.0431449 0.0608876 -0.0681778 0.0574885 0.124014 0.118372 -1.53834 0.0394419 0.352695 -0.260658 0.144807 0.255774 0.0667157 0.0352363 -1.0723 0.256418 0.101353 -0.0421885 0.122527 0.382697 -0.160295 0.147506 -0.433422 -0.0120515 -0.137362 -0.236525 -0.280547 -0.377849 -0.0819744 -0.00586242 0.0341689 -0.223323 0.0333133 -0.0202481 0.108348 0.0684649 0.178722 -0.0215156 -0.020731 -0.0661728 -0.0501308 0.201164 -0.0478742 0.30908 -0.349134 0.120264 0.00455598 0.30031 -0.156712 -0.146365 -0.0570163 -0.0503674 -0.31976 0.160286 0.142628 0.0105524 0.0284411 -0.0195566 -0.0275931 0.149945 -0.137088 -0.0381461 -0.124939 -0.139683 0.00919464 0.173995 -0.210937 -0.163423 -0.0233239 -0.277483 0.13491 -0.191472 -0.124729 -0.142222 0.184082 0.130671 -0.0617805 -0.187586 -0.359046 0.526056 0.250677 -0.306261 0.141884 0.284869 -0.39031 -0.31073 -0.279179 0.353208 0.0139471 0.089162 0.0914678 0.0501654 0.200536 -0.197894 0.258342 -0.353805 0.13157 -0.13421 -0.0540475 0.219589 -0.0779111 -0.011755 -0.587861 -0.102423 -0.245534 -0.0588509 -0.147234 0.149792 0.521226 -0.0524632 0.117636 -0.165377 0.141436 -0.197225 0.232586 0.180565 -0.201874 -0.166906 0.285654 -0.283792 0.187573 0.771959 0.101722 -0.0613706 0.0248435 0.0440755 0.308562 0.0256607 -0.0278602 -0.0213169 -0.233052 -0.145142 -0.0330192 -0.126384 -0.0251949 -0.00964793 -0.00821233 0.8293 -0.274858 -0.285102 0.188512 -0.216319 0.0180407 0.0643702 0.168087 0.218436 -0.0515469 -0.0776239 -0.00258956 -0.231027 0.105495 0.512892 0.010876 -0.0334675 -0.103695 -0.178384 0.120396 -0.227364 0.182711 0.115871 0.146065 0.252217 0.308096 0.203467 0.236381 -0.335482 0.0678624 -0.0629316 -0.197874 0.12653 -0.166691 0.330352 -0.00793378 0.291569 -0.0940921 0.00165524 -0.14346 -0.0829446 0.0587277 0.185977 0.295283 0.150576 0.00303932 0.269033 -0.0670812 -0.180088 0.137305 0.105989 -0.0251134 0.173514 -0.259867 -0.267342 -0.362452 0.102807 0.0466502 0.0642642 -0.337775 -0.256883 0.594942 0.104548 -0.412148 -0.514193 -0.148129 -0.180429 0.232444 0.252006 0.0556433 -0.104041 0.102172 0.167452 0.430726 -0.0582295 -0.565095 0.158338 -0.639132 0.623486 -0.113585 0.0485054 0.1285 0.303607 -0.626568 -0.388428 -0.0769128 -0.256687 -0.106396 -0.0415676 -0.493662 0.0318051 0.223821 -0.211112 0.0576398 0.288672 0.193412 0.33897 0.479571 -0.141124 0.0531796 -0.392719 0.0105848 0.020579 -0.170809 -0.0417512 0.352848 0.310637 0.170361 0.277958 -0.0919218 0.186247 0.0697815 0.154752 0.353367 -0.184462 -0.682667 -0.155491 0.234775 0.303058 -0.179318 0.0548961 -0.368741 0.10187 -0.29855 0.00605175 0.0575984 -0.167996 0.084102 -0.474149 0.0911199 -0.155221 0.468497 0.23428 0.204 -0.0299161 0.160973 0.0295833 -0.148852 0.226158 -0.302427 0.0491835 0.0532429 -0.158008 0.286059 0.209788 0.041772 -0.280611 0.316821 -0.35961 0.266354 0.293807 0.216581 -0.157335 0.157873 0.251667 0.282693 0.180125 0.0183858 0.169941 -0.118326
h -0.0848431 0.017307 0.139856 0.0188464 0.0596625 0.0404492 0.0843878 0.0401977 -0.0985024 -1.09029 0.234606 -0.205409 0.0167189 0.0904394 0.160135 0.0908654 0.25951 -1.13698 0.229183 0.131217 -0.0378102 0.104924 0.165588 -0.0455657 0.0940705 -0.207252 -0.0193487 0.153966 -0.0748572 -0.0971516 -0.134483 0.152309 0.0283294 0.132933 0.115449 0.065234 -0.389216 -0.0325015 0.0687195 0.227219 -0.127155 -0.184226 -0.139525 -0.105099 -0.060117 0.177394 0.326303 -0.220676 -0.147018 -0.212714 0.249592 -0.0187719 -0.20875 0.0991355 0.162795 -0.169179 0.138711 -0.156155 0.115403 -0.0993323 -0.0681188 -0.00867739 -0.197525 -0.195517 -0.247656 -0.223062 -0.175775 0.142835 0.183997 -0.122817 0.0207219 -0.0298245 0.0117663 -0.223258 -0.211239 -0.145259 0.0216693 0.115714 -0.0796018 -0.112318 -0.205016 0.102048 -0.0247308 0.295517 0.0143273 -0.206388 0.345369 -0.0818194 0.0737111 -0.176476 0.0949429 -0.0819706 -0.0724808 0.117291 0.0436067 0.20508 -0.232135 -0.294552 0.0121482 0.117061 0.038599 -0.080102 0.18335 -0.0946851 -0.138236 -0.557917 -0.356254 -0.0717145 0.128939 -0.295788 -0.303856 0.152649 -0.0738029 0.240313 -0.0224459 -0.00522536 -0.0988153 -0.252719 0.170184 0.0594306 -0.118404 0.107689 0.0800964 -0.211241 0.0101998 0.127396 -0.188328 -0.11608 0.123558 0.256397 -0.143804 -0.0750244 -0.0146507 -0.0769556 -0.0144469 -0.276941 -0.0806881 0.198255 -0.127925 0.188163 0.305369 -0.0992978 -0.251211 -0.143815 0.119154 0.271036 -0.105534 -0.048232 -0.15347 0.0174681 -0.166296 -0.35813 -0.0993651 -0.0513359 -0.16963 0.374995 -0.197071 0.143677 -0.19378 -0.0469003 0.0999497 -0.224742 -0.133449 0.33118 0.245467 -0.151692 -0.341649 0.0889677 0.0203779 0.0876033 -0.177387 0.0156631 0.173758 0.265171 0.040758 0.0244446 -0.27798 0.0357489 -0.100153 -0.149604 0.0234321 -0.31643 0.0357498 0.238284 -0.0809626 0.320402 0.0133578 -0.0950565 0.0878523 -0.143599 -0.170329 0.0619175 -0.161156 -0.173654 0.0883531 0.093275 0.306083 -0.200483 0.242546 -0.187939 -0.138844 0.00543927 -0.107999 -0.00637482 -0.131853 -0.20622 -0.0697164 0.140669 0.0710982 0.177878 0.330671 0.413053 0.017722 0.00736801 -0.208601 -0.0547906 -0.0339169 0.0555393 0.424692 -0.100377 -0.0320376 -0.549688 0.208342 -0.143034 -0.263409 0.172606 -0.0399704 -0.161257 0.190202 -0.0256098 0.188929 -0.0356427 -0.0468415 0.0986588 -0.0795492 0.0909881 0.149836 0.0760706 -0.0169741 0.431255 -0.285753 -0.0497785 0.0478033 0.17501 0.167629 0.344791 -0.0649688 0.260288 0.343633 -0.282162 -0.0556395 -0.122838 0.0239263 0.134653 0.0647078 -0.11698 0.25931 0.245937 -0.147456 -0.035989 -0.0453901 -0.235197 0.123949 0.0248241 0.0915013 0.380343 -0.0884197 0.0185254 -0.304244 0.133838 -0.268333 -0.0335776 0.136866 -0.0190716 0.110309 0.0401033 0.104143 0.374047 0.387326 0.138867 0.0266917 -0.177873 -0.33706 -0.150794 -0.0277758 0.109722 0.146453 -0.00955451 -0.187048 -0.134294 -0.0335818 -0.00239388 0.214578 -0.0879511 0.135675 -0.0305191 0.00359222 0.424549 -0.114267 -0.0743754
( -0.119791 0.382355 0.25952 0.43219 0.468319 -0.313418 0.155052 -0.0817113 -0.527027 -0.739011 0.351818 -0.0584554 0.0525227 -0.587104 0.190188 -0.00837673 -0.441619 -0.441876 0.454568 -0.0573818 0.0111339 -0.197482 -0.254739 0.250299 0.0810474 -0.0605122 -0.272267 -0.0470318 0.432586 -0.214722 0.879473 -0.529346 0.330509 -0.161025 0.515246 0.597901 -0.306658 0.308806 0.406835 0.00478476 -0.653436 -0.290155 -0.331989 -0.0577573 -0.0978074 0.230244 0.176261 0.663062 0.183317 -0.178064 0.313562 0.0602862 0.38522 -0.181347 0.0327767 0.326124 -0.342104 -0.0300249 0.780757 0.116374 -0.556859 0.238482 -0.0193189 0.356989 0.111837 0.365366 0.330517 0.352086 0.161949 -0.205289 -0.212788 -0.588561 -0.285304 0.0368649 0.233813 0.2801 -0.391703 -0.239799 0.215013 -0.20892 -0.8787 0.000684961 -0.272538 0.129118 -0.0802772 -0.0300186 0.693988 -0.247973 -0.149908 -0.395601 -0.712466 -0.00360663 0.204092 -0.192205 -0.217227 0.471578 -0.49301 0.185188 -0.0867334 -0.0611491 0.597353 -0.376062 0.514454 0.15909 0.379221 0.687233 -0.140222 -0.308838 -0.501952 -0.243628 0.731345 -0.225746 0.211138 0.420457 0.240455 -0.106356 0.429128 -0.647395 0.248701 0.227947 -0.287646 0.405534 0.213051 -0.28702 -0.0716141 -0.20065 0.866678 0.242574 -0.129011 0.202272 0.0796565 -0.0294295 -0.074857 0.11178 -0.498464 0.328857 0.458095 -0.0627604 -0.672936 -0.514918 1.36779 0.369349 -0.539757 -0.0395167 0.340893 0.508462 0.359809 0.0480626 -0.505937 0.176492 0.445077 0.424388 0.240929 0.249747 -0.266346 -0.292798 0.368693 -0.435787 0.175265 0.169758 0.419586 -0.775416 0.421546 -0.119335 0.381623 -0.104179 0.310475 -0.495922 -0.551236 0.210597 -0.433805 0.463288 0.0787742 -0.329106 0.0371543 0.205698 -0.209837 0.110358 -0.647636 -0.0869918 0.099148 0.219955 0.713454 0.0647623 0.188638 0.455419 -0.222045 0.222953 0.327113 0.450796 0.373199 -0.362486 0.472844 0.225307 -0.464249 0.302946 0.295401 0.167513 -0.0624749 0.0349617 0.283787 0.745247 -0.160961 -0.227253 0.103414 0.239925 -0.0458566 0.150675 0.380158 0.0868739 -0.109897 0.0937 0.347593 0.0584865 0.146333 -0.0258466 0.252408 -0.103075 0.0426119 0.445023 0.5978 -0.155742 -0.0794989 0.14472 -0.376525 -0.631387 -0.421814 -0.078039 0.182369 -0.320238 0.195368 0.165584 0.348763 0.190159 -0.0617506 0.24317 0.276402 -0.310153 0.312058 0.194429 0.226145 -0.172845 -0.0586392 0.631761 0.286891 0.378206 0.158102 0.55505 0.296764 -0.0697635 0.0269051 -0.0724758 -0.183083 0.0402969 -0.042385 -0.030537 -0.0728777 -0.660672 0.112686 -0.115087 -0.032257 -0.0177665 0.123507 -0.348359 0.017253 0.237928 0.0800013 0.336666 -0.248342 0.34338 0.0124208 0.702465 0.336398 0.11534 -0.365504 0.451957 0.0146705 -0.213131 -0.699404 -0.802594 -0.0940069 0.13384 0.7176 0.244118 0.363004 0.555909 -0.289843 -0.181629 -0.218274 0.0165136 0.216479 0.107973 -0.0626752 0.534279 -0.194956 0.00213291 -0.139514 -0.271164 0.0454798 0.16489
C 0.104529 -0.499499 -0.101365 0.249708 -0.302233 -0.230812 -0.204295 0.325053 0.196708 -1.43584 -0.0281441 -0.330715 -0.183479 0.281321 0.442991 0.172545 0.126539 -0.925755 -0.213739 0.192825 0.0343265 0.131452 0.045316 -0.0555473 -0.278014 -0.238927 0.107795 0.104435 -0.310393 -0.13519 0.0211487 -0.207075 0.30901 0.0187082 -0.354474 -0.0897501 0.287291 0.152778 0.229046 0.169018 0.226612 0.0904842 0.281208 0.149708 0.0882055 0.151563 0.393114 -0.296989 0.134902 0.0903278 0.0610515 -0.272079 0.260104 -0.426797 -0.421921 -0.0879703 -0.074838 0.000781038 -0.423464 -0.155029 0.0931217 0.428017 0.00684994 -0.314712 0.00348289 0.287156 -0.301483 -0.107188 -0.0709538 0.0323598 -0.144677 -0.192929 0.0564781 -0.0489195 -0.0420762 0.169248 -0.0943203 -0.258494 -0.0680419 -0.253731 0.192449 -0.157134 0.234435 0.0611296 0.257587 0.168178 -0.000937229 -0.000458296 -0.184508 0.0400291 0.0710508 0.333685 -0.0640883 0.0168858 -0.0256965 0.300205 -0.206786 0.14823 0.330721 0.278623 0.107694 -0.429321 0.434568 0.23178 -0.0347311 0.0262713 0.180078 -0.0967114 -0.146707 -0.00161506 0.110746 0.36212 -0.00216274 -0.124427 0.198187 0.24308 -0.169742 0.172904 -0.141628 0.324096 0.158044 -0.203695 0.00686652 0.395408 0.00812475 -0.157453 -0.253685 -0.0990791 0.175668 -0.164812 -0.0161288 0.257288 0.236052 0.0203023 0.153222 -0.176527 -0.155114 0.106282 0.0204477 0.326602 1.00996 -0.161701 -0.510595 0.315333 0.190599 -0.19756 -0.178716 0.200927 0.126715 0.151698 0.0330538 -0.127949 0.0326833 -0.0452144 0.297305 0.268583 0.0927043 -0.141159 0.0916962 -0.090985 0.100481 0.124392 -0.558836 0.410965 0.272786 -0.105459 -0.129866 0.0538817 0.0109357 -0.178974 0.160185 0.0459471 0.153922 0.144296 -0.00769301 0.17453 -0.127239 0.115094 -0.14912 0.185853 -0.172761 -0.00848869 0.0664989 -0.134192 0.366453 -0.182316 -0.0387801 -0.164209 0.00826278 -0.216339 0.0751804 -0.10853 0.115928 0.100658 0.216865 -0.310641 0.0249478 0.130916 0.201386 -0.345993 0.160404 -0.168595 -0.0191108 0.0494793 -0.0176767 -0.20612 0.268856 0.274747 0.00171549 -0.0111615 0.362377 0.358881 -0.045115 0.436297 0.00215962 0.323697 0.259329 -0.0256014 0.249949 -0.277699 0.107982 -0.320845 0.187207 -0.287791 -0.256507 -0.30245 -0.0361469 -0.245431 0.110415 0.0797096 -0.093044 0.212366 -0.31017 0.00747907 0.0521479 -0.112994 -0.200453 -0.307107 0.158906 0.0877512 0.11433 -0.215067 -0.362569 -0.566129 0.0605334 0.0579411 0.227887 0.0233977 -0.0817505 -0.0302019 0.11261 -0.0670169 -0.133973 0.0765426 -0.182122 -0.494934 -0.504888 0.151669 0.123028 0.129836 0.0576558 -0.134731 0.029875 0.215492 0.221011 0.115664 -0.0925476 0.0672608 -0.271553 -0.0238374 -0.136095 0.0885542 0.28089 0.843096 -0.233882 -0.0219055 0.239463 -0.0650395 -0.207089 -0.365492 -0.0191953 -0.267408 0.179106 0.0903489 0.190157 0.085251 -0.174573 0.0602912 0.263265 0.206227 0.189496 0.00586508 -0.085171 -0.159938 0.0263642 0.0858499 0.165632 -0.0283224 -0.397412 -0.152784
w 0.080462 -0.27895 0.0540439 0.16825 -0.00467234 0.329568 -0.334676 0.141931 -0.0376278 -1.15158 0.351833 0.158013 -0.224549 -0.212966 -0.065163 0.388275 0.0641221 -1.0257 0.378002 0.332516 -0.0793868 0.151713 0.0722797 0.141198 -0.0120409 -0.175442 0.355352 -0.0814845 0.0560412 0.252353 0.17477 0.0839922 -0.369649 -0.0660003 0.138335 0.457176 0.0383137 0.186554 -0.0397989 0.259961 -0.199511 -0.386079 -0.323403 -0.341352 -0.251878 -0.0820714 -0.215542 -0.242044 0.0298891 -0.408106 0.0914545 -0.108216 -0.0352354 -0.147464 0.229985 -0.29873 -0.141143 -0.00857065 -0.0127224 0.0127167 0.0190959 0.183047 0.21058 -0.274666 -0.216646 0.149038 -0.131566 -0.230056 -0.253619 -0.0740131 0.48404 -0.0489327 0.0282178 0.136983 -0.421658 -0.0563245 0.0717079 0.142186 0.424685 0.125516 0.159703 -0.15536 0.0137287 -0.481032 -0.160039 0.0200449 0.144657 0.0236088 -0.195285 -0.237591 -0.406834 -0.39062 0.00818486 0.483773 -0.185161 0.422347 0.235952 0.194638 -0.463959 0.00355097 -0.364165 -0.0120644 0.00726419 -0.0266209 -0.0421238 -0.667453 -0.0326631 -0.0911894 0.0285282 0.144872 0.254112 0.131786 -0.0069265 0.279637 0.271879 0.19703 0.272929 -0.256185 0.0813347 -0.205874 -0.101426 0.09858 -0.792751 0.123377 0.330976 0.472596 -0.114672 -0.281835 0.433456 0.040491 -0.170504 -0.143683 0.0345587 -0.0564094 -0.465801 -0.380846 0.0233508 0.248865 -0.253039 -0.0854827 0.45343 -0.13529 -0.322548 0.043437 0.212573 0.18411 0.00915548 -0.138555 0.00798763 0.249048 -0.210736 -0.161724 0.0110964 0.271589 0.209874 0.232086 -0.141307 0.0932497 0.0653888 -0.105908 0.336344 0.0557949 -0.146461 0.570249 0.0425334 0.269043 -0.0326435 0.126546 -0.251663 -0.157405 -0.0480258 0.133485 0.349035 0.0583535 0.245651 0.229401 -0.094013 0.0845842 -0.0598052 0.0915827 0.544094 -0.195954 0.0275714 0.0512827 0.276577 0.0359066 0.260201 0.165776 -0.0577644 -0.195566 0.448359 0.0475095 -0.141685 -0.14414 0.0744765 0.00742196 0.563149 -0.010441 0.524941 -0.397889 0.155172 0.153962 -0.016299 -0.0419035 -0.155229 0.190934 -0.073668 -0.362059 0.0506704 0.213541 0.0245081 0.0195633 -0.350161 0.134588 -0.20949 -0.289873 0.0776259 -0.0297693 0.522498 -0.00711947 0.303874 -0.194156 0.624839 -0.105166 0.0978151 -0.41697 -0.546274 -0.309738 -0.696781 -0.0441633 0.388152 0.0793086 0.276932 0.257909 -0.00547571 -0.0516638 -0.146795 -0.00140719 -0.00642293 -0.281703 -0.0631174 -0.00921594 -0.0904589 0.00450081 -0.0197805 0.0845185 0.13872 -0.14846 0.233767 0.152852 0.114338 -0.041954 0.14183 0.317408 -0.0526166 -0.540794 0.198553 0.185326 -0.446657 -0.343905 -0.304159 -0.457655 0.282179 -0.313206 -0.0219551 0.261842 0.0777195 0.518244 -0.432062 0.48882 -0.213237 0.258039 0.374257 0.0020955 -0.0535176 0.637484 -0.0227007 -0.288724 0.131724 -0.019195 -0.336784 -0.0147015 -0.313986 0.0412132 0.0163124 -0.094883 -0.406405 0.238946 -0.0636179 -0.0486814 0.400838 -0.320107 0.0577636 0.416607 -0.0225134 0.168611 -0.114196 0.285438 0.680982 0.0578169
@ 0.262143 0.133472 -0.0641865 0.383067 -0.521202 0.167345 0.0637421 -0.0931714 0.0292438 -1.48791 0.450746 0.0376681 0.077662 0.0531474 0.0674171 0.113466 -0.347786 -1.06909 0.498755 0.369015 -0.117828 0.638249 0.34868 0.163581 -0.671628 0.0678928 0.289059 0.636817 -0.408794 0.223949 0.188235 0.106118 0.296239 -0.0780114 -0.645171 0.549131 0.385608 0.433482 0.186782 -0.0587083 0.254782 -0.205901 -0.157322 -0.208023 0.11746 0.0577404 -0.493199 0.516632 -0.269724 0.427691 0.337406 0.506558 -0.0607667 -0.699861 -0.0546214 -0.576627 -0.225592 0.5117 0.203733 0.325741 -0.676534 -0.196703 0.0302698 -0.543207 -0.271937 0.575118 -0.0421009 0.43254 -0.00470676 0.067526 -0.308156 -0.541006 0.214319 0.0262785 -0.069384 0.0875097 0.222876 0.0607303 -0.12595 -0.0735647 -0.167105 0.57622 0.217527 0.176789 0.252364 0.44113 0.826337 -0.51289 -0.229663 -0.0551073 -0.00657224 -0.252078 0.203509 0.176367 0.121104 0.580646 -0.439911 -0.0667412 0.366073 0.0481318 0.316321 -0.843073 0.393659 0.232677 0.347384 -0.174851 -0.223883 -0.617592 -0.881955 -0.152622 -0.0287924 -0.19548 0.0975083 -0.112443 -0.275168 0.313272 -0.0156642 -0.551268 0.0590966 0.2341 -0.131731 -0.446478 0.396437 -0.422955 0.228501 0.0697329 0.26625 -0.481036 -0.412403 -0.0388883 -0.0580702 -0.151809 0.145139 0.142076 -0.494072 0.788964 0.159049 -0.0343841 -0.496635 -0.0768709 0.806367 -0.0459761 -0.692576 0.0596648 -0.17751 -0.0486082 0.0750244 0.555455 -0.422618 0.377964 0.123492 0.14469 -0.571739 -0.0345595 0.431892 -0.435933 0.456143 -0.674936 0.523316 0.241042 -0.140615 -0.262913 -0.145046 0.159824 0.269312 0.562013 0.1346 -0.20493 0.186545 0.11893 -0.068259 0.351049 -0.0538834 -0.427696 0.343799 -0.251969 0.432942 -0.295278 -0.32193 -0.542718 -0.377264 0.181297 0.143055 -0.52514 0.362105 0.126207 0.22596 -0.320487 -0.167436 0.375328 0.402789 -0.171994 -0.265706 -0.312388 -0.0732068 -0.334627 0.2278 0.154792 0.00128008 -0.377921 -0.120236 0.263855 0.425794 -0.287422 0.0133159 0.293385 0.627825 -0.252256 0.18171 -0.77024 0.414855 -0.0423567 0.458814 0.27995 -0.403112 -0.395886 0.221607 -0.00272782 0.548491 -0.12932 0.192997 0.458074 -0.0273301 0.0198513 -0.10535 -0.264669 -0.427422 -0.0130167 -0.215952 -0.643721 0.333583 -0.164256 -0.331009 -0.480113 -0.28728 0.76616 0.052931 0.0460101 0.333633 -0.150978 0.218884 0.0465193 -0.259601 -0.191138 0.0512207 0.232414 0.257016 0.449695 0.415205 0.309435 0.326904 0.104064 -0.293994 0.39932 -0.737983 -0.396011 -0.0536592 0.336765 0.0628734 -0.260792 0.0363209 0.152232 0.0157426 -0.799544 -0.393341 0.193838 0.116278 0.434076 -0.0706146 0.252151 -0.145674 -0.173547 -0.220114 0.309574 -0.185103 -0.160037 0.244182 0.0491335 -0.441975 -0.392329 -0.667213 -0.24806 -0.0940022 0.141814 -0.00802829 0.222155 -0.120608 -0.111514 0.0880323 0.542227 0.763857 -0.0725129 -0.193937 0.26024 -0.045939 -0.605646 0.479827 -0.0754609 -0.535565 -0.568861
V 0.245919 -0.320166 0.0205741 0.180891 -0.0441605 -0.143596 0.0779463 -0.0449221 -0.071371 -1.5764 0.684404 0.0316094 0.105152 0.0872989 -0.200637 -0.09389 0.0774146 -1.04285 0.696436 0.204459 -0.412257 0.246522 0.215724 -0.103442 0.0129654 -0.104499 0.158164 -0.206245 -0.0323946 0.0684775 0.020129 -0.00767796 -0.651574 -0.121639 -0.593437 0.138754 0.291396 -0.0425329 -0.295572 0.0270716 0.0132434 0.0751237 -0.488149 -0.36737 -0.221726 0.205319 0.236698 -0.129335 0.0237288 0.00436258 -0.0787689 -0.141183 -0.470544 -0.10125 0.0686254 -0.164531 -0.0807896 -0.0519911 -0.0754322 -0.0710197 -0.0446063 0.105891 0.404221 -0.346213 0.0864488 -0.315353 0.0213912 0.26242 -0.0666462 -0.212046 -0.288138 0.0801184 -0.159696 0.272995 -0.437378 -0.156401 -0.114738 0.145729 -0.103551 0.307774 0.0875685 -0.250371 0.240718 0.177815 0.100077 0.254017 0.338944 -0.18444 -0.22084 -0.246379 -0.132216 0.0405981 0.420131 0.0785066 -0.26368 0.0212599 0.103008 0.147887 -0.306547 -0.0847221 -0.194703 -0.262272 0.273869 -0.237433 -0.119066 -0.20533 -0.26065 -0.365926 -0.180957 -0.174053 0.127324 0.254582 0.289331 -0.203764 -0.162176 -0.211725 -0.164347 0.233672 0.0706434 -0.017202 0.0667045 0.308203 0.0827453 0.277889 0.228671 0.219239 -0.403581 0.161269 0.171506 0.250107 0.375442 -0.00819566 0.0158243 -0.3419 -0.551042 -0.159874 0.0873052 0.316495 -0.0228277 -0.0180818 0.831678 0.141959 -0.626459 0.307685 -0.078549 0.182401 -0.0811347 -0.173733 0.377956 -0.165865 -0.190025 0.0878009 0.19258 -0.175705 -0.00355483 -0.132578 0.0430053 0.0467484 0.136126 0.00714257 -0.358038 -0.0654741 0.103618 0.2087 0.379059 0.363807 0.086672 0.196966 -0.148177 -0.0264437 -0.351329 -0.422428 -0.123929 -0.0172738 0.436135 -0.0404488 0.187981 0.237988 0.0953504 0.00465118 0.116239 -0.0509639 -0.173573 -0.0293315 0.374322 -0.120858 0.064547 0.00173857 -0.516107 -0.14313 0.538661 -0.00652814 0.238128 -0.377225 -0.214811 -0.282933 0.155704 0.0123677 -0.106074 -0.181935 -0.10216 0.123261 -0.115749 -0.257129 -0.530342 -0.37838 0.216337 0.148413 0.0510395 -0.0418015 0.320258 0.138423 0.0819376 0.322444 0.135069 -0.538372 0.252987 -0.47802 0.632854 -0.253318 0.0340616 -0.0280023 0.130631 -0.695317 -0.00088077 -0.0888097 -0.065702 -0.204696 -0.109588 -0.224383 0.441924 0.0579164 0.0721374 0.219958 0.243619 0.254548 0.170365 0.054459 -0.139293 0.0527887 -0.353024 -0.236489 -0.103401 -0.389203 -0.199709 0.268656 0.117061 0.0243571 -0.15482 0.0079878 0.127698 -0.145826 0.360273 -0.163582 -0.416697 -0.795819 0.136266 0.335542 0.090458 -0.452383 0.0358812 -0.564106 0.260161 -0.225699 -0.172336 0.196874 0.203416 0.200076 -0.763149 0.0421677 0.00127578 0.455294 0.344471 0.165572 0.0802721 0.293205 0.0313453 0.0802367 -0.0775037 -0.662911 0.0797854 0.0486161 0.22591 0.236839 0.23838 0.111019 0.00594488 0.284098 -0.292293 0.0936284 0.238255 0.134562 0.0381181 0.388647 0.145591 0.0926058 0.269067 -0.340695 0.0909082 -0.244857
z -0.0643207 -0.113394 -0.174918 0.351298 -0.233671 0.125906 0.0637483 -0.0411032 -0.342783 -1.39831 0.22419 -0.045622 -0.240355 0.480637 0.238413 -0.377948 0.375227 -1.34997 0.321516 0.136706 -0.252205 0.0603271 0.0528632 -0.340928 -0.259769 0.0845208 0.185567 0.146144 -0.0382612 0.246484 0.0377442 -0.567506 -0.004562 -0.172323 -0.240139 0.0308636 -0.364165 0.0788069 -0.135127 -0.22752 0.267078 -0.4143 -0.137659 -0.164955 0.0896877 -0.0467838 -0.0983885 -0.063178 0.0563677 0.172477 0.0332808 0.0495218 0.00691918 0.350485 0.1697 0.0968402 -0.0361356 -0.0138036 -0.155212 0.306551 0.329914 0.266743 -0.187632 -0.171631 0.120537 -0.0704159 -0.332022 -0.298592 0.201719 -0.0886569 0.153224 -0.203929 -0.0812126 -0.12992 -0.180929 -0.428757 -0.135006 0.324337 -0.27207 0.139964 0.180442 0.281592 0.317672 0.178559 -0.239727 0.291137 0.581628 -0.312576 0.103031 -0.0679244 -0.0912898 0.0324352 0.0121174 0.261565 0.0812931 0.720729 -0.47878 -0.146988 -0.224679 -0.0359036 0.538116 0.405808 -0.147275 0.378885 0.14714 -0.465243 -0.104894 -0.385622 -0.1613 0.119076 -0.142485 0.0869228 -0.445611 -0.265302 -0.140551 -0.224685 0.0743655 -0.25819 0.157929 0.295067 0.0872805 0.103419 0.354252 0.144721 -0.0955039 -0.070437 -0.218176 -0.263711 0.198376 0.326964 -0.419465 -0.043139 0.232025 -0.0914144 -0.0525968 0.320863 0.00135745 -0.238893 -0.0586902 0.00123981 0.655636 -0.152767 -0.589681 -0.016417 0.245399 0.322115 0.138913 0.246557 0.113845 0.414699 0.108693 -0.33108 0.199112 -0.300032 -0.00575166 0.178952 0.0518311 -0.0583925 0.12593 -0.312613 0.204435 -0.045555 -0.0652833 0.0545077 0.56497 0.0466235 -0.287544 0.376356 0.263144 -0.275951 0.128267 0.253305 0.0451502 0.0702296 -0.124097 -0.179566 -0.159038 -0.0271909 -0.381293 0.00796656 0.0256228 -0.005321 0.145147 -0.10563 0.139838 0.0700185 0.245084 -0.313101 -0.0823575 -0.0406254 0.326726 0.0579145 0.54044 -0.351972 0.183173 0.136233 0.34861 0.225811 0.330931 -0.841829 -0.497046 0.0259762 -0.217916 -0.228509 -0.144214 -0.0076264 0.0831584 0.202458 0.0577866 0.0628271 0.258466 0.243034 0.148208 0.21918 -0.113358 -0.222786 -0.30767 0.0423804 0.540566 -0.0598233 0.0610341 -0.00679448 0.614412 -0.371438 -0.331851 -0.376737 0.243013 -0.219338 -0.0388791 -0.371599 -0.223788 -0.0376031 -0.117805 -0.652094 -0.160077 0.287505 -0.309182 0.0350845 0.0974991 -0.145658 -0.21361 0.046137 -0.00580024 -0.0374183 0.0102967 -0.169725 0.285312 -0.0778329 0.236131 0.0383962 0.252529 -0.279481 0.091689 -0.185942 0.0919928 -0.327612 -0.173847 -0.163227 -0.266776 -0.633331 0.241994 -0.281318 0.23379 -0.465677 -0.078373 0.168644 -0.0699954 -0.116219 -0.637456 -0.225467 -0.292908 -0.461514 0.121564 0.302103 0.0308911 -0.318588 -0.0925748 -0.127109 0.109126 0.0565153 0.0349118 -0.54464 -0.129496 -0.00944281 -0.192054 0.237284 0.195632 0.140866 -0.0681502 -0.00123997 0.255085 -0.180699 0.0733165 0.11779 0.189421 -0.0562599 -0.357702 0.258449 -0.0706191 0.207871
Z 0.491116 -0.0182894 0.143066 -0.00342799 -0.20131 0.049397 0.0122978 0.224019 -0.135023 -1.57131 0.723799 0.384642 0.086653 -0.0491608 0.192988 -0.16084 0.16057 -1.15014 -0.176954 -0.30554 0.373318 -0.0302955 0.594569 -0.315672 -0.0711962 -0.0165734 -0.0251692 -0.122828 -0.184156 0.111644 -0.117662 -0.0333196 0.0619125 -0.235567 -0.581171 -0.0723071 -0.289433 -0.201871 -0.253061 0.329779 -0.00396108 0.262424 -0.399817 -0.262478 0.128727 0.22859 0.13341 0.0253388 0.141107 0.309793 -0.404169 0.255109 -0.0778302 0.11113 -0.0627495 -0.286617 0.318259 -0.24983 -0.130891 0.314214 0.415705 0.0725328 -0.10589 -0.575857 0.0925851 -0.147276 -0.0753644 0.135464 -0.0449337 -0.0466066 -0.436231 0.0666346 -0.0721282 0.328973 0.017864 -0.451883 0.0693836 0.136436 0.242864 -0.0739484 -0.00203565 -0.585703 0.235012 -0.218958 0.263868 -0.0159128 0.31973 -0.253352 -0.365121 -0.31182 0.0065261 0.197988 0.572641 0.202504 0.0504018 0.366971 -0.0966351 0.127872 -0.154922 -0.0899259 0.23216 -0.13961 -0.349935 0.0886937 0.0655081 -0.579816 0.0251055 -0.430621 0.309819 -0.169864 0.122904 0.534599 -0.134435 0.001312 -0.356919 -0.0367701 -0.336697 -0.0577481 0.164787 0.198911 0.0971408 -0.303393 -0.0680292 0.224591 0.18482 0.0693795 0.00147151 0.306582 0.305643 0.253207 -0.30658 -0.26395 -0.165058 -0.297286 -0.2229 0.120007 -0.223662 0.0904717 0.0856483 0.302992 0.972798 0.0163161 -0.432597 0.00248672 0.0425123 0.324863 0.102892 -0.272412 0.619034 -0.155616 -0.155536 0.339343 0.12872 -0.322465 0.0641447 -0.162077 0.19324 -0.627074 -0.588028 -0.00368317 -0.215855 0.061979 0.0594774 0.483937 0.351062 0.510221 0.31136 0.489912 -0.278899 -0.351999 0.106899 0.0544438 0.399339 -0.311769 -0.0599509 -0.19719 -0.0252795 -0.076726 -0.435017 0.127138 -0.0319227 0.13986 0.285648 0.0431237 0.619114 -0.521673 0.186772 0.0080306 -0.0108131 0.0109883 0.458435 0.314213 0.619132 -0.283862 -0.784305 -0.164318 -0.237183 0.356145 -0.170942 -0.171201 -0.232855 0.456175 -0.351787 -0.462191 -0.0620942 -0.181821 0.298774 -0.233947 -0.147508 0.0408463 0.120338 0.214081 0.0309375 0.286503 0.290286 -0.276953 -0.00563711 -0.39714 0.566689 -0.264066 0.279778 0.314556 -0.109827 -0.322061 -0.313154 -0.342594 0.112943 0.0448467 -0.322873 -0.62879 0.0883109 0.165282 0.0780007 0.0979027 0.256661 0.437755 -0.190863 0.045803 -0.214209 -0.275908 -0.0864183 -0.233614 -0.192327 -0.218387 0.280304 -0.0875238 0.388906 -0.348392 0.449245 0.316424 0.0230031 -0.0481353 0.364576 0.158626 -0.396049 -0.629903 0.253481 0.239481 -0.0413802 -0.368641 0.0461643 -0.397279 -0.287096 -0.136627 -0.239489 -0.00189562 -0.265079 -0.267891 -0.586572 -0.0980136 0.135106 -0.370219 0.10889 0.295885 -0.0447352 -0.118031 -0.218494 -0.141808 0.514777 -0.631478 0.165343 -0.0693941 0.30711 0.151602 0.431525 0.00923302 -0.370852 0.376083 -0.310052 0.371622 0.215765 0.249982 0.0200198 0.204748 0.299593 0.0447592 0.202526 0.10803 0.440956 -0.118999
e 0.0938929 0.0438122 0.180603 -0.211526 -0.1535 0.25801 0.239995 0.117473 -0.0898593 -0.753143 0.226411 -0.109773 -0.0573605 -0.0535958 0.0739606 0.0418029 -0.0680152 -1.09723 0.293644 -0.00839204 -0.0333872 0.0451012 0.20006 0.0140127 -0.123378 0.328976 0.0116332 0.0107122 0.0243885 -0.12451 -0.0649838 0.0425535 0.102614 0.0880838 0.00436495 -0.0236085 0.0612419 0.298157 0.00511234 -0.00247336 -0.200061 0.157561 0.0422123 0.197872 -0.012368 -0.181187 0.187593 -0.0183816 -0.0330786 0.204469 -0.0432003 -0.0442657 0.155718 -0.123926 0.0781035 -0.164564 -0.200335 -0.167405 0.0793642 -0.121576 -0.0957865 0.0135311 0.101564 -0.252187 -0.0299957 0.168765 0.101055 -0.0801818 -0.0478405 -0.0892786 -0.109944 0.151402 0.0765702 -0.115384 -0.0797752 -0.0434167 0.307931 -0.0506271 0.0349057 -0.108749 0.148791 0.0779474 0.00142498 0.00217369 0.167552 0.122136 0.28259 0.0228276 -0.232283 0.0435047 0.147258 -0.227748 0.0163965 0.138719 -0.124391 0.24139 0.0769213 0.0675201 -0.0885802 0.108524 -0.267676 0.0601159 -0.032403 -0.198359 -0.0344364 -0.418066 -0.0867505 0.0191616 -0.274439 -0.317715 -0.0428804 0.194327 -0.16754 0.1313 0.266264 0.185005 -0.0132036 0.131837 0.244171 -0.215744 0.0549577 -0.0677163 0.0210947 0.147846 0.124752 -0.0168355 -0.15389 0.127968 0.0949851 0.217103 -0.208416 0.0459661 0.182691 0.175625 -0.0314087 -0.154071 0.0615992 0.196456 -0.175734 0.124751 0.28007 0.0679942 -0.112197 -0.0565889 -0.144505 -0.0543731 -0.0266588 -0.0658977 -0.0586758 0.0510958 0.228196 0.111898 -0.0783984 -0.225972 -0.0383505 -0.0173358 0.334825 -0.0680416 -0.0862043 0.159505 0.0421828 0.132442 0.0599057 0.047734 0.287171 0.0531176 -0.0402822 0.0868054 0.0904948 0.0187633 0.111082 -0.0572544 0.01935 0.36277 0.0175067 -0.0601589 0.0585271 -0.313839 0.0277396 -0.13356 0.154185 -0.0940542 0.0543505 -0.0276504 -0.0237787 -0.0213557 0.19997 0.139375 0.10175 0.0756522 0.119828 0.161812 -0.0891059 0.116583 0.0468355 0.143625 0.189196 0.0348814 0.0917302 0.0865131 -0.139094 0.000168357 0.0970736 0.0119111 0.0733052 -0.0683933 0.147455 0.0827544 0.136399 -0.221663 -0.0191107 -0.1899 0.103732 -0.0415312 -0.221458 -0.161348 0.0957494 0.152411 0.293593 -0.127295 0.0321077 0.0429621 0.125723 0.0350295 -0.00871684 -0.116514 -0.193863 -0.174224 -0.202054 0.026668 0.450537 0.214073 -0.0531452 0.19081 -0.139795 -0.0470463 0.0424981 0.0494485 0.225188 -0.08834 0.10861 0.0668919 -0.20319 -0.148155 -0.237883 -0.00689361 -0.0453377 0.0996504 0.105487 -0.031059 -0.258276 -0.0149419 -0.0130084 0.162348 0.0772549 -0.388261 -0.0467132 0.0941905 0.178199 0.246494 -0.255802 -0.00127591 0.0143107 -0.0611472 0.0345329 0.134286 0.11375 0.257863 -0.178635 0.261299 -0.0970252 0.0115158 0.160775 -0.0202566 0.119683 0.0410129 0.217721 -0.0598902 -0.0178928 -0.0758813 0.00564036 -0.0726113 0.193857 -0.194418 0.0269626 -0.0611202 -0.0638048 -0.0415881 -0.146958 -0.0225477 0.177558 -0.0247224 0.181428 -0.109558 -0.0917079 -0.121332 0.180425 -0.041553 -0.154244 -0.128963
i 0.315118 -0.194888 -0.106185 0.399666 0.111958 -0.0266824 0.333846 0.181474 -0.06485 -0.945529 0.032739 -0.200394 -0.166566 0.0790137 0.289893 0.0658432 -0.0967318 -1.10814 -0.137787 0.26547 -0.184423 0.118472 -0.328706 0.0979517 -0.279085 -0.045333 0.0841331 -0.262138 0.0911083 -0.157725 0.187271 -0.109564 0.0519614 -0.211293 -0.298974 0.227995 0.331239 -0.0284657 -0.00335967 -0.110978 -0.182307 0.229881 -0.243977 0.242082 -0.0309073 -0.201728 0.00464637 -0.0849722 0.15941 0.0547659 0.112978 -0.33653 0.0685544 0.0802985 0.111602 0.216085 0.0688906 -0.156201 -0.0185842 0.155149 0.0600158 -0.102318 0.0381572 -0.422425 0.224897 0.348546 -0.332876 -0.0854533 0.0086534 -0.0807282 -0.154929 0.145362 -0.465001 -0.195198 -0.0793125 -0.088495 -0.182678 -0.0595129 -0.103471 -0.0626643 -0.101669 -0.156593 -0.135444 0.332772 -0.25263 0.0409849 0.170206 -0.192404 -0.104022 -0.0391379 0.0712972 -0.0664537 -0.365074 0.00177059 -0.145317 0.170236 0.20812 -0.0666499 0.0597194 0.0859742 0.198767 -0.194409 -0.0294474 0.119935 0.174462 -0.754495 -0.0166906 -0.38728 -0.000466764 -0.147013 -0.270307 0.210511 -0.171951 -0.168654 -0.130213 -0.0246165 0.14529 -0.0845106 0.139153 0.114518 -0.28373 -0.118762 0.22549 -0.0772575 0.0979658 -0.082814 -0.185345 0.153512 0.0939334 -0.0403678 -0.125036 -0.156152 -0.012777 0.208591 -0.189739 0.245523 -0.0723928 -0.0765258 0.0744587 -0.222038 0.763308 -0.0363269 0.030647 -0.229542 -0.17993 -0.315128 0.175175 0.256983 -0.0175775 0.275679 -0.0251663 -0.1271 0.14466 -0.027372 0.379621 0.128368 -0.00632908 -0.181528 -0.0789961 0.103475 -0.0226894 -0.158417 0.212613 0.319065 0.215753 -0.0141561 -0.0592237 0.289586 -0.0640098 0.31015 -0.0365294 -0.015468 0.159444 0.185906 0.1008 -0.158896 0.120466 0.124751 -0.390463 -0.251686 -0.197392 -0.0362771 0.212511 0.0894133 0.0531159 0.0788771 0.140556 -0.0292637 0.0335905 0.205588 0.191012 0.00353969 0.167493 0.211498 -0.0979891 0.0597665 0.285034 0.120406 0.196153 -0.109347 -0.0835259 0.118805 0.247285 0.0515751 -0.139959 0.0676471 -0.0985784 0.364498 0.125686 0.0425329 -0.154708 0.0877508 -0.273921 0.25014 -0.115438 -0.287548 0.0442916 0.318534 0.289668 0.100948 -0.0851599 -0.0411322 0.293729 0.190876 0.224382 0.179725 0.135721 0.126807 -0.126513 0.192521 -0.117995 0.150317 -0.324527 -0.0853502 -0.00810679 0.109197 0.325302 0.169514 0.230675 0.0429429 -0.12536 -0.116082 0.129293 0.0199593 -0.0483896 0.127734 -0.033858 0.24547 0.224313 -0.0886457 -0.0178661 0.232056 -0.44026 0.0660958 -0.00959362 -0.197395 -0.204684 -0.0148652 0.000769943 0.187324 -0.310319 -0.180467 0.0306257 -0.0943395 -0.0206834 0.1421 -0.101547 -0.0743218 -0.393276 -0.169392 0.0775916 0.197213 -0.140968 0.336205 -0.221488 -0.0473175 0.0182749 0.0942729 0.321991 0.092176 -0.0280198 -0.0794545 0.0309331 -0.0904776 0.11434 0.287766 0.200515 -0.064825 0.0858123 -0.0154623 -0.0316231 -0.238933 -0.0140678 0.124269 0.194314 -0.0172931 0.0253024 0.303144 -0.0103084 0.0593173
y 0.210711 -0.192484 0.0401261 0.271211 -0.26025 -0.107201 -0.342025 0.120469 -0.0533656 -1.29802 0.302411 0.274047 0.0443012 -0.0325265 0.189578 -0.214596 -0.338857 -1.16866 0.104407 0.208983 0.091867 -0.271941 0.212059 -0.105383 -0.138384 0.0952262 -0.0294383 0.0612378 -0.00666795 0.0310999 -0.162081 -0.294555 0.227768 -0.0346698 -0.116897 -0.0326892 -0.260272 0.072424 -0.360872 0.418521 0.0839673 -0.19602 0.147536 0.294464 0.3771 -0.00545112 0.0347159 0.289452 -0.102442 0.0415111 -0.117335 0.266347 -0.247795 0.152822 0.127689 -0.0712854 0.524562 -0.242552 -0.006768 0.204229 0.114903 -0.0255567 -0.170838 -0.434963 -0.213055 0.0178996 -0.242532 0.402145 0.133442 0.0509473 0.0293289 0.0443344 -0.0690586 0.0602386 0.136216 0.0341795 0.197321 0.257996 -0.0716081 0.261805 0.101265 -0.0683678 -0.00307157 0.139454 0.222632 -0.0965715 0.144728 -0.00072626 -0.109439 -0.192147 0.119652 0.0864441 0.193052 0.207358 0.382752 0.38641 -0.215731 -0.153436 -0.265402 -0.103855 0.258248 -0.0559816 -0.271015 0.182948 -0.150265 -0.540179 0.173865 -0.193671 -0.0782511 -0.110619 0.0137942 0.328052 -0.280685 0.178637 -0.221369 0.172559 -0.00984901 -0.182122 -0.331579 0.236811 -0.0795454 -0.401583 0.0165171 0.0470741 0.194192 0.0145183 0.0684353 -0.251878 -0.0620554 0.316788 -0.241065 -0.195169 0.0932091 -0.228582 0.0928218 0.026166 0.0415918 -0.132591 0.012792 0.011072 0.359457 0.112218 -0.368104 -0.0883562 0.0524899 -0.19394 -0.204717 0.157005 0.128992 0.27523 0.00792091 0.0793005 -0.206673 -0.178482 0.0189093 -0.203434 0.0789685 -0.15701 -0.163036 -0.230638 0.162913 0.0889816 0.0597846 0.223279 0.271003 0.134427 0.0058499 0.230554 -0.0534001 -0.0529326 0.285916 0.484824 0.439553 -0.28006 -0.178672 -0.337118 -0.0952894 -0.21532 -0.262358 0.243775 0.256241 -0.134378 0.40314 0.0378762 0.349161 0.149631 0.195366 -0.198712 0.0544871 0.076432 -0.0133706 0.208441 0.375043 -0.245403 -0.285947 -0.117958 -0.0493158 0.145679 0.382085 -0.273272 -0.0915126 -0.111324 -0.0941245 -0.03903 -0.0527625 0.208222 -0.174507 -0.339289 0.0398205 -0.105523 0.155491 0.222323 0.212825 0.0871449 -0.258712 0.0140071 0.353397 0.0759974 0.69142 0.0255388 0.263282 0.0179852 0.000356677 0.0717537 -0.0150014 -0.0609194 -0.0467376 0.0917597 -0.0548037 -0.378603 0.271696 0.163428 0.0095608 0.0723245 -0.121812 0.429298 -0.119686 -0.412887 0.217519 0.0815722 -0.08714 -0.22636 -0.249483 0.402615 -0.00420015 -0.356592 -0.150446 -0.100797 0.378998 0.0440508 -0.0857994 0.0133886 0.293185 0.214976 -0.0490763 -0.0198632 0.103532 0.0424432 -0.0491354 -0.0957982 -0.1008 -0.00083714 -0.255495 -0.326746 0.258817 0.22492 -0.369093 0.301558 -0.523558 -0.0911132 -0.323394 -0.411966 -0.164819 0.179108 -0.289724 0.02121 0.0347096 0.0723183 0.232926 0.0883246 0.127537 -0.150733 -0.333545 -0.0529866 0.28144 0.099363 -0.242476 0.0727028 -0.317291 -0.0274223 0.0464263 -0.0302705 0.0738542 -0.00241268 -0.144828 -0.189238 0.259282 0.51955 0.10392 0.20384
T 0.563784 -0.453233 0.19423 0.0447773 -0.052464 -0.345424 0.0705833 -0.126393 0.280582 -1.21064 0.0448603 -0.181328 0.0291698 0.0826217 0.224081 -0.0979826 0.169898 -0.979041 0.486936 0.438253 -0.262459 -0.272367 0.290391 -0.177425 0.112934 -0.0638537 0.163838 0.3343 -0.277954 -0.048072 0.242325 -0.238577 0.133157 0.482161 0.103054 0.102747 0.050882 -0.188382 -0.124863 0.142741 0.0322651 0.168426 0.120849 -0.0231288 0.0381446 -0.355962 0.194663 -0.223721 -0.32019 -0.12006 0.316389 -0.238225 0.0826364 -0.223893 -0.218967 0.135063 -0.175722 0.0529259 -0.138597 -0.147143 -0.114738 0.302212 0.127847 -0.0674654 -0.310919 0.166219 -0.104703 -0.292309 -0.0266889 -0.209156 0.103291 -0.0910956 0.275851 0.127282 -0.343862 -0.180832 -0.177566 -0.244725 0.0733281 0.00894241 0.168003 -0.489613 0.147948 0.269959 -0.274212 0.327738 -0.0545822 -0.11863 -0.323186 -0.175054 -0.0546512 0.0551471 0.00691869 0.156154 -0.0706464 0.46533 0.119752 0.21259 -0.0623386 0.376558 -0.119057 0.159687 -0.0193878 0.0852455 -0.067171 -0.112924 -0.148515 -0.016694 -0.273971 -0.124376 -0.0770675 0.269524 -0.169897 -0.110825 0.104507 0.203298 0.140649 0.298713 0.0415784 0.109603 -0.0493956 -0.00913845 -0.166055 0.221901 0.0959925 0.105805 0.302572 -0.0725075 -0.0697 0.3178 -0.0441763 0.142545 -0.00330831 -0.074459 -0.18657 -0.099573 -0.251335 -0.00178079 -0.028757 -0.0407286 0.836609 -0.213223 -0.472292 0.149311 -0.0567058 0.197854 0.0493754 0.0806316 0.468056 0.110291 0.071981 0.215629 0.116788 0.0878658 0.376592 0.034058 0.105795 0.278099 0.1106 0.30833 -0.257592 -0.0949947 -0.0205995 0.00822854 0.13113 0.0142654 0.0137533 0.150575 -0.163763 -0.06712 0.204684 0.211927 -0.0578112 0.243234 0.0431068 0.0390186 -0.271201 -0.0457051 0.0644319 0.0616125 -0.0734224 0.132489 -0.0157597 0.0182723 0.143489 0.0218261 0.221542 0.0600131 -0.233725 0.0236861 -0.106295 -0.0582001 -0.200371 -0.151495 -0.114905 -0.276287 -0.0127782 0.255708 0.0401842 -0.424414 -0.00362202 0.465612 0.000895313 -0.116085 -0.349623 0.0153773 0.0858678 0.0255823 0.260881 0.138462 -0.0934804 0.201736 0.146219 0.428867 -0.181365 0.0133939 0.0190211 -0.554343 0.550638 -0.114822 0.00929258 0.102826 0.163686 -0.143206 -0.292712 -0.375754 0.194716 -0.321602 -0.281954 -0.293162 0.244005 0.16612 -0.101952 -0.305169 0.13839 -0.194157 -0.31519 0.132486 -0.0876124 0.0173998 -0.0380994 -0.251825 -0.0749466 -0.28325 -0.214834 0.17327 0.280828 0.187163 -0.0148163 0.156545 -0.0710278 -0.226726 0.237075 0.0329142 -0.0215532 -0.450213 -0.0814472 -0.138997 -0.00252615 -0.00754768 -0.0167251 -0.291245 0.049429 -0.0133063 0.0483424 -0.0232334 -0.222638 0.179261 -0.0386132 0.26767 -0.215137 0.320416 0.0501222 0.0571838 0.0765454 0.213379 -0.104671 -0.188186 -0.076718 -0.385419 0.205673 -0.0296164 -0.0724988 0.0632523 -0.086099 0.272648 -0.164918 0.14088 -0.0386889 -0.110989 0.358091 -0.350751 -0.0207286 -0.054343 0.120262 0.107871 -0.142889 -0.0795563 0.0368467 -0.0712824
q 0.542025 -0.135666 0.332714 -0.333257 -0.0113793 -0.280202 0.274469 -0.103613 -0.421016 -1.35661 0.533662 -0.356385 -0.300549 0.190698 0.0143003 -0.321162 0.0479881 -0.839245 0.49557 -0.0812322 0.540413 0.522542 0.54996 0.109648 0.147018 0.228719 -0.38429 0.171833 -0.225784 0.251133 0.264134 -0.487728 -0.291729 -0.209751 -0.490141 -0.125145 -0.270465 -0.269579 0.152343 -0.31207 0.236157 -0.301531 -0.53797 -0.123102 0.0419538 -0.080161 0.408985 0.0310676 -0.626382 0.440935 0.251046 0.458184 -0.145115 0.0791147 0.0466103 -0.0424942 0.161562 -0.0271309 0.452186 0.643363 -0.174638 0.457523 0.0326045 -0.115783 -0.384355 -0.24262 0.621007 -0.211076 0.0589056 -0.361359 0.197337 0.00149168 -0.277246 -0.00320124 -0.0325985 -0.731828 -0.077299 -0.0481133 -0.308853 -0.612618 0.0794804 -0.231929 0.0733612 0.0154917 -0.129453 0.446869 0.693372 0.352701 -0.489878 -0.327986 -0.250957 0.131753 0.557512 0.0697321 -0.17717 0.164186 0.0536564 0.558198 0.00353842 -0.181071 -0.164491 0.161463 0.106385 0.4126 0.217667 -0.655791 -0.398494 0.0826792 0.277457 0.169504 0.0557042 0.384562 -0.140152 -0.177472 -0.272637 0.0619393 0.499673 0.375992 -0.0752742 -0.236519 0.1887 -0.164002 0.385464 0.473534 -0.366258 0.0698561 -0.468852 0.568792 -0.343381 0.0
gitextract_as6ks4uh/
├── .gitignore
├── README.md
├── api.py
├── app.py
├── base/
│ ├── base_io.py
│ └── base_model.py
├── daanet/
│ ├── base.py
│ └── basic.py
├── dataio_utils/
│ ├── __init__.py
│ ├── flow_io.py
│ ├── full_load_io.py
│ └── helper.py
├── default.yaml
├── gpu_env.py
├── grid.yaml
├── grid_search.py
├── model_utils/
│ ├── __init__.py
│ └── helper.py
├── nlp/
│ ├── __init__.py
│ ├── encode_blocks.py
│ ├── match_blocks.py
│ ├── nn.py
│ └── seq2seq/
│ ├── __init__.py
│ ├── common.py
│ ├── pointer_generator.py
│ └── rnn.py
├── sample_data/
│ ├── sample.charembed.txt
│ ├── sample.embed.txt
│ ├── sample.json
│ └── sample.out.json
└── utils/
├── __init__.py
├── eval_4/
│ ├── __init__.py
│ ├── bleu_metric/
│ │ ├── __init__.py
│ │ ├── bleu.py
│ │ └── bleu_scorer.py
│ ├── eval.py
│ ├── exact_f1/
│ │ ├── __init__.py
│ │ └── exact_f1.py
│ ├── meteor/
│ │ ├── __init__.py
│ │ ├── meteor-1.5.jar
│ │ ├── meter.py
│ │ ├── reference
│ │ ├── res.txt
│ │ └── test
│ └── rouge_metric/
│ ├── __init__.py
│ └── rouge.py
├── helper.py
├── predictor.py
└── vocab.py
SYMBOL INDEX (278 symbols across 27 files)
FILE: api.py
function train (line 11) | def train(args):
function evaluate (line 26) | def evaluate(args):
function demo (line 32) | def demo(args):
FILE: app.py
function run (line 7) | def run():
FILE: base/base_io.py
class BaseDataIO (line 8) | class BaseDataIO:
method __init__ (line 9) | def __init__(self, args):
method next_batch (line 19) | def next_batch(self, batch_size: int, mode: ModeKeys):
method load_data (line 22) | def load_data(self, file_paths: List[str], mode: ModeKeys):
method make_mini_batch (line 25) | def make_mini_batch(self, data):
FILE: base/base_model.py
class BaseModel (line 16) | class BaseModel:
method __init__ (line 17) | def __init__(self, args):
method _build_graph (line 50) | def _build_graph(self):
method _set_learning_rate (line 53) | def _set_learning_rate(self):
method _set_fetches (line 112) | def _set_fetches(self):
method add_tfboard (line 121) | def add_tfboard(self, name, value, mode):
method add_fetch (line 135) | def add_fetch(self, name, value, mode):
method get_fetch (line 144) | def get_fetch(self, name, mode):
method init_session (line 147) | def init_session(self):
method write_num_pars (line 160) | def write_num_pars(self):
method is_graph_valid (line 183) | def is_graph_valid(self):
method batch2feed_dict (line 190) | def batch2feed_dict(self, batch, mode):
method run_sess_op (line 208) | def run_sess_op(self, batch, mode):
method is_effective_epoch (line 212) | def is_effective_epoch(self, metric, last_metric, best_metric):
method load_embedding (line 227) | def load_embedding(self):
method get_tfboard_vars (line 230) | def get_tfboard_vars(self):
method train (line 233) | def train(self, mode=ModeKeys.TRAIN):
method predict (line 260) | def predict(self, inputs, mode=ModeKeys.EVAL):
method evaluate (line 272) | def evaluate(self, epoch=-1, mode=ModeKeys.EVAL):
method save_eval_preds (line 292) | def save_eval_preds(self, preds, epoch=-1, mode=ModeKeys.EVAL):
method save_for_serving (line 339) | def save_for_serving(self, epoch):
method save (line 342) | def save(self, epoch):
method save_model (line 347) | def save_model(self, epoch, save='save'):
method save_args (line 354) | def save_args(self):
method restore (line 358) | def restore(self, epoch=-1, use_ema=True, use_partial_loader=False):
method _init_train_op (line 380) | def _init_train_op(self):
method _init_tensorboard (line 433) | def _init_tensorboard(self):
method reset (line 457) | def reset(self):
FILE: daanet/base.py
class RCBase (line 14) | class RCBase(base_model.BaseModel):
method __init__ (line 15) | def __init__(self, args):
method train (line 18) | def train(self, mode=ModeKeys.TRAIN):
method restore_evaluate (line 41) | def restore_evaluate(self, args):
method evaluate (line 50) | def evaluate(self, epoch=-1, mode=ModeKeys.EVAL):
method predict (line 80) | def predict(self, inputs, mode=ModeKeys.EVAL):
method save_eval_preds (line 83) | def save_eval_preds(self, preds, epoch=-1, mode=ModeKeys.EVAL):
method save_metrics (line 104) | def save_metrics(self, metrics, epoch=-1):
method parse_result (line 134) | def parse_result(self, batch, fetches):
FILE: daanet/basic.py
class RCCore (line 22) | class RCCore(RCBase):
method __init__ (line 23) | def __init__(self, args):
method _build_graph (line 26) | def _build_graph(self):
method _placeholders (line 39) | def _placeholders(self):
method _shortcuts (line 112) | def _shortcuts(self):
method _masks (line 122) | def _masks(self):
method _embed (line 149) | def _embed(self):
method _encode (line 211) | def _encode(self):
method _decode (line 376) | def _decode(self):
method _model_loss (line 450) | def _model_loss(self):
method load_embedding (line 465) | def load_embedding(self):
method get_tfboard_vars (line 474) | def get_tfboard_vars(self):
method get_training_decoder (line 482) | def get_training_decoder(self, decoder_inputs, decoder_length, decoder...
method get_inference_decoder (line 498) | def get_inference_decoder(self, decoder_cell, train_initial_state, pro...
method get_decode_cell_state (line 521) | def get_decode_cell_state(self, encoder_output, encoder_state, encoder...
method _loss_calc_helper (line 547) | def _loss_calc_helper(self, decode_length, decode_target, decode_mask,...
FILE: dataio_utils/flow_io.py
class FlowDataIO (line 11) | class FlowDataIO(base_io.BaseDataIO):
method __init__ (line 12) | def __init__(self, args):
method make_node (line 26) | def make_node(self, mode: ModeKeys):
method next_batch (line 31) | def next_batch(self, batch_size: int, mode: ModeKeys):
method load_data (line 34) | def load_data(self, file_paths: List[str], mode: ModeKeys):
method _dump_to_json (line 47) | def _dump_to_json(self, sample):
method _load_from_json (line 55) | def _load_from_json(self, batch):
method _filter_invalid_seq (line 58) | def _filter_invalid_seq(self, line):
method make_sample (line 61) | def make_sample(self, line, mode=ModeKeys.TRAIN):
method make_mini_batch (line 64) | def make_mini_batch(self, data, mode=ModeKeys.TRAIN):
method single2batch (line 67) | def single2batch(self, context):
FILE: dataio_utils/full_load_io.py
class DataIO (line 7) | class DataIO(base_io.BaseDataIO):
method __init__ (line 8) | def __init__(self, args):
method reset_pointer (line 24) | def reset_pointer(self, mode, shuffle=False):
method next_batch (line 30) | def next_batch(self, batch_size: int, mode: ModeKeys):
method post_process_train (line 44) | def post_process_train(self, sample):
method post_process_eval (line 55) | def post_process_eval(self, sample):
FILE: dataio_utils/helper.py
function _tokenize (line 7) | def _tokenize(x):
function _char_token_start_end (line 28) | def _char_token_start_end(char_start, answer_text, char_token_map, full_...
function _dump_to_json (line 39) | def _dump_to_json(sample):
function _load_from_json (line 43) | def _load_from_json(batch):
function _parse_line (line 47) | def _parse_line(line):
function _do_padding (line 51) | def _do_padding(token_ids, token_lengths, pad_id):
function _do_char_padding (line 56) | def _do_char_padding(char_ids, token_lengths, pad_id, char_pad_id):
function _dropout_word (line 64) | def _dropout_word(x, unk_id, dropout_keep_prob):
function _fast_copy (line 68) | def _fast_copy(x, ignore_keys):
function build_vocab (line 78) | def build_vocab(embd_files):
FILE: gpu_env.py
class SummaryType (line 14) | class SummaryType(Enum):
class ModeKeys (line 20) | class ModeKeys(Enum):
FILE: grid_search.py
function run (line 10) | def run():
FILE: model_utils/helper.py
function sample_element_from_var (line 11) | def sample_element_from_var(all_var):
function partial_restore (line 25) | def partial_restore(session, save_file):
function mblock (line 42) | def mblock(scope_name, device_name=None, reuse=None):
function get_filename (line 59) | def get_filename(args, mode: ModeKeys):
function write_dev_json (line 65) | def write_dev_json(f, pred_answers):
class LossCounter (line 71) | class LossCounter:
method __init__ (line 72) | def __init__(self, task_names, log_interval, batch_size, tb_writer):
method _reset_step_loss (line 85) | def _reset_step_loss(self):
method record (line 90) | def record(self, fetches):
method is_overfitted (line 101) | def is_overfitted(self, metric):
method _trigger (line 108) | def _trigger(self):
method show_status (line 111) | def show_status(self):
method _get_multitask_loss_str (line 121) | def _get_multitask_loss_str(loss_dict, normalizer=1.0, show_key=True, ...
FILE: nlp/encode_blocks.py
function LSTM_encode (line 6) | def LSTM_encode(seqs, scope='lstm_encode_block', reuse=None, **kwargs):
function TCN_encode (line 16) | def TCN_encode(seqs, num_layers, normalize_output=True, scope='tcn_encod...
function Res_DualCNN_encode (line 30) | def Res_DualCNN_encode(seqs, use_spatial_dropout=True, scope='res_biconv...
function CNN_encode (line 51) | def CNN_encode(seqs, filter_size=3, dilation=1,
FILE: nlp/match_blocks.py
function Attention_match (line 7) | def Attention_match(context, query, context_mask, query_mask,
function Transformer_match (line 26) | def Transformer_match(context,
function BiDaf_match (line 121) | def BiDaf_match(a, b, a_mask, b_mask, residual, scope=None, reuse=None, ...
function dot_attention (line 150) | def dot_attention(inputs, memory, mask, hidden_size, keep_prob=1.0, is_t...
FILE: nlp/nn.py
function minus_mask (line 14) | def minus_mask(x, mask, offset=1e30):
function mul_mask (line 25) | def mul_mask(x, mask):
function masked_reduce_mean (line 35) | def masked_reduce_mean(x, mask):
function masked_reduce_max (line 39) | def masked_reduce_max(x, mask):
function weighted_sparse_softmax_cross_entropy (line 43) | def weighted_sparse_softmax_cross_entropy(labels, preds, weights):
function get_bounded_class_weight (line 57) | def get_bounded_class_weight(labels, weights, ub=None):
function weighted_smooth_softmax_cross_entropy (line 69) | def weighted_smooth_softmax_cross_entropy(labels, num_labels, preds, wei...
function get_var (line 87) | def get_var(name, shape, dtype=tf.float32,
function layer_norm (line 96) | def layer_norm(inputs,
function linear_logit (line 126) | def linear_logit(x, units, act_fn=None, dropout_keep=1., use_layer_norm=...
function bilinear_logit (line 138) | def bilinear_logit(x, units, act_fn=None,
function label_smoothing (line 146) | def label_smoothing(inputs, epsilon=0.1):
function normalize_by_axis (line 182) | def normalize_by_axis(x, axis, smooth_factor=1e-5):
function get_cross_correlated_mat (line 187) | def get_cross_correlated_mat(num_out_A, num_out_B, learn_cooc='FIXED', c...
function get_self_correlated_mat (line 207) | def get_self_correlated_mat(num_out_A, scope=None, reuse=None):
function gate_filter (line 218) | def gate_filter(x, scope=None, reuse=None):
function focal_loss2 (line 228) | def focal_loss2(onehot_labels, prediction_tensor, alpha=0.25, gamma=2, ):
function focal_loss (line 248) | def focal_loss(prediction_tensor, target_tensor, weights=None, alpha=0.2...
function spatial_dropout (line 279) | def spatial_dropout(x, scope=None, reuse=None):
function get_last_output (line 289) | def get_last_output(output, seq_length, scope=None, reuse=None):
function get_lstm_init_state (line 301) | def get_lstm_init_state(batch_size, num_layers, num_units, direction, sc...
function highway_layer (line 311) | def highway_layer(x, scope=None, reuse=None, **kwargs):
function highway_network (line 322) | def highway_network(x, num_layers, scope=None, reuse=None, **kwargs):
FILE: nlp/seq2seq/common.py
function initializer (line 13) | def initializer(): return tf.contrib.layers.variance_scaling_initializer...
function rand_uniform_initializer (line 19) | def rand_uniform_initializer(
function truc_norm_initializer (line 23) | def truc_norm_initializer(
function initializer_relu (line 27) | def initializer_relu(): return tf.contrib.layers.variance_scaling_initia...
function get_var (line 36) | def get_var(name, shape, dtype=tf.float32,
function softmax_mask (line 45) | def softmax_mask(val, mask):
function dropout (line 49) | def dropout(args, keep_prob, is_train, mode="recurrent"):
function dense (line 64) | def dense(inputs, hidden_size, use_bias=True, scope=None):
function layer_norm (line 81) | def layer_norm(x, filters=None, epsilon=1e-6, scope=None, reuse=None):
function layer_norm_compute_python (line 94) | def layer_norm_compute_python(x, epsilon, scale, bias):
function get_scope_name (line 102) | def get_scope_name():
function make_var (line 106) | def make_var(name, shape, trainable=True):
function mblock (line 114) | def mblock(scope_name, device_name=None, reuse=None):
function get_init_state (line 132) | def get_init_state(args, name, q_type, shape):
function highway (line 147) | def highway(x, size=None, activation=tf.nn.relu,
function conv (line 164) | def conv(inputs, output_size, bias=None, activation=None, kernel_size=1,...
function sparse_nll_loss (line 195) | def sparse_nll_loss(probs, labels, epsilon=1e-9, scope=None):
function normalize_distribution (line 206) | def normalize_distribution(p, eps=1e-9):
function kl_divergence (line 212) | def kl_divergence(p, q, eps=1e-9):
function get_kl_loss (line 218) | def get_kl_loss(start_label, start_probs, bandwidth=1.0):
function sym_kl_divergence (line 226) | def sym_kl_divergence(p, q, eps=1e-9):
function get_conv_feature (line 230) | def get_conv_feature(x, out_dim, window_len, upsampling=False):
function upsampling_a2b (line 239) | def upsampling_a2b(a, b, D_a):
FILE: nlp/seq2seq/pointer_generator.py
class PointerGeneratorGreedyEmbeddingHelper (line 10) | class PointerGeneratorGreedyEmbeddingHelper(tf.contrib.seq2seq.GreedyEmb...
method __init__ (line 11) | def __init__(self, embeddings, start_tokens, end_token, unk_token):
method sample (line 16) | def sample(self, time, outputs, state, name=None):
method next_inputs (line 26) | def next_inputs(self, time, outputs, state, sample_ids, name=None):
class PointerGeneratorDecoder (line 44) | class PointerGeneratorDecoder(tf.contrib.seq2seq.BasicDecoder):
method __init__ (line 47) | def __init__(self, source_extend_tokens, source_oov_words, coverage, c...
method output_size (line 57) | def output_size(self):
method output_dtype (line 64) | def output_dtype(self):
method step (line 73) | def step(self, time, inputs, state, name=None):
class PointerGeneratorAttentionWrapper (line 157) | class PointerGeneratorAttentionWrapper(tf.contrib.seq2seq.AttentionWrapp...
method __init__ (line 158) | def __init__(self, cell,
method zero_state (line 183) | def zero_state(self, batch_size, dtype):
method set_history (line 234) | def set_history(self, history_inputs):
method call (line 237) | def call(self, inputs, state):
function _pg_bahdanau_score (line 349) | def _pg_bahdanau_score(processed_query, keys, coverage, coverage_vector):
class PointerGeneratorBahdanauAttention (line 383) | class PointerGeneratorBahdanauAttention(tf.contrib.seq2seq.BahdanauAtten...
method __init__ (line 384) | def __init__(self,
method __call__ (line 422) | def __call__(self, query, state):
FILE: nlp/seq2seq/rnn.py
function single_rnn_cell (line 10) | def single_rnn_cell(cell_name, num_units, is_train=None, keep_prob=0.75):
function multi_rnn_cell (line 32) | def multi_rnn_cell(cell_name, num_units, is_train=None, keep_prob=1.0, n...
function get_lstm_init_state (line 51) | def get_lstm_init_state(batch_size, num_layers, num_units, direction, sc...
function LSTM_encode (line 61) | def LSTM_encode(seqs, scope=None, reuse=None, **kwargs):
function custom_dynamic_rnn (line 72) | def custom_dynamic_rnn(cell, inputs, inputs_len, initial_state=None):
function reduce_state (line 134) | def reduce_state(fw_state, bw_state, hidden_size, act_fn=tf.nn.relu, sco...
class CudaRNN (line 145) | class CudaRNN:
method __init__ (line 146) | def __init__(self, num_layers, num_units, cell_type):
method __call__ (line 158) | def __call__(self, inputs, seq_len, keep_prob=1.0,
class native_gru (line 204) | class native_gru:
method __init__ (line 206) | def __init__(self, num_layers, num_units, batch_size, input_size, keep...
method __call__ (line 228) | def __call__(self, inputs, seq_len, keep_prob=1.0, is_train=None, conc...
FILE: utils/eval_4/bleu_metric/bleu.py
class Bleu (line 14) | class Bleu:
method __init__ (line 15) | def __init__(self, n=4):
method compute_score (line 21) | def compute_score(self, gts, res):
method compute_score_single_answer (line 35) | def compute_score_single_answer(self, res, ref):
method method (line 43) | def method(self):
FILE: utils/eval_4/bleu_metric/bleu_scorer.py
function precook (line 25) | def precook(s, n=4, out=False):
function cook_refs (line 38) | def cook_refs(refs, eff=None, n=4): ## lhuang: oracle will call with "a...
function cook_test (line 64) | def cook_test(test, xxx_todo_changeme, eff=None, n=4):
class BleuScorer (line 90) | class BleuScorer(object):
method copy (line 98) | def copy(self):
method __init__ (line 106) | def __init__(self, test=None, refs=None, n=4, special_reflen=None):
method cook_append (line 115) | def cook_append(self, test, refs):
method ratio (line 128) | def ratio(self, option=None):
method score_ratio (line 132) | def score_ratio(self, option=None):
method score_ratio_str (line 136) | def score_ratio_str(self, option=None):
method reflen (line 139) | def reflen(self, option=None):
method testlen (line 143) | def testlen(self, option=None):
method retest (line 147) | def retest(self, new_test):
method rescore (line 158) | def rescore(self, new_test):
method size (line 163) | def size(self):
method __iadd__ (line 167) | def __iadd__(self, other):
method compatible (line 181) | def compatible(self, other):
method single_reflen (line 184) | def single_reflen(self, option="average"):
method _single_reflen (line 187) | def _single_reflen(self, reflens, option=None, testlen=None):
method recompute_score (line 200) | def recompute_score(self, option=None, verbose=0):
method compute_score (line 204) | def compute_score(self, option=None, verbose=0):
FILE: utils/eval_4/eval.py
function normalize (line 7) | def normalize(s):
function compute_bleu_rouge (line 26) | def compute_bleu_rouge(pred_dict, ref_dict, bleu_order=4):
FILE: utils/eval_4/exact_f1/exact_f1.py
class f1_exact_eval (line 15) | class f1_exact_eval:
method __init__ (line 17) | def __init__(self):
method normalize_answer (line 21) | def normalize_answer(self, s):
method get_tokens (line 40) | def get_tokens(self, s):
method compute_exact (line 44) | def compute_exact(self, a_gold, a_pred):
method compute_f1 (line 47) | def compute_f1(self, a_gold, a_pred):
method compute_scores (line 62) | def compute_scores(self, res, ref):
FILE: utils/eval_4/meteor/meter.py
function compute_meter_score (line 4) | def compute_meter_score(pred, ref):
FILE: utils/eval_4/rouge_metric/rouge.py
function my_lcs (line 13) | def my_lcs(string, sub):
class Rouge (line 37) | class Rouge():
method __init__ (line 43) | def __init__(self):
method calc_score (line 47) | def calc_score(self, candidate, refs):
method compute_score (line 79) | def compute_score(self, gts, res):
method method (line 92) | def method(self):
FILE: utils/helper.py
function set_logger (line 27) | def set_logger(model_id=None):
function touch (line 46) | def touch(fname: str, times=None, create_dirs: bool = False):
function touch_dir (line 56) | def touch_dir(base_dir: str) -> None:
function millify (line 62) | def millify(n):
function args2hparam (line 70) | def args2hparam(args, vocab):
function runner (line 79) | def runner(main, *done):
function parse_yaml (line 89) | def parse_yaml(yaml_path, model_id):
function parse_args (line 106) | def parse_args(yaml_path, model_id, default_set, followup=None):
function add_param_recur (line 171) | def add_param_recur(root, p_tree):
function fill_gpu_jobs (line 181) | def fill_gpu_jobs(all_jobs, logger, job_parser,
function get_args_cli (line 213) | def get_args_cli(args):
function parse_arg (line 227) | def parse_arg(v: str):
function get_scope_name (line 251) | def get_scope_name():
function sparse_nll_loss (line 255) | def sparse_nll_loss(probs, labels, epsilon=1e-9, scope=None):
function normalize_distribution (line 265) | def normalize_distribution(p, eps=1e-9):
function kl_divergence (line 271) | def kl_divergence(p, q, eps=1e-9):
function get_kl_loss (line 277) | def get_kl_loss(start_label, start_probs, bandwidth=1.0):
function sym_kl_divergence (line 284) | def sym_kl_divergence(p, q, eps=1e-9):
function get_conv1d (line 288) | def get_conv1d(x, out_dim, window_len, name, act_fn):
function upsampling_a2b (line 292) | def upsampling_a2b(a, b, D_a):
function dropout (line 297) | def dropout(args, keep_prob, is_train, mode="recurrent"):
function get_tmp_yaml (line 312) | def get_tmp_yaml(par, prefix=None):
function build_model (line 319) | def build_model(args, reset_graph=True):
function get_last_output (line 326) | def get_last_output(output, sequence_length, name):
function import_class (line 337) | def import_class(import_str):
function delete_module (line 352) | def delete_module(modname):
FILE: utils/predictor.py
class Predictor (line 13) | class Predictor:
method __init__ (line 15) | def __init__(self, batch_size=32):
method predict (line 21) | def predict(self, model_id, inputs):
function import_class (line 36) | def import_class(import_str):
function delete_module (line 51) | def delete_module(modname):
function load_models (line 64) | def load_models(save_path):
function parse_args (line 91) | def parse_args(yaml_path, model_id):
function list_models (line 108) | def list_models(save_path):
FILE: utils/vocab.py
class Vocab (line 9) | class Vocab:
method load_from_pickle (line 12) | def load_from_pickle(fp):
method __init__ (line 16) | def __init__(self, embedding_files, lower=True):
method size (line 38) | def size(self):
method pretraind_size (line 41) | def pretraind_size(self):
method initial_tokens_size (line 46) | def initial_tokens_size(self):
method get_id (line 49) | def get_id(self, token, fallback_chars=False):
method get_token (line 57) | def get_token(self, idx):
method get_token_with_oovs (line 60) | def get_token_with_oovs(self, idx, oovs):
method add (line 70) | def add(self, token):
method load_pretrained (line 86) | def load_pretrained(self, embedding_path):
method tokens2ids (line 136) | def tokens2ids(self, tokens):
method tokens2ids_with_oovs (line 139) | def tokens2ids_with_oovs(self, tokens, init_oovs=[], dynamic_oovs=True):
Condensed preview — 49 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (5,995K chars).
[
{
"path": ".gitignore",
"chars": 1379,
"preview": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packagi"
},
{
"path": "README.md",
"chars": 2059,
"preview": "# DAANet: Dual Ask-Answer Network for Machine Reading Comprehension\n[![Python: 3.6](https://img.shields.io/badge/Python-"
},
{
"path": "api.py",
"chars": 1758,
"preview": "import logging\n\nfrom tensorflow.python.framework.errors_impl import NotFoundError, InvalidArgumentError\n\nfrom gpu_env im"
},
{
"path": "app.py",
"chars": 466,
"preview": "import sys\n\nfrom gpu_env import DEVICE_ID, MODEL_ID, CONFIG_SET\nfrom utils.helper import set_logger, parse_args, get_arg"
},
{
"path": "base/base_io.py",
"chars": 866,
"preview": "import logging\nfrom typing import List\n\nfrom dataio_utils.helper import build_vocab\nfrom gpu_env import APP_NAME, ModeKe"
},
{
"path": "base/base_model.py",
"chars": 20931,
"preview": "import importlib\nimport json\nimport logging\nimport os\nfrom collections import defaultdict\nfrom math import ceil\n\nimport "
},
{
"path": "daanet/base.py",
"chars": 7115,
"preview": "import json\nimport os\n\nimport tensorflow as tf\n\nfrom base import base_model\nfrom gpu_env import ModeKeys\nfrom model_util"
},
{
"path": "daanet/basic.py",
"chars": 30506,
"preview": "\"\"\"Sequence-to-Sequence with attention model.\n\"\"\"\n\nimport tensorflow as tf\nfrom tensorflow.python.layers import core as "
},
{
"path": "dataio_utils/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "dataio_utils/flow_io.py",
"chars": 2238,
"preview": "import json\nfrom typing import List\n\nimport tensorflow as tf\n\nfrom base import base_io\nfrom gpu_env import ModeKeys\n\n\n# "
},
{
"path": "dataio_utils/full_load_io.py",
"chars": 2018,
"preview": "import random\n\nfrom base import base_io\nfrom gpu_env import ModeKeys\n\n\nclass DataIO(base_io.BaseDataIO):\n def __init_"
},
{
"path": "dataio_utils/helper.py",
"chars": 2653,
"preview": "import copy\nimport json\nimport random\nimport re\n\n\ndef _tokenize(x):\n tokens = [v for v in re.findall(r\"\\w+|[^\\w]\", x,"
},
{
"path": "default.yaml",
"chars": 1695,
"preview": "# DO NOT CHANGE THIS FILE!!! This file contains all the default values\n# If you need to run a new experiment, please cre"
},
{
"path": "gpu_env.py",
"chars": 1139,
"preview": "import os\nfrom datetime import datetime\nfrom enum import Enum\n\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\nos.environ['CUDA"
},
{
"path": "grid.yaml",
"chars": 685,
"preview": "common:\n retry_delay: 300 # seconds, interval between checking available GPU\n wait_until_next: 60 # seconds, time del"
},
{
"path": "grid_search.py",
"chars": 1612,
"preview": "import itertools\nimport os\nimport sys\n\nfrom ruamel.yaml import YAML\n\nfrom utils.helper import set_logger, fill_gpu_jobs,"
},
{
"path": "model_utils/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "model_utils/helper.py",
"chars": 4728,
"preview": "import json\nimport logging\nimport os\nimport time\n\nimport tensorflow as tf\n\nfrom gpu_env import APP_NAME, ModeKeys\n\n\ndef "
},
{
"path": "nlp/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "nlp/encode_blocks.py",
"chars": 3388,
"preview": "import tensorflow as tf\n\nfrom nlp.nn import initializer, regularizer, spatial_dropout, get_lstm_init_state, layer_norm\n\n"
},
{
"path": "nlp/match_blocks.py",
"chars": 7338,
"preview": "import tensorflow as tf\n\nfrom nlp.nn import linear_logit, layer_norm\nfrom nlp.seq2seq.common import dropout, softmax_mas"
},
{
"path": "nlp/nn.py",
"chars": 14303,
"preview": "import tensorflow as tf\n\ninitializer = tf.contrib.layers.variance_scaling_initializer(factor=1.0,\n "
},
{
"path": "nlp/seq2seq/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "nlp/seq2seq/common.py",
"chars": 9076,
"preview": "# coding=utf-8\n\nimport math\nimport time\n\nimport tensorflow as tf\nfrom tensorflow.python.ops.image_ops_impl import Resize"
},
{
"path": "nlp/seq2seq/pointer_generator.py",
"chars": 22126,
"preview": "import tensorflow as tf\nfrom tensorflow.contrib.seq2seq.python.ops.attention_wrapper import _compute_attention\nfrom tens"
},
{
"path": "nlp/seq2seq/rnn.py",
"chars": 10472,
"preview": "# coding=utf-8\n\nimport tensorflow as tf\nimport tensorflow.contrib as tc\n\nimport gpu_env\nfrom .common import dropout, den"
},
{
"path": "sample_data/sample.charembed.txt",
"chars": 266698,
"preview": "Y 0.273228 -0.422742 0.154817 0.260701 -0.0980259 -0.211214 -0.231156 0.098635 -0.131988 -1.42048 0.296819 0.22757 -0.25"
},
{
"path": "sample_data/sample.embed.txt",
"chars": 2437361,
"preview": ", -0.055718 0.207441 0.003973 0.104525 -0.143629 -0.050006 -0.210321 0.131132 -0.289840 0.168434 -0.032015 0.007560 -0.0"
},
{
"path": "sample_data/sample.json",
"chars": 2276135,
"preview": "{\n \"version\": \"v2.0\",\n \"data\": [\n {\n \"paragraphs\": [\n {\n \"context\": \"Beyonc\\u00e9 Giselle Know"
},
{
"path": "sample_data/sample.out.json",
"chars": 0,
"preview": ""
},
{
"path": "utils/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "utils/eval_4/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "utils/eval_4/bleu_metric/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "utils/eval_4/bleu_metric/bleu.py",
"chars": 1297,
"preview": "#!/usr/bin/env python\n# \n# File Name : bleu.py\n#\n# Description : Wrapper for BLEU scorer.\n#\n# Creation Date : 06-01-2015"
},
{
"path": "utils/eval_4/bleu_metric/bleu_scorer.py",
"chars": 8791,
"preview": "#!/usr/bin/env python\n\n# bleu_scorer.py\n# David Chiang <chiang@isi.edu>\n\n# Copyright (c) 2004-2006 University of Marylan"
},
{
"path": "utils/eval_4/eval.py",
"chars": 1497,
"preview": "from .bleu_metric.bleu import Bleu\nfrom .exact_f1.exact_f1 import f1_exact_eval\nfrom .meteor.meter import compute_meter_"
},
{
"path": "utils/eval_4/exact_f1/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "utils/eval_4/exact_f1/exact_f1.py",
"chars": 2407,
"preview": "\"\"\"Official evaluation script for SQuAD version 2.0.\n\nIn addition to basic functionality, we also compute additional sta"
},
{
"path": "utils/eval_4/meteor/__init__.py",
"chars": 26,
"preview": "__author__ = 'larryjfyan'\n"
},
{
"path": "utils/eval_4/meteor/meter.py",
"chars": 635,
"preview": "import os\n\n\ndef compute_meter_score(pred, ref):\n cwd = os.path.dirname(__file__)\n test_path = '{}/test'.format(cwd"
},
{
"path": "utils/eval_4/meteor/reference",
"chars": 264914,
"preview": "When did Beyonce start becoming popular ?\nWhat areas did Beyonce compete in when she was growing up ?\nWhen did Beyonce l"
},
{
"path": "utils/eval_4/meteor/res.txt",
"chars": 176868,
"preview": "Meteor version: 1.5\n\nEval ID: meteor-1.5-wo-en-norm-0.85_0.2_0.6_0.75-ex_st_sy_pa-1.0_0.6_0.8_0.6\n\nLanguage: "
},
{
"path": "utils/eval_4/meteor/test",
"chars": 207944,
"preview": "what <_unk_> <_unk_> <_unk_> <_unk_> <_unk_> ?\nwhat <_unk_> <_unk_> <_unk_> <_unk_> <_unk_> ?\nwhat <_unk_> <_unk_> <_unk"
},
{
"path": "utils/eval_4/rouge_metric/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "utils/eval_4/rouge_metric/rouge.py",
"chars": 3391,
"preview": "#!/usr/bin/env python\n# \n# File Name : rouge.py\n#\n# Description : Computes ROUGE-L metric as described by Lin and Hovey "
},
{
"path": "utils/helper.py",
"chars": 11978,
"preview": "import importlib\nimport logging\nimport math\nimport os\nimport re\nimport shutil\nimport subprocess\nimport sys\nimport time\ni"
},
{
"path": "utils/predictor.py",
"chars": 3379,
"preview": "import os\nimport re\nimport sys\n\nimport tensorflow as tf\n\nfrom gpu_env import MODEL_ID\n\nMODEL_PATH = './ext'\nprint(sys.pa"
},
{
"path": "utils/vocab.py",
"chars": 5793,
"preview": "import logging\nimport pickle\n\nimport numpy as np\n\nfrom gpu_env import APP_NAME\n\n\nclass Vocab:\n\n @staticmethod\n def"
}
]
// ... and 1 more files (download for full content)
About this extraction
This page contains the full source code of the hanxiao/daanet GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 49 files (5.5 MB), approximately 1.5M tokens, and a symbol index with 278 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.