[
  {
    "path": "README.md",
    "content": "Speech-to-Text-WaveNet : End-to-end sentence level Chinese speech recognition using DeepMind's WaveNet\n=\nA tensorflow implementation for Chinese speech recognition based on DeepMind's WaveNet: A Generative Model for Raw Audio. ([Hereafter the Paper]( https://arxiv.org/abs/1609.03499))\n\nVersion\n---\nCurrent Version : 0.0.1\n\nDependencies\n---\n1. python == 3.5\n2. tensorflow == 1.0.0\n3. librosa == 0.5.0\n\nDataset\n---\n[清华30小时中文数据集](http://data.cslt.org/thchs30/standalone.html)\n\nDirectories\n---\n1. cache: save data featrue and word dictionary\n2. data: wav files and related labels\n3. model: save the models\n\nNetwork model\n---\n1. Data random shuffle per epoch\n2. Xavier initialization\n3. Adam optimization algorithms\n4. Batch Normalization\n\nTrain the network\n---\npython3 train.py\n\nTest the network\n---\npython3 test.py\n\nOther resources\n---\n1. [TensorFlow练习15: 中文语音识别](http://blog.topspeedsnail.com/archives/10696#more-10696)\n2. [ibab's WaveNet(speech synthesis) tensorflow implementationt](https://github.com/ibab/tensorflow-wavenet)\n3. [buriburisuri's WaveNet(English speech recognition) tensorflow and sugartensor implementationt](https://github.com/buriburisuri/speech-to-text-wavenet#version)\n"
  },
  {
    "path": "cache/readme.md",
    "content": "\n"
  },
  {
    "path": "data/readme.md",
    "content": "\n"
  },
  {
    "path": "model/readme.me",
    "content": "\n"
  },
  {
    "path": "model.py",
    "content": "#-*- coding:utf-8 -*-\r\n__author__ = 'Deeper'\r\nimport tensorflow as tf  # 1.0.0\r\nimport numpy as np\r\n\r\nclass Model():\r\n\tdef __init__(self, n_out, batch_size=1, n_mfcc=20, is_training=True):\r\n\t\tn_dim = 128\r\n\t\tself.is_training = is_training\r\n\r\n\t\tself.input_data = tf.placeholder(dtype=tf.float32, shape=[batch_size, None, n_mfcc])\r\n\t\tself.seq_len = tf.reduce_sum(tf.cast(tf.not_equal(tf.reduce_sum(self.input_data, reduction_indices=2), 0.), tf.int32), reduction_indices=1)\r\n\t\tself.targets = tf.placeholder(dtype=tf.int32, shape=[batch_size, None])\r\n\r\n\t\t# 1D convolution\r\n\t\tself.conv1d_index = 0\r\n\t\tout = self.conv1d_layer(self.input_data, dim=n_dim)\r\n\t\t\r\n\t\t# stack hole CNN\r\n\t\tn_blocks = 3\r\n\t\tskip = 0\r\n\t\tself.aconv1d_index = 0\r\n\t\tfor _ in range(n_blocks):\r\n\t\t\tfor r in [1, 2, 4, 8, 16]:\r\n\t\t\t\tout, s = self.residual_block(out, size=7, rate=r, dim=n_dim)\r\n\t\t\t\tskip += s\r\n\r\n\t\tlogit = self.conv1d_layer(skip, dim=skip.get_shape().as_list()[-1])\r\n\t\tself.logit = self.conv1d_layer(logit, dim=n_out, bias=True, activation=None)\r\n\r\n\t\t# CTC loss\r\n\t\tindices = tf.where(tf.not_equal(tf.cast(self.targets, tf.float32), 0.))\r\n\t\ttarget = tf.SparseTensor(indices=indices, values=tf.gather_nd(self.targets, indices)-1, dense_shape=tf.cast(tf.shape(self.targets), tf.int64))\r\n\t\tloss = tf.nn.ctc_loss(target, self.logit, self.seq_len, time_major=False)\r\n\t\tself.cost = tf.reduce_mean(loss)\r\n\r\n\t\t# optimizer\r\n\t\toptimizer = tf.train.AdamOptimizer()\r\n\t\tvar_list = [var for var in tf.trainable_variables()]\r\n\t\tgradient = optimizer.compute_gradients(self.cost, var_list=var_list)\r\n\t\tself.optimizer_op = optimizer.apply_gradients(gradient)\r\n\r\n\tdef residual_block(self, input_tensor, size, rate, dim):\r\n\t\tconv_filter = self.aconv1d_layer(input_tensor, size=size, rate=rate, activation='tanh')\r\n\t\tconv_gate = self.aconv1d_layer(input_tensor, size=size, rate=rate, activation='sigmoid')\r\n\t\tout = conv_filter * conv_gate\r\n\t\tout = self.conv1d_layer(out, size=1, dim=dim)\r\n\t\treturn out + input_tensor, out\r\n\r\n\tdef conv1d_layer(self, input_tensor, size=1, dim=128, bias=False, activation='tanh'):\r\n\t\twith tf.variable_scope('conv1d'+str(self.conv1d_index)):\r\n\t\t\tshape = input_tensor.get_shape().as_list()\r\n\t\t\tkernel = tf.get_variable('kernel', (size, shape[-1], dim), dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer())\r\n\t\t\tif bias:\r\n\t\t\t\tb = tf.get_variable('b', [dim], dtype=tf.float32, initializer=tf.constant_initializer(0))\r\n\t\t\tout = tf.nn.conv1d(input_tensor, kernel, stride=1, padding='SAME') + (b if bias else 0)\r\n\t\t\tif not bias:\r\n\t\t\t\tout = self.batch_norm_wrapper(out)\r\n\r\n\t\t\tout = self.activation_wrapper(out, activation)\r\n\t\t\t\r\n\t\t\tself.conv1d_index += 1\r\n\t\t\treturn out\r\n\r\n\tdef aconv1d_layer(self, input_tensor, size=7, rate=2, bias=False, activation='tanh'):\r\n\t\twith tf.variable_scope('aconv1d_'+str(self.aconv1d_index)):\r\n\t\t\tshape = input_tensor.get_shape().as_list()\r\n\t\t\tkernel = tf.get_variable('kernel',(1, size, shape[-1], shape[-1]), dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer())\r\n\t\t\tif bias:\r\n\t\t\t\tb = tf.get_variable('b', [shape[-1]], dtype=tf.float32, initializer=tf.constant_initializer(0))\r\n\t\t\tout = tf.nn.atrous_conv2d(tf.expand_dims(input_tensor, dim=1), kernel, rate=rate, padding='SAME')\r\n\t\t\tout = tf.squeeze(out, [1])\r\n\t\t\tif not bias:\r\n\t\t\t\tout = self.batch_norm_wrapper(out)\r\n\r\n\t\t\tout = self.activation_wrapper(out, activation)\r\n\t\t\t\r\n\t\t\tself.aconv1d_index += 1\r\n\t\t\treturn out\r\n\r\n\tdef batch_norm_wrapper(self, inputs, decay=0.999):\r\n\t\tepsilon = 1e-3\r\n\t\tshape = inputs.get_shape().as_list()\r\n\r\n\t\tbeta = tf.get_variable('beta', shape[-1], dtype=tf.float32, initializer=tf.constant_initializer(0))\r\n\t\tgamma = tf.get_variable('gamma', shape[-1], dtype=tf.float32, initializer=tf.constant_initializer(1))\r\n\t\tpop_mean = tf.get_variable('mean', shape[-1], dtype=tf.float32, initializer=tf.constant_initializer(0))\r\n\t\tpop_var = tf.get_variable('variance', shape[-1], dtype=tf.float32, initializer=tf.constant_initializer(1))\r\n\t\tif self.is_training:\r\n\t\t\tbatch_mean, batch_var = tf.nn.moments(inputs, axes=list(range(len(shape)-1)))\r\n\t\t\ttrain_mean = tf.assign(pop_mean, pop_mean*decay+batch_mean*(1-decay))\r\n\t\t\ttrain_var =tf.assign(pop_var, pop_var*decay+batch_var*(1-decay))\r\n\t\t\twith tf.control_dependencies([train_mean, train_var]):\r\n\t\t\t\treturn tf.nn.batch_normalization(inputs, batch_mean, batch_var, beta, gamma, epsilon)\r\n\t\telse:\r\n\t\t\treturn tf.nn.batch_normalization(inputs, pop_mean, pop_var, beta, gamma, epsilon)\r\n\r\n\tdef activation_wrapper(self, inputs, activation):\r\n\t\tout = inputs\r\n\r\n\t\tif activation == 'sigmoid':\r\n\t\t\tout = tf.nn.sigmoid(out)\r\n\t\telif activation == 'tanh':\r\n\t\t\tout = tf.nn.tanh(out)\r\n\t\telif activation == 'relu':\r\n\t\t\tout = tf.nn.relu(out)\r\n\r\n\t\treturn out"
  },
  {
    "path": "test.py",
    "content": "#-*- coding:utf-8 -*-\r\n\r\nfrom __future__ import print_function\r\nfrom model import Model\r\nfrom utils import SpeechLoader\r\n\r\nimport tensorflow as tf  # 1.0.0\r\nimport numpy as np\r\nimport librosa\r\nimport os\r\n \r\n# 语音识别\r\n# 把batch_size改为1\r\ndef speech_to_text():\r\n    n_mfcc = 60\r\n\r\n    # load data\r\n    speech_loader = SpeechLoader()\r\n\r\n    # load model\r\n    model = Model(speech_loader.vocab_size, n_mfcc=n_mfcc, is_training=False)\r\n\r\n    saver = tf.train.Saver()\r\n\r\n    with tf.Session() as sess:\r\n        for j in range(750,755):\r\n            # extract feature\r\n            wav_file = os.path.join(os.getcwd(),'data','wav','test','D4','D4_'+str(j)+'.wav')\r\n            wav, sr = librosa.load(wav_file, mono=True)\r\n            mfcc = np.transpose(np.expand_dims(librosa.feature.mfcc(wav, sr, n_mfcc=n_mfcc), axis=0), [0,2,1])\r\n            mfcc = mfcc.tolist()\r\n\r\n            # fill 0\r\n            while len(mfcc[0]) < speech_loader.wav_max_len:\r\n                mfcc[0].append([0] * n_mfcc)\r\n\r\n            # word dict\r\n            wmap = {value:key for key, value in speech_loader.wordmap.items()}\r\n\r\n            # recognition\r\n            saver.restore(sess, tf.train.latest_checkpoint('model'))\r\n            decoded = tf.transpose(model.logit, perm=[1, 0, 2])\r\n            decoded, probs = tf.nn.ctc_beam_search_decoder(decoded, model.seq_len, top_paths=1, merge_repeated=True)\r\n            predict = tf.sparse_to_dense(decoded[0].indices, decoded[0].dense_shape, decoded[0].values) + 1\r\n            output, probs = sess.run([predict, probs], feed_dict={model.input_data: mfcc})\r\n            \r\n            # print result\r\n            words = ''\r\n            for i in range(len(output[0])):\r\n                words += wmap.get(output[0][i], -1)\r\n\r\n            print(\"---------------------------\")\r\n            print(\"Input: \" + wav_file)\r\n            print(\"Output: \" + words)\r\n\r\n\r\nif __name__ == '__main__':\r\n        speech_to_text()\r\n        \r\n    "
  },
  {
    "path": "train.py",
    "content": "#-*- coding:utf-8 -*-\r\n\r\nfrom __future__ import print_function\r\nfrom utils import SpeechLoader\r\nfrom model import Model\r\nimport tensorflow as tf #1.0.0\r\nimport time\r\nimport os\r\n\r\ndef train():\r\n\t# setting parameters\r\n\tbatch_size = 32\r\n\tn_epoch = 100\r\n\tn_mfcc = 60\r\n\r\n\t# load speech data\r\n\twav_path = os.path.join(os.getcwd(),'data','wav','train')\r\n\tlabel_file = os.path.join(os.getcwd(),'data','doc','trans','train.word.txt')\r\n\tspeech_loader = SpeechLoader(wav_path, label_file, batch_size, n_mfcc)\r\n\tn_out = speech_loader.vocab_size\r\n\r\n\t# load model\r\n\tmodel = Model(n_out, batch_size=batch_size, n_mfcc=n_mfcc)\r\n\r\n\twith tf.Session() as sess:\r\n\t\tsess.run(tf.global_variables_initializer())\r\n\t\t\r\n\t\tsaver = tf.train.Saver(tf.global_variables())\r\n\r\n\t\tfor epoch in range(n_epoch):\r\n\t\t\tspeech_loader.create_batches() # random shuffle data\r\n\t\t\tspeech_loader.reset_batch_pointer()\r\n\t\t\tfor batch in range(speech_loader.n_batches):\r\n\t\t\t\tstart = time.time()\r\n\t\t\t\tbatches_wav, batches_label = speech_loader.next_batch()\r\n\t\t\t\tfeed = {model.input_data: batches_wav, model.targets: batches_label}\r\n\t\t\t\ttrain_loss, _ = sess.run([model.cost, model.optimizer_op], feed_dict=feed)\r\n\t\t\t\tend = time.time()\r\n\t\t\t\tprint(\"epoch: %d/%d, batch: %d/%d, loss: %s, time: %.3f.\"%(epoch, n_epoch, batch, speech_loader.n_batches, train_loss, end-start))\r\n\r\n\t\t\t# save models\r\n\t\t\tif epoch % 5 ==0:\r\n\t\t\t\tsaver.save(sess, os.path.join(os.getcwd(), 'model','speech.module'), global_step=epoch)\r\n\r\n\r\nif __name__ == '__main__':\r\n\ttrain()"
  },
  {
    "path": "utils.py",
    "content": "#-*- coding:utf-8 -*-\r\n__author__ = 'Deeper'\r\nimport tensorflow as tf \r\nimport numpy as np \r\nimport os\r\nimport codecs\r\nimport librosa\r\nfrom six.moves import cPickle, reduce, map\r\nfrom collections import Counter\r\nimport random\r\n\r\nclass SpeechLoader():\r\n\r\n\tdef __init__(self, wav_path=None, label_file=None, batch_size=1, n_mfcc=20, encoding='utf-8'):\r\n\t\tself.batch_size = batch_size\r\n\t\tself.encoding = encoding\r\n\t\tself.n_mfcc = n_mfcc\r\n\r\n\t\t# path setting\r\n\t\tdata_dir = os.path.join(os.getcwd(), 'cache', 'mfcc'+str(n_mfcc))\r\n\t\t# data cache\r\n\t\twavs_file = os.path.join(data_dir, \"wavs.file\")\r\n\t\tvocab_file = os.path.join(data_dir,\"vocab.file\")\r\n\t\tmfcc_tensor = os.path.join(data_dir, \"mfcc.tensor\")\r\n\t\tlabel_tensor = os.path.join(data_dir, \"label.tensor\")\r\n\r\n\t\t# data process\r\n\t\tif not (os.path.exists(vocab_file) and os.path.exists(mfcc_tensor) and os.path.exists(label_tensor)):\r\n\t\t\tprint(\"reading wav files\")\r\n\t\t\tself.preprocess(wav_path, label_file, wavs_file, vocab_file, mfcc_tensor, label_tensor)\r\n\t\telse:\r\n\t\t\tprint(\"loading preprocessed files\")\r\n\t\t\tself.load_preprocessed(vocab_file, mfcc_tensor, label_tensor)\r\n\r\n\t\t# minibatch\r\n\t\tself.create_batches()\r\n\t\t# pointer reset\r\n\t\tself.reset_batch_pointer()\r\n\r\n\tdef preprocess(self, wav_path, label_file, wavs_file, vocab_file, mfcc_tensor, label_tensor):\r\n\t\tdef handle_file(dirpath, filename):\r\n\t\t\tif filename.endswith('.wav') or filename.endswith('.WAV'):\r\n\t\t\t\tfilename_path = os.path.join(dirpath, filename)\r\n\t\t\t\tif os.stat(filename_path).st_size < 240000:\r\n\t\t\t\t\treturn\r\n\t\t\t\treturn filename_path\r\n\r\n\t\t# read label file\r\n\t\tlabels_dict = {}\r\n\t\twith codecs.open(label_file,\"r\", encoding=self.encoding) as f:\r\n\t\t\tfor label in f:\r\n\t\t\t\tlabel = label.strip('\\n')\r\n\t\t\t\tlabels_id = label.split(' ',1)[0]\r\n\t\t\t\tlabels_text = label.split(' ',1)[1]\r\n\t\t\t\tlabels_dict[labels_id] = labels_text\r\n\t\t# print(\"\",len(labels_dict)) # 10000\r\n\t\t\r\n\t\t# wav files\r\n\t\twav_files = []\r\n\t\tif wav_path:\r\n\t\t\tfor (dirpath, dirnames, filenames) in os.walk(wav_path):\r\n\t\t\t\tfor filename in filenames:\r\n\t\t\t\t\tif handle_file(dirpath,filename):\r\n\t\t\t\t\t\twav_files.append(handle_file(dirpath,filename))\r\n\t\tprint(\"初始样本数：\", len(wav_files)) #样本数\r\n\r\n\t\t# data filter and feature extraction\r\n\t\twav_files_filter = []\r\n\t\tlabels_filter = []\r\n\t\tself.mfcc_tensor = []\r\n\t\tself.wav_max_len = 0\r\n\t\tcnt = 0\r\n\t\tfor wav_file in wav_files:\r\n\t\t\twav_id = os.path.basename(wav_file).split('.')[0]\r\n\t\t\tif wav_id in labels_dict:\r\n\t\t\t\tprint('样本'+str(cnt), wav_file)\r\n\t\t\t\tlabels_filter.append(labels_dict[wav_id])\r\n\t\t\t\twav_files_filter.append(wav_file)\r\n\t\t\t\t# mfcc feature\r\n\t\t\t\twav_file, sr = librosa.load(wav_file, mono=True)\r\n\t\t\t\tmfcc = np.transpose(librosa.feature.mfcc(wav_file, sr, n_mfcc=self.n_mfcc),[1,0])\r\n\t\t\t\tself.mfcc_tensor.append(mfcc.tolist())\r\n\t\t\t\tcnt += 1\r\n\t\tself.wav_max_len = max(len(mfcc) for mfcc in self.mfcc_tensor)\r\n\t\tprint(\"样本总数：\", cnt)\r\n\t\tprint(\"最长的语音：\", self.wav_max_len)\r\n\t\t# print(len(wav_files_filter), len(labels_filter),len(wav2mfcc)) # assert check dimensions\r\n\r\n\t\twith open(wavs_file, 'wb') as f:\r\n\t\t\tcPickle.dump(wav_files_filter, f)\r\n\r\n\t\twith open(mfcc_tensor, 'wb') as f:\r\n\t\t\tcPickle.dump(self.mfcc_tensor, f)\r\n\r\n\t\t# vocab file\r\n\t\tvocabs = []\r\n\t\tfor label in labels_filter:\r\n\t\t\tvocabs += [word for word in label]\r\n\t\tcount = Counter(vocabs)\r\n\t\tcount_pairs = sorted(count.items(), key=lambda x:-x[1])\r\n\t\twords, _ = zip(*count_pairs)\r\n\t\tself.wordmap = dict(zip(words, range(len(words))))\r\n\r\n\t\tself.vocab_size = len(words)\r\n\t\tprint(\"词汇表大小：\",len(words))\r\n\r\n\t\twith open(vocab_file,'wb') as f:\r\n\t\t\tcPickle.dump(self.wordmap, f)\r\n\r\n\t\t# label vector\r\n\t\tlabel_encoder = lambda word: self.wordmap.get(word, len(words))\r\n\t\tself.label_tensor = [list(map(label_encoder, label)) for label in labels_filter]\r\n\t\tself.label_max_len = max(len(label) for label in self.label_tensor)\r\n\t\tprint(\"最长的句子：\", self.label_max_len)\r\n\r\n\t\twith open(label_tensor,'wb') as f:\r\n\t\t\tcPickle.dump(self.label_tensor, f)\r\n\r\n\tdef load_preprocessed(self, vocab_file, mfcc_tensor, label_tensor):\r\n\t\twith open(vocab_file, 'rb') as f:\r\n\t\t\tself.wordmap = cPickle.load(f) \r\n\t\tself.vocab_size = len(self.wordmap)\r\n\t\tprint(\"词汇表大小：\",self.vocab_size)\r\n\r\n\t\twith open(mfcc_tensor, 'rb') as f:\r\n\t\t\tself.mfcc_tensor = cPickle.load(f)\r\n\t\tself.wav_max_len = max(len(mfcc) for mfcc in self.mfcc_tensor)\r\n\t\tprint(\"最长的语音：\", self.wav_max_len)\r\n\r\n\t\twith open(label_tensor, 'rb') as f:\r\n\t\t\tself.label_tensor = cPickle.load(f)\r\n\t\tself.label_max_len = max(len(label) for label in self.label_tensor)\r\n\t\tprint(\"最长的句子：\", self.label_max_len)\r\n\r\n\tdef create_batches(self):\r\n\t\tself.n_batches = len(self.mfcc_tensor) // self.batch_size\r\n\t\tif self.n_batches==0:\r\n\t\t\tassert False, \"Not enough data. Make seq_length and batch_size small.\"\r\n\t\t\r\n\t\tself.mfcc_tensor = self.mfcc_tensor[:self.n_batches*self.batch_size]\r\n\t\tself.label_tensor = self.label_tensor[:self.n_batches*self.batch_size]\r\n\r\n\t\t# random shuffle the data\r\n\t\tif len(self.mfcc_tensor) != len(self.label_tensor):\r\n\t\t\tassert False, \"Data length does not match the label length!\"\r\n\r\n\t\tdata_tensor = []\r\n\t\tfor i in range(len(self.mfcc_tensor)):\r\n\t\t\tdata_tensor.append([self.mfcc_tensor[i], self.label_tensor[i]])\r\n\r\n\t\trandom.shuffle(data_tensor)\r\n\t\tself.mfcc_tensor = []\r\n\t\tself.label_tensor = []\r\n\t\tfor i in range(len(data_tensor)):\r\n\t\t\tself.mfcc_tensor.append(data_tensor[i][0])\r\n\t\t\tself.label_tensor.append(data_tensor[i][1])\r\n\r\n\t\t# create batches\r\n\t\tself.x_batches = []\r\n\t\tself.y_batches = []\r\n\r\n\t\tfor i in range(self.n_batches):\r\n\t\t\tfrom_index = i*self.batch_size\r\n\t\t\tto_index = from_index + self.batch_size\r\n\t\t\tmfcc_batches = self.mfcc_tensor[from_index:to_index]\r\n\t\t\tlabel_batches = self.label_tensor[from_index:to_index]\r\n\t\t\t# 补零对齐\r\n\t\t\tfor mfcc in mfcc_batches:\r\n\t\t\t\twhile len(mfcc) < self.wav_max_len:\r\n\t\t\t\t\tmfcc.append([0]*self.n_mfcc)\r\n\t\t\tfor label in label_batches:\r\n\t\t\t\twhile len(label) < self.label_max_len:\r\n\t\t\t\t\tlabel.append(0)\r\n\r\n\t\t\tself.x_batches.append(mfcc_batches)\r\n\t\t\tself.y_batches.append(label_batches)\r\n\r\n\tdef next_batch(self):\r\n\t\tx, y = self.x_batches[self.pointer], self.y_batches[self.pointer]\r\n\t\tself.pointer += 1\r\n\t\treturn x, y\r\n\r\n\tdef reset_batch_pointer(self):\r\n\t\tself.pointer = 0"
  }
]