[ { "path": "README.md", "content": "# dcase2017_task4_cvssp\n\nSource code for the DCASE2017 task4 \"Large-scale weakly supervised sound event detection for smart cars\" challenge. \n\n# Warning: this code is out of date, new version of our DCASE2017 code could be found here: https://github.com/qiuqiangkong/sound_event_detection_dcase2017_task4 \n\nThe detailed results can be found here: \nhttp://www.cs.tut.fi/sgn/arg/dcase2017/challenge/task-large-scale-sound-event-detection-results\n\nThe system submitted by CVSSP team is summarized here:\nhttp://www.cs.tut.fi/sgn/arg/dcase2017/documents/challenge_technical_reports/DCASE2017_Xu_146.pdf\n\nThe data set used in this chanllege is part of Google Audioset.\n\nDCASE 2017 challenge is organized by TUT, CMU and INRIA, sponsored by Google and Audio Analytic. The CVSSP team submitted four systems to the audio tagging sub-task,which took all the top four places on the result table, among the 29 systems submitted by a number of organisations. CVSSP's system is also ranked at the 3rd place in the sound event detection subtask, among 17 systems. The competitors include CMU, New York University, Bosch, USC, TUT, Singapore A*Star, Korean Advanced Institute of Science and Technology, Seoul National University, National Taiwan university, etc.\n\nFor downloading the data, please see https://groups.google.com/forum/#!searchin/dcase-discussions/own%7Csort:relevance/dcase-discussions/Lk2dTScX3A8/kvW17tlzAgAJ\n\nNumber of training files: 51172\nNumber of testing files: 488\nNumber of evaluation files: 1103\n\n@inproceedings{xu2018large,\n title={Large-scale weakly supervised audio classification using gated convolutional neural network},\n author={Xu, Yong and Kong, Qiuqiang and Wang, Wenwu and Plumbley, Mark D},\n booktitle={2018 IEEE international conference on acoustics, speech and signal processing (ICASSP)},\n pages={121--125},\n year={2018},\n organization={IEEE}\n}\n" }, { "path": "config.py", "content": "# workspace = \"\"\nworkspace = \"/vol/vssp/msos/qk/workspaces/ICASSP2018_dcase\"\n\n# config\nsample_rate = 16000.\nn_window = 1024\nn_overlap = 360 # ensure 240 frames in 10 seconds\nmax_len = 240 # sequence max length is 10 s, 240 frames. \nstep_time_in_sec = float(n_window - n_overlap) / sample_rate\n\n# Id of classes\nids = ['/m/0284vy3', '/m/05x_td', '/m/02mfyn', '/m/02rhddq', '/m/0199g', \n '/m/06_fw', '/m/012n7d', '/m/012ndj', '/m/0dgbq', '/m/04qvtq', \n '/m/03qc9zr', '/m/0k4j', '/t/dd00134', '/m/01bjv', '/m/07r04', \n '/m/04_sv', '/m/07jdr']\n\n# Name of classes\nlbs = ['Train horn', 'Air horn, truck horn', 'Car alarm', 'Reversing beeps', \n 'Bicycle', 'Skateboard', 'Ambulance (siren)', \n 'Fire engine, fire truck (siren)', 'Civil defense siren', \n 'Police car (siren)', 'Screaming', 'Car', 'Car passing by', 'Bus', \n 'Truck', 'Motorcycle', 'Train']\n \nidx_to_id = {index: id for index, id in enumerate(ids)}\nid_to_idx = {id: index for index, id in enumerate(ids)}\nidx_to_lb = {index: lb for index, lb in enumerate(lbs)}\nlb_to_idx = {lb: index for index, lb in enumerate(lbs)}\nnum_classes = len(lbs)" }, { "path": "data_generator.py", "content": "import numpy as np\nimport random\n\nclass BalanceDataGenerator(object):\n def __init__(self, batch_size, type, te_max_iter=100):\n assert type in ['train', 'test']\n self._batch_size_ = batch_size\n self._type_ = type\n self._te_max_iter_ = te_max_iter\n \n def generate(self, xs, ys):\n batch_size = self._batch_size_\n x = xs[0]\n y = ys[0]\n (n_samples, n_labs) = y.shape\n n_each = batch_size // n_labs \n \n index_list = []\n for i1 in xrange(n_labs):\n index_list.append(np.where(y[:, i1] == 1)[0])\n \n for i1 in xrange(n_labs):\n np.random.shuffle(index_list[i1])\n \n pointer_list = [0] * n_labs\n len_list = [len(e) for e in index_list]\n iter = 0\n while True:\n if (self._type_) == 'test' and (iter == self._te_max_iter_):\n break\n iter += 1\n batch_x = []\n batch_y = []\n for i1 in xrange(n_labs):\n if pointer_list[i1] >= len_list[i1]:\n pointer_list[i1] = 0\n np.random.shuffle(index_list[i1])\n \n batch_idx = index_list[i1][pointer_list[i1] : min(pointer_list[i1] + n_each, len_list[i1])]\n batch_x.append(x[batch_idx])\n batch_y.append(y[batch_idx])\n pointer_list[i1] += n_each\n batch_x = np.concatenate(batch_x, axis=0)\n batch_y = np.concatenate(batch_y, axis=0)\n yield batch_x, batch_y\n\nclass RatioDataGenerator(object):\n def __init__(self, batch_size, type, te_max_iter=100, verbose=1):\n assert type in ['train', 'test']\n self._batch_size_ = batch_size\n self._type_ = type\n self._te_max_iter_ = te_max_iter\n self._verbose_ = verbose\n \n def _get_lb_list(self, n_samples_list):\n lb_list = []\n for idx in xrange(len(n_samples_list)):\n n_samples = n_samples_list[idx]\n if n_samples < 1000:\n lb_list += [idx]\n elif n_samples < 2000:\n lb_list += [idx] * 2\n elif n_samples < 3000:\n lb_list += [idx] * 3\n elif n_samples < 4000:\n lb_list += [idx] * 4\n else:\n lb_list += [idx] * 5\n return lb_list\n \n def generate(self, xs, ys):\n batch_size = self._batch_size_\n x = xs[0]\n y = ys[0]\n (n_samples, n_labs) = y.shape\n \n n_samples_list = np.sum(y, axis=0)\n lb_list = self._get_lb_list(n_samples_list)\n \n if self._verbose_ == 1:\n print(\"n_samples_list: %s\" % (n_samples_list,))\n print(\"lb_list: %s\" % (lb_list,))\n print(\"len(lb_list): %d\" % len(lb_list))\n \n index_list = []\n for i1 in xrange(n_labs):\n index_list.append(np.where(y[:, i1] == 1)[0])\n \n for i1 in xrange(n_labs):\n np.random.shuffle(index_list[i1])\n \n queue = []\n pointer_list = [0] * n_labs\n len_list = [len(e) for e in index_list]\n iter = 0\n while True:\n if (self._type_) == 'test' and (iter == self._te_max_iter_):\n break\n iter += 1\n batch_x = []\n batch_y = []\n \n while len(queue) < batch_size:\n random.shuffle(lb_list)\n queue += lb_list\n \n batch_idx = queue[0 : batch_size]\n queue[0 : batch_size] = []\n \n n_per_class_list = [batch_idx.count(idx) for idx in xrange(n_labs)]\n \n for i1 in xrange(n_labs):\n if pointer_list[i1] >= len_list[i1]:\n pointer_list[i1] = 0\n np.random.shuffle(index_list[i1])\n \n per_class_batch_idx = index_list[i1][pointer_list[i1] : min(pointer_list[i1] + n_per_class_list[i1], len_list[i1])]\n batch_x.append(x[per_class_batch_idx])\n batch_y.append(y[per_class_batch_idx])\n pointer_list[i1] += n_per_class_list[i1]\n batch_x = np.concatenate(batch_x, axis=0)\n batch_y = np.concatenate(batch_y, axis=0)\n yield batch_x, batch_y\n\n" }, { "path": "download.py", "content": "" }, { "path": "main.py", "content": "\"\"\"Train a cldnn on the task4 of DCASE2017 dataset.\nGPU run command with Theano backend (with TensorFlow, the GPU is automatically used):\n THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatx=float32 python train*.py\n\nAuthor: Yong XU\nCreat date: 03/04/2017\n\"\"\"\nfrom __future__ import print_function \nimport sys\nimport cPickle\nimport numpy as np\nimport argparse\nimport time\nimport os\n\nimport keras\nfrom keras import backend as K\nfrom keras.models import Sequential,Model\nfrom keras.layers.core import Dense, Dropout, Activation, Flatten, Reshape, Permute,Lambda, RepeatVector\nfrom keras.layers.convolutional import ZeroPadding2D, AveragePooling2D, Conv2D,MaxPooling2D, Convolution1D,MaxPooling1D\nfrom keras.layers.pooling import GlobalMaxPooling2D\nfrom keras.layers import Merge, Input, merge\nfrom keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, LearningRateScheduler\nfrom keras.layers import LSTM, SimpleRNN, GRU, TimeDistributed, Bidirectional\nfrom keras.layers.normalization import BatchNormalization\nimport h5py\nfrom keras.layers.merge import Multiply\nfrom sklearn import preprocessing\nimport random\n\nimport config as cfg\nfrom prepare_data import create_folder, load_hdf5_data, calculate_scaler, do_scale\nfrom data_generator import RatioDataGenerator\n\n\ndef scheduler(epoch):\n initial_lrate = float(0.001)\n cur_lr=initial_lrate*pow(float(0.5),(epoch//5))\n print(\"learning rate: %f\" % cur_lr)\n return cur_lr\n\ndef block(input):\n cnn = Conv2D(128, (3, 3), padding=\"same\", activation=\"linear\", use_bias=False)(input)\n cnn = BatchNormalization(axis=-1)(cnn)\n\n cnn1 = Lambda(slice1, output_shape=slice1_output_shape)(cnn)\n cnn2 = Lambda(slice2, output_shape=slice2_output_shape)(cnn)\n\n cnn1 = Activation('linear')(cnn1)\n cnn2 = Activation('sigmoid')(cnn2)\n\n out = Multiply()([cnn1, cnn2])\n return out\n\ndef slice1(x):\n return x[:, :, :, 0:64]\n\ndef slice2(x):\n return x[:, :, :, 64:128]\n\ndef slice1_output_shape(input_shape):\n return tuple([input_shape[0],input_shape[1],input_shape[2],64])\n\ndef slice2_output_shape(input_shape):\n return tuple([input_shape[0],input_shape[1],input_shape[2],64])\n\ndef train():\n num_classes = cfg.num_classes\n \n # Load training & testing data\n (tr_x, tr_y, tr_na_list) = load_hdf5_data(args.tr_hdf5_path, verbose=1)\n # (tr_x, tr_y, tr_na_list) = load_hdf5(args.te_hdf5_path, verbose=1)\n (te_x, te_y, te_na_list) = load_hdf5_data(args.te_hdf5_path, verbose=1)\n print(\"\")\n\n # Scale data\n scaler = calculate_scaler(tr_x, verbose=2)\n tr_x = do_scale(tr_x, scaler, verbose=1)\n te_x = do_scale(te_x, scaler, verbose=1)\n pause\n # Build model\n (_, n_time, n_freq) = tr_x.shape\n input_logmel = Input(shape=(n_time, n_freq))\n a1 = Reshape((n_time, n_freq, 1))(input_logmel)\n \n cnn1 = block(a1)\n cnn1 = block(cnn1)\n cnn1 = MaxPooling2D(pool_size=(2, 2))(cnn1)\n \n cnn1 = block(cnn1)\n cnn1 = block(cnn1)\n cnn1 = MaxPooling2D(pool_size=(2, 2))(cnn1)\n \n cnn1 = block(cnn1)\n cnn1 = block(cnn1)\n cnn1 = MaxPooling2D(pool_size=(2, 2))(cnn1)\n \n cnn1 = block(cnn1)\n cnn3 = block(cnn1)\n cnn3 = MaxPooling2D(pool_size=(1, 2))(cnn1)\n \n cnnout = Conv2D(256, (3, 3), padding=\"same\", activation=\"relu\", use_bias=True)(cnn3)\n cnnout = MaxPooling2D(pool_size=(1, 4))(cnnout)\n \n cnnout = Reshape((30, 256))(cnnout) # Time step is downsampled to 30. \n \n rnnout = Bidirectional(GRU(128, activation='linear', return_sequences=True))(cnnout)\n rnnout_gate = Bidirectional(GRU(128, activation='sigmoid', return_sequences=True))(cnnout)\n out = Multiply()([rnnout, rnnout_gate])\n \n out = TimeDistributed(Dense(num_classes, activation='sigmoid'))(out)\n out = Lambda(lambda x: K.mean(x, axis=1),output_shape=(num_classes,))(out)\n \n model = Model(input_logmel, out)\n model.summary()\n \n # Compile model\n model.compile(loss='binary_crossentropy',\n optimizer='adam',\n metrics=['accuracy'])\n \n # Save model callback\n filepath = os.path.join(args.out_model_dir, \"gatedAct_rationBal44_lr0.001_normalization_at_cnnRNN_64newMel_240fr.{epoch:02d}-{val_acc:.4f}.hdf5\")\n create_folder(os.path.dirname(filepath))\n save_model = ModelCheckpoint(filepath=filepath,\n monitor='val_acc', \n verbose=0,\n save_best_only=False,\n save_weights_only=False,\n mode='auto',\n period=1) \n\n # Data generator\n # gen = RatioDataGenerator(batch_size=44, type='train')\n gen = RatioDataGenerator(batch_size=100, type='train')\n\n # Train\n model.fit_generator(generator=gen.generate([tr_x], [tr_y]), \n steps_per_epoch=10, # 100 iters is called an 'epoch'\n epochs=2000000010, # Maximum 'epoch' to train\n verbose=1, \n callbacks=[save_model], \n validation_data=(te_x, te_y))\n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser(description=\"\")\n subparsers = parser.add_subparsers(dest='mode')\n \n parser_train = subparsers.add_parser('train')\n parser_train.add_argument('--tr_hdf5_path', type=str)\n parser_train.add_argument('--te_hdf5_path', type=str)\n parser_train.add_argument('--out_model_dir', type=str)\n \n args = parser.parse_args()\n \n if args.mode == 'train':\n train()" }, { "path": "main_cnnrnn_at.py", "content": "'''Train a cldnn on the task4 of DCASE2017 dataset.\nGPU run command with Theano backend (with TensorFlow, the GPU is automatically used):\n THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatx=float32 python train*.py\n\nAuthor: Yong XU\nCreat date: 03/04/2017\n'''\n\n#from __future__ import print_function \n\nimport keras\nfrom keras import backend as K\nimport sys\nimport cPickle\nimport numpy as np\nfrom keras.models import Sequential,Model\n#from keras.layers.core import Dense, Dropout, Activation, Flatten, Flatten_last2d, Reshape,Permute,Lambda, RepeatVector\nfrom keras.layers.core import Dense, Dropout, Activation, Flatten, Reshape,Permute,Lambda, RepeatVector\nfrom keras.layers.convolutional import ZeroPadding2D, AveragePooling2D, Conv2D,MaxPooling2D, Convolution1D,MaxPooling1D\nfrom keras.layers.pooling import GlobalMaxPooling2D\nfrom keras.layers import Merge, Input, merge\nfrom keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, LearningRateScheduler\nfrom keras.layers import LSTM, SimpleRNN, GRU, TimeDistributed, Bidirectional\nfrom keras.layers.normalization import BatchNormalization\nimport h5py\nfrom keras.layers.merge import Multiply\nfrom sklearn import preprocessing\nimport random\n\ndef scheduler(epoch):\n initial_lrate = float(0.001)\n cur_lr=initial_lrate*pow(float(0.5),(epoch//5))\n print \"learning rate:\", cur_lr\n return cur_lr\n\nclass RatioDataGenerator(object):\n def __init__(self, batch_size, type, te_max_iter=100):\n assert type in ['train', 'test']\n self._batch_size_ = batch_size\n self._type_ = type\n self._te_max_iter_ = te_max_iter\n \n def _get_lb_list(self, n_samples_list):\n lb_list = []\n for idx in xrange(len(n_samples_list)):\n n_samples = n_samples_list[idx]\n if n_samples < 1000:\n lb_list += [idx]\n elif n_samples < 2000:\n lb_list += [idx] * 2\n elif n_samples < 3000:\n lb_list += [idx] * 3\n elif n_samples < 4000:\n lb_list += [idx] * 4\n else:\n lb_list += [idx] * 5\n return lb_list\n \n def generate(self, xs, ys):\n batch_size = self._batch_size_\n x = xs[0]\n y = ys[0]\n (n_samples, n_labs) = y.shape\n \n n_samples_list = np.sum(y, axis=0)\n lb_list = self._get_lb_list(n_samples_list)\n \n print \"n_samples_list:\", n_samples_list\n print \"lb_list:\", lb_list\n print \"len(lb_list):\", len(lb_list)\n \n \n index_list = []\n for i1 in xrange(n_labs):\n index_list.append(np.where(y[:, i1] == 1)[0])\n \n for i1 in xrange(n_labs):\n np.random.shuffle(index_list[i1])\n \n queue = []\n pointer_list = [0] * n_labs\n len_list = [len(e) for e in index_list]\n iter = 0\n while True:\n if (self._type_) == 'test' and (iter == self._te_max_iter_):\n break\n iter += 1\n batch_x = []\n batch_y = []\n \n while len(queue) < batch_size:\n random.shuffle(lb_list)\n queue += lb_list\n \n batch_idx = queue[0 : batch_size]\n queue[0 : batch_size] = []\n \n n_per_class_list = [batch_idx.count(idx) for idx in xrange(n_labs)]\n \n for i1 in xrange(n_labs):\n if pointer_list[i1] >= len_list[i1]:\n pointer_list[i1] = 0\n np.random.shuffle(index_list[i1])\n \n per_class_batch_idx = index_list[i1][pointer_list[i1] : min(pointer_list[i1] + n_per_class_list[i1], len_list[i1])]\n batch_x.append(x[per_class_batch_idx])\n batch_y.append(y[per_class_batch_idx])\n pointer_list[i1] += n_per_class_list[i1]\n batch_x = np.concatenate(batch_x, axis=0)\n batch_y = np.concatenate(batch_y, axis=0)\n yield batch_x, batch_y\n\nclass BalanceDataGenerator(object):\n def __init__(self, batch_size, type, max_iter=100):\n assert type in ['train', 'test']\n self._batch_size_ = batch_size\n self._type_ = type\n self._max_iter_ = max_iter\n \n def generate(self, xs, ys):\n batch_size = self._batch_size_\n x = xs[0]\n y = ys[0]\n #(n_samples, n_features) = x.shape ### yong xu commented\n (n_samples, n_labs) = y.shape\n n_each = batch_size // n_labs \n \n index_list = []\n for i1 in xrange(n_labs):\n index_list.append(np.where(y[:, i1] == 1)[0])\n \n for i1 in xrange(n_labs):\n np.random.shuffle(index_list[i1])\n \n pointer_list = [0] * n_labs\n iter = 0\n while True:\n if (self._type_) == 'test' and (iter == self._max_iter_):\n break\n iter += 1\n batch_x = []\n batch_y = []\n for i1 in xrange(n_labs):\n idx_num = len(index_list[i1])\n if pointer_list[i1] >= idx_num:\n pointer_list[i1] = 0\n np.random.shuffle(index_list[i1])\n \n batch_idx = index_list[i1][pointer_list[i1] : min(pointer_list[i1] + n_each, idx_num)]\n batch_x.append(x[batch_idx])\n batch_y.append(y[batch_idx])\n pointer_list[i1] += n_each\n batch_x = np.concatenate(batch_x, axis=0)\n batch_y = np.concatenate(batch_y, axis=0)\n yield batch_x, batch_y\n\n# resize data for fit into CNN. size: (batch_num*color_maps*height*weight)\ndef reshapeX1( X ):\n N = len(X)\n return X.reshape( (N, t_delay, feadim, 1, 1) )\n \n# resize data for fit into CNN. size: (batch_num*color_maps*height*weight)\ndef reshapeX2( X ):\n N = len(X)\n return X.reshape( (N, t_delay, feadim) )\n\ndef reshapeX3( X ):\n N = len(X)\n return X.reshape( (N, t_delay, feadim, 1) )\n\ndef reshapeX4( X ):\n N = len(X)\n return X.reshape( (N*t_delay, feadim) )\n\ndef reshapeX5( X , sample_num):\n N = len(X)\n return X.reshape( (sample_num, t_delay, feadim, 1) )\n\ndef outfunc(vects):\n x,y=vects\n #y=K.sum( y, axis=1 )\n y = K.clip( y, 1.0e-9, 1 ) # clip to avoid numerical underflow\n #z=Lambda(lambda x: K.sum(x, axis=1),output_shape=(8,))(y)\n y = K.sum(y, axis=1)\n #y = K.sum(y, axis=1)\n #z = RepeatVector(249)(z)\n #z=Permute((2,1))(z)\n #return K.sum( x / z, axis=1 )\n #x = K.sum( x, axis=(1,2) )\n x = K.sum( x, axis=1 )\n #x = K.sum( x, axis=1 ) \n return x / y\n\ndef slice1(x):\n return x[:,:,:, 0:64]\n\ndef slice2(x):\n return x[:,:,:, 64:128]\n\ndef slice1_output_shape(input_shape):\n return tuple([input_shape[0],input_shape[1],input_shape[2],64])\n\ndef slice2_output_shape(input_shape):\n return tuple([input_shape[0],input_shape[1],input_shape[2],64])\n\n#parameters:\nnum_classes=17\nfeadim=64\nt_delay=240 # the len of Utterance is 300\nmodel_out_path=\"/vol/vssp/msos/yx/t4_d2017/models_val\"\n\n# train and test sets:\nwith h5py.File(\"/vol/vssp/msos/Audioset/task4_dcase2017_features/packed_features/logmel/training_pack.h5\", 'r') as hf:\n tr_x = np.array(hf.get('x'))\n tr_y = np.array(hf.get('y'))\nprint tr_x.shape\nprint tr_y.shape\n\n\n\nwith h5py.File(\"/vol/vssp/msos/Audioset/task4_dcase2017_features/packed_features/logmel/testing_pack.h5\", 'r') as hf:\n va_x = np.array(hf.get('x'))\n va_y = np.array(hf.get('y'))\nprint va_x.shape\nprint va_y.shape\n\n###########normalization training and test set\ntr_x2=reshapeX4(tr_x)\nva_x2=reshapeX4(va_x)\nscaler = preprocessing.StandardScaler().fit(tr_x2)\nprint scaler.mean_, scaler.scale_\ntr_x2 = scaler.transform( tr_x2 )\nva_x2 = scaler.transform( va_x2 )\ntr_x=reshapeX5( tr_x2 , len(tr_x))\nva_x=reshapeX5( va_x2 , len(va_x))\n####################################\n#tr_x=reshapeX3(tr_x)\nprint tr_x.shape\n#va_x=reshapeX3(va_x)\nprint va_x.shape\n\ndef block(input):\n cnn1=Conv2D(128, (3, 3), padding=\"same\", activation=\"linear\", use_bias=False)(input)\n cnn1=BatchNormalization(axis=-1)(cnn1)\n\n cnn11 = Lambda(slice1,output_shape=slice1_output_shape)(cnn1)\n cnn12 = Lambda(slice2,output_shape=slice2_output_shape)(cnn1)\n\n cnn11=Activation('linear')(cnn11)\n cnn12=Activation('sigmoid')(cnn12)\n\n cnn1=Multiply()([cnn11,cnn12])\n return cnn1\n\n###build model by keras\n\ninput_audio=Input(shape=(t_delay, feadim, 1))\n\n#input_flat=TimeDistributed(Flatten())(input_audio)\n\n###detection factor for each tag (7 meaningful tags + 1 silence tag = 8 tags)\n#det =TimeDistributed(Dense(17,activation='softmax'))(input_flat) # The posterior sum of each tag is 1.0, now the dims of det are 33 frs * 8 tags\n\ncnn1 = block(input_audio)\ncnn1 = block(cnn1)\ncnn1=MaxPooling2D(pool_size=(2, 2))(cnn1)\n\ncnn1 = block(cnn1)\ncnn1 = block(cnn1)\ncnn1=MaxPooling2D(pool_size=(2, 2))(cnn1)\n\ncnn1 = block(cnn1)\ncnn1 = block(cnn1)\ncnn1=MaxPooling2D(pool_size=(2, 2))(cnn1)\n\ncnn1 = block(cnn1)\ncnn3 = block(cnn1)\ncnn3=MaxPooling2D(pool_size=(1, 3))(cnn1)\n\ncnnout=Conv2D(256, (3, 3), padding=\"same\", activation=\"relu\", use_bias=True)(cnn3)\ncnnout=MaxPooling2D(pool_size=(1, 2))(cnnout)\n\n#cnnout=Flatten_last2d()(cnnout)\ncnnout=Reshape((30,256))(cnnout)\n\nrnnout=Bidirectional(GRU(128, activation='linear', return_sequences=True))(cnnout)\nrnnout_gate=Bidirectional(GRU(128, activation='sigmoid', return_sequences=True))(cnnout)\nout=Multiply()([rnnout,rnnout_gate])\n\nout=TimeDistributed(Dense(17,activation='sigmoid'))(out)\n#det =TimeDistributed(Dense(17,activation='softmax'))(out)\n#out=Multiply()([out,det])\n#out=Lambda(outfunc,output_shape=(17,))([out,det])\nout=Lambda(lambda x: K.mean(x, axis=1),output_shape=(17,))(out)\n\nallmodel=Model(input_audio, out)\nallmodel.summary()\n\n# Let's train the model using RMSprop\nallmodel.compile(loss='binary_crossentropy',\n optimizer='adam',\n metrics=['accuracy'])\n\ndump_fd=model_out_path+'/gatedAct_rationBal44_lr0.001_normalization_at_cnnRNN_64newMel_240fr.{epoch:02d}-{val_acc:.4f}.hdf5'\n\neachmodel=ModelCheckpoint(dump_fd,monitor='val_acc',verbose=0,save_best_only=False,save_weights_only=False,mode='auto',period=10) \n\n#reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,\n# patience=5, min_lr=0.00000001)\n\n#reduce_lr = LearningRateScheduler(scheduler)\n\ngen = RatioDataGenerator(batch_size=44, type='train')\n#gen = BalanceDataGenerator(batch_size=52, type='train')\n#for (batch_x, batch_y) in gen.generate([x], [y]):\n# train_on_batch(batch_x, batch_y, class_weight=None, sample_weight=None)\n\nsteps_per_epoch=100\nallmodel.fit_generator(gen.generate([tr_x], [tr_y]), steps_per_epoch, epochs=2000000010, verbose=1, callbacks=[eachmodel], validation_data=(va_x, va_y))\n#allmodel.fit(tr_x, tr_y, batch_size=100, epochs=31,\n# verbose=1, validation_data=(va_x, va_y), callbacks=[eachmodel]) #, callbacks=[best_model]) \n" }, { "path": "main_cnnrnn_balbatch_norm_baseline_icassp2018.py", "content": "'''Train a cldnn on the task4 of DCASE2017 dataset.\nGPU run command with Theano backend (with TensorFlow, the GPU is automatically used):\n THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatx=float32 python train*.py\n\nAuthor: Yong XU\nCreat date: 03/04/2017\n'''\n\n#from __future__ import print_function\n\nimport keras\nfrom keras import backend as K\nimport sys\nimport cPickle\nimport numpy as np\nfrom keras.models import Sequential,Model\n#from keras.layers.core import Dense, Dropout, Activation, Flatten, Flatten_last2d, Reshape,Permute,Lambda, RepeatVector\nfrom keras.layers.core import Dense, Dropout, Activation, Flatten, Reshape,Permute,Lambda, RepeatVector\nfrom keras.layers.convolutional import ZeroPadding2D, AveragePooling2D, Conv2D,MaxPooling2D, Convolution1D,MaxPooling1D\nfrom keras.layers.pooling import GlobalMaxPooling2D\nfrom keras.layers import Merge, Input, merge\nfrom keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, LearningRateScheduler\nfrom keras.layers import LSTM, SimpleRNN, GRU, TimeDistributed, Bidirectional\nfrom keras.layers.normalization import BatchNormalization\nimport h5py\nfrom keras.layers.merge import Multiply\nfrom sklearn import preprocessing\nimport random\n\ndef scheduler(epoch):\n initial_lrate = float(0.001)\n cur_lr=initial_lrate*pow(float(0.5),(epoch//5))\n print \"learning rate:\", cur_lr\n return cur_lr\n\nclass RatioDataGenerator(object):\n def __init__(self, batch_size, type, te_max_iter=100):\n assert type in ['train', 'test']\n self._batch_size_ = batch_size\n self._type_ = type\n self._te_max_iter_ = te_max_iter\n \n def _get_lb_list(self, n_samples_list):\n lb_list = []\n for idx in xrange(len(n_samples_list)):\n n_samples = n_samples_list[idx]\n if n_samples < 1000:\n lb_list += [idx]\n elif n_samples < 2000:\n lb_list += [idx] * 2\n elif n_samples < 3000:\n lb_list += [idx] * 3\n elif n_samples < 4000:\n lb_list += [idx] * 4\n else:\n lb_list += [idx] * 5\n return lb_list\n \n def generate(self, xs, ys):\n batch_size = self._batch_size_\n x = xs[0]\n y = ys[0]\n (n_samples, n_labs) = y.shape\n \n n_samples_list = np.sum(y, axis=0)\n lb_list = self._get_lb_list(n_samples_list)\n \n print \"n_samples_list:\", n_samples_list\n print \"lb_list:\", lb_list\n print \"len(lb_list):\", len(lb_list)\n \n \n index_list = []\n for i1 in xrange(n_labs):\n index_list.append(np.where(y[:, i1] == 1)[0])\n \n for i1 in xrange(n_labs):\n np.random.shuffle(index_list[i1])\n \n queue = []\n pointer_list = [0] * n_labs\n len_list = [len(e) for e in index_list]\n iter = 0\n while True:\n if (self._type_) == 'test' and (iter == self._te_max_iter_):\n break\n iter += 1\n batch_x = []\n batch_y = []\n \n while len(queue) < batch_size:\n random.shuffle(lb_list)\n queue += lb_list\n \n batch_idx = queue[0 : batch_size]\n queue[0 : batch_size] = []\n \n n_per_class_list = [batch_idx.count(idx) for idx in xrange(n_labs)]\n \n for i1 in xrange(n_labs):\n if pointer_list[i1] >= len_list[i1]:\n pointer_list[i1] = 0\n np.random.shuffle(index_list[i1])\n \n per_class_batch_idx = index_list[i1][pointer_list[i1] : min(pointer_list[i1] + n_per_class_list[i1], len_list[i1])]\n batch_x.append(x[per_class_batch_idx])\n batch_y.append(y[per_class_batch_idx])\n pointer_list[i1] += n_per_class_list[i1]\n batch_x = np.concatenate(batch_x, axis=0)\n batch_y = np.concatenate(batch_y, axis=0)\n yield batch_x, batch_y\n\nclass BalanceDataGenerator(object):\n def __init__(self, batch_size, type, max_iter=100):\n assert type in ['train', 'test']\n self._batch_size_ = batch_size\n self._type_ = type\n self._max_iter_ = max_iter\n \n def generate(self, xs, ys):\n batch_size = self._batch_size_\n x = xs[0]\n y = ys[0]\n #(n_samples, n_features) = x.shape ### yong xu commented\n (n_samples, n_labs) = y.shape\n n_each = batch_size // n_labs \n \n index_list = []\n for i1 in xrange(n_labs):\n index_list.append(np.where(y[:, i1] == 1)[0])\n \n for i1 in xrange(n_labs):\n np.random.shuffle(index_list[i1])\n \n pointer_list = [0] * n_labs\n iter = 0\n while True:\n if (self._type_) == 'test' and (iter == self._max_iter_):\n break\n iter += 1\n batch_x = []\n batch_y = []\n for i1 in xrange(n_labs):\n idx_num = len(index_list[i1])\n if pointer_list[i1] >= idx_num:\n pointer_list[i1] = 0\n np.random.shuffle(index_list[i1])\n \n batch_idx = index_list[i1][pointer_list[i1] : min(pointer_list[i1] + n_each, idx_num)]\n batch_x.append(x[batch_idx])\n batch_y.append(y[batch_idx])\n pointer_list[i1] += n_each\n batch_x = np.concatenate(batch_x, axis=0)\n batch_y = np.concatenate(batch_y, axis=0)\n yield batch_x, batch_y\n\n# resize data for fit into CNN. size: (batch_num*color_maps*height*weight)\ndef reshapeX1( X ):\n N = len(X)\n return X.reshape( (N, t_delay, feadim, 1, 1) )\n \n# resize data for fit into CNN. size: (batch_num*color_maps*height*weight)\ndef reshapeX2( X ):\n N = len(X)\n return X.reshape( (N, t_delay, feadim) )\n\ndef reshapeX3( X ):\n N = len(X)\n return X.reshape( (N, t_delay, feadim, 1) )\n\ndef reshapeX4( X ):\n N = len(X)\n return X.reshape( (N*t_delay, feadim) )\n\ndef reshapeX5( X , sample_num):\n N = len(X)\n return X.reshape( (sample_num, t_delay, feadim, 1) )\n\ndef outfunc(vects):\n x,y=vects\n #y=K.sum( y, axis=1 )\n y = K.clip( y, 1.0e-9, 1 ) # clip to avoid numerical underflow\n #z=Lambda(lambda x: K.sum(x, axis=1),output_shape=(8,))(y)\n y = K.sum(y, axis=1)\n #y = K.sum(y, axis=1)\n #z = RepeatVector(249)(z)\n #z=Permute((2,1))(z)\n #return K.sum( x / z, axis=1 )\n #x = K.sum( x, axis=(1,2) )\n x = K.sum( x, axis=1 )\n #x = K.sum( x, axis=1 ) \n return x / y\n\ndef slice1(x):\n return x[:,:,:, 0:64]\n\ndef slice2(x):\n return x[:,:,:, 64:128]\n\ndef slice1_output_shape(input_shape):\n return tuple([input_shape[0],input_shape[1],input_shape[2],64])\n\ndef slice2_output_shape(input_shape):\n return tuple([input_shape[0],input_shape[1],input_shape[2],64])\n\n#parameters:\nnum_classes=17\nfeadim=64\nt_delay=240 # the len of Utterance is 300\nmodel_out_path=\"/vol/vssp/msos/yx/t4_d2017/models_val\"\n\n# train and test sets:\nwith h5py.File(\"/vol/vssp/msos/Audioset/task4_dcase2017_features/packed_features/logmel/training_pack.h5\", 'r') as hf:\n tr_x = np.array(hf.get('x'))\n tr_y = np.array(hf.get('y'))\nprint tr_x.shape\nprint tr_y.shape\n\n\n\nwith h5py.File(\"/vol/vssp/msos/Audioset/task4_dcase2017_features/packed_features/logmel/testing_pack.h5\", 'r') as hf:\n va_x = np.array(hf.get('x'))\n va_y = np.array(hf.get('y'))\nprint va_x.shape\nprint va_y.shape\n\n###########normalization training and test set\ntr_x2=reshapeX4(tr_x)\nva_x2=reshapeX4(va_x)\nscaler = preprocessing.StandardScaler().fit(tr_x2)\nprint scaler.mean_, scaler.scale_\ntr_x2 = scaler.transform( tr_x2 )\nva_x2 = scaler.transform( va_x2 )\ntr_x=reshapeX5( tr_x2 , len(tr_x))\nva_x=reshapeX5( va_x2 , len(va_x))\n####################################\n#tr_x=reshapeX3(tr_x)\nprint tr_x.shape\n#va_x=reshapeX3(va_x)\nprint va_x.shape\n\ndef block(input):\n cnn1=Conv2D(64, (3, 3), padding=\"same\", activation=\"linear\", use_bias=False)(input)\n cnn1=BatchNormalization(axis=-1)(cnn1)\n\n #cnn11 = Lambda(slice1,output_shape=slice1_output_shape)(cnn1)\n #cnn12 = Lambda(slice2,output_shape=slice2_output_shape)(cnn1)\n\n #cnn11=Activation('linear')(cnn11)\n #cnn12=Activation('sigmoid')(cnn12)\n\n #cnn1=Multiply()([cnn11,cnn12])\n return cnn1\n\n###build model by keras\n\ninput_audio=Input(shape=(t_delay, feadim, 1))\n\n#input_flat=TimeDistributed(Flatten())(input_audio)\n\n###detection factor for each tag (7 meaningful tags + 1 silence tag = 8 tags)\n#det =TimeDistributed(Dense(17,activation='softmax'))(input_flat) # The posterior sum of each tag is 1.0, now the dims of det are 33 frs * 8 tags\n\ncnn1 = block(input_audio)\ncnn1 = block(cnn1)\ncnn1=MaxPooling2D(pool_size=(2, 2))(cnn1)\n\ncnn1 = block(cnn1)\ncnn1 = block(cnn1)\ncnn1=MaxPooling2D(pool_size=(2, 2))(cnn1)\n\ncnn1 = block(cnn1)\ncnn1 = block(cnn1)\ncnn1=MaxPooling2D(pool_size=(2, 2))(cnn1)\n\ncnn1 = block(cnn1)\ncnn3 = block(cnn1)\ncnn3=MaxPooling2D(pool_size=(1, 3))(cnn1)\n\ncnnout=Conv2D(256, (3, 3), padding=\"same\", activation=\"relu\", use_bias=True)(cnn3)\ncnnout=MaxPooling2D(pool_size=(1, 2))(cnnout)\n\n#cnnout=Flatten_last2d()(cnnout)\ncnnout=Reshape((30,256))(cnnout)\n\nout=Bidirectional(GRU(128, activation='relu', return_sequences=True))(cnnout)\n#rnnout_gate=Bidirectional(GRU(128, activation='sigmoid', return_sequences=True))(cnnout)\n#out=Multiply()([rnnout,rnnout_gate])\n\nout=TimeDistributed(Dense(17,activation='sigmoid'))(out)\n#det =TimeDistributed(Dense(17,activation='softmax'))(out)\n#out=Multiply()([out,det])\n#out=Lambda(outfunc,output_shape=(17,))([out,det])\nout=Lambda(lambda x: K.mean(x, axis=1),output_shape=(17,))(out)\n\nallmodel=Model(input_audio, out)\nallmodel.summary()\n\n# Let's train the model using RMSprop\nallmodel.compile(loss='binary_crossentropy',\n optimizer='adam',\n metrics=['accuracy'])\n\ndump_fd=model_out_path+'/icassp_baseline_crnn_rationBal44_lr0.001_norm_64newMel_240fr.{epoch:02d}-{val_acc:.4f}.hdf5'\n\neachmodel=ModelCheckpoint(dump_fd,monitor='val_acc',verbose=0,save_best_only=False,save_weights_only=False,mode='auto',period=10) \n\n#reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,\n# patience=5, min_lr=0.00000001)\n\n#reduce_lr = LearningRateScheduler(scheduler)\n\ngen = RatioDataGenerator(batch_size=44, type='train')\n#gen = BalanceDataGenerator(batch_size=52, type='train')\n#for (batch_x, batch_y) in gen.generate([x], [y]):\n# train_on_batch(batch_x, batch_y, class_weight=None, sample_weight=None)\n\nsteps_per_epoch=100\nallmodel.fit_generator(gen.generate([tr_x], [tr_y]), steps_per_epoch, epochs=2000000010, verbose=1, callbacks=[eachmodel], validation_data=(va_x, va_y))\n#allmodel.fit(tr_x, tr_y, batch_size=100, epochs=31,\n# verbose=1, validation_data=(va_x, va_y), callbacks=[eachmodel]) #, callbacks=[best_model]) \n" }, { "path": "main_crnn_at.py", "content": "\"\"\"\nSummary: Train a CNN-RNN audio tagging classifier on the task4 of DCASE2017 \n dataset. \nAuthor: Yong XU, Qiuqiang Kong\nCreated: 03/04/2017\nModified: 24/09/2017\n--------------------------------------\n\"\"\"\nfrom __future__ import print_function \nimport sys\nimport cPickle\nimport numpy as np\nimport argparse\nimport glob\nimport time\nimport os\n\nimport keras\nfrom keras import backend as K\nfrom keras.models import Sequential,Model, load_model\nfrom keras.layers.core import (Dense, Dropout, Activation, Flatten, Reshape, \n Permute,Lambda, RepeatVector)\nfrom keras.layers.convolutional import (ZeroPadding2D, AveragePooling2D, \n Conv2D,MaxPooling2D, Convolution1D,MaxPooling1D)\nfrom keras.layers.pooling import GlobalMaxPooling2D\nfrom keras.layers import Merge, Input, merge\nfrom keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, LearningRateScheduler\nfrom keras.layers import LSTM, SimpleRNN, GRU, TimeDistributed, Bidirectional\nfrom keras.layers.normalization import BatchNormalization\nimport h5py\nfrom keras.layers.merge import Multiply\nfrom sklearn import preprocessing\nimport random\n\nimport config as cfg\nfrom prepare_data import create_folder, load_hdf5_data, do_scale\nfrom data_generator import RatioDataGenerator\nfrom evaluation import io_task4, evaluate\n\ndef scheduler(epoch):\n initial_lrate = float(0.001)\n cur_lr=initial_lrate*pow(float(0.5),(epoch//5))\n print(\"learning rate: %f\" % cur_lr)\n return cur_lr\n\ndef block(input):\n cnn = Conv2D(128, (3, 3), padding=\"same\", activation=\"linear\", use_bias=False)(input)\n cnn = BatchNormalization(axis=-1)(cnn)\n\n cnn1 = Lambda(slice1, output_shape=slice1_output_shape)(cnn)\n cnn2 = Lambda(slice2, output_shape=slice2_output_shape)(cnn)\n\n cnn1 = Activation('linear')(cnn1)\n cnn2 = Activation('sigmoid')(cnn2)\n\n out = Multiply()([cnn1, cnn2])\n return out\n\ndef slice1(x):\n return x[:, :, :, 0:64]\n\ndef slice2(x):\n return x[:, :, :, 64:128]\n\ndef slice1_output_shape(input_shape):\n return tuple([input_shape[0],input_shape[1],input_shape[2],64])\n\ndef slice2_output_shape(input_shape):\n return tuple([input_shape[0],input_shape[1],input_shape[2],64])\n\ndef train(args):\n num_classes = cfg.num_classes\n \n # Load training & testing data\n (tr_x, tr_y, tr_na_list) = load_hdf5_data(args.tr_hdf5_path, verbose=1)\n (te_x, te_y, te_na_list) = load_hdf5_data(args.te_hdf5_path, verbose=1)\n print(\"\")\n\n # Scale data\n tr_x = do_scale(tr_x, args.scaler_path, verbose=1)\n te_x = do_scale(te_x, args.scaler_path, verbose=1)\n \n # Build model\n (_, n_time, n_freq) = tr_x.shape\n input_logmel = Input(shape=(n_time, n_freq), name='in_layer')\n a1 = Reshape((n_time, n_freq, 1))(input_logmel)\n \n cnn1 = block(a1)\n cnn1 = block(cnn1)\n cnn1 = MaxPooling2D(pool_size=(2, 2))(cnn1)\n \n cnn1 = block(cnn1)\n cnn1 = block(cnn1)\n cnn1 = MaxPooling2D(pool_size=(2, 2))(cnn1)\n \n cnn1 = block(cnn1)\n cnn1 = block(cnn1)\n cnn1 = MaxPooling2D(pool_size=(2, 2))(cnn1)\n \n cnn1 = block(cnn1)\n cnn3 = block(cnn1)\n cnn3 = MaxPooling2D(pool_size=(1, 2))(cnn1)\n \n cnnout = Conv2D(256, (3, 3), padding=\"same\", activation=\"relu\", use_bias=True)(cnn3)\n cnnout = MaxPooling2D(pool_size=(1, 4))(cnnout)\n \n cnnout = Reshape((30, 256))(cnnout) # Time step is downsampled to 30. \n \n rnnout = Bidirectional(GRU(128, activation='linear', return_sequences=True))(cnnout)\n rnnout_gate = Bidirectional(GRU(128, activation='sigmoid', return_sequences=True))(cnnout)\n out = Multiply()([rnnout, rnnout_gate])\n \n out = TimeDistributed(Dense(num_classes, activation='sigmoid'), name='localization_layer')(out)\n out = Lambda(lambda x: K.mean(x, axis=1),output_shape=(num_classes,))(out)\n \n model = Model(input_logmel, out)\n model.summary()\n \n # Compile model\n model.compile(loss='binary_crossentropy',\n optimizer='adam',\n metrics=['accuracy'])\n \n # Save model callback\n filepath = os.path.join(args.out_model_dir, \n \"gatedAct_rationBal44_lr0.001_normalization_at_cnnRNN_64mel_240fr.{epoch:02d}-{val_acc:.4f}.hdf5\")\n create_folder(os.path.dirname(filepath))\n save_model = ModelCheckpoint(filepath=filepath,\n monitor='val_acc', \n verbose=0,\n save_best_only=False,\n save_weights_only=False,\n mode='auto',\n period=1) \n\n # Data generator\n gen = RatioDataGenerator(batch_size=44, type='train')\n\n # Train\n model.fit_generator(generator=gen.generate([tr_x], [tr_y]), \n steps_per_epoch=100, # 100 iters is called an 'epoch'\n epochs=2000000010, # Maximum 'epoch' to train\n verbose=1, \n callbacks=[save_model], \n validation_data=(te_x, te_y))\n\ndef run_func(func, x, batch_size):\n pred_all = []\n for ptr in xrange(0, len(x), batch_size):\n batch_x = x[ptr : ptr + batch_size]\n [pred] = func([batch_x, 0.])\n pred_all.append(pred)\n pred_all = np.concatenate(pred_all, axis=0)\n return pred_all\n\ndef recognize(args, at_bool, sed_bool):\n (te_x, te_y, te_na_list) = load_hdf5_data(args.te_hdf5_path, verbose=1)\n x = te_x\n y = te_y\n na_list = te_na_list\n \n x = do_scale(x, args.scaler_path, verbose=1)\n \n fusion_at_list = []\n fusion_sed_list = []\n # for epoch in range(19, 81, 10):\n for epoch in range(20, 30, 1):\n t1 = time.time()\n [model_path] = glob.glob(os.path.join(args.model_dir, \n \"*.%02d-0.*.hdf5\" % epoch))\n model = load_model(model_path)\n \n # Audio tagging\n if at_bool:\n pred = model.predict(x)\n fusion_at_list.append(pred)\n \n # Sound event detection\n if sed_bool:\n in_layer = model.get_layer('in_layer')\n loc_layer = model.get_layer('localization_layer')\n func_loc_output = K.function([in_layer.input, K.learning_phase()], \n [loc_layer.output])\n pred3d = run_func(func_loc_output, x, batch_size=64)\n fusion_sed_list.append(pred3d)\n \n print(\"Prediction time: %s\" % (time.time() - t1,))\n \n # Write out AT probabilities\n if at_bool:\n fusion_at = np.mean(np.array(fusion_at_list), axis=0)\n print(\"AT shape: %s\" % (fusion_at.shape,))\n io_task4.at_write_prob_mat_to_csv(\n na_list=na_list, \n prob_mat=fusion_at, \n out_path=os.path.join(args.out_dir, \"at_prob_mat.csv.gz\"))\n \n # Write out SED probabilites\n if sed_bool:\n fusion_sed = np.mean(np.array(fusion_sed_list), axis=0)\n print(\"SED shape:%s\" % (fusion_sed.shape,))\n io_task4.sed_write_prob_mat_list_to_csv(\n na_list=na_list, \n prob_mat_list=fusion_sed, \n out_path=os.path.join(args.out_dir, \"sed_prob_mat_list.csv.gz\"))\n \n print(\"Prediction finished!\")\n\ndef get_stat(args, at_bool, sed_bool):\n lbs = cfg.lbs\n step_time_in_sec = cfg.step_time_in_sec\n max_len = cfg.max_len\n thres_ary = [0.3] * len(lbs)\n\n # Calculate AT stat\n if at_bool:\n pd_prob_mat_csv_path = os.path.join(args.pred_dir, \"at_prob_mat.csv.gz\")\n at_stat_path = os.path.join(args.stat_dir, \"at_stat.csv\")\n at_submission_path = os.path.join(args.submission_dir, \"at_submission.csv\")\n \n at_evaluator = evaluate.AudioTaggingEvaluate(\n weak_gt_csv=\"meta_data/groundtruth_weak_label_testing_set.csv\", \n lbs=lbs)\n \n at_stat = at_evaluator.get_stats_from_prob_mat_csv(\n pd_prob_mat_csv=pd_prob_mat_csv_path, \n thres_ary=thres_ary)\n \n # Write out & print AT stat\n at_evaluator.write_stat_to_csv(stat=at_stat, \n stat_path=at_stat_path)\n at_evaluator.print_stat(stat_path=at_stat_path)\n \n # Write AT to submission format\n io_task4.at_write_prob_mat_csv_to_submission_csv(\n at_prob_mat_path=pd_prob_mat_csv_path, \n lbs=lbs, \n thres_ary=at_stat['thres_ary'], \n out_path=at_submission_path)\n \n # Calculate SED stat\n if sed_bool:\n sed_prob_mat_list_path = os.path.join(args.pred_dir, \"sed_prob_mat_list.csv.gz\")\n sed_stat_path = os.path.join(args.stat_dir, \"sed_stat.csv\")\n sed_submission_path = os.path.join(args.submission_dir, \"sed_submission.csv\")\n \n sed_evaluator = evaluate.SoundEventDetectionEvaluate(\n strong_gt_csv=\"meta_data/groundtruth_strong_label_testing_set.csv\", \n lbs=lbs, \n step_sec=step_time_in_sec, \n max_len=max_len)\n \n # Write out & print SED stat\n sed_stat = sed_evaluator.get_stats_from_prob_mat_list_csv(\n pd_prob_mat_list_csv=sed_prob_mat_list_path, \n thres_ary=thres_ary)\n \n # Write SED to submission format\n sed_evaluator.write_stat_to_csv(stat=sed_stat, \n stat_path=sed_stat_path) \n sed_evaluator.print_stat(stat_path=sed_stat_path)\n \n # Write SED to submission format\n io_task4.sed_write_prob_mat_list_csv_to_submission_csv(\n sed_prob_mat_list_path=sed_prob_mat_list_path, \n lbs=lbs, \n thres_ary=thres_ary, \n step_sec=step_time_in_sec, \n out_path=sed_submission_path)\n \n print(\"Calculating stat finished!\")\n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser(description=\"\")\n subparsers = parser.add_subparsers(dest='mode')\n \n parser_train = subparsers.add_parser('train')\n parser_train.add_argument('--tr_hdf5_path', type=str)\n parser_train.add_argument('--te_hdf5_path', type=str)\n parser_train.add_argument('--scaler_path', type=str)\n parser_train.add_argument('--out_model_dir', type=str)\n \n parser_recognize = subparsers.add_parser('recognize')\n parser_recognize.add_argument('--te_hdf5_path', type=str)\n parser_recognize.add_argument('--scaler_path', type=str)\n parser_recognize.add_argument('--model_dir', type=str)\n parser_recognize.add_argument('--out_dir', type=str)\n \n parser_get_stat = subparsers.add_parser('get_stat')\n parser_get_stat.add_argument('--pred_dir', type=str)\n parser_get_stat.add_argument('--stat_dir', type=str)\n parser_get_stat.add_argument('--submission_dir', type=str)\n \n args = parser.parse_args()\n \n if args.mode == 'train':\n train(args)\n elif args.mode == 'recognize':\n recognize(args, at_bool=True, sed_bool=False)\n elif args.mode == 'get_stat':\n get_stat(args, at_bool=True, sed_bool=False)\n else:\n raise Exception(\"Incorrect argument!\")" }, { "path": "main_crnn_sed.py", "content": "\"\"\"Train a cldnn on the task4 of DCASE2017 dataset.\nGPU run command with Theano backend (with TensorFlow, the GPU is automatically used):\n THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatx=float32 python train*.py\n\nAuthor: Yong XU\nCreat date: 03/04/2017\n\"\"\"\nfrom __future__ import print_function \nimport sys\nimport cPickle\nimport numpy as np\nimport argparse\nimport glob\nimport time\nimport os\n\nimport keras\nfrom keras import backend as K\nfrom keras.models import Sequential,Model, load_model\nfrom keras.layers.core import Dense, Dropout, Activation, Flatten, Reshape, Permute,Lambda, RepeatVector\nfrom keras.layers.convolutional import ZeroPadding2D, AveragePooling2D, Conv2D,MaxPooling2D, Convolution1D,MaxPooling1D\nfrom keras.layers.pooling import GlobalMaxPooling2D\nfrom keras.layers import Merge, Input, merge\nfrom keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, LearningRateScheduler\nfrom keras.layers import LSTM, SimpleRNN, GRU, TimeDistributed, Bidirectional\nfrom keras.layers.normalization import BatchNormalization\nimport h5py\nfrom keras.layers.merge import Multiply\nfrom sklearn import preprocessing\nimport random\n\nimport config as cfg\nfrom prepare_data import create_folder, load_hdf5_data, do_scale\nfrom data_generator import RatioDataGenerator\nfrom evaluation import io_task4, evaluate\nfrom main_crnn_at import block, slice1, slice2, slice1_output_shape, slice2_output_shape, run_func, recognize, get_stat\n\ndef outfunc(vects):\n x,y=vects\n y = K.clip(y, 1e-7, 1.) # clip to avoid numerical underflow\n y = K.sum(y, axis=1)\n x = K.sum(x, axis=1)\n return x / y\n\ndef train(args):\n num_classes = cfg.num_classes\n \n # Load training & testing data\n (tr_x, tr_y, tr_na_list) = load_hdf5_data(args.tr_hdf5_path, verbose=1)\n # (tr_x, tr_y, tr_na_list) = load_hdf5(args.te_hdf5_path, verbose=1)\n (te_x, te_y, te_na_list) = load_hdf5_data(args.te_hdf5_path, verbose=1)\n print(\"\")\n\n # Scale data\n tr_x = do_scale(tr_x, args.scaler_path, verbose=1)\n te_x = do_scale(te_x, args.scaler_path, verbose=1)\n \n # Build model\n (_, n_time, n_freq) = tr_x.shape\n input_logmel = Input(shape=(n_time, n_freq), name='in_layer')\n a1 = Reshape((n_time, n_freq, 1))(input_logmel)\n \n cnn1 = block(a1)\n cnn1 = block(cnn1)\n cnn1 = MaxPooling2D(pool_size=(1, 2))(cnn1)\n \n cnn1 = block(cnn1)\n cnn1 = block(cnn1)\n cnn1 = MaxPooling2D(pool_size=(1, 2))(cnn1)\n \n cnn1 = block(cnn1)\n cnn1 = block(cnn1)\n cnn1 = MaxPooling2D(pool_size=(1, 2))(cnn1)\n \n cnn1 = block(cnn1)\n cnn3 = block(cnn1)\n cnn3 = MaxPooling2D(pool_size=(1, 2))(cnn1)\n \n cnnout = Conv2D(256, (3, 3), padding=\"same\", activation=\"relu\", use_bias=True)(cnn3)\n cnnout = MaxPooling2D(pool_size=(1, 4))(cnnout)\n \n cnnout = Reshape((240, 256))(cnnout) # Time step is downsampled to 30. \n \n rnnout = Bidirectional(GRU(128, activation='linear', return_sequences=True))(cnnout)\n rnnout_gate = Bidirectional(GRU(128, activation='sigmoid', return_sequences=True))(cnnout)\n out = Multiply()([rnnout, rnnout_gate])\n \n out = TimeDistributed(Dense(num_classes, activation='sigmoid'), name='localization_layer')(out)\n det =TimeDistributed(Dense(num_classes, activation='softmax'))(out)\n out=Multiply()([out,det])\n out=Lambda(outfunc, output_shape=(num_classes,))([out, det])\n \n model = Model(input_logmel, out)\n model.summary()\n \n # Compile model\n model.compile(loss='binary_crossentropy',\n optimizer='adam',\n metrics=['accuracy'])\n \n # Save model callback\n filepath = os.path.join(args.out_model_dir, \"gatedAct_rationBal44_lr0.001_normalization_at_cnnRNN_64newMel_240fr.{epoch:02d}-{val_acc:.4f}.hdf5\")\n create_folder(os.path.dirname(filepath))\n save_model = ModelCheckpoint(filepath=filepath,\n monitor='val_acc', \n verbose=0,\n save_best_only=False,\n save_weights_only=False,\n mode='auto',\n period=1) \n\n # Data generator\n gen = RatioDataGenerator(batch_size=44, type='train')\n\n # Train\n model.fit_generator(generator=gen.generate([tr_x], [tr_y]), \n steps_per_epoch=100, # 100 iters is called an 'epoch'\n epochs=2000000010, # Maximum 'epoch' to train\n verbose=1, \n callbacks=[save_model], \n validation_data=(te_x, te_y))\n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser(description=\"\")\n subparsers = parser.add_subparsers(dest='mode')\n \n parser_train = subparsers.add_parser('train')\n parser_train.add_argument('--tr_hdf5_path', type=str)\n parser_train.add_argument('--te_hdf5_path', type=str)\n parser_train.add_argument('--scaler_path', type=str)\n parser_train.add_argument('--out_model_dir', type=str)\n \n parser_recognize = subparsers.add_parser('recognize')\n parser_recognize.add_argument('--te_hdf5_path', type=str)\n parser_recognize.add_argument('--scaler_path', type=str)\n parser_recognize.add_argument('--model_dir', type=str)\n parser_recognize.add_argument('--out_dir', type=str)\n \n parser_get_stat = subparsers.add_parser('get_stat')\n parser_get_stat.add_argument('--pred_dir', type=str)\n parser_get_stat.add_argument('--stat_dir', type=str)\n parser_get_stat.add_argument('--submission_dir', type=str)\n \n args = parser.parse_args()\n \n if args.mode == 'train':\n train(args)\n elif args.mode == 'recognize':\n recognize(args, at_bool=True, sed_bool=True)\n elif args.mode == 'get_stat':\n get_stat(args, at_bool=True, sed_bool=True)\n else:\n raise Exception(\"Incorrect argument!\")" }, { "path": "predict_audio_tagging_icassp2018_cnnrnn_baseline.py", "content": "'''Train a cldnn on the task4 of DCASE2017 dataset.\nGPU run command with Theano backend (with TensorFlow, the GPU is automatically used):\n THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatx=float32 python train*.py\n\nAuthor: Yong XU\nCreat date: 03/04/2017\n'''\n\n#from __future__ import print_function\n\nimport keras\nfrom keras import backend as K\nfrom keras.models import load_model\nimport sys\nimport cPickle as pickle\nimport numpy as np\nfrom keras.models import Sequential,Model\nfrom keras.layers.core import Dense, Dropout, Activation, Flatten, Reshape,Permute,Lambda, RepeatVector\nfrom keras.layers.convolutional import ZeroPadding2D, AveragePooling2D, Conv2D,MaxPooling2D, Convolution1D,MaxPooling1D\nfrom keras.layers import Merge, Input, merge\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.layers import LSTM, SimpleRNN, GRU, TimeDistributed, Bidirectional\nimport h5py\nimport os\nimport shutil\nfrom sklearn import preprocessing\nimport gzip\nimport glob\n\n# resize data for fit into CNN. size: (batch_num*color_maps*height*weight)\ndef reshapeX1( X ):\n N = len(X)\n return X.reshape( (1, N, feadim, 1, 1) )\n \n# resize data for fit into CNN. size: (batch_num*color_maps*height*weight)\ndef reshapeX2( X ):\n N = len(X)\n return X.reshape( (N, t_delay, feadim) )\n\ndef reshapeX3( X ):\n N = len(X)\n return X.reshape( (1, N, feadim) )\n\ndef reshapeX6( X ):\n N = len(X)\n return X.reshape( (1, N, feadim, 1) )\n\ndef reshapeX4( X ):\n N = len(X)\n return X.reshape( (N*t_delay, feadim) )\n\ndef reshapeX5( X , sample_num):\n N = len(X)\n return X.reshape( (sample_num, t_delay, feadim, 1) )\n\n#parameters:\nnum_classes=17\nfeadim=64\nt_delay=240 # the len of Utterance is 300\n\n## train sets:\n#with h5py.File(\"/vol/vssp/msos/Audioset/task4_dcase2017_features/packed_features/logmel/training_pack.h5\", 'r') as hf:\n# tr_x = np.array(hf.get('x'))\n# tr_y = np.array(hf.get('y'))\n#print tr_x.shape\n#print tr_y.shape\n\n#test sets:\nwith h5py.File(\"/vol/vssp/msos/Audioset/task4_dcase2017_features/packed_features/logmel/testing_pack.h5\", 'r') as hf:\n va_x = np.array(hf.get('x'))\n va_y = np.array(hf.get('y'))\n va_id = np.array(hf.get('na_list'))\n\nprint va_id.shape\nprint va_x.shape\nprint va_y.shape\n\n###########normalization training and test set\n#tr_x2=reshapeX4(tr_x)\nva_x2=reshapeX4(va_x)\n#scaler = preprocessing.StandardScaler().fit(tr_x2)\n#print scaler.mean_, scaler.scale_\n#with open('tr_norm.pickle', 'wb') as handle:\n# pickle.dump(scaler, handle, protocol=pickle.HIGHEST_PROTOCOL)\n\n\nwith open('tr_norm.pickle', 'rb') as handle:\n scaler = pickle.load(handle)\nva_x2 = scaler.transform( va_x2 )\nva_x=reshapeX5( va_x2 , len(va_x))\n####################################\nf=gzip.open('/vol/vssp/msos/yx/t4_d2017/dcase2017_task4_evaluation_code_20170712/data/at_prob_mat_icassp2018_crnn_baseline.csv.gz','w')\n#f2=gzip.open('/vol/vssp/msos/yx/t4_d2017/dcase2017_task4_evaluation_code_20170712/data/sed_prob_mat_list.csv.gz','w')\npred_fusion=[[0 for x in range(17)] for y in range(488)] \nprint len(pred_fusion)\nmodel_num=0\n#for epoch in range(20,110,10):\nfor epoch in range(19,81,10):\n#19-61: f1=56,59\n#19-71: f1=56,58.5\n#19-81: f1=0.567,0.596\n#19-91: f1=0.565,0.584\n print \"epoch:\", epoch\n model_num=model_num+1\n print \"model num:\", model_num\n \n path='/vol/vssp/msos/yx/t4_d2017/models_val/icassp_baseline_crnn_rationBal44_lr0.001_norm_64newMel_240fr.%d-0.*.hdf5'%epoch\n for model_f in glob.glob(path):\n print model_f\n md=load_model(model_f)\n #md.summary()\n \n# def recognize():\n \n p_y_pred = md.predict( va_x )\n pred_fusion=pred_fusion+p_y_pred\n \n #shutil.copy('at_prediction.csv', '/vol/vssp/msos/yx/t4_d2017/dcase2017_task4_evaluation_code_20170616/examples/.')\n #os.wait()\n #shutil.copy('prob_mat.csv', '/vol/vssp/msos/yx/t4_d2017/dcase2017_task4_evaluation_code_20170616/examples/.')\n #os.wait()\n \n #path=\"/vol/vssp/msos/yx/t4_d2017/dcase2017_task4_evaluation_code_20170616\"\n #os.chdir(path) \n #cmd=\"python runme.py\"\n #os.system(cmd)\n\n# do fusion\np_y_pred= pred_fusion/model_num\nprint p_y_pred.shape\np_y_pred=list(p_y_pred)\n#print p_y_pred\nfor i in range(len(va_x)):\n #print i\n p_y_pred_s = p_y_pred[i]\n f.write(\"%s.wav\" % (va_id[i]))\n for e in p_y_pred_s:\n# f.write(\" %s\" % e)\n f.write(\"\t%s\" % e)\n f.write(\"\\n\")\n #sys.exit()\n # copy the audio tagging results to form the sed results, simple mode\n #for j in range(240):\n # #print i\n # f2.write(\"%s.wav\" % (va_id[i]))\n # for e in p_y_pred_s:\n # f2.write(\"\t%s\" % e)\n # f2.write(\"\\n\") \n\nf.close\n#f2.close\n\n# if __name__ == '__main__':\n# recognize()\n" }, { "path": "predict_crnn_at.py", "content": "'''Train a cldnn on the task4 of DCASE2017 dataset.\nGPU run command with Theano backend (with TensorFlow, the GPU is automatically used):\n THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatx=float32 python train*.py\n\nAuthor: Yong XU\nCreat date: 03/04/2017 \n'''\n\n#from __future__ import print_function\n\nimport keras\nfrom keras import backend as K\nfrom keras.models import load_model\nimport sys\nimport cPickle as pickle\nimport numpy as np\nfrom keras.models import Sequential,Model\nfrom keras.layers.core import Dense, Dropout, Activation, Flatten, Reshape,Permute,Lambda, RepeatVector\nfrom keras.layers.convolutional import ZeroPadding2D, AveragePooling2D, Conv2D,MaxPooling2D, Convolution1D,MaxPooling1D\nfrom keras.layers import Merge, Input, merge\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.layers import LSTM, SimpleRNN, GRU, TimeDistributed, Bidirectional\nimport h5py\nimport os\nimport shutil\nfrom sklearn import preprocessing\nimport gzip\nimport glob\n\n# resize data for fit into CNN. size: (batch_num*color_maps*height*weight)\ndef reshapeX1( X ):\n N = len(X)\n return X.reshape( (1, N, feadim, 1, 1) )\n \n# resize data for fit into CNN. size: (batch_num*color_maps*height*weight)\ndef reshapeX2( X ):\n N = len(X)\n return X.reshape( (N, t_delay, feadim) )\n\ndef reshapeX3( X ):\n N = len(X)\n return X.reshape( (1, N, feadim) )\n\ndef reshapeX6( X ):\n N = len(X)\n return X.reshape( (1, N, feadim, 1) )\n\ndef reshapeX4( X ):\n N = len(X)\n return X.reshape( (N*t_delay, feadim) )\n\ndef reshapeX5( X , sample_num):\n N = len(X)\n return X.reshape( (sample_num, t_delay, feadim, 1) )\n\n#parameters:\nnum_classes=17\nfeadim=64\nt_delay=240 # the len of Utterance is 300\n\n## train sets:\n#with h5py.File(\"/vol/vssp/msos/Audioset/task4_dcase2017_features/packed_features/logmel/training_pack.h5\", 'r') as hf:\n# tr_x = np.array(hf.get('x'))\n# tr_y = np.array(hf.get('y'))\n#print tr_x.shape\n#print tr_y.shape\n\n#test sets:\nwith h5py.File(\"/vol/vssp/msos/Audioset/task4_dcase2017_features/packed_features/logmel/testing_pack.h5\", 'r') as hf:\n va_x = np.array(hf.get('x'))\n va_y = np.array(hf.get('y'))\n va_id = np.array(hf.get('na_list'))\n\nprint va_id.shape\nprint va_x.shape\nprint va_y.shape\n\n###########normalization training and test set\n#tr_x2=reshapeX4(tr_x)\nva_x2=reshapeX4(va_x)\n#scaler = preprocessing.StandardScaler().fit(tr_x2)\n#print scaler.mean_, scaler.scale_\n#with open('tr_norm.pickle', 'wb') as handle:\n# pickle.dump(scaler, handle, protocol=pickle.HIGHEST_PROTOCOL)\n\n\nwith open('tr_norm.pickle', 'rb') as handle:\n scaler = pickle.load(handle)\nva_x2 = scaler.transform( va_x2 )\nva_x=reshapeX5( va_x2 , len(va_x))\n####################################\nf=gzip.open('/vol/vssp/msos/yx/t4_d2017/dcase2017_task4_evaluation_code_20170712/data/at_prob_mat_icassp2018_crnn_baseline.csv.gz','w')\n#f2=gzip.open('/vol/vssp/msos/yx/t4_d2017/dcase2017_task4_evaluation_code_20170712/data/sed_prob_mat_list.csv.gz','w')\npred_fusion=[[0 for x in range(17)] for y in range(488)] \nprint len(pred_fusion)\nmodel_num=0\n#for epoch in range(20,110,10):\nfor epoch in range(19,81,10):\n#19-61: f1=56,59\n#19-71: f1=56,58.5\n#19-81: f1=0.567,0.596\n#19-91: f1=0.565,0.584\n print \"epoch:\", epoch\n model_num=model_num+1\n print \"model num:\", model_num\n \n path='/vol/vssp/msos/yx/t4_d2017/models_val/icassp_baseline_crnn_rationBal44_lr0.001_norm_64newMel_240fr.%d-0.*.hdf5'%epoch\n for model_f in glob.glob(path):\n print model_f\n md=load_model(model_f)\n #md.summary()\n \n# def recognize():\n \n p_y_pred = md.predict( va_x )\n pred_fusion=pred_fusion+p_y_pred\n \n #shutil.copy('at_prediction.csv', '/vol/vssp/msos/yx/t4_d2017/dcase2017_task4_evaluation_code_20170616/examples/.')\n #os.wait()\n #shutil.copy('prob_mat.csv', '/vol/vssp/msos/yx/t4_d2017/dcase2017_task4_evaluation_code_20170616/examples/.')\n #os.wait()\n \n #path=\"/vol/vssp/msos/yx/t4_d2017/dcase2017_task4_evaluation_code_20170616\"\n #os.chdir(path) \n #cmd=\"python runme.py\"\n #os.system(cmd)\n\n# do fusion\np_y_pred= pred_fusion/model_num\nprint p_y_pred.shape\np_y_pred=list(p_y_pred)\n#print p_y_pred\nfor i in range(len(va_x)):\n #print i\n p_y_pred_s = p_y_pred[i]\n f.write(\"%s.wav\" % (va_id[i]))\n for e in p_y_pred_s:\n# f.write(\" %s\" % e)\n f.write(\"\t%s\" % e)\n f.write(\"\\n\")\n #sys.exit()\n # copy the audio tagging results to form the sed results, simple mode\n #for j in range(240):\n # #print i\n # f2.write(\"%s.wav\" % (va_id[i]))\n # for e in p_y_pred_s:\n # f2.write(\"\t%s\" % e)\n # f2.write(\"\\n\") \n\nf.close\n#f2.close\n\n# if __name__ == '__main__':\n# recognize()\n" }, { "path": "predict_crnn_sed.py", "content": "'''Train a cldnn on the task4 of DCASE2017 dataset.\nGPU run command with Theano backend (with TensorFlow, the GPU is automatically used):\n THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatx=float32 python train*.py\n\nAuthor: Yong XU\nCreat date: 03/04/2017\n'''\n\n#from __future__ import print_function \n\nimport keras\nfrom keras import backend as K\nfrom keras.models import load_model\nimport sys\nimport cPickle as pickle\nimport numpy as np\nfrom keras.models import Sequential,Model\nfrom keras.layers.core import Dense, Dropout, Activation, Flatten, Reshape,Permute,Lambda, RepeatVector\nfrom keras.layers.convolutional import ZeroPadding2D, AveragePooling2D, Conv2D,MaxPooling2D, Convolution1D,MaxPooling1D\nfrom keras.layers import Merge, Input, merge\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.layers import LSTM, SimpleRNN, GRU, TimeDistributed, Bidirectional\nimport h5py\nimport os\nimport shutil\nfrom sklearn import preprocessing\nimport gzip\nimport glob\nimport matplotlib.pyplot as plt\n\n# resize data for fit into CNN. size: (batch_num*color_maps*height*weight)\ndef reshapeX1( X ):\n N = len(X)\n return X.reshape( (1, N, feadim, 1, 1) )\n \n# resize data for fit into CNN. size: (batch_num*color_maps*height*weight)\ndef reshapeX2( X ):\n N = len(X)\n return X.reshape( (N, t_delay, feadim) )\n\ndef reshapeX3( X ):\n N = len(X)\n return X.reshape( (1, N, feadim) )\n\ndef reshapeX6( X ):\n N = len(X)\n return X.reshape( (1, N, feadim, 1) )\n\ndef reshapeX4( X ):\n N = len(X)\n return X.reshape( (N*t_delay, feadim) )\n\ndef reshapeX5( X , sample_num):\n N = len(X)\n return X.reshape( (sample_num, t_delay, feadim, 1) )\n\ndef reshapeX7( X ):\n N = len(X)\n return X.reshape( (batch_size, t_delay, feadim, 1) )\n\n#parameters:\nnum_classes=17\nfeadim=64\nt_delay=240 # the len of Utterance is 300\nbatch_size=61\n\n## train sets:\n#with h5py.File(\"/vol/vssp/msos/Audioset/task4_dcase2017_features/packed_features/logmel/training_pack.h5\", 'r') as hf:\n# tr_x = np.array(hf.get('x'))\n# tr_y = np.array(hf.get('y'))\n#print tr_x.shape\n#print tr_y.shape\n\n#test sets:\nwith h5py.File(\"/vol/vssp/msos/Audioset/task4_dcase2017_features/packed_features/logmel/testing_pack.h5\", 'r') as hf:\n va_x = np.array(hf.get('x'))\n va_y = np.array(hf.get('y'))\n va_id = np.array(hf.get('na_list'))\n\nprint va_id.shape\nprint va_x.shape\nprint va_y.shape\nprint va_y\n\n###########normalization training and test set\n#tr_x2=reshapeX4(tr_x)\nva_x2=reshapeX4(va_x)\n#scaler = preprocessing.StandardScaler().fit(tr_x2)\n#print scaler.mean_, scaler.scale_\n#with open('tr_norm.pickle', 'wb') as handle:\n# pickle.dump(scaler, handle, protocol=pickle.HIGHEST_PROTOCOL)\n\n\nwith open('tr_norm.pickle', 'rb') as handle:\n scaler = pickle.load(handle)\nva_x2 = scaler.transform( va_x2 )\nva_x=reshapeX5( va_x2 , len(va_x))\n####################################\n#f=gzip.open('/vol/vssp/msos/yx/t4_d2017/dcase2017_task4_evaluation_code_20170712/data/at_prob_mat.csv.gz','w')\nf2=gzip.open('/vol/vssp/msos/yx/t4_d2017/dcase2017_task4_evaluation_code_20170712/data/sed_prob_mat_list.csv.gz','w')\npred_fusion=[[0 for x in range(17)] for y in range(488)] \nprint len(pred_fusion)\nmodel_num=0\n#for epoch in range(20,110,10):\n\ntotal_out=[[[0 for x in range(17)] for y in range(t_delay)] for z in range(len(va_x))] #[488,240,17]\nfor epoch in range(19,81,10):\n#19-61: f1=56,59\n#19-71: f1=56,58.5\n#19-81: f1=0.567,0.596\n#19-91: f1=0.565,0.584\n print \"epoch:\", epoch\n model_num=model_num+1\n print \"model num:\", model_num\n \n #path='/vol/vssp/msos/yx/t4_d2017/models_val/gatedAct_rationBal44_lr0.001_normalization_sigLOC240_cnnRNN_64newMel_240fr.%d-0.*.hdf5'%epoch\n path='/vol/vssp/msos/yx/t4_d2017/models_val/gatedAct_rationBal44_lr0.001_normalization_LOC240_cnnRNN_64newMel_240fr.%d-0.*.hdf5'%epoch # should be softmax ?\n #path='/vol/vssp/msos/yx/t4_d2017/models_val/gatedAct_rationBal44_lr0.001_normalization_LOC_cnnRNN_mfcc40dd_240fr.%d-0.*.hdf5'%epoch\n for model_f in glob.glob(path):\n print model_f\n md=load_model(model_f)\n #md.summary()\n \n# def recognize():\n \n #p_y_pred = md.predict( va_x )\n #pred_fusion=pred_fusion+p_y_pred\n\n batch_out=[]#for append, should be [batch_size, 240, 17]\n for utt in range(0,len(va_x),batch_size): \n print utt\n if 1: ### for localization\n get_3rd_layer_output = K.function([md.layers[0].input, K.learning_phase()], [md.layers[60].output]) ###60 should be the sigmoid layer, with quite large value 1.0, but with several 1.0 values \n ####61 should be the softmax layer\n ###62 seems to be much worse.\n layer_output = get_3rd_layer_output([reshapeX7(va_x[utt:utt+batch_size]), 0])[0]\n print layer_output\n print layer_output.shape\n #layer_output1=layer_output[:,:]\n #for utt in range(len(va_x)):\n print va_id[utt]\n #for fr in range(240):\n #f2.write(\"%s.wav\" % (va_id[utt]))###output id, wav name\n batch_out.extend(layer_output) #### append will be additional index in the first position;;; extend will concatenate in the first index without additional index \n #for jz in range(batch_size):\n \n # for fr in range(240):\n #f2.write(\"%s.wav\" % (va_id[utt]))\n # for e in layer_output[jz,fr,:]:\n # f2.write(\"\t%s\" % e)\n #f2.write(\"\\n\")\n # utt=utt+1\n #print utt\n #f2.write(\"\\n\")\n #f2.close \n batch_out=np.array(batch_out)\n print \"batch_out.shape:\", batch_out.shape\n total_out=total_out+batch_out\ntotal_out=total_out/model_num\n\n####do fusion print:\nfor utt in range(488):\n for fr in range(240):\n f2.write(\"%s.wav\" % (va_id[utt]))\n for tg in total_out[utt,fr,:]:\n f2.write(\"\t%s\" % tg)\n f2.write(\"\\n\")\nf2.close\n" }, { "path": "prepare_data.py", "content": "from __future__ import print_function\nimport numpy as np\nimport sys\nimport soundfile\nimport os\nimport librosa\nfrom scipy import signal\nimport pickle\nimport cPickle\nimport scipy\nimport time\nimport csv\nimport gzip\nimport h5py\nimport matplotlib.ticker as ticker\nimport matplotlib.pyplot as plt\nfrom sklearn import preprocessing\nfrom sklearn import metrics\nimport argparse\n\nimport config as cfg\n\n# Read wav\ndef read_audio(path, target_fs=None):\n (audio, fs) = soundfile.read(path)\n if audio.ndim > 1:\n audio = np.mean(audio, axis=1)\n if target_fs is not None and fs != target_fs:\n audio = librosa.resample(audio, orig_sr=fs, target_sr=target_fs)\n fs = target_fs\n return audio, fs\n \n# Write wav\ndef write_audio(path, audio, sample_rate):\n soundfile.write(file=path, data=audio, samplerate=sample_rate)\n\n# Create an empty folder\ndef create_folder(fd):\n if not os.path.exists(fd):\n os.makedirs(fd)\n\n### Feature extraction. \ndef extract_features(wav_dir, out_dir, recompute):\n \"\"\"Extract log mel spectrogram features. \n \n Args:\n wav_dir: string, directory of wavs. \n out_dir: string, directory to write out features. \n recompute: bool, if True recompute all features, if False skip existed\n extracted features. \n \n Returns:\n None\n \"\"\"\n fs = cfg.sample_rate\n n_window = cfg.n_window\n n_overlap = cfg.n_overlap\n \n create_folder(out_dir)\n names = [na for na in os.listdir(wav_dir) if na.endswith(\".wav\")]\n names = sorted(names)\n print(\"Total file number: %d\" % len(names))\n\n # Mel filter bank\n melW = librosa.filters.mel(sr=fs, \n n_fft=n_window, \n n_mels=64, \n fmin=0., \n fmax=8000.)\n \n cnt = 0\n t1 = time.time()\n for na in names:\n wav_path = wav_dir + '/' + na\n out_path = out_dir + '/' + os.path.splitext(na)[0] + '.p'\n \n # Skip features already computed\n if recompute or (not os.path.isfile(out_path)):\n print(cnt, out_path)\n (audio, _) = read_audio(wav_path, fs)\n \n # Skip corrupted wavs\n if audio.shape[0] == 0:\n print(\"File %s is corrupted!\" % wav_path)\n else:\n # Compute spectrogram\n ham_win = np.hamming(n_window)\n [f, t, x] = signal.spectral.spectrogram(\n x=audio, \n window=ham_win,\n nperseg=n_window, \n noverlap=n_overlap, \n detrend=False, \n return_onesided=True, \n mode='magnitude') \n x = x.T\n x = np.dot(x, melW.T)\n x = np.log(x + 1e-8)\n x = x.astype(np.float32)\n \n # Dump to pickle\n cPickle.dump(x, open(out_path, 'wb'), \n protocol=cPickle.HIGHEST_PROTOCOL)\n cnt += 1\n print(\"Extracting feature time: %s\" % (time.time() - t1,))\n\n### Pack features of hdf5 file\ndef pack_features_to_hdf5(fe_dir, csv_path, out_path):\n \"\"\"Pack extracted features to a single hdf5 file. \n \n This hdf5 file can speed up loading the features. This hdf5 file has \n structure:\n na_list: list of names\n x: bool array, (n_clips)\n y: float32 array, (n_clips, n_time, n_freq)\n \n Args: \n fe_dir: string, directory of features. \n csv_path: string | \"\", path of csv file. E.g. \"testing_set.csv\". If the \n string is empty, then pack features with all labels False. \n out_path: string, path to write out the created hdf5 file. \n \n Returns:\n None\n \"\"\"\n max_len = cfg.max_len\n create_folder(os.path.dirname(out_path))\n \n t1 = time.time()\n x_all, y_all, na_all = [], [], []\n \n if csv_path != \"\": # Pack from csv file (training & testing from dev. data)\n with open(csv_path, 'rb') as f:\n reader = csv.reader(f)\n lis = list(reader)\n cnt = 0\n for li in lis:\n [na, bgn, fin, lbs, ids] = li\n if cnt % 100 == 0: print(cnt)\n na = os.path.splitext(na)[0]\n bare_na = 'Y' + na + '_' + bgn + '_' + fin # Correspond to the wav name. \n fe_na = bare_na + \".p\"\n fe_path = os.path.join(fe_dir, fe_na)\n \n if not os.path.isfile(fe_path):\n print(\"File %s is in the csv file but the feature is not extracted!\" % fe_path)\n else:\n na_all.append(bare_na[1:] + \".wav\") # Remove 'Y' in the begining. \n x = cPickle.load(open(fe_path, 'rb'))\n x = pad_trunc_seq(x, max_len)\n x_all.append(x)\n ids = ids.split(',')\n y = ids_to_multinomial(ids)\n y_all.append(y)\n cnt += 1\n else: # Pack from features without ground truth label (dev. data)\n names = os.listdir(fe_dir)\n names = sorted(names)\n for fe_na in names:\n bare_na = os.path.splitext(fe_na)[0]\n fe_path = os.path.join(fe_dir, fe_na)\n na_all.append(bare_na + \".wav\")\n x = cPickle.load(open(fe_path, 'rb'))\n x = pad_trunc_seq(x, max_len)\n x_all.append(x)\n y_all.append(None)\n \n x_all = np.array(x_all, dtype=np.float32)\n y_all = np.array(y_all, dtype=np.bool)\n print(\"len(na_all): %d\", len(na_all))\n print(\"x_all.shape: %s, %s\" % (x_all.shape, x_all.dtype))\n print(\"y_all.shape: %s, %s\" % (y_all.shape, y_all.dtype))\n \n with h5py.File(out_path, 'w') as hf:\n hf.create_dataset('na_list', data=na_all)\n hf.create_dataset('x', data=x_all)\n hf.create_dataset('y', data=y_all)\n \n print(\"Save hdf5 to %s\" % out_path)\n print(\"Pack features time: %s\" % (time.time() - t1,))\n \ndef ids_to_multinomial(ids):\n \"\"\"Ids of wav to multinomial representation. \n \n Args:\n ids: list of id, e.g. ['/m/0284vy3', '/m/02mfyn']\n \n Returns:\n 1d array, multimonial representation, e.g. [1,0,1,0,0,...]\n \"\"\"\n y = np.zeros(len(cfg.lbs))\n for id in ids:\n index = cfg.id_to_idx[id]\n y[index] = 1\n return y\n \ndef pad_trunc_seq(x, max_len):\n \"\"\"Pad or truncate a sequence data to a fixed length. \n \n Args:\n x: ndarray, input sequence data. \n max_len: integer, length of sequence to be padded or truncated. \n \n Returns:\n ndarray, Padded or truncated input sequence data. \n \"\"\"\n L = len(x)\n shape = x.shape\n if L < max_len:\n pad_shape = (max_len - L,) + shape[1:]\n pad = np.zeros(pad_shape)\n x_new = np.concatenate((x, pad), axis=0)\n else:\n x_new = x[0:max_len]\n return x_new\n \n### Load data & scale data\ndef load_hdf5_data(hdf5_path, verbose=1):\n \"\"\"Load hdf5 data. \n \n Args:\n hdf5_path: string, path of hdf5 file. \n verbose: integar, print flag. \n \n Returns:\n x: ndarray (np.float32), shape: (n_clips, n_time, n_freq)\n y: ndarray (np.bool), shape: (n_clips, n_classes)\n na_list: list, containing wav names. \n \"\"\"\n t1 = time.time()\n with h5py.File(hdf5_path, 'r') as hf:\n x = np.array(hf.get('x'))\n y = np.array(hf.get('y'))\n na_list = list(hf.get('na_list'))\n \n if verbose == 1:\n print(\"--- %s ---\" % hdf5_path)\n print(\"x.shape: %s %s\" % (x.shape, x.dtype))\n print(\"y.shape: %s %s\" % (y.shape, y.dtype))\n print(\"len(na_list): %d\" % len(na_list))\n print(\"Loading time: %s\" % (time.time() - t1,))\n \n return x, y, na_list\n\ndef calculate_scaler(hdf5_path, out_path):\n \"\"\"Calculate scaler of input data on each frequency bin. \n \n Args:\n hdf5_path: string, path of packed hdf5 features file. \n out_path: string, path to write out the calculated scaler. \n \n Returns:\n None. \n \"\"\"\n t1 = time.time()\n (x, y, na_list) = load_hdf5_data(hdf5_path, verbose=1)\n (n_clips, n_time, n_freq) = x.shape\n x2d = x.reshape((n_clips * n_time, n_freq))\n scaler = preprocessing.StandardScaler().fit(x2d)\n print(\"Mean: %s\" % (scaler.mean_,))\n print(\"Std: %s\" % (scaler.scale_,))\n print(\"Calculating scaler time: %s\" % (time.time() - t1,))\n pickle.dump(scaler, open(out_path, 'wb'))\n \ndef do_scale(x3d, scaler_path, verbose=1):\n \"\"\"Do scale on the input sequence data. \n \n Args:\n x3d: ndarray, input sequence data, shape: (n_clips, n_time, n_freq)\n scaler_path: string, path of pre-calculated scaler. \n verbose: integar, print flag. \n \n Returns:\n Scaled input sequence data. \n \"\"\"\n t1 = time.time()\n scaler = pickle.load(open(scaler_path, 'rb'))\n (n_clips, n_time, n_freq) = x3d.shape\n x2d = x3d.reshape((n_clips * n_time, n_freq))\n x2d_scaled = scaler.transform(x2d)\n x3d_scaled = x2d_scaled.reshape((n_clips, n_time, n_freq))\n if verbose == 1:\n print(\"Scaling time: %s\" % (time.time() - t1,))\n return x3d_scaled\n\n### Main function\nif __name__ == '__main__':\n parser = argparse.ArgumentParser(description=\"\")\n subparsers = parser.add_subparsers(dest='mode')\n \n parser_ef = subparsers.add_parser('extract_features')\n parser_ef.add_argument('--wav_dir', type=str)\n parser_ef.add_argument('--out_dir', type=str)\n parser_ef.add_argument('--recompute', type=bool)\n \n parser_pf = subparsers.add_parser('pack_features')\n parser_pf.add_argument('--fe_dir', type=str)\n parser_pf.add_argument('--csv_path', type=str)\n parser_pf.add_argument('--out_path', type=str)\n \n parser_cs = subparsers.add_parser('calculate_scaler')\n parser_cs.add_argument('--hdf5_path', type=str)\n parser_cs.add_argument('--out_path', type=str)\n\n args = parser.parse_args()\n \n if args.mode == 'extract_features':\n extract_features(wav_dir=args.wav_dir, \n out_dir=args.out_dir, \n recompute=args.recompute)\n elif args.mode == 'pack_features':\n pack_features_to_hdf5(fe_dir=args.fe_dir, \n csv_path=args.csv_path, \n out_path=args.out_path)\n elif args.mode == 'calculate_scaler':\n calculate_scaler(hdf5_path=args.hdf5_path, \n out_path=args.out_path)\n else:\n raise Exception(\"Incorrect argument!\")" }, { "path": "runme.sh", "content": "#!/bin/bash\n\nTEST_WAV_DIR=\"/vol/vssp/AP_datasets/audio/audioset/task4_dcase2017_audio/official_downloads/testing\"\nTRAIN_WAV_DIR=\"/vol/vssp/AP_datasets/audio/audioset/task4_dcase2017_audio/official_downloads/training\"\nEVALUATION_WAV_DIR=\"/vol/vssp/datasets/audio/audioset/task4_dcase2017_audio/official_downloads/evaluation\"\n\nWORKSPACE=\"/vol/vssp/msos/qk/workspaces/ICASSP2018_dcase\"\n\n# Extract features\npython prepare_data.py extract_features --wav_dir=$TEST_WAV_DIR --out_dir=$WORKSPACE\"/features/logmel/testing\" --recompute=True\npython prepare_data.py extract_features --wav_dir=$TRAIN_WAV_DIR --out_dir=$WORKSPACE\"/features/logmel/training\" --recompute=True\npython prepare_data.py extract_features --wav_dir=$EVALUATION_WAV_DIR --out_dir=$WORKSPACE\"/features/logmel/evaluation\" --recompute=True\n\n# Pack features\npython prepare_data.py pack_features --fe_dir=$WORKSPACE\"/features/logmel/testing\" --csv_path=\"meta_data/testing_set.csv\" --out_path=$WORKSPACE\"/packed_features/logmel/testing.h5\"\npython prepare_data.py pack_features --fe_dir=$WORKSPACE\"/features/logmel/training\" --csv_path=\"meta_data/training_set.csv\" --out_path=$WORKSPACE\"/packed_features/logmel/training.h5\"\npython prepare_data.py pack_features --fe_dir=$WORKSPACE\"/features/logmel/evaluation\" --csv_path=\"\" --out_path=$WORKSPACE\"/packed_features/logmel/evaluation.h5\"\n\n# Calculate scaler\npython prepare_data.py calculate_scaler --hdf5_path=$WORKSPACE\"/packed_features/logmel/training.h5\" --out_path=$WORKSPACE\"/scalers/logmel/training.scaler\"\n\n# Train AT\nTHEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python main_crnn_at.py train --tr_hdf5_path=$WORKSPACE\"/packed_features/logmel/training.h5\" --te_hdf5_path=$WORKSPACE\"/packed_features/logmel/testing.h5\" --scaler_path=$WORKSPACE\"/scalers/logmel/training.scaler\" --out_model_dir=$WORKSPACE\"/models/crnn_at\"\n\n# Recognize AT\nTHEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python main_crnn_at.py recognize --te_hdf5_path=$WORKSPACE\"/packed_features/logmel/testing.h5\" --scaler_path=$WORKSPACE\"/scalers/logmel/training.scaler\" --model_dir=$WORKSPACE\"/models/crnn_at\" --out_dir=$WORKSPACE\"/preds/crnn_at\"\n\n# Get stat of AT\nTHEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python main_crnn_at.py get_stat --pred_dir=$WORKSPACE\"/preds/crnn_at\" --stat_dir=$WORKSPACE\"/stats/crnn_at\" --submission_dir=$WORKSPACE\"/submissions/crnn_at\"\n\n# Train SED\nTHEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python main_crnn_sed.py train --tr_hdf5_path=$WORKSPACE\"/packed_features/logmel/training.h5\" --te_hdf5_path=$WORKSPACE\"/packed_features/logmel/testing.h5\" --scaler_path=$WORKSPACE\"/scalers/logmel/training.scaler\" --out_model_dir=$WORKSPACE\"/models/crnn_sed\"\n\n# Recognize SED\nTHEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python main_crnn_at.py recognize --te_hdf5_path=$WORKSPACE\"/packed_features/logmel/testing.h5\" --scaler_path=$WORKSPACE\"/scalers/logmel/training.scaler\" --model_dir=$WORKSPACE\"/models/crnn_sed\" --out_dir=$WORKSPACE\"/preds/crnn_sed\"\n\n# Get stat of SED\nTHEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python main_crnn_at.py get_stat --pred_dir=$WORKSPACE\"/preds/crnn_sed\" --stat_dir=$WORKSPACE\"/stats/crnn_sed\" --submission_dir=$WORKSPACE\"/submissions/crnn_sed\"\n" } ]