[
  {
    "path": "README.md",
    "content": "# StarGAN-Voice-Conversion\nThis is a pytorch implementation of the paper: StarGAN-VC: Non-parallel many-to-many voice conversion with star generative adversarial networks  https://arxiv.org/abs/1806.02169 .\nNote that the model architecture is a little different from that of the original paper.\n\n# Dependencies\n* Python 3.6 (or 3.5)\n* Pytorch 0.4.0\n* pyworld\n* tqdm\n* librosa\n* tensorboardX and tensorboard\n\n# Usage\n## Download Dataset\n\nDownload and unzip [VCTK](https://homepages.inf.ed.ac.uk/jyamagis/page3/page58/page58.html) corpus to designated directories.\n\n```bash\nmkdir ./data\nwget https://datashare.is.ed.ac.uk/bitstream/handle/10283/2651/VCTK-Corpus.zip?sequence=2&isAllowed=y\nunzip VCTK-Corpus.zip -d ./data\n```\nIf the downloaded VCTK is in tar.gz, run this:\n\n```bash\ntar -xzvf VCTK-Corpus.tar.gz -C ./data\n```\n\nPreprocess data\n\nWe will use Mel-cepstral coefficients(MCEPs) here.\n\n```bash\npython preprocess.py --sample_rate 16000 \\\n                    --origin_wavpath data/VCTK-Corpus/wav48 \\\n                    --target_wavpath data/VCTK-Corpus/wav16 \\\n                    --mc_dir_train data/mc/train \\\n                    --mc_dir_test data/mc/test\n```\n\nTrain model\n\nNote: you may need to early stop the training process if the training-time test samples sounds good or the you can also see the training loss curves to determine early stop or not.\n\n```\npython main.py\n```\n\nConvert\n\nFor example: restore model at step 200000 and specify the source speaker and target speaker to `p262` and `p272`, respectively.\n\n```\nconvert.py --resume_iters 200000 --src_spk p262 --trg_spk p272\n```\n\n## To-Do list\n- [x] Post some converted samples (Please find some converted samples in the `converted_samples` folder).\n\n## Papers that use this repo:\n1. [AUTOVC: Zero-Shot Voice Style Transfer with Only Autoencoder Loss (ICML2019)](https://arxiv.org/pdf/1905.05879v2.pdf)\n2. [Blow: a single-scale hyperconditioned flow for non-parallel raw-audio voice conversion (NeurIPS 2019)](https://arxiv.org/pdf/1906.00794.pdf)\n3. [ADAGAN: ADAPTIVE GAN FOR MANY-TO-MANY NON-PARALLEL VOICE CONVERSION (under review for ICLR 2020)](https://openreview.net/pdf?id=HJlk-eHFwH)\n\n"
  },
  {
    "path": "convert.py",
    "content": "import argparse\nfrom model import Generator\nfrom torch.autograd import Variable\nimport torch\nimport torch.nn.functional as F\nimport numpy as np\nimport os\nfrom os.path import join, basename, dirname, split\nimport time\nimport datetime\nfrom data_loader import to_categorical\nimport librosa\nfrom utils import *\nimport glob\n\n# Below is the accent info for the used 10 speakers.\nspk2acc = {'262': 'Edinburgh', #F\n           '272': 'Edinburgh', #M\n           '229': 'SouthEngland', #F \n           '232': 'SouthEngland', #M\n           '292': 'NorthernIrishBelfast', #M \n           '293': 'NorthernIrishBelfast', #F \n           '360': 'AmericanNewJersey', #M\n           '361': 'AmericanNewJersey', #F\n           '248': 'India', #F\n           '251': 'India'} #M\n\nspeakers = ['p262', 'p272', 'p229', 'p232', 'p292', 'p293', 'p360', 'p361', 'p248', 'p251']\nspk2idx = dict(zip(speakers, range(len(speakers))))\n\nclass TestDataset(object):\n    \"\"\"Dataset for testing.\"\"\"\n    def __init__(self, config):\n        assert config.trg_spk in speakers, f'The trg_spk should be chosen from {speakers}, but you choose {trg_spk}.'\n        # Source speaker\n        self.src_spk = config.src_spk\n        self.trg_spk = config.trg_spk\n\n        self.mc_files = sorted(glob.glob(join(config.test_data_dir, f'{config.src_spk}*.npy')))\n        self.src_spk_stats = np.load(join(config.train_data_dir, f'{config.src_spk}_stats.npz'))\n        self.src_wav_dir = f'{config.wav_dir}/{config.src_spk}'\n\n        \n        self.trg_spk_stats = np.load(join(config.train_data_dir, f'{config.trg_spk}_stats.npz'))\n\n        self.logf0s_mean_src = self.src_spk_stats['log_f0s_mean']\n        self.logf0s_std_src = self.src_spk_stats['log_f0s_std']\n        self.logf0s_mean_trg = self.trg_spk_stats['log_f0s_mean']\n        self.logf0s_std_trg = self.trg_spk_stats['log_f0s_std']\n        self.mcep_mean_src = self.src_spk_stats['coded_sps_mean']\n        self.mcep_std_src = self.src_spk_stats['coded_sps_std']\n        self.mcep_mean_trg = self.trg_spk_stats['coded_sps_mean']\n        self.mcep_std_trg = self.trg_spk_stats['coded_sps_std']\n        \n        self.spk_idx = spk2idx[config.trg_spk]\n        spk_cat = to_categorical([self.spk_idx], num_classes=len(speakers))\n        self.spk_c_trg = spk_cat\n\n\n    def get_batch_test_data(self, batch_size=4):\n        batch_data = []\n        for i in range(batch_size):\n            mcfile = self.mc_files[i]\n            filename = basename(mcfile).split('-')[-1]\n            wavfile_path = join(self.src_wav_dir, filename.replace('npy', 'wav'))\n            batch_data.append(wavfile_path)\n        return batch_data \n\n\ndef load_wav(wavfile, sr=16000):\n    wav, _ = librosa.load(wavfile, sr=sr, mono=True)\n    return wav_padding(wav, sr=sr, frame_period=5, multiple = 4)  # TODO\n    # return wav\n\ndef test(config):\n    os.makedirs(join(config.convert_dir, str(config.resume_iters)), exist_ok=True)\n    sampling_rate, num_mcep, frame_period=16000, 36, 5\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n    G = Generator().to(device)\n    test_loader = TestDataset(config)\n    # Restore model\n    print(f'Loading the trained models from step {config.resume_iters}...')\n    G_path = join(config.model_save_dir, f'{config.resume_iters}-G.ckpt')\n    G.load_state_dict(torch.load(G_path, map_location=lambda storage, loc: storage))\n\n    # Read a batch of testdata\n    test_wavfiles = test_loader.get_batch_test_data(batch_size=config.num_converted_wavs)\n    test_wavs = [load_wav(wavfile, sampling_rate) for wavfile in test_wavfiles]\n\n    with torch.no_grad():\n        for idx, wav in enumerate(test_wavs):\n            print(len(wav))\n            wav_name = basename(test_wavfiles[idx])\n            # print(wav_name)\n            f0, timeaxis, sp, ap = world_decompose(wav=wav, fs=sampling_rate, frame_period=frame_period)\n            f0_converted = pitch_conversion(f0=f0, \n                mean_log_src=test_loader.logf0s_mean_src, std_log_src=test_loader.logf0s_std_src, \n                mean_log_target=test_loader.logf0s_mean_trg, std_log_target=test_loader.logf0s_std_trg)\n            coded_sp = world_encode_spectral_envelop(sp=sp, fs=sampling_rate, dim=num_mcep)\n            print(\"Before being fed into G: \", coded_sp.shape)\n            coded_sp_norm = (coded_sp - test_loader.mcep_mean_src) / test_loader.mcep_std_src\n            coded_sp_norm_tensor = torch.FloatTensor(coded_sp_norm.T).unsqueeze_(0).unsqueeze_(1).to(device)\n            spk_conds = torch.FloatTensor(test_loader.spk_c_trg).to(device)\n            # print(spk_conds.size())\n            coded_sp_converted_norm = G(coded_sp_norm_tensor, spk_conds).data.cpu().numpy()\n            coded_sp_converted = np.squeeze(coded_sp_converted_norm).T * test_loader.mcep_std_trg + test_loader.mcep_mean_trg\n            coded_sp_converted = np.ascontiguousarray(coded_sp_converted)\n            print(\"After being fed into G: \", coded_sp_converted.shape)\n            wav_transformed = world_speech_synthesis(f0=f0_converted, coded_sp=coded_sp_converted, \n                                                    ap=ap, fs=sampling_rate, frame_period=frame_period)\n            wav_id = wav_name.split('.')[0]\n            librosa.output.write_wav(join(config.convert_dir, str(config.resume_iters),\n                f'{wav_id}-vcto-{test_loader.trg_spk}.wav'), wav_transformed, sampling_rate)\n            if [True, False][0]:\n                wav_cpsyn = world_speech_synthesis(f0=f0, coded_sp=coded_sp, \n                                                ap=ap, fs=sampling_rate, frame_period=frame_period)\n                librosa.output.write_wav(join(config.convert_dir, str(config.resume_iters), f'cpsyn-{wav_name}'), wav_cpsyn, sampling_rate)\n\n\nif __name__ == '__main__':\n    parser = argparse.ArgumentParser()\n\n    # Model configuration.\n    parser.add_argument('--num_speakers', type=int, default=10, help='dimension of speaker labels')\n    parser.add_argument('--num_converted_wavs', type=int, default=8, help='number of wavs to convert.')\n    parser.add_argument('--resume_iters', type=int, default=None, help='step to resume for testing.')\n    parser.add_argument('--src_spk', type=str, default='p262', help = 'target speaker.')\n    parser.add_argument('--trg_spk', type=str, default='p272', help = 'target speaker.')\n\n    # Directories.\n    parser.add_argument('--train_data_dir', type=str, default='./data/mc/train')\n    parser.add_argument('--test_data_dir', type=str, default='./data/mc/test')\n    parser.add_argument('--wav_dir', type=str, default=\"./data/VCTK-Corpus/wav16\")\n    parser.add_argument('--log_dir', type=str, default='./logs')\n    parser.add_argument('--model_save_dir', type=str, default='./models')\n    parser.add_argument('--convert_dir', type=str, default='./converted')\n\n\n    config = parser.parse_args()\n    \n    print(config)\n    if config.resume_iters is None:\n        raise RuntimeError(\"Please specify the step number for resuming.\")\n    test(config)\n"
  },
  {
    "path": "converted_samples/Readme",
    "content": "cpsyn stands for copy-synthesis for references.\n\n\nThese converted samples were obtained from not-well-fine-tuned model. If you want to get better results, please tune the hyper-parameters carefully."
  },
  {
    "path": "data_loader.py",
    "content": "from torch.utils import data\nimport torch\nimport os\nimport random\nimport glob\nfrom os.path import join, basename, dirname, split\nimport numpy as np\n\n# Below is the accent info for the used 10 speakers.\nspk2acc = {'262': 'Edinburgh', #F\n           '272': 'Edinburgh', #M\n           '229': 'SouthEngland', #F \n           '232': 'SouthEngland', #M\n           '292': 'NorthernIrishBelfast', #M \n           '293': 'NorthernIrishBelfast', #F \n           '360': 'AmericanNewJersey', #M\n           '361': 'AmericanNewJersey', #F\n           '248': 'India', #F\n           '251': 'India'} #M\nmin_length = 256   # Since we slice 256 frames from each utterance when training.\n# Build a dict useful when we want to get one-hot representation of speakers.\nspeakers = ['p262', 'p272', 'p229', 'p232', 'p292', 'p293', 'p360', 'p361', 'p248', 'p251']\nspk2idx = dict(zip(speakers, range(len(speakers))))\n\ndef to_categorical(y, num_classes=None):\n    \"\"\"Converts a class vector (integers) to binary class matrix.\n    E.g. for use with categorical_crossentropy.\n    # Arguments\n        y: class vector to be converted into a matrix\n            (integers from 0 to num_classes).\n        num_classes: total number of classes.\n    # Returns\n        A binary matrix representation of the input. The classes axis\n        is placed last.\n    From Keras np_utils\n    \"\"\"\n    y = np.array(y, dtype='int')\n    input_shape = y.shape\n    if input_shape and input_shape[-1] == 1 and len(input_shape) > 1:\n        input_shape = tuple(input_shape[:-1])\n    y = y.ravel()\n    if not num_classes:\n        num_classes = np.max(y) + 1\n    n = y.shape[0]\n    categorical = np.zeros((n, num_classes), dtype=np.float32)\n    categorical[np.arange(n), y] = 1\n    output_shape = input_shape + (num_classes,)\n    categorical = np.reshape(categorical, output_shape)\n    return categorical\n\nclass MyDataset(data.Dataset):\n    \"\"\"Dataset for MCEP features and speaker labels.\"\"\"\n    def __init__(self, data_dir):\n        mc_files = glob.glob(join(data_dir, '*.npy'))\n        mc_files = [i for i in mc_files if basename(i)[:4] in speakers] \n        self.mc_files = self.rm_too_short_utt(mc_files)\n        self.num_files = len(self.mc_files)\n        print(\"\\t Number of training samples: \", self.num_files)\n        for f in self.mc_files:\n            mc = np.load(f)\n            if mc.shape[0] <= min_length:\n                print(f)\n                raise RuntimeError(f\"The data may be corrupted! We need all MCEP features having more than {min_length} frames!\") \n\n    def rm_too_short_utt(self, mc_files, min_length=min_length):\n        new_mc_files = []\n        for mcfile in mc_files:\n            mc = np.load(mcfile)\n            if mc.shape[0] > min_length:\n                new_mc_files.append(mcfile)\n        return new_mc_files\n\n    def sample_seg(self, feat, sample_len=min_length):\n        assert feat.shape[0] - sample_len >= 0\n        s = np.random.randint(0, feat.shape[0] - sample_len + 1)\n        return feat[s:s+sample_len, :]\n\n    def __len__(self):\n        return self.num_files\n\n    def __getitem__(self, index):\n        filename = self.mc_files[index]\n        spk = basename(filename).split('_')[0]\n        spk_idx = spk2idx[spk]\n        mc = np.load(filename)\n        mc = self.sample_seg(mc)\n        mc = np.transpose(mc, (1, 0))  # (T, D) -> (D, T), since pytorch need feature having shape\n        # to one-hot\n        spk_cat = np.squeeze(to_categorical([spk_idx], num_classes=len(speakers)))\n\n        return torch.FloatTensor(mc), torch.LongTensor([spk_idx]).squeeze_(), torch.FloatTensor(spk_cat)\n        \n\nclass TestDataset(object):\n    \"\"\"Dataset for testing.\"\"\"\n    def __init__(self, data_dir, wav_dir, src_spk='p262', trg_spk='p272'):\n        self.src_spk = src_spk\n        self.trg_spk = trg_spk\n        self.mc_files = sorted(glob.glob(join(data_dir, '{}*.npy'.format(self.src_spk))))\n\n        self.src_spk_stats = np.load(join(data_dir.replace('test', 'train'), '{}_stats.npz'.format(src_spk)))\n        self.trg_spk_stats = np.load(join(data_dir.replace('test', 'train'), '{}_stats.npz'.format(trg_spk)))\n        \n        self.logf0s_mean_src = self.src_spk_stats['log_f0s_mean']\n        self.logf0s_std_src = self.src_spk_stats['log_f0s_std']\n        self.logf0s_mean_trg = self.trg_spk_stats['log_f0s_mean']\n        self.logf0s_std_trg = self.trg_spk_stats['log_f0s_std']\n        self.mcep_mean_src = self.src_spk_stats['coded_sps_mean']\n        self.mcep_std_src = self.src_spk_stats['coded_sps_std']\n        self.mcep_mean_trg = self.trg_spk_stats['coded_sps_mean']\n        self.mcep_std_trg = self.trg_spk_stats['coded_sps_std']\n        self.src_wav_dir = f'{wav_dir}/{src_spk}'\n        self.spk_idx = spk2idx[trg_spk]\n        spk_cat = to_categorical([self.spk_idx], num_classes=len(speakers))\n        self.spk_c_trg = spk_cat\n\n    def get_batch_test_data(self, batch_size=8):\n        batch_data = []\n        for i in range(batch_size):\n            mcfile = self.mc_files[i]\n            filename = basename(mcfile).split('-')[-1]\n            wavfile_path = join(self.src_wav_dir, filename.replace('npy', 'wav'))\n            batch_data.append(wavfile_path)\n        return batch_data       \n\ndef get_loader(data_dir, batch_size=32, mode='train', num_workers=1):\n    dataset = MyDataset(data_dir)\n    data_loader = data.DataLoader(dataset=dataset,\n                                  batch_size=batch_size,\n                                  shuffle=(mode=='train'),\n                                  num_workers=num_workers,\n                                  drop_last=True)\n    return data_loader\n\n\nif __name__ == '__main__':\n    loader = get_loader('./data/mc/train')\n    data_iter = iter(loader)\n    for i in range(10):\n        mc, spk_idx, acc_idx, spk_acc_cat = next(data_iter)\n        print('-'*50)\n        print(mc.size())\n        print(spk_idx.size())\n        print(acc_idx.size())\n        print(spk_acc_cat.size())\n        print(spk_idx.squeeze_())\n        print(spk_acc_cat)\n        print('-'*50)\n\n\n\n\n\n\n\n"
  },
  {
    "path": "logger.py",
    "content": "# import tensorflow as tf\nfrom tensorboardX import SummaryWriter\n\n# class Logger(object):\n#     \"\"\"Tensorflow Tensorboard logger.\"\"\"\n\n#     def __init__(self, log_dir):\n#         \"\"\"Initialize summary writer.\"\"\"\n#         self.writer = tf.summary.FileWriter(log_dir)\n\n#     def scalar_summary(self, tag, value, step):\n#         \"\"\"Add scalar summary.\"\"\"\n#         summary = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=value)])\n#         self.writer.add_summary(summary, step)\n\nclass Logger(object):\n    \"\"\"Using tensorboardX such that need no dependency on tensorflow.\"\"\"\n\n    def __init__(self, log_dir):\n        \"\"\"Initialize summary writer.\"\"\"\n        self.writer = SummaryWriter(log_dir)\n\n    def scalar_summary(self, tag, value, step):\n        self.writer.add_scalar(tag, value, step)"
  },
  {
    "path": "main.py",
    "content": "import os\nimport argparse\nfrom solver import Solver\nfrom data_loader import get_loader, TestDataset\nfrom torch.backends import cudnn\n\n\ndef str2bool(v):\n    return v.lower() in ('true')\n\ndef main(config):\n    # For fast training.\n    cudnn.benchmark = True\n\n    # Create directories if not exist.\n    if not os.path.exists(config.log_dir):\n        os.makedirs(config.log_dir)\n    if not os.path.exists(config.model_save_dir):\n        os.makedirs(config.model_save_dir)\n    if not os.path.exists(config.sample_dir):\n        os.makedirs(config.sample_dir)\n\n    # Data loader.\n    train_loader = get_loader(config.train_data_dir, config.batch_size, 'train', num_workers=config.num_workers)\n    test_loader = TestDataset(config.test_data_dir, config.wav_dir, src_spk='p262', trg_spk='p272')\n\n    # Solver for training and testing StarGAN.\n    solver = Solver(train_loader, test_loader, config)\n\n    if config.mode == 'train':    \n        solver.train()\n\n    # elif config.mode == 'test':\n    #     solver.test()\n\n\nif __name__ == '__main__':\n    parser = argparse.ArgumentParser()\n\n    # Model configuration.\n    parser.add_argument('--num_speakers', type=int, default=10, help='dimension of speaker labels')\n    parser.add_argument('--lambda_cls', type=float, default=10, help='weight for domain classification loss')\n    parser.add_argument('--lambda_rec', type=float, default=10, help='weight for reconstruction loss')\n    parser.add_argument('--lambda_gp', type=float, default=10, help='weight for gradient penalty')\n    parser.add_argument('--sampling_rate', type=int, default=16000, help='sampling rate')\n    \n    # Training configuration.\n    parser.add_argument('--batch_size', type=int, default=32, help='mini-batch size')\n    parser.add_argument('--num_iters', type=int, default=200000, help='number of total iterations for training D')\n    parser.add_argument('--num_iters_decay', type=int, default=100000, help='number of iterations for decaying lr')\n    parser.add_argument('--g_lr', type=float, default=0.0001, help='learning rate for G')\n    parser.add_argument('--d_lr', type=float, default=0.0001, help='learning rate for D')\n    parser.add_argument('--n_critic', type=int, default=5, help='number of D updates per each G update')\n    parser.add_argument('--beta1', type=float, default=0.5, help='beta1 for Adam optimizer')\n    parser.add_argument('--beta2', type=float, default=0.999, help='beta2 for Adam optimizer')\n    parser.add_argument('--resume_iters', type=int, default=None, help='resume training from this step')\n\n    # Test configuration.\n    parser.add_argument('--test_iters', type=int, default=100000, help='test model from this step')\n\n    # Miscellaneous.\n    parser.add_argument('--num_workers', type=int, default=1)\n    parser.add_argument('--mode', type=str, default='train', choices=['train', 'test'])\n    parser.add_argument('--use_tensorboard', type=str2bool, default=True)\n\n    # Directories.\n    parser.add_argument('--train_data_dir', type=str, default='./data/mc/train')\n    parser.add_argument('--test_data_dir', type=str, default='./data/mc/test')\n    parser.add_argument('--wav_dir', type=str, default=\"./data/VCTK-Corpus/wav16\")\n    parser.add_argument('--log_dir', type=str, default='./logs')\n    parser.add_argument('--model_save_dir', type=str, default='./models')\n    parser.add_argument('--sample_dir', type=str, default='./samples')\n\n    # Step size.\n    parser.add_argument('--log_step', type=int, default=10)\n    parser.add_argument('--sample_step', type=int, default=1000)\n    parser.add_argument('--model_save_step', type=int, default=1000)\n    parser.add_argument('--lr_update_step', type=int, default=1000)\n\n    config = parser.parse_args()\n    print(config)\n    main(config)"
  },
  {
    "path": "model.py",
    "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\n# from data_loader import get_loader\n\n\nclass ResidualBlock(nn.Module):\n    \"\"\"Residual Block with instance normalization.\"\"\"\n    def __init__(self, dim_in, dim_out):\n        super(ResidualBlock, self).__init__()\n        self.main = nn.Sequential(\n            nn.Conv2d(dim_in, dim_out, kernel_size=3, stride=1, padding=1, bias=False),\n            nn.InstanceNorm2d(dim_out, affine=True, track_running_stats=True),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(dim_out, dim_out, kernel_size=3, stride=1, padding=1, bias=False),\n            nn.InstanceNorm2d(dim_out, affine=True, track_running_stats=True))\n\n    def forward(self, x):\n        return x + self.main(x)\n\nclass Generator(nn.Module):\n    \"\"\"Generator network.\"\"\"\n    def __init__(self, conv_dim=64, num_speakers=10, repeat_num=6):\n        super(Generator, self).__init__()\n        c_dim = num_speakers\n        layers = []\n        layers.append(nn.Conv2d(1+c_dim, conv_dim, kernel_size=(3, 9), padding=(1, 4), bias=False))\n        layers.append(nn.InstanceNorm2d(conv_dim, affine=True, track_running_stats=True))\n        layers.append(nn.ReLU(inplace=True))\n\n        # Down-sampling layers.\n        curr_dim = conv_dim\n        for i in range(2):\n            layers.append(nn.Conv2d(curr_dim, curr_dim*2, kernel_size=(4, 8), stride=(2, 2), padding=(1, 3), bias=False))\n            layers.append(nn.InstanceNorm2d(curr_dim*2, affine=True, track_running_stats=True))\n            layers.append(nn.ReLU(inplace=True))\n            curr_dim = curr_dim * 2\n\n        # Bottleneck layers.\n        for i in range(repeat_num):\n            layers.append(ResidualBlock(dim_in=curr_dim, dim_out=curr_dim))\n\n        # Up-sampling layers.\n        for i in range(2):\n            layers.append(nn.ConvTranspose2d(curr_dim, curr_dim//2, kernel_size=4, stride=2, padding=1, bias=False))\n            layers.append(nn.InstanceNorm2d(curr_dim//2, affine=True, track_running_stats=True))\n            layers.append(nn.ReLU(inplace=True))\n            curr_dim = curr_dim // 2\n\n        layers.append(nn.Conv2d(curr_dim, 1, kernel_size=7, stride=1, padding=3, bias=False))\n        self.main = nn.Sequential(*layers)\n\n    def forward(self, x, c):\n        # Replicate spatially and concatenate domain information.\n        c = c.view(c.size(0), c.size(1), 1, 1)\n        c = c.repeat(1, 1, x.size(2), x.size(3))\n        x = torch.cat([x, c], dim=1)\n        return self.main(x)\n\nclass Discriminator(nn.Module):\n    \"\"\"Discriminator network with PatchGAN.\"\"\"\n    def __init__(self, input_size=(36, 256), conv_dim=64, repeat_num=5, num_speakers=10):\n        super(Discriminator, self).__init__()\n        layers = []\n        layers.append(nn.Conv2d(1, conv_dim, kernel_size=4, stride=2, padding=1))\n        layers.append(nn.LeakyReLU(0.01))\n\n        curr_dim = conv_dim\n        for i in range(1, repeat_num):\n            layers.append(nn.Conv2d(curr_dim, curr_dim*2, kernel_size=4, stride=2, padding=1))\n            layers.append(nn.LeakyReLU(0.01))\n            curr_dim = curr_dim * 2\n\n        kernel_size_0 = int(input_size[0] / np.power(2, repeat_num)) # 1\n        kernel_size_1 = int(input_size[1] / np.power(2, repeat_num)) # 8\n        self.main = nn.Sequential(*layers)\n        self.conv_dis = nn.Conv2d(curr_dim, 1, kernel_size=(kernel_size_0, kernel_size_1), stride=1, padding=0, bias=False) # padding should be 0\n        self.conv_clf_spks = nn.Conv2d(curr_dim, num_speakers, kernel_size=(kernel_size_0, kernel_size_1), stride=1, padding=0, bias=False)  # for num_speaker\n        \n    def forward(self, x):\n        h = self.main(x)\n        out_src = self.conv_dis(h)\n        out_cls_spks = self.conv_clf_spks(h)\n        return out_src, out_cls_spks.view(out_cls_spks.size(0), out_cls_spks.size(1))\n\nif __name__ == '__main__':\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    train_loader = get_loader('/scratch/sxliu/data_exp/VCTK-Corpus-22.05k/mc/train', 16, 'train', num_workers=1)\n    data_iter = iter(train_loader)\n    G = Generator().to(device)\n    D = Discriminator().to(device)\n    for i in range(10):\n        mc_real, spk_label_org, acc_label_org, spk_acc_c_org = next(data_iter)\n        mc_real.unsqueeze_(1) # (B, D, T) -> (B, 1, D, T) for conv2d\n        mc_real = mc_real.to(device)                         # Input mc.\n        spk_label_org = spk_label_org.to(device)             # Original spk labels.\n        acc_label_org = acc_label_org.to(device)             # Original acc labels.\n        spk_acc_c_org = spk_acc_c_org.to(device)             # Original spk acc conditioning.\n        mc_fake = G(mc_real, spk_acc_c_org)\n        print(mc_fake.size())\n        out_src, out_cls_spks, out_cls_emos = D(mc_fake)\n\n\n\n"
  },
  {
    "path": "preprocess.py",
    "content": "import librosa\nimport numpy as np\nimport os, sys\nimport argparse\nimport pyworld\nfrom multiprocessing import cpu_count\nfrom concurrent.futures import ProcessPoolExecutor\nfrom functools import partial\nfrom utils import *\nfrom tqdm import tqdm\nfrom collections import defaultdict\nfrom collections import namedtuple\nfrom sklearn.model_selection import train_test_split\nimport glob\nfrom os.path import join, basename\nimport subprocess\n\ndef resample(spk, origin_wavpath, target_wavpath):\n    wavfiles = [i for i in os.listdir(join(origin_wavpath, spk)) if i.endswith(\".wav\")]\n    for wav in wavfiles:\n        folder_to = join(target_wavpath, spk)\n        os.makedirs(folder_to, exist_ok=True)\n        wav_to = join(folder_to, wav)\n        wav_from = join(origin_wavpath, spk, wav)\n        subprocess.call(['sox', wav_from, \"-r\", \"16000\", wav_to])\n    return 0\n\ndef resample_to_16k(origin_wavpath, target_wavpath, num_workers=1):\n    os.makedirs(target_wavpath, exist_ok=True)\n    spk_folders = os.listdir(origin_wavpath)\n    print(f\"> Using {num_workers} workers!\")\n    executor = ProcessPoolExecutor(max_workers=num_workers)\n    futures = []\n    for spk in spk_folders:\n        futures.append(executor.submit(partial(resample, spk, origin_wavpath, target_wavpath)))\n    result_list = [future.result() for future in tqdm(futures)]\n    print(result_list)\n\ndef split_data(paths):\n    indices = np.arange(len(paths))\n    test_size = 0.1\n    train_indices, test_indices = train_test_split(indices, test_size=test_size, random_state=1234)\n    train_paths = list(np.array(paths)[train_indices])\n    test_paths = list(np.array(paths)[test_indices])\n    return train_paths, test_paths\n\ndef get_spk_world_feats(spk_fold_path, mc_dir_train, mc_dir_test, sample_rate=16000):\n    paths = glob.glob(join(spk_fold_path, '*.wav'))\n    spk_name = basename(spk_fold_path)\n    train_paths, test_paths = split_data(paths)\n    f0s = []\n    coded_sps = []\n    for wav_file in train_paths:\n        f0, _, _, _, coded_sp = world_encode_wav(wav_file, fs=sample_rate)\n        f0s.append(f0)\n        coded_sps.append(coded_sp)\n    log_f0s_mean, log_f0s_std = logf0_statistics(f0s)\n    coded_sps_mean, coded_sps_std = coded_sp_statistics(coded_sps)\n    np.savez(join(mc_dir_train, spk_name+'_stats.npz'), \n            log_f0s_mean=log_f0s_mean,\n            log_f0s_std=log_f0s_std,\n            coded_sps_mean=coded_sps_mean,\n            coded_sps_std=coded_sps_std)\n    \n    for wav_file in tqdm(train_paths):\n        wav_nam = basename(wav_file)\n        f0, timeaxis, sp, ap, coded_sp = world_encode_wav(wav_file, fs=sample_rate)\n        normed_coded_sp = normalize_coded_sp(coded_sp, coded_sps_mean, coded_sps_std)\n        np.save(join(mc_dir_train, wav_nam.replace('.wav', '.npy')), normed_coded_sp, allow_pickle=False)\n    \n    for wav_file in tqdm(test_paths):\n        wav_nam = basename(wav_file)\n        f0, timeaxis, sp, ap, coded_sp = world_encode_wav(wav_file, fs=sample_rate)\n        normed_coded_sp = normalize_coded_sp(coded_sp, coded_sps_mean, coded_sps_std)\n        np.save(join(mc_dir_test, wav_nam.replace('.wav', '.npy')), normed_coded_sp, allow_pickle=False)\n    return 0\n\n\nif __name__ == '__main__':\n    parser = argparse.ArgumentParser()\n\n\n    sample_rate_default = 16000\n    origin_wavpath_default = \"./data/VCTK-Corpus/wav48\"\n    target_wavpath_default = \"./data/VCTK-Corpus/wav16\"\n    mc_dir_train_default = './data/mc/train'\n    mc_dir_test_default = './data/mc/test'\n\n    parser.add_argument(\"--sample_rate\", type = int, default = 16000, help = \"Sample rate.\")\n    parser.add_argument(\"--origin_wavpath\", type = str, default = origin_wavpath_default, help = \"The original wav path to resample.\")\n    parser.add_argument(\"--target_wavpath\", type = str, default = target_wavpath_default, help = \"The original wav path to resample.\")\n    parser.add_argument(\"--mc_dir_train\", type = str, default = mc_dir_train_default, help = \"The directory to store the training features.\")\n    parser.add_argument(\"--mc_dir_test\", type = str, default = mc_dir_test_default, help = \"The directory to store the testing features.\")\n    parser.add_argument(\"--num_workers\", type = int, default = None, help = \"The number of cpus to use.\")\n\n    argv = parser.parse_args()\n\n    sample_rate = argv.sample_rate\n    origin_wavpath = argv.origin_wavpath\n    target_wavpath = argv.target_wavpath\n    mc_dir_train = argv.mc_dir_train\n    mc_dir_test = argv.mc_dir_test\n    num_workers = argv.num_workers if argv.num_workers is not None else cpu_count()\n\n    # The original wav in VCTK is 48K, first we want to resample to 16K\n    resample_to_16k(origin_wavpath, target_wavpath, num_workers=num_workers)\n\n    # WE only use 10 speakers listed below for this experiment.\n    speaker_used = ['262', '272', '229', '232', '292', '293', '360', '361', '248', '251']\n    speaker_used = ['p'+i for i in speaker_used]\n\n    ## Next we are to extract the acoustic features (MCEPs, lf0) and compute the corresponding stats (means, stds). \n    # Make dirs to contain the MCEPs\n    os.makedirs(mc_dir_train, exist_ok=True)\n    os.makedirs(mc_dir_test, exist_ok=True)\n\n    num_workers = len(speaker_used) #cpu_count()\n    print(\"number of workers: \", num_workers)\n    executor = ProcessPoolExecutor(max_workers=num_workers)\n\n    work_dir = target_wavpath\n    # spk_folders = os.listdir(work_dir)\n    # print(\"processing {} speaker folders\".format(len(spk_folders)))\n    # print(spk_folders)\n\n    futures = []\n    for spk in speaker_used:\n        spk_path = os.path.join(work_dir, spk)\n        futures.append(executor.submit(partial(get_spk_world_feats, spk_path, mc_dir_train, mc_dir_test, sample_rate)))\n    result_list = [future.result() for future in tqdm(futures)]\n    print(result_list)\n    sys.exit(0)\n\n"
  },
  {
    "path": "solver.py",
    "content": "from model import Generator\nfrom model import Discriminator\nfrom torch.autograd import Variable\nfrom torchvision.utils import save_image\nimport torch\nimport torch.nn.functional as F\nimport numpy as np\nimport os\nfrom os.path import join, basename, dirname, split\nimport time\nimport datetime\nfrom data_loader import to_categorical\nimport librosa\nfrom utils import *\nfrom tqdm import tqdm\n\n\nclass Solver(object):\n    \"\"\"Solver for training and testing StarGAN.\"\"\"\n\n    def __init__(self, train_loader, test_loader, config):\n        \"\"\"Initialize configurations.\"\"\"\n\n        # Data loader.\n        self.train_loader = train_loader\n        self.test_loader = test_loader\n        self.sampling_rate = config.sampling_rate\n\n        # Model configurations.\n        self.num_speakers = config.num_speakers\n        self.lambda_cls = config.lambda_cls\n        self.lambda_rec = config.lambda_rec\n        self.lambda_gp = config.lambda_gp\n\n        # Training configurations.\n        self.batch_size = config.batch_size\n        self.num_iters = config.num_iters\n        self.num_iters_decay = config.num_iters_decay\n        self.g_lr = config.g_lr\n        self.d_lr = config.d_lr\n        self.n_critic = config.n_critic\n        self.beta1 = config.beta1\n        self.beta2 = config.beta2\n        self.resume_iters = config.resume_iters\n\n        # Test configurations.\n        self.test_iters = config.test_iters\n\n        # Miscellaneous.\n        self.use_tensorboard = config.use_tensorboard\n        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n        # Directories.\n        self.log_dir = config.log_dir\n        self.sample_dir = config.sample_dir\n        self.model_save_dir = config.model_save_dir\n\n        # Step size.\n        self.log_step = config.log_step\n        self.sample_step = config.sample_step\n        self.model_save_step = config.model_save_step\n        self.lr_update_step = config.lr_update_step\n\n        # Build the model and tensorboard.\n        self.build_model()\n        if self.use_tensorboard:\n            self.build_tensorboard()\n\n    def build_model(self):\n        \"\"\"Create a generator and a discriminator.\"\"\"\n        self.G = Generator(num_speakers=self.num_speakers)\n        self.D = Discriminator(num_speakers=self.num_speakers)\n\n        self.g_optimizer = torch.optim.Adam(self.G.parameters(), self.g_lr, [self.beta1, self.beta2])\n        self.d_optimizer = torch.optim.Adam(self.D.parameters(), self.d_lr, [self.beta1, self.beta2])\n        self.print_network(self.G, 'G')\n        self.print_network(self.D, 'D')\n            \n        self.G.to(self.device)\n        self.D.to(self.device)\n\n    def print_network(self, model, name):\n        \"\"\"Print out the network information.\"\"\"\n        num_params = 0\n        for p in model.parameters():\n            num_params += p.numel()\n        print(model)\n        print(name)\n        print(\"The number of parameters: {}\".format(num_params))\n\n    def restore_model(self, resume_iters):\n        \"\"\"Restore the trained generator and discriminator.\"\"\"\n        print('Loading the trained models from step {}...'.format(resume_iters))\n        G_path = os.path.join(self.model_save_dir, '{}-G.ckpt'.format(resume_iters))\n        D_path = os.path.join(self.model_save_dir, '{}-D.ckpt'.format(resume_iters))\n        self.G.load_state_dict(torch.load(G_path, map_location=lambda storage, loc: storage))\n        self.D.load_state_dict(torch.load(D_path, map_location=lambda storage, loc: storage))\n\n    def build_tensorboard(self):\n        \"\"\"Build a tensorboard logger.\"\"\"\n        from logger import Logger\n        self.logger = Logger(self.log_dir)\n\n    def update_lr(self, g_lr, d_lr):\n        \"\"\"Decay learning rates of the generator and discriminator.\"\"\"\n        for param_group in self.g_optimizer.param_groups:\n            param_group['lr'] = g_lr\n        for param_group in self.d_optimizer.param_groups:\n            param_group['lr'] = d_lr\n\n    def reset_grad(self):\n        \"\"\"Reset the gradient buffers.\"\"\"\n        self.g_optimizer.zero_grad()\n        self.d_optimizer.zero_grad()\n\n    def denorm(self, x):\n        \"\"\"Convert the range from [-1, 1] to [0, 1].\"\"\"\n        out = (x + 1) / 2\n        return out.clamp_(0, 1)\n\n    def gradient_penalty(self, y, x):\n        \"\"\"Compute gradient penalty: (L2_norm(dy/dx) - 1)**2.\"\"\"\n        weight = torch.ones(y.size()).to(self.device)\n        dydx = torch.autograd.grad(outputs=y,\n                                   inputs=x,\n                                   grad_outputs=weight,\n                                   retain_graph=True,\n                                   create_graph=True,\n                                   only_inputs=True)[0]\n\n        dydx = dydx.view(dydx.size(0), -1)\n        dydx_l2norm = torch.sqrt(torch.sum(dydx**2, dim=1))\n        return torch.mean((dydx_l2norm-1)**2)\n\n    def label2onehot(self, labels, dim):\n        \"\"\"Convert label indices to one-hot vectors.\"\"\"\n        batch_size = labels.size(0)\n        out = torch.zeros(batch_size, dim)\n        out[np.arange(batch_size), labels.long()] = 1\n        return out\n\n    def sample_spk_c(self, size):\n        spk_c = np.random.randint(0, self.num_speakers, size=size)\n        spk_c_cat = to_categorical(spk_c, self.num_speakers)\n        return torch.LongTensor(spk_c), torch.FloatTensor(spk_c_cat)\n\n    def classification_loss(self, logit, target):\n        \"\"\"Compute softmax cross entropy loss.\"\"\"\n        return F.cross_entropy(logit, target)\n\n    def load_wav(self, wavfile, sr=16000):\n        wav, _ = librosa.load(wavfile, sr=sr, mono=True)\n        return wav_padding(wav, sr=16000, frame_period=5, multiple = 4)  # TODO\n\n    def train(self):\n        \"\"\"Train StarGAN.\"\"\"\n        # Set data loader.\n        train_loader = self.train_loader\n\n        data_iter = iter(train_loader)\n\n        # Read a batch of testdata\n        test_wavfiles = self.test_loader.get_batch_test_data(batch_size=4)\n        test_wavs = [self.load_wav(wavfile) for wavfile in test_wavfiles]\n\n        # Determine whether do copysynthesize when first do training-time conversion test.\n        cpsyn_flag = [True, False][0]\n        # f0, timeaxis, sp, ap = world_decompose(wav = wav, fs = sampling_rate, frame_period = frame_period)\n\n        # Learning rate cache for decaying.\n        g_lr = self.g_lr\n        d_lr = self.d_lr\n\n        # Start training from scratch or resume training.\n        start_iters = 0\n        if self.resume_iters:\n            print(\"resuming step %d ...\"% self.resume_iters)\n            start_iters = self.resume_iters\n            self.restore_model(self.resume_iters)\n\n        # Start training.\n        print('Start training...')\n        start_time = time.time()\n        for i in range(start_iters, self.num_iters):\n\n            # =================================================================================== #\n            #                             1. Preprocess input data                                #\n            # =================================================================================== #\n\n            # Fetch labels.\n            try:\n                mc_real, spk_label_org, spk_c_org = next(data_iter)\n            except:\n                data_iter = iter(train_loader)\n                mc_real, spk_label_org, spk_c_org = next(data_iter)\n\n            mc_real.unsqueeze_(1) # (B, D, T) -> (B, 1, D, T) for conv2d\n\n            # Generate target domain labels randomly.\n            # spk_label_trg: int,   spk_c_trg:one-hot representation \n            spk_label_trg, spk_c_trg = self.sample_spk_c(mc_real.size(0)) \n\n            mc_real = mc_real.to(self.device)                         # Input mc.\n            spk_label_org = spk_label_org.to(self.device)             # Original spk labels.\n            spk_c_org = spk_c_org.to(self.device)                     # Original spk acc conditioning.\n            spk_label_trg = spk_label_trg.to(self.device)             # Target spk labels for classification loss for G.\n            spk_c_trg = spk_c_trg.to(self.device)                     # Target spk conditioning.\n\n            # =================================================================================== #\n            #                             2. Train the discriminator                              #\n            # =================================================================================== #\n\n            # Compute loss with real mc feats.\n            out_src, out_cls_spks = self.D(mc_real)\n            d_loss_real = - torch.mean(out_src)\n            d_loss_cls_spks = self.classification_loss(out_cls_spks, spk_label_org)\n            \n\n            # Compute loss with fake mc feats.\n            mc_fake = self.G(mc_real, spk_c_trg)\n            out_src, out_cls_spks = self.D(mc_fake.detach())\n            d_loss_fake = torch.mean(out_src)\n\n            # Compute loss for gradient penalty.\n            alpha = torch.rand(mc_real.size(0), 1, 1, 1).to(self.device)\n            x_hat = (alpha * mc_real.data + (1 - alpha) * mc_fake.data).requires_grad_(True)\n            out_src, _ = self.D(x_hat)\n            d_loss_gp = self.gradient_penalty(out_src, x_hat)\n\n            # Backward and optimize.\n            d_loss = d_loss_real + d_loss_fake + self.lambda_cls * d_loss_cls_spks + self.lambda_gp * d_loss_gp\n            self.reset_grad()\n            d_loss.backward()\n            self.d_optimizer.step()\n\n            # Logging.\n            loss = {}\n            loss['D/loss_real'] = d_loss_real.item()\n            loss['D/loss_fake'] = d_loss_fake.item()\n            loss['D/loss_cls_spks'] = d_loss_cls_spks.item()\n            loss['D/loss_gp'] = d_loss_gp.item()\n            \n            # =================================================================================== #\n            #                               3. Train the generator                                #\n            # =================================================================================== #\n            \n            if (i+1) % self.n_critic == 0:\n                # Original-to-target domain.\n                mc_fake = self.G(mc_real, spk_c_trg)\n                out_src, out_cls_spks = self.D(mc_fake)\n                g_loss_fake = - torch.mean(out_src)\n                g_loss_cls_spks = self.classification_loss(out_cls_spks, spk_label_trg)\n\n                # Target-to-original domain.\n                mc_reconst = self.G(mc_fake, spk_c_org)\n                g_loss_rec = torch.mean(torch.abs(mc_real - mc_reconst))\n\n                # Backward and optimize.\n                g_loss = g_loss_fake + self.lambda_rec * g_loss_rec + self.lambda_cls * g_loss_cls_spks\n                self.reset_grad()\n                g_loss.backward()\n                self.g_optimizer.step()\n\n                # Logging.\n                loss['G/loss_fake'] = g_loss_fake.item()\n                loss['G/loss_rec'] = g_loss_rec.item()\n                loss['G/loss_cls_spks'] = g_loss_cls_spks.item()\n\n            # =================================================================================== #\n            #                                 4. Miscellaneous                                    #\n            # =================================================================================== #\n\n            # Print out training information.\n            if (i+1) % self.log_step == 0:\n                et = time.time() - start_time\n                et = str(datetime.timedelta(seconds=et))[:-7]\n                log = \"Elapsed [{}], Iteration [{}/{}]\".format(et, i+1, self.num_iters)\n                for tag, value in loss.items():\n                    log += \", {}: {:.4f}\".format(tag, value)\n                print(log)\n\n                if self.use_tensorboard:\n                    for tag, value in loss.items():\n                        self.logger.scalar_summary(tag, value, i+1)\n\n            if (i+1) % self.sample_step == 0:\n                sampling_rate=16000\n                num_mcep=36\n                frame_period=5\n                with torch.no_grad():\n                    for idx, wav in tqdm(enumerate(test_wavs)):\n                        wav_name = basename(test_wavfiles[idx])\n                        # print(wav_name)\n                        f0, timeaxis, sp, ap = world_decompose(wav=wav, fs=sampling_rate, frame_period=frame_period)\n                        f0_converted = pitch_conversion(f0=f0, \n                            mean_log_src=self.test_loader.logf0s_mean_src, std_log_src=self.test_loader.logf0s_std_src, \n                            mean_log_target=self.test_loader.logf0s_mean_trg, std_log_target=self.test_loader.logf0s_std_trg)\n                        coded_sp = world_encode_spectral_envelop(sp=sp, fs=sampling_rate, dim=num_mcep)\n                        \n                        coded_sp_norm = (coded_sp - self.test_loader.mcep_mean_src) / self.test_loader.mcep_std_src\n                        coded_sp_norm_tensor = torch.FloatTensor(coded_sp_norm.T).unsqueeze_(0).unsqueeze_(1).to(self.device)\n                        conds = torch.FloatTensor(self.test_loader.spk_c_trg).to(self.device)\n                        # print(conds.size())\n                        coded_sp_converted_norm = self.G(coded_sp_norm_tensor, conds).data.cpu().numpy()\n                        coded_sp_converted = np.squeeze(coded_sp_converted_norm).T * self.test_loader.mcep_std_trg + self.test_loader.mcep_mean_trg\n                        coded_sp_converted = np.ascontiguousarray(coded_sp_converted)\n                        # decoded_sp_converted = world_decode_spectral_envelop(coded_sp = coded_sp_converted, fs = sampling_rate)\n                        wav_transformed = world_speech_synthesis(f0=f0_converted, coded_sp=coded_sp_converted, \n                                                                ap=ap, fs=sampling_rate, frame_period=frame_period)\n                        \n                        librosa.output.write_wav(\n                            join(self.sample_dir, str(i+1)+'-'+wav_name.split('.')[0]+'-vcto-{}'.format(self.test_loader.trg_spk)+'.wav'), wav_transformed, sampling_rate)\n                        if cpsyn_flag:\n                            wav_cpsyn = world_speech_synthesis(f0=f0, coded_sp=coded_sp, \n                                                        ap=ap, fs=sampling_rate, frame_period=frame_period)\n                            librosa.output.write_wav(join(self.sample_dir, 'cpsyn-'+wav_name), wav_cpsyn, sampling_rate)\n                    cpsyn_flag = False\n\n            # Save model checkpoints.\n            if (i+1) % self.model_save_step == 0:\n                G_path = os.path.join(self.model_save_dir, '{}-G.ckpt'.format(i+1))\n                D_path = os.path.join(self.model_save_dir, '{}-D.ckpt'.format(i+1))\n                torch.save(self.G.state_dict(), G_path)\n                torch.save(self.D.state_dict(), D_path)\n                print('Saved model checkpoints into {}...'.format(self.model_save_dir))\n\n            # Decay learning rates.\n            if (i+1) % self.lr_update_step == 0 and (i+1) > (self.num_iters - self.num_iters_decay):\n                g_lr -= (self.g_lr / float(self.num_iters_decay))\n                d_lr -= (self.d_lr / float(self.num_iters_decay))\n                self.update_lr(g_lr, d_lr)\n                print ('Decayed learning rates, g_lr: {}, d_lr: {}.'.format(g_lr, d_lr))\n\n\n"
  },
  {
    "path": "utils.py",
    "content": "import librosa\nimport numpy as np\nimport os\nimport pyworld\n\n\ndef load_wav(wav_file, sr):\n    wav, _ = librosa.load(wav_file, sr=sr, mono=True)\n    return wav\n\ndef world_decompose(wav, fs, frame_period = 5.0):\n    # Decompose speech signal into f0, spectral envelope and aperiodicity using WORLD\n    wav = wav.astype(np.float64)\n    f0, timeaxis = pyworld.harvest(wav, fs, frame_period = frame_period, f0_floor = 71.0, f0_ceil = 800.0)\n    sp = pyworld.cheaptrick(wav, f0, timeaxis, fs)\n    ap = pyworld.d4c(wav, f0, timeaxis, fs)\n    return f0, timeaxis, sp, ap\n\ndef world_encode_spectral_envelop(sp, fs, dim=36):\n    # Get Mel-cepstral coefficients (MCEPs)\n    #sp = sp.astype(np.float64)\n    coded_sp = pyworld.code_spectral_envelope(sp, fs, dim)\n    return coded_sp\n\ndef world_decode_spectral_envelop(coded_sp, fs):\n    # Decode Mel-cepstral to sp\n    fftlen = pyworld.get_cheaptrick_fft_size(fs)\n    decoded_sp = pyworld.decode_spectral_envelope(coded_sp, fs, fftlen)\n    return decoded_sp\n\ndef world_encode_wav(wav_file, fs, frame_period=5.0, coded_dim=36):\n    wav = load_wav(wav_file, sr=fs)\n    f0, timeaxis, sp, ap = world_decompose(wav=wav, fs=fs, frame_period=frame_period)\n    coded_sp = world_encode_spectral_envelop(sp = sp, fs = fs, dim = coded_dim)\n    return f0, timeaxis, sp, ap, coded_sp\n\ndef world_speech_synthesis(f0, coded_sp, ap, fs, frame_period):\n    decoded_sp = world_decode_spectral_envelop(coded_sp, fs)\n    # TODO\n    min_len = min([len(f0), len(coded_sp), len(ap)])\n    f0 = f0[:min_len]\n    coded_sp = coded_sp[:min_len]\n    ap = ap[:min_len]\n    wav = pyworld.synthesize(f0, decoded_sp, ap, fs, frame_period)\n    # Librosa could not save wav if not doing so\n    wav = wav.astype(np.float32)\n    return wav\n\ndef world_synthesis_data(f0s, coded_sps, aps, fs, frame_period):\n    wavs = list()\n    for f0, decoded_sp, ap in zip(f0s, coded_sps, aps):\n        wav = world_speech_synthesis(f0, coded_sp, ap, fs, frame_period)\n        wavs.append(wav)\n    return wavs\n\ndef coded_sps_normalization_fit_transoform(coded_sps):\n    coded_sps_concatenated = np.concatenate(coded_sps, axis = 1)\n    coded_sps_mean = np.mean(coded_sps_concatenated, axis = 1, keepdims = True)\n    coded_sps_std = np.std(coded_sps_concatenated, axis = 1, keepdims = True)\n    coded_sps_normalized = list()\n    for coded_sp in coded_sps:\n        coded_sps_normalized.append((coded_sp - coded_sps_mean) / coded_sps_std)\n    return coded_sps_normalized, coded_sps_mean, coded_sps_std\n\ndef coded_sp_statistics(coded_sps):\n    # sp shape (T, D)\n    coded_sps_concatenated = np.concatenate(coded_sps, axis = 0)\n    coded_sps_mean = np.mean(coded_sps_concatenated, axis = 0, keepdims = False)\n    coded_sps_std = np.std(coded_sps_concatenated, axis = 0, keepdims = False)\n    return coded_sps_mean, coded_sps_std\n\ndef normalize_coded_sp(coded_sp, coded_sp_mean, coded_sp_std):\n    normed = (coded_sp - coded_sp_mean) / coded_sp_std\n    return normed\n\ndef coded_sps_normalization_transoform(coded_sps, coded_sps_mean, coded_sps_std):\n\n    coded_sps_normalized = list()\n    for coded_sp in coded_sps:\n        coded_sps_normalized.append((coded_sp - coded_sps_mean) / coded_sps_std)\n    \n    return coded_sps_normalized\n\ndef coded_sps_normalization_inverse_transoform(normalized_coded_sps, coded_sps_mean, coded_sps_std):\n\n    coded_sps = list()\n    for normalized_coded_sp in normalized_coded_sps:\n        coded_sps.append(normalized_coded_sp * coded_sps_std + coded_sps_mean)\n\n    return coded_sps\n\ndef coded_sp_padding(coded_sp, multiple = 4):\n    num_features = coded_sp.shape[0]\n    num_frames = coded_sp.shape[1]\n    num_frames_padded = int(np.ceil(num_frames / multiple)) * multiple\n    num_frames_diff = num_frames_padded - num_frames\n    num_pad_left = num_frames_diff // 2\n    num_pad_right = num_frames_diff - num_pad_left\n    coded_sp_padded = np.pad(coded_sp, ((0, 0), (num_pad_left, num_pad_right)), 'constant', constant_values = 0)\n    return coded_sp_padded\n\ndef wav_padding(wav, sr, frame_period, multiple = 4):\n\n    assert wav.ndim == 1 \n    num_frames = len(wav)\n    num_frames_padded = int((np.ceil((np.floor(num_frames / (sr * frame_period / 1000)) + 1) / multiple + 1) * multiple - 1) * (sr * frame_period / 1000))\n    num_frames_diff = num_frames_padded - num_frames\n    num_pad_left = num_frames_diff // 2\n    num_pad_right = num_frames_diff - num_pad_left\n    wav_padded = np.pad(wav, (num_pad_left, num_pad_right), 'constant', constant_values = 0)\n\n    return wav_padded\n\ndef logf0_statistics(f0s):\n    log_f0s_concatenated = np.ma.log(np.concatenate(f0s))\n    log_f0s_mean = log_f0s_concatenated.mean()\n    log_f0s_std = log_f0s_concatenated.std()\n\n    return log_f0s_mean, log_f0s_std\n\ndef pitch_conversion(f0, mean_log_src, std_log_src, mean_log_target, std_log_target):\n\n    # Logarithm Gaussian normalization for Pitch Conversions\n    f0_converted = np.exp((np.ma.log(f0) - mean_log_src) / std_log_src * std_log_target + mean_log_target)\n\n    return f0_converted\n\ndef wavs_to_specs(wavs, n_fft = 1024, hop_length = None):\n\n    stfts = list()\n    for wav in wavs:\n        stft = librosa.stft(wav, n_fft = n_fft, hop_length = hop_length)\n        stfts.append(stft)\n\n    return stfts\n\n\ndef wavs_to_mfccs(wavs, sr, n_fft = 1024, hop_length = None, n_mels = 128, n_mfcc = 24):\n\n    mfccs = list()\n    for wav in wavs:\n        mfcc = librosa.feature.mfcc(y = wav, sr = sr, n_fft = n_fft, hop_length = hop_length, n_mels = n_mels, n_mfcc = n_mfcc)\n        mfccs.append(mfcc)\n\n    return mfccs\n\n\ndef mfccs_normalization(mfccs):\n\n    mfccs_concatenated = np.concatenate(mfccs, axis = 1)\n    mfccs_mean = np.mean(mfccs_concatenated, axis = 1, keepdims = True)\n    mfccs_std = np.std(mfccs_concatenated, axis = 1, keepdims = True)\n\n    mfccs_normalized = list()\n    for mfcc in mfccs:\n        mfccs_normalized.append((mfcc - mfccs_mean) / mfccs_std)\n    \n    return mfccs_normalized, mfccs_mean, mfccs_std\n\n\ndef sample_train_data(dataset_A, dataset_B, n_frames = 128):\n\n    num_samples = min(len(dataset_A), len(dataset_B))\n    train_data_A_idx = np.arange(len(dataset_A))\n    train_data_B_idx = np.arange(len(dataset_B))\n    np.random.shuffle(train_data_A_idx)\n    np.random.shuffle(train_data_B_idx)\n    train_data_A_idx_subset = train_data_A_idx[:num_samples]\n    train_data_B_idx_subset = train_data_B_idx[:num_samples]\n\n    train_data_A = list()\n    train_data_B = list()\n\n    for idx_A, idx_B in zip(train_data_A_idx_subset, train_data_B_idx_subset):\n        data_A = dataset_A[idx_A]\n        frames_A_total = data_A.shape[1]\n        assert frames_A_total >= n_frames\n        start_A = np.random.randint(frames_A_total - n_frames + 1)\n        end_A = start_A + n_frames\n        train_data_A.append(data_A[:,start_A:end_A])\n\n        data_B = dataset_B[idx_B]\n        frames_B_total = data_B.shape[1]\n        assert frames_B_total >= n_frames\n        start_B = np.random.randint(frames_B_total - n_frames + 1)\n        end_B = start_B + n_frames\n        train_data_B.append(data_B[:,start_B:end_B])\n\n    train_data_A = np.array(train_data_A)\n    train_data_B = np.array(train_data_B)\n\n    return train_data_A, train_data_B"
  }
]