[ { "path": ".dockerignore", "content": "# The .dockerignore file excludes files from the container build process.\n#\n# https://docs.docker.com/engine/reference/builder/#dockerignore-file\n\n# Exclude Git files\n.git\n.github\n.gitignore\n\n# Exclude Python cache files\n__pycache__\n.mypy_cache\n.pytest_cache\n.ruff_cache\n\n# Exclude Python virtual environment\n/venv\n" }, { "path": ".gitignore", "content": "__pycache__/\n*.py[cod]\n*$py.class\n*.egg-info\n.pytest_cache\n.ipynb_checkpoints\n\nthumbs.db\n.DS_Store\n.idea\n*.log\n*.pdf\n*.mkv\n*.mp4\n*.png\n*.wav\n*.mp3\n*.pth\n*.th\n*.json\n\n*durip*\n*rtx*\n*l40*\n*a40*\n\nsrc/audiocraft\n\n!/demo/\n!/demo/*\n/demo/temp/*.txt\n!/demo/temp/84_121550_000074_000000.txt\n.cog/tmp/*" }, { "path": "Dockerfile", "content": "FROM jupyter/base-notebook:python-3.9.13\n\nUSER root\n\n# Install OS dependencies\nRUN apt-get update && apt-get install -y git-core ffmpeg espeak-ng && \\\n apt-get clean && \\\n rm -rf /var/lib/apt/lists/*\n\n# Update Conda, create the voicecraft environment, and install dependencies\nRUN conda update -y -n base -c conda-forge conda && \\\n conda create -y -n voicecraft python=3.9.16 && \\\n conda run -n voicecraft conda install -y -c conda-forge montreal-forced-aligner=2.2.17 openfst=1.8.2 kaldi=5.5.1068 && \\\n conda run -n voicecraft mfa model download dictionary english_us_arpa && \\\n conda run -n voicecraft mfa model download acoustic english_us_arpa && \\\n conda run -n voicecraft pip install -e git+https://github.com/facebookresearch/audiocraft.git@c5157b5bf14bf83449c17ea1eeb66c19fb4bc7f0#egg=audiocraft && \\\n conda run -n voicecraft pip install xformers==0.0.22 && \\\n conda run -n voicecraft pip install torch==2.0.1 && \\\n conda run -n voicecraft pip install torchaudio==2.0.2 && \\\n conda run -n voicecraft pip install tensorboard==2.16.2 && \\\n conda run -n voicecraft pip install phonemizer==3.2.1 && \\\n conda run -n voicecraft pip install datasets==2.16.0 && \\\n conda run -n voicecraft pip install torchmetrics==0.11.1 && \\\n conda run -n voicecraft pip install huggingface_hub==0.22.2\n \n\n# Install the Jupyter kernel\nRUN conda install -n voicecraft ipykernel --update-deps --force-reinstall -y && \\\n conda run -n voicecraft python -m ipykernel install --name=voicecraft" }, { "path": "LICENSE-CODE", "content": "Attribution-NonCommercial-ShareAlike 4.0 International\n\n=======================================================================\n\nCreative Commons Corporation (\"Creative Commons\") is not a law firm and\ndoes not provide legal services or legal advice. Distribution of\nCreative Commons public licenses does not create a lawyer-client or\nother relationship. Creative Commons makes its licenses and related\ninformation available on an \"as-is\" basis. Creative Commons gives no\nwarranties regarding its licenses, any material licensed under their\nterms and conditions, or any related information. Creative Commons\ndisclaims all liability for damages resulting from their use to the\nfullest extent possible.\n\nUsing Creative Commons Public Licenses\n\nCreative Commons public licenses provide a standard set of terms and\nconditions that creators and other rights holders may use to share\noriginal works of authorship and other material subject to copyright\nand certain other rights specified in the public license below. The\nfollowing considerations are for informational purposes only, are not\nexhaustive, and do not form part of our licenses.\n\n Considerations for licensors: Our public licenses are\n intended for use by those authorized to give the public\n permission to use material in ways otherwise restricted by\n copyright and certain other rights. Our licenses are\n irrevocable. Licensors should read and understand the terms\n and conditions of the license they choose before applying it.\n Licensors should also secure all rights necessary before\n applying our licenses so that the public can reuse the\n material as expected. Licensors should clearly mark any\n material not subject to the license. This includes other CC-\n licensed material, or material used under an exception or\n limitation to copyright. More considerations for licensors:\n wiki.creativecommons.org/Considerations_for_licensors\n\n Considerations for the public: By using one of our public\n licenses, a licensor grants the public permission to use the\n licensed material under specified terms and conditions. If\n the licensor's permission is not necessary for any reason--for\n example, because of any applicable exception or limitation to\n copyright--then that use is not regulated by the license. Our\n licenses grant only permissions under copyright and certain\n other rights that a licensor has authority to grant. Use of\n the licensed material may still be restricted for other\n reasons, including because others have copyright or other\n rights in the material. A licensor may make special requests,\n such as asking that all changes be marked or described.\n Although not required by our licenses, you are encouraged to\n respect those requests where reasonable. More considerations\n for the public:\n wiki.creativecommons.org/Considerations_for_licensees\n\n=======================================================================\n\nCreative Commons Attribution-NonCommercial-ShareAlike 4.0 International\nPublic License\n\nBy exercising the Licensed Rights (defined below), You accept and agree\nto be bound by the terms and conditions of this Creative Commons\nAttribution-NonCommercial-ShareAlike 4.0 International Public License\n(\"Public License\"). To the extent this Public License may be\ninterpreted as a contract, You are granted the Licensed Rights in\nconsideration of Your acceptance of these terms and conditions, and the\nLicensor grants You such rights in consideration of benefits the\nLicensor receives from making the Licensed Material available under\nthese terms and conditions.\n\n\nSection 1 -- Definitions.\n\n a. Adapted Material means material subject to Copyright and Similar\n Rights that is derived from or based upon the Licensed Material\n and in which the Licensed Material is translated, altered,\n arranged, transformed, or otherwise modified in a manner requiring\n permission under the Copyright and Similar Rights held by the\n Licensor. For purposes of this Public License, where the Licensed\n Material is a musical work, performance, or sound recording,\n Adapted Material is always produced where the Licensed Material is\n synched in timed relation with a moving image.\n\n b. Adapter's License means the license You apply to Your Copyright\n and Similar Rights in Your contributions to Adapted Material in\n accordance with the terms and conditions of this Public License.\n\n c. BY-NC-SA Compatible License means a license listed at\n creativecommons.org/compatiblelicenses, approved by Creative\n Commons as essentially the equivalent of this Public License.\n\n d. Copyright and Similar Rights means copyright and/or similar rights\n closely related to copyright including, without limitation,\n performance, broadcast, sound recording, and Sui Generis Database\n Rights, without regard to how the rights are labeled or\n categorized. For purposes of this Public License, the rights\n specified in Section 2(b)(1)-(2) are not Copyright and Similar\n Rights.\n\n e. Effective Technological Measures means those measures that, in the\n absence of proper authority, may not be circumvented under laws\n fulfilling obligations under Article 11 of the WIPO Copyright\n Treaty adopted on December 20, 1996, and/or similar international\n agreements.\n\n f. Exceptions and Limitations means fair use, fair dealing, and/or\n any other exception or limitation to Copyright and Similar Rights\n that applies to Your use of the Licensed Material.\n\n g. License Elements means the license attributes listed in the name\n of a Creative Commons Public License. The License Elements of this\n Public License are Attribution, NonCommercial, and ShareAlike.\n\n h. Licensed Material means the artistic or literary work, database,\n or other material to which the Licensor applied this Public\n License.\n\n i. Licensed Rights means the rights granted to You subject to the\n terms and conditions of this Public License, which are limited to\n all Copyright and Similar Rights that apply to Your use of the\n Licensed Material and that the Licensor has authority to license.\n\n j. Licensor means the individual(s) or entity(ies) granting rights\n under this Public License.\n\n k. NonCommercial means not primarily intended for or directed towards\n commercial advantage or monetary compensation. For purposes of\n this Public License, the exchange of the Licensed Material for\n other material subject to Copyright and Similar Rights by digital\n file-sharing or similar means is NonCommercial provided there is\n no payment of monetary compensation in connection with the\n exchange.\n\n l. Share means to provide material to the public by any means or\n process that requires permission under the Licensed Rights, such\n as reproduction, public display, public performance, distribution,\n dissemination, communication, or importation, and to make material\n available to the public including in ways that members of the\n public may access the material from a place and at a time\n individually chosen by them.\n\n m. Sui Generis Database Rights means rights other than copyright\n resulting from Directive 96/9/EC of the European Parliament and of\n the Council of 11 March 1996 on the legal protection of databases,\n as amended and/or succeeded, as well as other essentially\n equivalent rights anywhere in the world.\n\n n. You means the individual or entity exercising the Licensed Rights\n under this Public License. Your has a corresponding meaning.\n\n\nSection 2 -- Scope.\n\n a. License grant.\n\n 1. Subject to the terms and conditions of this Public License,\n the Licensor hereby grants You a worldwide, royalty-free,\n non-sublicensable, non-exclusive, irrevocable license to\n exercise the Licensed Rights in the Licensed Material to:\n\n a. reproduce and Share the Licensed Material, in whole or\n in part, for NonCommercial purposes only; and\n\n b. produce, reproduce, and Share Adapted Material for\n NonCommercial purposes only.\n\n 2. Exceptions and Limitations. For the avoidance of doubt, where\n Exceptions and Limitations apply to Your use, this Public\n License does not apply, and You do not need to comply with\n its terms and conditions.\n\n 3. Term. The term of this Public License is specified in Section\n 6(a).\n\n 4. Media and formats; technical modifications allowed. The\n Licensor authorizes You to exercise the Licensed Rights in\n all media and formats whether now known or hereafter created,\n and to make technical modifications necessary to do so. The\n Licensor waives and/or agrees not to assert any right or\n authority to forbid You from making technical modifications\n necessary to exercise the Licensed Rights, including\n technical modifications necessary to circumvent Effective\n Technological Measures. For purposes of this Public License,\n simply making modifications authorized by this Section 2(a)\n (4) never produces Adapted Material.\n\n 5. Downstream recipients.\n\n a. Offer from the Licensor -- Licensed Material. Every\n recipient of the Licensed Material automatically\n receives an offer from the Licensor to exercise the\n Licensed Rights under the terms and conditions of this\n Public License.\n\n b. Additional offer from the Licensor -- Adapted Material.\n Every recipient of Adapted Material from You\n automatically receives an offer from the Licensor to\n exercise the Licensed Rights in the Adapted Material\n under the conditions of the Adapter's License You apply.\n\n c. No downstream restrictions. You may not offer or impose\n any additional or different terms or conditions on, or\n apply any Effective Technological Measures to, the\n Licensed Material if doing so restricts exercise of the\n Licensed Rights by any recipient of the Licensed\n Material.\n\n 6. No endorsement. Nothing in this Public License constitutes or\n may be construed as permission to assert or imply that You\n are, or that Your use of the Licensed Material is, connected\n with, or sponsored, endorsed, or granted official status by,\n the Licensor or others designated to receive attribution as\n provided in Section 3(a)(1)(A)(i).\n\n b. Other rights.\n\n 1. Moral rights, such as the right of integrity, are not\n licensed under this Public License, nor are publicity,\n privacy, and/or other similar personality rights; however, to\n the extent possible, the Licensor waives and/or agrees not to\n assert any such rights held by the Licensor to the limited\n extent necessary to allow You to exercise the Licensed\n Rights, but not otherwise.\n\n 2. Patent and trademark rights are not licensed under this\n Public License.\n\n 3. To the extent possible, the Licensor waives any right to\n collect royalties from You for the exercise of the Licensed\n Rights, whether directly or through a collecting society\n under any voluntary or waivable statutory or compulsory\n licensing scheme. In all other cases the Licensor expressly\n reserves any right to collect such royalties, including when\n the Licensed Material is used other than for NonCommercial\n purposes.\n\n\nSection 3 -- License Conditions.\n\nYour exercise of the Licensed Rights is expressly made subject to the\nfollowing conditions.\n\n a. Attribution.\n\n 1. If You Share the Licensed Material (including in modified\n form), You must:\n\n a. retain the following if it is supplied by the Licensor\n with the Licensed Material:\n\n i. identification of the creator(s) of the Licensed\n Material and any others designated to receive\n attribution, in any reasonable manner requested by\n the Licensor (including by pseudonym if\n designated);\n\n ii. a copyright notice;\n\n iii. a notice that refers to this Public License;\n\n iv. a notice that refers to the disclaimer of\n warranties;\n\n v. a URI or hyperlink to the Licensed Material to the\n extent reasonably practicable;\n\n b. indicate if You modified the Licensed Material and\n retain an indication of any previous modifications; and\n\n c. indicate the Licensed Material is licensed under this\n Public License, and include the text of, or the URI or\n hyperlink to, this Public License.\n\n 2. You may satisfy the conditions in Section 3(a)(1) in any\n reasonable manner based on the medium, means, and context in\n which You Share the Licensed Material. For example, it may be\n reasonable to satisfy the conditions by providing a URI or\n hyperlink to a resource that includes the required\n information.\n 3. If requested by the Licensor, You must remove any of the\n information required by Section 3(a)(1)(A) to the extent\n reasonably practicable.\n\n b. ShareAlike.\n\n In addition to the conditions in Section 3(a), if You Share\n Adapted Material You produce, the following conditions also apply.\n\n 1. The Adapter's License You apply must be a Creative Commons\n license with the same License Elements, this version or\n later, or a BY-NC-SA Compatible License.\n\n 2. You must include the text of, or the URI or hyperlink to, the\n Adapter's License You apply. You may satisfy this condition\n in any reasonable manner based on the medium, means, and\n context in which You Share Adapted Material.\n\n 3. You may not offer or impose any additional or different terms\n or conditions on, or apply any Effective Technological\n Measures to, Adapted Material that restrict exercise of the\n rights granted under the Adapter's License You apply.\n\n\nSection 4 -- Sui Generis Database Rights.\n\nWhere the Licensed Rights include Sui Generis Database Rights that\napply to Your use of the Licensed Material:\n\n a. for the avoidance of doubt, Section 2(a)(1) grants You the right\n to extract, reuse, reproduce, and Share all or a substantial\n portion of the contents of the database for NonCommercial purposes\n only;\n\n b. if You include all or a substantial portion of the database\n contents in a database in which You have Sui Generis Database\n Rights, then the database in which You have Sui Generis Database\n Rights (but not its individual contents) is Adapted Material,\n including for purposes of Section 3(b); and\n\n c. You must comply with the conditions in Section 3(a) if You Share\n all or a substantial portion of the contents of the database.\n\nFor the avoidance of doubt, this Section 4 supplements and does not\nreplace Your obligations under this Public License where the Licensed\nRights include other Copyright and Similar Rights.\n\n\nSection 5 -- Disclaimer of Warranties and Limitation of Liability.\n\n a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE\n EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS\n AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF\n ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,\n IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,\n WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR\n PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,\n ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT\n KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT\n ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.\n\n b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE\n TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,\n NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,\n INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,\n COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR\n USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN\n ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR\n DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR\n IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.\n\n c. The disclaimer of warranties and limitation of liability provided\n above shall be interpreted in a manner that, to the extent\n possible, most closely approximates an absolute disclaimer and\n waiver of all liability.\n\n\nSection 6 -- Term and Termination.\n\n a. This Public License applies for the term of the Copyright and\n Similar Rights licensed here. However, if You fail to comply with\n this Public License, then Your rights under this Public License\n terminate automatically.\n\n b. Where Your right to use the Licensed Material has terminated under\n Section 6(a), it reinstates:\n\n 1. automatically as of the date the violation is cured, provided\n it is cured within 30 days of Your discovery of the\n violation; or\n\n 2. upon express reinstatement by the Licensor.\n\n For the avoidance of doubt, this Section 6(b) does not affect any\n right the Licensor may have to seek remedies for Your violations\n of this Public License.\n\n c. For the avoidance of doubt, the Licensor may also offer the\n Licensed Material under separate terms or conditions or stop\n distributing the Licensed Material at any time; however, doing so\n will not terminate this Public License.\n\n d. Sections 1, 5, 6, 7, and 8 survive termination of this Public\n License.\n\n\nSection 7 -- Other Terms and Conditions.\n\n a. The Licensor shall not be bound by any additional or different\n terms or conditions communicated by You unless expressly agreed.\n\n b. Any arrangements, understandings, or agreements regarding the\n Licensed Material not stated herein are separate from and\n independent of the terms and conditions of this Public License.\n\n\nSection 8 -- Interpretation.\n\n a. For the avoidance of doubt, this Public License does not, and\n shall not be interpreted to, reduce, limit, restrict, or impose\n conditions on any use of the Licensed Material that could lawfully\n be made without permission under this Public License.\n\n b. To the extent possible, if any provision of this Public License is\n deemed unenforceable, it shall be automatically reformed to the\n minimum extent necessary to make it enforceable. If the provision\n cannot be reformed, it shall be severed from this Public License\n without affecting the enforceability of the remaining terms and\n conditions.\n\n c. No term or condition of this Public License will be waived and no\n failure to comply consented to unless expressly agreed to by the\n Licensor.\n\n d. Nothing in this Public License constitutes or may be interpreted\n as a limitation upon, or waiver of, any privileges and immunities\n that apply to the Licensor or You, including from the legal\n processes of any jurisdiction or authority.\n\n=======================================================================\n\nCreative Commons is not a party to its public\nlicenses. Notwithstanding, Creative Commons may elect to apply one of\nits public licenses to material it publishes and in those instances\nwill be considered the “Licensor.” The text of the Creative Commons\npublic licenses is dedicated to the public domain under the CC0 Public\nDomain Dedication. Except for the limited purpose of indicating that\nmaterial is shared under a Creative Commons public license or as\notherwise permitted by the Creative Commons policies published at\ncreativecommons.org/policies, Creative Commons does not authorize the\nuse of the trademark \"Creative Commons\" or any other trademark or logo\nof Creative Commons without its prior written consent including,\nwithout limitation, in connection with any unauthorized modifications\nto any of its public licenses or any other arrangements,\nunderstandings, or agreements concerning use of licensed material. For\nthe avoidance of doubt, this paragraph does not form part of the\npublic licenses.\n\nCreative Commons may be contacted at creativecommons.org.\n" }, { "path": "LICENSE-MODEL", "content": "Coqui Public Model License 1.0.0\nhttps://coqui.ai/cpml.txt\n\nThis license allows only non-commercial use of a machine learning model and its outputs.\n\nAcceptance\nIn order to get any license under these terms, you must agree to them as both strict obligations and conditions to all your licenses.\n\nLicenses\nThe licensor grants you a copyright license to do everything you might do with the model that would otherwise infringe the licensor's copyright in it, for any non-commercial purpose. The licensor grants you a patent license that covers patent claims the licensor can license, or becomes able to license, that you would infringe by using the model in the form provided by the licensor, for any non-commercial purpose.\n\nNon-commercial Purpose\nNon-commercial purposes include any of the following uses of the model or its output, but only so far as you do not receive any direct or indirect payment arising from the use of the model or its output.\n\nPersonal use for research, experiment, and testing for the benefit of public knowledge, personal study, private entertainment, hobby projects, amateur pursuits, or religious observance.\nUse by commercial or for-profit entities for testing, evaluation, or non-commercial research and development. Use of the model to train other models for commercial use is not a non-commercial purpose.\nUse by any charitable organization for charitable purposes, or for testing or evaluation. Use for revenue-generating activity, including projects directly funded by government grants, is not a non-commercial purpose.\nNotices\nYou must ensure that anyone who gets a copy of any part of the model, or any modification of the model, or their output, from you also gets a copy of these terms or the URL for them above.\n\nNo Other Rights\nThese terms do not allow you to sublicense or transfer any of your licenses to anyone else, or prevent the licensor from granting licenses to anyone else. These terms do not imply any other licenses.\n\nPatent Defense\nIf you make any written claim that the model infringes or contributes to infringement of any patent, your licenses for the model granted under these terms ends immediately. If your company makes such a claim, your patent license ends immediately for work on behalf of your company.\n\nViolations\nThe first time you are notified in writing that you have violated any of these terms, or done anything with the model or its output that is not covered by your licenses, your licenses can nonetheless continue if you come into full compliance with these terms, and take practical steps to correct past violations, within 30 days of receiving notice. Otherwise, all your licenses end immediately.\n\nNo Liability\nAS FAR AS THE LAW ALLOWS, THE MODEL AND ITS OUTPUT COME AS IS, WITHOUT ANY WARRANTY OR CONDITION, AND THE LICENSOR WILL NOT BE LIABLE TO YOU FOR ANY DAMAGES ARISING OUT OF THESE TERMS OR THE USE OR NATURE OF THE MODEL OR ITS OUTPUT, UNDER ANY KIND OF LEGAL CLAIM. IF THIS PROVISION IS NOT ENFORCEABLE IN YOUR JURISDICTION, YOUR LICENSES ARE VOID.\n\nDefinitions\nThe licensor is the individual or entity offering these terms, and the model is the model the licensor makes available under these terms, including any documentation or similar information about the model.\n\nYou refers to the individual or entity agreeing to these terms.\n\nYour company is any legal entity, sole proprietorship, or other kind of organization that you work for, plus all organizations that have control over, are under the control of, or are under common control with that organization. Control means ownership of substantially all the assets of an entity, or the power to direct its management and policies by vote, contract, or otherwise. Control can be direct or indirect.\n\nYour licenses are all the licenses granted to you under these terms.\n\nUse means anything you do with the model or its output requiring one of your licenses." }, { "path": "README.md", "content": "# VoiceCraft: Zero-Shot Speech Editing and Text-to-Speech in the Wild\n[![Paper](https://img.shields.io/badge/arXiv-2403.16973-brightgreen.svg?style=flat-square)](https://arxiv.org/pdf/2403.16973.pdf) [![HuggingFace](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue)](https://huggingface.co/spaces/pyp1/VoiceCraft_gradio) [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1IOjpglQyMTO2C3Y94LD9FY0Ocn-RJRg6?usp=sharing) [![Replicate](https://replicate.com/cjwbw/voicecraft/badge)](https://replicate.com/cjwbw/voicecraft) [![YouTube demo](https://img.shields.io/youtube/views/eikybOi8iwU)](https://youtu.be/eikybOi8iwU) [![Demo page](https://img.shields.io/badge/Audio_Samples-blue?logo=Github&style=flat-square)](https://jasonppy.github.io/VoiceCraft_web/)\n\n\n### TL;DR\nVoiceCraft is a token infilling neural codec language model, that achieves state-of-the-art performance on both **speech editing** and **zero-shot text-to-speech (TTS)** on in-the-wild data including audiobooks, internet videos, and podcasts.\n\nTo clone or edit an unseen voice, VoiceCraft needs only a few seconds of reference.\n\n## How to run inference\nThere are three ways (besides running Gradio in Colab):\n\n1. More flexible inference beyond Gradio UI in Google Colab. see [quickstart colab](#quickstart-colab)\n2. with docker. see [quickstart docker](#quickstart-docker)\n3. without docker. see [environment setup](#environment-setup). You can also run gradio locally if you choose this option\n4. As a standalone script that you can easily integrate into other projects.\nsee [quickstart command line](#quickstart-command-line).\n\nWhen you are inside the docker image or you have installed all dependencies, Checkout [`inference_tts.ipynb`](./inference_tts.ipynb).\n\nIf you want to do model development such as training/finetuning, I recommend following [envrionment setup](#environment-setup) and [training](#training).\n\n## News\n:star: 03/15/2025: change inference sampling from topp=1 to topk=40 massively improve editing and TTS performance\n\n:star: 04/22/2024: 330M/830M TTS Enhanced Models are up [here](https://huggingface.co/pyp1), load them through [`gradio_app.py`](./gradio_app.py) or [`inference_tts.ipynb`](./inference_tts.ipynb)! Replicate demo is up, major thanks to [@chenxwh](https://github.com/chenxwh)!\n\n:star: 04/11/2024: VoiceCraft Gradio is now available on HuggingFace Spaces [here](https://huggingface.co/spaces/pyp1/VoiceCraft_gradio)! Major thanks to [@zuev-stepan](https://github.com/zuev-stepan), [@Sewlell](https://github.com/Sewlell), [@pgsoar](https://github.com/pgosar) [@Ph0rk0z](https://github.com/Ph0rk0z).\n\n:star: 04/05/2024: I finetuned giga330M with the TTS objective on gigaspeech and 1/5 of librilight. Weights are [here](https://huggingface.co/pyp1/VoiceCraft/tree/main). Make sure maximal prompt + generation length <= 16 seconds (due to our limited compute, we had to drop utterances longer than 16s in training data). Even stronger models forthcomming, stay tuned!\n\n:star: 03/28/2024: Model weights for giga330M and giga830M are up on HuggingFace🤗 [here](https://huggingface.co/pyp1/VoiceCraft/tree/main)!\n\n## TODO\n- [x] Codebase upload\n- [x] Environment setup\n- [x] Inference demo for speech editing and TTS\n- [x] Training guidance\n- [x] RealEdit dataset and training manifest\n- [x] Model weights\n- [x] Better guidance on training/finetuning\n- [x] Colab notebooks\n- [x] HuggingFace Spaces demo\n- [x] Command line\n- [ ] Improve efficiency\n\n## QuickStart Colab\n\n:star: To try out speech editing or TTS Inference with VoiceCraft, the simplest way is using Google Colab.\nInstructions to run are on the Colab itself.\n\n1. To try [Speech Editing](https://colab.research.google.com/drive/1FV7EC36dl8UioePY1xXijXTMl7X47kR_?usp=sharing)\n2. To try [TTS Inference](https://colab.research.google.com/drive/1lch_6it5-JpXgAQlUTRRI2z2_rk5K67Z?usp=sharing)\n\n## QuickStart Command Line\n\n:star: To use it as a standalone script, check out tts_demo.py and speech_editing_demo.py.\nBe sure to first [setup your environment](#environment-setup).\nWithout arguments, they will run the standard demo arguments used as an example elsewhere\nin this repository. You can use the command line arguments to specify unique input audios,\ntarget transcripts, and inference hyperparameters. Run the help command for more information:\n`python3 tts_demo.py -h`\n\n## QuickStart Docker\n:star: To try out TTS inference with VoiceCraft, you can also use docker. Thank [@ubergarm](https://github.com/ubergarm) and [@jayc88](https://github.com/jay-c88) for making this happen.\n\nTested on Linux and Windows and should work with any host with docker installed.\n```bash\n# 1. clone the repo on in a directory on a drive with plenty of free space\ngit clone git@github.com:jasonppy/VoiceCraft.git\ncd VoiceCraft\n\n# 2. assumes you have docker installed with nvidia container container-toolkit (windows has this built into the driver)\n# https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/1.13.5/install-guide.html\n# sudo apt-get install -y nvidia-container-toolkit-base || yay -Syu nvidia-container-toolkit || echo etc...\n\n# 3. First build the docker image\ndocker build --tag \"voicecraft\" .\n\n# 4. Try to start an existing container otherwise create a new one passing in all GPUs\n./start-jupyter.sh # linux\nstart-jupyter.bat # windows\n\n# 5. now open a webpage on the host box to the URL shown at the bottom of:\ndocker logs jupyter\n\n# 6. optionally look inside from another terminal\ndocker exec -it jupyter /bin/bash\nexport USER=(your_linux_username_used_above)\nexport HOME=/home/$USER\nsudo apt-get update\n\n# 7. confirm video card(s) are visible inside container\nnvidia-smi\n\n# 8. Now in browser, open inference_tts.ipynb and work through one cell at a time\necho GOOD LUCK\n```\n\n## Environment setup\n```bash\nconda create -n voicecraft python=3.9.16\nconda activate voicecraft\n\npip install -e git+https://github.com/facebookresearch/audiocraft.git@c5157b5bf14bf83449c17ea1eeb66c19fb4bc7f0#egg=audiocraft\npip install xformers==0.0.22\npip install torchaudio==2.0.2 torch==2.0.1 # this assumes your system is compatible with CUDA 11.7, otherwise checkout https://pytorch.org/get-started/previous-versions/#v201\napt-get install ffmpeg # if you don't already have ffmpeg installed\napt-get install espeak-ng # backend for the phonemizer installed below\npip install tensorboard==2.16.2\npip install phonemizer==3.2.1\npip install datasets==2.16.0\npip install torchmetrics==0.11.1\npip install huggingface_hub==0.22.2\n# install MFA for getting forced-alignment, this could take a few minutes\nconda install -c conda-forge montreal-forced-aligner=2.2.17 openfst=1.8.2 kaldi=5.5.1068\n# install MFA english dictionary and model\nmfa model download dictionary english_us_arpa\nmfa model download acoustic english_us_arpa\n# pip install huggingface_hub\n# conda install pocl # above gives an warning for installing pocl, not sure if really need this\n\n# to run ipynb\nconda install -n voicecraft ipykernel --no-deps --force-reinstall\n```\n\nIf you have encountered version issues when running things, checkout [environment.yml](./environment.yml) for exact matching.\n\n## Inference Examples\nCheckout [`inference_speech_editing.ipynb`](./inference_speech_editing.ipynb) and [`inference_tts.ipynb`](./inference_tts.ipynb)\n\n## Gradio\n### Run in colab\n\n[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1IOjpglQyMTO2C3Y94LD9FY0Ocn-RJRg6?usp=sharing)\n\n### Run locally\nAfter environment setup install additional dependencies:\n```bash\napt-get install -y espeak espeak-data libespeak1 libespeak-dev\napt-get install -y festival*\napt-get install -y build-essential\napt-get install -y flac libasound2-dev libsndfile1-dev vorbis-tools\napt-get install -y libxml2-dev libxslt-dev zlib1g-dev\npip install -r gradio_requirements.txt\n```\n\nRun gradio server from terminal or [`gradio_app.ipynb`](./gradio_app.ipynb):\n```bash\npython gradio_app.py\n```\nIt is ready to use on [default url](http://127.0.0.1:7860).\n\n### How to use it\n1. (optionally) Select models\n2. Load models\n3. Transcribe\n4. (optionally) Tweak some parameters\n5. Run\n6. (optionally) Rerun part-by-part in Long TTS mode\n\n### Some features\nSmart transcript: write only what you want to generate\n\nTTS mode: Zero-shot TTS\n\nEdit mode: Speech editing\n\nLong TTS mode: Easy TTS on long texts\n\n\n## Training\nTo train an VoiceCraft model, you need to prepare the following parts:\n1. utterances and their transcripts\n2. encode the utterances into codes using e.g. Encodec\n3. convert transcripts into phoneme sequence, and a phoneme set (we named it vocab.txt)\n4. manifest (i.e. metadata)\n\nStep 1,2,3 are handled in [./data/phonemize_encodec_encode_hf.py](./data/phonemize_encodec_encode_hf.py), where\n1. Gigaspeech is downloaded through HuggingFace. Note that you need to sign an agreement in order to download the dataset (it needs your auth token)\n2. phoneme sequence and encodec codes are also extracted using the script.\n\nAn example run:\n\n```bash\nconda activate voicecraft\nexport CUDA_VISIBLE_DEVICES=0\ncd ./data\npython phonemize_encodec_encode_hf.py \\\n--dataset_size xs \\\n--download_to path/to/store_huggingface_downloads \\\n--save_dir path/to/store_extracted_codes_and_phonemes \\\n--encodec_model_path path/to/encodec_model \\\n--mega_batch_size 120 \\\n--batch_size 32 \\\n--max_len 30000\n```\nwhere encodec_model_path is avaliable [here](https://huggingface.co/pyp1/VoiceCraft). This model is trained on Gigaspeech XL, it has 56M parameters, 4 codebooks, each codebook has 2048 codes. Details are described in our [paper](https://jasonppy.github.io/assets/pdfs/VoiceCraft.pdf). If you encounter OOM during extraction, try decrease the batch_size and/or max_len.\nThe extracted codes, phonemes, and vocab.txt will be stored at `path/to/store_extracted_codes_and_phonemes/${dataset_size}/{encodec_16khz_4codebooks,phonemes,vocab.txt}`.\n\nAs for manifest, please download train.txt and validation.txt from [here](https://huggingface.co/datasets/pyp1/VoiceCraft_RealEdit/tree/main), and put them under `path/to/store_extracted_codes_and_phonemes/manifest/`. Please also download vocab.txt from [here](https://huggingface.co/datasets/pyp1/VoiceCraft_RealEdit/tree/main) if you want to use our pretrained VoiceCraft model (so that the phoneme-to-token matching is the same).\n\nNow, you are good to start training!\n\n```bash\nconda activate voicecraft\ncd ./z_scripts\nbash e830M.sh\n```\n\nIt's the same procedure to prepare your own custom dataset. Make sure that if\n\n## Finetuning\nYou also need to do step 1-4 as Training, and I recommend to use AdamW for optimization if you finetune a pretrained model for better stability. checkout script `./z_scripts/e830M_ft.sh`.\n\nIf your dataset introduce new phonemes (which is very likely) that doesn't exist in the giga checkpoint, make sure you combine the original phonemes with the phoneme from your data when construction vocab. And you need to adjust `--text_vocab_size` and `--text_pad_token` so that the former is bigger than or equal to you vocab size, and the latter has the same value as `--text_vocab_size` (i.e. `--text_pad_token` is always the last token). Also since the text embedding are now of a different size, make sure you modify the weights loading part so that I won't crash (you could skip loading `text_embedding` or only load the existing part, and randomly initialize the new)\n\n## License\nThe codebase is under CC BY-NC-SA 4.0 ([LICENSE-CODE](./LICENSE-CODE)), and the model weights are under Coqui Public Model License 1.0.0 ([LICENSE-MODEL](./LICENSE-MODEL)). Note that we use some of the code from other repository that are under different licenses: `./models/codebooks_patterns.py` is under MIT license; `./models/modules`, `./steps/optim.py`, `data/tokenizer.py` are under Apache License, Version 2.0; the phonemizer we used is under GNU 3.0 License.\n\n## Acknowledgement\nWe thank Feiteng for his [VALL-E reproduction](https://github.com/lifeiteng/vall-e), and we thank audiocraft team for open-sourcing [encodec](https://github.com/facebookresearch/audiocraft).\n\n## Citation\n```\n@article{peng2024voicecraft,\n author = {Peng, Puyuan and Huang, Po-Yao and Mohamed, Abdelrahman and Harwath, David},\n title = {VoiceCraft: Zero-Shot Speech Editing and Text-to-Speech in the Wild},\n journal = {arXiv},\n year = {2024},\n}\n```\n\n## Disclaimer\nAny organization or individual is prohibited from using any technology mentioned in this paper to generate or edit someone's speech without his/her consent, including but not limited to government leaders, political figures, and celebrities. If you do not comply with this item, you could be in violation of copyright laws.\n\n" }, { "path": "RealEdit.txt", "content": "wav_fn\torig_transcript\tnew_transcript\torig_masked_span\tnew_masked_span\ttype\nYOU1000000102_S0000137.wav\tif i had never dropped out. i would have never dropped in on that calligraphy class and personal computers might not have the wonderful typography that they do.\tif i had never dropped out. i would have never stopped by that calligraphy class and personal computers might not have the wonderful typography that they do.\t10,12\t10,11\tsubstitution\nYOU1000000124_S0000174.wav\tso people in symbolic era. i mean we all agree that symbols come in and symbols come out, okay.\tso people in symbolic era. i mean we all agree that signals go in and symbols come out, okay.\t11,12\t11,12\tsubstitution\nYOU1000000006_S0000016.wav\tand then we can actually go through and generate a lead. so we've got this first call to action here and this is sending them to a landing page.\tand then we can actually go through and generate a lead. so we've got this first action here and this is sending them to a landing page.\t16,17\t15,16\tdeletion\nYOU1000000149_S0000172.wav\tthe dry shampoo honestly like weirdly styles it enough to where i'm like satisfied with it but i do like to add a little bit of hair spray.\tthe dry shampoo honestly like weirdly styles it enough to where i'm like satisfied with it but sometimes it needs a little extra i do like to add a little bit of hair spray.\t16,17\t17,22\tinsertion\nYOU1000000102_S0000031.wav\tsteve also cofounded pixar animation studios. which has revolutionized the film industry in it short history with brilliant use of technology.\tsteve also cofounded pixar animation studios. which has revolutionized the film industry in it short history with films like toy story that showcase brilliant use of technology.\t16,17\t17,22\tinsertion\nYOU1000000148_S0000041.wav\tso i was actually happy to wear such a heavy dress and to be able to wear proper wool, you know mountain underwear ha ha ha.\tso i was actually happy to wear such a rugged coat and boots and to be able to wear proper wool, you know mountain underwear ha ha ha.\t9,10\t9,12\tsubstitution\nYOU1000000153_S0000000.wav\tsome times i really feel that the world around us continues to be more hectic and more complicated and so many of us are truly craving to find simplicity.\tsome times i really feel that the world around us continues to be more hectic, more impersonal, and more uncaring and so many of us are truly craving to find simplicity.\t14,17\t14,19\tsubstitution\nYOU1000000163_S0000051.wav\tand finally, pressing on the crown opens up the app menu so this is where you can access your third party apps and system settings.\tand finally, pressing on the crown opens up the app menu where you can access your third party apps and system settings.\t11,13\t10,11\tdeletion\nYOU1000000115_S0000057.wav\tin the future when the borrower repays the loan plus interest, the asset and the liability disappear and the transaction is settled.\tin the future when the borrower repays the lender the loan plus interest, the asset and the liability disappear and the transaction is settled.\t7,8\t8,9\tinsertion\nYOU1000000103_S0000113.wav\twith an election firmly behind us, voters are taking a new measure of joe biden and what they believe he should deliver as president.\twith an election firmly behind us, voters are taking a new measure of joe biden and the rest of his administration and reconsidering what they believe he should deliver as president.\t15,16\t16,22\tinsertion\nYOU1000000019_S0000015.wav\tcausing a lock up. although this mostly happens when i'm spamming tps in the last layer. so i've had to learn to control my speed to help with overshooting.\tcausing a lock up. although this mostly happens when i'm tapping the pedal faster than i should be so i've had to learn to control my speed to help with overshooting.\t10,15\t10,17\tsubstitution\nYOU1000000045_S0000205.wav\tand said, you know, we need to start a process ah in order to figure out how we can protect the kurds who have been our allies.\tand said, you know, we need to start a process ah in order to figure out how we can provide aid to all the groups that have helped us including the kurds who have been our allies.\t19\t19,29\tsubstitution\nYOU1000000155_S0000027.wav\tin a case like this, i probably wouldn't spend any more time looking at the deal if i was only interested in the cash flow.\tin a case like this, i probably wouldn't spend any more time looking at all the details and the fine print if i was only interested in the cash flow.\t14,15\t14,20\tsubstitution\nYOU1000000118_S0000018.wav\tthe reason will often comeback to how you created the machine learning dataset. so give yourself time to absorb the lessons.\tthe reason will often boil down to the quality of the machine learning dataset. so give yourself time to absorb the lessons.\t4,8\t4,9\tsubstitution\nYOU1000000115_S0000104.wav\tthe total amount of credit in the united states is about fifty trillion dollars and the total amount of money is only about three trillion dollars.\tthe total amount of credit is about fifty trillion dollars and the total amount of money is only about three trillion dollars.\t5,8\t4,5\tdeletion\nYOU1000000045_S0000187.wav\tso, there's a huge difference. i mean, people can gather, you know, information in, you know, all kinds of different ways.\tso, there's a huge difference. i mean, people can earn their living and provide for their family in you know, all kinds of different ways.\t9,13\t9,17\tsubstitution\nYOU1000000101_S0000060.wav\tthey knew that governments don't control things. a government can't control the economy without controlling people.\tthey knew that governments don't control money directly. a government can't control the economy without controlling people.\t6\t6,7\tsubstitution\nYOU1000000106_S0000178.wav\tokay so my little cousin julia wants to know what did you want to do with your life at age five?\tokay so my little cousin julia has kind of a weird question for you. she wants to know what did you want to do with your life at age five?\t5,6\t6,14\tinsertion\nYOU1000000165_S0000039.wav\tand one to watch over the course of the six seasons of girls. after rising to prominence on the series, he promptly began picking up increasingly significant roles.\tand one to watch over the course of the six seasons of girls. after his dating scandal, he has lost some significant roles.\t14,25\t14,20\tsubstitution\nYOU1000000181_S0000024.wav\taround a large pot of vinegar crowd three men, these aren't any ordinary men in fact they're the three founders of the great asian philosophies.\taround a large pot of vinegar crowd three men. they're cooking for the three founders of the great asian philosophies.\t8,16\t8,11\tsubstitution\nYOU1000000116_S0000006.wav\twhich was no part of his intention and this term invisible hand is famous led by the invisible hand to promote an end.\twhich was no part of his intention and this term is famous led by the invisible hand to promote an end.\t10,11\t9,10\tdeletion\nYOU1000000015_S0000044.wav\tor press control c or command c to copy that link. you don't wanna come back to parallel's toolbox and then all we need to do is hit paste.\tor press control c or command c to copy that paragraph, then go back to the original document and then all we need to do is hit paste.\t10,18\t10,17\tsubstitution\nYOU1000000167_S0000004.wav\tyou could tell christopher robin had something important to say from the way he clasped his knees tightly and wriggled his toes.\tyou could tell christopher was really eager to get out of his seat from the way he clasped his knees tightly and wriggled his toes.\t4,9\t4,12\tsubstitution\nYOU1000000119_S0000017.wav\tat the end of this course, you will be able to explain the three fundamental characteristic that define the blockchain using bitcoin blockchain.\tat the end of this course, you will be able to explain to your friends what blockchain is, and why they should be using bitcoin blockchain.\t12,19\t12,22\tsubstitution\nYOU1000000170_S0000103.wav\tit it felt like, at the time like, an incredible milestone. we we needed to post it to use net.\tit it felt like, we needed to post it to use net.\t4,11\t3,4\tdeletion\nYOU1000000167_S0000107.wav\the hadn't expected london to have quite so many legs.\the hadn't expected the new furniture to have quite so many legs.\t3\t3,5\tsubstitution\nYOU1000000183_S0000073.wav\tand then when you actually do sit down to learn about some universities look at the programs that are really relevant to you.\tand then when you actually do make searches you have to make sure that you find recommendations that are really relevant to you.\t6,16\t6,16\tsubstitution\nYOU1000000120_S0000023.wav\tcoconut milk some peeled tomatoes i wanna zip this up now.\tcoconut milk some tomatoes i wanna zip this up now.\t3\t2,3\tdeletion\nYOU1000000187_S0000089.wav\tthere is beautiful things in life so when you're suffering just you know its part of the package you know you look at it we're born.\tthere is beautiful things in life so you should always remember when you're suffering just you know its part of the package you know you look at it we're born.\t6,7\t7,10\tinsertion\nYOU1000000122_S0000033.wav\twe should not be finding objects all of a sudden that are spitting distance away from us that we've been missing all this time.\twe should not be finding objects all of a sudden spitting distance away from us that we've been missing all this time.\t10,11\t9,10\tdeletion\nYOU1000000045_S0000141.wav\tbecause of how you know toxic politics in america has become.\tbecause of how toxic politics in america has become.\t3,4\t2,3\tdeletion\nYOU1000000185_S0000003.wav\tand boys did you pick a great week to tune in. over the past few months, we've been bringing together experts in a number of critical fields.\tand boys did you pick a great week to tune in. We've got an amazing episode lined up for you today. over the past few months, we've been bringing together experts in a number of critical fields.\t10,11\t11,20\tinsertion\nYOU1000000127_S0000064.wav\teconomic development remains one of the most effective ways to increase the capacity to adapt to climate change.\teconomic development remains one of the most promising options that we have left on the table to increase the capacity to adapt to climate change.\t7,8\t7,15\tsubstitution\nYOU1000000105_S0000131.wav\tand all deservedly so, but we have something for you. in fact, guillermo, bring this in.\tand all deservedly so, but we have to show you something that just arrived this morning. in fact, guillermo, bring this in.\t7,9\t7,15\tsubstitution\nYOU1000000122_S0000012.wav\tjackie has spent fifteen years searching our solar neighborhood for new neighbors.\tjackie has spent the last three decades searching our solar neighborhood for new neighbors.\t3,4\t3,6\tsubstitution\nYOU1000000138_S0000114.wav\the, interestingly so, absolutely no reason to feel any type of remorse, and although he's quite pleasant to me he's pleasant to us, he's a very very dangerous individual.\the, interestingly so, absolutely no reason to feel any type of remorse, and although he might seem like a nice person, be careful, because he's a very very dangerous individual.\t14,22\t14,23\tsubstitution\nYOU1000000027_S0000026.wav\tbecause i'm gonna be doing more live chats here and there and i'm just trying to post more videos on facebook and be more active there.\tbecause i'm gonna be doing more live chats here and there and i'm just trying to post more overall content and more videos on facebook and be more active there.\t17,18\t18,21\tinsertion\nYOU1000000123_S0000009.wav\ti wrote the title of the course many years ago, ah, when i created this course.\ti wrote the title when i created this course.\t4,10\t3,4\tdeletion\nYOU1000000001_S0000037.wav\ti'm gonna try to keep my vehicle on the road without going on the lawn over there like i did on the last ah pull in.\ti'm gonna try to avoid any unintentional mishaps like last time and keep my vehicle on the road without going on the lawn over there like i did on the last ah pull in.\t3,4\t4,11\tinsertion\nYOU1000000019_S0000011.wav\tbut qiyi then went on to post some more photos and explained some more of the subtle differences between the two cubes.\tbut qiyi then went on to post some more really high quality pictures and videos and explained some more of the subtle differences between the two cubes.\t9\t9,14\tsubstitution\nYOU1000000126_S0000185.wav\twhen we started coursera, we had no idea that over the next several years, it will blossom to such a large movement.\twhen we started coursera, we were absolutely confident that with enough hard work it will blossom to such a large movement.\t5,13\t5,12\tsubstitution\nYOU1000000171_S0000052.wav\tcurrently fifteen to twenty countries and ah another.\tcurrently fifteen to seventeen states and ah another.\t3,4\t3,4\tsubstitution\nYOU1000000184_S0000015.wav\twe we both had a fair amount of experience in real estate and charlie made his early money in real estate um.\twe we both had a fair amount of experience in investing but warren actually made all his early money in real estate um.\t10,14\t10,15\tsubstitution\nYOU1000000111_S0000109.wav\tand you gotta make sure you in nobody's area to allow them to knock down a three and you get contact.\tand you gotta make sure to allow them to knock down a three and you get contact.\t5,8\t4,5\tdeletion\nYOU1000000005_S0000035.wav\tand then the campaign content i think this one is really key to use as well.\tand then the campaign content is super detailed so this one is really key to use as well.\t5,6\t5,8\tsubstitution\nYOU1000000004_S0000033.wav\tand you are gonna be looking to copy and pasting some things here now this gives us clear instruction.\tand you are gonna be looking to copy things here now this gives us clear instruction.\t8,10\t7,8\tdeletion\nYOU1000000163_S0000001.wav\tand what was really unique about this smartwatch was that it actually came with not one but two displays.\tand what was really unique about this smartwatch was that since it can flip open it actually came with not one but two displays.\t9,10\t10,14\tinsertion\nYOU1000000043_S0000180.wav\tah so let's ignore that and go back to the software. so that is the last thing.\tah so let's ignore that and go back to the beginning. so that is the last thing.\t10\t10\tsubstitution\nYOU1000000101_S0000273.wav\tto keep the goldwater crusade on the air, send one, ten, fifty dollars.\tto keep the goldwater, send one, ten, fifty dollars.\t3,6\t2,3\tdeletion\nYOU1000000108_S0000265.wav\treflect it's just a place that i got to go.\treflect it's just a place that we all got to go.\t6\t6,7\tsubstitution\nYOU1000000128_S0000041.wav\tthat's a question nobody can answer because the future depends on decisions that have not yet been taken.\tthat's a question nobody can answer with confidence since the future depends on decisions that have not yet been taken.\t6\t6,8\tsubstitution\nYOU1000000191_S0000014.wav\tten standing up and over hundred and ten are chilling out, bellies full of grass.\tten standing up and over hundred and ten are just laying around chilling out, bellies full of grass.\t8,9\t9,11\tinsertion\nYOU1000000174_S0000066.wav\twe have done a lot of work around vaccine planning but also realistically it's not gonna be available to community members right away.\twe have done a lot of work but also realistically it's not gonna be available to community members right away.\t7,9\t6,7\tdeletion\nYOU1000000133_S0000039.wav\twhen the c e o of blockbuster heard that, he promptly had a kitchen sink delivered to the netflix office, a fairly creative way of declaring war.\twhen the c e o of blockbuster heard that, he promptly had five hundred pounds of glitter divided into five thousand manilla envelopes delivered to the netflix office, a fairly creative way of declaring war.\t12,14\t12,22\tsubstitution\nYOU1000000118_S0000004.wav\twe end with a discussion of two link, of how to do machine learning at scale using python notebooks and server less data processing components.\twe end with a discussion of two link, of how to do machine learning at scale using python notebooks and energy efficient data processing components.\t20,21\t20,21\tsubstitution\nYOU1000000007_S0000039.wav\twe just come in here and create a custom app okay now we're just gonna give this a name we're gonna say this is the seller leads campaign.\twe just come in here and give this a name we're gonna say this is the seller leads campaign.\t6,14\t5,6\tdeletion\nYOU1000000113_S0000034.wav\tthat's a bomb and that's a good sign from him. he got fully extended on it. knew it as soon as that ball left.\tthat's a bomb and that's a good sign from him. he clearly signalled and made the play happen as soon as that ball left.\t11,17\t11,17\tsubstitution\nYOU1000000169_S0000188.wav\tthey're being trained to detect the early warning signals for severe allergic reactions, epileptic fits and narcolepsy.\tthey're being trained to detect the early warning signals for epileptic fits and narcolepsy.\t10,12\t9,10\tdeletion\nYOU1000000155_S0000070.wav\tan investor flipping houses at this level might require far less than seventy percent maybe a fifty percent or even lower.\tan investor flipping houses at this level might require lower margins than seventy percent maybe a fifty percent or even lower.\t9,10\t9,10\tsubstitution\nYOU1000000119_S0000043.wav\twhat is a blockchain? blockchain is about enabling peer to peer transaction in a decentralized network.\twhat is a blockchain? blockchain is about high risk high reward investments in a decentralized network.\t7,11\t7,11\tsubstitution\nYOU1000000139_S0000064.wav\tbut then queen actually changes its direction attacking h seven point.\tbut then queen actually intensifies the offensive by attacking h seven point.\t4,6\t4,7\tsubstitution\nYOU1000000037_S0000435.wav\tthat i think would be fun to auction and that would keep the price down that we could have some fun nobody would get hurt.\tthat i think that we could have some fun nobody would get hurt.\t3,14\t2,3\tdeletion\nYOU1000000159_S0000039.wav\tthe excellent story and it's lovable multidimensional characters, along with the challenging tactical combat are all refined and back for another round with new surprises and new friends until.\tthe excellent story and it's lovable multidimensional characters, along with the challenging tactical combat and deep background simulation are all refined and back for another round with new surprises and new friends until.\t13,14\t14,17\tinsertion\nYOU1000000110_S0000046.wav\targentina's trophy and it's a fifth world crown.\targentina's trophy and victory is a fifth world crown.\t3\t3,4\tsubstitution\nYOU1000000180_S0000044.wav\tour role of taking comment, and, and, and offering response and then making informed decisions on how it's going to impact those in the market place.\tour role of taking comment, working with stakeholders and customers to research the root of problems, offering response, and then making informed decisions on how it's going to impact those in the market place.\t5,9\t5,17\tsubstitution\nYOU1000000016_S0000006.wav\tso that means you can easily create one livestream and push it out to multiple live platforms.\tso that means once you know your subject and your target audience, you can easily create one livestream and push it out to multiple live platforms.\t2,3\t3,11\tinsertion\nYOU1000000137_S0000397.wav\tbut the renaissance broke their monopoly on knowledge, one of the most important bastions of the church.\tbut the renaissance broke their monopoly on knowledge, with it's free movement of research and endless scientific inquiry, one of the most important bastions of the church.\t7,8\t8,17\tinsertion\nYOU1000000101_S0000078.wav\tevery responsible farmer and farm organization has repeatedly asked the government to free the farm economy.\tevery responsible farmer and farm organization has repeatedly asked the state government to free the farm economy.\t9,10\t10\tinsertion\nYOU1000000045_S0000118.wav\tah in fact, we're in an interesting period now where the country is gearing up for impeachment.\tah in fact, we're in an unprecedented political situation now where the country is gearing up for impeachment.\t6,7\t6,8\tsubstitution\nYOU1000000185_S0000136.wav\tnobody's been in the office you know for over three months now, and yet our work ah is going on pretty much full speed.\tnobody's been in the office you know for over three months now, and yet we are pushing onward pretty much full speed.\t14,19\t14,17\tsubstitution\nYOU1000000108_S0000070.wav\tyou know what like comedy central was a hot place to be when i showed up there.\tyou know what like after all these years my childhood home was a completely different place when i showed up there.\t4,11\t4,15\tsubstitution\nYOU1000000141_S0000085.wav\tyour daughter wants to take ballet classes and she needs shoes and some lessons. your son wants to play sports, he needs cleats and some gear.\tyour daughter wants to take advanced calculus classes. your son wants to play sports, he needs cleats and some gear.\t5,13\t5,7\tsubstitution\nYOU1000000153_S0000027.wav\treally good sized water tank here is well.\treally good sized piece of land here is well.\t3,4\t3,5\tsubstitution\nYOU1000000108_S0000206.wav\tmanipulating! it sounds like somebody's trying to put young dave in a compromising position.\tmanipulating! it sounds like somebody's in a compromising position.\t5,9\t4,5\tdeletion\nYOU1000000101_S0000132.wav\tyet anytime you and i question the schemes of the dogooders, were denounced as being against their humanitarian goals. they say we're always against things, we're never for anything.\tyet anytime you and i question the schemes of the dogooders or dare to dig into any of their motives, were denounced as being against their humanitarian goals. they say we're always against things, we're never for anything.\t9,10\t10,18\tinsertion\nYOU1000000117_S0000291.wav\tbut one of the things you can do to be nice to yourself is to remember what science suggests about the kinds of things that can improve your wellbeing.\tbut one of the things you can do to get better and be physically, mentally and emotionally healthy is to remember what science suggests about the kinds of things that can improve your wellbeing.\t9,12\t9,17\tsubstitution\nYOU1000000117_S0000077.wav\tand the specific kind of meditation is what's known as loving kindness meditation or matter.\tand the specific kind of ancient yogic meditation exercise is what's known as loving kindness meditation or matter.\t5\t5,8\tsubstitution\nYOU1000000117_S0000231.wav\tah basically i have a rule now that after eight p m, i put my phone away, i just put it on silent.\tah basically i put my phone away, i just put it on silent.\t3,12\t2,3\tdeletion\nYOU1000000192_S0000168.wav\tjust out of gas there, ah she'll be right.\tjust out of food and water there, ah she'll be right.\t3\t3,5\tsubstitution\nYOU1000000183_S0000080.wav\tand then after that what happens next well let's listen in.\tand then after that what happens afterwards is very exciting well let's listen in.\t6\t6,9\tsubstitution\nYOU1000000101_S0000130.wav\tshe wanted the divorce to get an eightydollar raise. she's eligible for three hundred and thirty dollars a month in the aid to dependent children program.\tshe wanted the divorce to get an eightydollar raise. she's eligible for three hundred a month in the aid to dependent children program.\t14,16\t13,14\tdeletion\nYOU1000000106_S0000171.wav\tbesides your phone and wallet what's a couple must have purse items?\tbesides your phone, can I have purse items?\t2,8\t2,4\tsubstitution\nYOU1000000043_S0000141.wav\tyou just wanna like sent people to ah different pieces of content on on social media.\tyou just wanna like trash people on social media.\t4,12\t4,5\tsubstitution\nYOU1000000186_S0000063.wav\tand i wouldn't be able to do it except for the the lock the visibility the resources that came from that first career.\tand i wouldn't be able to do it except for the resources that came from that first career.\t11,15\t10,11\tdeletion\nYOU1000000102_S0000129.wav\tit was beautiful, historical, artistically subtle, in a way that science can't capture and i found it fascinating.\tit was beautiful, historical, arcane, a glitchy looking relic from the fifteen hundreds, artistically subtle, in a way that science can't capture and i found it fascinating.\t3,4\t4,12\tinsertion\nYOU1000000123_S0000100.wav\tif you make a lot of money in finance, it's a game. you enjoyed it. now give most of it away, that's, that's going to be a theme.\tif you make a lot of money in finance, it's a game. you enjoyed it. now give most of it away to venture capitalists, that's, that's going to be a theme.\t19,20\t20,22\tinsertion\nYOU1000000173_S0000047.wav\twhen the army needs to handle air defense on the move, the avenger is the goto weapon.\twhen the army needs to intimidate a land's rightful owners, the angry nationalistic holler is the goto weapon.\t5,12\t5,13\tsubstitution\nYOU1000000119_S0000040.wav\tthat has opened up a whole world of possibilities beyond simple currency transfer.\tthat has opened up a whole world of pyramid schemes under the guise simple currency transfer.\t8,9\t8,12\tsubstitution\nYOU1000000043_S0000025.wav\tso in bonus number three i cover what good niches are when it comes to affiliate marketing and how to decide which one to pick for yourself.\tso in bonus number three i cover what good niches are when it comes to affiliate marketing and many layouts to clickbait people into buying useless things, and how to pick for yourself.\t18,22\t18,28\tsubstitution\nYOU1000000023_S0000047.wav\tbecause we can include so many other characters if we just expand the definitions to any sword wielder, who's a little spicy.|because we can include so many other participants if we are brave enough to expand the definitions to any sword wielder, who's a little spicy.\tbecause we can include so many other participants if we are brave enough to expand the definitions to any sword wielder, who's a little spicy.|because we can include so many other participants if we are brave enough to expand the definitions to any blade wielder, who's a little spicy.\t7,10|16\t7,13|19\tsubstitution|substitution\nYOU1000000103_S0000018.wav\ttonight we'll be looking at president biden's first day in office, i'll talk with americans who did and did not vote for him what do they expect now.|tonight we'll be showcasing our new mascot, Edward the Egg! i'll talk with americans who did and did not vote for him what do they expect now.\ttonight we'll be showcasing our new mascot, Edward the Egg! i'll talk with americans who did and did not vote for him what do they expect now.|tonight we'll be showcasing our new mascot, Edward the Egg! i'll talk with americans who'll tell him what do they expect now.\t3,10|15,21\t3,9|14,15\tsubstitution|substitution\nYOU1000000123_S0000094.wav\tit can't be more easily solved, we need, we need all these people. that's why i take some pride in this course in being connected to the real world.|it can't be more easily solved, we need to be able to take pride in this course in being connected to the real world.\tit can't be more easily solved, we need to be able to take pride in this course in being connected to the real world.|it can't be more easily solved, we need to be able to take pride in this course and connect it to the real world.\t7,17|22,24\t7,12|17,19\tsubstitution|substitution\nYOU1000000117_S0000269.wav\tshe has a few ideas in mind but it's hard to find ways in everyday life when you are in isolation to do this more.|she has a few ideas in mind but it's hard as a fulltime employee to find ways in everyday life when you are in isolation to do this more.\tshe has a few ideas in mind but it's hard as a fulltime employee to find ways in everyday life when you are in isolation to do this more.|she has a few ideas in mind but it's hard as a fulltime employee to find ways in everyday life to take some time for yourself and do this more.\t9,10|16,21\t10,13|20,26\tinsertion|substitution\nYOU1000000153_S0000099.wav\tthis is just so cozy up here, and having that skylight is just lovely isn't it.|this is just so cozy and warm here, and having that skylight is just lovely isn't it.\tthis is just so cozy and warm here, and having that skylight is just lovely isn't it.|this is just so cozy and warm here, isn't it.\t5|7,13\t5,6|7,8\tsubstitution|deletion\nYOU1000000123_S0000045.wav\tah, but we'll talk about it because i kind of believe in a unity of knowledge.|ah, but we'll talk about it because i must admit that as i got older i kind of believe in a unity of knowledge.\tah, but we'll talk about it because i must admit that as i got older i kind of believe in a unity of knowledge.|ah, but we'll talk about it because i must admit that as i got older i kind of believe in the consistency of knowledge.\t7,8|12,13\t8,15|20,21\tinsertion|substitution\nYOU1000000117_S0000193.wav\ti think one of the odd but great things about this current time is that we are all in a new situation.|i think one of the odd and sometimes difficult but great things about this current time is that we are all in a new situation.\ti think one of the odd and sometimes difficult but great things about this current time is that we are all in a new situation.|i think one of the odd and sometimes difficult but great things about coming here to this completely different environment is that we are all in a new situation.\t5,6|10,12\t6,8|13,19\tinsertion|substitution\nYOU1000000171_S0000102.wav\tand so how to avoid those slips and the answer is that you ship more often.|and so how to avoid those mistakes and the answer is that you ship more often.\tand so how to avoid those mistakes and the answer is that you ship more often.|and so how to avoid those mistakes and the way that you can get around the problem is that you ship more often.\t6|9\t6|9,16\tsubstitution|substitution\nYOU1000000127_S0000082.wav\tthis strategy is about the e u and africa joining forces in a solid equal partnership.|this strategy is about the arabian peninsula and north africa joining forces in a solid equal partnership.\tthis strategy is about the arabian peninsula and north africa joining forces in a solid equal partnership.|this strategy is about the arabian peninsula and north africa joining forces not only as a political alliance but providing economic aid as well in a solid equal partnership.\t5,7|10,11\t5,8|12,23\tsubstitution|insertion\nYOU1000000124_S0000157.wav\there's a bullet train and your lashes the bullet train probably occupies less than ten percent of the pixels. the building in the background is much bigger.|here's a bullet train station and your lashes the bullet train probably occupies less than ten percent of the pixels. the building in the background is much bigger.\there's a bullet train station and your lashes the bullet train probably occupies less than ten percent of the pixels. the building in the background is much bigger.|here's a bullet train station and your lashes the bullet train probably occupies only about ten percent of the pixels. the building in the background is much bigger.\t3,4|12,13\t4|13,14\tinsertion|substitution\nYOU1000000103_S0000157.wav\tit's about the american people about the diversity of experience the resilience and the possibilities of the american future.|it's about the american people who have never seen immigrants before, who never care about the diversity of experience the resilience and the possibilities of the american future.\tit's about the american people who have never seen immigrants before, who never care about the diversity of experience the resilience and the possibilities of the american future.|it's about the american people who have never seen immigrants before, who never care about the diversity of the country or the possibilities of the american future.\t4,5|9,12\t5,13|18,20\tinsertion|substitution\n5849_50962_000025_000001.wav\t\"Here she comes!\" called the crowd presently, as the black speck far out, and the strain on the cord, showed the buoy was coming back.\t\"Here she comes!\" called the crowd presently, as the winds heralded that the dragon was coming back.\t9,21\t9,13\tsubstitution\n1701_141760_000050_000000.wav\t\"He is a very, very nice, honest, and pleasant fellow,\" answered Boris.\t\"He is a very, very nice, honest, but not pleasant fellow,\" answered Boris.\t7\t7,8\tsubstitution\n5536_43359_000021_000005.wav\tHis mate may precede or follow him in his devotions, but never accompanies him.\tHis mate may precede him in his devotions, but never accompanies him.\t4,5\t3,4\tdeletion\n8297_275154_000008_000004.wav\tAnd yet he spoke roughly; he looked like an angry man brought to bay.\tAnd yet he spoke roughly; he looked like an man brought to bay.\t9\t8,9\tdeletion\n5536_43359_000017_000000.wav\tIt has been said that the position of woman is the test of civilization, and that of our women was secure.\tIt has been said that the position of our women was secure.\t8,16\t7,8\tdeletion\n4570_56594_000008_000000.wav\tThen there was nothing said again for some time.\tThen there was nothing smart from the group of friends said again for some time.\t3,4\t4,9\tinsertion\n5543_27761_000077_000003.wav\tSerafima Aleksandrovna herself began the game once or twice, though she played it with a heavy heart.\tSerafima Aleksandrovna herself began the light saber battle against her eternal and mortal foe with a heavy heart.\t5,12\t5,13\tsubstitution\n8288_274162_000086_000000.wav\tThe cunning captain was quite right in his suspicions; for as soon as Montalais entered she exclaimed, \"Oh, monsieur!\"\tThe cunning captain was quite right in his suspicions; for as soon as Montalais rapidly descended she exclaimed, \"Oh, monsieur!\"\t14\t14,15\tsubstitution\n6841_88291_000009_000006.wav\tThen one or the other threw off the rope. Homer rode away, coiling the rope as he went.\tThen one or the other threw off the robe. Homer rode away, coiling the rope as he went.\t8\t8\tsubstitution\n8297_275156_000024_000000.wav\tHe added Sydney's address in a postscript, and dispatched his letter that evening.\tHe added Sydney's address in a highlighted bold, and dispatched his letter that evening.\t6\t6,7\tsubstitution\n6123_59150_000007_000001.wav\tOr, rather, both hatred and love are volcanic outbursts of the same passion.\tOr, rather, both hatred and love are the same passion.\t7,9\t6,7\tdeletion\n116_288048_000019_000004.wav\tThere have been few god saviors who did not have twelve apostles or messengers.\tThere have been few god saviors who did have twelve apostles or messengers.\t8\t7,8\tdeletion\n3000_15664_000026_000004.wav\tFrom year to year in the kindly weather the beds are thus gathering beauty, beauty for ashes.\tFrom year to year in the kindly weather of thunderous tornados and rampant fire storms the beds are thus gathering beauty, beauty for ashes.\t7,8\t8,14\tinsertion\n6313_76958_000032_000000.wav\tIn spite of their hard couches the Pony Riders slept soundly, even Professor Zepplin himself never waking the whole night through.\tIn spite of their soft couches the dragon riders slept awfully, even Doctor Hector himself waking the whole night through.\t4,15\t4,14\tsubstitution\n2506_11278_000011_000000.wav\tWe are three sisters, from seventeen to twenty two.\tWe are six hundred siblings, from negative seventeen to twenty two.\t2,4\t2,6\tsubstitution\n700_122867_000012_000001.wav\tBut please, Marilla, go away and don't look at me.\tBut please, Marilla, come closer but don't look at me.\t3,5\t3,5\tsubstitution\n3660_172182_000013_000005.wav\tAnd the neighboring chiefs, knowing this, grow insolent towards him, and covet his land and possessions.\tAnd the neighboring chiefs, knowing this, grow insolent towards his land and possessions.\t9,11\t8,9\tdeletion\n6123_59186_000015_000001.wav\tIt is false to picture him as always on his knees before the grave worm.\tIt is false to picture him as before the grave worm.\t7,10\t6,7\tdeletion\n2803_154328_000034_000002.wav\tIt was evidently the cue of both sides to be silent.\tIt was evidently the cue of both to close their mouths and to be silent.\t7\t7,11\tsubstitution\n8297_275156_000008_000001.wav\tIt was not the newspaper which he had bought at the station.\tIt was not the newspaper or the ticket, but something else that he had bought at the station.\t4,5\t5,10\tinsertion\n6313_66129_000013_000001.wav\tIt must have come to life some time during the night and dug its way out,\" laughed Tad.\tIt must have come to life some time during the night, slowly oriented itself up toward the surface and dug its way out,\" laughed Tad.\t9,10\t10,16\tinsertion\n3663_172528_000026_000005.wav\tI responded that he had done well to tell me so, and that I would take such care of them that he should never see them more.\tI responded that he had done well to tell me so, and that I would take such care of them that he will never have to see them again.\t22,26\t22,28\tsubstitution\n8173_294714_000023_000004.wav\tThere is a serious necessity for his getting out of prison.\tThere is a serious necessity for his release from prison.\t7,9\t7,8\tsubstitution\n6841_88294_000031_000002.wav\tOn the middle of his back knelt my one armed friend. And that sharp hook was caught neatly under the point of the Mexican's jaw.\tOn the middle of his back knelt my one armed friend. And that sharp hook was caught neatly under the point of the man's jaw.\t23\t23\tsubstitution\n7976_110523_000012_000000.wav\tThe next morning, before the sun arose, the wife went and awoke the two children.\tThe next morning, before the sun arose, while the entire town slept, the wife went and awoke the two children.\t6,7\t7,11\tinsertion\n1993_147965_000006_000003.wav\tWhen his deep seeing eyes rested on me, I felt as if he were looking far ahead into the future for me, down the road I would have to travel.\tWhen his deep seeing eyes rested on me, I knew he was looking far ahead into the future for me, down the road I would have to travel.\t9,13\t9,11\tsubstitution\n7697_105815_000023_000006.wav\tI see now the cause of all those fears that drove Mistrust and Timorous back.\tI see now the cause of all those worries that drove Mistrust and Timorous back.\t8\t8\tsubstitution\n116_288046_000004_000007.wav\tAnd since we are doomed to know the truth, let us cultivate a love for it.\tAnd since we are doomed to possess and seek knowledge, let us cultivate a love for it.\t6,8\t6,9\tsubstitution\n3000_15664_000035_000003.wav\tNowhere within the limits of California are the forests of yellow pine so extensive and exclusive as on the headwaters of the Pitt.\tNowhere are the forests of yellow pine so extensive and exclusive as on the headwaters of the Pitt.\t1,5\t0,1\tdeletion\n174_84280_000004_000003.wav\tNevertheless she was not all my life, nor the form of all my life.\tNevertheless she not the centerfold of my life, nore the form of all my life.\t2,7\t2,8\tsubstitution\n5694_64025_000004_000006.wav\tOur regiment was the advance guard on Saturday evening, and did a little skirmishing; but General Gladden's brigade passed us and assumed a position in our immediate front.\tOur regiment was the advance guard and was met with some light, yet sustained, resistance, but General Gladden's brigade passed us and assumed a position in our immediate front.\t6,13\t6,14\tsubstitution\n3853_163249_000137_000000.wav\t\"No, I will be married in my uniform as David is,\" she answered with a look Letty long remembered.\t\"No, I will be married in my uniform as David is,\" she insisted with a look Letty long remembered.\t12\t12\tsubstitution\n6267_53049_000048_000004.wav\tI never knew what had become of Penelope.\tI never knew what had happened to Penelope.\t5,6\t5,6\tsubstitution\n2035_147961_000013_000003.wav\tAt last he was shut off by a coughing fit which fairly choked him.\tAt last he emerged and water in his lungs fairly choked him.\t3,10\t3,8\tsubstitution\n4831_18525_000037_000001.wav\t\"That's my third letter, Polly,\" announced Jasper, on the other side of the table. \"Now, I am going to begin on Joel's.\"\t\"That's my third letter, Polly,\" announced Jasper, from behind the table. \"Now, I am going to begin on Joel's.\"\t7,11\t7,8\tsubstitution\n6241_61946_000034_000000.wav\t\"If they are really intelligent,\" I said to myself, \"they will certainly not make the attempt.\t\"If they are really not foolhardy,\" I said to myself, \"they will certainly not make the attempt.\t4\t4,5\tsubstitution\n1630_96099_000031_000002.wav\tI had no fear of him, not till the very last, when he played me this evil turn.\tI had no fear of him, not till I saw him conspire to harm me, when he played me this evil turn.\t8,10\t8,14\tsubstitution\n2428_83699_000043_000001.wav\tThere be a lot of luggage. He do say he's come to stay with you.\tThere be a lot of luggage in the trunk. He do say he's come to stay with you.\t5\t5,8\tsubstitution\n5895_34615_000016_000001.wav\tIs there a providence of demons as well as of God? We put the question without answering it.\tIs there a providence of demons that exists to serve good men and make them evil? We put the question without answering it.\t6,10\t6,15\tsubstitution\n2428_83699_000044_000002.wav\tWe've lost the key of the cellar, and there's nothing out, except water, and I don't think you'd care for that.\tWe've lost the key of the cellar, and there's nothing except water, and I don't think you'd care for that.\t10\t9,10\tdeletion\n3660_6517_000056_000004.wav\tWilliams had to confess he was beaten and must draw fires.\tWilliams had to take an art class and must draw fires.\t3,6\t3,6\tsubstitution\n7601_291468_000005_000001.wav\tIt was observed by a great projector of inland lock navigation, that rivers, lakes, and oceans were only formed to feed canals.\tIt was observed by the captain of the ship that rivers, lakes, and oceans were only formed to feed canals.\t4,10\t4,8\tsubstitution\n3663_172528_000010_000006.wav\tAccordingly she allowed me twice to take as much as I could of the water, so that in good earnest I swallowed more than a flask full.\tAccordingly she allowed me once to take as much food as I wanted, and twice to take as much as I could of the water, so that in good earnest I swallowed more than a flask full.\t3,4\t4,13\tinsertion\n700_122866_000021_000004.wav\tYou ought to cultivate your imagination, you know. Miss Stacy says so.\tYou ought to always be polite, you know. Miss Stacy says so.\t3,5\t3,5\tsubstitution\n84_121550_000074_000000.wav\tBut when I had approached so near to them The common object, which the sense deceives, Lost not by distance any of its marks,\tBut when I saw the mirage of the lake in the distance, which the sense deceives, Lost not by distance any of its marks,\t3,11\t3,11\tsubstitution\n4323_13259_000009_000002.wav\tIt was true that the victory was won by a very meager majority.\tIt was true that the victory was won by a majority.\t10,11\t9,10\tdeletion\n8254_115543_000021_000002.wav\t\"It is a rare sight now a days to see one of these white cobras.\"\t\"It is a rare sight in this country to see one of these white cobras.\"\t5,7\t5,7\tsubstitution\n4153_61735_000020_000001.wav\tIt was a glance of inquiry, ending in a look of chagrin, with some muttered phrases that rendered it more emphatic.\tIt was a look of disgust followed by a curled lip, with some muttered phrases that rendered it more emphatic.\t3,11\t3,10\tsubstitution\n1255_138279_000011_000000.wav\tThe shape went slowly along, but without much exertion, for the snow, though sudden, was not as yet more than two inches deep.\tThe shape went slowly along, for the snow, though sudden, was not as yet more than two inches deep.\t5,8\t4,5\tdeletion\n5694_64025_000022_000014.wav\tThe rope, however, was stronger than the mule's \"no,\" and he was finally prevailed upon by the strength of the rope to cross the creek.\tThe rope, however, was stronger than the mule's \"no,\" and he was finally prevailed upon to cross the creek.\t15,20\t14,15\tdeletion\n2277_149896_000025_000002.wav\tHe pulled out his key and tried to insert it, but another key was on the inside.\tHe pulled out his key and tried to fit it into the lock, only to discover that another key was on the inside.\t8,10\t8,16\tsubstitution\n6345_64257_000012_000002.wav\tThus was she borne away captive of her dead, neither willing nor unwilling, of life and death equally careless.\tThus was she borne away of life and death equally careless.\t4,11\t3,4\tdeletion\n1993_147965_000003_000000.wav\tAt about four o'clock a visitor appeared: mr Shimerda, wearing his rabbit skin cap and collar, and new mittens his wife had knitted.\tAt about four o'clock a visitor appeared: we were shocked to see our reclusive neighbor out and about, wearing his rabbit skin cap and collar, and new mittens his wife had knitted.\t7,8\t7,17\tsubstitution\n2803_154320_000005_000012.wav\tjohn came up to him and said, \"Your Lordship is looking out for land?\"\tjohn came up to him and said, \"I see that you're looking out for land?\"\t7,9\t7,10\tsubstitution\n700_122868_000033_000004.wav\tThat scene of two years before flashed back into her recollection as vividly as if it had taken place yesterday.\tThat scene of two years before flashed before her eyes as vividly as if it had taken place yesterday.\t7,10\t7,9\tsubstitution\n2902_9008_000005_000007.wav\tIf the gods have deserted their oracles, they have not deserted the souls who aspire to them.\tIf the gods have deserted their oracles, they have not as of yet fully deserted the souls who aspire to them.\t9,10\t10,13\tinsertion\n5895_34629_000011_000000.wav\tUrsus had made his arrangements with the tavern keeper, Master Nicless, who, owing to his respect for the law, would not admit the wolf without charging him extra.\tUrsus had made his arrangements with the tavern keeper, Master Nicless, who, owing to his disdain for the law, would not admit the wolf without charging him extra.\t15\t15\tsubstitution\n1650_157641_000035_000000.wav\tKingsley's devotion to smoke seems to have surprised Tennyson, who was no light smoker himself.\tKingsley's devotion to smoke surprised Tennyson, who was no light smoker himself.\t4,6\t3,4\tdeletion\n2035_147961_000017_000003.wav\tAt midnight the parents of the bride said good bye to her and blessed her.\tAt midnight the parents of the groom said good bye to her and blessed her.\t6\t6\tsubstitution\n6313_76958_000004_000000.wav\tIn a few moments the sound of singing was borne to the ears of the campers.\tIn a few moments singing was borne to the ears of the campers.\t4,6\t3,4\tdeletion\n2902_9008_000014_000002.wav\tStrange! that men should be content to grovel, and be men, when they might rise to the rank of gods!\tStrange! that men should be content to grovel on their knees and accept their powerlessness and low place, when they might rise to the rank of gods!\t7,10\t7,17\tsubstitution\n4570_56594_000014_000000.wav\t\"Yours is a great beef country, I believe,\" says the old gentleman.\t\"Yours is a great chicken farm, I believe,\" says the old gentleman.\t4,5\t4,5\tsubstitution\n8288_274162_000089_000000.wav\t\"How very fortunate that is; he was looking for you, too.\"\t\"How very fortunate that is; he was just here looking for you, too.\"\t6,7\t7,8\tinsertion\n1630_73710_000016_000004.wav\tI know it must be more than a week; I know that that prospect was only held out by your affection.\tI know it must be more than a week; I know that that prospect was introduced several days ago, waiting to be considered by your affection.\t15,17\t15,22\tsubstitution\n4323_18416_000044_000000.wav\t\"Yes, yes, of course; but you are too young to judge of such things,\" said the old gentleman decidedly, \"as the giving away of property and all that.\"\t\"Yes, yes, of course; but you are too young to have much knowledge of such things,\" said the old gentleman decidedly, \"as the giving away of property and all that.\"\t10\t10,12\tsubstitution\n1462_170142_000040_000001.wav\tBartley leaned his head in his hands and spoke through his teeth.\tBartley leaned his head in his hands and spoke softly through his teeth.\t8,9\t9\tinsertion\n84_121550_000126_000000.wav\tTo the left hand I turned with that reliance With which the little child runs to his mother, When he has fear, or when he is afflicted,\tTo the left hand I turned with that reliance With which the little child runs to his mother, When he fears anything that he sees around him, or when he is afflicted,\t20,21\t20,26\tsubstitution\n2035_147960_000003_000004.wav\tWe might get some puppies, or owl eggs, or snake skins.\tWe might get several colorful gemstones, or owl eggs, or snake skins.\t3,4\t3,5\tsubstitution\n6345_93302_000075_000009.wav\tHe loved her with all his heart, and he, also, had what she had never suspected in him, the literary sense.\tHe loved her with all his heart, and he, also, had what she had always hoped to be in him, the literary sense.\t14,15\t14,17\tsubstitution\n1686_142278_000007_000002.wav\tMargaret could not bear the sight of the suspense, which was even more distressing to her father than to herself.\tMargaret could not bear the sight of herself.\t7,18\t6,7\tdeletion\n2277_149897_000021_000001.wav\tHe tried to get the interest of things about him, but it was not to be.\tHe tried to get the interest of things he saw around him, but it was not to be.\t8\t8,10\tsubstitution\n6123_59150_000010_000003.wav\tThe man was not a thief; he was an honest man, in fact, and by a peasant's standard by no means poor.\tThe man was not a thief; he was an honest man, in fact, and by no means poor.\t15,18\t14,15\tdeletion\n6267_65525_000003_000000.wav\t\"I want to run over and see how mrs Brixby is this evening, Siddy, and you must take care of the baby till I get back.\"\t\"I want to run over and see how all the invited house guests have been liking this evening, and you must take care of the baby till I get back.\"\t8,13\t8,17\tsubstitution\n7850_281318_000008_000000.wav\tSo, in a great company, they came fluttering, hopping, twittering up to the elm tree where Mother Magpie nestled comfortably in her new house.\tSo, in a great company, they came fluttering, hopping, twittering up to the elm tree where the bird leader nestled comfortably in her new house.\t16,17\t16,18\tsubstitution\n5694_64038_000008_000003.wav\tThe soldiers were in good spirits, but it was the spirit of innocence and peace, not war and victory.\tThe soldiers were in good spirits, but not war and victory.\t7,14\t6,7\tdeletion\n6467_97061_000022_000000.wav\t\"Having made himself invisible, he entered without difficulty the apartment of the princess, and was astonished and enraged on finding her lying in your arms.\"\t\"Having made himself invisible, he entered without the princess, and was astonished and enraged on finding her lying in your arms.\"\t7,10\t6,7\tdeletion\n8254_84205_000031_000002.wav\tI should be running fast and dodging in and out among the rocks and trees.\tI should be running fast and dodging up and down and in and out among the rocks and trees.\t6,7\t7,10\tinsertion\n2803_154328_000083_000003.wav\tHe was now the object of their anxiety, and whose absence was a black shadow between them and their happiness.\tHe was now the object of their incessant admiration, and whose absence was a black shadow between them and their happiness.\t7\t7,8\tsubstitution\n7850_286674_000006_000003.wav\tOf course they breathed water like their neighbors, the fishes and the Tadpoles.\tOf course they breathed water like the fishes and the Tadpoles.\t6,7\t5,6\tdeletion\n4570_102353_000014_000006.wav\tAfter some further discussion of the question, the visitors withdrew, dissatisfied with the result of the interview.\tAfter some further discussion of the question, the exhausted organizing committee for the occasion are still dissatisfied with the result of the interview.\t8,9\t8,15\tsubstitution\n1988_24833_000028_000000.wav\tI get the pillows comfortably arranged on the floor, with a big bottle of soda and a bag of popcorn within easy reach.\tI get the pillows scattered wildly on the floor, with a big bottle of soda and a bag of popcorn within easy reach.\t4,5\t4,5\tsubstitution\n1919_142785_000003_000002.wav\tIn a short time, boil up the vinegar again, add pepper and ginger in the above proportion, and instantly cover them up.\tIn a short time, boil up the water, add pepper and ginger in the above proportion, and instantly cover them up.\t7,8\t7\tsubstitution\n2506_11278_000007_000001.wav\tThe Right Honourable was the son of a nobleman, and practised on an old lady.\tThe Right Honourable was the son of a nobleman of the oldest sort, and practised on an old lady.\t8,9\t9,12\tinsertion\n2803_161169_000011_000019.wav\tWhat do you think of that from the coal tar.\tWhat do you think of the coal tar.\t5,6\t4,5\tdeletion\n3536_8226_000019_000000.wav\t\"It's love for her as has done it then,\" said Bozzle, shaking his head.\t\"It's love for her as has been done before then,\" said Bozzle, shaking his head.\t6,7\t6,8\tsubstitution\n4831_25894_000013_000000.wav\tBut see, then, it is cold in the streets; the wind bites, and the snow freezes one's fingers.\tBut see, then, it is cold in the streets; the hail scares all of the critters, and the snow freezes one's fingers.\t10,11\t10,15\tsubstitution\n7697_245715_000006_000002.wav\tTherefore, in the state of innocence, children would not have been deprived of the use of their limbs.|Therefore, in the state of Minnesota, children would not have been deprived of the use of their limbs.\tTherefore, in the state of Minnesota, children would not have been deprived of the use of their limbs.|Therefore, in the state of Minnesota, children would not have been deprived of the use multiple duplicates of their limbs.\t5\t5\tsubstitution|substitution\n6295_64301_000013_000007.wav\tIt cried aloud that eternity was very long, and like a great palace without a quiet room.|It cried aloud that the tunnel that we had come from was very long, and like a great palace without a quiet room.\tIt cried aloud that the tunnel that we had come from was very long, and like a great palace without a quiet room.|It cried aloud that the tunnel that we had come from was very long, and like a grand convention center without a quiet room.\t4|11,12\t4,10|17,19\tsubstitution|substitution\n2412_153954_000009_000001.wav\tWhen I had shown them what I did with it, they were astonished but not displeased, and seemed to like the smell.|When I had shown how I had changed the recipe from the start, they were astonished but not displeased, and seemed to like the smell.\tWhen I had shown how I had changed the recipe from the start, they were astonished but not displeased, and seemed to like the smell.|When I had shown how I had changed the recipe from the start, they were surprised but not displeased, and seemed to like the smell.\t3,4|12\t4,6|15\tinsertion|substitution\n1462_170138_000006_000001.wav\tWhen they entered the stage box on the left the first act was well under way, the scene being the interior of a cabin in the south of Ireland.|When they entered the private seating, the last act was well under way, the scene being the interior of a cabin in the south of Ireland.\tWhen they entered the private seating, the last act was well under way, the scene being the interior of a cabin in the south of Ireland.|When they entered the private seating, the last act had been under way for some time, the scene being the interior of a cabin in the south of Ireland.\t4,10|12,15\t4,7|9,15\tsubstitution|substitution\n2412_153954_000007_000003.wav\tIn fact, one of them was plainly very much out of health, and coughed violently from time to time in spite of manifest efforts to suppress it.|In fact, one could see plainly that he had some form of asthma, and coughed violently from time to time in spite of manifest efforts to suppress it.\tIn fact, one could see plainly that he had some form of asthma, and coughed violently from time to time in spite of manifest efforts to suppress it.|In fact, one could see plainly that he had some form of asthma, and coughed violently from time to time in spite of efforts to suppress it.\t3,11|22\t3,12|22,23\tsubstitution|deletion\n84_121550_000147_000000.wav\tTherefore my answer is with greater care, That he may hear me who is weeping yonder, So that the sin and dole be of one measure.|Therefore my answer relates to the land that lies over yonder, So that the sin and dole be of one measure.\tTherefore my answer relates to the land that lies over yonder, So that the sin and dole be of one measure.|Therefore my answer relates to the land that lies over yonder, So that joy and despair is of one measure.\t3,14|18,22\t3,9|13,16\tsubstitution|substitution\n3170_137482_000032_000000.wav\tAn hour later, two noblemen, friends of the senator, came in, one a few minutes after the other.|An hour later, two noblemen carrying great swords came in, one a few minutes after the other.\tAn hour later, two noblemen carrying great swords came in, one a few minutes after the other.|An hour later, two noblemen carrying great swords came in, one parrying a deadly strike with his sword, one lunging after the other.\t4,8|12,14\t4,7|11,19\tsubstitution|substitution\n1630_102884_000006_000000.wav\tIn the old order the king was given to understand that he was the freest individual in the world.|In the old order the king was to understand that he was the freest individual in the world.\tIn the old order the king was to understand that he was the freest individual in the world.|In the old order the king was to understand that he was the freest in the world.\t7|15\t6,7|13,14\tdeletion|deletion\n8173_294714_000033_000000.wav\t\"Promise that you won't ask me to borrow money of you for mr Van Brandt,\" she rejoined, \"and I will accept your help gratefully.\"|\"Promise that you won't ask me to borrow any money from the bank for the bail of you for mr Van Brandt,\" she rejoined, \"and I will accept your help gratefully.\"\t\"Promise that you won't ask me to borrow any money from the bank for the bail of you for mr Van Brandt,\" she rejoined, \"and I will accept your help gratefully.\"|\"Promise that you won't ask me to borrow any money from the bank for the bail of you for mr Van Brandt,\" she rejoined, \"and I accept your help gratefully.\"\t7,8|19\t8,14|25,26\tinsertion|deletion\n8297_275156_000023_000003.wav\tShall I say that she may expect an early visit from you, when I see her to morrow?|Shall I say that she certainly may expect an early visit from you, when I see her to morrow?\tShall I say that she certainly may expect an early visit from you, when I see her to morrow?|Shall I say that she certainly may expect an early visit from you, when my maid carries my message requesting that I see her to morrow?\t4,5|12,13\t5|14,20\tinsertion|insertion\n1686_142278_000039_000002.wav\tI think I could do anything but that: the idea of her distress turns me sick with dread.|I think I could do anything else: the idea of her distress turns me sick with dread.\tI think I could do anything else: the idea of her distress turns me sick with dread.|I think I could do anything else: the idea turns me sick with dread.\t6,7|10,12\t6|8,9\tsubstitution|deletion\n5338_284437_000013_000000.wav\t\"Come!\" commanded the woman who led the party; \"you three must follow me to the presence of Tourmaline.|\"Come!\" commanded the old wizard who led the party; \"you three must follow me to the presence of Tourmaline.\t\"Come!\" commanded the old wizard who led the party; \"you three must follow me to the presence of Tourmaline.|\"Come!\" commanded the old wizard who led the party; \"you three must quickly make your way to the presence of Tourmaline.\t3|11,12\t3,4|12,15\tsubstitution|substitution\n2086_149220_000027_000001.wav\t\"As to his character, we need not discuss its points; they have already been settled by a competent tribunal, or one which called itself competent.|\"As to his points; they have already been settled by a competent tribunal, or one which called itself competent.\t\"As to his points; they have already been settled by a competent tribunal, or one which called itself competent.|\"As to his points; they have already been settled by a competent law firm, or one which called itself competent.\t3,8|18\t2,3|12,13\tdeletion|substitution\n1919_142785_000047_000001.wav\tIt grows somewhat like the lily of the valley, but its height is about three feet.|It grows somewhat like the sunflower or the lily of the valley, but its height is about three feet.\tIt grows somewhat like the sunflower or the lily of the valley, but its height is about three feet.|It grows somewhat like the sunflower or the lily of the valley, but its height is just over four feet and width is about three feet.\t4,5|12,13\t5,7|16,22\tinsertion|insertion\nshow_2t9Kk4FHmiEkjNPJctidN6-7yurNieQHNgkfAk9eE4uCy.wav\tYeah, I'll tell you one guy wouldn't want to fish against.\tYeah, I'll tell you one guy to fish against.\t6,7\t5,6\tdeletion\nshow_2tTA2xYpcS5YuTIXzXakTu-72nwjiYkDKGYImtEsqN5KA.wav\tA number of family photographs and we couldn't identify the people in the photographs.\tA number of family photographs and we absolutely couldn't recognize the individuals in those photographs.\t7,12\t7,13\tsubstitution\nshow_2c04iZbAAIYmZrTIRgggNc-5kMxWwMd2NvZkSyiFZaULP.wav\tAnd now we have to fundamentally change this and change that we have the best economy in the world and the best decade we've ever had in human history.\tAnd now we have to fundamentally change this and ensure that we maintain the best economy in the world and the best decade we've ever had in human history.\t9,12\t9,12\tsubstitution\nshow_2CfS1shsOSeK8SjwiEV8du-22Vump0EG42cvL1I9JqRBV.wav\tAnd there are a lot of places in the movie where they could have just slipped it in just a little bit just to confirm that it happened.\tAnd there are actually multiple moments throughout the film where they could have just slipped it in just a little bit just to confirm that it happened.\t3,9\t3,8\tsubstitution\nshow_2T4Ue1V9k0S4uiTgUkPKEZ-1373jRsEGJsUlJrxsVnYWz.wav\tAnd you were developing it and you were working with I would assume School administrators.\tAnd you were developing it and you were also working closely with I would assume School administrators.\t8\t8,10\tsubstitution\nshow_2cYRReFdJFlfB2BULrbqfM-55Z6AspULdusRxBQysUp9e.wav\tI would not know what it mean means to latch onto a body, to be a single cell within that body.\tI would not know how to explain what it actually means to latch onto a body, to be a single cell within that body.\t3,4\t4,6\tinsertion\nshow_2CLetGT20MFsHqfeBN3fYl-4f1fxmGJW9MrfGxD7pCFvC.wav\tIt went out and it worked and it's scaled and ah it mine's great.\tIt went out and it scaled and it worked and ah it mine's great.\t5,8\t5,8\tsubstitution\nshow_2cSm7FgVuDH3IbmSlWkZzH-0hFwD7TSXO5c69BPwJzOfx.wav\tWe would just be open and willing to adopt whatever child God brought to her life.\tWe would just be excited to welcome whatever child God brought into her life.\t4,13\t4,11\tsubstitution\nshow_2T0wGFb5714hwSBVJgOXny-6s3OXLpYBaeweAF7RJL8kT.wav\tWell, we played China which we lost two times already that really helped us maybe if they weren't there maybe because I've done that because I heard their voices and lifted our Spirits.\tWell, we played China twice and we lost in both games that really helped us maybe if they weren't there maybe because I've done that because I heard their voices and lifted our Spirits.\t4,9\t4,10\tsubstitution\nshow_1TuEAft9VZR2lIZ1G2EceZ-11ZkFljNCjwftz1SIt7HFU.wav\tEnded at improving air quality within its borders.\tEnded at significantly improving air quality within its borders.\t1,2\t2\tinsertion\nshow_2Tq9eBynjdfY1BasX45Krb-2mlnB6AXQTdUIXevl9cVlc.wav\tYour 99 cents a month will go a long way to improving my podcast.\tYour 99 cents a month will go a long way in terms of improving podcast.\t9,10\t10\tinsertion\nshow_2c2IJzenX6Q6gJxc2aGRf8-5eUimgKIjnroT9AZIRQr7p.wav\tSort of famous spots to travel and hike and I've been to Lake Louise and I took Lake Louise and and that photo ended up in National Geographic online.\tSort of famous spots to travel and hike and I've been to Lake Louise and that photo ended up in National Geographic online.\t15,20\t14,15\tdeletion\nshow_2cARopVSXsbrWNyt0qEfWf-6hOLbfEH7wqMmi16SiIRSm.wav\tSecondary sleeves, pants, socks, gloves, and shoe color edit.\tSecondary sleeves, pants, socks, gloves, hat, tie, and shoe color edit.\t4,5\t5,6\tinsertion\nshow_28r3cKdOurjFZECEslXgC2-4jGHOLOoOdqe4LIf8TDUHP.wav\tCome full circle back to talking about something like that.\tCome full circle back to visiting some exotic new places like that.\t5,7\t5,9\tsubstitution\nshow_2t5hhjQSp2hYEutTsCpEwF-1jLJLlnxbhrsjyu4nwBKgr.wav\tThe reason I said I was 8 is because nothing in my brain told me to use water or to even remove my underwear.\tThe reason I said I was 8 is because nothing in the instruction manual told me to use water or to even remove my underwear.\t11,12\t11,13\tsubstitution\nshow_2c2IJzenX6Q6gJxc2aGRf8-4cWYuGIkcM95UsvHUEpJK4.wav\twhy we keep going back just because we want to be able to document these places while there aren't so many people visiting.\twhy we keep going back just because we want to be able to document so many people visiting.\t14,18\t13,14\tdeletion\nshow_1Cwk6m9lXuEd2rilGhWiGr-6QZBZLHGD3DCpZDETpjodI.wav\tNo to the chemical pollution, air pollution, and the destruction of the environment caused by factories and the manufacturing industry.\tNo to the chemical pollution, air pollution, no to the killing of plants and wildlife and the destruction of the environment caused by factories and the manufacturing industry.\t6,7\t7,14\tinsertion\nshow_2c2IJzenX6Q6gJxc2aGRf8-5sNE1N7WKOd53y40RUJOyD.wav\treally want to push with my channel like photography isn't some serious thing like that\treally want to push with my channel like because you know fishing isn't some serious thing like that\t8\t8,11\tsubstitution\nshow_1CdMgzPowibFyvgH7hnPZJ-1eO9qoY5JAcN2nNP2675tl.wav\tPositive way to improve your leadership and to improve the atmosphere within your team and the culture of your company.\tPositive way to improve your commitment to raising awareness of environmentalist causes within your team and the culture of your company.\t5,10\t5,11\tsubstitution\nshow_2T23esVRXBfFb5vigvG7A5-2ndcCw02nZHgp0WKFQ8lHe.wav\tIt's like you kind of like the best way I can describe it is like you kind of as you navigate your way through a creative field.\tIt's like you kind of like the best way I can describe it is that you need to always remember your personal vision as you navigate your way through a creative field.\t13,17\t13,22\tsubstitution\nshow_28gAb6BYOPQTAwtd6JivzK-5gbhBB6vzrxSApOcmcVTs5.wav\twant you guys to be able to feel comfortable being vulnerable with us.\twant you guys to be able to feel comfortable with being honest and truthful with us.\t9,10\t9,13\tsubstitution\nshow_2cDEjUoE1xIZqEMHdy2iLg-3BhVKbLFPasrUOxLgZNUbd.wav\tthat schedule is one per week and it will probably be like a Wednesday night thing because I plan on doing one to two videos per week.\tthat schedule is one per week and you will start to see a lot more content arriving because I plan on doing one to two videos per week.\t7,15\t7,16\tsubstitution\nshow_2CLetGT20MFsHqfeBN3fYl-0lZ9jnYVgXx7HmKUcdTnJO.wav\thedge a little bit in a traditional Market since and whether its buying a bunch of gpus and then reselling them for 2x the price ah that happened.\thedge a little bit in a traditional Market by buying up a whole lot of single family homes and then reselling them for 2x the price ah that happened.\t8,16\t8,17\tsubstitution\nshow_2TPvj8tyUhY2UHOzU9kyu4-6lnZyS5yzd3S4Vaqw5TrHy.wav\tOkay, so then he moves on he says so I understand now before we look at everything in more detail.\tOkay, so then he moves on he says so I understand now before inspecting everything in more detail.\t13,15\t13\tsubstitution\nshow_2cH1Sf7Tg3TiDdGpD3oLiR-0opjgwiSz3AWOOoE49L9pi.wav\tThe Patriots will just skiing blocking but their backs and tight ends, maybe tighten the formations a little bit.\tThe Patriots will just focus on their wide receivers and tight ends, maybe tighten the formations a little bit.\t4,8\t4,8\tsubstitution\nshow_28hFGrNqCyS73hMP94FALm-3NRcwegtutZLbw1YC2DhhM.wav\tThe GP did not recommend talking therapies.\tThe GP did not suggest talking therapies.\t4\t4\tsubstitution\nshow_2CJ6f4oLCccT3fsUaWAk9k-3fVgo6u94DJHpK7uP1Qb7V.wav\tAnd, like comment subscribe give me feedback give me feedback.\tAnd, like comment subscribe give me your thoughts and any feedback.\t6,8\t6,9\tsubstitution\nshow_2cNIhBNwWmamJs75G3tMxY-4UyKjZff8srwoG8A71ql2K.wav\tfeel safe to get naked emotionally and mentally to share how they overcome their pain and suffering and how they grew into these bright lights.\tfeel safe to get so incredibly overwhelmed emotionally and mentally to share how they overcome their pain and suffering and how they grew into these bright lights.\t4\t4,6\tsubstitution\nshow_2cNIhBNwWmamJs75G3tMxY-1xe6sJ3hUH8wCHnPxCjbwu.wav\tAnd so I looked at it as very much like an organic problem and I said okay my brain needs this.\tAnd so I looked at it as very much like a complex issue and I said okay my brain needs this.\t10,12\t10,12\tsubstitution\nshow_28qfNqUaAXdF3TcEGapJ1d-7uzzKzrT6ggIoqv9gRPz7E.wav\tBut yeah, I was I was never the best student first year college was actually really really good and I even had a full time job at a time.\tBut yeah, I would say that I was never the best student in class, but first year college was actually really really good and I even had a full time job at a time.\t3,9\t3,14\tsubstitution\nshow_1tl5wg2z0fzjWR18MHKARa-3auJGSBu9ERKSjw44eKkhj.wav\tIt was because partly was because I had such an Early Peek.\tIt was because at least partly because I had such an Early Peek.\t3,4\t3,5\tsubstitution\nshow_1ChaMDlb8CNR7Bta8ZxODC-6gI5xAKjYcPiQ2cANcnG9q.wav\tAnd if we don't make it a priority the distractions will get us one of the things I love so much about Jesus is if you go back and look at the gospels, he was so focused.\tAnd if we don't make it a priority the distractions will get us one of the things I admire about Jesus is if you go back and look at the gospels, he was so focused.\t18,20\t18\tsubstitution\nshow_2chnqxY9vGUWIxn4JvvRpZ-5WanEbFbdssEwK77TcezxC.wav\treally supportive friends and firefighters in the fire service who've inspired me to go out there and do this.\treally supportive colleagues, friends, and family and they inspired me to go out there and do this.\t2,9\t2,7\tsubstitution\nshow_1CHJvc14dYPq0IsX5T0YAP-2FKEiw2NrWejqlw9IkaB1X.wav\tWithin I'd say within like half a week things changed in my house the energy changed in my house the relationship with my wife started to change and I was like does the magic started to happen?\tWithin I'd say within like half a week things changed in my house the relationship with my wife started to change and I was like does the magic started to happen?\t14,19\t13,14\tdeletion\nshow_2cyslpwM45TtVfznjVlCnL-4ECn8gmQeSRJJmM3eHg4TF.wav\tHis music worked better when he did live action TV shows for suspense and humanistic reactions to like scenes and intense situations.\tHis music worked better when he did live action TV shows for suspense and emotional responses to like scenes and intense situations.\t14,15\t14,15\tsubstitution\nshow_2TXTWqxZkLHF6k4cd5F8XN-1N8w7dUDxxqtRErZObJrbG.wav\tHelping me find my identity and high school, which I also was fortunate enough to find this amazing woman.\tHelping me find my high school, which I also was fortunate enough to find this amazing woman.\t4,5\t3,4\tdeletion\nshow_2Cz04p7U4u2lLSofHLYIeH-0KUpyjDzatq7f6TWjWgAdf.wav\tpassion and you're not gonna see those results and it's gonna stop you from actually making it get into it because you love it.\tpassion and you're not gonna like them and it's gonna stop you from actually making it get into it because you love it.\t5,7\t5,6\tsubstitution\nshow_2TjptLx9uQUaHhp6YB8jhW-0YIfNwpxL2ztlaSsmBTCKL.wav\tAnother thing is anything inside the parentheses turns the opposite so negative becomes positive and a positive becomes negative.\tAnother thing is that in this equation anything inside the parentheses turns the opposite so negative becomes positive and a positive becomes negative.\t2,3\t3,6\tinsertion\nshow_2Chp07kvTN1qDImtrnXm4O-7mdEKKOHue6R5MLo283EQm.wav\thope hope you got a better feel of organic versus paid look definitely keep an eye out for upcoming episodes because I'm a dive a lot more deep into the paid social world.\thope hope you got a better feel of how instagram stars fund their extravagant vacations definitely keep an eye out for upcoming episodes because I'm a dive a lot more deep into the paid social world.\t8,11\t8,14\tsubstitution\nshow_2ctsjdVxkuzqftlC9TJASy-6iadzuoEBJ9AOLXaXPmagP.wav\tSo if you've been following my story, you will remember that I said earlier in this podcast that the Grammy nominations came out.\tSo if you've been following my story, you will remember that I said earlier that this week we had super exciting stuff to talk about because Grammy nominations came out.\t14,18\t14,25\tsubstitution\nshow_1cPkxhnrYWUvCzd0uXMKwo-1auCDHN3NrKq4Bn0OrE0lM.wav\tfreedom is made with a key.\tfreedom is made by effort not with a key.\t2,3\t3,5\tinsertion\nshow_2CZeMpXywYmWy53SV2kWEm-2Ic0xbN3defufBYR46ooEi.wav\tSo for more craziness now that French was conquered we have to join forces to Great Britain.\tSo for more craziness now that French was conquered by the Germans, we have to join forces to Great Britain.\t8,9\t9,11\tinsertion\nshow_2CyWjLhTGlpGHeSpJOvxj2-6Hvd5G0lyzP62VYPmP1jQj.wav\tIt was one of those things, you know, you have project sometimes you start on the bottom and sometimes you start at the top.\tIt was one of those things, you know, some people just start on the bottom and sometimes you start at the top.\t8,12\t8,10\tsubstitution\nshow_2cNgsFoVxaxZkUnVU3ehQu-0mgpNxV3cnsvy7RXtF9OHv.wav\tTwenty years later it became 20 thousand times worse.\tTwenty years later it became thousand times worse.\t5\t4,5\tdeletion\nshow_2czbki8aNirvUjlYcO3I1t-7kdfTr9l9Egod1iFPzIqkK.wav\tAs a body and some individuals on ways and means we were speaking about possibly just passing the sales price disclosure on residential property.\tAs a body and some individuals on ways and means we were speaking about possibly just banning them on residential property.\t16,20\t16,17\tsubstitution\nshow_2cFZSZNdkxKdiTEE7yrAMB-06KUOjkKFLQgxfnC59GMtT.wav\tTyler also introduces the lack of closure that will bother him increasingly throughout the album's front half.\tTyler also introduces in great detail its outcome on the album as a whole, the lack of closure that will bother him increasingly throughout the album's front half.\t2,3\t3,13\tinsertion\nshow_2C2dO6pWL4cPzOJ2Bu7QRA-3I34aJLdGXgCEuY6rd90Tm.wav\tAnd then the other matchup is Seattle visits Philadelphia now Philadelphia has a worse record, but because they were not able to win their division.\tAnd then the other matchup is Seattle visits Philadelphia now Seattle has the better record, but because they were not able to win their division.\t10,13\t10,13\tsubstitution\nshow_2c2IJzenX6Q6gJxc2aGRf8-0z0etCBM2PrHOLc9gxc25E.wav\tMore of a base and infrastructure to tell those stories rather than doing it out of a out of a tent with solar power.\tMore of a base and infrastructure to fight these battles instead of out of a tent with solar power.\t7,16\t7,11\tsubstitution\nshow_1CxjAV2kY4pypL256BmRQ6-531d49VH5hVVMzXfKhsMFf.wav\tSo the fourth episode May the fourth be with you.\tSo the fourth place winner will chat with you.\t3,7\t3,6\tsubstitution\nshow_2TI9Upbk0gXEdxsYTPzB9W-4x4pLTV2ZiA98TIIWkK0JH.wav\tThis year and and the word started to spread in the lacrosse community and just in general.\tThis year and and the word started to spread rapidly and without any sense of control in the slightest in the lacrosse community and just in general.\t8,9\t9,18\tinsertion\nshow_2cOpF3UhdxdvlZZzyOVPHt-3GXDpd2ZHpy9YTMkLAjuZq.wav\tJoey Scott and Richie all sat together at the creek.\tJoey Scott and Albert enjoyed fishing trout at the creek.\t3,6\t3,6\tsubstitution\nshow_2TXUkJOq3oBEn2ROormwza-6ogJCqF2Ya7qZ2L23AAoPd.wav\tAre equipped with sensors to monitor the proximity of surroundings and can open in even the tightest parking spaces.\tAre equipped with sensors to detect collisions and can open in even the tightest parking spaces.\t5,9\t5,6\tsubstitution\nshow_28OttmVaPSfuB6e4cqX0yu-2tB2ldmTxblsCsC8QH7jnm.wav\tAnd we're at this point.\tAnd we're all extremely excited at this point.\t1,2\t2,4\tinsertion\nshow_28DAnHzOfbUoRkpj5OMqVI-58LUphKYwegjJP2ZQFjmUH.wav\tYou know best to study every day after your classes, especially because you know, you're still in the mood you're still in that flow.\tYou know best to study when well rested and well fed and when you're still in that flow.\t5,18\t5,12\tsubstitution\nshow_2T8QRK60cWaPQflfo6Wuc4-4oTO10xL7hQQS2fuXBy1d7.wav\tIn the pursuit of lightness minimal stress ultimate fulfillment.\tIn the pursuit of calm serenity an escape from stress ultimate fulfillment.\t4,5\t4,8\tsubstitution\nshow_2CN1XNYxo4NFClfUajCtSM-1PrlYFjZzosbVg4BvKzbLJ.wav\tand and you know that she has been around for a few decades now longer and it has such a story that it it it was established in basketball.\tand and you know that she has been around for as long as many young people today have been watching and it has such a story that it it it was established in basketball.\t10,14\t10,19\tsubstitution\nshow_2t94ceh3K4qorKbKXJw7NV-5r2eumfkrw5Ym2WYexqEpK.wav\tHello and welcome to the first of hopefully many you cannot ingest podcast.\tHello and welcome to the second of hopefully many you cannot ingest podcast.\t5\t5\tsubstitution\nshow_2c3EDnMjSm9bAr1fQgLmMg-0jzn6k4JPy4s3XOiQZFFF8.wav\tThey sit down that these plot points need to happen because they have a whole Board of index cards full of notes that need to happen in the story.\tThey sit down that these plot points need to happen because they have a whole Board of like fifty index cards full of notes that need to happen in the story.\t16,17\t17,18\tinsertion\nshow_2cQVtitXsGYcp9kIYBi9VJ-7wZR6aZIx7PTYFcShbre2k.wav\tCommunity so I didn't ever feel that openness until I moved back and I wasn't in that realm anymore.\tCommunity so I didn't ever feel that openness until I moved across the country and I wasn't in that realm anymore.\t11\t11,13\tsubstitution\nshow_1C49KB0vYZsFe9eoFAr2Cq-6ZkOgQuv6e4y74xhDNKc4Y.wav\tIf you ask me what you are I would have Alex Caruso as the starting point guard for the Lakers.\tIf you ask me what you are I would have Alex Caruso out on the court playing as the starting point guard for the Lakers.\t11,12\t12,16\tinsertion\nshow_2CMZqwsTyimKMEGMIdOFCz-2xWHgQryE2ruRadxVVbdbD.wav\tReally going to talk about why us and why now for this podcast.\tReally going to talk about who we are why we're here and why now for this podcast.\t5,6\t5,10\tsubstitution\nshow_2t5PIVQePC6L3CFRpAUnaf-0f0tl83ucovdSpJHoftEU5.wav\tNo words just lightning breaking darkness and crashing into the Earth with brilliant presence.\tNo words just lightning breaking darkness and crashing into the surface of the Earth with brilliant presence.\t9,10\t10,12\tinsertion\nshow_2cGQMNoS6MuKFozuNYjCOQ-6qmAgAKLoYpSknCgQ0y6ET.wav\tFor making the title though because I need to get my numbers way up before I get there, but I'm gonna get there title of Iceland is definitely going to sign me and um, yeah.\tFor making the title though because I need to get my numbers way up before I get there, but I'm gonna get there title of Iceland is going to sign me and um, yeah.\t27\t26,27\tdeletion\nshow_2tgc74udMU420iVPvl597O-0fORVMXyI1aCUobzMKm5Ll.wav\tFeeling into the eyes and the temples and the entire facial structure.\tFeeling into the eyes and getting a sense of the entire facial structure.\t5,7\t5,8\tsubstitution\nshow_2csQINhTs2YQOWmpmy5gmJ-3J4UWcEHj2lQvk4lRog4LZ.wav\tIt had proven to be an exciting challenge the last time he subdued a couple and Israel's expecting his second victim to be by soon.\tIt had proven to be an exciting challenge the last time he subdued a couple and he couldn't wait for his victim to be by soon.\t16,19\t16,20\tsubstitution\nshow_2tYdWKnaDR4D2qgCHml2Ax-1ntgzgJweV4WPav6lZUeK9.wav\tHave three rounds of attack before you switch on the defense and then you're going to have three rounds of Defense before you switch on to attack.\tHave three rounds of attack before you switch on the defense and then you're going to have three rounds of Defense after which you repeat again on to attack.\t21,23\t21,25\tsubstitution\nshow_2TuwSyFIHWD1UxyBCMLnWT-4N24WWvGmHzFS0BoqfGNRE.wav\tbreak out of their shell a little bit and you see oh gosh, they're really way more Hardy than I thought or funny than I thought or whatever because it's the moment they've drench themselves like the other day.\tbreak out of their shell a little bit and you see oh gosh, they're really way more Hardy than I thought or funny than I thought or whatever because it's the moment they recovered from harsh cold weather of the other day.\t32,35\t32,38\tsubstitution\nshow_1CzCdrVrUH7JwgyZnVGYLh-6Q5gndks7qNDYlyWQNThsb.wav\tHey guys, Tim Jennings here with soul heart with another episode of search engine optimization tips and tricks trying to get your site found on Google and other search engines like that.\tHey guys, Tim Robinson hosting the show today with another episode of search engine optimization tips and tricks trying to get your site found on Google and other search engines like that.\t3,7\t3,7\tsubstitution\nshow_1CUdmqDR1A47vPMjsiK6m2-0vStekPNMu57qySDbgL4Bz.wav\tAnd I think that's uh it's it's a fascinating uh Dynamic but uh I wanna thank you.\tAnd I think that's absolutely a super fresh, new, and exciting Dynamic but uh I wanna thank you.\t4,9\t4,10\tsubstitution\nshow_1tUHam5eF5aw1ANOoTTNHY-00rYk6fUFND3sgPVnFfOx5.wav\tYou know your body first and foremost because that is so important.\tYou know your friends and family first and foremost because that is so important.\t3\t3,5\tsubstitution\nshow_1Tazwk3AUA0uz6jQk0X2qx-6AnUhuFsREJdKTZ5YtJN67.wav\tI actually got I took the BET and I bet on the Cavs winning.\tI actually got I took the BET and I bet on the Patriots with Tom Brady winning.\t12\t12,15\tsubstitution\nshow_1c7paeaWBSC8lM2WmoE7oI-7ngmygKXeMj6llnxB9E5W5.wav\tWe also have to be able to observe ourselves and how we behave, why do we refuse to rest?\tWe also have to be able to observe ourselves and contemplate our decisions, such as why do we refuse to rest?\t10,12\t10,14\tsubstitution\nshow_1tPIbAQXvAfaZ9w2aUDVn5-2Otc04LmTGhUMuBF8U36Bt.wav\tteachers in elementary school and middle school are not that different.\tteachers in elementary school and in most universities are not that different.\t5,6\t5,7\tsubstitution\nshow_1CnVxnXxFzJyqAVe0gxVao-2MZDt0KXGXSM6ciptUHsJI.wav\tmaybe take like two shots a day of like that drink and I'm sorry sure when I saw my doctor that Monday she did an ultrasound.\tmaybe take like a day off or something because after the weekend when I saw my doctor that Monday she did an ultrasound.\t3,14\t3,11\tsubstitution\nshow_1TH2TkfOKETXMhheVKhnSF-4OR6mYxdRwIdRfZAZyIg0d.wav\tInteresting and I think this is a comment a much more common phenomenon nowadays is that she just found out that she has a fifth sibling?\tInteresting and I think she just found out that she has a fifth sibling?\t4,15\t3,4\tdeletion\nshow_1CmHgwWKnKTU94RPoVJVhm-2FjSP20WACh8xlrfgo27lv.wav\tNow maybe should we why don't we push this out to situation Nation?\tNow maybe should we think about if we want to push this out to situation Nation?\t4,6\t4,9\tsubstitution\nshow_1cY9S6222J0jGzYbHGQKPs-7ImJPzwLhq2ZjRrbYBMANb.wav\tOkay, what's um, what's one genre people will be shocked to know that you read.\tOkay, what's um, what's one genre people will be shocked to know you really like to read.\t12,13\t12,15\tsubstitution\nshow_1T69Xe0EJ4n0gOO4RD9qv0-42elfJEncMhPZCtpdwUX2Q.wav\tTo buy that vodka or putting the money at the dock passive and there's one other thing that actually I'm not sure if you you guys are really aware.\tTo buy that vodka or any other alcohol you need to show proof of age and there's one other thing that actually I'm not sure if you you guys are really aware.\t5,11\t5,14\tsubstitution\nshow_1t3ZatwPEux3wUnXMUE62z-2VItlcCcQodDtxIm1hEgLc.wav\tI knew that if I didn't just start even if it wasn't perfect that I would never start.\tI knew that if I didn't just get to work on it even if it wasn't perfect that I would never start.\t7\t7,11\tsubstitution\nshow_1T6df6cejtcf12QJZN0yUu-5mvPGk6dS60f0LCPGjbR4D.wav\tJust because someone isn't saying anything while you're talking doesn't mean they are processing a single word.\tJust because someone is quiet while you're talking doesn't mean they are processing a single word.\t3,5\t3,4\tsubstitution\nshow_1cKkutPWS7rRmyOtrSwouo-79txjRa3xuTvzkjLhRvBgR.wav\tIs an infinitely constricting Paradox if I try and Define how much needs to be done before I can enjoy an emotional experience.\tIs an infinitely constricting Paradox if I try and figure out everything I have to do before I can enjoy an emotional experience.\t9,15\t9,15\tsubstitution\nshow_1C98g10rH9mj7aiTgRGEtH-416OTsevOhXjjB6asFyyK9.wav\tAnd like I look back to our branding uh two years ago, like you can see what the website looked like back then horrendous.\tAnd like I look back to our branding back when we were first starting out as a company, like you can see what the website looked like back then horrendous.\t8,11\t8,17\tsubstitution\nshow_1CMwbrEPRtk46eseEmxFOd-6FlNy7MMX7L4J5KCyVYWYM.wav\tIf you screenshot somebody though, like if they send a picture and you take a screenshot, I think that person knows that you have screenshotted them.\tIf you screenshot somebody though, like if they send a picture and you take a screenshot, I think there's a little notification in the app telling them that you have screenshotted them.\t18,20\t18,26\tsubstitution\nshow_1TTQPQzpjtXPKadzUx5vo6-1nVtCzKPNliNLJ91Ck5TSq.wav\tSo my last question for you guys talk to me about your most memorable moment or the aha moment or the that feeling of just it was so wonderful to be part of this club.\tSo my last question for you guys talk to me about that feeling of just it was so wonderful to be part of this club.\t11,20\t10,11\tdeletion\nshow_1c4MlC6ClLyP8osRCtdTUs-2GXhBpBNH9GrdHorlNBcPo.wav\tPersonal trainers pretty much pretty much a teacher is just a teacher teaching you in assisting Youth of the goals that you want.\tPersonal trainers pretty much are just coaches I mean really they're like pretty much a teacher is just a teacher teaching you in assisting Youth of the goals that you want.\t3,4\t4,11\tinsertion\nshow_1coo0trh2Do1KR6ev6Fczv-3Zr9rSN8B9pjuNMjZVcYVM.wav\tSo that's why there's a definite divided in people's opinion on this and that's why it's been such a highly talked about issue.\tSo that's why there's a definite divided in people's opinion on this which is likely also why it's been such a highly talked about issue.\t12,13\t12,15\tsubstitution\nshow_1cVKtsokch166IPm3tRg0U-4XdMT9MFto7G1IWnaH7fZV.wav\tProhibits that so by getting rid of homework.\tProhibits that so we're getting rid of homework.\t3\t3\tsubstitution\nshow_1CLAwGQAgTZIzV0TBj254v-51nbmcVD8allN0g5hLbhN7.wav\tWent Caspian and said Lord King slay me speedily as a great traitor for buy my silence.\tWent Caspian and said if you'll see me slain I ask only that it be done speedily as a great traitor for buy my silence.\t4,7\t4,15\tsubstitution\nshow_1TsDtgHbctWFu1B856QLI0-6LykTfQQJFPhKfoVzNdtgA.wav\tEnvironment has changed and yes, it is easy to say our environment is out to get us and it might not be your fault.\tEnvironment has changed and yes, it is easy to say our environment is dying right now and it might not be your fault.\t13,16\t13,15\tsubstitution\nshow_1cOsDxbQjLADedwZaG7Bm1-6mLz726LYCcgN69RWWlrOJ.wav\tFast cars, that had the nice clothes, that had the money, they was criminals.\tFast cars, that had the nice clothes, that had expensive gold watches, that had the money, they was criminals.\t8,9\t9,13\tinsertion\nshow_1ttEqOUCJnc7JAGNCWaAqq-471ymSOOetlSAecJK9Bfhr.wav\tKind of a great time he kind of gets to pair with uh Quinn Priester and those to kind of get to come up through the minor leagues together.\tKind of a great time he gets to team up with Quinn Priester and those to kind of get to come up through the minor leagues together.\t6,12\t6,10\tsubstitution\nshow_1CB8GHgtZAnsn6ihBUOWKo-1DyrYzUYIq5Zx4edeHge5Y.wav\tback the Coca Cola, and then everyone be happy and buy it more, then they make more money.\tback the Coca Cola, and then everyone buy it more, then they make more money.\t7,9\t6,7\tdeletion\nshow_1tzXR6tf3WGxV9nNWolDMN-5ZayFE8KG7W18jlM9t5d44.wav\tAnd, just either let the balloon go she would count down uh and then we would all think of what we wanted to let go and then we would let the balloon go or blow out our candles and then it was gone.\tAnd, just either let the balloon go she would count down uh and then we would celebrate this moment togather and then we would let the balloon go or blow out our candles and then it was gone.\t16,24\t16,19\tsubstitution\nshow_1tAewNZS0q8QPQpIIEUQQ0-3juUg3wFn3w7OFFo0sGe6R.wav\tWhen they killed them they turned back into the packed humans that were there.\tWhen they killed them they turned back into the terrified ponies that were there.\t9,10\t9,10\tsubstitution\nshow_1T15rqmPErKONqSx9rzr9H-726cSurFjtPFS9fiEAMT6b.wav\tUse those email templates verbatim verbatim, but make sure y'all very careful with them because there's merge codes that they have preselected that might not be.\tUse those email templates verbatim, but make sure y'all very careful with them because there's merge codes that they have preselected that might not be.\t4\t3,4\tdeletion\nshow_1cdpRq4rWNv1xYw3yab7b7-1BDuArBpFR2bZmv4cNafcl.wav\tSo I'm pretty sure I wanted to be a teacher so I could just tell everyone what to do.|So I'm pretty convinced that I wanted to be a teacher so I could just tell everyone what to do.\tSo I'm pretty convinced that I wanted to be a teacher so I could just tell everyone what to do.|So I'm pretty convinced that I wanted to be a teacher so that I can tell everyone what to do.\t3|11,13\t3,4|12,14\tsubstitution|substitution\nshow_2C0AgUOt4eCULjFjb3mynN-6Uv8Y4yw6o4V2zCKZrzrPg.wav\tSee why it's extremely valuable to it's kind of like it's kind of like having a wall hack to watch a demo.|See why it's extremely important right? it's kind of like it's kind of like having a wall hack to watch a demo.\tSee why it's extremely important right? it's kind of like it's kind of like having a wall hack to watch a demo.|See why it's extremely important right? it's kind of like having a rough time to watch a demo.\t4,5|10,17\t4,5|10,13\tsubstitution|substitution\nshow_2czpNd58pfuIxOCvO2czHu-2U9S7MpxvVUR6sPTdFRUWR.wav\tSo um yea that's it for this episode of the podcast what I will let say just eh that just as we come to the end of the podcast.|So um yea that's pretty much all for this episode of the podcast what I will let say just eh that just as we come to the end of the podcast.\tSo um yea that's pretty much all for this episode of the podcast what I will let say just eh that just as we come to the end of the podcast.|So um yea that's pretty much all for this episode of the podcast as we come to the end of the podcast.\t4|11,19\t4,6|12,13\tsubstitution|deletion\nshow_2c04iZbAAIYmZrTIRgggNc-4zacmnwJi3osMV5beOYGLu.wav\tThen at the same time in my mind mentally I kept screaming and yelling and mentally I'm thinking to myself I'm screaming and yelling at the top of my lungs.|Then at the same time I feel like I'm trapped mentally and I'm thinking to myself I'm screaming and yelling at the top of my lungs.\tThen at the same time I feel like I'm trapped mentally and I'm thinking to myself I'm screaming and yelling at the top of my lungs.|Then at the same time I feel like I'm trapped mentally and I'm thinking to myself I'm screaming at the top of my lungs.\t5,15|22,23\t5,11|17,18\tsubstitution|deletion\nshow_2co4uJEBlUoi9JanlRz6ls-1QGXaI6j7lYMr3jofmM7Vy.wav\tIf you ever wondered how I make my podcast guys, well I use anchor anchor is free.|If you ever thought about how I make my podcast guys, well I use anchor anchor is free.\tIf you ever thought about how I make my podcast guys, well I use anchor anchor is free.|If you ever thought about how I make my podcast guys, well the key is a tool called anchor anchor is free.\t3|11,12\t3,4|12,17\tsubstitution|substitution\nshow_1TMFh5H29pHWPD6KizMrlq-0nCgrfdU9zWUl6XJUnbsUX.wav\tserved an incredibly big purpose doing that that being said all of that was me trying to instill those things into my life.|served an incredibly long time doing that that being said all of that was me trying to instill those things into my life.\tserved an incredibly long time doing that that being said all of that was me trying to instill those things into my life.|served an incredibly long time doing that that being said all of that was me trying to instill things into my life.\t3,4|18\t3,4|17,18\tsubstitution|deletion\nshow_28ZmEMgyEUzHCmoW9DdwK3-12YTpVg7Ko2mRc4Si6OtDu.wav\tBe a good stress as well something that you know, you could be controlling something that won't you know, take a mental toll on you.|Be a good stress as well you know, you could be controlling something that won't you know, take a mental toll on you.\tBe a good stress as well you know, you could be controlling something that won't you know, take a mental toll on you.|Be a good stress as well you know, you could be controlling something that will not take a mental toll on you.\t6,7|16,18\t5,6|14,15\tdeletion|substitution\nshow_2toX0f3dPmI8gmUSOKZicx-1fJ8FUGSZLyb5fpbd3QDSi.wav\tThis year has been like my entire Journey so far in the music business and I'm just looking forward to what's to come.|This year has been the best part of my Journey so far in the music business and I'm just looking forward to what's to come.\tThis year has been the best part of my Journey so far in the music business and I'm just looking forward to what's to come.|This year has been the best part of my Journey so far in the acting business and I'm just looking forward to what's to come.\t4,6|12\t4,8|14\tsubstitution|substitution\nshow_2txZW3TWakg6Pr41kcbiA6-2MlY5WOs8ScY5zTfGRJHDc.wav\tAnd but with this job it was like I was staring at a computer like for 10 hours 8 hours and then maybe doing therapy like for nine hours a week.|And but with this job I had to just be like reading or typing away on a computer like for 10 hours 8 hours and then maybe doing therapy like for nine hours a week.\tAnd but with this job I had to just be like reading or typing away on a computer like for 10 hours 8 hours and then maybe doing therapy like for nine hours a week.|And but with this job I had to just be like reading or typing away on a computer like for 10 hours 8 hours and then maybe doing digital art for nine hours a week.\t5,11|24,25\t5,15|28,29\tsubstitution|substitution\nshow_2CN1XNYxo4NFClfUajCtSM-75tS2ZoJCscJetPxi0aukT.wav\tSo, you know like there was a there was an example where I bought two or three pair and literally just gave them to Goodwill as I was moving because the I never wore them they still had tags on them.|So, you know like there was an example where I bought two or three pair and literally just gave them to Goodwill as I was moving because the I never wore them they still had tags on them.\tSo, you know like there was an example where I bought two or three pair and literally just gave them to Goodwill as I was moving because the I never wore them they still had tags on them.|So, you know like there was an example where I bought two or three pair and then just gave them to Goodwill as I was moving because the I never wore them they still had tags on them.\t6,8|19\t5,6|16\tdeletion|substitution\nshow_2Cp52s1B4vepSJi2F8gmU9-36PtNWkMxWZtxbqBJ4mUXD.wav\tAnd I want to mention that there were a couple of teachers who really helped me during that time.|And I want to mention that there were a couple of great teachers who really helped me during that time.\tAnd I want to mention that there were a couple of great teachers who really helped me during that time.|And I want to mention that there were a couple of great teachers who noticed and reached out when I was struggling and helped me during that time.\t10,11|13\t11|14,22\tinsertion|substitution\nshow_2T23esVRXBfFb5vigvG7A5-6JivmdWNP3UnZiIOplv953.wav\tIt's hard to say how I get things off the ground because I just get going like I don't know how to explain it.|It's hard to say how I get the system running so quickly because I just get going like I don't know how to explain it.\tIt's hard to say how I get the system running so quickly because I just get going like I don't know how to explain it.|It's hard to say how I get the system running so quickly because I just get going I don't know how to explain it.\t7,10|16\t7,11|16,17\tsubstitution|deletion\nshow_2T3Pjyw6MJEwPE9uixrYak-7M5817WSsGaldlQfZptcyf.wav\tI know now how extremely lucky that truly is what a blessing Not only was I going to be a mom but besides nausea all my other symptoms went away.|I know now how incredibly lucky I really was, what a blessing Not only was I going to be a mom but besides nausea all my other symptoms went away.\tI know now how incredibly lucky I really was, what a blessing Not only was I going to be a mom but besides nausea all my other symptoms went away.|I know now how incredibly lucky I really was, what a blessing Not only was I going to be a mom but except for nausea all my other symptoms went away.\t4,8|22\t4,8|22,23\tsubstitution|substitution\nshow_1C5B3zuyd67j7v9XRKNb2L-4Az7OglwsgYR94ikvOZdCf.wav\tPast something and I have so many things that I have planned for this podcast, but first before we get into any of that I'm going to introduce myself.|Past something and I have so many things that I have planned to talk about today, but first before we get into any of that I'm going to introduce myself.\tPast something and I have so many things that I have planned to talk about today, but first before we get into any of that I'm going to introduce myself.|Past something and I have so many things that I have planned to talk about today, but first before we get into any of that I would like to introduce myself.\t12,14|24,25\t12,15|25,27\tsubstitution|substitution\nshow_1c8f0MS5LcfbSvwexFC9mn-1MY1u1xOyiFAWXGRqyPJj7.wav\tYou know with hesitation everything is counted to the T and says if every drink is measured, how are you going to give a regular an honest poor?|You know with hesitation everything is counted to the T and says if you're calculating the nutritional information, how are you going to give a regular an honest poor?\tYou know with hesitation everything is counted to the T and says if you're calculating the nutritional information, how are you going to give a regular an honest poor?|You know with hesitation everything is counted to the T and says if you're calculating the nutritional information, how are you going to provide the same service to an honest poor?\t13,16|22,24\t13,17|23,27\tsubstitution|substitution" }, { "path": "cog.yaml", "content": "# Configuration for Cog ⚙️\n# Reference: https://github.com/replicate/cog/blob/main/docs/yaml.md\n\nbuild:\n gpu: true\n system_packages:\n - libgl1-mesa-glx\n - libglib2.0-0\n - ffmpeg\n - espeak-ng\n python_version: \"3.11\"\n python_packages:\n - torch==2.1.0\n - torchaudio==2.1.0 \n - xformers\n - phonemizer==3.2.1\n - whisperx==3.1.1\n - openai-whisper>=20231117\n run:\n - git clone https://github.com/facebookresearch/audiocraft && pip install -e ./audiocraft\n - pip install \"pydantic<2.0.0\"\n - curl -o /usr/local/bin/pget -L \"https://github.com/replicate/pget/releases/download/v0.6.0/pget_linux_x86_64\" && chmod +x /usr/local/bin/pget\n - mkdir -p /root/.cache/torch/hub/checkpoints/ && wget --output-document \"/root/.cache/torch/hub/checkpoints/wav2vec2_fairseq_base_ls960_asr_ls960.pth\" \"https://download.pytorch.org/torchaudio/models/wav2vec2_fairseq_base_ls960_asr_ls960.pth\"\npredict: \"predict.py:Predictor\"\n" }, { "path": "config.py", "content": "import argparse\n\n\ndef MyParser():\n parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n # general training \n parser.add_argument(\"--seed\", type=int, default=1)\n parser.add_argument(\"--precision\", type=str, default=\"float16\")\n parser.add_argument(\"--num_workers\", type=int, default=8)\n parser.add_argument(\"--resume\", action=\"store_true\", default=False)\n parser.add_argument(\"--tb_write_every_n_steps\", type=int, default=100)\n parser.add_argument(\"--print_every_n_steps\", type=int, default=400)\n parser.add_argument(\"--val_every_n_steps\", type=int, default=800)\n parser.add_argument(\"--lr\", type=float, default=0.05)\n parser.add_argument(\"--batch_size\", type=int, default=100, help=\"this is the effective batch size, no matter whether using gradient_accumulation_steps, not used if we specified max_num_tokens\")\n parser.add_argument(\"--max_num_tokens\", type=int, default=100000, help=\"max number of encodec tokens per gpu, this is only used when using dynamic batching, will ignore batch size. Note this is the final effective batch size per GPU, i.e. gradient accumulated batch size per gpu\")\n parser.add_argument(\"--val_max_num_tokens\", type=int, default=None, help=\"FOR validation\")\n parser.add_argument(\"--num_buckets\", type=int, default=6, help='used for dynamic batching, bucketing the samples based on the number of tokens')\n parser.add_argument(\"--dynamic_batching\", type=int, default=0)\n parser.add_argument(\"--weight_decay\", type=float, default=1e-2)\n parser.add_argument(\"--warmup_fraction\", type=float, default=0.01, help=\"use linear warmup, the proportion of the training steps that are used for warming up\")\n parser.add_argument(\"--num_epochs\", type=int, default=10)\n parser.add_argument(\"--num_steps\", type=int, default=None, help=\"if not None, will ignore n_epochs and use num_steps as the total number of amount of training, can try e.g. 400000 i.e. 400k steps\")\n parser.add_argument(\"--gradient_accumulation_steps\", type=int, default=1)\n parser.add_argument(\"--gradient_clip_val\", type=float, default=1.0, help=\"the value for torch.nn.utils.clip_grad_norm_(), not used if we use ScaledAdam optimizer\")\n parser.add_argument(\"--early_stop_step\", type=int, default=3200, help=\"stop training after this many steps of non-improvement\")\n parser.add_argument(\"--early_stop_threshold\", type=float, default=-1.0, help=\"early stop after the improvement is below this threshold for certain number of steps\")\n\n # optimizer focused\n parser.add_argument(\"--optimizer_name\", type=str, default=\"AdamW\", help=\"can also use ScaledAdam, in which case we'll also use the Eden scheduler\")\n parser.add_argument(\"--reduce_lr_start_step\", type=int, default=3000, help='after which significantly reduce the lr. a param for the eden optimizer')\n parser.add_argument(\"--pseudo_epoch_size\", type=int, default=3000, help=\"only use for Eden scheduler.\")\n parser.add_argument(\"--reduce_lr_start_epoch\", type=int, default=4)\n parser.add_argument(\"--clipping_update_period\", type=int, default=600)\n\n\n # path\n parser.add_argument(\"--exp_dir\", type=str, default=None, help=\"will be combined with dataset name\")\n parser.add_argument(\"--dataset\", type=str, help=\"e.g. 'libritts', 'gigaspeech', they are folder name in the data dir also\")\n parser.add_argument(\"--dataset_dir\", type=str, help=\"need to be compatible with corresponding dataset py file\")\n parser.add_argument(\"--phn_folder_name\", type=str, default=\"phonemes\", help=\"for libritts I also have arpa phns, in which case should be phonemes_arpa\")\n parser.add_argument(\"--encodec_folder_name\", type=str, default=\"encodec_16khz_4codebooks\", help=\"folder where encodec codes are stored\")\n parser.add_argument(\"--manifest_name\", type=str, default=\"manifest\", help=\"metadata filename\")\n\n # data focused\n parser.add_argument(\"--pad_x\", type=int, default=1, help=\"whether or not always pad x to have text_max_length. select 1 to get the maximal memory consumption, but the actual case should be smaller, better to have it being 0\")\n parser.add_argument(\"--audio_max_length\", type=float, default=20, help=\"in second, crop or drop the audio is length is longer than this\")\n parser.add_argument(\"--audio_min_length\", type=float, default=2, help=\"in second, drop the audio if length is shorter than this\")\n parser.add_argument(\"--text_max_length\", type=int, default=400, help='if too long, we crop or drop')\n parser.add_argument(\"--text_min_length\", type=float, default=10, help=\"if too short, will drop\")\n parser.add_argument(\"--encodec_sr\", type=int, default=50, help=\"for my encodec that takes 16kHz audio with a downsample rate of 320, the codec sample rate is 50Hz, i.e. 50 codes (x n_codebooks) per second\")\n parser.add_argument(\"--drop_long\", type=int, default=0, help=\"if this is true, will drop example whose encodec sequence or phone sequence is too long, rather than cropping, to reduce hellucination\")\n\n # encodec and token rearrangement\n parser.add_argument('--mask_len_min', type=int, default=1, help='Minimum mask length')\n parser.add_argument('--mask_len_max', type=int, default=600, help='Maximum mask length')\n parser.add_argument(\"--eos\", type=int, default=-1, help=\"this is to be used with reduced_eog, where we end the utterance with eos, and end the generated segment with eog, also when this is used, the n_special should be 4\")\n parser.add_argument(\"--reduced_eog\", type=int, default=0, help=\"for the non-final segments, do not insert eog at the end, this could hopefully solve the early stopping issue when doing tts\")\n parser.add_argument(\"--special_first\", type=int, default=0, help=\"if 1, need to have special tokens to be the first few tokens, e.g. 0, 1, 2, which means we need to adjust the preprocessing and postprocessing of the encodec codes. note that we hard coded to have 3 special tokens\")\n parser.add_argument(\"--n_special\", type=int, default=3, help=\"empty, eog, pad, (eos)\")\n parser.add_argument(\"--codebook_weight\", type=str, default=None, help=\"e.g. ['5','1','0.5','0.1']\")\n parser.add_argument(\"--max_mask_portion\",type=float,default=0.7,help=\"should mask a utterance for more than this portion\")\n parser.add_argument(\"--max_n_spans\", type=int, default=3, help='maximal number of spans, only use when using multicm3, this is used to decide number of mask_embedding, and max clamp value if use Poisson distribution, if use uniform distribution to sample number of spans if will be uniform(1,max_n_spans)')\n parser.add_argument(\"--shuffle_mask_embedding\", type=int, default=0, help=\"whether shuffle the mask embedding, so that mask:0 is not the most well trained, default is not shuffling. The default has it's benefit, as it make sure that mask:0 always appear the first\")\n parser.add_argument(\"--mask_sample_dist\", type=str, default=\"poisson1\", help=\"uniform or poissonx, e.g. poisson1, meaning the parameter lambda is 1, it will most likely sample 1 masks\")\n parser.add_argument(\"--min_gap\", type=int, default=5, help=\"after sampled starts, delete later one if it closer to the former start than the min_gap\")\n parser.add_argument('--n_codebooks', type=int, default=4)\n parser.add_argument('--text_vocab_size', type=int, default=100, help='Size of text vocabulary')\n parser.add_argument('--text_pad_token', type=int, default=100, help='padding of the text tokens, not attended')\n parser.add_argument('--audio_vocab_size', type=str, default='2048', help=\"Size of audio vocabulary\")\n parser.add_argument(\"--empty_token\", default=2048, type=int, help=\"indicating the no token at the position for the codebook\")\n parser.add_argument('--eog', type=int, default=2049, help='End of generation token')\n parser.add_argument('--audio_pad_token', type=int, default=2050, help='padding of the encodec codes, not attended')\n\n # model focused\n parser.add_argument('--d_model', type=int, default=2048, help='Model dimension')\n parser.add_argument('--audio_embedding_dim', type=int, default=2048, help='dimension for encodec continues embedding (before being quantized)')\n parser.add_argument('--text_embedding_dropout', type=float, default=0.1, help='Dropout for text embedding')\n parser.add_argument('--audio_embedding_dropout', type=float, default=0, help='Dropout for audio embedding')\n parser.add_argument('--text_positional_embedding_dropout', type=float, default=0.1, help='Dropout for text positional embedding')\n parser.add_argument('--audio_positional_embedding_dropout', type=float, default=0.1, help='Dropout for audio positional embedding')\n parser.add_argument('--trm_dropout', type=float, default=0.1, help='Dropout for transformer')\n parser.add_argument('--nhead', type=int, default=16, help='Number of attention heads')\n parser.add_argument('--num_decoder_layers', type=int, default=16, help='Number of decoder layers')\n parser.add_argument('--load_model_from', type=str, default=None, help='Path to load model from, this will be effective last, so will overwrite all previous load, including resume')\n return parser" }, { "path": "data/__init__.py", "content": "" }, { "path": "data/gigaspeech.py", "content": "import os\nimport torch\nimport random\nimport copy\nimport logging\nimport shutil\n\nclass dataset(torch.utils.data.Dataset):\n def __init__(self, args, split):\n super().__init__()\n self.args = args\n self.split = split\n assert self.split in ['train', 'validation', 'test']\n manifest_fn = os.path.join(self.args.dataset_dir, self.args.manifest_name, self.split+\".txt\")\n\n with open(manifest_fn, \"r\") as rf:\n data = [l.strip().split(\"\\t\") for l in rf.readlines()]\n lengths_list = [int(item[-1]) for item in data]\n self.data = []\n self.lengths_list = []\n for d, l in zip(data, lengths_list):\n if l >= self.args.encodec_sr*self.args.audio_min_length:\n if self.args.drop_long and l > self.args.encodec_sr*self.args.audio_max_length:\n continue\n self.data.append(d)\n self.lengths_list.append(l)\n logging.info(f\"number of data points for {self.split} split: {len(self.lengths_list)}\")\n\n # phoneme vocabulary\n vocab_fn = os.path.join(self.args.dataset_dir,\"vocab.txt\")\n shutil.copy(vocab_fn, os.path.join(self.args.exp_dir, \"vocab.txt\"))\n with open(vocab_fn, \"r\") as f:\n temp = [l.strip().split(\" \") for l in f.readlines() if len(l) != 0]\n self.phn2num = {item[1]:int(item[0]) for item in temp}\n \n self.symbol_set = set([\"\", \"\", \"\", \"\"])\n \n def __len__(self):\n return len(self.lengths_list)\n \n def _load_phn_enc(self, index):\n item = self.data[index]\n pf = os.path.join(self.args.dataset_dir, self.args.phn_folder_name, item[1]+\".txt\")\n ef = os.path.join(self.args.dataset_dir, self.args.encodec_folder_name, item[1]+\".txt\")\n try:\n with open(pf, \"r\") as p, open(ef, \"r\") as e:\n phns = [l.strip() for l in p.readlines()]\n assert len(phns) == 1, phns\n x = [self.phn2num[item] for item in phns[0].split(\" \") if item not in self.symbol_set] # drop [\"\", \"\", \"\", \"\"], as they are not in training set annotation\n encos = [l.strip().split() for k, l in enumerate(e.readlines()) if k < self.args.n_codebooks]\n \n assert len(encos) == self.args.n_codebooks, ef\n if self.args.special_first:\n y = [[int(n)+self.args.n_special for n in l] for l in encos]\n else:\n y = [[int(n) for n in l] for l in encos]\n except Exception as e:\n logging.info(f\"loading failed for {pf} and {ef}, maybe files don't exist or are corrupted\")\n logging.info(f\"error message: {e}\")\n return [], [[]]\n\n return x, y\n\n def __getitem__(self, index):\n x, y = self._load_phn_enc(index)\n x_len, y_len = len(x), len(y[0])\n\n if x_len == 0 or y_len == 0:\n return {\n \"x\": None, \n \"x_len\": None, \n \"y\": None, \n \"y_len\": None, \n \"y_mask_interval\": None, # index y_mask_interval[1] is the position of start_of_continue token\n \"extra_mask_start\": None # this is only used in VE1\n }\n while y_len < self.args.encodec_sr*self.args.audio_min_length:\n assert not self.args.dynamic_batching\n index = random.choice(range(len(self))) # regenerate an index\n x, y = self._load_phn_enc(index)\n x_len, y_len = len(x), len(y[0])\n if self.args.drop_long:\n while x_len > self.args.text_max_length or y_len > self.args.encodec_sr*self.args.audio_max_length:\n index = random.choice(range(len(self))) # regenerate an index\n x, y = self._load_phn_enc(index)\n x_len, y_len = len(x), len(y[0])\n\n ### padding and cropping below ###\n ### padding and cropping below ###\n # adjust the length of encodec codes, pad to max_len or randomly crop\n orig_y_len = copy.copy(y_len)\n max_len = int(self.args.audio_max_length * self.args.encodec_sr)\n if y_len > max_len:\n audio_start = random.choice(range(0, y_len-max_len))\n for i in range(len(y)):\n y[i] = y[i][audio_start:(audio_start+max_len)]\n y_len = max_len\n else:\n audio_start = 0\n if not self.args.dynamic_batching:\n pad = [0] * (max_len - y_len) if self.args.sep_special_token else [self.args.audio_pad_token] * (max_len - y_len)\n for i in range(len(y)):\n y[i] = y[i] + pad\n \n # adjust text\n # if audio is cropped, and text is longer than max, crop max based on how audio is cropped\n if audio_start > 0 and len(x) > self.args.text_max_length: # if audio is longer than max and text is long than max, start text the way audio started\n x = x[int(len(x)*audio_start/orig_y_len):]\n if len(x) > self.args.text_max_length: # if text is still longer than max, cut the end\n x = x[:self.args.text_max_length]\n \n x_len = len(x)\n if x_len > self.args.text_max_length:\n text_start = random.choice(range(0, x_len - self.args.text_max_length))\n x = x[text_start:text_start+self.args.text_max_length]\n x_len = self.args.text_max_length\n elif self.args.pad_x and x_len <= self.args.text_max_length:\n pad = [0] * (self.args.text_max_length - x_len) if self.args.sep_special_token else [self.args.text_pad_token] * (self.args.text_max_length - x_len)\n x = x + pad\n ### padding and cropping above ###\n ### padding and cropping above ###\n\n return {\n \"x\": torch.LongTensor(x), \n \"x_len\": x_len, \n \"y\": torch.LongTensor(y), \n \"y_len\": y_len\n }\n \n\n def collate(self, batch):\n out = {key:[] for key in batch[0]}\n for item in batch:\n if item['x'] == None: # deal with load failure\n continue\n for key, val in item.items():\n out[key].append(val)\n res = {}\n if self.args.pad_x:\n res[\"x\"] = torch.stack(out[\"x\"], dim=0)\n else:\n res[\"x\"] = torch.nn.utils.rnn.pad_sequence(out[\"x\"], batch_first=True, padding_value=self.args.text_pad_token)\n res[\"x_lens\"] = torch.LongTensor(out[\"x_len\"])\n if self.args.dynamic_batching:\n if out['y'][0].ndim==2:\n res['y'] = torch.nn.utils.rnn.pad_sequence([item.transpose(1,0) for item in out['y']],padding_value=self.args.audio_pad_token)\n res['y'] = res['y'].permute(1,2,0) # T B K -> B K T\n else:\n assert out['y'][0].ndim==1, out['y'][0].shape\n res['y'] = torch.nn.utils.rnn.pad_sequence(out['y'], batch_first=True, padding_value=self.args.audio_pad_token)\n else:\n res['y'] = torch.stack(out['y'], dim=0)\n res[\"y_lens\"] = torch.LongTensor(out[\"y_len\"])\n res[\"text_padding_mask\"] = torch.arange(res['x'][0].shape[-1]).unsqueeze(0) >= res['x_lens'].unsqueeze(1)\n res[\"audio_padding_mask\"] = torch.arange(res['y'][0].shape[-1]).unsqueeze(0) >= res['y_lens'].unsqueeze(1)\n return res" }, { "path": "data/phonemize_encodec_encode_hf.py", "content": "import argparse\ndef parse_args():\n parser = argparse.ArgumentParser(description=\"encode the librilight dataset using encodec model\")\n parser.add_argument(\"--dataset_size\", type=str, default='xs', help='sizes of gigaspeech, xs, s, m, l, xl. we use xl for VoiceCraft training, xs is good for debugging')\n parser.add_argument('--download_to', type=str, default=\"/data/scratch/pyp/datasets/gigaspeech_debug\", help=\"dir where you want the huggingface gigaspeech dataset to be downloaded to\")\n parser.add_argument('--save_dir', type=str, default=\"/data/scratch/pyp/datasets/gigaspeech_phn_enc_manifest_debug\", help=\"path to the manifest, phonemes, and encodec codes dirs\")\n parser.add_argument('--encodec_model_path', type=str, default=\"/data/scratch/pyp/exp_pyp/audiocraft/encodec/xps/6f79c6a8/checkpoint.th\")\n parser.add_argument('--n_workers', type=int, default=4, help=\"Number of parallel worker processes\")\n parser.add_argument('--mega_batch_size', type=int, default=100, help=\"Number of samples in each mega batch for multiprocess dataloading\")\n parser.add_argument('--batch_size', type=int, default=4, help=\"batch size for encodec encoding, decrease it if OOM. This is the sum of batch size *over each gpu*, so increase it if you are using more gpus\")\n parser.add_argument('--model_sr', type=int, default=16000, help='encodec input audio sample rate')\n parser.add_argument('--downsample_rate', type=int, default=320, help='encodec downsample rate')\n parser.add_argument('--model_code_sr', type=int, default=50, help='encodec model code sample rate')\n parser.add_argument('--len_cap', type=float, default=35.0, help='will drop audios that are longer than this number')\n parser.add_argument('--max_len', type=int, default=30000, help='max length of audio in samples, if exceed, will cut a batch into half to process, decrease this number if OOM on your machine')\n return parser.parse_args()\nif __name__ == \"__main__\":\n import logging\n formatter = (\n \"%(asctime)s [%(levelname)s] %(filename)s:%(lineno)d || %(message)s\"\n )\n logging.basicConfig(format=formatter, level=logging.INFO)\n args = parse_args()\n\n import os\n import numpy as np\n import torch\n import tqdm\n import time\n from datasets import load_dataset, DownloadConfig\n\n from tokenizer import TextTokenizer, tokenize_text\n \n # get the path\n phn_save_root = os.path.join(args.save_dir, args.dataset_size, \"phonemes\")\n codes_save_root = os.path.join(args.save_dir, args.dataset_size, \"encodec_16khz_4codebooks\")\n vocab_fn = os.path.join(args.save_dir, args.dataset_size, \"vocab.txt\")\n os.makedirs(phn_save_root, exist_ok=True)\n os.makedirs(codes_save_root, exist_ok=True)\n\n\n def sort_by_audio_len(lens):\n inds = np.argsort(lens).tolist()\n logging.info(f\"longest: {lens[inds[-1]]*args.model_code_sr} encodec codes, {lens[inds[-1]]:.2f} sec.\")\n logging.info(f\"shortest: {lens[inds[0]]*args.model_code_sr} encodec codes, {lens[inds[0]]:.2f} sec.\")\n logging.info(f\"median: {lens[inds[len(inds)//2]]*args.model_code_sr} encodec codes, {lens[inds[len(inds)//2]]:.2f} sec.\")\n logging.info(f\"95 percentile longest: {lens[inds[int(len(inds)*0.95)]]*args.model_code_sr} encodec codes, {lens[inds[int(len(inds)*0.95)]]:.2f} sec.\")\n return inds[::-1]\n \n def write_array_to_txt_file(array, filename):\n with open(filename, 'w') as f:\n for a in array[:-1]:\n f.write(' '.join(map(str, a))+'\\n')\n f.write(' '.join(map(str, array[-1])))\n \n\n ### phonemization\n # load tokenizer\n # load the encodec model\n from audiocraft.solvers import CompressionSolver\n model = CompressionSolver.model_from_checkpoint(args.encodec_model_path)\n model = model.cuda()\n model = model.eval()\n text_tokenizer = TextTokenizer()\n\n\n # https://github.com/SpeechColab/GigaSpeech\n # there are only four different punctuations\n # need to check whether there are other < started strings\n punc2sym = {\" \": \",\", \" \": \".\", \" \": \"?\", \" \": \"!\"} # note the space in front of each punc name\n gar2sym = {\"\": \"#%#\", \"\": \"##%\", \"\": \"%%#\", \"\":\"%#%\"} # so that they are savely keep as the original sym when using tokenize_text\n punc2sym.update(gar2sym)\n\n word2sym = { \"h æ ʃ h ɐ ʃ p ɚ s ɛ n t\": \"\", \"h æ ʃ p ɚ s ɛ n t h æ ʃ\": \"\", \"p ɚ s ɛ n t h ɐ ʃ p ɚ s ɛ n t\": \"\", \"p ɚ s ɛ n t p ɚ s ɛ n t h æ ʃ\": \"\"}\n forbidden_words = set(['#%#', '##%', '%%#', '%#%'])\n\n dc = DownloadConfig(cache_dir=args.download_to)\n stime = time.time()\n logging.info(\"loading the dataset...\")\n gs = load_dataset(\"speechcolab/gigaspeech\", args.dataset_size, use_auth_token=True, cache_dir = args.download_to, download_config=dc)\n logging.info(f\"time spend on loading the dataset: {time.time() - stime:.2f} seconds\")\n\n splits = ['validation', 'test', 'train']\n \n logging.info(f\"gigaspeech dataset {args.dataset_size} info: {gs}\")\n logging.info(f\"phonemizing...\")\n phn_vocab = set()\n all_lens = []\n \n # you will see a ton of [WARNING] words_mismatch.py:88......, it's not a issue\n for split in tqdm.tqdm(splits):\n skip = 0\n logging.info(f\"now processing split {split}...\")\n for item in tqdm.tqdm(gs[split]):\n save_fn = os.path.join(phn_save_root, item['segment_id']+\".txt\")\n text = item['text']\n if sum(word in forbidden_words for word in text.split(\" \")):\n logging.info(f\"skip {item['segment_id']}, because it contains forbiden words. It's transcript: {text}\")\n skip += 1\n continue\n for k, v in punc2sym.items():\n text = text.replace(k, v)\n phn = tokenize_text(text_tokenizer, text)\n phn_seq = \" \".join(phn)\n for k, v in word2sym.items():\n phn_seq = phn_seq.replace(k, v)\n phn_vocab.update(phn_seq.split(\" \"))\n all_lens.append(len(phn_seq.split(\" \")))\n with open(save_fn, \"w\") as f:\n f.write(phn_seq)\n logging.info(f\"split {split} has {len(gs[split])} samples in total, skipped {skip} due to forbiden words\")\n\n print(f\"phn vocab size: {len(list(phn_vocab))}\")\n print(\"phn sequence stats: \")\n print(f\"longest: {max(all_lens)}\")\n print(f\"shortest: {min(all_lens)}\")\n print(f\"median: {np.quantile(all_lens, 0.5)}\")\n print(f\"95 percentile longest: {np.quantile(all_lens, 0.95)}\")\n print(\"write vocabulary to \", vocab_fn)\n with open(vocab_fn, \"w\") as f:\n for i, phn in enumerate(list(phn_vocab)):\n if i < len(list(phn_vocab)) - 1:\n f.write(f\"{str(i)} {phn}\\n\")\n else:\n f.write(f\"{str(i)} {phn}\")\n\n class mydataset(torch.utils.data.Dataset):\n def __init__(self, split):\n super().__init__()\n self.data = gs[split]\n def __len__(self):\n return len(self.data)\n def __getitem__(self, ind):\n try:\n segment_id, audio, sr, text, begin_time, end_time = self.data[ind]['segment_id'], torch.from_numpy(self.data[ind]['audio']['array']).float(), self.data[ind]['audio']['sampling_rate'], self.data[ind]['text'], self.data[ind]['begin_time'], self.data[ind]['end_time']\n except:\n return None, None, None, None, None, None\n \n return segment_id, audio, sr, text, begin_time, end_time\n def collate(self, batch):\n res = {'segment_id': [], \"audio\": [], \"sr\": [], \"text\": [], \"begin_time\": [], \"end_time\": []}\n for item in batch:\n if item[0] != None:\n res['segment_id'].append(item[0])\n res['audio'].append(item[1])\n res['sr'].append(item[2])\n res['text'].append(item[3])\n res['begin_time'].append(item[4])\n res['end_time'].append(item[5])\n return res\n\n\n ## encodec codes extraction\n logging.info(\"encodec encoding...\")\n train_dataset = mydataset('train')\n train_loader = torch.torch.utils.data.DataLoader(train_dataset, batch_size=args.mega_batch_size, shuffle=False, drop_last=False, num_workers=args.n_workers, collate_fn=train_dataset.collate)\n validation_dataset = mydataset('validation')\n validation_loader = torch.torch.utils.data.DataLoader(validation_dataset, batch_size=args.mega_batch_size, shuffle=False, drop_last=False, num_workers=args.n_workers, collate_fn=validation_dataset.collate)\n test_dataset = mydataset('test')\n test_loader = torch.torch.utils.data.DataLoader(test_dataset, batch_size=args.mega_batch_size, shuffle=False, drop_last=False, num_workers=args.n_workers, collate_fn=test_dataset.collate)\n splits = ['validation', 'test', 'train']\n loaders = [validation_loader, test_loader, train_loader]\n # splits = ['validation'] # for debug\n # loaders = [validation_loader]\n for split, loader in zip(splits, loaders):\n skip = 0\n logging.info(f\"now processing split {split}...\")\n mega_n_steps = int(np.ceil(len(gs[split]) / args.mega_batch_size))\n logging.info(f\"partition the split {split} into {mega_n_steps} parts, each has {args.mega_batch_size} samples\")\n for m, mega_batch in enumerate(loader):\n logging.info(f\"====================================\")\n logging.info(f\"====================================\")\n logging.info(f\"now processing mega step {m+1}/{mega_n_steps}\")\n lengths = np.array(mega_batch['end_time']) - np.array(mega_batch['begin_time'])\n sorted_inds = sort_by_audio_len(lengths)\n for j in range(len(sorted_inds))[::-1]:\n if lengths[sorted_inds[j]] < 0.2 or lengths[sorted_inds[j]] > args.len_cap: # skip samples that are too short (shorter than 0.2s), or too big (bigger than 80s)\n skip += 1\n del sorted_inds[j]\n \n n_steps = int(np.ceil(len(sorted_inds) / args.batch_size))\n for n in tqdm.tqdm(range(n_steps), disable=True):\n inds_used = sorted_inds[n*args.batch_size:(n+1)*args.batch_size]\n audio_batch = [mega_batch['audio'][id] for id in inds_used]\n sr_batch = [mega_batch['sr'][id] for id in inds_used]\n segment_id_batch = [mega_batch['segment_id'][id] for id in inds_used]\n text_batch = [mega_batch['text'][id] for id in inds_used]\n padded_wav = torch.nn.utils.rnn.pad_sequence(audio_batch, batch_first=True).unsqueeze(1) # [B, T] -> [B, 1, T]\n all_lens = [lengths[id] for id in inds_used]\n with torch.no_grad():\n if max(all_lens) > args.max_len and len(all_lens) > 1: # NOTE decrease args.max_len if OOM, or chunk it into more than 2 forward passes\n codes = []\n inwav = padded_wav.cuda()\n codes.append(model.encode(inwav[:len(inwav)//2])[0].cpu())\n codes.append(model.encode(inwav[len(inwav)//2:])[0].cpu())\n codes = torch.cat(codes, dim=0)\n else:\n encoded_frames = model.encode(padded_wav.cuda())\n # logging.info(f\"encoded_frames: {encoded_frames[0].shape}\")\n codes = encoded_frames[0].cpu()\n\n for i, length in enumerate(all_lens):\n save_fn = os.path.join(codes_save_root, segment_id_batch[i]+\".txt\")\n actual_len = round(length * args.model_code_sr) # 320 is downsample rate for this model\n cur_code = codes[i].tolist() if type(codes) == list else codes[i, :, :actual_len].tolist()\n write_array_to_txt_file(cur_code, save_fn)\n" }, { "path": "data/tokenizer.py", "content": "# cp from https://github.com/lifeiteng/vall-e/blob/main/valle/data/tokenizer.py\n# Copyright 2023 (authors: Feiteng Li)\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport re\nfrom dataclasses import asdict, dataclass\nfrom typing import Any, Dict, List, Optional, Pattern, Union\n\nimport numpy as np\nimport torch\nimport torchaudio\n# from lhotse.features import FeatureExtractor\n# from lhotse.utils import Seconds, compute_num_frames\nfrom phonemizer.backend import EspeakBackend\nfrom phonemizer.backend.espeak.language_switch import LanguageSwitch\nfrom phonemizer.backend.espeak.words_mismatch import WordMismatch\nfrom phonemizer.punctuation import Punctuation\nfrom phonemizer.separator import Separator\n\n\n\nclass TextTokenizer:\n \"\"\"Phonemize Text.\"\"\"\n\n def __init__(\n self,\n language=\"en-us\",\n backend=\"espeak\",\n separator=Separator(word=\"_\", syllable=\"-\", phone=\"|\"),\n preserve_punctuation=True,\n punctuation_marks: Union[str, Pattern] = Punctuation.default_marks(),\n with_stress: bool = False,\n tie: Union[bool, str] = False,\n language_switch: LanguageSwitch = \"keep-flags\",\n words_mismatch: WordMismatch = \"ignore\",\n ) -> None:\n phonemizer = EspeakBackend(\n language,\n punctuation_marks=punctuation_marks,\n preserve_punctuation=preserve_punctuation,\n with_stress=with_stress,\n tie=tie,\n language_switch=language_switch,\n words_mismatch=words_mismatch,\n )\n \n self.backend = phonemizer\n self.separator = separator\n\n def to_list(self, phonemized: str) -> List[str]:\n fields = []\n for word in phonemized.split(self.separator.word):\n # \"ɐ m|iː|n?\" ɹ|ɪ|z|ɜː|v; h|ɪ|z.\n pp = re.findall(r\"\\w+|[^\\w\\s]\", word, re.UNICODE)\n fields.extend(\n [p for p in pp if p != self.separator.phone]\n + [self.separator.word]\n )\n assert len(\"\".join(fields[:-1])) == len(phonemized) - phonemized.count(\n self.separator.phone\n )\n return fields[:-1]\n\n def __call__(self, text, strip=True) -> List[List[str]]:\n if isinstance(text, str):\n text = [text]\n\n phonemized = self.backend.phonemize(\n text, separator=self.separator, strip=strip, njobs=1\n )\n return [self.to_list(p) for p in phonemized]\n\n\ndef tokenize_text(tokenizer: TextTokenizer, text: str) -> List[str]:\n phonemes = tokenizer([text.strip()])\n return phonemes[0] # k2symbols\n\ndef convert_audio(wav: torch.Tensor, sr: int, target_sr: int, target_channels: int):\n assert wav.shape[0] in [1, 2], \"Audio must be mono or stereo.\"\n if target_channels == 1:\n wav = wav.mean(0, keepdim=True)\n elif target_channels == 2:\n *shape, _, length = wav.shape\n wav = wav.expand(*shape, target_channels, length)\n elif wav.shape[0] == 1:\n wav = wav.expand(target_channels, -1)\n wav = torchaudio.transforms.Resample(sr, target_sr)(wav)\n return wav\n\nclass AudioTokenizer:\n \"\"\"EnCodec audio.\"\"\"\n\n def __init__(\n self,\n device: Any = None,\n signature = None\n ) -> None:\n from audiocraft.solvers import CompressionSolver\n model = CompressionSolver.model_from_checkpoint(signature)\n self.sample_rate = model.sample_rate\n self.channels = model.channels\n \n if not device:\n device = torch.device(\"cpu\")\n if torch.cuda.is_available():\n device = torch.device(\"cuda:0\")\n\n self._device = device\n\n self.codec = model.to(device)\n\n @property\n def device(self):\n return self._device\n\n def encode(self, wav: torch.Tensor) -> torch.Tensor:\n codes = self.codec.encode(wav.to(self.device))\n return [(codes[0], None)]\n\n def decode(self, frames: torch.Tensor) -> torch.Tensor:\n frames = frames[0][0] # [1,4,T]\n return self.codec.decode(frames)\n \n\n\ndef tokenize_audio(tokenizer: AudioTokenizer, audio_path: str, offset = -1, num_frames=-1):\n # Load and pre-process the audio waveform\n if offset != -1 and num_frames!=-1:\n wav, sr = torchaudio.load(audio_path, frame_offset=offset, num_frames=num_frames)\n else:\n wav, sr = torchaudio.load(audio_path)\n wav = convert_audio(wav, sr, tokenizer.sample_rate, tokenizer.channels)\n wav = wav.unsqueeze(0)\n\n # Extract discrete codes from EnCodec\n with torch.no_grad():\n encoded_frames = tokenizer.encode(wav)\n return encoded_frames\n" }, { "path": "demo/temp/84_121550_000074_000000.txt", "content": "But when I had approached so near to them The common object, which the sense deceives, Lost not by distance any of its marks," }, { "path": "demo/temp/mfa_alignments/5895_34622_000026_000002.csv", "content": "Begin,End,Label,Type,Speaker\r\n0.04,0.58,gwynplaine,words,temp\r\n0.58,0.94,had,words,temp\r\n0.94,1.45,besides,words,temp\r\n1.45,1.62,for,words,temp\r\n1.62,1.86,his,words,temp\r\n1.86,2.16,work,words,temp\r\n2.16,2.31,and,words,temp\r\n2.31,2.49,for,words,temp\r\n2.49,2.71,his,words,temp\r\n2.71,3.03,feats,words,temp\r\n3.03,3.12,of,words,temp\r\n3.12,3.61,strength,words,temp\r\n3.95,4.25,round,words,temp\r\n4.25,4.45,his,words,temp\r\n4.45,4.7,neck,words,temp\r\n4.7,4.81,and,words,temp\r\n4.81,5.04,over,words,temp\r\n5.04,5.22,his,words,temp\r\n5.22,5.83,shoulders,words,temp\r\n6.16,6.31,an,words,temp\r\n6.41,7.15,esclavine,words,temp\r\n7.15,7.29,of,words,temp\r\n7.29,7.7,leather,words,temp\r\n0.04,0.1,G,phones,temp\r\n0.1,0.13,W,phones,temp\r\n0.13,0.22,IH1,phones,temp\r\n0.22,0.3,N,phones,temp\r\n0.3,0.38,P,phones,temp\r\n0.38,0.42,L,phones,temp\r\n0.42,0.53,EY1,phones,temp\r\n0.53,0.58,N,phones,temp\r\n0.58,0.71,HH,phones,temp\r\n0.71,0.86,AE1,phones,temp\r\n0.86,0.94,D,phones,temp\r\n0.94,0.97,B,phones,temp\r\n0.97,1.01,IH0,phones,temp\r\n1.01,1.14,S,phones,temp\r\n1.14,1.34,AY1,phones,temp\r\n1.34,1.4,D,phones,temp\r\n1.4,1.45,Z,phones,temp\r\n1.45,1.52,F,phones,temp\r\n1.52,1.55,AO1,phones,temp\r\n1.55,1.62,R,phones,temp\r\n1.62,1.69,HH,phones,temp\r\n1.69,1.76,IH1,phones,temp\r\n1.76,1.86,Z,phones,temp\r\n1.86,1.95,W,phones,temp\r\n1.95,2.07,ER1,phones,temp\r\n2.07,2.16,K,phones,temp\r\n2.16,2.23,AH0,phones,temp\r\n2.23,2.26,N,phones,temp\r\n2.26,2.31,D,phones,temp\r\n2.31,2.38,F,phones,temp\r\n2.38,2.41,AO1,phones,temp\r\n2.41,2.49,R,phones,temp\r\n2.49,2.55,HH,phones,temp\r\n2.55,2.62,IH1,phones,temp\r\n2.62,2.71,Z,phones,temp\r\n2.71,2.8,F,phones,temp\r\n2.8,2.9,IY1,phones,temp\r\n2.9,2.98,T,phones,temp\r\n2.98,3.03,S,phones,temp\r\n3.03,3.07,AH0,phones,temp\r\n3.07,3.12,V,phones,temp\r\n3.12,3.2,S,phones,temp\r\n3.2,3.26,T,phones,temp\r\n3.26,3.32,R,phones,temp\r\n3.32,3.39,EH1,phones,temp\r\n3.39,3.48,NG,phones,temp\r\n3.48,3.53,K,phones,temp\r\n3.53,3.61,TH,phones,temp\r\n3.95,4.03,R,phones,temp\r\n4.03,4.16,AW1,phones,temp\r\n4.16,4.21,N,phones,temp\r\n4.21,4.25,D,phones,temp\r\n4.25,4.29,HH,phones,temp\r\n4.29,4.36,IH1,phones,temp\r\n4.36,4.45,Z,phones,temp\r\n4.45,4.53,N,phones,temp\r\n4.53,4.62,EH1,phones,temp\r\n4.62,4.7,K,phones,temp\r\n4.7,4.74,AH0,phones,temp\r\n4.74,4.77,N,phones,temp\r\n4.77,4.81,D,phones,temp\r\n4.81,4.92,OW1,phones,temp\r\n4.92,4.97,V,phones,temp\r\n4.97,5.04,ER0,phones,temp\r\n5.04,5.11,HH,phones,temp\r\n5.11,5.18,IH1,phones,temp\r\n5.18,5.22,Z,phones,temp\r\n5.22,5.34,SH,phones,temp\r\n5.34,5.47,OW1,phones,temp\r\n5.47,5.51,L,phones,temp\r\n5.51,5.58,D,phones,temp\r\n5.58,5.71,ER0,phones,temp\r\n5.71,5.83,Z,phones,temp\r\n6.16,6.23,AE1,phones,temp\r\n6.23,6.31,N,phones,temp\r\n6.41,7.15,spn,phones,temp\r\n7.15,7.21,AH0,phones,temp\r\n7.21,7.29,V,phones,temp\r\n7.29,7.36,L,phones,temp\r\n7.36,7.44,EH1,phones,temp\r\n7.44,7.49,DH,phones,temp\r\n7.49,7.7,ER0,phones,temp\r\n" }, { "path": "demo/temp/mfa_alignments/84_121550_000074_000000.csv", "content": "Begin,End,Label,Type,Speaker\r\n0.03,0.18,but,words,temp\r\n0.18,0.32,when,words,temp\r\n0.32,0.48,i,words,temp\r\n0.48,0.64,had,words,temp\r\n0.64,1.19,approached,words,temp\r\n1.22,1.58,so,words,temp\r\n1.58,1.91,near,words,temp\r\n1.91,2.07,to,words,temp\r\n2.07,2.42,them,words,temp\r\n2.53,2.61,the,words,temp\r\n2.61,3.01,common,words,temp\r\n3.05,3.62,object,words,temp\r\n3.68,3.93,which,words,temp\r\n3.93,4.02,the,words,temp\r\n4.02,4.34,sense,words,temp\r\n4.34,4.97,deceives,words,temp\r\n5.04,5.54,lost,words,temp\r\n5.54,6.0,not,words,temp\r\n6.0,6.14,by,words,temp\r\n6.14,6.67,distance,words,temp\r\n6.79,7.05,any,words,temp\r\n7.05,7.18,of,words,temp\r\n7.18,7.34,its,words,temp\r\n7.34,7.87,marks,words,temp\r\n0.03,0.06,B,phones,temp\r\n0.06,0.09,AH1,phones,temp\r\n0.09,0.18,T,phones,temp\r\n0.18,0.23,W,phones,temp\r\n0.23,0.27,EH1,phones,temp\r\n0.27,0.32,N,phones,temp\r\n0.32,0.48,AY1,phones,temp\r\n0.48,0.49,HH,phones,temp\r\n0.49,0.6,AE1,phones,temp\r\n0.6,0.64,D,phones,temp\r\n0.64,0.7,AH0,phones,temp\r\n0.7,0.83,P,phones,temp\r\n0.83,0.88,R,phones,temp\r\n0.88,0.99,OW1,phones,temp\r\n0.99,1.12,CH,phones,temp\r\n1.12,1.19,T,phones,temp\r\n1.22,1.4,S,phones,temp\r\n1.4,1.58,OW1,phones,temp\r\n1.58,1.7,N,phones,temp\r\n1.7,1.84,IH1,phones,temp\r\n1.84,1.91,R,phones,temp\r\n1.91,2.01,T,phones,temp\r\n2.01,2.07,AH0,phones,temp\r\n2.07,2.13,DH,phones,temp\r\n2.13,2.3,EH1,phones,temp\r\n2.3,2.42,M,phones,temp\r\n2.53,2.55,DH,phones,temp\r\n2.55,2.61,AH0,phones,temp\r\n2.61,2.73,K,phones,temp\r\n2.73,2.85,AA1,phones,temp\r\n2.85,2.9,M,phones,temp\r\n2.9,2.95,AH0,phones,temp\r\n2.95,3.01,N,phones,temp\r\n3.05,3.22,AA1,phones,temp\r\n3.22,3.27,B,phones,temp\r\n3.27,3.34,JH,phones,temp\r\n3.34,3.48,EH0,phones,temp\r\n3.48,3.54,K,phones,temp\r\n3.54,3.62,T,phones,temp\r\n3.68,3.69,HH,phones,temp\r\n3.69,3.76,W,phones,temp\r\n3.76,3.8,IH1,phones,temp\r\n3.8,3.93,CH,phones,temp\r\n3.93,3.95,DH,phones,temp\r\n3.95,4.02,AH0,phones,temp\r\n4.02,4.12,S,phones,temp\r\n4.12,4.21,EH1,phones,temp\r\n4.21,4.27,N,phones,temp\r\n4.27,4.34,S,phones,temp\r\n4.34,4.42,D,phones,temp\r\n4.42,4.45,IH0,phones,temp\r\n4.45,4.59,S,phones,temp\r\n4.59,4.79,IY1,phones,temp\r\n4.79,4.87,V,phones,temp\r\n4.87,4.97,Z,phones,temp\r\n5.04,5.12,L,phones,temp\r\n5.12,5.33,AO1,phones,temp\r\n5.33,5.42,S,phones,temp\r\n5.42,5.54,T,phones,temp\r\n5.54,5.7,N,phones,temp\r\n5.7,5.89,AA1,phones,temp\r\n5.89,6.0,T,phones,temp\r\n6.0,6.05,B,phones,temp\r\n6.05,6.14,AY1,phones,temp\r\n6.14,6.24,D,phones,temp\r\n6.24,6.3,IH1,phones,temp\r\n6.3,6.38,S,phones,temp\r\n6.38,6.45,T,phones,temp\r\n6.45,6.51,AH0,phones,temp\r\n6.51,6.57,N,phones,temp\r\n6.57,6.67,S,phones,temp\r\n6.79,6.89,EH1,phones,temp\r\n6.89,6.95,N,phones,temp\r\n6.95,7.05,IY0,phones,temp\r\n7.05,7.13,AH0,phones,temp\r\n7.13,7.18,V,phones,temp\r\n7.18,7.22,IH0,phones,temp\r\n7.22,7.29,T,phones,temp\r\n7.29,7.34,S,phones,temp\r\n7.34,7.39,M,phones,temp\r\n7.39,7.5,AA1,phones,temp\r\n7.5,7.58,R,phones,temp\r\n7.58,7.7,K,phones,temp\r\n7.7,7.87,S,phones,temp\r\n" }, { "path": "edit_utils.py", "content": "def get_span(orig, new, editType):\n orig_list = orig.split(\" \")\n new_list = new.split(\" \")\n \n flag = False # this indicate whether the actual edit follow the specified editType\n if editType == \"deletion\":\n assert len(orig_list) > len(new_list), f\"the edit type is deletion, but new is not shorter than original:\\n new: {new}\\n orig: {orig}\"\n diff = len(orig_list) - len(new_list)\n for i, (o, n) in enumerate(zip(orig_list, new_list)):\n if o != n: # assume the index of the first different word is the starting index of the orig_span\n \n orig_span = [i, i + diff - 1] # assume that the indices are starting and ending index of the deleted part\n new_span = [i-1, i] # but for the new span, the starting and ending index is the two words that surround the deleted part\n flag = True\n break\n\n\n elif editType == \"insertion\": \n assert len(orig_list) < len(new_list), f\"the edit type is insertion, but the new is not longer than the original:\\n new: {new}\\n orig: {orig}\"\n diff = len(new_list) - len(orig_list)\n for i, (o, n) in enumerate(zip(orig_list, new_list)):\n if o != n: # insertion is just the opposite of deletion\n new_span = [i, i + diff - 1] # NOTE if only inserted one word, s and e will be the same\n orig_span = [i-1, i]\n flag = True\n break\n\n elif editType == \"substitution\":\n new_span = []\n orig_span = []\n for i, (o, n) in enumerate(zip(orig_list, new_list)):\n if o != n:\n new_span = [i]\n orig_span = [i]\n break\n assert len(new_span) == 1 and len(orig_span) == 1, f\"new_span: {new_span}, orig_span: {orig_span}\"\n for j, (o, n) in enumerate(zip(orig_list[::-1], new_list[::-1])):\n if o != n:\n new_span.append(len(new_list) - j -1)\n orig_span.append(len(orig_list) - j - 1)\n flag = True\n break\n else:\n raise RuntimeError(f\"editType unknown: {editType}\")\n\n if not flag:\n raise RuntimeError(f\"wrong editing with the specified edit type:\\n original: {orig}\\n new: {new}\\n, editType: {editType}\")\n\n return orig_span, new_span " }, { "path": "environment.yml", "content": "name: voicecraft\nchannels:\n - conda-forge\n - defaults\ndependencies:\n - _libgcc_mutex=0.1=conda_forge\n - _openmp_mutex=4.5=2_gnu\n - aom=3.8.2=h59595ed_0\n - asttokens=2.4.1=pyhd8ed1ab_0\n - atk-1.0=2.38.0=hd4edc92_1\n - audioread=3.0.1=py39hf3d152e_1\n - backcall=0.2.0=pyh9f0ad1d_0\n - baumwelch=0.3.7=h00ab1b0_5\n - biopython=1.79=py39hb9d737c_3\n - brotli=1.1.0=hd590300_1\n - brotli-bin=1.1.0=hd590300_1\n - brotli-python=1.1.0=py39h3d6467e_1\n - bzip2=1.0.8=hd590300_5\n - ca-certificates=2024.2.2=hbcca054_0\n - cairo=1.18.0=h3faef2a_0\n - certifi=2024.2.2=pyhd8ed1ab_0\n - cffi=1.16.0=py39h7a31438_0\n - charset-normalizer=3.3.2=pyhd8ed1ab_0\n - click=8.1.7=unix_pyh707e725_0\n - colorama=0.4.6=pyhd8ed1ab_0\n - comm=0.2.2=pyhd8ed1ab_0\n - contourpy=1.2.0=py39h7633fee_0\n - cycler=0.12.1=pyhd8ed1ab_0\n - dataclassy=1.0.1=pyhd8ed1ab_0\n - dav1d=1.2.1=hd590300_0\n - debugpy=1.8.1=py39h3d6467e_0\n - decorator=5.1.1=pyhd8ed1ab_0\n - executing=2.0.1=pyhd8ed1ab_0\n - expat=2.6.2=h59595ed_0\n - ffmpeg=6.1.1=gpl_h38e077a_106\n - font-ttf-dejavu-sans-mono=2.37=hab24e00_0\n - font-ttf-inconsolata=3.000=h77eed37_0\n - font-ttf-source-code-pro=2.038=h77eed37_0\n - font-ttf-ubuntu=0.83=h77eed37_1\n - fontconfig=2.14.2=h14ed4e7_0\n - fonts-conda-ecosystem=1=0\n - fonts-conda-forge=1=0\n - fonttools=4.49.0=py39hd1e30aa_0\n - freetype=2.12.1=h267a509_2\n - fribidi=1.0.10=h36c2ea0_0\n - gdk-pixbuf=2.42.10=h829c605_5\n - gettext=0.21.1=h27087fc_0\n - giflib=5.2.1=h0b41bf4_3\n - gmp=6.3.0=h59595ed_1\n - gnutls=3.7.9=hb077bed_0\n - graphite2=1.3.13=h58526e2_1001\n - graphviz=9.0.0=h78e8752_1\n - greenlet=3.0.3=py39h3d6467e_0\n - gtk2=2.24.33=h280cfa0_4\n - gts=0.7.6=h977cf35_4\n - harfbuzz=8.3.0=h3d44ed6_0\n - hdbscan=0.8.33=py39h44dd56e_4\n - icu=73.2=h59595ed_0\n - idna=3.6=pyhd8ed1ab_0\n - importlib-metadata=7.0.2=pyha770c72_0\n - importlib-resources=6.3.0=pyhd8ed1ab_0\n - importlib_metadata=7.0.2=hd8ed1ab_0\n - importlib_resources=6.3.0=pyhd8ed1ab_0\n - ipykernel=6.29.3=pyhd33586a_0\n - jedi=0.19.1=pyhd8ed1ab_0\n - joblib=1.3.2=pyhd8ed1ab_0\n - jupyter_client=8.6.1=pyhd8ed1ab_0\n - jupyter_core=5.7.2=py39hf3d152e_0\n - kaldi=5.5.1068=cpu_h31769b2_2\n - keyutils=1.6.1=h166bdaf_0\n - kiwisolver=1.4.5=py39h7633fee_1\n - kneed=0.8.5=pyhd8ed1ab_0\n - krb5=1.21.2=h659d440_0\n - lame=3.100=h166bdaf_1003\n - lazy_loader=0.3=pyhd8ed1ab_0\n - lcms2=2.16=hb7c19ff_0\n - ld_impl_linux-64=2.40=h41732ed_0\n - lerc=4.0.0=h27087fc_0\n - libabseil=20240116.1=cxx17_h59595ed_2\n - libass=0.17.1=h8fe9dca_1\n - libblas=3.9.0=21_linux64_openblas\n - libbrotlicommon=1.1.0=hd590300_1\n - libbrotlidec=1.1.0=hd590300_1\n - libbrotlienc=1.1.0=hd590300_1\n - libcblas=3.9.0=21_linux64_openblas\n - libclang-cpp15=15.0.7=default_hb11cfb5_4\n - libdeflate=1.19=hd590300_0\n - libdrm=2.4.120=hd590300_0\n - libedit=3.1.20191231=he28a2e2_2\n - libexpat=2.6.2=h59595ed_0\n - libffi=3.4.2=h7f98852_5\n - libflac=1.4.3=h59595ed_0\n - libgcc-ng=13.2.0=h807b86a_5\n - libgd=2.3.3=h119a65a_9\n - libgfortran-ng=13.2.0=h69a702a_5\n - libgfortran5=13.2.0=ha4646dd_5\n - libglib=2.80.0=hf2295e7_0\n - libgomp=13.2.0=h807b86a_5\n - libhwloc=2.9.3=default_h554bfaf_1009\n - libiconv=1.17=hd590300_2\n - libidn2=2.3.7=hd590300_0\n - libjpeg-turbo=3.0.0=hd590300_1\n - liblapack=3.9.0=21_linux64_openblas\n - liblapacke=3.9.0=21_linux64_openblas\n - libllvm14=14.0.6=hcd5def8_4\n - libllvm15=15.0.7=hb3ce162_4\n - libllvmspirv15=15.0.0=h0cdce71_1\n - libnsl=2.0.1=hd590300_0\n - libogg=1.3.4=h7f98852_1\n - libopenblas=0.3.26=pthreads_h413a1c8_0\n - libopenvino=2024.0.0=h2e90f83_1\n - libopenvino-auto-batch-plugin=2024.0.0=hd5fc58b_1\n - libopenvino-auto-plugin=2024.0.0=hd5fc58b_1\n - libopenvino-hetero-plugin=2024.0.0=h3ecfda7_1\n - libopenvino-intel-cpu-plugin=2024.0.0=h2e90f83_1\n - libopenvino-intel-gpu-plugin=2024.0.0=h2e90f83_1\n - libopenvino-ir-frontend=2024.0.0=h3ecfda7_1\n - libopenvino-onnx-frontend=2024.0.0=h757c851_1\n - libopenvino-paddle-frontend=2024.0.0=h757c851_1\n - libopenvino-pytorch-frontend=2024.0.0=h59595ed_1\n - libopenvino-tensorflow-frontend=2024.0.0=hca94c1a_1\n - libopenvino-tensorflow-lite-frontend=2024.0.0=h59595ed_1\n - libopus=1.3.1=h7f98852_1\n - libpciaccess=0.18=hd590300_0\n - libpng=1.6.43=h2797004_0\n - libpq=16.2=h33b98f1_0\n - libprotobuf=4.25.3=h08a7969_0\n - librosa=0.10.1=pyhd8ed1ab_0\n - librsvg=2.56.3=he3f83f7_1\n - libsndfile=1.2.2=hc60ed4a_1\n - libsodium=1.0.18=h36c2ea0_1\n - libsqlite=3.45.2=h2797004_0\n - libstdcxx-ng=13.2.0=h7e041cc_5\n - libtasn1=4.19.0=h166bdaf_0\n - libtiff=4.6.0=ha9c0a0a_2\n - libunistring=0.9.10=h7f98852_0\n - libuuid=2.38.1=h0b41bf4_0\n - libva=2.21.0=hd590300_0\n - libvorbis=1.3.7=h9c3ff4c_0\n - libvpx=1.14.0=h59595ed_0\n - libwebp=1.3.2=h658648e_1\n - libwebp-base=1.3.2=hd590300_0\n - libxcb=1.15=h0b41bf4_0\n - libxcrypt=4.4.36=hd590300_1\n - libxml2=2.12.5=h232c23b_0\n - libzlib=1.2.13=hd590300_5\n - llvm-spirv-15=15.0.0=h0cdce71_1\n - mad=0.15.1b=h9c3ff4c_1\n - markdown-it-py=3.0.0=pyhd8ed1ab_0\n - matplotlib-base=3.8.3=py39he9076e7_0\n - matplotlib-inline=0.1.6=pyhd8ed1ab_0\n - mdurl=0.1.2=pyhd8ed1ab_0\n - montreal-forced-aligner=2.2.17=pyhd8ed1ab_0\n - mpg123=1.32.4=h59595ed_0\n - msgpack-python=1.0.7=py39h7633fee_0\n - munkres=1.1.4=pyh9f0ad1d_0\n - ncurses=6.4=h59595ed_2\n - nest-asyncio=1.6.0=pyhd8ed1ab_0\n - nettle=3.9.1=h7ab15ed_0\n - ngram=1.3.14=h924138e_2\n - numba=0.59.0=py39h615d6bd_1\n - numpy=1.26.4=py39h474f0d3_0\n - ocl-icd=2.3.2=hd590300_0\n - openfst=1.8.2=h924138e_2\n - openh264=2.4.1=h59595ed_0\n - openjpeg=2.5.2=h488ebb8_0\n - openssl=3.2.1=hd590300_0\n - p11-kit=0.24.1=hc5aa10d_0\n - packaging=24.0=pyhd8ed1ab_0\n - pandas=2.2.1=py39hddac248_0\n - pango=1.52.1=ha41ecd1_0\n - parso=0.8.3=pyhd8ed1ab_0\n - patsy=0.5.6=pyhd8ed1ab_0\n - pcre2=10.43=hcad00b1_0\n - pexpect=4.9.0=pyhd8ed1ab_0\n - pgvector-python=0.2.5=pyhe093146_0\n - pickleshare=0.7.5=py_1003\n - pillow=10.2.0=py39had0adad_0\n - pip=24.0=pyhd8ed1ab_0\n - pixman=0.43.2=h59595ed_0\n - platformdirs=4.2.0=pyhd8ed1ab_0\n - pocl=5.0=h03a6ac1_2\n - pocl-core=5.0=hdaecddf_2\n - pocl-cpu=5.0=he901f76_2\n - pocl-cpu-minimal=5.0=h5ccd973_2\n - pocl-cuda=5.0=hdaecddf_2\n - pocl-remote=5.0=h5ccd973_2\n - pooch=1.8.1=pyhd8ed1ab_0\n - postgresql=16.2=h7387d8b_0\n - prompt-toolkit=3.0.42=pyha770c72_0\n - prompt_toolkit=3.0.42=hd8ed1ab_0\n - psutil=5.9.8=py39hd1e30aa_0\n - psycopg2=2.9.9=py39h89197e3_0\n - pthread-stubs=0.4=h36c2ea0_1001\n - ptyprocess=0.7.0=pyhd3deb0d_0\n - pugixml=1.14=h59595ed_0\n - pure_eval=0.2.2=pyhd8ed1ab_0\n - pycparser=2.21=pyhd8ed1ab_0\n - pygments=2.17.2=pyhd8ed1ab_0\n - pyparsing=3.1.2=pyhd8ed1ab_0\n - pysocks=1.7.1=pyha2e5f31_6\n - pysoundfile=0.12.1=pypyhd8ed1ab_1\n - python=3.9.18=h0755675_1_cpython\n - python-tzdata=2024.1=pyhd8ed1ab_0\n - python_abi=3.9=4_cp39\n - pytz=2024.1=pyhd8ed1ab_0\n - pyyaml=6.0.1=py39hd1e30aa_1\n - pyzmq=25.1.2=py39h8c080ef_0\n - readline=8.2=h8228510_1\n - requests=2.31.0=pyhd8ed1ab_0\n - rich=13.7.1=pyhd8ed1ab_0\n - rich-click=1.7.4=pyhd8ed1ab_0\n - scikit-learn=1.2.2=py39hc236052_2\n - scipy=1.12.0=py39h474f0d3_2\n - seaborn=0.13.2=hd8ed1ab_0\n - seaborn-base=0.13.2=pyhd8ed1ab_0\n - setuptools=69.2.0=pyhd8ed1ab_0\n - six=1.16.0=pyh6c4a22f_0\n - snappy=1.1.10=h9fff704_0\n - sox=14.4.2=ha5cc309_1018\n - soxr=0.1.3=h0b41bf4_3\n - soxr-python=0.3.7=py39h44dd56e_0\n - sqlalchemy=2.0.28=py39hd1e30aa_0\n - sqlite=3.45.2=h2c6b66d_0\n - stack_data=0.6.2=pyhd8ed1ab_0\n - statsmodels=0.14.1=py39h44dd56e_0\n - svt-av1=1.8.0=h59595ed_0\n - tbb=2021.11.0=h00ab1b0_1\n - threadpoolctl=3.3.0=pyhc1e730c_0\n - tk=8.6.13=noxft_h4845f30_101\n - tornado=6.4=py39hd1e30aa_0\n - tqdm=4.66.2=pyhd8ed1ab_0\n - traitlets=5.14.2=pyhd8ed1ab_0\n - typing-extensions=4.10.0=hd8ed1ab_0\n - typing_extensions=4.10.0=pyha770c72_0\n - tzcode=2024a=h3f72095_0\n - tzdata=2024a=h0c530f3_0\n - unicodedata2=15.1.0=py39hd1e30aa_0\n - urllib3=2.2.1=pyhd8ed1ab_0\n - wcwidth=0.2.13=pyhd8ed1ab_0\n - wheel=0.42.0=pyhd8ed1ab_0\n - x264=1!164.3095=h166bdaf_2\n - x265=3.5=h924138e_3\n - xorg-fixesproto=5.0=h7f98852_1002\n - xorg-kbproto=1.0.7=h7f98852_1002\n - xorg-libice=1.1.1=hd590300_0\n - xorg-libsm=1.2.4=h7391055_0\n - xorg-libx11=1.8.7=h8ee46fc_0\n - xorg-libxau=1.0.11=hd590300_0\n - xorg-libxdmcp=1.1.3=h7f98852_0\n - xorg-libxext=1.3.4=h0b41bf4_2\n - xorg-libxfixes=5.0.3=h7f98852_1004\n - xorg-libxrender=0.9.11=hd590300_0\n - xorg-renderproto=0.11.1=h7f98852_1002\n - xorg-xextproto=7.3.0=h0b41bf4_1003\n - xorg-xproto=7.0.31=h7f98852_1007\n - xz=5.2.6=h166bdaf_0\n - yaml=0.2.5=h7f98852_2\n - zeromq=4.3.5=h59595ed_1\n - zipp=3.17.0=pyhd8ed1ab_0\n - zlib=1.2.13=hd590300_5\n - zstd=1.5.5=hfc55251_0\n - pip:\n - absl-py==2.1.0\n - aiofiles==23.2.1\n - aiohttp==3.9.3\n - aiosignal==1.3.1\n - altair==5.2.0\n - antlr4-python3-runtime==4.9.3\n - anyio==4.3.0\n - async-timeout==4.0.3\n - attrs==23.2.0\n - av==11.0.0\n - babel==2.14.0\n - beautifulsoup4==4.12.3\n - bibtexparser==2.0.0b7\n - bleach==6.1.0\n - blis==0.7.11\n - catalogue==2.0.10\n - clldutils==3.22.2\n - cloudpickle==3.0.0\n - cmake==3.28.3\n - colorlog==6.8.2\n - confection==0.1.4\n - csvw==3.3.0\n - cymem==2.0.8\n - cython==0.29.37\n - datasets==2.16.0\n - defusedxml==0.7.1\n - demucs==4.0.1\n - dill==0.3.6\n - dlinfo==1.2.1\n - docopt==0.6.2\n - dora-search==0.1.12\n - einops==0.7.0\n - encodec==0.1.1\n - exceptiongroup==1.2.0\n - fastapi==0.110.0\n - fastjsonschema==2.19.1\n - ffmpy==0.3.2\n - filelock==3.13.1\n - flashy==0.0.2\n - frozenlist==1.4.1\n - fsspec==2023.10.0\n - gradio==3.50.2\n - gradio-client==0.6.1\n - grpcio==1.62.1\n - h11==0.14.0\n - httpcore==1.0.4\n - httpx==0.27.0\n - huggingface-hub==0.22.2\n - hydra-colorlog==1.2.0\n - hydra-core==1.3.2\n - ipython==8.12.3\n - isodate==0.6.1\n - jinja2==3.1.3\n - jsonschema==4.21.1\n - jsonschema-specifications==2023.12.1\n - julius==0.2.7\n - jupyterlab-pygments==0.3.0\n - lameenc==1.7.0\n - langcodes==3.3.0\n - language-tags==1.2.0\n - lit==18.1.1\n - llvmlite==0.42.0\n - lxml==5.1.0\n - markdown==3.5.2\n - markupsafe==2.1.5\n - mistune==3.0.2\n - mpmath==1.3.0\n - msgpack==1.0.8\n - multidict==6.0.5\n - multiprocess==0.70.14\n - murmurhash==1.0.10\n - nbclient==0.10.0\n - nbconvert==7.16.3\n - nbformat==5.10.3\n - networkx==3.2.1\n - num2words==0.5.13\n - nvidia-cublas-cu11==11.10.3.66\n - nvidia-cuda-cupti-cu11==11.7.101\n - nvidia-cuda-nvrtc-cu11==11.7.99\n - nvidia-cuda-runtime-cu11==11.7.99\n - nvidia-cudnn-cu11==8.5.0.96\n - nvidia-cufft-cu11==10.9.0.58\n - nvidia-curand-cu11==10.2.10.91\n - nvidia-cusolver-cu11==11.4.0.1\n - nvidia-cusparse-cu11==11.7.4.91\n - nvidia-nccl-cu11==2.14.3\n - nvidia-nvtx-cu11==11.7.91\n - omegaconf==2.3.0\n - openunmix==1.2.1\n - orjson==3.9.15\n - pandocfilters==1.5.1\n - pathlib-abc==0.1.1\n - pathy==0.11.0\n - pgvector==0.2.2\n - phonemizer==3.2.1\n - pipreqs==0.5.0\n - praatio==6.2.0\n - preshed==3.0.9\n - protobuf==4.25.3\n - pyarrow==15.0.2\n - pyarrow-hotfix==0.6\n - pydantic==1.10.14\n - pydub==0.25.1\n - pylatexenc==2.10\n - pynini==2.1.6\n - pypinyin==0.48.0\n - python-dateutil==2.9.0.post0\n - python-multipart==0.0.9\n - rdflib==7.0.0\n - referencing==0.33.0\n - regex==2023.12.25\n - responses==0.18.0\n - retrying==1.3.4\n - rfc3986==1.5.0\n - rpds-py==0.18.0\n - safetensors==0.4.2\n - segments==2.2.1\n - semantic-version==2.10.0\n - sentencepiece==0.2.0\n - smart-open==6.4.0\n - sniffio==1.3.1\n - soupsieve==2.5\n - spacy==3.5.2\n - spacy-legacy==3.0.12\n - spacy-loggers==1.0.5\n - srsly==2.4.8\n - starlette==0.36.3\n - submitit==1.5.1\n - sympy==1.12\n - tabulate==0.9.0\n - tensorboard==2.16.2\n - tensorboard-data-server==0.7.2\n - thinc==8.1.12\n - tinycss2==1.2.1\n - tokenizers==0.15.2\n - toolz==0.12.1\n - torch==2.0.1\n - torchaudio==2.0.2\n - torchmetrics==0.11.1\n - transformers==4.38.2\n - treetable==0.2.5\n - triton==2.0.0\n - typer==0.7.0\n - uritemplate==4.1.1\n - uvicorn==0.28.0\n - wasabi==1.1.2\n - webencodings==0.5.1\n - websockets==11.0.3\n - werkzeug==3.0.1\n - xformers==0.0.22\n - xxhash==3.4.1\n - yarg==0.1.9\n - yarl==1.9.4\nprefix: /home/pyp/miniconda3/envs/voicecraft\n" }, { "path": "gradio_app.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"id\": \"9b6a0c92\",\n \"metadata\": {},\n \"source\": [\n \"### Only do the below if you are using docker\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"961faa43\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"!source ~/.bashrc && \\\\\\n\",\n \" apt-get update && \\\\\\n\",\n \" apt-get install -y espeak espeak-data libespeak1 libespeak-dev && \\\\\\n\",\n \" apt-get install -y festival* && \\\\\\n\",\n \" apt-get install -y build-essential && \\\\\\n\",\n \" apt-get install -y flac libasound2-dev libsndfile1-dev vorbis-tools && \\\\\\n\",\n \" apt-get install -y libxml2-dev libxslt-dev zlib1g-dev\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"598d75cf\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"!source ~/.bashrc && \\\\\\n\",\n \" conda activate voicecraft && \\\\\\n\",\n \" pip install -r gradio_requirements.txt\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"8b9c4436\",\n \"metadata\": {},\n \"source\": [\n \"# STOP\\n\",\n \"You have to do this part manually using the mouse/keyboard and the tabs at the top.\\n\",\n \"\\n\",\n \"* Refresh your browser to make sure it picks up the new kernel.\\n\",\n \"* Kernel -> Change Kernel -> Select Kernel -> voicecraft\\n\",\n \"* Kernel -> Restart Kernel -> Yes\\n\",\n \"\\n\",\n \"Now you can run the rest of the notebook and get an audio sample output. It will automatically download more models and such. The next time you use this container, you can just start below here as the dependencies will remain available until you delete the docker container.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"f089aa96\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from gradio_app import app\\n\",\n \"app.launch()\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"voicecraft\",\n \"language\": \"python\",\n \"name\": \"voicecraft\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.9.19\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 5\n}\n" }, { "path": "gradio_app.py", "content": "import os\nimport re\nfrom num2words import num2words\nimport gradio as gr\nimport torch\nimport torchaudio\nfrom data.tokenizer import (\n AudioTokenizer,\n TextTokenizer,\n)\nfrom models import voicecraft\nimport io\nimport numpy as np\nimport random\nimport uuid\nimport nltk\nnltk.download('punkt')\n\nDEMO_PATH = os.getenv(\"DEMO_PATH\", \"./demo\")\nTMP_PATH = os.getenv(\"TMP_PATH\", \"./demo/temp\")\nMODELS_PATH = os.getenv(\"MODELS_PATH\", \"./pretrained_models\")\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nwhisper_model, align_model, voicecraft_model = None, None, None\n_whitespace_re = re.compile(r\"\\s+\")\n\ndef get_random_string():\n return \"\".join(str(uuid.uuid4()).split(\"-\"))\n\n\ndef seed_everything(seed):\n if seed != -1:\n os.environ['PYTHONHASHSEED'] = str(seed)\n random.seed(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.benchmark = False\n torch.backends.cudnn.deterministic = True\n\n\nclass WhisperxAlignModel:\n def __init__(self):\n from whisperx import load_align_model\n self.model, self.metadata = load_align_model(language_code=\"en\", device=device)\n\n def align(self, segments, audio_path):\n from whisperx import align, load_audio\n audio = load_audio(audio_path)\n return align(segments, self.model, self.metadata, audio, device, return_char_alignments=False)[\"segments\"]\n\n\nclass WhisperModel:\n def __init__(self, model_name):\n from whisper import load_model\n self.model = load_model(model_name, device)\n\n from whisper.tokenizer import get_tokenizer\n tokenizer = get_tokenizer(multilingual=False)\n self.supress_tokens = [-1] + [\n i\n for i in range(tokenizer.eot)\n if all(c in \"0123456789\" for c in tokenizer.decode([i]).removeprefix(\" \"))\n ]\n\n def transcribe(self, audio_path):\n return self.model.transcribe(audio_path, suppress_tokens=self.supress_tokens, word_timestamps=True)[\"segments\"]\n\n\nclass WhisperxModel:\n def __init__(self, model_name, align_model: WhisperxAlignModel):\n from whisperx import load_model\n self.model = load_model(model_name, device, asr_options={\"suppress_numerals\": True, \"max_new_tokens\": None, \"clip_timestamps\": None, \"hallucination_silence_threshold\": None})\n self.align_model = align_model\n\n def transcribe(self, audio_path):\n segments = self.model.transcribe(audio_path, batch_size=8)[\"segments\"]\n for segment in segments:\n segment['text'] = replace_numbers_with_words(segment['text'])\n return self.align_model.align(segments, audio_path)\n\n\ndef load_models(whisper_backend_name, whisper_model_name, alignment_model_name, voicecraft_model_name):\n global transcribe_model, align_model, voicecraft_model\n\n if voicecraft_model_name == \"330M\":\n voicecraft_model_name = \"giga330M\"\n elif voicecraft_model_name == \"830M\":\n voicecraft_model_name = \"giga830M\"\n elif voicecraft_model_name == \"330M_TTSEnhanced\":\n voicecraft_model_name = \"330M_TTSEnhanced\"\n elif voicecraft_model_name == \"830M_TTSEnhanced\":\n voicecraft_model_name = \"830M_TTSEnhanced\"\n\n if alignment_model_name is not None:\n align_model = WhisperxAlignModel()\n\n if whisper_model_name is not None:\n if whisper_backend_name == \"whisper\":\n transcribe_model = WhisperModel(whisper_model_name)\n else:\n if align_model is None:\n raise gr.Error(\"Align model required for whisperx backend\")\n transcribe_model = WhisperxModel(whisper_model_name, align_model)\n\n voicecraft_name = f\"{voicecraft_model_name}.pth\"\n model = voicecraft.VoiceCraft.from_pretrained(f\"pyp1/VoiceCraft_{voicecraft_name.replace('.pth', '')}\")\n phn2num = model.args.phn2num\n config = model.args\n model.to(device)\n\n encodec_fn = f\"{MODELS_PATH}/encodec_4cb2048_giga.th\"\n if not os.path.exists(encodec_fn):\n os.system(f\"wget https://huggingface.co/pyp1/VoiceCraft/resolve/main/encodec_4cb2048_giga.th -O \" + encodec_fn)\n\n voicecraft_model = {\n \"config\": config,\n \"phn2num\": phn2num,\n \"model\": model,\n \"text_tokenizer\": TextTokenizer(backend=\"espeak\"),\n \"audio_tokenizer\": AudioTokenizer(signature=encodec_fn)\n }\n return gr.Accordion()\n\n\ndef get_transcribe_state(segments):\n words_info = [word_info for segment in segments for word_info in segment[\"words\"]]\n transcript = \" \".join([segment[\"text\"] for segment in segments])\n transcript = transcript[1:] if transcript[0] == \" \" else transcript\n return {\n \"segments\": segments,\n \"transcript\": transcript,\n \"words_info\": words_info,\n \"transcript_with_start_time\": \" \".join([f\"{word['start']} {word['word']}\" for word in words_info]),\n \"transcript_with_end_time\": \" \".join([f\"{word['word']} {word['end']}\" for word in words_info]),\n \"word_bounds\": [f\"{word['start']} {word['word']} {word['end']}\" for word in words_info]\n }\n\n\ndef transcribe(seed, audio_path):\n if transcribe_model is None:\n raise gr.Error(\"Transcription model not loaded\")\n seed_everything(seed)\n\n segments = transcribe_model.transcribe(audio_path)\n state = get_transcribe_state(segments)\n\n return [\n state[\"transcript\"], state[\"transcript_with_start_time\"], state[\"transcript_with_end_time\"],\n gr.Dropdown(value=state[\"word_bounds\"][-1], choices=state[\"word_bounds\"], interactive=True), # prompt_to_word\n gr.Dropdown(value=state[\"word_bounds\"][0], choices=state[\"word_bounds\"], interactive=True), # edit_from_word\n gr.Dropdown(value=state[\"word_bounds\"][-1], choices=state[\"word_bounds\"], interactive=True), # edit_to_word\n state\n ]\n\n\ndef align_segments(transcript, audio_path):\n from aeneas.executetask import ExecuteTask\n from aeneas.task import Task\n import json\n config_string = 'task_language=eng|os_task_file_format=json|is_text_type=plain'\n\n tmp_transcript_path = os.path.join(TMP_PATH, f\"{get_random_string()}.txt\")\n tmp_sync_map_path = os.path.join(TMP_PATH, f\"{get_random_string()}.json\")\n with open(tmp_transcript_path, \"w\") as f:\n f.write(transcript)\n\n task = Task(config_string=config_string)\n task.audio_file_path_absolute = os.path.abspath(audio_path)\n task.text_file_path_absolute = os.path.abspath(tmp_transcript_path)\n task.sync_map_file_path_absolute = os.path.abspath(tmp_sync_map_path)\n ExecuteTask(task).execute()\n task.output_sync_map_file()\n\n with open(tmp_sync_map_path, \"r\") as f:\n return json.load(f)\n\n\ndef align(seed, transcript, audio_path):\n if align_model is None:\n raise gr.Error(\"Align model not loaded\")\n seed_everything(seed)\n transcript = replace_numbers_with_words(transcript).replace(\" \", \" \").replace(\" \", \" \")\n fragments = align_segments(transcript, audio_path)\n segments = [{\n \"start\": float(fragment[\"begin\"]),\n \"end\": float(fragment[\"end\"]),\n \"text\": \" \".join(fragment[\"lines\"])\n } for fragment in fragments[\"fragments\"]]\n segments = align_model.align(segments, audio_path)\n state = get_transcribe_state(segments)\n\n return [\n state[\"transcript_with_start_time\"], state[\"transcript_with_end_time\"],\n gr.Dropdown(value=state[\"word_bounds\"][-1], choices=state[\"word_bounds\"], interactive=True), # prompt_to_word\n gr.Dropdown(value=state[\"word_bounds\"][0], choices=state[\"word_bounds\"], interactive=True), # edit_from_word\n gr.Dropdown(value=state[\"word_bounds\"][-1], choices=state[\"word_bounds\"], interactive=True), # edit_to_word\n state\n ]\n\n\ndef get_output_audio(audio_tensors, codec_audio_sr):\n result = torch.cat(audio_tensors, 1)\n buffer = io.BytesIO()\n torchaudio.save(buffer, result, int(codec_audio_sr), format=\"wav\")\n buffer.seek(0)\n return buffer.read()\n\ndef replace_numbers_with_words(sentence):\n sentence = re.sub(r'(\\d+)', r' \\1 ', sentence) # add spaces around numbers\n def replace_with_words(match):\n num = match.group(0)\n try:\n return num2words(num) # Convert numbers to words\n except:\n return num # In case num2words fails (unlikely with digits but just to be safe)\n return re.sub(r'\\b\\d+\\b', replace_with_words, sentence) # Regular expression that matches numbers\n\ndef run(seed, left_margin, right_margin, codec_audio_sr, codec_sr, top_k, top_p, temperature,\n stop_repetition, sample_batch_size, kvcache, silence_tokens,\n audio_path, transcribe_state, transcript, smart_transcript,\n mode, prompt_end_time, edit_start_time, edit_end_time,\n split_text, selected_sentence, previous_audio_tensors):\n if voicecraft_model is None:\n raise gr.Error(\"VoiceCraft model not loaded\")\n if smart_transcript and (transcribe_state is None):\n raise gr.Error(\"Can't use smart transcript: whisper transcript not found\")\n\n seed_everything(seed)\n transcript = replace_numbers_with_words(transcript).replace(\" \", \" \").replace(\" \", \" \") # replace numbers with words, so that the phonemizer can do a better job\n\n if mode == \"Long TTS\":\n if split_text == \"Newline\":\n sentences = transcript.split('\\n')\n else:\n from nltk.tokenize import sent_tokenize\n sentences = sent_tokenize(transcript.replace(\"\\n\", \" \"))\n elif mode == \"Rerun\":\n colon_position = selected_sentence.find(':')\n selected_sentence_idx = int(selected_sentence[:colon_position])\n sentences = [selected_sentence[colon_position + 1:]]\n else:\n sentences = [transcript.replace(\"\\n\", \" \")]\n\n info = torchaudio.info(audio_path)\n audio_dur = info.num_frames / info.sample_rate\n\n audio_tensors = []\n inference_transcript = \"\"\n for sentence in sentences:\n decode_config = {\"top_k\": top_k, \"top_p\": top_p, \"temperature\": temperature, \"stop_repetition\": stop_repetition,\n \"kvcache\": kvcache, \"codec_audio_sr\": codec_audio_sr, \"codec_sr\": codec_sr,\n \"silence_tokens\": silence_tokens, \"sample_batch_size\": sample_batch_size}\n if mode != \"Edit\":\n from inference_tts_scale import inference_one_sample\n\n if smart_transcript:\n target_transcript = \"\"\n for word in transcribe_state[\"words_info\"]:\n if word[\"end\"] < prompt_end_time:\n target_transcript += word[\"word\"] + (\" \" if word[\"word\"][-1] != \" \" else \"\")\n elif (word[\"start\"] + word[\"end\"]) / 2 < prompt_end_time:\n # include part of the word it it's big, but adjust prompt_end_time\n target_transcript += word[\"word\"] + (\" \" if word[\"word\"][-1] != \" \" else \"\")\n prompt_end_time = word[\"end\"]\n break\n else:\n break\n target_transcript += f\" {sentence}\"\n else:\n target_transcript = sentence\n\n inference_transcript += target_transcript + \"\\n\"\n target_transcript = re.sub(_whitespace_re, \" \", target_transcript)\n prompt_end_frame = int(min(audio_dur, prompt_end_time) * info.sample_rate)\n _, gen_audio = inference_one_sample(voicecraft_model[\"model\"],\n voicecraft_model[\"config\"],\n voicecraft_model[\"phn2num\"],\n voicecraft_model[\"text_tokenizer\"], voicecraft_model[\"audio_tokenizer\"],\n audio_path, target_transcript, device, decode_config,\n prompt_end_frame)\n else:\n from inference_speech_editing_scale import inference_one_sample\n\n if smart_transcript:\n target_transcript = \"\"\n for word in transcribe_state[\"words_info\"]:\n if word[\"start\"] < edit_start_time:\n target_transcript += word[\"word\"] + (\" \" if word[\"word\"][-1] != \" \" else \"\")\n else:\n break\n target_transcript += f\" {sentence}\"\n for word in transcribe_state[\"words_info\"]:\n if word[\"end\"] > edit_end_time:\n target_transcript += word[\"word\"] + (\" \" if word[\"word\"][-1] != \" \" else \"\")\n else:\n target_transcript = sentence\n\n inference_transcript += target_transcript + \"\\n\"\n target_transcript = re.sub(_whitespace_re, \" \", target_transcript)\n morphed_span = (max(edit_start_time - left_margin, 1 / codec_sr), min(edit_end_time + right_margin, audio_dur))\n mask_interval = [[round(morphed_span[0]*codec_sr), round(morphed_span[1]*codec_sr)]]\n mask_interval = torch.LongTensor(mask_interval)\n\n _, gen_audio = inference_one_sample(voicecraft_model[\"model\"],\n voicecraft_model[\"config\"],\n voicecraft_model[\"phn2num\"],\n voicecraft_model[\"text_tokenizer\"], voicecraft_model[\"audio_tokenizer\"],\n audio_path, target_transcript, mask_interval, device, decode_config)\n gen_audio = gen_audio[0].cpu()\n audio_tensors.append(gen_audio)\n\n if mode != \"Rerun\":\n output_audio = get_output_audio(audio_tensors, codec_audio_sr)\n sentences = [f\"{idx}: {text}\" for idx, text in enumerate(sentences)]\n component = gr.Dropdown(choices=sentences, value=sentences[0])\n return output_audio, inference_transcript, component, audio_tensors\n else:\n previous_audio_tensors[selected_sentence_idx] = audio_tensors[0]\n output_audio = get_output_audio(previous_audio_tensors, codec_audio_sr)\n sentence_audio = get_output_audio(audio_tensors, codec_audio_sr)\n return output_audio, inference_transcript, sentence_audio, previous_audio_tensors\n\n\ndef update_input_audio(audio_path):\n if audio_path is None:\n return 0, 0, 0\n\n info = torchaudio.info(audio_path)\n max_time = round(info.num_frames / info.sample_rate, 2)\n return [\n gr.Slider(maximum=max_time, value=max_time),\n gr.Slider(maximum=max_time, value=0),\n gr.Slider(maximum=max_time, value=max_time),\n ]\n\n\ndef change_mode(mode):\n # tts_mode_controls, edit_mode_controls, edit_word_mode, split_text, long_tts_sentence_editor\n return [\n gr.Group(visible=mode != \"Edit\"),\n gr.Group(visible=mode == \"Edit\"),\n gr.Radio(visible=mode == \"Edit\"),\n gr.Radio(visible=mode == \"Long TTS\"),\n gr.Group(visible=mode == \"Long TTS\"),\n ]\n\n\ndef load_sentence(selected_sentence, codec_audio_sr, audio_tensors):\n if selected_sentence is None:\n return None\n colon_position = selected_sentence.find(':')\n selected_sentence_idx = int(selected_sentence[:colon_position])\n return get_output_audio([audio_tensors[selected_sentence_idx]], codec_audio_sr)\n\n\ndef update_bound_word(is_first_word, selected_word, edit_word_mode):\n if selected_word is None:\n return None\n\n word_start_time = float(selected_word.split(' ')[0])\n word_end_time = float(selected_word.split(' ')[-1])\n if edit_word_mode == \"Replace half\":\n bound_time = (word_start_time + word_end_time) / 2\n elif is_first_word:\n bound_time = word_start_time\n else:\n bound_time = word_end_time\n\n return bound_time\n\n\ndef update_bound_words(from_selected_word, to_selected_word, edit_word_mode):\n return [\n update_bound_word(True, from_selected_word, edit_word_mode),\n update_bound_word(False, to_selected_word, edit_word_mode),\n ]\n\n\nsmart_transcript_info = \"\"\"\nIf enabled, the target transcript will be constructed for you:
\n - In TTS and Long TTS mode just write the text you want to synthesize.
\n - In Edit mode just write the text to replace selected editing segment.
\nIf disabled, you should write the target transcript yourself:
\n - In TTS mode write prompt transcript followed by generation transcript.
\n - In Long TTS select split by newline (SENTENCE SPLIT WON'T WORK) and start each line with a prompt transcript.
\n - In Edit mode write full prompt
\n\"\"\"\n\ndemo_original_transcript = \"Gwynplaine had, besides, for his work and for his feats of strength, round his neck and over his shoulders, an esclavine of leather.\"\n\ndemo_text = {\n \"TTS\": {\n \"smart\": \"I cannot believe that the same model can also do text to speech synthesis too!\",\n \"regular\": \"Gwynplaine had, besides, for his work and for his feats of strength, I cannot believe that the same model can also do text to speech synthesis too!\"\n },\n \"Edit\": {\n \"smart\": \"take over the stage for half an hour,\",\n \"regular\": \"Gwynplaine had, besides, for his work and for his feats of strength, take over the stage for half an hour, an esclavine of leather.\"\n },\n \"Long TTS\": {\n \"smart\": \"You can run the model on a big text!\\n\"\n \"Just write it line-by-line. Or sentence-by-sentence.\\n\"\n \"If some sentences sound odd, just rerun the model on them, no need to generate the whole text again!\",\n \"regular\": \"Gwynplaine had, besides, for his work and for his feats of strength, You can run the model on a big text!\\n\"\n \"Gwynplaine had, besides, for his work and for his feats of strength, Just write it line-by-line. Or sentence-by-sentence.\\n\"\n \"Gwynplaine had, besides, for his work and for his feats of strength, If some sentences sound odd, just rerun the model on them, no need to generate the whole text again!\"\n }\n}\n\nall_demo_texts = {vv for k, v in demo_text.items() for kk, vv in v.items()}\n\ndemo_words = ['0.069 Gwynplain 0.611', '0.671 had, 0.912', '0.952 besides, 1.414', '1.494 for 1.634', '1.695 his 1.835', '1.915 work 2.136', '2.196 and 2.297', '2.337 for 2.517', '2.557 his 2.678', '2.758 feats 3.019', '3.079 of 3.139', '3.2 strength, 3.561', '4.022 round 4.263', '4.303 his 4.444', '4.524 neck 4.705', '4.745 and 4.825', '4.905 over 5.086', '5.146 his 5.266', '5.307 shoulders, 5.768', '6.23 an 6.33', '6.531 esclavine 7.133', '7.213 of 7.293', '7.353 leather. 7.614']\n\ndemo_words_info = [{'word': 'Gwynplain', 'start': 0.069, 'end': 0.611, 'score': 0.833}, {'word': 'had,', 'start': 0.671, 'end': 0.912, 'score': 0.879}, {'word': 'besides,', 'start': 0.952, 'end': 1.414, 'score': 0.863}, {'word': 'for', 'start': 1.494, 'end': 1.634, 'score': 0.89}, {'word': 'his', 'start': 1.695, 'end': 1.835, 'score': 0.669}, {'word': 'work', 'start': 1.915, 'end': 2.136, 'score': 0.916}, {'word': 'and', 'start': 2.196, 'end': 2.297, 'score': 0.766}, {'word': 'for', 'start': 2.337, 'end': 2.517, 'score': 0.808}, {'word': 'his', 'start': 2.557, 'end': 2.678, 'score': 0.786}, {'word': 'feats', 'start': 2.758, 'end': 3.019, 'score': 0.97}, {'word': 'of', 'start': 3.079, 'end': 3.139, 'score': 0.752}, {'word': 'strength,', 'start': 3.2, 'end': 3.561, 'score': 0.742}, {'word': 'round', 'start': 4.022, 'end': 4.263, 'score': 0.916}, {'word': 'his', 'start': 4.303, 'end': 4.444, 'score': 0.666}, {'word': 'neck', 'start': 4.524, 'end': 4.705, 'score': 0.908}, {'word': 'and', 'start': 4.745, 'end': 4.825, 'score': 0.882}, {'word': 'over', 'start': 4.905, 'end': 5.086, 'score': 0.847}, {'word': 'his', 'start': 5.146, 'end': 5.266, 'score': 0.791}, {'word': 'shoulders,', 'start': 5.307, 'end': 5.768, 'score': 0.729}, {'word': 'an', 'start': 6.23, 'end': 6.33, 'score': 0.854}, {'word': 'esclavine', 'start': 6.531, 'end': 7.133, 'score': 0.803}, {'word': 'of', 'start': 7.213, 'end': 7.293, 'score': 0.772}, {'word': 'leather.', 'start': 7.353, 'end': 7.614, 'score': 0.896}]\n\n\ndef update_demo(mode, smart_transcript, edit_word_mode, transcript, edit_from_word, edit_to_word):\n if transcript not in all_demo_texts:\n return transcript, edit_from_word, edit_to_word\n\n replace_half = edit_word_mode == \"Replace half\"\n change_edit_from_word = edit_from_word == demo_words[2] or edit_from_word == demo_words[3]\n change_edit_to_word = edit_to_word == demo_words[11] or edit_to_word == demo_words[12]\n demo_edit_from_word_value = demo_words[2] if replace_half else demo_words[3]\n demo_edit_to_word_value = demo_words[12] if replace_half else demo_words[11]\n return [\n demo_text[mode][\"smart\" if smart_transcript else \"regular\"],\n demo_edit_from_word_value if change_edit_from_word else edit_from_word,\n demo_edit_to_word_value if change_edit_to_word else edit_to_word,\n ]\n\n\ndef get_app():\n with gr.Blocks() as app:\n with gr.Row():\n with gr.Column(scale=2):\n load_models_btn = gr.Button(value=\"Load models\")\n with gr.Column(scale=5):\n with gr.Accordion(\"Select models\", open=False) as models_selector:\n with gr.Row():\n voicecraft_model_choice = gr.Radio(label=\"VoiceCraft model\", value=\"830M_TTSEnhanced\",\n choices=[\"330M\", \"830M\", \"330M_TTSEnhanced\", \"830M_TTSEnhanced\"])\n whisper_backend_choice = gr.Radio(label=\"Whisper backend\", value=\"whisperX\", choices=[\"whisperX\", \"whisper\"])\n whisper_model_choice = gr.Radio(label=\"Whisper model\", value=\"base.en\",\n choices=[None, \"base.en\", \"small.en\", \"medium.en\", \"large\"])\n align_model_choice = gr.Radio(label=\"Forced alignment model\", value=\"whisperX\", choices=[\"whisperX\", None])\n\n with gr.Row():\n with gr.Column(scale=2):\n input_audio = gr.Audio(value=f\"{DEMO_PATH}/5895_34622_000026_000002.wav\", label=\"Input Audio\", type=\"filepath\", interactive=True)\n with gr.Group():\n original_transcript = gr.Textbox(label=\"Original transcript\", lines=5, value=demo_original_transcript,\n info=\"Use whisperx model to get the transcript. Fix and align it if necessary.\")\n with gr.Accordion(\"Word start time\", open=False):\n transcript_with_start_time = gr.Textbox(label=\"Start time\", lines=5, interactive=False, info=\"Start time before each word\")\n with gr.Accordion(\"Word end time\", open=False):\n transcript_with_end_time = gr.Textbox(label=\"End time\", lines=5, interactive=False, info=\"End time after each word\")\n\n transcribe_btn = gr.Button(value=\"Transcribe\")\n align_btn = gr.Button(value=\"Align\")\n\n with gr.Column(scale=3):\n with gr.Group():\n transcript = gr.Textbox(label=\"Text\", lines=7, value=demo_text[\"TTS\"][\"smart\"])\n with gr.Row():\n smart_transcript = gr.Checkbox(label=\"Smart transcript\", value=True)\n with gr.Accordion(label=\"?\", open=False):\n info = gr.Markdown(value=smart_transcript_info)\n\n with gr.Row():\n mode = gr.Radio(label=\"Mode\", choices=[\"TTS\", \"Edit\", \"Long TTS\"], value=\"TTS\")\n split_text = gr.Radio(label=\"Split text\", choices=[\"Newline\", \"Sentence\"], value=\"Newline\",\n info=\"Split text into parts and run TTS for each part.\", visible=False)\n edit_word_mode = gr.Radio(label=\"Edit word mode\", choices=[\"Replace half\", \"Replace all\"], value=\"Replace all\",\n info=\"What to do with first and last word\", visible=False)\n\n with gr.Group() as tts_mode_controls:\n prompt_to_word = gr.Dropdown(label=\"Last word in prompt\", choices=demo_words, value=demo_words[11], interactive=True)\n prompt_end_time = gr.Slider(label=\"Prompt end time\", minimum=0, maximum=7.614, step=0.001, value=3.600)\n\n with gr.Group(visible=False) as edit_mode_controls:\n with gr.Row():\n edit_from_word = gr.Dropdown(label=\"First word to edit\", choices=demo_words, value=demo_words[12], interactive=True)\n edit_to_word = gr.Dropdown(label=\"Last word to edit\", choices=demo_words, value=demo_words[18], interactive=True)\n with gr.Row():\n edit_start_time = gr.Slider(label=\"Edit from time\", minimum=0, maximum=7.614, step=0.001, value=4.022)\n edit_end_time = gr.Slider(label=\"Edit to time\", minimum=0, maximum=7.614, step=0.001, value=5.768)\n\n run_btn = gr.Button(value=\"Run\")\n\n with gr.Column(scale=2):\n output_audio = gr.Audio(label=\"Output Audio\")\n with gr.Accordion(\"Inference transcript\", open=False):\n inference_transcript = gr.Textbox(label=\"Inference transcript\", lines=5, interactive=False,\n info=\"Inference was performed on this transcript.\")\n with gr.Group(visible=False) as long_tts_sentence_editor:\n sentence_selector = gr.Dropdown(label=\"Sentence\", value=None,\n info=\"Select sentence you want to regenerate\")\n sentence_audio = gr.Audio(label=\"Sentence Audio\", scale=2)\n rerun_btn = gr.Button(value=\"Rerun\")\n\n with gr.Row():\n with gr.Accordion(\"Generation Parameters - change these if you are unhappy with the generation\", open=False):\n stop_repetition = gr.Radio(label=\"stop_repetition\", choices=[-1, 1, 2, 3, 4], value=3,\n info=\"if there are long silence in the generated audio, reduce the stop_repetition to 2 or 1. -1 = disabled\")\n sample_batch_size = gr.Number(label=\"speech rate\", value=3, precision=0,\n info=\"The higher the number, the faster the output will be. \"\n \"Under the hood, the model will generate this many samples and choose the shortest one. \"\n \"For giga330M_TTSEnhanced, 1 or 2 should be fine since the model is trained to do TTS.\")\n seed = gr.Number(label=\"seed\", value=-1, precision=0, info=\"random seeds always works :)\")\n kvcache = gr.Radio(label=\"kvcache\", choices=[0, 1], value=1,\n info=\"set to 0 to use less VRAM, but with slower inference\")\n left_margin = gr.Number(label=\"left_margin\", value=0.08, info=\"margin to the left of the editing segment\")\n right_margin = gr.Number(label=\"right_margin\", value=0.08, info=\"margin to the right of the editing segment\")\n top_p = gr.Number(label=\"top_p\", value=1, info=\"do not do topp sampling therefore set it to 1\")\n temperature = gr.Number(label=\"temperature\", value=1, info=\"haven't try other values, do not recommend to change\")\n top_k = gr.Number(label=\"top_k\", value=40, info=\"40 is a good default, can also try 20, 30\")\n codec_audio_sr = gr.Number(label=\"codec_audio_sr\", value=16000, info='encodec specific, Do not change')\n codec_sr = gr.Number(label=\"codec_sr\", value=50, info='encodec specific, Do not change')\n silence_tokens = gr.Textbox(label=\"silence tokens\", value=\"[1388,1898,131]\", info=\"encodec specific, do not change\")\n\n\n audio_tensors = gr.State()\n transcribe_state = gr.State(value={\"words_info\": demo_words_info})\n\n\n mode.change(fn=update_demo,\n inputs=[mode, smart_transcript, edit_word_mode, transcript, edit_from_word, edit_to_word],\n outputs=[transcript, edit_from_word, edit_to_word])\n edit_word_mode.change(fn=update_demo,\n inputs=[mode, smart_transcript, edit_word_mode, transcript, edit_from_word, edit_to_word],\n outputs=[transcript, edit_from_word, edit_to_word])\n smart_transcript.change(fn=update_demo,\n inputs=[mode, smart_transcript, edit_word_mode, transcript, edit_from_word, edit_to_word],\n outputs=[transcript, edit_from_word, edit_to_word])\n\n load_models_btn.click(fn=load_models,\n inputs=[whisper_backend_choice, whisper_model_choice, align_model_choice, voicecraft_model_choice],\n outputs=[models_selector])\n\n input_audio.upload(fn=update_input_audio,\n inputs=[input_audio],\n outputs=[prompt_end_time, edit_start_time, edit_end_time])\n transcribe_btn.click(fn=transcribe,\n inputs=[seed, input_audio],\n outputs=[original_transcript, transcript_with_start_time, transcript_with_end_time,\n prompt_to_word, edit_from_word, edit_to_word, transcribe_state])\n align_btn.click(fn=align,\n inputs=[seed, original_transcript, input_audio],\n outputs=[transcript_with_start_time, transcript_with_end_time,\n prompt_to_word, edit_from_word, edit_to_word, transcribe_state])\n\n mode.change(fn=change_mode,\n inputs=[mode],\n outputs=[tts_mode_controls, edit_mode_controls, edit_word_mode, split_text, long_tts_sentence_editor])\n\n run_btn.click(fn=run,\n inputs=[\n seed, left_margin, right_margin,\n codec_audio_sr, codec_sr,\n top_k, top_p, temperature,\n stop_repetition, sample_batch_size,\n kvcache, silence_tokens,\n input_audio, transcribe_state, transcript, smart_transcript,\n mode, prompt_end_time, edit_start_time, edit_end_time,\n split_text, sentence_selector, audio_tensors\n ],\n outputs=[output_audio, inference_transcript, sentence_selector, audio_tensors])\n\n sentence_selector.change(fn=load_sentence,\n inputs=[sentence_selector, codec_audio_sr, audio_tensors],\n outputs=[sentence_audio])\n rerun_btn.click(fn=run,\n inputs=[\n seed, left_margin, right_margin,\n codec_audio_sr, codec_sr,\n top_k, top_p, temperature,\n stop_repetition, sample_batch_size,\n kvcache, silence_tokens,\n input_audio, transcribe_state, transcript, smart_transcript,\n gr.State(value=\"Rerun\"), prompt_end_time, edit_start_time, edit_end_time,\n split_text, sentence_selector, audio_tensors\n ],\n outputs=[output_audio, inference_transcript, sentence_audio, audio_tensors])\n\n prompt_to_word.change(fn=update_bound_word,\n inputs=[gr.State(False), prompt_to_word, gr.State(\"Replace all\")],\n outputs=[prompt_end_time])\n edit_from_word.change(fn=update_bound_word,\n inputs=[gr.State(True), edit_from_word, edit_word_mode],\n outputs=[edit_start_time])\n edit_to_word.change(fn=update_bound_word,\n inputs=[gr.State(False), edit_to_word, edit_word_mode],\n outputs=[edit_end_time])\n edit_word_mode.change(fn=update_bound_words,\n inputs=[edit_from_word, edit_to_word, edit_word_mode],\n outputs=[edit_start_time, edit_end_time])\n return app\n\n\nif __name__ == \"__main__\":\n import argparse\n\n parser = argparse.ArgumentParser(description=\"VoiceCraft gradio app.\")\n \n parser.add_argument(\"--demo-path\", default=\"./demo\", help=\"Path to demo directory\")\n parser.add_argument(\"--tmp-path\", default=\"./demo/temp\", help=\"Path to tmp directory\")\n parser.add_argument(\"--models-path\", default=\"./pretrained_models\", help=\"Path to voicecraft models directory\")\n parser.add_argument(\"--port\", default=7860, type=int, help=\"App port\")\n parser.add_argument(\"--share\", action=\"store_true\", help=\"Launch with public url\")\n parser.add_argument(\"--server_name\", default=\"127.0.0.1\", type=str, help=\"Server name for launching the app. 127.0.0.1 for localhost; 0.0.0.0 to allow access from other machines in the local network. Might also give access to external users depends on the firewall settings.\")\n\n os.environ[\"USER\"] = os.getenv(\"USER\", \"user\")\n args = parser.parse_args()\n DEMO_PATH = args.demo_path\n TMP_PATH = args.tmp_path\n MODELS_PATH = args.models_path\n\n app = get_app()\n app.queue().launch(share=args.share, server_name=args.server_name, server_port=args.port)\n" }, { "path": "gradio_requirements.txt", "content": "gradio==3.50.2\nnltk>=3.8.1\nopenai-whisper>=20231117\naeneas>=1.7.3.0\nwhisperx>=3.1.1\nhuggingface_hub==0.22.2\nnum2words==0.5.13\n" }, { "path": "inference_speech_editing.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"os.environ[\\\"CUDA_DEVICE_ORDER\\\"]=\\\"PCI_BUS_ID\\\" \\n\",\n \"os.environ[\\\"CUDA_VISIBLE_DEVICES\\\"]=\\\"7\\\"\\n\",\n \"os.environ[\\\"USER\\\"] = \\\"YOUR_USERNAME\\\" # TODO change this to your username\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 2,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"/home/pyp/miniconda3/envs/voicecraft/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\\n\",\n \" from .autonotebook import tqdm as notebook_tqdm\\n\"\n ]\n }\n ],\n \"source\": [\n \"# import libs\\n\",\n \"import torch\\n\",\n \"import torchaudio\\n\",\n \"import numpy as np\\n\",\n \"import random\\n\",\n \"from argparse import Namespace\\n\",\n \"\\n\",\n \"from data.tokenizer import (\\n\",\n \" AudioTokenizer,\\n\",\n \" TextTokenizer,\\n\",\n \")\\n\",\n \"\\n\",\n \"from models import voicecraft\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# install MFA models and dictionaries if you haven't done so already\\n\",\n \"!source ~/.bashrc && \\\\\\n\",\n \" conda activate voicecraft && \\\\\\n\",\n \" mfa model download dictionary english_us_arpa && \\\\\\n\",\n \" mfa model download acoustic english_us_arpa\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# hyperparameters for inference\\n\",\n \"left_margin = 0.08\\n\",\n \"right_margin = 0.08\\n\",\n \"codec_audio_sr = 16000\\n\",\n \"codec_sr = 50\\n\",\n \"top_k = 40\\n\",\n \"top_p = 1\\n\",\n \"temperature = 1\\n\",\n \"kvcache = 0\\n\",\n \"# adjust the below three arguments if the generation is not as good\\n\",\n \"seed = 1 # random seed magic\\n\",\n \"silence_tokens = [1388,1898,131] # if there are long silence in the generated audio, reduce the stop_repetition to 3, 2 or even 1\\n\",\n \"stop_repetition = -1 # -1 means do not adjust prob of silence tokens. if there are long silence or unnaturally strecthed words, increase sample_batch_size to 2, 3 or even 4\\n\",\n \"# what this will do to the model is that the model will run sample_batch_size examples of the same audio, and pick the one that's the shortest\\n\",\n \"def seed_everything(seed):\\n\",\n \" os.environ['PYTHONHASHSEED'] = str(seed)\\n\",\n \" random.seed(seed)\\n\",\n \" np.random.seed(seed)\\n\",\n \" torch.manual_seed(seed)\\n\",\n \" torch.cuda.manual_seed(seed)\\n\",\n \" torch.backends.cudnn.benchmark = False\\n\",\n \" torch.backends.cudnn.deterministic = True\\n\",\n \"seed_everything(seed)\\n\",\n \"device = \\\"cuda\\\" if torch.cuda.is_available() else \\\"cpu\\\"\\n\",\n \"# load model, tokenizer, and other necessary files\\n\",\n \"voicecraft_name=\\\"giga330M.pth\\\" # or gigaHalfLibri330M_TTSEnhanced_max16s.pth, giga830M.pth\\n\",\n \"\\n\",\n \"# the new way of loading the model, with huggingface, recommended\\n\",\n \"from models import voicecraft\\n\",\n \"model = voicecraft.VoiceCraft.from_pretrained(f\\\"pyp1/VoiceCraft_{voicecraft_name.replace('.pth', '')}\\\")\\n\",\n \"phn2num = model.args.phn2num\\n\",\n \"config = vars(model.args)\\n\",\n \"model.to(device)\\n\",\n \"\\n\",\n \"# # the old way of loading the model\\n\",\n \"# from models import voicecraft\\n\",\n \"# filepath = hf_hub_download(repo_id=\\\"pyp1/VoiceCraft\\\", filename=voicecraft_name, repo_type=\\\"model\\\")\\n\",\n \"# ckpt = torch.load(filepath, map_location=\\\"cpu\\\")\\n\",\n \"# model = voicecraft.VoiceCraft(ckpt[\\\"config\\\"])\\n\",\n \"# model.load_state_dict(ckpt[\\\"model\\\"])\\n\",\n \"# config = vars(model.args)\\n\",\n \"# phn2num = ckpt[\\\"phn2num\\\"]\\n\",\n \"# model.to(device)\\n\",\n \"# model.eval()\\n\",\n \"\\n\",\n \"encodec_fn = \\\"./pretrained_models/encodec_4cb2048_giga.th\\\"\\n\",\n \"if not os.path.exists(encodec_fn):\\n\",\n \" os.system(f\\\"wget https://huggingface.co/pyp1/VoiceCraft/resolve/main/encodec_4cb2048_giga.th\\\")\\n\",\n \" os.system(f\\\"mv encodec_4cb2048_giga.th ./pretrained_models/encodec_4cb2048_giga.th\\\")\\n\",\n \"audio_tokenizer = AudioTokenizer(signature=encodec_fn) # will also put the neural codec model on gpu\\n\",\n \"\\n\",\n \"text_tokenizer = TextTokenizer(backend=\\\"espeak\\\")\\n\",\n \"\\n\",\n \"# point to the original file or record the file\\n\",\n \"# write down the transcript for the file, or run whisper to get the transcript (and you can modify it if it's not accurate), save it as a .txt file\\n\",\n \"orig_audio = \\\"./demo/84_121550_000074_000000.wav\\\"\\n\",\n \"orig_transcript = \\\"But when I had approached so near to them The common object, which the sense deceives, Lost not by distance any of its marks,\\\"\\n\",\n \"# move the audio and transcript to temp folder\\n\",\n \"temp_folder = \\\"./demo/temp\\\"\\n\",\n \"os.makedirs(temp_folder, exist_ok=True)\\n\",\n \"os.system(f\\\"cp {orig_audio} {temp_folder}\\\")\\n\",\n \"filename = os.path.splitext(orig_audio.split(\\\"/\\\")[-1])[0]\\n\",\n \"with open(f\\\"{temp_folder}/{filename}.txt\\\", \\\"w\\\") as f:\\n\",\n \" f.write(orig_transcript)\\n\",\n \"# run MFA to get the alignment\\n\",\n \"align_temp = f\\\"{temp_folder}/mfa_alignments\\\"\\n\",\n \"os.makedirs(align_temp, exist_ok=True)\\n\",\n \"os.system(f\\\"mfa align -j 1 --output_format csv {temp_folder} english_us_arpa english_us_arpa {align_temp}\\\")\\n\",\n \"# if it fail, it could be because the audio is too hard for the alignment model, increasing the beam size usually solves the issue\\n\",\n \"# os.system(f\\\"mfa align -j 1 --output_format csv {temp_folder} english_us_arpa english_us_arpa {align_temp} --beam 1000 --retry_beam 2000\\\")\\n\",\n \"audio_fn = f\\\"{temp_folder}/{filename}.wav\\\"\\n\",\n \"transcript_fn = f\\\"{temp_folder}/{filename}.txt\\\"\\n\",\n \"align_fn = f\\\"{align_temp}/{filename}.csv\\\"\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"original:\\n\"\n ]\n },\n {\n \"data\": {\n \"text/html\": [\n \"\\n\",\n \" \\n\",\n \" \"\n ],\n \"text/plain\": [\n \"\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n },\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"edited:\\n\"\n ]\n },\n {\n \"data\": {\n \"text/html\": [\n \"\\n\",\n \" \\n\",\n \" \"\n ],\n \"text/plain\": [\n \"\"\n ]\n },\n \"metadata\": {},\n \"output_type\": \"display_data\"\n }\n ],\n \"source\": [\n \"editTypes_set = set(['substitution', 'insertion', 'deletion'])\\n\",\n \"# propose what do you want the target modified transcript to be\\n\",\n \"target_transcript = \\\"But when I saw the mirage of the lake in the distance, which the sense deceives, Lost not by distance any of its marks,\\\"\\n\",\n \"edit_type = \\\"substitution\\\"\\n\",\n \"assert edit_type in editTypes_set, f\\\"Invalid edit type {edit_type}. Must be one of {editTypes_set}.\\\"\\n\",\n \"\\n\",\n \"# if you want to do a second modification on top of the first one, write down the second modification (target_transcript2, type_of_modification2)\\n\",\n \"# make sure the two modification do not overlap, if they do, you need to combine them into one modification\\n\",\n \"\\n\",\n \"# run the script to turn user input to the format that the model can take\\n\",\n \"from edit_utils import get_span\\n\",\n \"orig_span, new_span = get_span(orig_transcript, target_transcript, edit_type)\\n\",\n \"if orig_span[0] > orig_span[1]:\\n\",\n \" RuntimeError(f\\\"example {audio_fn} failed\\\")\\n\",\n \"if orig_span[0] == orig_span[1]:\\n\",\n \" orig_span_save = [orig_span[0]]\\n\",\n \"else:\\n\",\n \" orig_span_save = orig_span\\n\",\n \"if new_span[0] == new_span[1]:\\n\",\n \" new_span_save = [new_span[0]]\\n\",\n \"else:\\n\",\n \" new_span_save = new_span\\n\",\n \"\\n\",\n \"orig_span_save = \\\",\\\".join([str(item) for item in orig_span_save])\\n\",\n \"new_span_save = \\\",\\\".join([str(item) for item in new_span_save])\\n\",\n \"from inference_speech_editing_scale import get_mask_interval\\n\",\n \"\\n\",\n \"start, end = get_mask_interval(align_fn, orig_span_save, edit_type)\\n\",\n \"info = torchaudio.info(audio_fn)\\n\",\n \"audio_dur = info.num_frames / info.sample_rate\\n\",\n \"morphed_span = (max(start - left_margin, 1/codec_sr), min(end + right_margin, audio_dur)) # in seconds\\n\",\n \"\\n\",\n \"# span in codec frames\\n\",\n \"mask_interval = [[round(morphed_span[0]*codec_sr), round(morphed_span[1]*codec_sr)]]\\n\",\n \"mask_interval = torch.LongTensor(mask_interval) # [M,2], M==1 for now\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"# run the model to get the output\\n\",\n \"from inference_speech_editing_scale import inference_one_sample\\n\",\n \"\\n\",\n \"decode_config = {'top_k': top_k, 'top_p': top_p, 'temperature': temperature, 'stop_repetition': stop_repetition, 'kvcache': kvcache, \\\"codec_audio_sr\\\": codec_audio_sr, \\\"codec_sr\\\": codec_sr, \\\"silence_tokens\\\": silence_tokens}\\n\",\n \"orig_audio, new_audio = inference_one_sample(model, Namespace(**config), phn2num, text_tokenizer, audio_tokenizer, audio_fn, target_transcript, mask_interval, device, decode_config)\\n\",\n \" \\n\",\n \"# save segments for comparison\\n\",\n \"orig_audio, new_audio = orig_audio[0].cpu(), new_audio[0].cpu()\\n\",\n \"# logging.info(f\\\"length of the resynthesize orig audio: {orig_audio.shape}\\\")\\n\",\n \"\\n\",\n \"# display the audio\\n\",\n \"from IPython.display import Audio\\n\",\n \"print(\\\"original:\\\")\\n\",\n \"display(Audio(orig_audio, rate=codec_audio_sr))\\n\",\n \"\\n\",\n \"print(\\\"edited:\\\")\\n\",\n \"display(Audio(new_audio, rate=codec_audio_sr))\\n\",\n \"\\n\",\n \"# # save the audio\\n\",\n \"# # output_dir\\n\",\n \"# output_dir = \\\"./demo/generated_se\\\"\\n\",\n \"# os.makedirs(output_dir, exist_ok=True)\\n\",\n \"\\n\",\n \"# save_fn_new = f\\\"{output_dir}/{os.path.basename(audio_fn)[:-4]}_new_seed{seed}.wav\\\"\\n\",\n \"\\n\",\n \"# torchaudio.save(save_fn_new, new_audio, codec_audio_sr)\\n\",\n \"\\n\",\n \"# save_fn_orig = f\\\"{output_dir}/{os.path.basename(audio_fn)[:-4]}_orig.wav\\\"\\n\",\n \"# if not os.path.isfile(save_fn_orig):\\n\",\n \"# orig_audio, orig_sr = torchaudio.load(audio_fn)\\n\",\n \"# if orig_sr != codec_audio_sr:\\n\",\n \"# orig_audio = torchaudio.transforms.Resample(orig_sr, codec_audio_sr)(orig_audio)\\n\",\n \"# torchaudio.save(save_fn_orig, orig_audio, codec_audio_sr)\\n\",\n \"\\n\",\n \"# # if you get error importing T5 in transformers\\n\",\n \"# # try \\n\",\n \"# # pip uninstall Pillow\\n\",\n \"# # pip install Pillow\\n\",\n \"# # you are likely to get warning looks like WARNING:phonemizer:words count mismatch on 300.0% of the lines (3/1), this can be safely ignored\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": []\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"voicecraft\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.9.18\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n" }, { "path": "inference_speech_editing_scale.py", "content": "import argparse, pickle\nimport logging\nimport os, random\nimport numpy as np\nimport torch\nimport torchaudio\n\nfrom data.tokenizer import (\n AudioTokenizer,\n TextTokenizer,\n tokenize_audio,\n tokenize_text\n)\n\nfrom models import voicecraft\nimport argparse, time, tqdm\n\n# this script only works for the musicgen architecture\ndef get_args():\n parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n parser.add_argument(\"--manifest_fn\", type=str, default=\"path/to/eval_metadata_file\")\n parser.add_argument(\"--audio_root\", type=str, default=\"path/to/audio_folder\")\n parser.add_argument(\"--exp_dir\", type=str, default=\"path/to/model_folder\")\n parser.add_argument(\"--left_margin\", type=float, default=0.08, help=\"extra space on the left to the word boundary\")\n parser.add_argument(\"--right_margin\", type=float, default=0.08, help=\"extra space on the right to the word boundary\")\n parser.add_argument(\"--seed\", type=int, default=1)\n parser.add_argument(\"--codec_audio_sr\", type=int, default=16000, help='the sample rate of audio that the codec is trained for')\n parser.add_argument(\"--codec_sr\", type=int, default=50, help='the sample rate of the codec codes')\n parser.add_argument(\"--top_k\", type=int, default=-1, help=\"sampling param\")\n parser.add_argument(\"--top_p\", type=float, default=0.8, help=\"sampling param\")\n parser.add_argument(\"--temperature\", type=float, default=1.0, help=\"sampling param\")\n parser.add_argument(\"--output_dir\", type=str, default=None)\n parser.add_argument(\"--device\", type=str, default=\"cuda\")\n parser.add_argument(\"--signature\", type=str, default=None, help=\"path to the encodec model\")\n parser.add_argument(\"--stop_repetition\", type=int, default=2, help=\"used for inference, when the number of consecutive repetition of a token is bigger than this, stop it\")\n parser.add_argument(\"--kvcache\", type=int, default=1, help='if true, use kv cache, which is 4-8x faster than without')\n parser.add_argument(\"--silence_tokens\", type=str, default=\"[1388,1898,131]\", help=\"note that if you are not using the pretrained encodec 6f79c6a8, make sure you specified it yourself, rather than using the default\")\n return parser.parse_args()\n\n@torch.no_grad()\ndef inference_one_sample(model, model_args, phn2num, text_tokenizer, audio_tokenizer, audio_fn, target_text, mask_interval, device, decode_config):\n # phonemize\n text_tokens = [phn2num[phn] for phn in\n tokenize_text(\n text_tokenizer, text=target_text.strip()\n ) if phn in phn2num\n ]\n text_tokens = torch.LongTensor(text_tokens).unsqueeze(0)\n text_tokens_lens = torch.LongTensor([text_tokens.shape[-1]])\n\n encoded_frames = tokenize_audio(audio_tokenizer, audio_fn)\n original_audio = encoded_frames[0][0].transpose(2,1) # [1,T,K]\n assert original_audio.ndim==3 and original_audio.shape[0] == 1 and original_audio.shape[2] == model_args.n_codebooks, original_audio.shape\n logging.info(f\"with direct encodec encoding before input, original audio length: {original_audio.shape[1]} codec frames, which is {original_audio.shape[1]/decode_config['codec_sr']:.2f} sec.\")\n\n # forward\n stime = time.time()\n encoded_frames = model.inference(\n text_tokens.to(device),\n text_tokens_lens.to(device),\n original_audio[...,:model_args.n_codebooks].to(device), # [1,T,8]\n mask_interval=mask_interval.unsqueeze(0).to(device),\n top_k=decode_config['top_k'],\n top_p=decode_config['top_p'],\n temperature=decode_config['temperature'],\n stop_repetition=decode_config['stop_repetition'],\n kvcache=decode_config['kvcache'],\n silence_tokens=eval(decode_config['silence_tokens']) if type(decode_config['silence_tokens']) == str else decode_config['silence_tokens'],\n ) # output is [1,K,T]\n logging.info(f\"inference on one sample take: {time.time() - stime:.4f} sec.\")\n if type(encoded_frames) == tuple:\n encoded_frames = encoded_frames[0]\n logging.info(f\"generated encoded_frames.shape: {encoded_frames.shape}, which is {encoded_frames.shape[-1]/decode_config['codec_sr']} sec.\")\n \n\n # decode (both original and generated)\n original_sample = audio_tokenizer.decode(\n [(original_audio.transpose(2,1), None)] # [1,T,8] -> [1,8,T]\n )\n generated_sample = audio_tokenizer.decode(\n [(encoded_frames, None)]\n )\n\n return original_sample, generated_sample\n\ndef get_model(exp_dir, device=None):\n with open(os.path.join(exp_dir, \"args.pkl\"), \"rb\") as f:\n model_args = pickle.load(f)\n\n logging.info(\"load model weights...\")\n model = voicecraft.VoiceCraft(model_args)\n ckpt_fn = os.path.join(exp_dir, \"best_bundle.pth\")\n ckpt = torch.load(ckpt_fn, map_location='cpu')['model']\n phn2num = torch.load(ckpt_fn, map_location='cpu')['phn2num']\n model.load_state_dict(ckpt)\n del ckpt\n logging.info(\"done loading weights...\")\n if device == None:\n device = torch.device(\"cpu\")\n if torch.cuda.is_available():\n device = torch.device(\"cuda:0\")\n model.to(device)\n model.eval()\n return model, model_args, phn2num\n\n\ndef get_mask_interval(ali_fn, word_span_ind, editType):\n with open(ali_fn, \"r\") as rf:\n data = [l.strip().split(\",\") for l in rf.readlines()]\n data = data[1:]\n tmp = word_span_ind.split(\",\")\n s, e = int(tmp[0]), int(tmp[-1])\n start = None\n for j, item in enumerate(data):\n if j == s and item[3] == \"words\":\n if editType == 'insertion':\n start = float(item[1])\n else:\n start = float(item[0])\n if j == e and item[3] == \"words\":\n if editType == 'insertion':\n end = float(item[0])\n else:\n end = float(item[1])\n assert start != None\n break\n return (start, end)\n\nif __name__ == \"__main__\":\n def seed_everything(seed):\n os.environ['PYTHONHASHSEED'] = str(seed)\n random.seed(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.benchmark = False\n torch.backends.cudnn.deterministic = True\n formatter = (\n \"%(asctime)s [%(levelname)s] %(filename)s:%(lineno)d || %(message)s\"\n )\n logging.basicConfig(format=formatter, level=logging.INFO)\n args = get_args()\n # args.device = 'cpu'\n args.allowed_repeat_tokens = eval(args.allowed_repeat_tokens)\n seed_everything(args.seed)\n\n # load model\n stime = time.time()\n logging.info(f\"loading model from {args.exp_dir}\")\n model, model_args, phn2num = get_model(args.exp_dir)\n if not os.path.isfile(model_args.exp_dir):\n model_args.exp_dir = args.exp_dir\n logging.info(f\"loading model done, took {time.time() - stime:.4f} sec\")\n\n # setup text and audio tokenizer\n text_tokenizer = TextTokenizer(backend=\"espeak\")\n audio_tokenizer = AudioTokenizer(signature=args.signature) # will also put the neural codec model on gpu\n\n with open(args.manifest_fn, \"r\") as rf:\n manifest = [l.strip().split(\"\\t\") for l in rf.readlines()]\n manifest = manifest[1:]\n \n # wav_fn\ttxt_fn\talingment_fn\tnum_words\tword_span_ind\n audio_fns = []\n target_texts = []\n mask_intervals = []\n edit_types = []\n new_spans = []\n orig_spans = []\n os.makedirs(args.output_dir, exist_ok=True)\n if args.crop_concat:\n mfa_temp = f\"{args.output_dir}/mfa_temp\"\n os.makedirs(mfa_temp, exist_ok=True)\n for item in manifest:\n audio_fn = os.path.join(args.audio_root, item[0])\n temp = torchaudio.info(audio_fn)\n audio_dur = temp.num_frames/temp.sample_rate\n audio_fns.append(audio_fn)\n target_text = item[2].split(\"|\")[-1]\n edit_types.append(item[5].split(\"|\"))\n new_spans.append(item[4].split(\"|\"))\n orig_spans.append(item[3].split(\"|\"))\n target_texts.append(target_text) # the last transcript is the target\n # mi needs to be created from word_ind_span and alignment_fn, along with args.left_margin and args.right_margin\n mis = []\n all_ind_intervals = item[3].split(\"|\")\n editTypes = item[5].split(\"|\")\n smaller_indx = []\n alignment_fn = os.path.join(args.audio_root, \"aligned\", item[0].replace(\".wav\", \".csv\"))\n if not os.path.isfile(alignment_fn):\n alignment_fn = alignment_fn.replace(\"/aligned/\", \"/aligned_csv/\")\n assert os.path.isfile(alignment_fn), alignment_fn\n for ind_inter,editType in zip(all_ind_intervals, editTypes):\n # print(ind_inter)\n mi = get_mask_interval(alignment_fn, ind_inter, editType)\n mi = (max(mi[0] - args.left_margin, 1/args.codec_sr), min(mi[1] + args.right_margin, audio_dur)) # in seconds\n mis.append(mi)\n smaller_indx.append(mi[0])\n ind = np.argsort(smaller_indx)\n mis = [mis[id] for id in ind]\n mask_intervals.append(mis)\n\n\n\n for i, (audio_fn, target_text, mask_interval) in enumerate(tqdm.tqdm(zip(audio_fns, target_texts, mask_intervals))):\n orig_mask_interval = mask_interval\n mask_interval = [[round(cmi[0]*args.codec_sr), round(cmi[1]*args.codec_sr)] for cmi in mask_interval]\n # logging.info(f\"i: {i}, mask_interval: {mask_interval}\")\n mask_interval = torch.LongTensor(mask_interval) # [M,2]\n orig_audio, new_audio = inference_one_sample(model, model_args, phn2num, text_tokenizer, audio_tokenizer, audio_fn, target_text, mask_interval, args.device, vars(args))\n \n # save segments for comparison\n orig_audio, new_audio = orig_audio[0].cpu(), new_audio[0].cpu()\n # logging.info(f\"length of the resynthesize orig audio: {orig_audio.shape}\")\n\n save_fn_new = f\"{args.output_dir}/{os.path.basename(audio_fn)[:-4]}_new_seed{args.seed}.wav\"\n \n torchaudio.save(save_fn_new, new_audio, args.codec_audio_sr)\n\n save_fn_orig = f\"{args.output_dir}/{os.path.basename(audio_fn)[:-4]}_orig.wav\"\n if not os.path.isfile(save_fn_orig):\n orig_audio, orig_sr = torchaudio.load(audio_fn)\n if orig_sr != args.codec_audio_sr:\n orig_audio = torchaudio.transforms.Resample(orig_sr, args.codec_audio_sr)(orig_audio)\n torchaudio.save(save_fn_orig, orig_audio, args.codec_audio_sr)\n\n" }, { "path": "inference_tts.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"VoiceCraft Inference Text To Speech Demo\\n\",\n \"===\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### Select 'voicecraft' as the kernel\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# import libs\\n\",\n \"# if this throws an error, something went wrong installing dependencies or changing the kernel above!\\n\",\n \"import os\\n\",\n \"os.environ[\\\"CUDA_DEVICE_ORDER\\\"]=\\\"PCI_BUS_ID\\\" \\n\",\n \"os.environ[\\\"CUDA_VISIBLE_DEVICES\\\"]=\\\"0\\\"\\n\",\n \"os.environ[\\\"USER\\\"] = \\\"me\\\" # TODO change this to your username\\n\",\n \"\\n\",\n \"import torch\\n\",\n \"import torchaudio\\n\",\n \"import numpy as np\\n\",\n \"import random\\n\",\n \"from argparse import Namespace\\n\",\n \"\\n\",\n \"from data.tokenizer import (\\n\",\n \" AudioTokenizer,\\n\",\n \" TextTokenizer,\\n\",\n \")\\n\",\n \"from huggingface_hub import hf_hub_download\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 2,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# # install MFA models and dictionaries if you haven't done so already, already done in the dockerfile or envrionment setup\\n\",\n \"# !source ~/.bashrc && \\\\\\n\",\n \"# conda activate voicecraft && \\\\\\n\",\n \"# mfa model download dictionary english_us_arpa && \\\\\\n\",\n \"# mfa model download acoustic english_us_arpa\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Dora directory: /tmp/audiocraft_me\\n\"\n ]\n }\n ],\n \"source\": [\n \"# load model, encodec, and phn2num\\n\",\n \"# # load model, tokenizer, and other necessary files\\n\",\n \"device = \\\"cuda\\\" if torch.cuda.is_available() else \\\"cpu\\\"\\n\",\n \"voicecraft_name=\\\"830M_TTSEnhanced.pth\\\" # or giga330M.pth, 330M_TTSEnhanced.pth, giga830M.pth\\n\",\n \"\\n\",\n \"# the new way of loading the model, with huggingface, recommended\\n\",\n \"from models import voicecraft\\n\",\n \"model = voicecraft.VoiceCraft.from_pretrained(f\\\"pyp1/VoiceCraft_{voicecraft_name.replace('.pth', '')}\\\")\\n\",\n \"phn2num = model.args.phn2num\\n\",\n \"config = vars(model.args)\\n\",\n \"model.to(device)\\n\",\n \"\\n\",\n \"\\n\",\n \"# # the old way of loading the model\\n\",\n \"# from models import voicecraft\\n\",\n \"# filepath = hf_hub_download(repo_id=\\\"pyp1/VoiceCraft\\\", filename=voicecraft_name, repo_type=\\\"model\\\")\\n\",\n \"# ckpt = torch.load(filepath, map_location=\\\"cpu\\\")\\n\",\n \"# model = voicecraft.VoiceCraft(ckpt[\\\"config\\\"])\\n\",\n \"# model.load_state_dict(ckpt[\\\"model\\\"])\\n\",\n \"# config = vars(model.args)\\n\",\n \"# phn2num = ckpt[\\\"phn2num\\\"]\\n\",\n \"# model.to(device)\\n\",\n \"# model.eval()\\n\",\n \"\\n\",\n \"\\n\",\n \"encodec_fn = \\\"./pretrained_models/encodec_4cb2048_giga.th\\\"\\n\",\n \"if not os.path.exists(encodec_fn):\\n\",\n \" os.system(f\\\"wget https://huggingface.co/pyp1/VoiceCraft/resolve/main/encodec_4cb2048_giga.th\\\")\\n\",\n \" os.system(f\\\"mv encodec_4cb2048_giga.th ./pretrained_models/encodec_4cb2048_giga.th\\\")\\n\",\n \"audio_tokenizer = AudioTokenizer(signature=encodec_fn, device=device) # will also put the neural codec model on gpu\\n\",\n \"\\n\",\n \"text_tokenizer = TextTokenizer(backend=\\\"espeak\\\")\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 4,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Prepare your audio\\n\",\n \"# point to the original audio whose speech you want to clone\\n\",\n \"# write down the transcript for the file, or run whisper to get the transcript (and you can modify it if it's not accurate), save it as a .txt file\\n\",\n \"orig_audio = \\\"./demo/5895_34622_000026_000002.wav\\\"\\n\",\n \"orig_transcript = \\\"Gwynplaine had, besides, for his work and for his feats of strength, round his neck and over his shoulders, an esclavine of leather.\\\"\\n\",\n \"\\n\",\n \"# move the audio and transcript to temp folder\\n\",\n \"temp_folder = \\\"./demo/temp\\\"\\n\",\n \"os.makedirs(temp_folder, exist_ok=True)\\n\",\n \"os.system(f\\\"cp {orig_audio} {temp_folder}\\\")\\n\",\n \"filename = os.path.splitext(orig_audio.split(\\\"/\\\")[-1])[0]\\n\",\n \"with open(f\\\"{temp_folder}/{filename}.txt\\\", \\\"w\\\") as f:\\n\",\n \" f.write(orig_transcript)\\n\",\n \"# run MFA to get the alignment\\n\",\n \"align_temp = f\\\"{temp_folder}/mfa_alignments\\\"\\n\",\n \"!source ~/.bashrc && \\\\\\n\",\n \" conda activate voicecraft && \\\\\\n\",\n \" mfa align -v --clean -j 1 --output_format csv {temp_folder} \\\\\\n\",\n \" english_us_arpa english_us_arpa {align_temp}\\n\",\n \"\\n\",\n \"# # if the above fails, it could be because the audio is too hard for the alignment model, increasing the beam size usually solves the issue\\n\",\n \"# !source ~/.bashrc && \\\\\\n\",\n \"# conda activate voicecraft && \\\\\\n\",\n \"# mfa align -v --clean -j 1 --output_format csv {temp_folder} \\\\\\n\",\n \"# english_us_arpa english_us_arpa {align_temp} --beam 1000 --retry_beam 2000\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# take a look at demo/temp/mfa_alignment, decide which part of the audio to use as prompt\\n\",\n \"cut_off_sec = 3.6 # NOTE: according to forced-alignment file demo/temp/mfa_alignments/5895_34622_000026_000002.wav, the word \\\"strength\\\" stop as 3.561 sec, so we use first 3.6 sec as the prompt. this should be different for different audio\\n\",\n \"target_transcript = \\\"Gwynplaine had, besides, for his work and for his feats of strength, I cannot believe that the same model can also do text to speech synthesis too!\\\"\\n\",\n \"# NOTE: 3 sec of reference is generally enough for high quality voice cloning, but longer is generally better, try e.g. 3~6 sec.\\n\",\n \"audio_fn = f\\\"{temp_folder}/{filename}.wav\\\"\\n\",\n \"info = torchaudio.info(audio_fn)\\n\",\n \"audio_dur = info.num_frames / info.sample_rate\\n\",\n \"\\n\",\n \"assert cut_off_sec < audio_dur, f\\\"cut_off_sec {cut_off_sec} is larger than the audio duration {audio_dur}\\\"\\n\",\n \"prompt_end_frame = int(cut_off_sec * info.sample_rate)\\n\",\n \"\\n\",\n \"# run the model to get the output\\n\",\n \"# hyperparameters for inference\\n\",\n \"codec_audio_sr = 16000\\n\",\n \"codec_sr = 50\\n\",\n \"top_k = 40 # can also try 20, 30, 50\\n\",\n \"top_p = 1 # 1 means do not do top-p sampling\\n\",\n \"temperature = 1\\n\",\n \"silence_tokens=[1388,1898,131]\\n\",\n \"kvcache = 1 # NOTE if OOM, change this to 0, or try the 330M model\\n\",\n \"\\n\",\n \"# NOTE adjust the below three arguments if the generation is not as good\\n\",\n \"stop_repetition = 3 # NOTE if the model generate long silence, reduce the stop_repetition to 3, 2 or even 1\\n\",\n \"sample_batch_size = 3 # NOTE: if the if there are long silence or unnaturally strecthed words, increase sample_batch_size to 4 or higher. What this will do to the model is that the model will run sample_batch_size examples of the same audio, and pick the one that's the shortest. So if the speech rate of the generated is too fast change it to a smaller number.\\n\",\n \"seed = 1 # change seed if you are still unhappy with the result\\n\",\n \"\\n\",\n \"def seed_everything(seed):\\n\",\n \" os.environ['PYTHONHASHSEED'] = str(seed)\\n\",\n \" random.seed(seed)\\n\",\n \" np.random.seed(seed)\\n\",\n \" torch.manual_seed(seed)\\n\",\n \" torch.cuda.manual_seed(seed)\\n\",\n \" torch.backends.cudnn.benchmark = False\\n\",\n \" torch.backends.cudnn.deterministic = True\\n\",\n \"seed_everything(seed)\\n\",\n \"\\n\",\n \"decode_config = {'top_k': top_k, 'top_p': top_p, 'temperature': temperature, 'stop_repetition': stop_repetition, 'kvcache': kvcache, \\\"codec_audio_sr\\\": codec_audio_sr, \\\"codec_sr\\\": codec_sr, \\\"silence_tokens\\\": silence_tokens, \\\"sample_batch_size\\\": sample_batch_size}\\n\",\n \"from inference_tts_scale import inference_one_sample\\n\",\n \"concated_audio, gen_audio = inference_one_sample(model, Namespace(**config), phn2num, text_tokenizer, audio_tokenizer, audio_fn, target_transcript, device, decode_config, prompt_end_frame)\\n\",\n \" \\n\",\n \"# save segments for comparison\\n\",\n \"concated_audio, gen_audio = concated_audio[0].cpu(), gen_audio[0].cpu()\\n\",\n \"# logging.info(f\\\"length of the resynthesize orig audio: {orig_audio.shape}\\\")\\n\",\n \"\\n\",\n \"\\n\",\n \"# display the audio\\n\",\n \"from IPython.display import Audio\\n\",\n \"print(\\\"concatenate prompt and generated:\\\")\\n\",\n \"display(Audio(concated_audio, rate=codec_audio_sr))\\n\",\n \"\\n\",\n \"print(\\\"generated:\\\")\\n\",\n \"display(Audio(gen_audio, rate=codec_audio_sr))\\n\",\n \"\\n\",\n \"# # save the audio\\n\",\n \"# # output_dir\\n\",\n \"# output_dir = \\\"/home/pyp/VoiceCraft/demo/generated_tts\\\"\\n\",\n \"# os.makedirs(output_dir, exist_ok=True)\\n\",\n \"# seg_save_fn_gen = f\\\"{output_dir}/{os.path.basename(audio_fn)[:-4]}_gen_seed{seed}.wav\\\"\\n\",\n \"# seg_save_fn_concat = f\\\"{output_dir}/{os.path.basename(audio_fn)[:-4]}_concat_seed{seed}.wav\\\" \\n\",\n \"\\n\",\n \"# torchaudio.save(seg_save_fn_gen, gen_audio, codec_audio_sr)\\n\",\n \"# torchaudio.save(seg_save_fn_concat, concated_audio, codec_audio_sr)\\n\",\n \"\\n\",\n \"# you are might get warnings like WARNING:phonemizer:words count mismatch on 300.0% of the lines (3/1), this can be safely ignored\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": []\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"voicecraft\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.9.18\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 4\n}\n" }, { "path": "inference_tts_scale.py", "content": "import argparse, pickle\nimport logging\nimport os, random\nimport numpy as np\nimport torch\nimport torchaudio\n\nfrom data.tokenizer import (\n AudioTokenizer,\n TextTokenizer,\n tokenize_audio,\n tokenize_text\n)\n\nfrom models import voicecraft\nimport argparse, time, tqdm\n\n\n# this script only works for the musicgen architecture\ndef get_args():\n parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n parser.add_argument(\"--manifest_fn\", type=str, default=\"path/to/eval_metadata_file\")\n parser.add_argument(\"--audio_root\", type=str, default=\"path/to/audio_folder\")\n parser.add_argument(\"--exp_dir\", type=str, default=\"path/to/model_folder\")\n parser.add_argument(\"--seed\", type=int, default=1)\n parser.add_argument(\"--codec_audio_sr\", type=int, default=16000, help='the sample rate of audio that the codec is trained for')\n parser.add_argument(\"--codec_sr\", type=int, default=50, help='the sample rate of the codec codes')\n parser.add_argument(\"--top_k\", type=int, default=40, help=\"sampling param\")\n parser.add_argument(\"--top_p\", type=float, default=1, help=\"sampling param\")\n parser.add_argument(\"--temperature\", type=float, default=1.0, help=\"sampling param\")\n parser.add_argument(\"--output_dir\", type=str, default=None)\n parser.add_argument(\"--device\", type=str, default=\"cuda\")\n parser.add_argument(\"--signature\", type=str, default=None, help=\"path to the encodec model\")\n parser.add_argument(\"--crop_concat\", type=int, default=0)\n parser.add_argument(\"--stop_repetition\", type=int, default=-1, help=\"used for inference, when the number of consecutive repetition of a token is bigger than this, stop it\")\n parser.add_argument(\"--kvcache\", type=int, default=1, help='if true, use kv cache, which is 4-8x faster than without')\n parser.add_argument(\"--sample_batch_size\", type=int, default=1, help=\"batch size for sampling, NOTE that it's not running inference for several samples, but duplicate one input sample batch_size times, and during inference, we only return the shortest generation\")\n parser.add_argument(\"--silence_tokens\", type=str, default=\"[1388,1898,131]\", help=\"note that if you are not using the pretrained encodec 6f79c6a8, make sure you specified it yourself, rather than using the default\")\n return parser.parse_args()\n\n\n@torch.no_grad()\ndef inference_one_sample(model, model_args, phn2num, text_tokenizer, audio_tokenizer, audio_fn, target_text, device, decode_config, prompt_end_frame):\n # phonemize\n text_tokens = [phn2num[phn] for phn in\n tokenize_text(\n text_tokenizer, text=target_text.strip()\n ) if phn in phn2num\n ]\n text_tokens = torch.LongTensor(text_tokens).unsqueeze(0)\n text_tokens_lens = torch.LongTensor([text_tokens.shape[-1]])\n\n # encode audio\n encoded_frames = tokenize_audio(audio_tokenizer, audio_fn, offset=0, num_frames=prompt_end_frame)\n original_audio = encoded_frames[0][0].transpose(2,1) # [1,T,K]\n assert original_audio.ndim==3 and original_audio.shape[0] == 1 and original_audio.shape[2] == model_args.n_codebooks, original_audio.shape\n logging.info(f\"original audio length: {original_audio.shape[1]} codec frames, which is {original_audio.shape[1]/decode_config['codec_sr']:.2f} sec.\")\n\n # forward\n stime = time.time()\n if decode_config['sample_batch_size'] <= 1:\n logging.info(f\"running inference with batch size 1\")\n concat_frames, gen_frames = model.inference_tts(\n text_tokens.to(device),\n text_tokens_lens.to(device),\n original_audio[...,:model_args.n_codebooks].to(device), # [1,T,8]\n top_k=decode_config['top_k'],\n top_p=decode_config['top_p'],\n temperature=decode_config['temperature'],\n stop_repetition=decode_config['stop_repetition'],\n kvcache=decode_config['kvcache'],\n silence_tokens=eval(decode_config['silence_tokens']) if type(decode_config['silence_tokens'])==str else decode_config['silence_tokens']\n ) # output is [1,K,T]\n else:\n logging.info(f\"running inference with batch size {decode_config['sample_batch_size']}, i.e. return the shortest among {decode_config['sample_batch_size']} generations.\")\n concat_frames, gen_frames = model.inference_tts_batch(\n text_tokens.to(device),\n text_tokens_lens.to(device),\n original_audio[...,:model_args.n_codebooks].to(device), # [1,T,8]\n top_k=decode_config['top_k'],\n top_p=decode_config['top_p'],\n temperature=decode_config['temperature'],\n stop_repetition=decode_config['stop_repetition'],\n kvcache=decode_config['kvcache'],\n batch_size = decode_config['sample_batch_size'],\n silence_tokens=eval(decode_config['silence_tokens']) if type(decode_config['silence_tokens'])==str else decode_config['silence_tokens']\n ) # output is [1,K,T]\n logging.info(f\"inference on one sample take: {time.time() - stime:.4f} sec.\")\n\n logging.info(f\"generated encoded_frames.shape: {gen_frames.shape}, which is {gen_frames.shape[-1]/decode_config['codec_sr']} sec.\")\n \n # for timestamp, codes in enumerate(gen_frames[0].transpose(1,0)):\n # logging.info(f\"{timestamp}: {codes.tolist()}\")\n # decode (both original and generated)\n concat_sample = audio_tokenizer.decode(\n [(concat_frames, None)] # [1,T,8] -> [1,8,T]\n )\n gen_sample = audio_tokenizer.decode(\n [(gen_frames, None)]\n )\n #Empty cuda cache between runs\n if torch.cuda.is_available():\n torch.cuda.empty_cache()\n # return\n return concat_sample, gen_sample\n\ndef get_model(exp_dir, device=None):\n with open(os.path.join(exp_dir, \"args.pkl\"), \"rb\") as f:\n model_args = pickle.load(f)\n\n logging.info(\"load model weights...\")\n model = voicecraft.VoiceCraft(model_args)\n ckpt_fn = os.path.join(exp_dir, \"best_bundle.pth\")\n ckpt = torch.load(ckpt_fn, map_location='cpu')['model']\n phn2num = torch.load(ckpt_fn, map_location='cpu')['phn2num']\n model.load_state_dict(ckpt)\n del ckpt\n logging.info(\"done loading weights...\")\n if device == None:\n device = torch.device(\"cpu\")\n if torch.cuda.is_available():\n device = torch.device(\"cuda:0\")\n model.to(device)\n model.eval()\n return model, model_args, phn2num\n\nif __name__ == \"__main__\":\n def seed_everything(seed):\n os.environ['PYTHONHASHSEED'] = str(seed)\n random.seed(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.benchmark = False\n torch.backends.cudnn.deterministic = True\n formatter = (\n \"%(asctime)s [%(levelname)s] %(filename)s:%(lineno)d || %(message)s\"\n )\n logging.basicConfig(format=formatter, level=logging.INFO)\n args = get_args()\n # args.device='cpu'\n seed_everything(args.seed)\n \n os.makedirs(args.output_dir, exist_ok=True)\n # load model\n\n with open(args.manifest_fn, \"r\") as rf:\n manifest = [l.strip().split(\"\\t\") for l in rf.readlines()]\n manifest = manifest[1:]\n manifest = [[item[0], item[2], item[3], item[1], item[5]] for item in manifest]\n \n stime = time.time()\n logging.info(f\"loading model from {args.exp_dir}\")\n model, model_args, phn2num = get_model(args.exp_dir)\n logging.info(f\"loading model done, took {time.time() - stime:.4f} sec\")\n\n # setup text and audio tokenizer\n text_tokenizer = TextTokenizer(backend=\"espeak\")\n audio_tokenizer = AudioTokenizer(signature=args.signature) # will also put the neural codec model on gpu\n \n audio_fns = []\n texts = []\n prompt_end_frames = []\n new_audio_fns = []\n text_to_syn = []\n\n for item in manifest:\n audio_fn = os.path.join(args.audio_root, item[0])\n audio_fns.append(audio_fn)\n temp = torchaudio.info(audio_fn)\n prompt_end_frames.append(round(float(item[2])*temp.sample_rate))\n texts.append(item[1])\n new_audio_fns.append(item[-2])\n all_text = item[1].split(\" \")\n start_ind = int(item[-1].split(\",\")[0])\n text_to_syn.append(\" \".join(all_text[start_ind:]))\n\n for i, (audio_fn, text, prompt_end_frame, new_audio_fn, to_syn) in enumerate(tqdm.tqdm((zip(audio_fns, texts, prompt_end_frames, new_audio_fns, text_to_syn)))):\n output_expected_sr = args.codec_audio_sr\n concated_audio, gen_audio = inference_one_sample(model, model_args, phn2num, text_tokenizer, audio_tokenizer, audio_fn, text, args.device, vars(args), prompt_end_frame)\n \n # save segments for comparison\n concated_audio, gen_audio = concated_audio[0].cpu(), gen_audio[0].cpu()\n if output_expected_sr != args.codec_audio_sr:\n gen_audio = torchaudio.transforms.Resample(output_expected_sr, args.codec_audio_sr)(gen_audio)\n concated_audio = torchaudio.transforms.Resample(output_expected_sr, args.codec_audio_sr)(concated_audio)\n\n seg_save_fn_gen = f\"{args.output_dir}/gen_{new_audio_fn[:-4]}_{i}_seed{args.seed}.wav\"\n seg_save_fn_concat = f\"{args.output_dir}/concat_{new_audio_fn[:-4]}_{i}_seed{args.seed}.wav\" \n\n torchaudio.save(seg_save_fn_gen, gen_audio, args.codec_audio_sr)\n torchaudio.save(seg_save_fn_concat, concated_audio, args.codec_audio_sr)\n" }, { "path": "main.py", "content": "from pathlib import Path\nimport torch\nimport pickle\nimport argparse\nimport logging\nimport torch.distributed as dist\nfrom config import MyParser\nfrom steps import trainer\n\n\nif __name__ == \"__main__\":\n formatter = (\n \"%(asctime)s [%(levelname)s] %(filename)s:%(lineno)d || %(message)s\"\n )\n logging.basicConfig(format=formatter, level=logging.INFO)\n \n torch.cuda.empty_cache()\n args = MyParser().parse_args()\n logging.info(args)\n exp_dir = Path(args.exp_dir)\n exp_dir.mkdir(exist_ok=True, parents=True)\n logging.info(f\"exp_dir: {str(exp_dir)}\")\n\n if args.resume:\n resume = args.resume\n assert(bool(args.exp_dir))\n with open(\"%s/args.pkl\" % args.exp_dir, \"rb\") as f:\n old_args = pickle.load(f)\n new_args = vars(args)\n old_args = vars(old_args)\n for key in new_args:\n if key not in old_args or old_args[key] != new_args[key]:\n old_args[key] = new_args[key]\n args = argparse.Namespace(**old_args)\n args.resume = resume\n else:\n with open(\"%s/args.pkl\" % args.exp_dir, \"wb\") as f:\n pickle.dump(args, f)\n\n dist.init_process_group(backend='nccl', init_method='env://')\n rank = dist.get_rank()\n world_size = dist.get_world_size()\n torch.cuda.set_device(rank)\n my_trainer = trainer.Trainer(args, world_size, rank)\n my_trainer.train()" }, { "path": "models/codebooks_patterns.py", "content": "# Copyright (c) Meta Platforms, Inc. and affiliates.\n# All rights reserved.\n#\n# This source code is licensed under the license found in the\n# LICENSE file in the root directory of this source tree.\n\nfrom collections import namedtuple\nfrom dataclasses import dataclass\nfrom functools import lru_cache\nimport logging\nimport typing as tp\n\nfrom abc import ABC, abstractmethod\nimport torch\n\nLayoutCoord = namedtuple('LayoutCoord', ['t', 'q']) # (timestep, codebook index)\nPatternLayout = tp.List[tp.List[LayoutCoord]] # Sequence of coordinates\n\n\n@dataclass\nclass Pattern:\n \"\"\"Base implementation of a pattern over a sequence with multiple codebooks.\n\n The codebook pattern consists in a layout, defining for each sequence step\n the list of coordinates of each codebook timestep in the resulting interleaved sequence.\n The first item of the pattern is always an empty list in order to properly insert a special token\n to start with. For convenience, we also keep track of ``n_q`` the number of codebooks used for the pattern\n and ``timesteps`` the number of timesteps corresponding to the original sequence.\n\n The pattern provides convenient methods to build and revert interleaved sequences from it:\n ``build_pattern_sequence`` maps a given a dense input tensor of multi-codebook sequence from [B, K, T]\n to the interleaved sequence of shape [B, K, S] applying the pattern, with S being the batch size,\n K being the number of codebooks, T the number of original timesteps and S the number of sequence steps\n for the output sequence. The unfilled positions are replaced with a special token and the built sequence\n is returned along with a mask indicating valid tokens.\n ``revert_pattern_sequence`` maps back an interleaved sequence of shape [B, K, S] to the original alignment\n of codebooks across timesteps to an output tensor of shape [B, K, T], using again a special token and a mask\n to fill and specify invalid positions if needed.\n See the dedicated methods for more details.\n \"\"\"\n # Pattern layout, for each sequence step, we have a list of coordinates\n # corresponding to the original codebook timestep and position.\n # The first list is always an empty list in order to properly insert\n # a special token to start with.\n layout: PatternLayout\n timesteps: int\n n_q: int\n\n def __post_init__(self):\n assert len(self.layout) > 0\n assert self.layout[0] == []\n self._validate_layout()\n self._build_reverted_sequence_scatter_indexes = lru_cache(100)(self._build_reverted_sequence_scatter_indexes)\n self._build_pattern_sequence_scatter_indexes = lru_cache(100)(self._build_pattern_sequence_scatter_indexes)\n # logging.info(\"New pattern, time steps: %d, sequence steps: %d\", self.timesteps, len(self.layout))\n\n def _validate_layout(self):\n \"\"\"Runs checks on the layout to ensure a valid pattern is defined.\n A pattern is considered invalid if:\n - Multiple timesteps for a same codebook are defined in the same sequence step\n - The timesteps for a given codebook are not in ascending order as we advance in the sequence\n (this would mean that we have future timesteps before past timesteps).\n \"\"\"\n q_timesteps = {q: 0 for q in range(self.n_q)}\n for s, seq_coords in enumerate(self.layout):\n if len(seq_coords) > 0:\n qs = set()\n for coord in seq_coords:\n qs.add(coord.q)\n last_q_timestep = q_timesteps[coord.q]\n assert coord.t >= last_q_timestep, \\\n f\"Past timesteps are found in the sequence for codebook = {coord.q} at step {s}\"\n q_timesteps[coord.q] = coord.t\n # each sequence step contains at max 1 coordinate per codebook\n assert len(qs) == len(seq_coords), \\\n f\"Multiple entries for a same codebook are found at step {s}\"\n\n @property\n def num_sequence_steps(self):\n return len(self.layout) - 1\n\n @property\n def max_delay(self):\n max_t_in_seq_coords = 0\n for seq_coords in self.layout[1:]:\n for coords in seq_coords:\n max_t_in_seq_coords = max(max_t_in_seq_coords, coords.t + 1)\n return max_t_in_seq_coords - self.timesteps\n\n @property\n def valid_layout(self):\n valid_step = len(self.layout) - self.max_delay\n return self.layout[:valid_step]\n\n def get_sequence_coords_with_timestep(self, t: int, q: tp.Optional[int] = None):\n \"\"\"Get codebook coordinates in the layout that corresponds to the specified timestep t\n and optionally to the codebook q. Coordinates are returned as a tuple with the sequence step\n and the actual codebook coordinates.\n \"\"\"\n assert t <= self.timesteps, \"provided timesteps is greater than the pattern's number of timesteps\"\n if q is not None:\n assert q <= self.n_q, \"provided number of codebooks is greater than the pattern's number of codebooks\"\n coords = []\n for s, seq_codes in enumerate(self.layout):\n for code in seq_codes:\n if code.t == t and (q is None or code.q == q):\n coords.append((s, code))\n return coords\n\n def get_steps_with_timestep(self, t: int, q: tp.Optional[int] = None) -> tp.List[int]:\n return [step for step, coords in self.get_sequence_coords_with_timestep(t, q)]\n\n def get_first_step_with_timesteps(self, t: int, q: tp.Optional[int] = None) -> tp.Optional[int]:\n steps_with_timesteps = self.get_steps_with_timestep(t, q)\n return steps_with_timesteps[0] if len(steps_with_timesteps) > 0 else None\n\n def _build_pattern_sequence_scatter_indexes(self, timesteps: int, n_q: int, keep_only_valid_steps: bool,\n device: tp.Union[torch.device, str] = 'cpu'):\n \"\"\"Build scatter indexes corresponding to the pattern, up to the provided sequence_steps.\n\n Args:\n timesteps (int): Maximum number of timesteps steps to consider.\n keep_only_valid_steps (bool): Restrict the pattern layout to match only valid steps.\n device (Union[torch.device, str]): Device for created tensors.\n Returns:\n indexes (torch.Tensor): Indexes corresponding to the sequence, of shape [K, S].\n mask (torch.Tensor): Mask corresponding to indexes that matches valid indexes, of shape [K, S].\n \"\"\"\n assert n_q == self.n_q, f\"invalid number of codebooks for the sequence and the pattern: {n_q} != {self.n_q}\"\n assert timesteps <= self.timesteps, \"invalid number of timesteps used to build the sequence from the pattern\"\n # use the proper layout based on whether we limit ourselves to valid steps only or not,\n # note that using the valid_layout will result in a truncated sequence up to the valid steps\n ref_layout = self.valid_layout if keep_only_valid_steps else self.layout\n # single item indexing being super slow with pytorch vs. numpy, so we use numpy here\n indexes = torch.zeros(n_q, len(ref_layout), dtype=torch.long).numpy()\n mask = torch.zeros(n_q, len(ref_layout), dtype=torch.bool).numpy()\n # fill indexes with last sequence step value that will correspond to our special token\n # the last value is n_q * timesteps as we have flattened z and append special token as the last token\n # which will correspond to the index: n_q * timesteps\n indexes[:] = n_q * timesteps\n # iterate over the pattern and fill scattered indexes and mask\n for s, sequence_coords in enumerate(ref_layout):\n for coords in sequence_coords:\n if coords.t < timesteps:\n indexes[coords.q, s] = coords.t + coords.q * timesteps\n mask[coords.q, s] = 1\n indexes = torch.from_numpy(indexes).to(device)\n mask = torch.from_numpy(mask).to(device)\n return indexes, mask\n\n def build_pattern_sequence(self, z: torch.Tensor, special_token: int, keep_only_valid_steps: bool = False):\n \"\"\"Build sequence corresponding to the pattern from the input tensor z.\n The sequence is built using up to sequence_steps if specified, and non-pattern\n coordinates are filled with the special token.\n\n Args:\n z (torch.Tensor): Input tensor of multi-codebooks sequence, of shape [B, K, T].\n special_token (int): Special token used to fill non-pattern coordinates in the new sequence.\n keep_only_valid_steps (bool): Build a sequence from the pattern up to valid (= fully defined) steps.\n Steps that are beyond valid steps will be replaced by the special_token in that case.\n Returns:\n values (torch.Tensor): Interleaved sequence matching the pattern, of shape [B, K, S] with S\n corresponding either to the sequence_steps if provided, otherwise to the length of the pattern.\n indexes (torch.Tensor): Indexes corresponding to the interleaved sequence, of shape [K, S].\n mask (torch.Tensor): Mask corresponding to indexes that matches valid indexes of shape [K, S].\n \"\"\"\n B, K, T = z.shape\n indexes, mask = self._build_pattern_sequence_scatter_indexes(\n T, K, keep_only_valid_steps=keep_only_valid_steps, device=str(z.device)\n )\n z = z.view(B, -1)\n # we append the special token as the last index of our flattened z tensor\n z = torch.cat([z, torch.zeros_like(z[:, :1]) + special_token], dim=1)\n values = z[:, indexes.view(-1)]\n values = values.view(B, K, indexes.shape[-1])\n return values, indexes, mask\n\n def _build_reverted_sequence_scatter_indexes(self, sequence_steps: int, n_q: int,\n keep_only_valid_steps: bool = False,\n is_model_output: bool = False,\n device: tp.Union[torch.device, str] = 'cpu'):\n \"\"\"Builds scatter indexes required to retrieve the original multi-codebook sequence\n from interleaving pattern.\n\n Args:\n sequence_steps (int): Sequence steps.\n n_q (int): Number of codebooks.\n keep_only_valid_steps (bool): Build a sequence from the pattern up to valid (= fully defined) steps.\n Steps that are beyond valid steps will be replaced by the special_token in that case.\n is_model_output (bool): Whether to keep the sequence item corresponding to initial special token or not.\n device (Union[torch.device, str]): Device for created tensors.\n Returns:\n torch.Tensor: Indexes for reconstructing the output, of shape [K, T].\n mask (torch.Tensor): Mask corresponding to indexes that matches valid indexes of shape [K, T].\n \"\"\"\n ref_layout = self.valid_layout if keep_only_valid_steps else self.layout\n # TODO(jade): Do we want to further truncate to only valid timesteps here as well?\n timesteps = self.timesteps\n assert n_q == self.n_q, f\"invalid number of codebooks for the sequence and the pattern: {n_q} != {self.n_q}\"\n assert sequence_steps <= len(ref_layout), \\\n f\"sequence to revert is longer than the defined pattern: {sequence_steps} > {len(ref_layout)}\"\n\n # ensure we take the appropriate indexes to keep the model output from the first special token as well\n if is_model_output:\n ref_layout = ref_layout[1:]\n\n # single item indexing being super slow with pytorch vs. numpy, so we use numpy here\n indexes = torch.zeros(n_q, timesteps, dtype=torch.long).numpy()\n mask = torch.zeros(n_q, timesteps, dtype=torch.bool).numpy()\n # fill indexes with last sequence step value that will correspond to our special token\n indexes[:] = n_q * sequence_steps\n for s, sequence_codes in enumerate(ref_layout):\n if s < sequence_steps:\n for code in sequence_codes:\n if code.t < timesteps:\n indexes[code.q, code.t] = s + code.q * sequence_steps\n mask[code.q, code.t] = 1\n indexes = torch.from_numpy(indexes).to(device)\n mask = torch.from_numpy(mask).to(device)\n return indexes, mask\n\n def revert_pattern_sequence(self, s: torch.Tensor, special_token: int, keep_only_valid_steps: bool = False):\n \"\"\"Revert a sequence built from the pattern back to the original multi-codebook sequence without interleaving.\n The sequence is reverted using up to timesteps if specified, and non-pattern coordinates\n are filled with the special token.\n\n Args:\n s (torch.Tensor): Interleaved sequence tensor obtained from the pattern, of shape [B, K, S].\n special_token (int or float): Special token used to fill non-pattern coordinates in the new sequence.\n Returns:\n values (torch.Tensor): Interleaved sequence matching the pattern, of shape [B, K, T] with T\n corresponding either to the timesteps if provided, or the total timesteps in pattern otherwise.\n indexes (torch.Tensor): Indexes corresponding to the interleaved sequence, of shape [K, T].\n mask (torch.Tensor): Mask corresponding to indexes that matches valid indexes of shape [K, T].\n \"\"\"\n B, K, S = s.shape\n indexes, mask = self._build_reverted_sequence_scatter_indexes(\n S, K, keep_only_valid_steps, is_model_output=False, device=str(s.device)\n )\n s = s.view(B, -1)\n # we append the special token as the last index of our flattened z tensor\n s = torch.cat([s, torch.zeros_like(s[:, :1]) + special_token], dim=1)\n values = s[:, indexes.view(-1)]\n values = values.view(B, K, indexes.shape[-1])\n return values, indexes, mask\n\n def revert_pattern_logits(self, logits: torch.Tensor, special_token: float, keep_only_valid_steps: bool = False):\n \"\"\"Revert model logits obtained on a sequence built from the pattern\n back to a tensor matching the original sequence.\n\n This method is similar to ``revert_pattern_sequence`` with the following specificities:\n 1. It is designed to work with the extra cardinality dimension\n 2. We return the logits for the first sequence item that matches the special_token and\n which matching target in the original sequence is the first item of the sequence,\n while we skip the last logits as there is no matching target\n \"\"\"\n B, card, K, S = logits.shape\n indexes, mask = self._build_reverted_sequence_scatter_indexes(\n S, K, keep_only_valid_steps, is_model_output=True, device=logits.device\n )\n logits = logits.reshape(B, card, -1)\n # we append the special token as the last index of our flattened z tensor\n logits = torch.cat([logits, torch.zeros_like(logits[:, :, :1]) + special_token], dim=-1) # [B, card, K x S]\n values = logits[:, :, indexes.view(-1)]\n values = values.view(B, card, K, indexes.shape[-1])\n return values, indexes, mask\n\n\nclass CodebooksPatternProvider(ABC):\n \"\"\"Abstraction around providing pattern for interleaving codebooks.\n\n The CodebooksPatternProvider abstraction allows to implement various strategies to\n define interleaving pattern of sequences composed of multiple codebooks. For a given\n number of codebooks `n_q`, the pattern provider can generate a specified pattern\n corresponding to a sequence of `T` timesteps with `n_q` parallel codebooks. This pattern\n can be used to construct a new sequence from the original codes respecting the specified\n pattern. The pattern is defined as a list of list of code coordinates, code coordinate\n being a tuple with the original timestep and codebook to build the new sequence.\n Note that all patterns must start with an empty list that is then used to insert a first\n sequence step of special tokens in the newly generated sequence.\n\n Args:\n n_q (int): number of codebooks.\n cached (bool): if True, patterns for a given length are cached. In general\n that should be true for efficiency reason to avoid synchronization points.\n \"\"\"\n def __init__(self, n_q: int, cached: bool = True):\n assert n_q > 0\n self.n_q = n_q\n self.get_pattern = lru_cache(100)(self.get_pattern) # type: ignore\n\n @abstractmethod\n def get_pattern(self, timesteps: int) -> Pattern:\n \"\"\"Builds pattern with specific interleaving between codebooks.\n\n Args:\n timesteps (int): Total numer of timesteps.\n \"\"\"\n raise NotImplementedError()\n\n\nclass DelayedPatternProvider(CodebooksPatternProvider):\n \"\"\"Provider for delayed pattern across delayed codebooks.\n Codebooks are delayed in the sequence and sequence steps will contain codebooks\n from different timesteps.\n\n Example:\n Taking timesteps=4 and n_q=3, delays=None, the multi-codebook sequence:\n [[1, 2, 3, 4],\n [1, 2, 3, 4],\n [1, 2, 3, 4]]\n The resulting sequence obtained from the returned pattern is:\n [[S, 1, 2, 3, 4],\n [S, S, 1, 2, 3],\n [S, S, S, 1, 2]]\n (with S being a special token)\n\n Args:\n n_q (int): Number of codebooks.\n delays (Optional[List[int]]): Delay for each of the codebooks.\n If delays not defined, each codebook is delayed by 1 compared to the previous one.\n flatten_first (int): Flatten the first N timesteps.\n empty_initial (int): Prepend with N empty list of coordinates.\n \"\"\"\n def __init__(self, n_q: int, delays: tp.Optional[tp.List[int]] = None,\n flatten_first: int = 0, empty_initial: int = 0):\n super().__init__(n_q)\n if delays is None:\n delays = list(range(n_q))\n self.delays = delays\n self.flatten_first = flatten_first\n self.empty_initial = empty_initial\n assert len(self.delays) == self.n_q\n assert sorted(self.delays) == self.delays\n\n def get_pattern(self, timesteps: int) -> Pattern:\n out: PatternLayout = [[]]\n max_delay = max(self.delays)\n if self.empty_initial:\n out += [[] for _ in range(self.empty_initial)]\n if self.flatten_first:\n for t in range(min(timesteps, self.flatten_first)):\n for q in range(self.n_q):\n out.append([LayoutCoord(t, q)])\n for t in range(self.flatten_first, timesteps + max_delay):\n v = []\n for q, delay in enumerate(self.delays):\n t_for_q = t - delay\n if t_for_q >= self.flatten_first:\n v.append(LayoutCoord(t_for_q, q))\n out.append(v)\n return Pattern(out, n_q=self.n_q, timesteps=timesteps)\n\n\nclass ParallelPatternProvider(DelayedPatternProvider):\n \"\"\"Provider for parallel pattern across codebooks.\n This pattern provider is a special case of the delayed pattern with actually no delay,\n hence delays=repeat(0, n_q).\n\n Args:\n n_q (int): Number of codebooks.\n \"\"\"\n def __init__(self, n_q: int):\n super().__init__(n_q, [0] * n_q)\n\n\nclass UnrolledPatternProvider(CodebooksPatternProvider):\n \"\"\"Provider for unrolling codebooks pattern.\n This pattern provider enables to represent the codebook flattened completely or only to some extend\n while also specifying a given delay between the flattened codebooks representation, allowing to\n unroll the codebooks in the sequence.\n\n Example:\n 1. Flattening of the codebooks.\n By default, the pattern provider will fully flatten the codebooks such as flattening=range(n_q),\n taking n_q = 3 and timesteps = 4:\n [[1, 2, 3, 4],\n [1, 2, 3, 4],\n [1, 2, 3, 4]]\n will result into:\n [[S, S, 1, S, S, 2, S, S, 3, S, S, 4],\n [S, 1, S, S, 2, S, S, 3, S, S, 4, S],\n [1, S, S, 2, S, S, 3, S, S, 4, S, S]]\n 2. Partial flattening of the codebooks. The ``flattening`` parameter allows to specify the inner step\n for each of the codebook, allowing to define which codebook to flatten (or keep in parallel), for example\n taking n_q = 3, timesteps = 4 and flattening = [0, 1, 1]:\n [[1, 2, 3, 4],\n [1, 2, 3, 4],\n [1, 2, 3, 4]]\n will result into:\n [[S, 1, S, S, 2, S, S, 3, S, S, 4, S],\n [S, 1, S, S, 2, S, S, 3, S, S, 4, S],\n [1, S, S, 2, S, S, 3, S, S, 4, S, S]]\n 3. Flattening with delay. The ``delay`` parameter allows to further unroll the sequence of codebooks\n allowing to specify the delay per codebook. Note that the delay between codebooks flattened to the\n same inner timestep should be coherent. For example, taking n_q = 3, timesteps = 4, flattening = [0, 1, 1]\n and delays = [0, 3, 3]:\n [[1, 2, 3, 4],\n [1, 2, 3, 4],\n [1, 2, 3, 4]]\n will result into:\n [[S, S, S, 1, S, 2, S, 3, S, 4],\n [S, S, S, 1, S, 2, S, 3, S, 4],\n [1, 2, 3, S, 4, S, 5, S, 6, S]]\n\n Args:\n n_q (int): Number of codebooks.\n flattening (Optional[List[int]]): Flattening schema over the codebooks. If not defined,\n the codebooks will be flattened to 1 codebook per step, meaning that the sequence will\n have n_q extra steps for each timestep.\n delays (Optional[List[int]]): Delay for each of the codebooks. If not defined,\n no delay is added and therefore will default to [0] * ``n_q``.\n Note that two codebooks that will be flattened to the same inner step\n should have the same delay, otherwise the pattern is considered as invalid.\n \"\"\"\n FlattenedCodebook = namedtuple('FlattenedCodebook', ['codebooks', 'delay'])\n\n def __init__(self, n_q: int, flattening: tp.Optional[tp.List[int]] = None,\n delays: tp.Optional[tp.List[int]] = None):\n super().__init__(n_q)\n if flattening is None:\n flattening = list(range(n_q))\n if delays is None:\n delays = [0] * n_q\n assert len(flattening) == n_q\n assert len(delays) == n_q\n assert sorted(flattening) == flattening\n assert sorted(delays) == delays\n self._flattened_codebooks = self._build_flattened_codebooks(delays, flattening)\n self.max_delay = max(delays)\n\n def _build_flattened_codebooks(self, delays: tp.List[int], flattening: tp.List[int]):\n \"\"\"Build a flattened codebooks representation as a dictionary of inner step\n and the actual codebook indices corresponding to the flattened codebook. For convenience, we\n also store the delay associated to the flattened codebook to avoid maintaining an extra mapping.\n \"\"\"\n flattened_codebooks: dict = {}\n for q, (inner_step, delay) in enumerate(zip(flattening, delays)):\n if inner_step not in flattened_codebooks:\n flat_codebook = UnrolledPatternProvider.FlattenedCodebook(codebooks=[q], delay=delay)\n else:\n flat_codebook = flattened_codebooks[inner_step]\n assert flat_codebook.delay == delay, (\n \"Delay and flattening between codebooks is inconsistent: \",\n \"two codebooks flattened to the same position should have the same delay.\"\n )\n flat_codebook.codebooks.append(q)\n flattened_codebooks[inner_step] = flat_codebook\n return flattened_codebooks\n\n @property\n def _num_inner_steps(self):\n \"\"\"Number of inner steps to unroll between timesteps in order to flatten the codebooks.\n \"\"\"\n return max([inner_step for inner_step in self._flattened_codebooks.keys()]) + 1\n\n def num_virtual_steps(self, timesteps: int) -> int:\n return timesteps * self._num_inner_steps + 1\n\n def get_pattern(self, timesteps: int) -> Pattern:\n \"\"\"Builds pattern for delay across codebooks.\n\n Args:\n timesteps (int): Total numer of timesteps.\n \"\"\"\n # the PatternLayout is built as a tuple of sequence position and list of coordinates\n # so that it can be reordered properly given the required delay between codebooks of given timesteps\n indexed_out: list = [(-1, [])]\n max_timesteps = timesteps + self.max_delay\n for t in range(max_timesteps):\n # for each timestep, we unroll the flattened codebooks,\n # emitting the sequence step with the corresponding delay\n for step in range(self._num_inner_steps):\n if step in self._flattened_codebooks:\n # we have codebooks at this virtual step to emit\n step_codebooks = self._flattened_codebooks[step]\n t_for_q = t + step_codebooks.delay\n coords = [LayoutCoord(t, q) for q in step_codebooks.codebooks]\n if t_for_q < max_timesteps and t < max_timesteps:\n indexed_out.append((t_for_q, coords))\n else:\n # there is no codebook in this virtual step so we emit an empty list\n indexed_out.append((t, []))\n out = [coords for _, coords in sorted(indexed_out)]\n return Pattern(out, n_q=self.n_q, timesteps=timesteps)\n\n\nclass VALLEPattern(CodebooksPatternProvider):\n \"\"\"Almost VALL-E style pattern. We futher allow some delays for the\n codebooks other than the first one.\n\n Args:\n n_q (int): Number of codebooks.\n delays (Optional[List[int]]): Delay for each of the codebooks.\n If delays not defined, each codebook is delayed by 1 compared to the previous one.\n \"\"\"\n def __init__(self, n_q: int, delays: tp.Optional[tp.List[int]] = None):\n super().__init__(n_q)\n if delays is None:\n delays = [0] * (n_q - 1)\n self.delays = delays\n assert len(self.delays) == self.n_q - 1\n assert sorted(self.delays) == self.delays\n\n def get_pattern(self, timesteps: int) -> Pattern:\n out: PatternLayout = [[]]\n for t in range(timesteps):\n out.append([LayoutCoord(t, 0)])\n max_delay = max(self.delays)\n for t in range(timesteps + max_delay):\n v = []\n for q, delay in enumerate(self.delays):\n t_for_q = t - delay\n if t_for_q >= 0:\n v.append(LayoutCoord(t_for_q, q + 1))\n out.append(v)\n return Pattern(out, n_q=self.n_q, timesteps=timesteps)\n\n\nclass MusicLMPattern(CodebooksPatternProvider):\n \"\"\"Almost MusicLM style pattern. This is equivalent to full flattening\n but in a different order.\n\n Args:\n n_q (int): Number of codebooks.\n group_by (int): Number of codebooks to group together.\n \"\"\"\n def __init__(self, n_q: int, group_by: int = 2):\n super().__init__(n_q)\n self.group_by = group_by\n\n def get_pattern(self, timesteps: int) -> Pattern:\n out: PatternLayout = [[]]\n for offset in range(0, self.n_q, self.group_by):\n for t in range(timesteps):\n for q in range(offset, offset + self.group_by):\n out.append([LayoutCoord(t, q)])\n return Pattern(out, n_q=self.n_q, timesteps=timesteps)" }, { "path": "models/modules/__init__.py", "content": "" }, { "path": "models/modules/activation.py", "content": "# cp from https://github.com/lifeiteng/vall-e/blob/main/valle/modules/activation.py, modified by Puyuan Peng, 2024\nfrom typing import Optional, Tuple\n\nimport torch\nfrom torch import Tensor\nfrom torch.nn import Linear, Module\nfrom torch.nn import functional as F\nfrom torch.nn.init import constant_, xavier_normal_, xavier_uniform_\nfrom torch.nn.modules.linear import NonDynamicallyQuantizableLinear\nfrom torch.nn.parameter import Parameter\nimport logging\nfrom typing import Callable, List, Optional, Tuple, Union\nfrom typing import TYPE_CHECKING\nif TYPE_CHECKING:\n from torch.types import _dtype as DType\nelse:\n # The JIT doesn't understand Union, nor torch.dtype here\n DType = int\n\ndef _canonical_mask(\n mask: Optional[Tensor],\n mask_name: str,\n other_type: Optional[DType],\n other_name: str,\n target_type: DType,\n check_other: bool = True,\n) -> Optional[Tensor]:\n\n if mask is not None:\n _mask_dtype = mask.dtype\n _mask_is_float = torch.is_floating_point(mask)\n if _mask_dtype != torch.bool and not _mask_is_float:\n raise AssertionError(\n f\"only bool and floating types of {mask_name} are supported\")\n if check_other and other_type is not None:\n if _mask_dtype != other_type:\n warnings.warn(\n f\"Support for mismatched {mask_name} and {other_name} \"\n \"is deprecated. Use same type for both instead.\"\n )\n if not _mask_is_float:\n mask = (\n torch.zeros_like(mask, dtype=target_type)\n .masked_fill_(mask, float(\"-inf\"))\n )\n return mask\n\ndef _in_projection_packed(\n q: Tensor,\n k: Tensor,\n v: Tensor,\n w: Tensor,\n b: Optional[Tensor] = None,\n) -> List[Tensor]:\n r\"\"\"\n Performs the in-projection step of the attention operation, using packed weights.\n Output is a triple containing projection tensors for query, key and value.\n\n Args:\n q, k, v: query, key and value tensors to be projected. For self-attention,\n these are typically the same tensor; for encoder-decoder attention,\n k and v are typically the same tensor. (We take advantage of these\n identities for performance if they are present.) Regardless, q, k and v\n must share a common embedding dimension; otherwise their shapes may vary.\n w: projection weights for q, k and v, packed into a single tensor. Weights\n are packed along dimension 0, in q, k, v order.\n b: optional projection biases for q, k and v, packed into a single tensor\n in q, k, v order.\n\n Shape:\n Inputs:\n - q: :math:`(..., E)` where E is the embedding dimension\n - k: :math:`(..., E)` where E is the embedding dimension\n - v: :math:`(..., E)` where E is the embedding dimension\n - w: :math:`(E * 3, E)` where E is the embedding dimension\n - b: :math:`E * 3` where E is the embedding dimension\n\n Output:\n - in output list :math:`[q', k', v']`, each output tensor will have the\n same shape as the corresponding input tensor.\n \"\"\"\n E = q.size(-1)\n if k is v:\n if q is k:\n # self-attention\n proj = F.linear(q, w, b)\n # reshape to 3, E and not E, 3 is deliberate for better memory coalescing and keeping same order as chunk()\n proj = proj.unflatten(-1, (3, E)).unsqueeze(0).transpose(0, -2).squeeze(-2).contiguous()\n return proj[0], proj[1], proj[2]\n else:\n # encoder-decoder attention\n w_q, w_kv = w.split([E, E * 2])\n if b is None:\n b_q = b_kv = None\n else:\n b_q, b_kv = b.split([E, E * 2])\n q_proj = F.linear(q, w_q, b_q)\n kv_proj = F.linear(k, w_kv, b_kv)\n # reshape to 2, E and not E, 2 is deliberate for better memory coalescing and keeping same order as chunk()\n kv_proj = kv_proj.unflatten(-1, (2, E)).unsqueeze(0).transpose(0, -2).squeeze(-2).contiguous()\n return (q_proj, kv_proj[0], kv_proj[1])\n else:\n w_q, w_k, w_v = w.chunk(3)\n if b is None:\n b_q = b_k = b_v = None\n else:\n b_q, b_k, b_v = b.chunk(3)\n return F.linear(q, w_q, b_q), F.linear(k, w_k, b_k), F.linear(v, w_v, b_v)\n\ndef _none_or_dtype(input: Optional[Tensor]) -> Optional[DType]:\n if input is None:\n return None\n elif isinstance(input, torch.Tensor):\n return input.dtype\n raise RuntimeError(\"input to _none_or_dtype() must be None or torch.Tensor\")\nclass MultiheadAttention(Module):\n r\"\"\"Allows the model to jointly attend to information\n from different representation subspaces as described in the paper:\n `Attention Is All You Need `_.\n\n Multi-Head Attention is defined as:\n\n .. math::\n \\text{MultiHead}(Q, K, V) = \\text{Concat}(head_1,\\dots,head_h)W^O\n\n where :math:`head_i = \\text{Attention}(QW_i^Q, KW_i^K, VW_i^V)`.\n\n ``forward()`` will use a special optimized implementation if all of the following\n conditions are met:\n\n - self attention is being computed (i.e., ``query``, ``key``, and ``value`` are the same tensor. This\n restriction will be loosened in the future.)\n - Either autograd is disabled (using ``torch.inference_mode`` or ``torch.no_grad``) or no tensor argument ``requires_grad``\n - training is disabled (using ``.eval()``)\n - dropout is 0\n - ``add_bias_kv`` is ``False``\n - ``add_zero_attn`` is ``False``\n - ``batch_first`` is ``True`` and the input is batched\n - ``kdim`` and ``vdim`` are equal to ``embed_dim``\n - at most one of ``key_padding_mask`` or ``attn_mask`` is passed\n - if a `NestedTensor `_ is passed, neither ``key_padding_mask``\n nor ``attn_mask`` is passed\n\n If the optimized implementation is in use, a\n `NestedTensor `_ can be passed for\n ``query``/``key``/``value`` to represent padding more efficiently than using a\n padding mask. In this case, a `NestedTensor `_\n will be returned, and an additional speedup proportional to the fraction of the input\n that is padding can be expected.\n\n Args:\n embed_dim: Total dimension of the model.\n num_heads: Number of parallel attention heads. Note that ``embed_dim`` will be split\n across ``num_heads`` (i.e. each head will have dimension ``embed_dim // num_heads``).\n dropout: Dropout probability on ``attn_output_weights``. Default: ``0.0`` (no dropout).\n bias: If specified, adds bias to input / output projection layers. Default: ``True``.\n add_bias_kv: If specified, adds bias to the key and value sequences at dim=0. Default: ``False``.\n add_zero_attn: If specified, adds a new batch of zeros to the key and value sequences at dim=1.\n Default: ``False``.\n kdim: Total number of features for keys. Default: ``None`` (uses ``kdim=embed_dim``).\n vdim: Total number of features for values. Default: ``None`` (uses ``vdim=embed_dim``).\n batch_first: If ``True``, then the input and output tensors are provided\n as (batch, seq, feature). Default: ``False`` (seq, batch, feature).\n\n Examples::\n\n >>> # xdoctest: +SKIP\n >>> multihead_attn = nn.MultiheadAttention(embed_dim, num_heads)\n >>> attn_output, attn_output_weights = multihead_attn(query, key, value)\n\n \"\"\"\n __constants__ = [\"batch_first\"]\n bias_k: Optional[torch.Tensor]\n bias_v: Optional[torch.Tensor]\n\n def __init__(\n self,\n embed_dim,\n num_heads,\n dropout=0.0,\n bias=True,\n add_bias_kv=False,\n add_zero_attn=False,\n kdim=None,\n vdim=None,\n batch_first=False,\n linear1_cls=Linear,\n linear2_cls=Linear,\n device=None,\n dtype=None,\n ) -> None:\n factory_kwargs = {\"device\": device, \"dtype\": dtype}\n super(MultiheadAttention, self).__init__()\n self.embed_dim = embed_dim\n self.kdim = kdim if kdim is not None else embed_dim\n self.vdim = vdim if vdim is not None else embed_dim\n self._qkv_same_embed_dim = (\n self.kdim == embed_dim and self.vdim == embed_dim\n )\n\n self.num_heads = num_heads\n self.dropout = dropout\n self.batch_first = batch_first\n self.head_dim = embed_dim // num_heads\n assert (\n self.head_dim * num_heads == self.embed_dim\n ), \"embed_dim must be divisible by num_heads\"\n\n if add_bias_kv:\n self.bias_k = Parameter(\n torch.empty((1, 1, embed_dim), **factory_kwargs)\n )\n self.bias_v = Parameter(\n torch.empty((1, 1, embed_dim), **factory_kwargs)\n )\n else:\n self.bias_k = self.bias_v = None\n\n if linear1_cls == Linear:\n if not self._qkv_same_embed_dim:\n self.q_proj_weight = Parameter(\n torch.empty((embed_dim, embed_dim), **factory_kwargs)\n )\n self.k_proj_weight = Parameter(\n torch.empty((embed_dim, self.kdim), **factory_kwargs)\n )\n self.v_proj_weight = Parameter(\n torch.empty((embed_dim, self.vdim), **factory_kwargs)\n )\n self.register_parameter(\"in_proj_weight\", None)\n else:\n # go down this route with voicecraft\n self.in_proj_weight = Parameter(\n torch.empty((3 * embed_dim, embed_dim), **factory_kwargs)\n )\n self.register_parameter(\"q_proj_weight\", None)\n self.register_parameter(\"k_proj_weight\", None)\n self.register_parameter(\"v_proj_weight\", None)\n\n if bias: # True by default\n self.in_proj_bias = Parameter(\n torch.empty(3 * embed_dim, **factory_kwargs)\n )\n else:\n self.register_parameter(\"in_proj_bias\", None)\n self.out_proj = NonDynamicallyQuantizableLinear(\n embed_dim, embed_dim, bias=bias, **factory_kwargs\n )\n\n self._reset_parameters()\n else:\n if not self._qkv_same_embed_dim:\n raise NotImplementedError\n else:\n self.in_proj_linear = linear1_cls(\n embed_dim, 3 * embed_dim, bias=bias, **factory_kwargs\n )\n self.in_proj_weight = self.in_proj_linear.weight\n\n self.register_parameter(\"q_proj_weight\", None)\n self.register_parameter(\"k_proj_weight\", None)\n self.register_parameter(\"v_proj_weight\", None)\n\n if bias:\n self.in_proj_bias = self.in_proj_linear.bias\n else:\n self.register_parameter(\"in_proj_bias\", None)\n\n self.out_proj = linear2_cls(\n embed_dim, embed_dim, bias=bias, **factory_kwargs\n )\n\n if self.bias_k is not None:\n xavier_normal_(self.bias_k)\n if self.bias_v is not None:\n xavier_normal_(self.bias_v)\n\n self.add_zero_attn = add_zero_attn\n\n def _reset_parameters(self):\n if self._qkv_same_embed_dim:\n xavier_uniform_(self.in_proj_weight)\n else:\n xavier_uniform_(self.q_proj_weight)\n xavier_uniform_(self.k_proj_weight)\n xavier_uniform_(self.v_proj_weight)\n\n if self.in_proj_bias is not None:\n constant_(self.in_proj_bias, 0.0)\n constant_(self.out_proj.bias, 0.0)\n\n if self.bias_k is not None:\n xavier_normal_(self.bias_k)\n if self.bias_v is not None:\n xavier_normal_(self.bias_v)\n\n def __setstate__(self, state):\n # Support loading old MultiheadAttention checkpoints generated by v1.1.0\n if \"_qkv_same_embed_dim\" not in state:\n state[\"_qkv_same_embed_dim\"] = True\n\n super(MultiheadAttention, self).__setstate__(state)\n\n def forward(\n self,\n query: Tensor,\n key: Tensor,\n value: Tensor,\n key_padding_mask: Optional[Tensor] = None,\n need_weights: bool = True,\n attn_mask: Optional[Tensor] = None,\n average_attn_weights: bool = True,\n past: Optional[Tensor] = None,\n ) -> Tuple[Tensor, Optional[Tensor]]:\n r\"\"\"\n Args:\n query: Query embeddings of shape :math:`(L, E_q)` for unbatched input, :math:`(L, N, E_q)` when ``batch_first=False``\n or :math:`(N, L, E_q)` when ``batch_first=True``, where :math:`L` is the target sequence length,\n :math:`N` is the batch size, and :math:`E_q` is the query embedding dimension ``embed_dim``.\n Queries are compared against key-value pairs to produce the output.\n See \"Attention Is All You Need\" for more details.\n key: Key embeddings of shape :math:`(S, E_k)` for unbatched input, :math:`(S, N, E_k)` when ``batch_first=False``\n or :math:`(N, S, E_k)` when ``batch_first=True``, where :math:`S` is the source sequence length,\n :math:`N` is the batch size, and :math:`E_k` is the key embedding dimension ``kdim``.\n See \"Attention Is All You Need\" for more details.\n value: Value embeddings of shape :math:`(S, E_v)` for unbatched input, :math:`(S, N, E_v)` when\n ``batch_first=False`` or :math:`(N, S, E_v)` when ``batch_first=True``, where :math:`S` is the source\n sequence length, :math:`N` is the batch size, and :math:`E_v` is the value embedding dimension ``vdim``.\n See \"Attention Is All You Need\" for more details.\n key_padding_mask: If specified, a mask of shape :math:`(N, S)` indicating which elements within ``key``\n to ignore for the purpose of attention (i.e. treat as \"padding\"). For unbatched `query`, shape should be :math:`(S)`.\n Binary and byte masks are supported.\n For a binary mask, a ``True`` value indicates that the corresponding ``key`` value will be ignored for\n the purpose of attention. For a float mask, it will be directly added to the corresponding ``key`` value.\n need_weights: If specified, returns ``attn_output_weights`` in addition to ``attn_outputs``.\n Default: ``True``.\n attn_mask: If specified, a 2D or 3D mask preventing attention to certain positions. Must be of shape\n :math:`(L, S)` or :math:`(N\\cdot\\text{num\\_heads}, L, S)`, where :math:`N` is the batch size,\n :math:`L` is the target sequence length, and :math:`S` is the source sequence length. A 2D mask will be\n broadcasted across the batch while a 3D mask allows for a different mask for each entry in the batch.\n Binary, byte, and float masks are supported. For a binary mask, a ``True`` value indicates that the\n corresponding position is not allowed to attend. For a byte mask, a non-zero value indicates that the\n corresponding position is not allowed to attend. For a float mask, the mask values will be added to\n the attention weight.\n average_attn_weights: If true, indicates that the returned ``attn_weights`` should be averaged across\n heads. Otherwise, ``attn_weights`` are provided separately per head. Note that this flag only has an\n effect when ``need_weights=True``. Default: ``True`` (i.e. average weights across heads)\n\n Outputs:\n - **attn_output** - Attention outputs of shape :math:`(L, E)` when input is unbatched,\n :math:`(L, N, E)` when ``batch_first=False`` or :math:`(N, L, E)` when ``batch_first=True``,\n where :math:`L` is the target sequence length, :math:`N` is the batch size, and :math:`E` is the\n embedding dimension ``embed_dim``.\n - **attn_output_weights** - Only returned when ``need_weights=True``. If ``average_attn_weights=True``,\n returns attention weights averaged across heads of shape :math:`(L, S)` when input is unbatched or\n :math:`(N, L, S)`, where :math:`N` is the batch size, :math:`L` is the target sequence length, and\n :math:`S` is the source sequence length. If ``average_attn_weights=False``, returns attention weights per\n head of shape :math:`(\\text{num\\_heads}, L, S)` when input is unbatched or :math:`(N, \\text{num\\_heads}, L, S)`.\n\n .. note::\n `batch_first` argument is ignored for unbatched inputs.\n \"\"\"\n is_batched = query.dim() == 3\n if key_padding_mask is not None:\n _kpm_dtype = key_padding_mask.dtype\n if _kpm_dtype != torch.bool and not torch.is_floating_point(\n key_padding_mask\n ):\n raise AssertionError(\n \"only bool and floating types of key_padding_mask are supported\"\n )\n why_not_fast_path = \"\"\n if not is_batched:\n why_not_fast_path = f\"input not batched; expected query.dim() of 3 but got {query.dim()}\"\n elif query is not key or key is not value:\n # When lifting this restriction, don't forget to either\n # enforce that the dtypes all match or test cases where\n # they don't!\n why_not_fast_path = \"non-self attention was used (query, key, and value are not the same Tensor)\"\n elif (\n self.in_proj_bias is not None\n and query.dtype != self.in_proj_bias.dtype\n ):\n why_not_fast_path = f\"dtypes of query ({query.dtype}) and self.in_proj_bias ({self.in_proj_bias.dtype}) don't match\"\n elif (\n self.in_proj_weight is not None\n and query.dtype != self.in_proj_weight.dtype\n ):\n # this case will fail anyway, but at least they'll get a useful error message.\n why_not_fast_path = f\"dtypes of query ({query.dtype}) and self.in_proj_weight ({self.in_proj_weight.dtype}) don't match\"\n elif self.training:\n why_not_fast_path = \"training is enabled\"\n elif not self.batch_first:\n why_not_fast_path = \"batch_first was not True\"\n elif self.bias_k is not None:\n why_not_fast_path = \"self.bias_k was not None\"\n elif self.bias_v is not None:\n why_not_fast_path = \"self.bias_v was not None\"\n elif self.dropout:\n why_not_fast_path = f\"dropout was {self.dropout}, required zero\"\n elif self.add_zero_attn:\n why_not_fast_path = \"add_zero_attn was enabled\"\n elif not self._qkv_same_embed_dim:\n why_not_fast_path = \"_qkv_same_embed_dim was not True\"\n elif attn_mask is not None:\n why_not_fast_path = \"attn_mask was not None\"\n elif query.is_nested and key_padding_mask is not None:\n why_not_fast_path = (\n \"key_padding_mask is not supported with NestedTensor input\"\n )\n elif self.num_heads % 2 == 1:\n why_not_fast_path = \"num_heads is odd\"\n elif torch.is_autocast_enabled():\n why_not_fast_path = \"autocast is enabled\"\n\n if not why_not_fast_path:\n tensor_args = (\n query,\n key,\n value,\n self.in_proj_weight,\n self.in_proj_bias,\n self.out_proj.weight,\n self.out_proj.bias,\n )\n # We have to use list comprehensions below because TorchScript does not support\n # generator expressions.\n if torch.overrides.has_torch_function(tensor_args):\n why_not_fast_path = \"some Tensor argument has_torch_function\"\n elif not all(\n [\n (x is None or x.is_cuda or \"cpu\" in str(x.device))\n for x in tensor_args\n ]\n ):\n why_not_fast_path = (\n \"some Tensor argument is neither CUDA nor CPU\"\n )\n elif torch.is_grad_enabled() and any(\n [x is not None and x.requires_grad for x in tensor_args]\n ):\n why_not_fast_path = (\n \"grad is enabled and at least one of query or the \"\n \"input/output projection weights or biases requires_grad\"\n )\n if not why_not_fast_path:\n return torch._native_multi_head_attention(\n query,\n key,\n value,\n self.embed_dim,\n self.num_heads,\n self.in_proj_weight,\n self.in_proj_bias,\n self.out_proj.weight,\n self.out_proj.bias,\n key_padding_mask\n if key_padding_mask is not None\n else attn_mask,\n need_weights,\n average_attn_weights,\n 1\n if key_padding_mask is not None\n else 0\n if attn_mask is not None\n else None,\n )\n\n any_nested = query.is_nested or key.is_nested or value.is_nested\n assert not any_nested, (\n \"MultiheadAttention does not support NestedTensor outside of its fast path. \"\n + f\"The fast path was not hit because {why_not_fast_path}\"\n )\n\n if self.batch_first and is_batched:\n # make sure that the transpose op does not affect the \"is\" property\n if key is value:\n if query is key:\n query = key = value = query.transpose(1, 0)\n else:\n query, key = [x.transpose(1, 0) for x in (query, key)]\n value = key\n else:\n query, key, value = [\n x.transpose(1, 0) for x in (query, key, value)\n ]\n\n if not self._qkv_same_embed_dim:\n attn_output, attn_output_weights = F.multi_head_attention_forward(\n query,\n key,\n value,\n self.embed_dim,\n self.num_heads,\n self.in_proj_weight,\n self.in_proj_bias,\n self.bias_k,\n self.bias_v,\n self.add_zero_attn,\n self.dropout,\n self.out_proj.weight,\n self.out_proj.bias,\n training=self.training,\n key_padding_mask=key_padding_mask,\n need_weights=need_weights,\n attn_mask=attn_mask,\n use_separate_proj_weight=True,\n q_proj_weight=self.q_proj_weight,\n k_proj_weight=self.k_proj_weight,\n v_proj_weight=self.v_proj_weight,\n average_attn_weights=average_attn_weights,\n )\n else:\n # re-write the self.attention here, to get k, v cache\n tgt_len, bsz, embed_dim = query.shape\n src_len, _, _ = key.shape\n num_heads = self.num_heads\n key_padding_mask = _canonical_mask(\n mask=key_padding_mask,\n mask_name=\"key_padding_mask\",\n other_type=_none_or_dtype(attn_mask),\n other_name=\"attn_mask\",\n target_type=query.dtype\n )\n attn_mask = _canonical_mask(\n mask=attn_mask,\n mask_name=\"attn_mask\",\n other_type=None,\n other_name=\"\",\n target_type=query.dtype,\n check_other=False,\n )\n head_dim = self.embed_dim // self.num_heads\n assert head_dim * self.num_heads == self.embed_dim, f\"embed_dim {self.embed_dim} not divisible by num_heads {self.num_heads}\"\n assert key.shape == value.shape, f\"key shape {key.shape} does not match value shape {value.shape}\"\n q, k, v = _in_projection_packed(query, key, value, self.in_proj_weight, self.in_proj_bias)\n # k_present, v_present = k, v\n \n #\n # reshape q, k, v for multihead attention and make em batch first\n #\n \n q = q.view(tgt_len, bsz * num_heads, head_dim).transpose(0, 1)\n k = k.view(k.shape[0], bsz * num_heads, head_dim).transpose(0, 1)\n v = v.view(v.shape[0], bsz * num_heads, head_dim).transpose(0, 1) # (bsz * num_heads, src_len, head_dim)\n src_len = k.size(1)\n if past is not None and past.ndim > 2:\n expected_src_len = src_len + past[0].shape[-2]\n else:\n expected_src_len = src_len\n\n\n # ensure attn_mask's dim is 3\n if attn_mask.dim() == 2:\n correct_2d_size = (tgt_len, expected_src_len)\n if attn_mask.shape != correct_2d_size:\n raise RuntimeError(f\"The shape of the 2D attn_mask is {attn_mask.shape}, but should be {correct_2d_size}.\")\n attn_mask = attn_mask.unsqueeze(0)\n elif attn_mask.dim() == 3:\n correct_3d_size = (bsz * num_heads, tgt_len, expected_src_len)\n if attn_mask.shape != correct_3d_size:\n raise RuntimeError(f\"The shape of the 3D attn_mask is {attn_mask.shape}, but should be {correct_3d_size}.\")\n else:\n raise RuntimeError(f\"attn_mask's dimension {attn_mask.dim()} is not supported\")\n \n if key_padding_mask is not None:\n assert key_padding_mask.shape == (bsz, expected_src_len), \\\n f\"expecting key_padding_mask shape of {(bsz, expected_src_len)}, but got {key_padding_mask.shape}\"\n key_padding_mask = key_padding_mask.view(bsz, 1, 1, expected_src_len). \\\n expand(-1, num_heads, -1, -1).reshape(bsz * num_heads, 1, expected_src_len)\n if attn_mask is None:\n attn_mask = key_padding_mask\n else:\n attn_mask = attn_mask + key_padding_mask\n \n if not self.training:\n dropout_p = 0.0\n else:\n dropout_p = self.dropout\n\n if need_weights:\n raise NotImplementedError(\"need_weights not implemented for voicecraft\")\n # B, Nt, E = q.shape\n # q_scaled = q / math.sqrt(E)\n\n # assert not (is_causal and attn_mask is None), \"FIXME: is_causal not implemented for need_weights\"\n\n # if attn_mask is not None:\n # attn_output_weights = torch.baddbmm(attn_mask, q_scaled, k.transpose(-2, -1))\n # else:\n # attn_output_weights = torch.bmm(q_scaled, k.transpose(-2, -1))\n # attn_output_weights = softmax(attn_output_weights, dim=-1)\n # if dropout_p > 0.0:\n # attn_output_weights = dropout(attn_output_weights, p=dropout_p)\n\n # attn_output = torch.bmm(attn_output_weights, v)\n\n # attn_output = attn_output.transpose(0, 1).contiguous().view(tgt_len * bsz, embed_dim)\n # attn_output = linear(attn_output, out_proj_weight, out_proj_bias)\n # attn_output = attn_output.view(tgt_len, bsz, attn_output.size(1))\n\n # # optionally average attention weights over heads\n # attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len)\n # if average_attn_weights:\n # attn_output_weights = attn_output_weights.mean(dim=1)\n\n # if not is_batched:\n # # squeeze the output if input was unbatched\n # attn_output = attn_output.squeeze(1)\n # attn_output_weights = attn_output_weights.squeeze(0)\n # return attn_output, attn_output_weights\n else:\n # attn_mask can be either (L,S) or (N*num_heads, L, S)\n # if attn_mask's shape is (1, L, S) we need to unsqueeze to (1, 1, L, S)\n # in order to match the input for SDPA of (N, num_heads, L, S)\n if attn_mask is not None:\n if attn_mask.size(0) == 1 and attn_mask.dim() == 3:\n attn_mask = attn_mask.unsqueeze(0)\n else:\n attn_mask = attn_mask.view(bsz, num_heads, -1, expected_src_len)\n\n q = q.view(bsz, num_heads, tgt_len, head_dim)\n k = k.view(bsz, num_heads, src_len, head_dim)\n v = v.view(bsz, num_heads, src_len, head_dim)\n # logging.info(f\"shape of past: {past.shape}\")\n if past is not None:\n present = torch.stack([k, v], dim=0) # (2, bsz, num_heads, src_len, head_dim)\n if past.ndim > 2: # this means we use kvcache, otherwise we just pass in a placeholder, but not actually using kvcache\n pk, pv = past\n k = torch.cat([pk, k], dim=-2)\n v = torch.cat([pv, v], dim=-2)\n else:\n present = None\n attn_output = F.scaled_dot_product_attention(q, k, v, attn_mask, dropout_p, is_causal=False)\n attn_output = attn_output.permute(2, 0, 1, 3).contiguous().view(bsz * tgt_len, embed_dim)\n\n attn_output = F.linear(attn_output, self.out_proj.weight, self.out_proj.bias)\n attn_output = attn_output.view(tgt_len, bsz, attn_output.size(1))\n if not is_batched:\n # squeeze the output if input was unbatched\n attn_output = attn_output.squeeze(1)\n # if self.training:\n # return attn_output, None\n # else:\n # return (attn_output, present), None\n\n # harded coded, the code do not support returning attn weigths yet\n attn_output_weights=None\n if self.batch_first and is_batched:\n return attn_output.transpose(1, 0), present\n else:\n return attn_output, present\n\n" }, { "path": "models/modules/embedding.py", "content": "# cp from https://github.com/lifeiteng/vall-e/blob/main/valle/modules/embedding.py\n# Copyright 2023 (authors: Feiteng Li)\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport math\n\nimport torch\nimport torch.nn as nn\n\n\nclass TokenEmbedding(nn.Module):\n def __init__(\n self,\n dim_model: int,\n vocab_size: int,\n dropout: float = 0.0,\n ):\n super().__init__()\n\n self.vocab_size = vocab_size\n self.dim_model = dim_model\n\n self.dropout = torch.nn.Dropout(p=dropout)\n self.word_embeddings = nn.Embedding(self.vocab_size, self.dim_model)\n\n @property\n def weight(self) -> torch.Tensor:\n return self.word_embeddings.weight\n\n def embedding(self, index: int) -> torch.Tensor:\n return self.word_embeddings.weight[index : index + 1]\n\n def forward(self, x: torch.Tensor):\n X = self.word_embeddings(x)\n X = self.dropout(X)\n\n return X\n\n\nclass SinePositionalEmbedding(nn.Module):\n def __init__(\n self,\n dim_model: int,\n dropout: float = 0.0,\n scale: bool = False,\n alpha: bool = False,\n ):\n super().__init__()\n self.dim_model = dim_model\n self.x_scale = math.sqrt(dim_model) if scale else 1.0\n self.alpha = nn.Parameter(torch.ones(1), requires_grad=alpha)\n self.dropout = torch.nn.Dropout(p=dropout)\n\n self.reverse = False\n self.pe = None\n self.extend_pe(torch.tensor(0.0).expand(1, 4000))\n\n def extend_pe(self, x):\n \"\"\"Reset the positional encodings.\"\"\"\n if self.pe is not None:\n if self.pe.size(1) >= x.size(1):\n if self.pe.dtype != x.dtype or self.pe.device != x.device:\n self.pe = self.pe.to(dtype=x.dtype, device=x.device)\n return\n pe = torch.zeros(x.size(1), self.dim_model)\n if self.reverse:\n position = torch.arange(\n x.size(1) - 1, -1, -1.0, dtype=torch.float32\n ).unsqueeze(1)\n else:\n position = torch.arange(\n 0, x.size(1), dtype=torch.float32\n ).unsqueeze(1)\n div_term = torch.exp(\n torch.arange(0, self.dim_model, 2, dtype=torch.float32)\n * -(math.log(10000.0) / self.dim_model)\n )\n pe[:, 0::2] = torch.sin(position * div_term)\n pe[:, 1::2] = torch.cos(position * div_term)\n pe = pe.unsqueeze(0)\n self.pe = pe.to(device=x.device, dtype=x.dtype).detach()\n\n def forward(self, x: torch.Tensor) -> torch.Tensor:\n self.extend_pe(x)\n output = x.unsqueeze(-1) if x.ndim == 2 else x\n output = output * self.x_scale + self.alpha * self.pe[:, : x.size(1)]\n return self.dropout(output)" }, { "path": "models/modules/sampling.py", "content": "import torch\nimport torch.nn.functional as F\n\ndef top_k_top_p_filtering(\n logits, top_k=0, top_p=1.0, filter_value=-float(\"Inf\"), min_tokens_to_keep=1\n):\n \"\"\"Filter a distribution of logits using top-k and/or nucleus (top-p) filtering\n Args:\n logits: logits distribution shape (batch size, vocabulary size)\n if top_k > 0: keep only top k tokens with highest probability (top-k filtering).\n if top_p < 1.0: keep the top tokens with cumulative probability >= top_p (nucleus filtering).\n Nucleus filtering is described in Holtzman et al. (http://arxiv.org/abs/1904.09751)\n Make sure we keep at least min_tokens_to_keep per batch example in the output\n From: https://gist.github.com/thomwolf/1a5a29f6962089e871b94cbd09daf317\n \"\"\"\n if top_k > 0:\n top_k = min(\n max(top_k, min_tokens_to_keep), logits.size(-1)\n ) # Safety check\n # Remove all tokens with a probability less than the last token of the top-k\n indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None]\n logits[indices_to_remove] = filter_value\n\n if top_p < 1.0:\n sorted_logits, sorted_indices = torch.sort(logits, descending=True)\n cumulative_probs = torch.cumsum(\n F.softmax(sorted_logits, dim=-1), dim=-1\n )\n\n # Remove tokens with cumulative probability above the threshold (token with 0 are kept)\n sorted_indices_to_remove = cumulative_probs > top_p\n if min_tokens_to_keep > 1:\n # Keep at least min_tokens_to_keep (set to min_tokens_to_keep-1 because we add the first one below)\n sorted_indices_to_remove[..., :min_tokens_to_keep] = 0\n # Shift the indices to the right to keep also the first token above the threshold\n sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[\n ..., :-1\n ].clone()\n sorted_indices_to_remove[..., 0] = 0\n\n # scatter sorted tensors to original indexing\n indices_to_remove = sorted_indices_to_remove.scatter(\n 1, sorted_indices, sorted_indices_to_remove\n )\n logits[indices_to_remove] = filter_value\n return logits\n \ndef topk_sampling(logits, top_k=10, top_p=1.0, temperature=1.0):\n # temperature: (`optional`) float\n # The value used to module the next token probabilities. Must be strictly positive. Default to 1.0.\n # top_k: (`optional`) int\n # The number of highest probability vocabulary tokens to keep for top-k-filtering. Between 1 and infinity. Default to 50.\n # top_p: (`optional`) float\n # The cumulative probability of parameter highest probability vocabulary tokens to keep for nucleus sampling. Must be between 0 and 1. Default to 1.\n\n # Temperature (higher temperature => more likely to sample low probability tokens)\n if temperature != 1.0:\n logits = logits / temperature\n # Top-p/top-k filtering\n logits = top_k_top_p_filtering(logits, top_k=top_k, top_p=top_p)\n # Sample\n token = torch.multinomial(F.softmax(logits, dim=-1), num_samples=1)\n return token" }, { "path": "models/modules/scaling.py", "content": "# cp from https://github.com/lifeiteng/vall-e/blob/main/valle/modules/scaling.py\n# Copyright 2022 Xiaomi Corp. (authors: Daniel Povey)\n#\n# See ../../../../LICENSE for clarification regarding multiple authors\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nimport collections\nimport logging\nimport random\nimport math\nfrom functools import reduce\nfrom itertools import repeat\nfrom typing import Optional, Tuple, Union\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch import Tensor\nfrom torch.nn import Embedding as ScaledEmbedding\n\n# from valle.utils import Transpose\n\nclass Transpose(nn.Identity):\n \"\"\"(N, T, D) -> (N, D, T)\"\"\"\n\n def forward(self, input: torch.Tensor) -> torch.Tensor:\n return input.transpose(1, 2)\n \nclass ActivationBalancerFunction(torch.autograd.Function):\n @staticmethod\n def forward(\n ctx,\n x: Tensor,\n scale_factor: Tensor,\n sign_factor: Optional[Tensor],\n channel_dim: int,\n ) -> Tensor:\n if channel_dim < 0:\n channel_dim += x.ndim\n ctx.channel_dim = channel_dim\n xgt0 = x > 0\n if sign_factor is None:\n ctx.save_for_backward(xgt0, scale_factor)\n else:\n ctx.save_for_backward(xgt0, scale_factor, sign_factor)\n return x\n\n @staticmethod\n def backward(ctx, x_grad: Tensor) -> Tuple[Tensor, None, None, None]:\n if len(ctx.saved_tensors) == 3:\n xgt0, scale_factor, sign_factor = ctx.saved_tensors\n for _ in range(ctx.channel_dim, x_grad.ndim - 1):\n scale_factor = scale_factor.unsqueeze(-1)\n sign_factor = sign_factor.unsqueeze(-1)\n factor = sign_factor + scale_factor * (xgt0.to(x_grad.dtype) - 0.5)\n else:\n xgt0, scale_factor = ctx.saved_tensors\n for _ in range(ctx.channel_dim, x_grad.ndim - 1):\n scale_factor = scale_factor.unsqueeze(-1)\n factor = scale_factor * (xgt0.to(x_grad.dtype) - 0.5)\n neg_delta_grad = x_grad.abs() * factor\n return (\n x_grad - neg_delta_grad,\n None,\n None,\n None,\n )\n\n\ndef _compute_scale_factor(\n x: Tensor,\n channel_dim: int,\n min_abs: float,\n max_abs: float,\n gain_factor: float,\n max_factor: float,\n) -> Tensor:\n if channel_dim < 0:\n channel_dim += x.ndim\n sum_dims = [d for d in range(x.ndim) if d != channel_dim]\n x_abs_mean = torch.mean(x.abs(), dim=sum_dims).to(torch.float32)\n\n if min_abs == 0.0:\n below_threshold = 0.0\n else:\n # below_threshold is 0 if x_abs_mean > min_abs, can be at most max_factor if\n # x_abs)_mean , min_abs.\n below_threshold = (\n (min_abs - x_abs_mean) * (gain_factor / min_abs)\n ).clamp(min=0, max=max_factor)\n\n above_threshold = ((x_abs_mean - max_abs) * (gain_factor / max_abs)).clamp(\n min=0, max=max_factor\n )\n\n return below_threshold - above_threshold\n\n\ndef _compute_sign_factor(\n x: Tensor,\n channel_dim: int,\n min_positive: float,\n max_positive: float,\n gain_factor: float,\n max_factor: float,\n) -> Tensor:\n if channel_dim < 0:\n channel_dim += x.ndim\n sum_dims = [d for d in range(x.ndim) if d != channel_dim]\n proportion_positive = torch.mean((x > 0).to(torch.float32), dim=sum_dims)\n if min_positive == 0.0:\n factor1 = 0.0\n else:\n # 0 if proportion_positive >= min_positive, else can be\n # as large as max_factor.\n factor1 = (\n (min_positive - proportion_positive) * (gain_factor / min_positive)\n ).clamp_(min=0, max=max_factor)\n\n if max_positive == 1.0:\n factor2 = 0.0\n else:\n # 0 if self.proportion_positive <= max_positive, else can be\n # as large as -max_factor.\n factor2 = (\n (proportion_positive - max_positive)\n * (gain_factor / (1.0 - max_positive))\n ).clamp_(min=0, max=max_factor)\n sign_factor = factor1 - factor2\n # require min_positive != 0 or max_positive != 1:\n assert not isinstance(sign_factor, float)\n return sign_factor\n\n\nclass ActivationScaleBalancerFunction(torch.autograd.Function):\n \"\"\"\n This object is used in class ActivationBalancer when the user specified\n min_positive=0, max_positive=1, so there are no constraints on the signs\n of the activations and only the absolute value has a constraint.\n \"\"\"\n\n @staticmethod\n def forward(\n ctx,\n x: Tensor,\n sign_factor: Tensor,\n scale_factor: Tensor,\n channel_dim: int,\n ) -> Tensor:\n if channel_dim < 0:\n channel_dim += x.ndim\n ctx.channel_dim = channel_dim\n xgt0 = x > 0\n ctx.save_for_backward(xgt0, sign_factor, scale_factor)\n return x\n\n @staticmethod\n def backward(ctx, x_grad: Tensor) -> Tuple[Tensor, None, None, None]:\n xgt0, sign_factor, scale_factor = ctx.saved_tensors\n for _ in range(ctx.channel_dim, x_grad.ndim - 1):\n sign_factor = sign_factor.unsqueeze(-1)\n scale_factor = scale_factor.unsqueeze(-1)\n\n factor = sign_factor + scale_factor * (xgt0.to(x_grad.dtype) - 0.5)\n neg_delta_grad = x_grad.abs() * factor\n return (\n x_grad - neg_delta_grad,\n None,\n None,\n None,\n )\n\n\nclass RandomClampFunction(torch.autograd.Function):\n @staticmethod\n def forward(\n ctx,\n x: Tensor,\n min: Optional[float],\n max: Optional[float],\n prob: float,\n reflect: float,\n ) -> Tensor:\n x_clamped = torch.clamp(x, min=min, max=max)\n mask = torch.rand_like(x) < prob\n ans = torch.where(mask, x_clamped, x)\n if x.requires_grad:\n ctx.save_for_backward(ans == x)\n ctx.reflect = reflect\n if reflect != 0.0:\n ans = ans * (1.0 + reflect) - (x * reflect)\n return ans\n\n @staticmethod\n def backward(\n ctx, ans_grad: Tensor\n ) -> Tuple[Tensor, None, None, None, None]:\n (is_same,) = ctx.saved_tensors\n x_grad = ans_grad * is_same.to(ans_grad.dtype)\n reflect = ctx.reflect\n if reflect != 0.0:\n x_grad = x_grad * (1.0 + reflect) - (ans_grad * reflect)\n return x_grad, None, None, None, None\n\n\ndef random_clamp(\n x: Tensor,\n min: Optional[float] = None,\n max: Optional[float] = None,\n prob: float = 0.5,\n reflect: float = 0.0,\n):\n return RandomClampFunction.apply(x, min, max, prob, reflect)\n\n\ndef random_cast_to_half(x: Tensor, min_abs: float = 5.0e-06) -> Tensor:\n \"\"\"\n A randomized way of casting a floating point value to half precision.\n \"\"\"\n if x.dtype == torch.float16:\n return x\n x_abs = x.abs()\n is_too_small = x_abs < min_abs\n # for elements where is_too_small is true, random_val will contain +-min_abs with\n # probability (x.abs() / min_abs), and 0.0 otherwise. [so this preserves expectations,\n # for those elements].\n random_val = min_abs * x.sign() * (torch.rand_like(x) * min_abs < x_abs)\n return torch.where(is_too_small, random_val, x).to(torch.float16)\n\n\nclass RandomGradFunction(torch.autograd.Function):\n \"\"\"\n Does nothing in forward pass; in backward pass, gets rid of very small grads using\n randomized approach that preserves expectations (intended to reduce roundoff).\n \"\"\"\n\n @staticmethod\n def forward(ctx, x: Tensor, min_abs: float) -> Tensor:\n ctx.min_abs = min_abs\n return x\n\n @staticmethod\n def backward(ctx, ans_grad: Tensor) -> Tuple[Tensor, None]:\n if ans_grad.dtype == torch.float16:\n return (\n random_cast_to_half(\n ans_grad.to(torch.float32), min_abs=ctx.min_abs\n ),\n None,\n )\n else:\n return ans_grad, None\n\n\nclass RandomGrad(torch.nn.Module):\n \"\"\"\n Gets rid of very small gradients using an expectation-preserving method, intended to increase\n accuracy of training when using amp (automatic mixed precision)\n \"\"\"\n\n def __init__(self, min_abs: float = 5.0e-06):\n super(RandomGrad, self).__init__()\n self.min_abs = min_abs\n\n def forward(self, x: Tensor):\n if (\n torch.jit.is_scripting()\n or not self.training\n or torch.jit.is_tracing()\n ):\n return x\n else:\n return RandomGradFunction.apply(x, self.min_abs)\n\n\nclass SoftmaxFunction(torch.autograd.Function):\n \"\"\"\n Tries to handle half-precision derivatives in a randomized way that should\n be more accurate for training than the default behavior.\n \"\"\"\n\n @staticmethod\n def forward(ctx, x: Tensor, dim: int):\n ans = x.softmax(dim=dim)\n # if x dtype is float16, x.softmax() returns a float32 because\n # (presumably) that op does not support float16, and autocast\n # is enabled.\n if torch.is_autocast_enabled():\n ans = ans.to(torch.float16)\n ctx.save_for_backward(ans)\n ctx.x_dtype = x.dtype\n ctx.dim = dim\n return ans\n\n @staticmethod\n def backward(ctx, ans_grad: Tensor):\n (ans,) = ctx.saved_tensors\n with torch.cuda.amp.autocast(enabled=False):\n ans_grad = ans_grad.to(torch.float32)\n ans = ans.to(torch.float32)\n x_grad = ans_grad * ans\n x_grad = x_grad - ans * x_grad.sum(dim=ctx.dim, keepdim=True)\n return x_grad, None\n\n\ndef softmax(x: Tensor, dim: int):\n if torch.jit.is_scripting() or torch.jit.is_tracing():\n return x.softmax(dim)\n\n return SoftmaxFunction.apply(x, dim)\n\n\nclass MaxEigLimiterFunction(torch.autograd.Function):\n @staticmethod\n def forward(\n ctx,\n x: Tensor,\n coeffs: Tensor,\n direction: Tensor,\n channel_dim: int,\n grad_scale: float,\n ) -> Tensor:\n ctx.channel_dim = channel_dim\n ctx.grad_scale = grad_scale\n ctx.save_for_backward(x.detach(), coeffs.detach(), direction.detach())\n return x\n\n @staticmethod\n def backward(ctx, x_grad, *args):\n with torch.enable_grad():\n (x_orig, coeffs, new_direction) = ctx.saved_tensors\n x_orig.requires_grad = True\n num_channels = x_orig.shape[ctx.channel_dim]\n x = x_orig.transpose(ctx.channel_dim, -1).reshape(-1, num_channels)\n new_direction.requires_grad = False\n x = x - x.mean(dim=0)\n x_var = (x ** 2).mean()\n x_residual = x - coeffs * new_direction\n x_residual_var = (x_residual ** 2).mean()\n # `variance_proportion` is the proportion of the variance accounted for\n # by the top eigen-direction. This is to be minimized.\n variance_proportion = (x_var - x_residual_var) / (x_var + 1.0e-20)\n variance_proportion.backward()\n x_orig_grad = x_orig.grad\n x_extra_grad = (\n x_orig.grad\n * ctx.grad_scale\n * x_grad.norm()\n / (x_orig_grad.norm() + 1.0e-20)\n )\n return x_grad + x_extra_grad.detach(), None, None, None, None\n\n\nclass BasicNorm(torch.nn.Module):\n \"\"\"\n This is intended to be a simpler, and hopefully cheaper, replacement for\n LayerNorm. The observation this is based on, is that Transformer-type\n networks, especially with pre-norm, sometimes seem to set one of the\n feature dimensions to a large constant value (e.g. 50), which \"defeats\"\n the LayerNorm because the output magnitude is then not strongly dependent\n on the other (useful) features. Presumably the weight and bias of the\n LayerNorm are required to allow it to do this.\n\n So the idea is to introduce this large constant value as an explicit\n parameter, that takes the role of the \"eps\" in LayerNorm, so the network\n doesn't have to do this trick. We make the \"eps\" learnable.\n\n Args:\n num_channels: the number of channels, e.g. 512.\n channel_dim: the axis/dimension corresponding to the channel,\n interprted as an offset from the input's ndim if negative.\n shis is NOT the num_channels; it should typically be one of\n {-2, -1, 0, 1, 2, 3}.\n eps: the initial \"epsilon\" that we add as ballast in:\n scale = ((input_vec**2).mean() + epsilon)**-0.5\n Note: our epsilon is actually large, but we keep the name\n to indicate the connection with conventional LayerNorm.\n learn_eps: if true, we learn epsilon; if false, we keep it\n at the initial value.\n eps_min: float\n eps_max: float\n \"\"\"\n\n def __init__(\n self,\n num_channels: int,\n channel_dim: int = -1, # CAUTION: see documentation.\n eps: float = 0.25,\n learn_eps: bool = True,\n eps_min: float = -3.0,\n eps_max: float = 3.0,\n ) -> None:\n super(BasicNorm, self).__init__()\n self.num_channels = num_channels\n self.channel_dim = channel_dim\n if learn_eps:\n self.eps = nn.Parameter(torch.tensor(eps).log().detach())\n else:\n self.register_buffer(\"eps\", torch.tensor(eps).log().detach())\n self.eps_min = eps_min\n self.eps_max = eps_max\n\n def forward(self, x: Tensor) -> Tensor:\n assert x.shape[self.channel_dim] == self.num_channels\n eps = self.eps\n if self.training and random.random() < 0.25:\n # with probability 0.25, in training mode, clamp eps between the min\n # and max; this will encourage it to learn parameters within the\n # allowed range by making parameters that are outside the allowed\n # range noisy.\n\n # gradients to allow the parameter to get back into the allowed region if it happens to exit it.\n eps = eps.clamp(min=self.eps_min, max=self.eps_max)\n scales = (\n torch.mean(x ** 2, dim=self.channel_dim, keepdim=True) + eps.exp()\n ) ** -0.5\n return x * scales\n\n\ndef ScaledLinear(*args, initial_scale: float = 1.0, **kwargs) -> nn.Linear:\n \"\"\"\n Behaves like a constructor of a modified version of nn.Linear\n that gives an easy way to set the default initial parameter scale.\n\n Args:\n Accepts the standard args and kwargs that nn.Linear accepts\n e.g. in_features, out_features, bias=False.\n\n initial_scale: you can override this if you want to increase\n or decrease the initial magnitude of the module's output\n (affects the initialization of weight_scale and bias_scale).\n Another option, if you want to do something like this, is\n to re-initialize the parameters.\n \"\"\"\n ans = nn.Linear(*args, **kwargs)\n with torch.no_grad():\n ans.weight[:] *= initial_scale\n if ans.bias is not None:\n torch.nn.init.uniform_(\n ans.bias, -0.1 * initial_scale, 0.1 * initial_scale\n )\n return ans\n\n\ndef ScaledConv1d(\n *args,\n initial_scale: float = 1.0,\n kernel_size: int = 3,\n padding: str = \"same\",\n **kwargs,\n) -> nn.Conv1d:\n \"\"\"\n Behaves like a constructor of a modified version of nn.Conv1d\n that gives an easy way to set the default initial parameter scale.\n\n Args:\n Accepts the standard args and kwargs that nn.Linear accepts\n e.g. in_features, out_features, bias=False.\n\n initial_scale: you can override this if you want to increase\n or decrease the initial magnitude of the module's output\n (affects the initialization of weight_scale and bias_scale).\n Another option, if you want to do something like this, is\n to re-initialize the parameters.\n \"\"\"\n ans = nn.Conv1d(*args, kernel_size=kernel_size, padding=padding, **kwargs)\n with torch.no_grad():\n ans.weight[:] *= initial_scale\n if ans.bias is not None:\n torch.nn.init.uniform_(\n ans.bias, -0.1 * initial_scale, 0.1 * initial_scale\n )\n return ans\n\n\ndef TransposeScaledConv1d(\n *args,\n initial_scale: float = 1.0,\n kernel_size: int = 3,\n padding: str = \"same\",\n **kwargs,\n) -> nn.Sequential:\n \"\"\"\n Transpose -> ScaledConv1d\n \"\"\"\n return nn.Sequential(\n Transpose(),\n ScaledConv1d(\n *args,\n initial_scale=initial_scale,\n kernel_size=kernel_size,\n padding=padding,\n **kwargs,\n ),\n )\n\n\ndef ScaledConv1dTranspose(\n *args,\n initial_scale: float = 1.0,\n kernel_size: int = 3,\n padding: str = \"same\",\n **kwargs,\n) -> nn.Sequential:\n \"\"\"\n Transpose -> ScaledConv1d\n \"\"\"\n return nn.Sequential(\n ScaledConv1d(\n *args,\n initial_scale=initial_scale,\n kernel_size=kernel_size,\n padding=padding,\n **kwargs,\n ),\n Transpose(),\n )\n\n\ndef TransposeConv1d(\n *args, kernel_size: int = 3, padding: str = \"same\", **kwargs\n) -> nn.Sequential:\n \"\"\"\n Transpose -> Conv1d\n \"\"\"\n return nn.Sequential(\n Transpose(),\n nn.Conv1d(*args, kernel_size=kernel_size, padding=padding, **kwargs),\n )\n\n\ndef Conv1dTranspose(\n *args, kernel_size: int = 3, padding: str = \"same\", **kwargs\n) -> nn.Sequential:\n \"\"\"\n ScaledConv1d -> Transpose\n \"\"\"\n return nn.Sequential(\n nn.Conv1d(*args, kernel_size=kernel_size, padding=padding, **kwargs),\n Transpose(),\n )\n\n\nclass SRLinear(nn.Linear):\n \"\"\"https://arxiv.org/abs/2303.06296\n Stabilizing Transformer Training by Preventing Attention Entropy Collapse\n \"\"\"\n\n def __init__(self, in_features, out_features, bias=True, **kwargs):\n super().__init__(in_features, out_features, bias=bias, **kwargs)\n self.register_buffer(\n \"u\", nn.functional.normalize(torch.randn(in_features), dim=0)\n )\n with torch.no_grad():\n sigma = self.get_sigma()\n self.register_buffer(\"spectral_norm\", sigma)\n self.sigma = nn.Parameter(torch.ones(1))\n\n def get_sigma(self):\n with torch.no_grad():\n u = self.u\n v = self.weight.mv(u)\n v = nn.functional.normalize(v, dim=0)\n u = self.weight.T.mv(v)\n u = nn.functional.normalize(u, dim=0)\n self.u.data.copy_(u)\n return torch.einsum(\"c,cd,d->\", v, self.weight, u)\n\n def get_weight(self):\n sigma = self.get_sigma()\n if self.training:\n self.spectral_norm.data.copy_(sigma)\n weight = (self.sigma / sigma) * self.weight\n return weight\n\n def forward(self, x):\n return nn.functional.linear(x, self.get_weight(), self.bias)\n\n\nclass SRConv1d(SRLinear):\n def __init__(\n self,\n in_features,\n out_features,\n kernel_size,\n stride: int = 1,\n padding: str = \"same\",\n bias: bool = True,\n **kwargs,\n ):\n in_features = in_features * kernel_size\n super().__init__(in_features, out_features, bias=bias, **kwargs)\n nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5))\n self.kernel_size = kernel_size\n self.stride = stride\n self.padding = padding\n\n def forward(self, x):\n in_features = self.in_features // self.kernel_size\n weight = self.get_weight().view(\n self.out_features, in_features, self.kernel_size\n )\n return nn.functional.conv1d(\n x, weight, bias=self.bias, stride=self.stride, padding=self.padding\n )\n\n\ndef TransposeSRConv1d(\n *args, kernel_size: int = 3, padding: str = \"same\", **kwargs\n) -> nn.Sequential:\n \"\"\"\n Transpose -> SRConv1d\n \"\"\"\n return nn.Sequential(\n Transpose(),\n SRConv1d(*args, kernel_size=kernel_size, padding=padding, **kwargs),\n )\n\n\ndef SRConv1dTranspose(\n *args, kernel_size: int = 3, padding: str = \"same\", **kwargs\n) -> nn.Sequential:\n \"\"\"\n SRConv1d -> Transpose\n \"\"\"\n return nn.Sequential(\n SRConv1d(*args, kernel_size=kernel_size, padding=padding, **kwargs),\n Transpose(),\n )\n\n\nclass ActivationBalancer(torch.nn.Module):\n \"\"\"\n Modifies the backpropped derivatives of a function to try to encourage, for\n each channel, that it is positive at least a proportion `threshold` of the\n time. It does this by multiplying negative derivative values by up to\n (1+max_factor), and positive derivative values by up to (1-max_factor),\n interpolated from 1 at the threshold to those extremal values when none\n of the inputs are positive.\n\n Args:\n num_channels: the number of channels\n channel_dim: the dimension/axis corresponding to the channel, e.g.\n -1, 0, 1, 2; will be interpreted as an offset from x.ndim if negative.\n min_positive: the minimum, per channel, of the proportion of the time\n that (x > 0), below which we start to modify the derivatives.\n max_positive: the maximum, per channel, of the proportion of the time\n that (x > 0), above which we start to modify the derivatives.\n max_factor: the maximum factor by which we modify the derivatives for\n either the sign constraint or the magnitude constraint;\n e.g. with max_factor=0.02, the the derivatives would be multiplied by\n values in the range [0.98..1.02].\n sign_gain_factor: determines the 'gain' with which we increase the\n change in gradient once the constraints on min_positive and max_positive\n are violated.\n scale_gain_factor: determines the 'gain' with which we increase the\n change in gradient once the constraints on min_abs and max_abs\n are violated.\n min_abs: the minimum average-absolute-value difference from the mean\n value per channel, which we allow, before we start to modify\n the derivatives to prevent this.\n max_abs: the maximum average-absolute-value difference from the mean\n value per channel, which we allow, before we start to modify\n the derivatives to prevent this.\n min_prob: determines the minimum probability with which we modify the\n gradients for the {min,max}_positive and {min,max}_abs constraints,\n on each forward(). This is done randomly to prevent all layers\n from doing it at the same time. Early in training we may use\n higher probabilities than this; it will decay to this value.\n \"\"\"\n\n def __init__(\n self,\n num_channels: int,\n channel_dim: int,\n min_positive: float = 0.05,\n max_positive: float = 0.95,\n max_factor: float = 0.04,\n sign_gain_factor: float = 0.01,\n scale_gain_factor: float = 0.02,\n min_abs: float = 0.2,\n max_abs: float = 100.0,\n min_prob: float = 0.1,\n ):\n super(ActivationBalancer, self).__init__()\n self.num_channels = num_channels\n self.channel_dim = channel_dim\n self.min_positive = min_positive\n self.max_positive = max_positive\n self.max_factor = max_factor\n self.min_abs = min_abs\n self.max_abs = max_abs\n self.min_prob = min_prob\n self.sign_gain_factor = sign_gain_factor\n self.scale_gain_factor = scale_gain_factor\n\n # count measures how many times the forward() function has been called.\n # We occasionally sync this to a tensor called `count`, that exists to\n # make sure it is synced to disk when we load and save the model.\n self.cpu_count = 0\n self.register_buffer(\"count\", torch.tensor(0, dtype=torch.int64))\n\n def forward(self, x: Tensor) -> Tensor:\n if (\n torch.jit.is_scripting()\n or not x.requires_grad\n or torch.jit.is_tracing()\n ):\n return _no_op(x)\n\n count = self.cpu_count\n self.cpu_count += 1\n\n if random.random() < 0.01:\n # Occasionally sync self.cpu_count with self.count.\n # count affects the decay of 'prob'. don't do this on every iter,\n # because syncing with the GPU is slow.\n self.cpu_count = max(self.cpu_count, self.count.item())\n self.count.fill_(self.cpu_count)\n\n # the prob of doing some work exponentially decreases from 0.5 till it hits\n # a floor at min_prob (==0.1, by default)\n prob = max(self.min_prob, 0.5 ** (1 + (count / 4000.0)))\n\n if random.random() < prob:\n sign_gain_factor = 0.5\n if self.min_positive != 0.0 or self.max_positive != 1.0:\n sign_factor = _compute_sign_factor(\n x,\n self.channel_dim,\n self.min_positive,\n self.max_positive,\n gain_factor=self.sign_gain_factor / prob,\n max_factor=self.max_factor,\n )\n else:\n sign_factor = None\n\n scale_factor = _compute_scale_factor(\n x.detach(),\n self.channel_dim,\n min_abs=self.min_abs,\n max_abs=self.max_abs,\n gain_factor=self.scale_gain_factor / prob,\n max_factor=self.max_factor,\n )\n return ActivationBalancerFunction.apply(\n x,\n scale_factor,\n sign_factor,\n self.channel_dim,\n )\n else:\n return _no_op(x)\n\n\ndef penalize_abs_values_gt(x: Tensor, limit: float, penalty: float) -> Tensor:\n \"\"\"\n Returns x unmodified, but in backprop will put a penalty for the excess of\n the absolute values of elements of x over the limit \"limit\". E.g. if\n limit == 10.0, then if x has any values over 10 it will get a penalty.\n\n Caution: the value of this penalty will be affected by grad scaling used\n in automatic mixed precision training. For this reasons we use this,\n it shouldn't really matter, or may even be helpful; we just use this\n to disallow really implausible values of scores to be given to softmax.\n \"\"\"\n x_sign = x.sign()\n over_limit = (x.abs() - limit) > 0\n # The following is a memory efficient way to penalize the absolute values of\n # x that's over the limit. (The memory efficiency comes when you think\n # about which items torch needs to cache for the autograd, and which ones it\n # can throw away). The numerical value of aux_loss as computed here will\n # actually be larger than it should be, by limit * over_limit.sum(), but it\n # has the same derivative as the real aux_loss which is penalty * (x.abs() -\n # limit).relu().\n aux_loss = penalty * ((x_sign * over_limit).to(torch.int8) * x)\n # note: we don't do sum() here on aux)_loss, but it's as if we had done\n # sum() due to how with_loss() works.\n x = with_loss(x, aux_loss)\n # you must use x for something, or this will be ineffective.\n return x\n\n\ndef _diag(x: Tensor): # like .diag(), but works for tensors with 3 dims.\n if x.ndim == 2:\n return x.diag()\n else:\n (batch, dim, dim) = x.shape\n x = x.reshape(batch, dim * dim)\n x = x[:, :: dim + 1]\n assert x.shape == (batch, dim)\n return x\n\n\ndef _whitening_metric(x: Tensor, num_groups: int):\n \"\"\"\n Computes the \"whitening metric\", a value which will be 1.0 if all the eigenvalues of\n of the centered feature covariance are the same within each group's covariance matrix\n and also between groups.\n Args:\n x: a Tensor of shape (*, num_channels)\n num_groups: the number of groups of channels, a number >=1 that divides num_channels\n Returns:\n Returns a scalar Tensor that will be 1.0 if the data is \"perfectly white\" and\n greater than 1.0 otherwise.\n \"\"\"\n assert x.dtype != torch.float16\n x = x.reshape(-1, x.shape[-1])\n (num_frames, num_channels) = x.shape\n assert num_channels % num_groups == 0\n channels_per_group = num_channels // num_groups\n x = x.reshape(num_frames, num_groups, channels_per_group).transpose(0, 1)\n # x now has shape (num_groups, num_frames, channels_per_group)\n # subtract the mean so we use the centered, not uncentered, covariance.\n # My experience has been that when we \"mess with the gradients\" like this,\n # it's better not do anything that tries to move the mean around, because\n # that can easily cause instability.\n x = x - x.mean(dim=1, keepdim=True)\n # x_covar: (num_groups, channels_per_group, channels_per_group)\n x_covar = torch.matmul(x.transpose(1, 2), x)\n x_covar_mean_diag = _diag(x_covar).mean()\n # the following expression is what we'd get if we took the matrix product\n # of each covariance and measured the mean of its trace, i.e.\n # the same as _diag(torch.matmul(x_covar, x_covar)).mean().\n x_covarsq_mean_diag = (x_covar ** 2).sum() / (\n num_groups * channels_per_group\n )\n # this metric will be >= 1.0; the larger it is, the less 'white' the data was.\n metric = x_covarsq_mean_diag / (x_covar_mean_diag ** 2 + 1.0e-20)\n return metric\n\n\nclass WhiteningPenaltyFunction(torch.autograd.Function):\n @staticmethod\n def forward(\n ctx,\n x: Tensor,\n num_groups: int,\n whitening_limit: float,\n grad_scale: float,\n ) -> Tensor:\n ctx.save_for_backward(x)\n ctx.num_groups = num_groups\n ctx.whitening_limit = whitening_limit\n ctx.grad_scale = grad_scale\n return x\n\n @staticmethod\n def backward(ctx, x_grad: Tensor):\n (x_orig,) = ctx.saved_tensors\n with torch.enable_grad():\n with torch.cuda.amp.autocast(enabled=False):\n x_detached = x_orig.to(torch.float32).detach()\n x_detached.requires_grad = True\n\n metric = _whitening_metric(x_detached, ctx.num_groups)\n\n if random.random() < 0.005 or __name__ == \"__main__\":\n logging.info(\n f\"Whitening: num_groups={ctx.num_groups}, num_channels={x_orig.shape[-1]}, \"\n f\"metric={metric.item():.2f} vs. limit={ctx.whitening_limit}\"\n )\n\n (metric - ctx.whitening_limit).relu().backward()\n penalty_grad = x_detached.grad\n scale = ctx.grad_scale * (\n x_grad.to(torch.float32).norm()\n / (penalty_grad.norm() + 1.0e-20)\n )\n penalty_grad = penalty_grad * scale\n return x_grad + penalty_grad.to(x_grad.dtype), None, None, None\n\n\nclass Whiten(nn.Module):\n def __init__(\n self,\n num_groups: int,\n whitening_limit: float,\n prob: Union[float, Tuple[float, float]],\n grad_scale: float,\n ):\n \"\"\"\n Args:\n num_groups: the number of groups to divide the channel dim into before\n whitening. We will attempt to make the feature covariance\n within each group, after mean subtraction, as \"white\" as possible,\n while having the same trace across all groups.\n whitening_limit: a value greater than 1.0, that dictates how much\n freedom we have to violate the constraints. 1.0 would mean perfectly\n white, with exactly the same trace across groups; larger values\n give more freedom. E.g. 2.0.\n prob: the probability with which we apply the gradient modification\n (also affects the grad scale). May be supplied as a float,\n or as a pair (min_prob, max_prob)\n\n grad_scale: determines the scale on the gradient term from this object,\n relative to the rest of the gradient on the attention weights.\n E.g. 0.02 (you may want to use smaller values than this if prob is large)\n \"\"\"\n super(Whiten, self).__init__()\n assert num_groups >= 1\n assert whitening_limit >= 1\n assert grad_scale >= 0\n self.num_groups = num_groups\n self.whitening_limit = whitening_limit\n if isinstance(prob, float):\n assert 0 < prob <= 1\n self.prob = prob\n else:\n (self.min_prob, self.max_prob) = prob\n assert 0 < self.min_prob < self.max_prob <= 1\n self.prob = self.max_prob\n\n self.grad_scale = grad_scale\n\n def forward(self, x: Tensor) -> Tensor:\n \"\"\"\n In the forward pass, this function just returns the input unmodified.\n In the backward pass, it will modify the gradients to ensure that the\n distribution in each group has close to (lambda times I) as the covariance\n after mean subtraction, with the same lambda across groups.\n For whitening_limit > 1, there will be more freedom to violate this\n constraint.\n\n Args:\n x: the input of shape (*, num_channels)\n\n Returns:\n x, unmodified. You should make sure\n you use the returned value, or the graph will be freed\n and nothing will happen in backprop.\n \"\"\"\n if (\n not x.requires_grad\n or random.random() > self.prob\n or self.grad_scale == 0\n ):\n return _no_op(x)\n else:\n if hasattr(self, \"min_prob\") and random.random() < 0.25:\n # occasionally switch between min_prob and max_prob, based on whether\n # we are above or below the threshold.\n if (\n _whitening_metric(x.to(torch.float32), self.num_groups)\n > self.whitening_limit\n ):\n # there would be a change to the grad.\n self.prob = self.max_prob\n else:\n self.prob = self.min_prob\n\n return WhiteningPenaltyFunction.apply(\n x, self.num_groups, self.whitening_limit, self.grad_scale\n )\n\n\nclass WithLoss(torch.autograd.Function):\n @staticmethod\n def forward(ctx, x: Tensor, y: Tensor):\n ctx.y_shape = y.shape\n return x\n\n @staticmethod\n def backward(ctx, ans_grad: Tensor):\n return ans_grad, torch.ones(\n ctx.y_shape, dtype=ans_grad.dtype, device=ans_grad.device\n )\n\n\ndef with_loss(x, y):\n if torch.jit.is_scripting() or torch.jit.is_tracing():\n return x\n # returns x but adds y.sum() to the loss function.\n return WithLoss.apply(x, y)\n\n\ndef _no_op(x: Tensor) -> Tensor:\n if torch.jit.is_scripting() or torch.jit.is_tracing():\n return x\n else:\n # a no-op function that will have a node in the autograd graph,\n # to avoid certain bugs relating to backward hooks\n return x.chunk(1, dim=-1)[0]\n\n\nclass Identity(torch.nn.Module):\n def __init__(self):\n super(Identity, self).__init__()\n\n def forward(self, x):\n return _no_op(x)\n\n\nclass MaxEig(torch.nn.Module):\n \"\"\"\n Modifies the backpropped derivatives of a function to try to discourage\n that any given direction in activation space accounts for more than\n a specified proportion of the covariance (e.g. 0.2).\n\n\n Args:\n num_channels: the number of channels\n channel_dim: the dimension/axis corresponding to the channel, e.g.\n -1, 0, 1, 2; will be interpreted as an offset from x.ndim if negative.\n max_var_per_eig: the maximum proportion of the variance of the\n features/channels, after mean subtraction, that can come from\n any given eigenvalue.\n min_prob: the minimum probability with which we apply this during any invocation\n of forward(), assuming last time we applied the constraint it was\n not active; supplied for speed.\n scale: determines the scale with which we modify the gradients, relative\n to the existing / unmodified gradients\n \"\"\"\n\n def __init__(\n self,\n num_channels: int,\n channel_dim: int,\n max_var_per_eig: float = 0.2,\n min_prob: float = 0.01,\n scale: float = 0.01,\n ):\n super(MaxEig, self).__init__()\n self.num_channels = num_channels\n self.channel_dim = channel_dim\n self.scale = scale\n assert max_var_per_eig == 0.0 or max_var_per_eig > 1.0 / num_channels\n self.max_var_per_eig = max_var_per_eig\n\n # we figure out the dominant direction using the power method: starting with\n # a random vector, keep multiplying by the covariance and renormalizing.\n with torch.no_grad():\n # arbitrary.. would use randn() but want to leave the rest of the model's\n # random parameters unchanged for comparison\n direction = torch.arange(num_channels).to(torch.float)\n direction = direction / direction.norm()\n self.register_buffer(\"max_eig_direction\", direction)\n\n self.min_prob = min_prob\n # cur_prob is the current probability we'll use to apply the ActivationBalancer.\n # We'll regress this towards prob, each tiem we try to apply it and it is not\n # active.\n self.cur_prob = 1.0\n\n def forward(self, x: Tensor) -> Tensor:\n if (\n torch.jit.is_scripting()\n or self.max_var_per_eig <= 0\n or random.random() > self.cur_prob\n or torch.jit.is_tracing()\n ):\n return _no_op(x)\n\n with torch.cuda.amp.autocast(enabled=False):\n eps = 1.0e-20\n orig_x = x\n x = x.to(torch.float32)\n with torch.no_grad():\n x = x.transpose(self.channel_dim, -1).reshape(\n -1, self.num_channels\n )\n x = x - x.mean(dim=0)\n new_direction, coeffs = self._find_direction_coeffs(\n x, self.max_eig_direction\n )\n x_var = (x ** 2).mean()\n x_residual = x - coeffs * new_direction\n x_residual_var = (x_residual ** 2).mean()\n\n # `variance_proportion` is the proportion of the variance accounted for\n # by the top eigen-direction.\n variance_proportion = (x_var - x_residual_var) / (\n x_var + 1.0e-20\n )\n\n # ensure new direction is nonzero even if x == 0, by including `direction`.\n self._set_direction(\n 0.1 * self.max_eig_direction + new_direction\n )\n\n if random.random() < 0.01 or __name__ == \"__main__\":\n logging.info(\n f\"variance_proportion = {variance_proportion.item()}, shape={tuple(orig_x.shape)}, cur_prob={self.cur_prob}\"\n )\n\n if variance_proportion >= self.max_var_per_eig:\n # The constraint is active. Note, we should quite rarely\n # reach here, only near the beginning of training if we are\n # starting to diverge, should this constraint be active.\n cur_prob = self.cur_prob\n self.cur_prob = (\n 1.0 # next time, do the update with probability 1.0.\n )\n return MaxEigLimiterFunction.apply(\n orig_x, coeffs, new_direction, self.channel_dim, self.scale\n )\n else:\n # let self.cur_prob exponentially approach self.min_prob, as\n # long as the constraint is inactive.\n self.cur_prob = 0.75 * self.cur_prob + 0.25 * self.min_prob\n return orig_x\n\n def _set_direction(self, direction: Tensor):\n \"\"\"\n Sets self.max_eig_direction to a normalized version of `direction`\n \"\"\"\n direction = direction.detach()\n direction = direction / direction.norm()\n direction_sum = direction.sum().item()\n if direction_sum - direction_sum == 0: # no inf/nan\n self.max_eig_direction[:] = direction\n else:\n logging.info(\n f\"Warning: sum of direction in MaxEig is {direction_sum}, \"\n \"num_channels={self.num_channels}, channel_dim={self.channel_dim}\"\n )\n\n def _find_direction_coeffs(\n self, x: Tensor, prev_direction: Tensor\n ) -> Tuple[Tensor, Tensor, Tensor]:\n \"\"\"\n Figure out (an approximation to) the proportion of the variance of a set of\n feature vectors that can be attributed to the top eigen-direction.\n Args:\n x: a Tensor of shape (num_frames, num_channels), with num_frames > 1.\n prev_direction: a Tensor of shape (num_channels,), that is our previous estimate\n of the top eigen-direction, or a random direction if this is the first\n iteration. Does not have to be normalized, but should be nonzero.\n\n Returns: (cur_direction, coeffs), where:\n cur_direction: a Tensor of shape (num_channels,) that is the current\n estimate of the top eigen-direction.\n coeffs: a Tensor of shape (num_frames, 1) that minimizes, or\n approximately minimizes, (x - coeffs * cur_direction).norm()\n \"\"\"\n (num_frames, num_channels) = x.shape\n assert num_channels > 1 and num_frames > 1\n assert prev_direction.shape == (num_channels,)\n # `coeffs` are the coefficients of `prev_direction` in x.\n # actually represent the coeffs up to a constant positive factor.\n coeffs = (x * prev_direction).sum(dim=1, keepdim=True) + 1.0e-10\n cur_direction = (x * coeffs).sum(dim=0) / (\n (coeffs ** 2).sum() + 1.0e-20\n )\n return cur_direction, coeffs\n\n\nclass DoubleSwishFunction(torch.autograd.Function):\n \"\"\"\n double_swish(x) = x * torch.sigmoid(x-1)\n This is a definition, originally motivated by its close numerical\n similarity to swish(swish(x)), where swish(x) = x * sigmoid(x).\n\n Memory-efficient derivative computation:\n double_swish(x) = x * s, where s(x) = torch.sigmoid(x-1)\n double_swish'(x) = d/dx double_swish(x) = x * s'(x) + x' * s(x) = x * s'(x) + s(x).\n Now, s'(x) = s(x) * (1-s(x)).\n double_swish'(x) = x * s'(x) + s(x).\n = x * s(x) * (1-s(x)) + s(x).\n = double_swish(x) * (1-s(x)) + s(x)\n ... so we just need to remember s(x) but not x itself.\n \"\"\"\n\n @staticmethod\n def forward(ctx, x: Tensor) -> Tensor:\n requires_grad = x.requires_grad\n x_dtype = x.dtype\n if x.dtype == torch.float16:\n x = x.to(torch.float32)\n\n s = torch.sigmoid(x - 1.0)\n y = x * s\n\n if requires_grad:\n deriv = y * (1 - s) + s\n # notes on derivative of x * sigmoid(x - 1):\n # https://www.wolframalpha.com/input?i=d%2Fdx+%28x+*+sigmoid%28x-1%29%29\n # min \\simeq -0.043638. Take floor as -0.043637 so it's a lower bund\n # max \\simeq 1.1990. Take ceil to be 1.2 so it's an upper bound.\n # the combination of \"+ torch.rand_like(deriv)\" and casting to torch.uint8 (which\n # floors), should be expectation-preserving.\n floor = -0.043637\n ceil = 1.2\n d_scaled = (deriv - floor) * (\n 255.0 / (ceil - floor)\n ) + torch.rand_like(deriv)\n if __name__ == \"__main__\":\n # for self-testing only.\n assert d_scaled.min() >= 0.0\n assert d_scaled.max() < 256.0\n d_int = d_scaled.to(torch.uint8)\n ctx.save_for_backward(d_int)\n if x.dtype == torch.float16 or torch.is_autocast_enabled():\n y = y.to(torch.float16)\n return y\n\n @staticmethod\n def backward(ctx, y_grad: Tensor) -> Tensor:\n (d,) = ctx.saved_tensors\n # the same constants as used in forward pass.\n floor = -0.043637\n ceil = 1.2\n d = d * ((ceil - floor) / 255.0) + floor\n return y_grad * d\n\n\nclass DoubleSwish(torch.nn.Module):\n def forward(self, x: Tensor) -> Tensor:\n \"\"\"Return double-swish activation function which is an approximation to Swish(Swish(x)),\n that we approximate closely with x * sigmoid(x-1).\n \"\"\"\n if torch.jit.is_scripting() or torch.jit.is_tracing():\n return x * torch.sigmoid(x - 1.0)\n return DoubleSwishFunction.apply(x)\n\n\ndef BalancedDoubleSwish(\n d_model, channel_dim=-1, max_abs=10.0, min_prob=0.25\n) -> nn.Sequential:\n \"\"\"\n ActivationBalancer -> DoubleSwish\n \"\"\"\n balancer = ActivationBalancer(\n d_model, channel_dim=channel_dim, max_abs=max_abs, min_prob=min_prob\n )\n return nn.Sequential(\n balancer,\n DoubleSwish(),\n )\n\n\ndef _test_max_eig():\n for proportion in [0.1, 0.5, 10.0]:\n logging.info(f\"proportion = {proportion}\")\n x = torch.randn(100, 128)\n direction = torch.randn(128)\n coeffs = torch.randn(100, 1)\n x += proportion * direction * coeffs\n\n x.requires_grad = True\n\n num_channels = 128\n m = MaxEig(\n num_channels, 1, 0.5, scale=0.1 # channel_dim # max_var_per_eig\n ) # grad_scale\n\n for _ in range(4):\n y = m(x)\n\n y_grad = torch.randn_like(x)\n y.backward(gradient=y_grad)\n\n if proportion < 0.2:\n assert torch.allclose(x.grad, y_grad, atol=1.0e-02)\n elif proportion > 1.0:\n assert not torch.allclose(x.grad, y_grad)\n\n\ndef _test_whiten():\n for proportion in [0.1, 0.5, 10.0]:\n logging.info(f\"_test_whiten(): proportion = {proportion}\")\n x = torch.randn(100, 128)\n direction = torch.randn(128)\n coeffs = torch.randn(100, 1)\n x += proportion * direction * coeffs\n\n x.requires_grad = True\n\n num_channels = 128\n m = Whiten(\n 1, 5.0, prob=1.0, grad_scale=0.1 # num_groups # whitening_limit,\n ) # grad_scale\n\n for _ in range(4):\n y = m(x)\n\n y_grad = torch.randn_like(x)\n y.backward(gradient=y_grad)\n\n if proportion < 0.2:\n assert torch.allclose(x.grad, y_grad)\n elif proportion > 1.0:\n assert not torch.allclose(x.grad, y_grad)\n\n\ndef _test_activation_balancer_sign():\n probs = torch.arange(0, 1, 0.01)\n N = 1000\n x = 1.0 * (\n (2.0 * (torch.rand(probs.numel(), N) < probs.unsqueeze(-1))) - 1.0\n )\n x = x.detach()\n x.requires_grad = True\n m = ActivationBalancer(\n probs.numel(),\n channel_dim=0,\n min_positive=0.05,\n max_positive=0.95,\n max_factor=0.2,\n min_abs=0.0,\n )\n\n y_grad = torch.sign(torch.randn(probs.numel(), N))\n\n y = m(x)\n y.backward(gradient=y_grad)\n print(\"_test_activation_balancer_sign: x = \", x)\n print(\"_test_activation_balancer_sign: y grad = \", y_grad)\n print(\"_test_activation_balancer_sign: x grad = \", x.grad)\n\n\ndef _test_activation_balancer_magnitude():\n magnitudes = torch.arange(0, 1, 0.01)\n N = 1000\n x = torch.sign(torch.randn(magnitudes.numel(), N)) * magnitudes.unsqueeze(\n -1\n )\n x = x.detach()\n x.requires_grad = True\n m = ActivationBalancer(\n magnitudes.numel(),\n channel_dim=0,\n min_positive=0.0,\n max_positive=1.0,\n max_factor=0.2,\n min_abs=0.2,\n max_abs=0.8,\n min_prob=1.0,\n )\n\n y_grad = torch.sign(torch.randn(magnitudes.numel(), N))\n\n y = m(x)\n y.backward(gradient=y_grad)\n print(\"_test_activation_balancer_magnitude: x = \", x)\n print(\"_test_activation_balancer_magnitude: y grad = \", y_grad)\n print(\"_test_activation_balancer_magnitude: x grad = \", x.grad)\n\n\ndef _test_basic_norm():\n num_channels = 128\n m = BasicNorm(num_channels=num_channels, channel_dim=1)\n\n x = torch.randn(500, num_channels)\n\n y = m(x)\n\n assert y.shape == x.shape\n x_rms = (x ** 2).mean().sqrt()\n y_rms = (y ** 2).mean().sqrt()\n print(\"x rms = \", x_rms)\n print(\"y rms = \", y_rms)\n assert y_rms < x_rms\n assert y_rms > 0.5 * x_rms\n\n\ndef _test_double_swish_deriv():\n x = torch.randn(10, 12, dtype=torch.double) * 3.0\n x.requires_grad = True\n m = DoubleSwish()\n\n tol = (1.2 - (-0.043637)) / 255.0\n torch.autograd.gradcheck(m, x, atol=tol)\n\n # for self-test.\n x = torch.randn(1000, 1000, dtype=torch.double) * 3.0\n x.requires_grad = True\n y = m(x)\n\n\ndef _test_softmax():\n a = torch.randn(2, 10, dtype=torch.float64)\n b = a.clone()\n a.requires_grad = True\n b.requires_grad = True\n a.softmax(dim=1)[:, 0].sum().backward()\n print(\"a grad = \", a.grad)\n softmax(b, dim=1)[:, 0].sum().backward()\n print(\"b grad = \", b.grad)\n assert torch.allclose(a.grad, b.grad)\n\n\nif __name__ == \"__main__\":\n logging.getLogger().setLevel(logging.INFO)\n torch.set_num_threads(1)\n torch.set_num_interop_threads(1)\n _test_softmax()\n _test_whiten()\n _test_max_eig()\n _test_activation_balancer_sign()\n _test_activation_balancer_magnitude()\n _test_basic_norm()\n _test_double_swish_deriv()" }, { "path": "models/modules/transformer.py", "content": "# cp from https://github.com/lifeiteng/vall-e/blob/main/valle/modules/transformer.py, modified by Puyuan Peng 2024\nimport copy\nimport numbers\nfrom functools import partial\nfrom typing import Any, Callable, List, Optional, Tuple, Union\n\nimport torch\nfrom torch import Tensor, nn\nfrom torch.nn import functional as F\n\nfrom .activation import MultiheadAttention\nfrom .scaling import ActivationBalancer, BalancedDoubleSwish\nfrom .scaling import BasicNorm as _BasicNorm\n\n_shape_t = Union[int, List[int], torch.Size]\n\n\nclass LayerNorm(nn.Module):\n __constants__ = [\"normalized_shape\", \"eps\", \"elementwise_affine\"]\n normalized_shape: Tuple[int, ...]\n eps: float\n elementwise_affine: bool\n\n def __init__(\n self,\n normalized_shape: _shape_t,\n eps: float = 1e-5,\n elementwise_affine: bool = True,\n device=None,\n dtype=None,\n ) -> None:\n factory_kwargs = {\"device\": device, \"dtype\": dtype}\n super(LayerNorm, self).__init__()\n if isinstance(normalized_shape, numbers.Integral):\n # mypy error: incompatible types in assignment\n normalized_shape = (normalized_shape,) # type: ignore[assignment]\n self.normalized_shape = tuple(normalized_shape) # type: ignore[arg-type]\n self.eps = eps\n self.elementwise_affine = elementwise_affine\n if self.elementwise_affine:\n self.weight = nn.Parameter(\n torch.empty(self.normalized_shape, **factory_kwargs)\n )\n self.bias = nn.Parameter(\n torch.empty(self.normalized_shape, **factory_kwargs)\n )\n else:\n self.register_parameter(\"weight\", None)\n self.register_parameter(\"bias\", None)\n\n self.reset_parameters()\n\n def reset_parameters(self) -> None:\n if self.elementwise_affine:\n nn.init.ones_(self.weight)\n nn.init.zeros_(self.bias)\n\n def forward(self, input: Tensor, embedding: Any = None) -> Tensor:\n if isinstance(input, tuple):\n input, embedding = input\n return (\n F.layer_norm(\n input,\n self.normalized_shape,\n self.weight,\n self.bias,\n self.eps,\n ),\n embedding,\n )\n\n assert embedding is None\n return F.layer_norm(\n input, self.normalized_shape, self.weight, self.bias, self.eps\n )\n\n def extra_repr(self) -> str:\n return (\n \"{normalized_shape}, eps={eps}, \"\n \"elementwise_affine={elementwise_affine}\".format(**self.__dict__)\n )\n\n\nclass AdaptiveLayerNorm(nn.Module):\n r\"\"\"Adaptive Layer Normalization\"\"\"\n\n def __init__(self, d_model, norm) -> None:\n super(AdaptiveLayerNorm, self).__init__()\n self.project_layer = nn.Linear(d_model, 2 * d_model)\n self.norm = norm\n self.d_model = d_model\n self.eps = self.norm.eps\n\n def forward(self, input: Tensor, embedding: Tensor = None) -> Tensor:\n if isinstance(input, tuple):\n input, embedding = input\n weight, bias = torch.split(\n self.project_layer(embedding),\n split_size_or_sections=self.d_model,\n dim=-1,\n )\n return (weight * self.norm(input) + bias, embedding)\n\n weight, bias = torch.split(\n self.project_layer(embedding),\n split_size_or_sections=self.d_model,\n dim=-1,\n )\n return weight * self.norm(input) + bias\n\n\nclass BasicNorm(_BasicNorm):\n def __init__(\n self,\n d_model: int,\n eps: float = 1e-5,\n device=None,\n dtype=None,\n ):\n super(BasicNorm, self).__init__(d_model, eps=eps)\n\n def forward(self, input: Tensor, embedding: Any = None) -> Tensor:\n if isinstance(input, tuple):\n input, embedding = input\n return (\n super(BasicNorm, self).forward(input),\n embedding,\n )\n\n assert embedding is None\n return super(BasicNorm, self).forward(input)\n\n\nclass BalancedBasicNorm(nn.Module):\n def __init__(\n self,\n d_model: int,\n eps: float = 1e-5,\n device=None,\n dtype=None,\n ):\n super(BalancedBasicNorm, self).__init__()\n self.balancer = ActivationBalancer(\n d_model,\n channel_dim=-1,\n min_positive=0.45,\n max_positive=0.55,\n max_abs=6.0,\n )\n self.norm = BasicNorm(d_model, eps, device=device, dtype=dtype)\n\n def forward(self, input: Tensor, embedding: Any = None) -> Tensor:\n if isinstance(input, tuple):\n input, embedding = input\n return self.norm((self.balancer(input), embedding))\n\n assert embedding is None\n return self.norm(self.balancer(input))\n\n\nclass IdentityNorm(nn.Module):\n def __init__(\n self,\n d_model: int,\n eps: float = 1e-5,\n device=None,\n dtype=None,\n ) -> None:\n super(IdentityNorm, self).__init__()\n\n def forward(self, input: Tensor, embedding: Any = None) -> Tensor:\n if isinstance(input, tuple):\n return input\n\n assert embedding is None\n return input\n\n\nclass TransformerEncoderLayer(nn.Module):\n __constants__ = [\"batch_first\", \"norm_first\"]\n\n def __init__(\n self,\n d_model: int,\n nhead: int,\n dim_feedforward: int = 2048,\n dropout: float = 0.1,\n activation: Union[str, Callable[[Tensor], Tensor]] = F.relu,\n batch_first: bool = False,\n norm_first: bool = False,\n device=None,\n dtype=None,\n linear1_self_attention_cls: nn.Module = nn.Linear,\n linear2_self_attention_cls: nn.Module = nn.Linear,\n linear1_feedforward_cls: nn.Module = nn.Linear,\n linear2_feedforward_cls: nn.Module = nn.Linear,\n layer_norm_cls: nn.Module = LayerNorm,\n layer_norm_eps: float = 1e-5,\n adaptive_layer_norm=False,\n ) -> None:\n factory_kwargs = {\"device\": device, \"dtype\": dtype}\n super(TransformerEncoderLayer, self).__init__()\n self.self_attn = MultiheadAttention(\n d_model,\n nhead,\n dropout=dropout,\n batch_first=batch_first,\n linear1_cls=linear1_self_attention_cls,\n linear2_cls=linear2_self_attention_cls,\n **factory_kwargs,\n )\n\n # Implementation of Feedforward model\n self.linear1 = linear1_feedforward_cls(\n d_model, dim_feedforward, **factory_kwargs\n )\n self.dropout = nn.Dropout(dropout)\n self.linear2 = linear2_feedforward_cls(\n dim_feedforward, d_model, **factory_kwargs\n )\n\n self.norm_first = norm_first\n self.dropout1 = nn.Dropout(dropout)\n self.dropout2 = nn.Dropout(dropout)\n\n # Legacy string support for activation function.\n if isinstance(activation, str):\n activation = _get_activation_fn(activation)\n elif isinstance(activation, partial):\n activation = activation(d_model)\n elif activation == BalancedDoubleSwish:\n activation = BalancedDoubleSwish(d_model)\n\n # # We can't test self.activation in forward() in TorchScript,\n # # so stash some information about it instead.\n # if activation is F.relu or isinstance(activation, torch.nn.ReLU):\n # self.activation_relu_or_gelu = 1\n # elif activation is F.gelu or isinstance(activation, torch.nn.GELU):\n # self.activation_relu_or_gelu = 2\n # else:\n # self.activation_relu_or_gelu = 0\n self.activation = activation\n\n norm1 = layer_norm_cls(d_model, eps=layer_norm_eps, **factory_kwargs)\n if layer_norm_cls == IdentityNorm:\n norm2 = BalancedBasicNorm(\n d_model, eps=layer_norm_eps, **factory_kwargs\n )\n else:\n norm2 = layer_norm_cls(\n d_model, eps=layer_norm_eps, **factory_kwargs\n )\n\n if adaptive_layer_norm:\n self.norm1 = AdaptiveLayerNorm(d_model, norm1)\n self.norm2 = AdaptiveLayerNorm(d_model, norm2)\n else:\n self.norm1 = norm1\n self.norm2 = norm2\n\n def __setstate__(self, state):\n super(TransformerEncoderLayer, self).__setstate__(state)\n if not hasattr(self, \"activation\"):\n self.activation = F.relu\n\n def forward(\n self,\n src: Tensor,\n src_mask: Optional[Tensor] = None,\n src_key_padding_mask: Optional[Tensor] = None,\n need_weights: Optional[bool] = False,\n past: Optional[Tensor] = None,\n ) -> Tensor:\n r\"\"\"Pass the input through the encoder layer.\n\n Args:\n src: the sequence to the encoder layer (required).\n src_mask: the mask for the src sequence (optional).\n src_key_padding_mask: the mask for the src keys per batch (optional).\n\n Shape:\n see the docs in Transformer class.\n \"\"\"\n x, stage_embedding = src, None\n is_src_tuple = False\n if isinstance(src, tuple):\n x, stage_embedding = src\n is_src_tuple = True\n\n if src_key_padding_mask is not None:\n _skpm_dtype = src_key_padding_mask.dtype\n if _skpm_dtype != torch.bool and not torch.is_floating_point(\n src_key_padding_mask\n ):\n raise AssertionError(\n \"only bool and floating types of key_padding_mask are supported\"\n )\n if need_weights:\n if self.norm_first:\n out, attn = self._sa_block_attn(\n self.norm1(x, stage_embedding),\n src_mask,\n src_key_padding_mask,\n past\n )\n out, present = out # present is the kvcache of the present timestep\n x = x + out\n x = x + self._ff_block(self.norm2(x, stage_embedding))\n else:\n out, attn = self._sa_block_attn(x, src_mask, src_key_padding_mask, past)\n out, present = out # present is the kvcache of the present timestep\n x = self.norm1(\n x + out,\n stage_embedding,\n )\n x = self.norm2(x + self._ff_block(x), stage_embedding)\n assert not is_src_tuple\n # return (x, stage_embedding)\n return (x, attn)\n else:\n if self.norm_first:\n out = self._sa_block(\n self.norm1(x, stage_embedding),\n src_mask,\n src_key_padding_mask, past\n )\n out, present = out # present is the kvcache of the present timestep\n x = x + out\n x = x + self._ff_block(self.norm2(x, stage_embedding))\n else:\n out = self._sa_block(x, src_mask, src_key_padding_mask)\n out, present = out # present is the kvcache of the present timestep\n x = self.norm1(\n x + out,\n stage_embedding, past\n )\n x = self.norm2(x + self._ff_block(x), stage_embedding)\n \n if is_src_tuple:\n x = (x, stage_embedding)\n if present != None:\n x = [x, present]\n return x\n\n # self-attention block\n def _sa_block(\n self,\n x: Tensor,\n attn_mask: Optional[Tensor],\n key_padding_mask: Optional[Tensor],\n past: Optional[Tensor] = None,\n ) -> Tensor:\n x = self.self_attn(\n x,\n x,\n x,\n attn_mask=attn_mask,\n key_padding_mask=key_padding_mask,\n need_weights=False,\n past=past\n )\n x, present = x\n return self.dropout1(x), present\n\n # self-attention block, also return attention weights\n def _sa_block_attn(\n self,\n x: Tensor,\n attn_mask: Optional[Tensor],\n key_padding_mask: Optional[Tensor],\n past: Optional[Tensor] = None,\n ) -> Tensor:\n x, attn = self.self_attn(\n x,\n x,\n x,\n attn_mask=attn_mask,\n key_padding_mask=key_padding_mask,\n need_weights=True,\n past=past\n )\n x, present = x\n return (self.dropout1(x), present), attn\n\n # feed forward block\n def _ff_block(self, x: Tensor) -> Tensor:\n x = self.linear2(self.dropout(self.activation(self.linear1(x))))\n return self.dropout2(x)\n\n\nclass TransformerEncoder(nn.Module):\n r\"\"\"TransformerEncoder is a stack of N encoder layers. Users can build the\n BERT(https://arxiv.org/abs/1810.04805) model with corresponding parameters.\n\n Args:\n encoder_layer: an instance of the TransformerEncoderLayer() class (required).\n num_layers: the number of sub-encoder-layers in the encoder (required).\n norm: the layer normalization component (optional).\n enable_nested_tensor: if True, input will automatically convert to nested tensor\n (and convert back on output). This will improve the overall performance of\n TransformerEncoder when padding rate is high. Default: ``True`` (enabled).\n\n Examples::\n >>> encoder_layer = TransformerEncoderLayer(d_model=512, nhead=8)\n >>> transformer_encoder = TransformerEncoder(encoder_layer, num_layers=6)\n >>> src = torch.rand(10, 32, 512)\n >>> out = transformer_encoder(src)\n \"\"\"\n __constants__ = [\"norm\"]\n\n def __init__(self, encoder_layer, num_layers, norm=None):\n super(TransformerEncoder, self).__init__()\n self.layers = _get_clones(encoder_layer, num_layers)\n self.num_layers = num_layers\n self.norm = norm\n\n def forward(\n self,\n src: Tensor,\n mask: Optional[Tensor] = None,\n src_key_padding_mask: Optional[Tensor] = None,\n return_layer_states: bool = False,\n need_weights:Optional[bool] = False,\n past: Optional[Tensor] = None,\n ) -> Tensor:\n r\"\"\"Pass the input through the encoder layers in turn.\n\n Args:\n src: the sequence to the encoder (required).\n mask: the mask for the src sequence (optional).\n src_key_padding_mask: the mask for the src keys per batch (optional).\n return_layer_states: return layers' state (optional).\n\n Shape:\n see the docs in Transformer class.\n \"\"\"\n if return_layer_states:\n assert not need_weights\n layer_states = [] # layers' output\n output = src\n for mod in self.layers:\n output = mod(\n output,\n src_mask=mask,\n src_key_padding_mask=src_key_padding_mask,\n past=past\n )\n layer_states.append(output[0])\n\n if self.norm is not None:\n output = self.norm(output)\n\n return layer_states, output\n if need_weights:\n assert not return_layer_states\n layer_attn = [] # layers' output\n output = src\n for mod in self.layers:\n output = mod(\n output,\n src_mask=mask,\n src_key_padding_mask=src_key_padding_mask,\n need_weights=True,\n past=past\n )\n layer_attn.append(output[1])\n\n if self.norm is not None:\n output = self.norm(output)\n\n return layer_attn, output\n \n output = src\n all_present = []\n for n_layer, mod in enumerate(self.layers):\n output = mod(\n output, src_mask=mask, src_key_padding_mask=src_key_padding_mask, past=None if past is None else past[n_layer]\n )\n if isinstance(output, list):\n output, present = output\n all_present.append(present)\n\n if self.norm is not None:\n output = self.norm(output)\n if all_present != []:\n all_present = torch.stack(all_present, dim=0) # (num_layers, 2, batch_size, num_heads, seq_len, head_dim)\n output = [output, all_present]\n return output\n\n\nclass TransformerDecoderLayer(nn.Module):\n __constants__ = [\"batch_first\", \"norm_first\"]\n\n def __init__(\n self,\n d_model: int,\n nhead: int,\n dim_feedforward: int = 2048,\n dropout: float = 0.1,\n activation: Union[str, Callable[[Tensor], Tensor]] = F.relu,\n linear1_self_attention_cls: nn.Module = nn.Linear,\n linear2_self_attention_cls: nn.Module = nn.Linear,\n linear1_feedforward_cls: nn.Module = nn.Linear,\n linear2_feedforward_cls: nn.Module = nn.Linear,\n batch_first: bool = False,\n norm_first: bool = False,\n device=None,\n dtype=None,\n layer_norm_cls: nn.Module = LayerNorm,\n layer_norm_eps: float = 1e-5,\n adaptive_layer_norm=False,\n ) -> None:\n factory_kwargs = {\"device\": device, \"dtype\": dtype}\n super(TransformerDecoderLayer, self).__init__()\n self.self_attn = MultiheadAttention(\n d_model,\n nhead,\n dropout=dropout,\n batch_first=batch_first,\n linear1_cls=linear1_self_attention_cls,\n linear2_cls=linear2_self_attention_cls,\n **factory_kwargs,\n )\n self.multihead_attn = MultiheadAttention(\n d_model,\n nhead,\n dropout=dropout,\n batch_first=batch_first,\n linear1_cls=linear1_self_attention_cls,\n linear2_cls=linear2_self_attention_cls,\n **factory_kwargs,\n )\n # Implementation of Feedforward model\n self.linear1 = linear1_feedforward_cls(\n d_model, dim_feedforward, **factory_kwargs\n )\n self.dropout = nn.Dropout(dropout)\n self.linear2 = linear2_feedforward_cls(\n dim_feedforward, d_model, **factory_kwargs\n )\n\n self.norm_first = norm_first\n self.dropout1 = nn.Dropout(dropout)\n self.dropout2 = nn.Dropout(dropout)\n self.dropout3 = nn.Dropout(dropout)\n\n # Legacy string support for activation function.\n if isinstance(activation, str):\n self.activation = _get_activation_fn(activation)\n elif isinstance(activation, partial):\n self.activation = activation(d_model)\n elif activation == BalancedDoubleSwish:\n self.activation = BalancedDoubleSwish(d_model)\n else:\n self.activation = activation\n\n if adaptive_layer_norm:\n norm1 = layer_norm_cls(\n d_model, eps=layer_norm_eps, **factory_kwargs\n )\n norm2 = layer_norm_cls(\n d_model, eps=layer_norm_eps, **factory_kwargs\n )\n norm3 = layer_norm_cls(\n d_model, eps=layer_norm_eps, **factory_kwargs\n )\n\n self.norm1 = AdaptiveLayerNorm(d_model, norm1)\n self.norm2 = AdaptiveLayerNorm(d_model, norm2)\n self.norm3 = AdaptiveLayerNorm(d_model, norm3)\n else:\n self.norm1 = layer_norm_cls(\n d_model, eps=layer_norm_eps, **factory_kwargs\n )\n self.norm2 = layer_norm_cls(\n d_model, eps=layer_norm_eps, **factory_kwargs\n )\n if layer_norm_cls == IdentityNorm:\n self.norm3 = BalancedBasicNorm(\n d_model, eps=layer_norm_eps, **factory_kwargs\n )\n else:\n self.norm3 = layer_norm_cls(\n d_model, eps=layer_norm_eps, **factory_kwargs\n )\n\n def forward(\n self,\n tgt: Tensor,\n memory: Tensor,\n tgt_mask: Optional[Tensor] = None,\n memory_mask: Optional[Tensor] = None,\n tgt_key_padding_mask: Optional[Tensor] = None,\n memory_key_padding_mask: Optional[Tensor] = None,\n ) -> Tensor:\n r\"\"\"Pass the inputs (and mask) through the decoder layer.\n\n Args:\n tgt: the sequence to the decoder layer (required).\n memory: the sequence from the last layer of the encoder (required).\n tgt_mask: the mask for the tgt sequence (optional).\n memory_mask: the mask for the memory sequence (optional).\n tgt_key_padding_mask: the mask for the tgt keys per batch (optional).\n memory_key_padding_mask: the mask for the memory keys per batch (optional).\n\n Shape:\n see the docs in Transformer class.\n \"\"\"\n tgt_is_tuple = False\n if isinstance(tgt, tuple):\n x, stage_embedding = tgt\n tgt_is_tuple = True\n else:\n x, stage_embedding = tgt, None\n\n if self.norm_first:\n x = x + self._sa_block(\n self.norm1(x, stage_embedding), tgt_mask, tgt_key_padding_mask\n )\n x = x + self._mha_block(\n self.norm2(x, stage_embedding),\n memory,\n memory_mask,\n memory_key_padding_mask,\n )\n x = x + self._ff_block(self.norm3(x, stage_embedding))\n else:\n x = self.norm1(\n x + self._sa_block(x, tgt_mask, tgt_key_padding_mask),\n stage_embedding,\n )\n x = self.norm2(\n x\n + self._mha_block(\n x, memory, memory_mask, memory_key_padding_mask\n ),\n stage_embedding,\n )\n x = self.norm3(x + self._ff_block(x), stage_embedding)\n\n if tgt_is_tuple:\n return (x, stage_embedding)\n return x\n\n # self-attention block\n def _sa_block(\n self,\n x: Tensor,\n attn_mask: Optional[Tensor],\n key_padding_mask: Optional[Tensor],\n ) -> Tensor:\n x = self.self_attn(\n x,\n x,\n x,\n attn_mask=attn_mask,\n key_padding_mask=key_padding_mask,\n need_weights=False,\n )[0]\n return self.dropout1(x)\n\n # multihead attention block\n def _mha_block(\n self,\n x: Tensor,\n mem: Tensor,\n attn_mask: Optional[Tensor],\n key_padding_mask: Optional[Tensor],\n ) -> Tensor:\n x = self.multihead_attn(\n x,\n mem,\n mem,\n attn_mask=attn_mask,\n key_padding_mask=key_padding_mask,\n need_weights=False,\n )[0]\n return self.dropout2(x)\n\n # feed forward block\n def _ff_block(self, x: Tensor) -> Tensor:\n x = self.linear2(self.dropout(self.activation(self.linear1(x))))\n return self.dropout3(x)\n\n\ndef _get_clones(module, N):\n return nn.ModuleList([copy.deepcopy(module) for i in range(N)])\n\n\ndef _get_activation_fn(activation: str) -> Callable[[Tensor], Tensor]:\n if activation == \"relu\":\n return F.relu\n elif activation == \"gelu\":\n return F.gelu\n\n raise RuntimeError(\n \"activation should be relu/gelu, not {}\".format(activation)\n )" }, { "path": "models/modules/utils.py", "content": "# cp from https://github.com/lifeiteng/vall-e/blob/main/valle/modules/transformer.py, modified by Puyuan Peng\nimport torch\n\n\ndef make_pad_mask(lengths: torch.Tensor, max_len: int = 0) -> torch.Tensor:\n \"\"\"\n Args:\n lengths:\n A 1-D tensor containing sentence lengths.\n max_len:\n The length of masks.\n Returns:\n Return a 2-D bool tensor, where masked positions\n are filled with `True` and non-masked positions are\n filled with `False`.\n\n >>> lengths = torch.tensor([1, 3, 2, 5])\n >>> make_pad_mask(lengths)\n tensor([[False, True, True, True, True],\n [False, False, False, True, True],\n [False, False, True, True, True],\n [False, False, False, False, False]])\n \"\"\"\n assert lengths.ndim == 1, lengths.ndim\n max_len = max(max_len, lengths.max())\n n = lengths.size(0)\n seq_range = torch.arange(0, max_len, device=lengths.device)\n expaned_lengths = seq_range.unsqueeze(0).expand(n, max_len)\n\n return expaned_lengths >= lengths.unsqueeze(-1)\n\ndef generate_partial_autoregressive_mask(sz, start, end):\n mask = torch.zeros(sz, sz).bool()\n mask[start:end, start:end] = torch.triu(torch.ones(end-start, end-start,dtype=torch.bool), diagonal=1)\n mask[:start, start:end] = True\n mask[end:, start:end] = True\n return mask\n" }, { "path": "models/voicecraft.py", "content": "import random\n\nimport numpy as np\nimport logging\nimport argparse, copy\nfrom typing import Dict, Optional\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torchmetrics.classification import MulticlassAccuracy\n\nfrom .modules.utils import make_pad_mask\n\nfrom .modules.embedding import SinePositionalEmbedding, TokenEmbedding\nfrom .modules.transformer import (\n LayerNorm,\n TransformerEncoder,\n TransformerEncoderLayer,\n)\nfrom .codebooks_patterns import DelayedPatternProvider\n\nfrom argparse import Namespace\nfrom huggingface_hub import PyTorchModelHubMixin\n\n\ndef top_k_top_p_filtering(\n logits, top_k=0, top_p=1.0, filter_value=-float(\"Inf\"), min_tokens_to_keep=1\n):\n \"\"\"Filter a distribution of logits using top-k and/or nucleus (top-p) filtering\n Args:\n logits: logits distribution shape (batch size, vocabulary size)\n if top_k > 0: keep only top k tokens with highest probability (top-k filtering).\n if top_p < 1.0: keep the top tokens with cumulative probability >= top_p (nucleus filtering).\n Nucleus filtering is described in Holtzman et al. (http://arxiv.org/abs/1904.09751)\n Make sure we keep at least min_tokens_to_keep per batch example in the output\n From: https://gist.github.com/thomwolf/1a5a29f6962089e871b94cbd09daf317\n \"\"\"\n if top_k > 0:\n top_k = min(\n max(top_k, min_tokens_to_keep), logits.size(-1)\n ) # Safety check\n # Remove all tokens with a probability less than the last token of the top-k\n indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None]\n logits[indices_to_remove] = filter_value\n\n if top_p < 1.0:\n sorted_logits, sorted_indices = torch.sort(logits, descending=True)\n cumulative_probs = torch.cumsum(\n F.softmax(sorted_logits, dim=-1), dim=-1\n )\n\n # Remove tokens with cumulative probability above the threshold (token with 0 are kept)\n sorted_indices_to_remove = cumulative_probs > top_p\n if min_tokens_to_keep > 1:\n # Keep at least min_tokens_to_keep (set to min_tokens_to_keep-1 because we add the first one below)\n sorted_indices_to_remove[..., :min_tokens_to_keep] = 0\n # Shift the indices to the right to keep also the first token above the threshold\n sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[\n ..., :-1\n ].clone()\n sorted_indices_to_remove[..., 0] = 0\n\n # scatter sorted tensors to original indexing\n indices_to_remove = sorted_indices_to_remove.scatter(\n 1, sorted_indices, sorted_indices_to_remove\n )\n logits[indices_to_remove] = filter_value\n return logits\n\n\ndef topk_sampling(logits, top_k=10, top_p=1.0, temperature=1.0):\n # temperature: (`optional`) float\n # The value used to module the next token probabilities. Must be strictly positive. Default to 1.0.\n # top_k: (`optional`) int\n # The number of highest probability vocabulary tokens to keep for top-k-filtering. Between 1 and infinity. Default to 50.\n # top_p: (`optional`) float\n # The cumulative probability of parameter highest probability vocabulary tokens to keep for nucleus sampling. Must be between 0 and 1. Default to 1.\n\n # Temperature (higher temperature => more likely to sample low probability tokens)\n if temperature != 1.0:\n logits = logits / temperature\n # Top-p/top-k filtering\n logits = top_k_top_p_filtering(logits, top_k=top_k, top_p=top_p)\n # Sample\n token = torch.multinomial(F.softmax(logits, dim=-1), num_samples=1)\n return token\n\n\n\nclass VoiceCraft(\n nn.Module,\n PyTorchModelHubMixin,\n library_name=\"voicecraft\",\n repo_url=\"https://github.com/jasonppy/VoiceCraft\",\n tags=[\"text-to-speech\"],\n ):\n def __new__(cls, args: Optional[Namespace] = None, config: Optional[Dict] = None, **kwargs) -> \"VoiceCraft\":\n # If initialized from Namespace args => convert to dict config for 'PyTorchModelHubMixin' to serialize it as config.json\n # Won't affect instance initialization\n if args is not None:\n if config is not None:\n raise ValueError(\"Cannot provide both `args` and `config`.\")\n config = vars(args)\n return super().__new__(cls, args=args, config=config, **kwargs)\n\n def __init__(self, args: Optional[Namespace] = None, config: Optional[Dict] = None):\n super().__init__()\n\n # If loaded from HF Hub => convert config.json to Namespace args before initializing\n if args is None:\n if config is None:\n raise ValueError(\"Either `args` or `config` must be provided.\")\n args = Namespace(**config)\n\n self.args = copy.copy(args)\n self.pattern = DelayedPatternProvider(n_q=self.args.n_codebooks)\n if not getattr(self.args, \"special_first\", False):\n self.args.special_first = 0\n if not getattr(self.args, \"n_special\", False):\n self.args.n_special = 3\n self.args.eos = getattr(self.args, \"eos\", -1)\n self.eog = nn.Parameter(torch.full((self.args.n_codebooks, 1), self.args.eog, dtype=torch.long), requires_grad=False) # [K 1]\n if self.args.eos > 0:\n assert self.args.eos != self.args.audio_pad_token and self.args.eos != self.args.empty_token, self.args.eos\n self.eos = nn.Parameter(torch.full((self.args.n_codebooks, 1), self.args.eos, dtype=torch.long), requires_grad=False) # [K 1]\n if isinstance(self.args.audio_vocab_size, str):\n self.args.audio_vocab_size = eval(self.args.audio_vocab_size)\n\n self.n_text_tokens = self.args.text_vocab_size + 1\n assert self.args.text_pad_token == self.args.text_vocab_size, f\"self.args.text_vocab_size: {self.args.text_vocab_size}, self.args.text_pad_token: {self.args.text_pad_token}\"\n\n self.n_audio_tokens = [self.args.audio_vocab_size + self.args.n_special] * self.args.n_codebooks # special tokens: empty token, EOG token, audio pad token\n assert self.args.audio_vocab_size == self.args.empty_token, self.args.empty_token\n assert self.args.eog == self.args.audio_vocab_size + 1, self.args.eog\n assert self.args.audio_pad_token == self.args.audio_vocab_size + 2, self.args.audio_pad_token\n\n self.text_embedding = TokenEmbedding(\n dim_model=self.args.d_model,\n vocab_size=self.n_text_tokens, \n dropout=self.args.text_embedding_dropout\n )\n\n self.audio_embedding = nn.ModuleList(\n [\n TokenEmbedding(\n dim_model=self.args.audio_embedding_dim, \n vocab_size=self.n_audio_tokens[k], \n dropout=self.args.audio_embedding_dropout\n ) for k in range(self.args.n_codebooks)\n ]\n )\n self.mask_embedding = nn.Parameter(torch.randn(self.args.max_n_spans, self.args.d_model), requires_grad=True)\n self.text_positional_embedding = SinePositionalEmbedding(\n self.args.d_model,\n dropout=self.args.text_positional_embedding_dropout,\n scale=False,\n alpha=True, # learnable scaler, scale the volume of positional embedding\n )\n self.audio_positional_embedding = SinePositionalEmbedding(\n self.args.d_model,\n dropout=self.args.audio_positional_embedding_dropout,\n scale=False,\n alpha=True, # learnable scaler, scale the volume of positional embedding\n )\n\n dec_layer = TransformerEncoderLayer(\n self.args.d_model,\n self.args.nhead,\n dim_feedforward=self.args.d_model * 4,\n dropout=self.args.trm_dropout,\n batch_first=True,\n norm_first=True,\n layer_norm_cls=LayerNorm\n )\n self.decoder = TransformerEncoder(\n dec_layer,\n num_layers=self.args.num_decoder_layers,\n norm=LayerNorm(self.args.d_model),\n )\n \n self.predict_layer = nn.ModuleList(\n [\n nn.Sequential(nn.Linear(self.args.d_model, self.args.audio_vocab_size//2), nn.GELU(), nn.Linear(self.args.audio_vocab_size//2, self.n_audio_tokens[k])) for k in range(self.args.n_codebooks)\n ]\n )\n \n self.accuracy_metrics = nn.ModuleList(\n [MulticlassAccuracy(\n self.n_audio_tokens[k],\n top_k=10,\n average=\"micro\",\n multidim_average=\"global\",\n ignore_index=None,\n ) for k in range(self.args.n_codebooks)]\n )\n\n \n def prepare_mask_intervals(self, y_lens):\n mask_intervals = []\n non_mask_intervals = []\n\n for i, y_len in enumerate(y_lens):\n if self.args.mask_sample_dist == \"uniform\":\n n_spans = random.choice(range(1, self.args.max_n_spans+1))\n elif \"poisson\" in self.args.mask_sample_dist.lower():\n param = float(self.args.mask_sample_dist[len(\"poisson\"):])\n poisson_sample = torch.poisson(torch.tensor([param]))\n n_spans = int(poisson_sample.clamp(1, self.args.max_n_spans).item())\n\n starts = random.sample(range(1, y_len-1-self.args.mask_len_min), n_spans)\n starts = sorted(starts)\n\n for j in range(len(starts)-1, 0, -1):\n if starts[j] - starts[j-1] < self.args.min_gap:\n del starts[j] # If elements are too close, delete the later one\n assert len(starts) > 0, f\"there is no masked span left, y_len: {y_len}, sampled n_spans: {n_spans}\"\n\n temp_starts = starts + [y_len]\n gaps = [temp_starts[j+1] - temp_starts[j] for j in range(len(temp_starts)-1)]\n\n ends = []\n\n for j, (start, gap) in enumerate(zip(starts, gaps)):\n mask_len = random.randint(self.args.mask_len_min, self.args.mask_len_max)\n # if mask_len > gap * self.args.max_mask_portion: # make sure the masks are not overlapping with each other\n if mask_len > gap - 1: # make sure the masks are not overlapping with each other\n # temp_mask_start = int(0.6*gap*self.args.max_mask_portion)\n # temp_mask_end = int(gap*self.args.max_mask_portion)\n temp_mask_start = 1\n temp_mask_end = gap - 1\n mask_len = random.randint(temp_mask_start, temp_mask_end)\n ends.append(start + mask_len)\n \n mask_intervals.append([(s,e) for s,e in zip(starts, ends)])\n non_mask_intervals.append([(ns,ne) for ns, ne in zip([0]+ends, starts+[y_len])])\n\n return mask_intervals, non_mask_intervals\n \n def rearrange(self, y, non_mask_intervals, mask_intervals):\n reduced_eog = getattr(self.args, \"reduced_eog\", 0)\n rearranged_y = []\n for i in range(len(y)):\n if self.args.eos > 0:\n assert reduced_eog\n cur_y = [y[i, :, item[0]: item[1]] for item in non_mask_intervals[i][:-1]] + [torch.cat([y[i, :, non_mask_intervals[i][-1][0]: non_mask_intervals[i][-1][1]], self.eos], dim=-1)] + [torch.cat([y[i, :, item[0]: item[1]], self.eog], dim=-1) for item in mask_intervals[i]] # only insert eog to the last non-mask-interval, which is when the utterance actual ends\n else:\n if reduced_eog:\n cur_y = [y[i, :, item[0]: item[1]] for item in non_mask_intervals[i][:-1]] + [torch.cat([y[i, :, non_mask_intervals[i][-1][0]: non_mask_intervals[i][-1][1]], self.eog], dim=-1)] + [torch.cat([y[i, :, item[0]: item[1]], self.eog], dim=-1) for item in mask_intervals[i]] # only insert eog to the last non-mask-interval, which is when the utterance actual ends\n else:\n cur_y = [torch.cat([y[i, :, item[0]: item[1]], self.eog], dim=-1) for item in non_mask_intervals[i]] + [torch.cat([y[i, :, item[0]: item[1]], self.eog], dim=-1) for item in mask_intervals[i]] # eog is added to each section TODO this is not correct, I should add eog to non_mask_intervals if that segment is not the ending segment (as there is no way for the model to predict eog for those segments, and this will do harm to tts experiment, where the model randomly output eog for the first segment)\n rearranged_y.append(cur_y)\n return rearranged_y\n \n def shift(self, rearranged_y):\n shifted_y = []\n patterns = []\n for i in range(len(rearranged_y)):\n cur_patterns = [self.pattern.get_pattern(cur_y.shape[1]) for cur_y in rearranged_y[i]]\n out = [cur_pattern.build_pattern_sequence(z=cur_y.unsqueeze(0).contiguous(), special_token=self.args.empty_token, keep_only_valid_steps=False) for cur_pattern, cur_y in zip(cur_patterns, rearranged_y[i])]\n shifted_y.append([item[0].squeeze(0) for item in out]) # the first item is values, later two are indexes and mask\n patterns.append(cur_patterns)\n return shifted_y, patterns\n \n def insert_mask(self, shifted_y):\n inserted_y = []\n mask_position = []\n mask_value = []\n for i in range(len(shifted_y)):\n num_masks = (len(shifted_y[i]) - 1) // 2\n assert num_masks == (len(shifted_y[i]) - 1) / 2, len(shifted_y[i])\n emb_inds = list(range(self.args.max_n_spans))\n if self.args.shuffle_mask_embedding:\n random.shuffle(emb_inds)\n emb_inds_use = emb_inds[:num_masks]\n emb_inds_use = emb_inds_use + emb_inds_use\n mask_value.append(emb_inds_use)\n cur_inserted_y = []\n cur_mask_position = []\n for j in range(len(shifted_y[i])-1):\n cur_inserted_y.append(shifted_y[i][j])\n cur_mask_position.append(sum([item.shape[1] for item in cur_inserted_y])) # each item is of shape [K S], so take shape[1]\n cur_inserted_y.append(self.eog) # insert mask token of shape [K, 1], BUT we are actually using the eog token as a place holder here, as the real mask will be inserted in embed_y function\n\n cur_inserted_y.append(shifted_y[i][-1])\n\n inserted_y.append(cur_inserted_y)\n mask_position.append(cur_mask_position)\n return inserted_y, mask_position, mask_value\n \n def cat_y(self, inserted_y, mask_position, y_lens):\n reduced_eog = getattr(self.args, \"reduced_eog\", 0)\n cated_y = []\n new_y_lens = []\n for i in range(len(inserted_y)):\n cur_cated_y = torch.cat(inserted_y[i], dim=1) #[K S]\n cur_cated_y = cur_cated_y.transpose(1,0) # [S K]\n cur_cated_y_len = cur_cated_y.shape[0]\n if reduced_eog:\n assert cur_cated_y_len == y_lens[i] + len(mask_position[i]) + (len(mask_position[i]) + 1) * self.args.n_codebooks + (len(mask_position[i])/2 + 1), f\"cur_cated_y_len == {cur_cated_y_len}, but it should be y_lens[i] ({y_lens[i]}) + len(mask_position[i]) ({len(mask_position[i])}) + (len(mask_position[i]) + 1) * self.args.n_codebooks ({(len(mask_position[i]) + 1) * self.args.n_codebooks}) + (len(mask_position[i])/2 + 1) ({len(mask_position[i])/2 + 1})={y_lens[i] + len(mask_position[i]) + (len(mask_position[i]) + 1) * self.args.n_codebooks + (len(mask_position[i])/2 + 1)}\"\n else:\n assert cur_cated_y_len == y_lens[i] + len(mask_position[i]) + (len(mask_position[i]) + 1) * self.args.n_codebooks + (len(mask_position[i]) + 1), f\"cur_cated_y_len == {cur_cated_y_len}, but it should be y_lens[i] ({y_lens[i]}) + len(mask_position[i]) ({len(mask_position[i])}) + (len(mask_position[i]) + 1) * self.args.n_codebooks ({(len(mask_position[i]) + 1) * self.args.n_codebooks}) + (len(mask_position[i]) + 1) ({len(mask_position[i]) + 1})\" # the last term represent the inserted eog token, originally it's inserted at the end of every token, but this is wrong\n new_y_lens.append(cur_cated_y_len)\n cated_y.append(cur_cated_y)\n\n cated_y = torch.nn.utils.rnn.pad_sequence(cated_y, batch_first=False, padding_value=self.args.audio_pad_token)\n assert cated_y.shape == torch.Size([max(new_y_lens),len(inserted_y), self.args.n_codebooks]), f\"cated_y.shape: {cated_y.shape}, but it should be {torch.Size([max(new_y_lens,len(inserted_y), self.args.n_codebooks)])}\"\n cated_y = cated_y.permute(2,0,1) # [T,B,K]->[K,T,B]\n assert cated_y.shape[0] == self.args.n_codebooks, cated_y.shape\n return cated_y, torch.LongTensor(new_y_lens).to(cated_y.device)\n\n def embed_y(self, cated_y, mask_position, mask_value):\n embedded_y = torch.stack([self.audio_embedding[k](cated_y[k]) for k in range(self.args.n_codebooks)], dim=0) # [K, T, B, D]\n assert embedded_y.shape[0] == self.args.n_codebooks, embedded_y.shape\n assert embedded_y.shape[-1] == self.args.d_model, embedded_y.shape\n embedded_y = embedded_y.sum(dim=0) # [K,T,B,D]->[T,B,D]\n embedded_y = embedded_y.transpose(1,0) # [T,B,D]->[B,T,D]\n for i in range(len(embedded_y)):\n if len(mask_position[i]) > 0:\n embedded_y[i, mask_position[i]] = self.mask_embedding[mask_value[i]] \n return embedded_y\n \n def prepare_input_target(self, y, y_lens):\n # rearrange y\n # assume y shape: [B T K], K is n_codebooks\n assert y.shape[1] == self.args.n_codebooks, y.shape\n # sample mask_intervals\n mask_intervals, non_mask_intervals = self.prepare_mask_intervals(y_lens)\n\n # need to have EOG in each section (SOG will be generated by the pattern class)\n # but mask can be inserted later after we have shifted the input\n # y could be rearranged in this way:\n # [\n # [tensor[4, 12], tensor[4, 45], tensor[4, 102], tensor[4, 32]], tensor[4, 22]],\n # [tensor[4, 44], tensor[4, 56], tensor[4, 19]],\n # ...\n # ]\n # for the first list of tensors (4 tensors), first 3 tensors are non_masked part, last 2 are masked part.\n # NOTE #non_masked_part = #masked_part + 1\n # NOTE *these are also the targets*\n # added eog at the end of each segment (masked segment and unmasked segment)\n rearranged_y = self.rearrange(y, non_mask_intervals, mask_intervals)\n targets = rearranged_y # each element in each sample is of shape [K T]\n assert targets[0][0].shape[0] == self.args.n_codebooks, targets[0][0].shape\n\n # next we need to apply pattern shifting to each tensor, after which, we'll replace the starting tokens of each section with a token that's different from the special padding token\n # [[5, 1, 2, 3, 4, 5, 5],\n # [5, 5, 1, 2, 3, 4, 5],\n # [5, 5, 5, 1, 2, 3, 4]]\n shifted_y, patterns = self.shift(rearranged_y) # each element [K S]\n assert shifted_y[0][0].shape[0] == self.args.n_codebooks, shifted_y[0][0].shape[0]\n\n\n # then, insert mask token at the intersection of each tensor (we want to decide the arrangement of the mask (shuffle or not)), we better have a separate nn.embedding for it\n # we also need to record the position of the inserted mask\n inserted_y, mask_position, mask_value = self.insert_mask(shifted_y)\n assert inserted_y[0][0].shape[0] == self.args.n_codebooks, inserted_y[0][0].shape[0]\n assert inserted_y[0][1].shape == torch.Size((self.args.n_codebooks, 1)), f\"this should be a mask, so should have shape {(self.args.n_codebooks, 1)}, but it's {inserted_y[0][1].shape}\"\n \n # then concat tensors that belong to the same sample (in order) then get the length of each sample, and then stack them in batch dimension, pad them with pad_token\n cated_y, new_y_lens = self.cat_y(inserted_y, mask_position, y_lens) # KTB\n assert cated_y.shape == torch.Size((self.args.n_codebooks, cated_y.shape[1], len(inserted_y)))\n \n\n # embed remember to separately embed the mask tokens\n embedded_y = self.embed_y(cated_y, mask_position, mask_value) #BTD\n assert embedded_y.shape[1:] == torch.Size((max(new_y_lens), self.args.d_model)), embedded_y.shape\n \n # positional embedding\n y_input = self.audio_positional_embedding(embedded_y)\n\n # make attention mask and padding mask\n y_padding_mask = make_pad_mask(new_y_lens).to(y.device)\n y_attention_mask = torch.triu(torch.ones(y_input.shape[1], y_input.shape[1]), diagonal=1).bool().to(y_padding_mask.device)\n return y_input, new_y_lens, targets, y_padding_mask, y_attention_mask, mask_position, patterns\n\n def remove_mask(self, logits, mask_position, new_y_lens):\n # logits: [B K S card]\n logits_use = []\n for i in range(len(logits)):\n non_mask_positions = [-1] + mask_position[i] + [new_y_lens[i]]\n non_mask_intervals = [[non_mask_positions[i]+1, non_mask_positions[i+1]] for i in range(len(non_mask_positions)-1)]\n cur_logits_use = [logits[i, :, l:r] for l,r in non_mask_intervals]\n logits_use.append(cur_logits_use)\n \n return logits_use\n \n def revert_pattern(self, patterns, logits_use):\n logits_final = []\n logit_masks = []\n for i in range(len(logits_use)):\n cur_logits = [\n item.unsqueeze(0).permute(0, 3, 1, 2).contiguous() for item in logits_use[i]\n ] # each item is of shape [1 K S card] [1 card K S]\n cur_logits_final = [\n cur_pattern.revert_pattern_logits(\n item, 0, keep_only_valid_steps=False\n )\n for cur_pattern, item in zip(patterns[i], cur_logits)\n ] # if input output order doesn't match, this step will give an error\n cur_logits_final_ret = [item[0].permute(0,2,3,1).squeeze(0) for item in cur_logits_final] # each element is of shape [K,T,card]\n logits_final.append(cur_logits_final_ret)\n logit_masks.append([item[2] for item in cur_logits_final])\n\n return logits_final, logit_masks\n\n def dec_forward(\n self, \n x_input, \n x_lens,\n x_attention_mask,\n x_padding_mask,\n y_input,\n new_y_lens,\n y_attention_mask,\n y_padding_mask,\n past=None,\n last_3_tokens=False\n ):\n x_attn_mask = F.pad(\n x_attention_mask,\n (0, new_y_lens.max()),\n value=True,\n ) # x attn to all x, doesn't attn to any y, this follow figure 3 of the valle paper\n y_attn_mask = F.pad(\n y_attention_mask,\n (x_lens.max(), 0), # y is padded at the front\n value=False,\n ) # y attn to all x, for y itself use lower triangle mask to ensure autoregressive\n xy_attn_mask = torch.concat([x_attn_mask, y_attn_mask], dim=0)\n\n # merge key padding and attention masks\n bsz, src_len = x_input.shape[0], x_lens.max() + new_y_lens.max()\n xy_padding_mask = torch.concat([x_padding_mask, y_padding_mask], dim=1)\n _xy_padding_mask = (\n xy_padding_mask.view(bsz, 1, 1, src_len)\n .expand(-1, self.args.nhead, -1, -1)\n .reshape(bsz * self.args.nhead, 1, src_len)\n )\n # Check shapes and resize+broadcast as necessary\n if xy_attn_mask.shape != _xy_padding_mask.shape:\n assert xy_attn_mask.ndim + 1 == _xy_padding_mask.ndim, f\"xy_attn_mask.shape: {xy_attn_mask.shape}, _xy_padding_mask: {_xy_padding_mask.shape}\"\n xy_attn_mask = xy_attn_mask.unsqueeze(0).repeat(_xy_padding_mask.shape[0], 1, 1) # Example approach\n xy_attn_mask = xy_attn_mask.logical_or(_xy_padding_mask)\n\n new_attn_mask = torch.zeros_like(xy_attn_mask)\n new_attn_mask.masked_fill_(xy_attn_mask, float(\"-inf\"))\n xy_attn_mask = new_attn_mask\n\n xy_input = torch.cat([x_input, y_input], dim=1)\n\n if past == None: # do not use kvcache\n out, _ = self.decoder((xy_input, None), mask=xy_attn_mask)\n return out[:, x_lens.max():], None\n else: # use kvcache\n if past.ndim > 3: # uses kvcache, only need to pass the last tokens, this doesn't work with multi-span speech editing yet\n if last_3_tokens:\n xy_input = xy_input[:, -3:]\n xy_attn_mask = xy_attn_mask[:, -3:]\n else:\n xy_input = xy_input[:, -1:]\n xy_attn_mask = xy_attn_mask[:, -1:]\n\n out, present = self.decoder((xy_input, None), mask=xy_attn_mask, past=past)\n if isinstance(out, tuple): # get rid of stage_embedding\n out = out[0]\n\n if out.shape[1] > x_lens.max(): # the first pass, not kvcache yet\n return out[:, x_lens.max():], present\n else: # used kvcache\n return out, present\n\n def forward(self, batch):\n \"\"\"\n Args:\n x:\n A 2-D tensor of shape (N, S).\n x_lens:\n A 1-D tensor of shape (N,). It contains the number of tokens in `x`\n before padding.\n y:\n A 3-D tensor of shape (N, K, T).\n where K is the number of codebooks\n y_lens:\n A 1-D tensor of shape (N,). It contains the number of tokens in `x`\n before padding.\n \"\"\"\n x, x_lens, y, y_lens = batch[\"x\"], batch[\"x_lens\"], batch[\"y\"], batch[\"y_lens\"]\n if len(x) == 0:\n return None\n x = x[:, :x_lens.max()] # this deal with gradient accumulation, where x_lens.max() might not be longer than the length of the current slice of x\n y = y[:, :, :y_lens.max()]\n assert x.ndim == 2, x.shape\n assert x_lens.ndim == 1, x_lens.shape\n assert y.ndim == 3 and y.shape[1] == self.args.n_codebooks, y.shape\n assert y_lens.ndim == 1, y_lens.shape\n # makes attention mask and padding mask for x\n x_padding_mask = make_pad_mask(x_lens).to(x.device)\n x_attention_mask = torch.triu(torch.ones(x.shape[1], x.shape[1]), diagonal=1).bool().to(x_padding_mask.device)\n x_input = self.text_embedding(x)\n x_input = self.text_positional_embedding(x_input)\n y_input, new_y_lens, targets, y_padding_mask, y_attention_mask, mask_position, patterns = self.prepare_input_target(y, y_lens)\n y_out = self.dec_forward(\n x_input, \n x_lens,\n x_attention_mask,\n x_padding_mask,\n y_input,\n new_y_lens,\n y_attention_mask,\n y_padding_mask\n )\n y_out = y_out[0] # no kv-caching during training\n assert y_out.shape == y_input.shape, f\"y_out.shape: {y_out.shape}, y_input.shape: {y_input.shape}\" # [B S D]\n \n logits = torch.stack([self.predict_layer[i](y_out) for i in range(self.args.n_codebooks)], dim=1) # [B K S card]\n # take out the mask token (using mask_position and new_y_lens) and revert (using function provided by self.pattern)\n assert logits.shape[1] == self.args.n_codebooks and logits.shape[3] == self.n_audio_tokens[0], logits.shape\n\n logits_use = self.remove_mask(logits, mask_position, new_y_lens)\n \n # revert the pattern shift for each logits section in each sample\n logits_final, logit_masks = self.revert_pattern(patterns, logits_use)\n assert logits_final[0][0].shape[0] == self.args.n_codebooks and logits_final[0][0].shape[2] == self.n_audio_tokens[0], f\"it is: {logits_final[0][0].shape}, but should be [K, T, card]\"\n # testing\n sample_to_test = 0\n assert len(logits_final[sample_to_test]) == len(targets[sample_to_test]), f\"{len(logits_final[sample_to_test])}, {len(targets[sample_to_test])}\"\n temp = sum([logits_final[sample_to_test][i].shape[:-1] != targets[sample_to_test][i].shape for i in range(len(targets[sample_to_test]))])\n assert temp == 0, f\"none equal positions: {temp}, total number of elements: {len(targets[sample_to_test])}\"\n\n logit_masked = sum([(item==False).any() for cur_mask in logit_masks for item in cur_mask])\n assert logit_masked == 0, logit_masks\n\n logits = torch.cat([torch.cat(item, dim=1) for item in logits_final], dim=1) # [K, T1+T2+T3+..., card]\n targets = torch.cat([torch.cat(item, dim=1) for item in targets], dim=1) # [K, T1+T2+T3+...]\n assert targets.shape[0] == logits.shape[0], f\"{targets.shape}, {logits.shape}\"\n loss = []\n ntokens = []\n top10acc = []\n for k, (logit, target) in enumerate(zip(logits, targets)):\n loss.append(F.cross_entropy(logit, target, reduction='mean'))\n top10acc.append(self.accuracy_metrics[k](logit.detach(), target))\n ntokens.append(len(logit))\n \n all_ntokens = sum(ntokens)\n if self.args.codebook_weight != None:\n codebook_weight = eval(self.args.codebook_weight)\n else:\n codebook_weight = [1.] * self.args.n_codebooks\n loss = sum([l*nt*cw for l, nt, cw in zip(loss, ntokens, codebook_weight)])\n top10acc_by_codebook = [t10a*nt for t10a, nt in zip(top10acc, ntokens)]\n top10acc = sum(top10acc_by_codebook)\n ntokens = torch.tensor(all_ntokens).to(logits.device)\n\n return {\n \"loss\": loss,\n \"top10acc\": top10acc,\n \"top10acc_by_codebook\": top10acc_by_codebook,\n \"effective_ntoken\": ntokens,\n }\n \n def inference(\n self,\n x: torch.Tensor,\n x_lens: torch.Tensor,\n y: torch.Tensor,\n mask_interval: list[torch.Tensor],\n top_k: int=-100,\n top_p: float=1.0,\n temperature: float=1.0,\n stop_repetition: int=-1,\n kvcache: int=1,\n silence_tokens: list[int]=[1388,1898,131],\n ) -> torch.Tensor:\n \"\"\"\n Args:\n x:\n A 2-D tensor of shape (1, L).\n x_lens:\n A 1-D tensor of shape (1,). It contains the number of tokens in `x`\n before padding.\n y:\n A 3-D tensor of shape (1, T, K).\n mask_interval:\n a list of tensors of shape (M, 2). contains M mask_start and mask_end. list length is actually 1, because we only support single sample inference for now\n top_k: (`optional`) int\n The number of highest probability tokens to keep for top-k-filtering. Default to -100.\n top_p: (`optional`) float\n For Neucleus sampling\n temperature: (`optional`) float\n The value used to module the next token probabilities. Must be strictly positive. Default to 1.0.\n eog_coef: (`optional`) float\n if 0, no change to eog token logits, otherwise, will adjust eog token logit based on the difference between acoustic token and phn token length\n stop_repetition (`optional`) int\n if not -1, will set the logits of a token that repeated this many times to be -100000, to avoid generating it again. This only apply to tokens from the first codebook\n allowed_repeat_tokens (`optional`) list of ints\n by inspecting the validation set, get a few tokens that indeed repeat a significant amount of time, and exclude those tokens from prevent repetition\n ultimate_stop_repetition (`optional`) int\n no matter that token it is, stop repetition once after this number\n \"\"\"\n assert x.ndim == 2, x.shape\n assert x_lens.ndim == 1, x_lens.shape\n assert y.ndim == 3, y.shape\n if self.args.special_first:\n y = y + int(self.args.n_special)\n y = y.transpose(2,1) # [1,T,K] -> [1,K,T]\n assert y.shape[0] == 1 and y.shape[1] == self.args.n_codebooks, y.shape # there is no padding\n assert mask_interval.shape == torch.Size((1, mask_interval.shape[1], 2)), mask_interval\n\n # make x attention mask and x_input\n x_attention_mask = torch.triu(torch.ones(x.shape[1], x.shape[1]), diagonal=1).bool().to(x.device)\n # x_attention_mask = torch.zeros(x.shape[1], x.shape[1]).bool().to(x.device)\n x_input = self.text_embedding(x)\n x_input = self.text_positional_embedding(x_input)\n\n # make initial y_input\n\n # make mask_interval and non_mask_interval\n y_len = y.shape[2]\n y_lens = torch.LongTensor([y_len]).to(y.device)\n mask_interval = mask_interval[0]\n starts = [item[0].item() for item in mask_interval] + [y_len]\n ends = [0] + [item[1].item() for item in mask_interval]\n mask_intervals = [[\n (item[0].item(), item[1].item()) for item in mask_interval\n ]] # a werid name change, mask_interval is input, now is mask_intervals, with one more dimension\n non_mask_intervals = [[\n (ns, ne) for ns, ne in zip(ends, starts)\n ]]\n \n # rearrange y\n # will add have EOG in each section (SOG will be generated by the pattern class)\n # but mask can be inserted later after we have shifted the input\n # y could be rearranged in this way:\n # [\n # [tensor[4, 12], tensor[4, 45], tensor[4, 102], tensor[4, 32]], tensor[4, 22]],\n # [tensor[4, 44], tensor[4, 56], tensor[4, 19]],\n # ...\n # ]\n # for the first list of tensors (4 tensors), first 3 tensors are non_masked part, last 2 are masked part.\n # NOTE #non_masked_part = #masked_part + 1\n rearranged_y = self.rearrange(y, non_mask_intervals, mask_intervals)\n assert rearranged_y[0][0].shape[0] == self.args.n_codebooks, rearranged_y[0][0].shape\n\n # shift each element of y\n # next we need to apply pattern shifting to each tensor, after which, we'll replace the starting tokens of each section with a token that's different from the special padding token\n # [\n # [empty, 1, 2, 3, eog, empty, empty, empty],\n # [empty, empty, 1, 2, 3, eog, empty, empty],\n # [empty, empty, empty, 1, 2, 3, eog, empty],\n # [empty, empty, empty, empty, 1, 2, 3, eog]\n # ]\n shifted_y, patterns = self.shift(rearranged_y) # each element [K S], patterns is not used, as we directly use the original input y\n assert shifted_y[0][0].shape[0] == self.args.n_codebooks, shifted_y[0][0].shape\n\n # insert mask token at the intersction of each tensor, but *actually inserted eog as place holder*\n # the position of inserted mask is also recorded\n # and the mask_value, the index of the mask emb is recorded\n inserted_y, mask_position, mask_value = self.insert_mask(shifted_y)\n assert inserted_y[0][0].shape[0] == self.args.n_codebooks, inserted_y[0][0].shape[0]\n assert inserted_y[0][1].shape == torch.Size((self.args.n_codebooks, 1)), f\"this should be a mask, so should have shape {(self.args.n_codebooks, 1)}, but it's {inserted_y[0][1].shape}\"\n \n # then concat tensors that belong to the same sample (in order) then get the length of each sample, and then stack them in batch dimension, pad them with pad_token\n cated_y, new_y_lens = self.cat_y(inserted_y, mask_position, y_lens) # KTB\n assert cated_y.shape == torch.Size((self.args.n_codebooks, cated_y.shape[1], len(inserted_y)))\n assert not (cated_y == self.args.audio_pad_token).any(), cated_y\n\n ### NOTE this is different from forward, as we will remove the masked tokens\n ### say there are two masked region\n ### the cated_y should be like\n ### [empty a a a a mask0 empty b b b mask1 empty c c mask0 empty]\n ### which means we need to take the part after the last empty out\n num_mask = len(mask_position[0])//2\n assert num_mask == len(mask_position[0])/2, mask_position\n cated_y = cated_y[:, :mask_position[0][num_mask]+2] # of shape [K,T,B]\n # logging.info(f\"mask_position[0][num_mask]+2: {mask_position[0][num_mask]+2}\")\n more_mask_value = mask_value[0][num_mask+1:] # NOTE this will be used in the generation loop for reference for inserting mask embedding\n new_y_lens[0] = mask_position[0][num_mask]+2\n mask_position[0] = mask_position[0][:num_mask+1]\n assert mask_position[0][num_mask]+2 == cated_y.shape[1], f\"num_mask: {num_mask}, mask_position: {mask_position}, cated_y.shape: {cated_y.shape}\"\n\n # embed: remember to separately embed the mask tokens\n embedded_y = self.embed_y(cated_y, mask_position, [mask_value[0][:num_mask+1]]) #BTD\n # assert embedded_y.shape == torch.Size((y.shape[0], max(new_y_lens), self.args.d_model)), embedded_y.shape\n\n # positional embedding\n y_input = self.audio_positional_embedding(embedded_y)\n\n # make attention mask and padding mask\n y_attention_mask = torch.triu(torch.ones(y_input.shape[1], y_input.shape[1]), diagonal=1).bool().to(y.device)\n # y_lens = torch.LongTensor([y_input.shape[1]]).to(y.device)\n\n x_padding_mask = torch.full((1,x_lens[0]), False).to(x.device)\n y_padding_mask = torch.full((1,new_y_lens[0]), False).to(y.device)\n\n\n codebook_eog = [False] * self.args.n_codebooks\n generated = [] # doesn't contain any empty_token, contains eog\n cur_generated = []\n # say 0 is empty, 4 is eog\n # tensor([[ 1, 2, 3, 4, 0, 0],\n # [ 0, 1, 2, 3, 4, 0],\n # [ 0, 0, 1, 2, 3, 4]])\n num_gen = []\n cur_num_gen = 0\n ##################### silence repetition handling #####################\n ##################### silence repetition handling #####################\n logging.info(f\"silence tokens: {silence_tokens}, note that if you are not using the pretrained encodec 6f79c6a8, make sure you specified it yourself, rather than using the default\")\n consec_silence_count = 0\n prev_token = None\n ##################### silence repetition handling #####################\n ##################### silence repetition handling #####################\n # prepare the cache placeholder\n # n_layers, 2, bsz, num_heads, src_len, head_dim\n past = torch.ones([self.args.num_decoder_layers, 2, x.shape[0]], device=x.device, dtype=torch.float32) if kvcache else None\n # handle multi-span kv-cache\n new_masked_span = False\n \n def sample_helper(n_eog, logits, codebook_eog, top_k, top_p, temperature, prev_token, consec_silence_count, stop_repetition, silence_tokens, cur_num_gen):\n if n_eog == 0:\n logits_adjust = logits\n for jj in range(1,self.args.n_codebooks):\n logits_adjust[jj][self.args.eog] = -10000\n logits_adjust[jj][self.args.empty_token] = -10000\n ##################### silence repetition handling #####################\n if stop_repetition > 0 and prev_token in silence_tokens and consec_silence_count > stop_repetition:\n if logits_adjust[0, prev_token] < 0:\n logits_adjust[0, prev_token] = logits_adjust[0, prev_token] * (consec_silence_count - (stop_repetition-1))\n else:\n logits_adjust[0, prev_token] = logits_adjust[0, prev_token] / (consec_silence_count - (stop_repetition-1))\n ##################### silence repetition handling #####################\n if type(logits_adjust) == list:\n samples_list= []\n for logit in logits_adjust:\n # print(logit)\n # print(logit.shape)\n cur_sample = topk_sampling(\n logit.unsqueeze(0), top_k=top_k, top_p=top_p, temperature=temperature\n ) # [1, 1]\n samples_list.append(cur_sample)\n samples = torch.cat(samples_list, dim=0) # [K, 1]\n else:\n samples = topk_sampling(\n logits_adjust, top_k=top_k, top_p=top_p, temperature=temperature\n ) # [K, 1]\n assert samples.shape == torch.Size((self.args.n_codebooks, 1)), f\"samples.shape: {samples.shape}\"\n if cur_num_gen < self.args.n_codebooks-1:\n for jj in range(1, self.args.n_codebooks - cur_num_gen):\n samples[-jj, 0] = self.args.empty_token\n\n if (\n samples[0,0] == self.args.eog or torch.argmax(logits[0], dim=-1) == self.args.eog or y_input.shape[1] > x_lens[0] * 10\n ): # last one means y is already too long, shouldn't happen, but put it here\n samples[0,0] = self.args.eog\n codebook_eog[0] = True\n ##################### silence repetition handling #####################\n ##################### silence repetition handling #####################\n if samples[0,0] in silence_tokens and samples[0,0] == prev_token:\n consec_silence_count += 1\n else:\n consec_silence_count = 0\n prev_token = samples[0,0]\n ##################### silence repetition handling #####################\n ##################### silence repetition handling #####################\n return samples, codebook_eog, prev_token, consec_silence_count\n else:\n assert sum(codebook_eog[i] for i in range(n_eog)) == n_eog, f\"codebook_eog: {codebook_eog}, but n_eog: {n_eog}\"\n logits_adjust = logits\n for jj in range(n_eog+1,self.args.n_codebooks):\n logits_adjust[jj][self.args.eog] = -10000\n logits_adjust[jj][self.args.empty_token] = -10000\n if type(logits_adjust) == list:\n samples_list= []\n for logit in logits_adjust:\n cur_sample = topk_sampling(\n logit.unsqueeze(0), top_k=top_k, top_p=top_p, temperature=temperature\n ) # [1, 1]\n samples_list.append(cur_sample)\n samples = torch.cat(samples_list, dim=0) # [K, 1]\n else:\n samples = topk_sampling(\n logits_adjust, top_k=top_k, top_p=top_p, temperature=temperature\n ) # [K, 1]\n for jj in range(n_eog):\n samples[jj, 0] = self.args.empty_token\n samples[n_eog, 0] = self.args.eog\n codebook_eog[n_eog] = True\n return samples, codebook_eog, prev_token, consec_silence_count\n\n while True:\n y_out, present = self.dec_forward(\n x_input, \n x_lens,\n x_attention_mask,\n x_padding_mask,\n y_input,\n new_y_lens,\n y_attention_mask,\n y_padding_mask,\n past=past,\n last_3_tokens = new_masked_span\n )\n if new_masked_span:\n new_masked_span = False\n\n if past != None:\n past = torch.cat([past, present.to(past.dtype)], dim=-2) if past.ndim > 3 else present.to(past.dtype)\n\n y_out = y_out[:, -1:] # only take the last one\n\n logits = torch.stack([self.predict_layer[i](y_out) for i in range(self.args.n_codebooks)], dim=1) # [B K S card], B==S==1, so [1 K 1 card]\n logits = logits.squeeze(0).squeeze(1) # [K card]\n assert logits.shape == torch.Size((self.args.n_codebooks, self.n_audio_tokens[0])), f\"{logits.shape}\"\n\n n_eog = sum(codebook_eog)\n assert n_eog < self.args.n_codebooks\n if self.args.eos > 0: # eos stands for end-of-sentence, which shouldn't be used as we are doing speech editing\n for jj in range(self.args.n_codebooks):\n logits[jj][self.args.eos] = -10000.\n # need to use a helper function to hand different n_eog cases\n samples, codebook_eog, prev_token, consec_silence_count = sample_helper(n_eog, logits, codebook_eog, top_k, top_p, temperature, prev_token, consec_silence_count, stop_repetition, silence_tokens, cur_num_gen)\n cur_num_gen += 1\n cur_generated.append(samples.squeeze(-1)) # [K,1] -> [K]\n # get samples_emb\n samples_emb = torch.stack([self.audio_embedding[k](samples[k]) for k in range(self.args.n_codebooks)], dim=0) # [K,1,D]\n samples_emb = samples_emb.sum(dim=0,keepdim=True) # [1,1,D]\n\n if sum(codebook_eog) == self.args.n_codebooks: # generation for the current span is done\n # re-init\n codebook_eog = [False] * self.args.n_codebooks\n num_gen.append(cur_num_gen)\n cur_num_gen = 0\n generated.append(cur_generated)\n cur_generated = []\n\n # if the current mask span is the last span, then all done\n # else\n # append the next mask token and the four empty tokens to start the next generation \n if len(more_mask_value) > 0:\n next_mask_ind = more_mask_value.pop(0)\n mask_emb = self.mask_embedding[next_mask_ind].unsqueeze(0).unsqueeze(0) # [1,1,D]\n assert mask_emb.shape == torch.Size((1,1,self.args.d_model)), mask_emb.shape\n empty_token = torch.LongTensor([self.args.empty_token]).to(y.device)\n empty_emb = torch.stack([\n self.audio_embedding[k](empty_token) for k in range(self.args.n_codebooks)], dim=0\n ).sum(dim=0, keepdim=True) # [1,1,D]\n assert empty_emb.shape == torch.Size((1,1,self.args.d_model)), empty_emb.shape\n extra_emb = torch.cat([mask_emb, empty_emb], dim=1) # [1,2,D]\n samples_emb = torch.cat([samples_emb, extra_emb], dim=1) # [1,3,D] # prev_last_token, mask_token, empty token\n assert samples_emb.shape == torch.Size((1,3,self.args.d_model)), f\"samples_emb.shape: {samples_emb.shape}\"\n ##################### silence repetition handling #####################\n ##################### silence repetition handling #####################\n consec_silence_count = 0\n prev_token = None\n ##################### silence repetition handling #####################\n ##################### silence repetition handling #####################\n\n # handling kv-caching for multi-span editing\n new_masked_span = True\n else:\n break\n else:\n assert samples_emb.shape == torch.Size((1,1,self.args.d_model)), f\"samples_emb.shape: {samples_emb.shape}\"\n\n embedded_y = torch.cat([embedded_y, samples_emb], dim=1)\n # positional embedding\n y_input = self.audio_positional_embedding(embedded_y) # [B T D]\n # make attention mask and padding mask\n y_attention_mask = torch.triu(torch.ones(y_input.shape[1], y_input.shape[1]), diagonal=1).bool().to(y.device)\n new_y_lens = torch.LongTensor([y_input.shape[1]]).to(y.device)\n y_padding_mask = torch.full((1,new_y_lens[0]), False).to(y.device)\n \n assert len(generated) == num_mask, f\"len(generated): {len(generated)}, num_mask: {num_mask}\"\n\n # # combine non_masked_span with generated spans\n # first need to shift the generated part back\n flatten_gen = []\n for l, orig_span in enumerate(generated):\n span = torch.stack(orig_span, dim=0) # [T K]\n span = span.transpose(1,0) # [K, T]\n assert span.shape[0] == self.args.n_codebooks, span.shape\n unshifted_span = []\n for j, s in enumerate(span):\n start_from = j\n end_at = - (self.args.n_codebooks - start_from)\n unshifted_span.append(s[start_from:end_at])\n unshifted_span = torch.stack(unshifted_span, dim=0)\n\n assert unshifted_span.shape[1] == num_gen[l] - self.args.n_codebooks, f\"len(unshifted_spans[0]): {len(unshifted_span[0])}, num_gen[l]: {num_gen[l]}\"\n flatten_gen.append(unshifted_span)\n # logging.info(f\"unshfited_span: {unshifted_span.shape}\")\n # raise\n assert len(non_mask_intervals[0]) - 1 == len(flatten_gen), f\"len(non_mask_intervals[0]): {len(non_mask_intervals[0])}, len(flatten_gen): {len(flatten_gen)}\"\n res = []\n for orig_interval, gen in zip(non_mask_intervals[0], flatten_gen):\n res.append(y[0, :, orig_interval[0]:orig_interval[1]])\n res.append(gen)\n res.append(y[0, :, non_mask_intervals[0][-1][0]:non_mask_intervals[0][-1][1]])\n res = torch.cat(res, dim=1).unsqueeze(0) # [K,new_T] -> [1, K, new_T]\n\n expected_y_len = y_len - sum([item[1] - item[0] for item in mask_intervals[0]]) + sum([item - self.args.n_codebooks for item in num_gen])\n assert res.shape == torch.Size((1, self.args.n_codebooks, expected_y_len)), f\"res.shape: {res.shape}, expected_y_len: {expected_y_len}. y_len - sum([item[1] - item[0] for item in mask_interval]) + sum([item - self.args.n_codebooks for item in num_gen]): {y_len}-{sum([item[1] - item[0] for item in mask_interval])} + {sum([item - self.args.n_codebooks for item in num_gen])}\"\n \n if self.args.special_first:\n res = res - int(self.args.n_special)\n\n return res\n\n def inference_tts(\n self,\n x: torch.Tensor,\n x_lens: torch.Tensor,\n y: torch.Tensor,\n top_k: int=-100,\n top_p: float=1.0,\n temperature: float=1.0,\n stop_repetition: int=3,\n kvcache: int=1,\n silence_tokens: list[int]=[1388,1898,131],\n *kargs\n ) -> torch.Tensor:\n \"\"\"\n different from inference_tts, this implementation uses kvcache, which should have significant speed up\n Args:\n x:\n A 2-D tensor of shape (1, L).\n x_lens:\n A 1-D tensor of shape (1,). It contains the number of tokens in `x`\n before padding.\n y:\n A 3-D tensor of shape (1, T, K).\n top_k: (`optional`) int\n The number of highest probability tokens to keep for top-k-filtering. Default to -100.\n top_p: (`optional`) float\n For Neucleus sampling\n temperature: (`optional`) float\n The value used to module the next token probabilities. Must be strictly positive. Default to 1.0.\n \"\"\"\n eog_inference = self.args.eos if self.args.eos>0 else self.args.eog\n assert x.ndim == 2, x.shape\n assert x_lens.ndim == 1, x_lens.shape\n assert y.ndim == 3, y.shape\n if self.args.special_first:\n y = y + int(self.args.n_special)\n y = y.transpose(2,1) # [1,T,K] -> [1,K,T]\n assert y.shape[0] == 1 and y.shape[1] == self.args.n_codebooks, y.shape # there is no padding\n\n # make x attention mask and x_input\n x_attention_mask = torch.triu(torch.ones(x.shape[1], x.shape[1]), diagonal=1).bool().to(x.device)\n # x_attention_mask = torch.zeros(x.shape[1], x.shape[1]).bool().to(x.device)\n x_input = self.text_embedding(x)\n x_input = self.text_positional_embedding(x_input)\n\n y_len = y.shape[2]\n y_lens = torch.LongTensor([y_len]).to(y.device)\n\n # rearrange y, we don't add eog to the end, this doesn't actually do anything in the tts scenario\n rearranged_y = [[y[0]]]\n assert rearranged_y[0][0].shape[0] == self.args.n_codebooks, rearranged_y[0][0].shape\n\n # shift y to create the delayed pattern\n shifted_y, patterns = self.shift(rearranged_y) # each element [K S], patterns is not used, as we directly use the original input y\n assert shifted_y[0][0].shape[0] == self.args.n_codebooks, shifted_y[0][0].shape\n assert len(shifted_y[0]) == 1, len(shifted_y[0])\n\n # below is different from forward or inference\n # where we cut this shifted part \n shifted_y[0][0] = shifted_y[0][0][:, :-(self.args.n_codebooks-1)]\n assert not (shifted_y[0][0][self.args.n_codebooks:] == self.args.empty_token).any() and not (shifted_y[0][0][self.args.n_codebooks:] == self.args.eog).any(), shifted_y[0][0]\n\n # next section in inference is insert mask at the intersection of each tensor in a sample, but we don't need to do that \n # next section is concate tensors of each sample to one tensor, which we also don't need\n cated_y = shifted_y[0][0].unsqueeze(-1) #[K,S]->[K,S,B]\n new_y_lens = torch.LongTensor([cated_y.shape[1]]).to(cated_y.device)\n assert cated_y.shape == torch.Size((self.args.n_codebooks, cated_y.shape[1], 1))\n assert not (cated_y == self.args.audio_pad_token).any(), cated_y\n\n # replace tokens in y with the embeddings, add sum codebooks up\n embedded_y = torch.stack([self.audio_embedding[k](cated_y[k]) for k in range(self.args.n_codebooks)], dim=0) # [K, S, B, D]\n assert embedded_y.shape[0] == self.args.n_codebooks, embedded_y.shape\n assert embedded_y.shape[-1] == self.args.d_model, embedded_y.shape\n embedded_y = embedded_y.sum(dim=0) # [K,S,B,D]->[S,B,D]\n embedded_y = embedded_y.transpose(1,0) # [S,B,D]->[B,S,D]\n \n # positional embedding\n y_input = self.audio_positional_embedding(embedded_y)\n\n # make attention mask and padding mask\n y_attention_mask = torch.triu(torch.ones(y_input.shape[1], y_input.shape[1]), diagonal=1).bool().to(y.device)\n\n x_padding_mask = torch.full((1,x_lens[0]), False).to(x.device)\n y_padding_mask = torch.full((1,new_y_lens[0]), False).to(y.device)\n\n # entering the generation stage\n # starting from line 708\n codebook_eog = [False] * self.args.n_codebooks\n generated = [] # doesn't contain any empty token, contain eog\n cur_generated = []\n # say 0 is empty, 4 is eog\n # tensor([[ 1, 2, 3, 4, 0, 0],\n # [ 0, 1, 2, 3, 4, 0],\n # [ 0, 0, 1, 2, 3, 4]])\n num_gen = []\n cur_num_gen = 0\n ##################### silence repetition handling #####################\n ##################### silence repetition handling #####################\n logging.info(f\"silence tokens: {silence_tokens}, note that if you are not using the pretrained encodec 6f79c6a8, make sure you specified it yourself, rather than using the default\")\n consec_silence_count = 0\n prev_token = None\n ##################### silence repetition handling #####################\n ##################### silence repetition handling #####################\n\n # prepare the cache placeholder\n # n_layers, 2, bsz, num_heads, src_len, head_dim\n past = torch.ones([self.args.num_decoder_layers, 2, x.shape[0]], device=x.device, dtype=torch.float32) if kvcache else None\n # logging.info(f\"number of decoder layers: {self.args.num_decoder_layers}\")\n # logging.info(f\"number of decoder layers: {self.args.num_decoder_layers}\")\n # logging.info(f\"number of decoder layers: {self.args.num_decoder_layers}\")\n def sample_helper(n_eog, logits, codebook_eog, top_k, top_p, temperature, prev_token, consec_silence_count, stop_repetition, silence_tokens, cur_num_gen):\n if n_eog == 0:\n logits_adjust = logits\n for jj in range(1,self.args.n_codebooks):\n logits_adjust[jj][eog_inference] = -10000\n logits_adjust[jj][self.args.empty_token] = -10000\n if cur_num_gen <= self.args.encodec_sr // 5: # this shouldn't happen, but just in case the model stopped too early\n logits_adjust[0][eog_inference] = -10000\n ##################### silence repetition handling #####################\n if stop_repetition > 0 and prev_token in silence_tokens and consec_silence_count > stop_repetition:\n if logits_adjust[0, prev_token] < 0:\n logits_adjust[0, prev_token] = logits_adjust[0, prev_token] * (consec_silence_count - (stop_repetition-1))\n else:\n logits_adjust[0, prev_token] = logits_adjust[0, prev_token] / (consec_silence_count - (stop_repetition-1))\n ##################### silence repetition handling #####################\n samples = topk_sampling(\n logits_adjust, top_k=top_k, top_p=top_p, temperature=temperature\n ) # [K, 1]\n assert samples.shape == torch.Size((self.args.n_codebooks, 1)), f\"samples.shape: {samples.shape}\"\n if cur_num_gen < self.args.n_codebooks-1:\n for jj in range(1, self.args.n_codebooks - cur_num_gen):\n samples[-jj, 0] = self.args.empty_token\n\n if (\n samples[0,0] == eog_inference or torch.argmax(logits[0], dim=-1) == eog_inference or y_input.shape[1] > x_lens[0] * (self.args.encodec_sr//5)\n ): # last one means y is already too long, shouldn't happen, but put it here\n samples[0,0] = eog_inference\n codebook_eog[0] = True\n ##################### silence repetition handling #####################\n if samples[0,0] in silence_tokens and samples[0,0] == prev_token:\n consec_silence_count += 1\n else:\n consec_silence_count = 0\n prev_token = samples[0,0]\n ##################### silence repetition handling #####################\n return samples, codebook_eog, prev_token, consec_silence_count\n else:\n assert sum(codebook_eog[i] for i in range(n_eog)) == n_eog, f\"codebook_eog: {codebook_eog}, but n_eog: {n_eog}\"\n logits_adjust = logits\n for jj in range(n_eog+1,self.args.n_codebooks):\n logits_adjust[jj][eog_inference] = -10000\n logits_adjust[jj][self.args.empty_token] = -10000\n samples = topk_sampling(\n logits_adjust, top_k=top_k, top_p=top_p, temperature=temperature\n ) # [K, 1]\n for jj in range(n_eog):\n samples[jj, 0] = self.args.empty_token\n samples[n_eog, 0] = eog_inference\n codebook_eog[n_eog] = True\n return samples, codebook_eog, prev_token, consec_silence_count\n while True:\n y_out, present = self.dec_forward(\n x_input, \n x_lens,\n x_attention_mask,\n x_padding_mask,\n y_input,\n new_y_lens,\n y_attention_mask,\n y_padding_mask,\n past=past\n )\n if past != None:\n past = torch.cat([past, present.to(past.dtype)], dim=-2) if past.ndim > 3 else present.to(past.dtype)\n\n\n y_out = y_out[:, -1:] # only take the last token\n logits = torch.stack([self.predict_layer[i](y_out) for i in range(self.args.n_codebooks)], dim=1) # [B K S card], B==S==1, so [1 K 1 card]\n logits = logits.squeeze(0).squeeze(1) # [K card]\n assert logits.shape == torch.Size((self.args.n_codebooks, self.n_audio_tokens[0])), f\"{logits.shape}\"\n\n n_eog = sum(codebook_eog)\n assert n_eog < self.args.n_codebooks\n if self.args.eos > 0: # if we are using end-of-sentence token (which is used by default), eog shouldn't be used here, as there is no masked spans\n for jj in range(self.args.n_codebooks):\n logits[jj][self.args.eog] = -10000.\n \n samples, codebook_eog, prev_token, consec_silence_count = sample_helper(n_eog, logits, codebook_eog, top_k, top_p, temperature, prev_token, consec_silence_count, stop_repetition, silence_tokens, cur_num_gen)\n \n cur_num_gen += 1\n cur_generated.append(samples.squeeze(-1)) # [K,1] -> [K]\n\n # samples.shape is [K,1]\n # ge samples_emb\n samples_emb = torch.stack([self.audio_embedding[k](samples[k]) for k in range(self.args.n_codebooks)], dim=0) # [K,1,D]\n samples_emb = samples_emb.sum(dim=0,keepdim=True) # [1,1,D]\n\n if sum(codebook_eog) == self.args.n_codebooks: # generation for the current span is done\n codebook_eog = [False] * self.args.n_codebooks\n num_gen.append(cur_num_gen)\n cur_num_gen = 0\n generated.append(cur_generated)\n cur_generated = []\n break\n else:\n assert samples_emb.shape == torch.Size((1,1,self.args.d_model)), f\"samples_emb.shape: {samples_emb.shape}\"\n \n embedded_y = torch.cat([embedded_y, samples_emb], dim=1)\n y_input = self.audio_positional_embedding(embedded_y) # [B T D]\n # make attention mask and padding mask\n y_attention_mask = torch.triu(torch.ones(y_input.shape[1], y_input.shape[1]), diagonal=1).bool().to(y.device)\n new_y_lens = torch.LongTensor([y_input.shape[1]]).to(y.device)\n y_padding_mask = torch.full((1,new_y_lens[0]), False).to(y.device)\n \n assert len(generated) == 1, f\"len(generated): {len(generated)}\"\n\n # revert the pattern\n flatten_gen = []\n for l, orig_span in enumerate(generated):\n span = torch.stack(orig_span, dim=0) # [T, K]\n span = span.transpose(1,0) # [K, T]\n assert span.shape[0] == self.args.n_codebooks, span.shape\n unshifted_span = []\n for j, s in enumerate(span):\n start_from = j\n end_at = - (self.args.n_codebooks - start_from)\n unshifted_span.append(s[start_from:end_at])\n unshifted_span = torch.stack(unshifted_span, dim=0)\n\n assert unshifted_span.shape[1] == num_gen[l] - self.args.n_codebooks, f\"len(unshifted_spans[0]): {len(unshifted_span[0])}, num_gen[l]: {num_gen[l]}\"\n\n flatten_gen.append(unshifted_span)\n assert len(flatten_gen) == 1, len(flatten_gen)\n \n # combine \n res = [y[0], flatten_gen[0]]\n res = torch.cat(res, dim=1).unsqueeze(0) # [K, new_t] -> [1, K, new_T]\n\n expected_y_len = y_len + sum([item - self.args.n_codebooks for item in num_gen])\n assert res.shape == torch.Size((1, self.args.n_codebooks, expected_y_len)), f\"res.shape: {res.shape}, expected_y_len: {expected_y_len}. y_len + sum([item - self.args.n_codebooks for item in num_gen]): {y_len} + {sum([item - self.args.n_codebooks for item in num_gen])}\"\n\n if self.args.special_first:\n res = res - int(self.args.n_special)\n flatten_gen = flatten_gen - int(self.args.n_special)\n\n return res, flatten_gen[0].unsqueeze(0)\n\n\n def inference_tts_batch(\n self,\n x: torch.Tensor,\n x_lens: torch.Tensor,\n y: torch.Tensor,\n top_k: int=-100,\n top_p: float=1.0,\n temperature: float=1.0,\n stop_repetition: int=3,\n kvcache: int=1,\n batch_size: int=5,\n silence_tokens: list[int]=[1388,1898,131],\n *kargs\n ) -> torch.Tensor:\n \"\"\"\n have a batch size when forward passing, but they are equivalant to same example but different random seed, therefore as long as one example generated eog, we can drop all other samlpes\n different from inference_tts, this implementation uses kvcache, which should have significant speed up\n Args:\n x:\n A 2-D tensor of shape (1, L).\n x_lens:\n A 1-D tensor of shape (1,). It contains the number of tokens in `x`\n before padding.\n y:\n A 3-D tensor of shape (1, T, K).\n top_k: (`optional`) int\n The number of highest probability tokens to keep for top-k-filtering. Default to -100.\n top_p: (`optional`) float\n For Neucleus sampling\n temperature: (`optional`) float\n The value used to module the next token probabilities. Must be strictly positive. Default to 1.0.\n \"\"\"\n eog_inference = self.args.eos if self.args.eos>0 else self.args.eog\n assert x.ndim == 2, x.shape\n assert x_lens.ndim == 1, x_lens.shape\n assert y.ndim == 3, y.shape\n if self.args.special_first:\n y = y + int(self.args.n_special)\n y = y.transpose(2,1) # [1,T,K] -> [1,K,T]\n assert y.shape[0] == 1 and y.shape[1] == self.args.n_codebooks, y.shape # there is no padding\n\n # make x attention mask and x_input\n x_attention_mask = torch.triu(torch.ones(x.shape[1], x.shape[1]), diagonal=1).bool().to(x.device)\n # x_attention_mask = torch.zeros(x.shape[1], x.shape[1]).bool().to(x.device)\n x_input = self.text_embedding(x)\n x_input = self.text_positional_embedding(x_input)\n\n y_len = y.shape[2]\n y_lens = torch.LongTensor([y_len]).to(y.device)\n\n # rearrange y, we don't add eog to the end, this doesn't actually do anything in the tts scenario\n rearranged_y = [[y[0]]]\n assert rearranged_y[0][0].shape[0] == self.args.n_codebooks, rearranged_y[0][0].shape\n\n # shift y to create the delayed pattern\n shifted_y, patterns = self.shift(rearranged_y) # each element [K S], patterns is not used, as we directly use the original input y\n assert shifted_y[0][0].shape[0] == self.args.n_codebooks, shifted_y[0][0].shape\n assert len(shifted_y[0]) == 1, len(shifted_y[0])\n\n # below is different from forward or inference\n # where we cut this shifted part \n shifted_y[0][0] = shifted_y[0][0][:, :-(self.args.n_codebooks-1)]\n assert not (shifted_y[0][0][self.args.n_codebooks:] == self.args.empty_token).any() and not (shifted_y[0][0][self.args.n_codebooks:] == self.args.eog).any(), shifted_y[0][0]\n\n # next section in inference is insert mask at the intersection of each tensor in a sample, but we don't need to do that \n # next section is concate tensors of each sample to one tensor, which we also don't need\n cated_y = shifted_y[0][0].unsqueeze(-1) #[K,S]->[K,S,B]\n new_y_lens = torch.LongTensor([cated_y.shape[1]]).to(cated_y.device)\n assert cated_y.shape == torch.Size((self.args.n_codebooks, cated_y.shape[1], 1))\n assert not (cated_y == self.args.audio_pad_token).any(), cated_y\n\n # replace tokens in y with the embeddings, add sum codebooks up\n embedded_y = torch.stack([self.audio_embedding[k](cated_y[k]) for k in range(self.args.n_codebooks)], dim=0) # [K, S, B, D]\n assert embedded_y.shape[0] == self.args.n_codebooks, embedded_y.shape\n assert embedded_y.shape[-1] == self.args.d_model, embedded_y.shape\n embedded_y = embedded_y.sum(dim=0) # [K,S,B,D]->[S,B,D]\n embedded_y = embedded_y.transpose(1,0) # [S,B,D]->[B,S,D]\n \n # positional embedding\n y_input = self.audio_positional_embedding(embedded_y)\n\n # make attention mask and padding mask\n y_attention_mask = torch.triu(torch.ones(y_input.shape[1], y_input.shape[1]), diagonal=1).bool().to(y.device)\n\n x_padding_mask = torch.full((1,x_lens[0]), False).to(x.device)\n y_padding_mask = torch.full((1,new_y_lens[0]), False).to(y.device)\n\n # entering the generation stage\n # starting from line 708\n codebook_eog = [False] * self.args.n_codebooks\n generated = [] # doesn't contain any empty token, contain eog\n cur_generated = [[] for _ in range(batch_size)]\n # say 0 is empty, 4 is eog\n # tensor([[ 1, 2, 3, 4, 0, 0],\n # [ 0, 1, 2, 3, 4, 0],\n # [ 0, 0, 1, 2, 3, 4]])\n num_gen = []\n cur_num_gen = 0\n ##################### silence repetition handling #####################\n ##################### silence repetition handling #####################\n logging.info(f\"silence tokens: {silence_tokens}, note that if you are not using the pretrained encodec 6f79c6a8, make sure you specified it yourself, rather than using the default\")\n consec_silence_counts = [0 for _ in range(batch_size)]\n prev_tokens = [None for _ in range(batch_size)]\n ##################### silence repetition handling #####################\n ##################### silence repetition handling #####################\n\n # prepare the cache placeholder\n # n_layers, 2, bsz, num_heads, src_len, head_dim\n past = torch.ones([self.args.num_decoder_layers, 2, x.shape[0]], device=x.device, dtype=torch.float32) if kvcache else None\n # logging.info(f\"number of decoder layers: {self.args.num_decoder_layers}\")\n # logging.info(f\"number of decoder layers: {self.args.num_decoder_layers}\")\n # logging.info(f\"number of decoder layers: {self.args.num_decoder_layers}\")\n keep = None # NOTE: this very important, tells which sample to keep\n def sample_helper(n_eog, logits, codebook_eog, top_k, top_p, temperature, prev_tokens, consec_silence_counts, stop_repetition, silence_tokens, cur_num_gen, keep):\n if n_eog == 0:\n logits_adjust = logits\n for jj in range(1,self.args.n_codebooks):\n logits_adjust[:,jj,eog_inference] = -10000\n logits_adjust[:,jj,self.args.empty_token] = -10000\n if cur_num_gen <= self.args.encodec_sr // 5: # this shouldn't happen, but just in case the model stopped too early\n logits_adjust[:,:,eog_inference] = -10000\n ##################### silence repetition handling #####################\n for b in range(batch_size):\n prev_token = prev_tokens[b]\n consec_silence_count = consec_silence_counts[b]\n if stop_repetition > 0 and prev_token in silence_tokens and consec_silence_count > stop_repetition:\n if logits_adjust[b, 0, prev_token] < 0:\n logits_adjust[b, 0, prev_token] = logits_adjust[b, 0, prev_token] * (consec_silence_count - (stop_repetition-1))\n else:\n logits_adjust[b, 0, prev_token] = logits_adjust[b, 0, prev_token] / (consec_silence_count - (stop_repetition-1))\n ##################### silence repetition handling #####################\n samples = topk_sampling(\n logits_adjust.reshape(batch_size * self.args.n_codebooks, logits_adjust.shape[-1]), top_k=top_k, top_p=top_p, temperature=temperature\n ) # [B*K, 1]\n samples = samples.reshape(batch_size, self.args.n_codebooks, 1)\n assert samples.shape == torch.Size((batch_size, self.args.n_codebooks, 1)), f\"samples.shape: {samples.shape}\"\n for b in range(batch_size):\n if cur_num_gen < self.args.n_codebooks-1:\n for jj in range(1, self.args.n_codebooks - cur_num_gen):\n samples[b, -jj, 0] = self.args.empty_token\n\n if (\n samples[b,0,0] == eog_inference or torch.argmax(logits[b,0], dim=-1) == eog_inference or y_input.shape[1] > x_lens[b] * (self.args.encodec_sr//5)\n ): # last one means y is already too long, shouldn't happen, but put it here\n samples[b,0,0] = eog_inference\n codebook_eog[0] = True\n keep = b # NOTE keep is a very important variable, we only return this one, note that if eog shows up in two samples, keep will be overwritten by the later one (or the last one)\n ##################### silence repetition handling #####################\n if samples[b,0,0] in silence_tokens and samples[b,0,0] == prev_tokens[b]:\n consec_silence_counts[b] += 1\n else:\n consec_silence_counts[b] = 0\n prev_tokens[b] = samples[b,0,0]\n ##################### silence repetition handling #####################\n return samples, codebook_eog, prev_tokens, consec_silence_counts, keep\n else:\n assert sum(codebook_eog[i] for i in range(n_eog)) == n_eog, f\"codebook_eog: {codebook_eog}, but n_eog: {n_eog}\"\n logits_adjust = logits\n for jj in range(n_eog+1,self.args.n_codebooks):\n logits_adjust[:,jj,eog_inference] = -10000\n logits_adjust[:,jj,self.args.empty_token] = -10000\n samples = topk_sampling(\n logits_adjust.reshape(batch_size * self.args.n_codebooks, logits_adjust.shape[-1]), top_k=top_k, top_p=top_p, temperature=temperature\n ) # [B, K, 1]\n samples = samples.reshape(batch_size, self.args.n_codebooks, 1)\n for jj in range(n_eog):\n samples[keep, jj, 0] = self.args.empty_token\n samples[keep, n_eog, 0] = eog_inference\n codebook_eog[n_eog] = True\n return samples, codebook_eog, prev_tokens, consec_silence_counts, keep\n while True:\n # if cur_num_gen > 0, should have everything in kvcache, so only pass in the last token\n # in the first generation step, we repeat each tensor to make their first dimension of length the batch size \n if cur_num_gen == 0:\n assert x_input.ndim == 3 and x_input.shape[0] == 1, x_input.shape\n assert x_padding_mask.ndim == 2 and x_padding_mask.shape[0] == 1, x_padding_mask.shape\n assert y_input.ndim == 3 and y_input.shape[0] == 1 and y_input.shape[1] == new_y_lens[0], y_input.shape\n assert embedded_y.ndim == 3 and embedded_y.shape[0] == 1 and embedded_y.shape[1] == new_y_lens[0], embedded_y.shape\n x_input = x_input.repeat(batch_size, 1, 1)\n x_lens = x_lens.repeat(batch_size)\n # x_attention_mask = x_attention_mask.repeat(batch_size, 1, 1) # no need to work with attention mask, it doesn't contain batch dimension\n x_padding_mask = x_padding_mask.repeat(batch_size, 1)\n y_input = y_input.repeat(batch_size, 1, 1)\n new_y_lens = new_y_lens.repeat(batch_size)\n # y_attention_mask = y_attention_mask.repeat(batch_size, 1, 1) # no need to work with attention mask, it doesn't contain batch dimension\n y_padding_mask = y_padding_mask.repeat(batch_size, 1)\n embedded_y = embedded_y.repeat(batch_size, 1, 1) # will be used to concat with newly generated token embedding\n past = past.repeat(1, 1, batch_size) if past != None else None\n else:\n assert x_input.shape[0] == batch_size and x_padding_mask.shape[0] == batch_size and y_input.shape[0] == batch_size and new_y_lens.shape[0] == batch_size, f\"x_input.shape: {x_input.shape}, x_padding_mask.shape: {x_padding_mask.shape}, y_input.shape: {y_input.shape}, new_y_lens.shape: {new_y_lens.shape}\"\n y_out, present = self.dec_forward(\n x_input, \n x_lens,\n x_attention_mask,\n x_padding_mask,\n y_input,\n new_y_lens,\n y_attention_mask,\n y_padding_mask,\n past=past\n )\n if past != None:\n past = torch.cat([past, present.to(past.dtype)], dim=-2) if past.ndim > 3 else present.to(past.dtype)\n\n # if no eog emerges, y_out should have batch size of batch_size\n if sum(codebook_eog) == 0:\n assert y_out.shape[0] == batch_size and y_out.ndim == 3, y_out.shape\n y_out = y_out[:, -1:] # only take the last token\n logits = torch.stack([self.predict_layer[i](y_out) for i in range(self.args.n_codebooks)], dim=1) # [B K S card], S==1, so [B K 1 card]\n logits = logits.squeeze(2) # [B K card]\n assert logits.shape == torch.Size((batch_size, self.args.n_codebooks, self.n_audio_tokens[0])), f\"{logits.shape}\"\n\n n_eog = sum(codebook_eog)\n if self.args.eos > 0:\n for jj in range(self.args.n_codebooks):\n logits[:,jj,self.args.eog] = -10000.\n samples, codebook_eog, prev_tokens, consec_silence_counts, keep = sample_helper(n_eog, logits, codebook_eog, top_k, top_p, temperature, prev_tokens, consec_silence_counts, stop_repetition, silence_tokens, cur_num_gen, keep)\n \n cur_num_gen += 1\n if sum(codebook_eog) == 0: # no eog yet, keep batch_size of samples\n assert keep == None\n for b in range(batch_size):\n cur_generated[b].append(samples[b].squeeze(-1))\n elif sum(codebook_eog) == 1: # the first eog just showed up in this step\n assert keep != None\n cur_generated = cur_generated[keep]\n cur_generated.append(samples[keep].squeeze(-1))\n else: # we are generating the rest eogs for the 'keep' sample \n cur_generated.append(samples[keep].squeeze(-1))\n\n # samples.shape is [K,1]\n # ge samples_emb\n samples_emb = torch.stack([self.audio_embedding[k](samples[:, k]) for k in range(self.args.n_codebooks)], dim=1) # [B, K,1,D]\n assert samples_emb.shape == torch.Size([batch_size, self.args.n_codebooks, 1, self.args.d_model])\n samples_emb = samples_emb.sum(dim=1,keepdim=False) # [B,1,D]\n if sum(codebook_eog) == self.args.n_codebooks: # generation for the current span is done\n codebook_eog = [False] * self.args.n_codebooks\n num_gen.append(cur_num_gen)\n cur_num_gen = 0\n generated.append(cur_generated)\n cur_generated = [[] for _ in range(batch_size)]\n break\n else:\n assert samples_emb.shape == torch.Size((batch_size,1,self.args.d_model)), f\"samples_emb.shape: {samples_emb.shape}\"\n \n embedded_y = torch.cat([embedded_y, samples_emb], dim=1)\n y_input = self.audio_positional_embedding(embedded_y) # [B T D]\n # make attention mask and padding mask\n y_attention_mask = torch.triu(torch.ones(y_input.shape[1], y_input.shape[1]), diagonal=1).bool().to(y.device)\n new_y_lens = torch.LongTensor([y_input.shape[1]]).to(y.device).repeat(batch_size)\n y_padding_mask = torch.full((batch_size,new_y_lens[0]), False).to(y.device)\n \n assert len(generated) == 1, f\"len(generated): {len(generated)}\"\n\n # revert the pattern\n flatten_gen = []\n for l, orig_span in enumerate(generated):\n span = torch.stack(orig_span, dim=0) # [T, K]\n span = span.transpose(1,0) # [K, T]\n assert span.shape[0] == self.args.n_codebooks, span.shape\n unshifted_span = []\n for j, s in enumerate(span):\n start_from = j\n end_at = - (self.args.n_codebooks - start_from)\n unshifted_span.append(s[start_from:end_at])\n unshifted_span = torch.stack(unshifted_span, dim=0)\n\n assert unshifted_span.shape[1] == num_gen[l] - self.args.n_codebooks, f\"len(unshifted_spans[0]): {len(unshifted_span[0])}, num_gen[l]: {num_gen[l]}\"\n\n flatten_gen.append(unshifted_span)\n assert len(flatten_gen) == 1, len(flatten_gen)\n \n # combine \n res = [y[0], flatten_gen[0]]\n res = torch.cat(res, dim=1).unsqueeze(0) # [K, new_t] -> [1, K, new_T]\n\n expected_y_len = y_len + sum([item - self.args.n_codebooks for item in num_gen])\n assert res.shape == torch.Size((1, self.args.n_codebooks, expected_y_len)), f\"res.shape: {res.shape}, expected_y_len: {expected_y_len}. y_len + sum([item - self.args.n_codebooks for item in num_gen]): {y_len} + {sum([item - self.args.n_codebooks for item in num_gen])}\"\n\n if self.args.special_first:\n res = res - int(self.args.n_special)\n flatten_gen = flatten_gen - int(self.args.n_special)\n\n return res, flatten_gen[0].unsqueeze(0)\n" }, { "path": "predict.py", "content": "# Prediction interface for Cog ⚙️\n# https://github.com/replicate/cog/blob/main/docs/python.md\n\nimport os\nimport time\nimport random\nimport getpass\nimport shutil\nimport subprocess\nimport torch\nimport numpy as np\nimport torchaudio\nfrom cog import BasePredictor, Input, Path, BaseModel\n\nos.environ[\"USER\"] = getpass.getuser()\n\nfrom data.tokenizer import (\n AudioTokenizer,\n TextTokenizer,\n)\nfrom models import voicecraft\nfrom inference_tts_scale import inference_one_sample\nfrom edit_utils import get_span\nfrom inference_speech_editing_scale import (\n inference_one_sample as inference_one_sample_editing,\n)\n\n\nMODEL_URL = \"https://weights.replicate.delivery/default/pyp1/VoiceCraft-models.tar\" # all the models are cached and uploaded to replicate.delivery for faster booting\nMODEL_CACHE = \"model_cache\"\n\n\nclass ModelOutput(BaseModel):\n whisper_transcript_orig_audio: str\n generated_audio: Path\n\n\nclass WhisperxAlignModel:\n def __init__(self):\n from whisperx import load_align_model\n\n self.model, self.metadata = load_align_model(\n language_code=\"en\", device=\"cuda:0\"\n )\n\n def align(self, segments, audio_path):\n from whisperx import align, load_audio\n\n audio = load_audio(audio_path)\n return align(\n segments,\n self.model,\n self.metadata,\n audio,\n device=\"cuda:0\",\n return_char_alignments=False,\n )[\"segments\"]\n\n\nclass WhisperxModel:\n def __init__(self, model_name, align_model: WhisperxAlignModel, device=\"cuda\"):\n from whisperx import load_model\n\n # the model weights are cached from Systran/faster-whisper-base.en etc\n self.model = load_model(\n model_name,\n device,\n asr_options={\n \"suppress_numerals\": True,\n \"max_new_tokens\": None,\n \"clip_timestamps\": None,\n \"hallucination_silence_threshold\": None,\n },\n )\n self.align_model = align_model\n\n def transcribe(self, audio_path):\n segments = self.model.transcribe(audio_path, language=\"en\", batch_size=8)[\n \"segments\"\n ]\n return self.align_model.align(segments, audio_path)\n\n\ndef download_weights(url, dest):\n start = time.time()\n print(\"downloading url: \", url)\n print(\"downloading to: \", dest)\n subprocess.check_call([\"pget\", \"-x\", url, dest], close_fds=False)\n print(\"downloading took: \", time.time() - start)\n\n\nclass Predictor(BasePredictor):\n def setup(self):\n \"\"\"Load the model into memory to make running multiple predictions efficient\"\"\"\n self.device = \"cuda\"\n\n if not os.path.exists(MODEL_CACHE):\n download_weights(MODEL_URL, MODEL_CACHE)\n\n encodec_fn = f\"{MODEL_CACHE}/encodec_4cb2048_giga.th\"\n self.models, self.ckpt, self.phn2num = {}, {}, {}\n for voicecraft_name in [\n \"giga830M.pth\",\n \"giga330M.pth\",\n \"gigaHalfLibri330M_TTSEnhanced_max16s.pth\",\n ]:\n ckpt_fn = f\"{MODEL_CACHE}/{voicecraft_name}\"\n\n self.ckpt[voicecraft_name] = torch.load(ckpt_fn, map_location=\"cpu\")\n self.models[voicecraft_name] = voicecraft.VoiceCraft(\n self.ckpt[voicecraft_name][\"config\"]\n )\n self.models[voicecraft_name].load_state_dict(\n self.ckpt[voicecraft_name][\"model\"]\n )\n self.models[voicecraft_name].to(self.device)\n self.models[voicecraft_name].eval()\n\n self.phn2num[voicecraft_name] = self.ckpt[voicecraft_name][\"phn2num\"]\n\n self.text_tokenizer = TextTokenizer(backend=\"espeak\")\n self.audio_tokenizer = AudioTokenizer(signature=encodec_fn, device=self.device)\n\n align_model = WhisperxAlignModel()\n self.transcribe_models = {\n k: WhisperxModel(f\"{MODEL_CACHE}/whisperx_{k.split('.')[0]}\", align_model)\n for k in [\"base.en\", \"small.en\", \"medium.en\"]\n }\n\n def predict(\n self,\n task: str = Input(\n description=\"Choose a task\",\n choices=[\n \"speech_editing-substitution\",\n \"speech_editing-insertion\",\n \"speech_editing-deletion\",\n \"zero-shot text-to-speech\",\n ],\n default=\"zero-shot text-to-speech\",\n ),\n voicecraft_model: str = Input(\n description=\"Choose a model\",\n choices=[\"giga830M.pth\", \"giga330M.pth\", \"giga330M_TTSEnhanced.pth\"],\n default=\"giga330M_TTSEnhanced.pth\",\n ),\n orig_audio: Path = Input(description=\"Original audio file\"),\n orig_transcript: str = Input(\n description=\"Optionally provide the transcript of the input audio. Leave it blank to use the WhisperX model below to generate the transcript. Inaccurate transcription may lead to error TTS or speech editing\",\n default=\"\",\n ),\n whisperx_model: str = Input(\n description=\"If orig_transcript is not provided above, choose a WhisperX model for generating the transcript. Inaccurate transcription may lead to error TTS or speech editing. You can modify the generated transcript and provide it directly to orig_transcript above\",\n choices=[\n \"base.en\",\n \"small.en\",\n \"medium.en\",\n ],\n default=\"base.en\",\n ),\n target_transcript: str = Input(\n description=\"Transcript of the target audio file\",\n ),\n cut_off_sec: float = Input(\n description=\"Only used for for zero-shot text-to-speech task. The first seconds of the original audio that are used for zero-shot text-to-speech. 3 sec of reference is generally enough for high quality voice cloning, but longer is generally better, try e.g. 3~6 sec\",\n default=3.01,\n ),\n kvcache: int = Input(\n description=\"Set to 0 to use less VRAM, but with slower inference\",\n choices=[0, 1],\n default=1,\n ),\n left_margin: float = Input(\n description=\"Margin to the left of the editing segment\",\n default=0.08,\n ),\n right_margin: float = Input(\n description=\"Margin to the right of the editing segment\",\n default=0.08,\n ),\n temperature: float = Input(\n description=\"Adjusts randomness of outputs, greater than 1 is random and 0 is deterministic. Do not recommend to change\",\n default=1,\n ),\n top_p: float = Input(\n description=\"Default value for TTS is 0.9, and 0.8 for speech editing\",\n default=1,\n ),\n stop_repetition: int = Input(\n default=3,\n description=\"Default value for TTS is 3, and -1 for speech editing. -1 means do not adjust prob of silence tokens. if there are long silence or unnaturally stretched words, increase sample_batch_size to 2, 3 or even 4\",\n ),\n sample_batch_size: int = Input(\n description=\"Default value for TTS is 4, and 1 for speech editing. The higher the number, the faster the output will be. Under the hood, the model will generate this many samples and choose the shortest one\",\n default=4,\n ),\n seed: int = Input(\n description=\"Random seed. Leave blank to randomize the seed\", default=None\n ),\n ) -> ModelOutput:\n \"\"\"Run a single prediction on the model\"\"\"\n\n if seed is None:\n seed = int.from_bytes(os.urandom(2), \"big\")\n print(f\"Using seed: {seed}\")\n\n seed_everything(seed)\n\n segments = self.transcribe_models[whisperx_model].transcribe(\n str(orig_audio)\n )\n\n state = get_transcribe_state(segments)\n\n whisper_transcript = state[\"transcript\"].strip()\n\n if len(orig_transcript.strip()) == 0:\n orig_transcript = whisper_transcript\n\n print(f\"The transcript from the Whisper model: {whisper_transcript}\")\n\n temp_folder = \"exp_dir\"\n if os.path.exists(temp_folder):\n shutil.rmtree(temp_folder)\n\n os.makedirs(temp_folder)\n\n filename = \"orig_audio\"\n audio_fn = str(orig_audio)\n\n info = torchaudio.info(audio_fn)\n audio_dur = info.num_frames / info.sample_rate\n\n # hyperparameters for inference\n codec_audio_sr = 16000\n codec_sr = 50\n top_k = 40\n silence_tokens = [1388, 1898, 131]\n\n if voicecraft_model == \"giga330M_TTSEnhanced.pth\":\n voicecraft_model = \"gigaHalfLibri330M_TTSEnhanced_max16s.pth\"\n\n if task == \"zero-shot text-to-speech\":\n assert (\n cut_off_sec < audio_dur\n ), f\"cut_off_sec {cut_off_sec} is larger than the audio duration {audio_dur}\"\n prompt_end_frame = int(cut_off_sec * info.sample_rate)\n\n idx = find_closest_cut_off_word(state[\"word_bounds\"], cut_off_sec)\n orig_transcript_until_cutoff_time = \" \".join(\n [word_bound[\"word\"] for word_bound in state[\"word_bounds\"][: idx + 1]]\n )\n else:\n edit_type = task.split(\"-\")[-1]\n orig_span, new_span = get_span(\n orig_transcript, target_transcript, edit_type\n )\n if orig_span[0] > orig_span[1]:\n RuntimeError(f\"example {audio_fn} failed\")\n if orig_span[0] == orig_span[1]:\n orig_span_save = [orig_span[0]]\n else:\n orig_span_save = orig_span\n if new_span[0] == new_span[1]:\n new_span_save = [new_span[0]]\n else:\n new_span_save = new_span\n orig_span_save = \",\".join([str(item) for item in orig_span_save])\n new_span_save = \",\".join([str(item) for item in new_span_save])\n\n start, end = get_mask_interval_from_word_bounds(\n state[\"word_bounds\"], orig_span_save, edit_type\n )\n\n # span in codec frames\n morphed_span = (\n max(start - left_margin, 1 / codec_sr),\n min(end + right_margin, audio_dur),\n ) # in seconds\n mask_interval = [\n [round(morphed_span[0] * codec_sr), round(morphed_span[1] * codec_sr)]\n ]\n mask_interval = torch.LongTensor(mask_interval) # [M,2], M==1 for now\n\n decode_config = {\n \"top_k\": top_k,\n \"top_p\": top_p,\n \"temperature\": temperature,\n \"stop_repetition\": stop_repetition,\n \"kvcache\": kvcache,\n \"codec_audio_sr\": codec_audio_sr,\n \"codec_sr\": codec_sr,\n \"silence_tokens\": silence_tokens,\n }\n\n if task == \"zero-shot text-to-speech\":\n decode_config[\"sample_batch_size\"] = sample_batch_size\n _, gen_audio = inference_one_sample(\n self.models[voicecraft_model],\n self.ckpt[voicecraft_model][\"config\"],\n self.phn2num[voicecraft_model],\n self.text_tokenizer,\n self.audio_tokenizer,\n audio_fn,\n orig_transcript_until_cutoff_time.strip()\n + \" \"\n + target_transcript.strip(),\n self.device,\n decode_config,\n prompt_end_frame,\n )\n else:\n _, gen_audio = inference_one_sample_editing(\n self.models[voicecraft_model],\n self.ckpt[voicecraft_model][\"config\"],\n self.phn2num[voicecraft_model],\n self.text_tokenizer,\n self.audio_tokenizer,\n audio_fn,\n target_transcript,\n mask_interval,\n self.device,\n decode_config,\n )\n\n # save segments for comparison\n gen_audio = gen_audio[0].cpu()\n\n out = \"/tmp/out.wav\"\n torchaudio.save(out, gen_audio, codec_audio_sr)\n return ModelOutput(\n generated_audio=Path(out), whisper_transcript_orig_audio=whisper_transcript\n )\n\n\ndef seed_everything(seed):\n os.environ[\"PYTHONHASHSEED\"] = str(seed)\n random.seed(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.benchmark = False\n torch.backends.cudnn.deterministic = True\n\n\ndef get_transcribe_state(segments):\n words_info = [word_info for segment in segments for word_info in segment[\"words\"]]\n return {\n \"transcript\": \" \".join([segment[\"text\"].strip() for segment in segments]),\n \"word_bounds\": [\n {\"word\": word[\"word\"], \"start\": word[\"start\"], \"end\": word[\"end\"]}\n for word in words_info\n ],\n }\n\n\ndef find_closest_cut_off_word(word_bounds, cut_off_sec):\n min_distance = float(\"inf\")\n\n for i, word_bound in enumerate(word_bounds):\n distance = abs(word_bound[\"start\"] - cut_off_sec)\n\n if distance < min_distance:\n min_distance = distance\n\n if word_bound[\"end\"] > cut_off_sec:\n break\n\n return i\n\n\ndef get_mask_interval_from_word_bounds(word_bounds, word_span_ind, editType):\n tmp = word_span_ind.split(\",\")\n s, e = int(tmp[0]), int(tmp[-1])\n start = None\n for j, item in enumerate(word_bounds):\n if j == s:\n if editType == \"insertion\":\n start = float(item[\"end\"])\n else:\n start = float(item[\"start\"])\n if j == e:\n if editType == \"insertion\":\n end = float(item[\"start\"])\n else:\n end = float(item[\"end\"])\n assert start is not None\n break\n return (start, end)\n" }, { "path": "pretrained_models/.gitkeep", "content": "" }, { "path": "start-jupyter.bat", "content": "@echo off\n\necho Creating and running the Jupyter container...\n\ndocker run -it -d ^\n --gpus all ^\n -p 8888:8888 ^\n -p 7860:7860 ^\n --name jupyter ^\n --user root ^\n -e NB_USER=\"%username%\" ^\n -e CHOWN_HOME=yes ^\n -e GRANT_SUDO=yes ^\n -e JUPYTER_TOKEN=mytoken ^\n -w \"/home/%username%\" ^\n -v \"%cd%\":\"/home/%username%/work\" ^\n voicecraft\n\nif %errorlevel% == 0 (\n echo Jupyter container created and running.\n\n echo Jupyter container is running.\n echo To access the Jupyter web UI, please follow these steps:\n echo 1. Open your web browser\n echo 2. Navigate to http://localhost:8888/?token=mytoken\n echo 3. !! The default token is \"mytoken\" and should be changed. !!\n pause\n) else (\n echo Failed to create and run the Jupyter container.\n)\n" }, { "path": "start-jupyter.sh", "content": "#!/usr/bin/env bash\n## Assumes you have docker installed with nvidia container container-toolkit\n# https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/1.13.5/install-guide.html\n# sudo apt-get install -y nvidia-container-toolkit-base || yay -Syu nvidia-container-toolkit || echo etc...\n## Try to start an existing container otherwise create a new one\ndocker start jupyter 2> /dev/null || \\\ndocker run -it \\\n -d \\\n --gpus all \\\n -p 8888:8888 \\\n -p 7860:7860 \\\n --name jupyter \\\n --user root \\\n -e NB_USER=\"$USER\" \\\n -e CHOWN_HOME=yes \\\n -e GRANT_SUDO=yes \\\n -w \"/home/${NB_USER}\" \\\n -v \"$PWD\":\"/home/$USER/work\" \\\n voicecraft\n\n## `docker logs jupyter` to get the URL link and token e.g.\n## http://127.0.0.1:8888/lab?token=blahblahblahblabhlaabhalbhalbhal\n" }, { "path": "steps/__init__.py", "content": "" }, { "path": "steps/optim.py", "content": "# Copyright 2022 Xiaomi Corp. (authors: Daniel Povey)\n#\n# See ../LICENSE for clarification regarding multiple authors\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport contextlib\nimport logging\nimport random\nfrom collections import defaultdict\nfrom typing import List, Optional, Tuple, Union\n\nimport torch\nimport torch.nn as nn\nfrom torch import Tensor\nfrom torch.optim import Optimizer\n\n\nclass BatchedOptimizer(Optimizer):\n \"\"\"\n This class adds to class Optimizer the capability to optimize parameters in batches:\n it will stack the parameters and their grads for you so the optimizer can work\n on tensors with an extra leading dimension. This is intended for speed with GPUs,\n as it reduces the number of kernels launched in the optimizer.\n\n Args:\n params:\n \"\"\"\n\n def __init__(self, params, defaults):\n super(BatchedOptimizer, self).__init__(params, defaults)\n\n @contextlib.contextmanager\n def batched_params(self, param_group, group_params_names):\n \"\"\"\n This function returns (technically, yields) a list of\n of tuples (p, state), where\n p is a `fake` parameter that is stacked (over axis 0) from real parameters\n that share the same shape, and its gradient is also stacked;\n `state` is the state corresponding to this batch of parameters\n (it will be physically located in the \"state\" for one of the real\n parameters, the last one that has any particular shape and dtype).\n\n This function is decorated as a context manager so that it can\n write parameters back to their \"real\" locations.\n\n The idea is, instead of doing:\n \n for p in group[\"params\"]:\n state = self.state[p]\n ...\n \n you can do:\n \n with self.batched_params(group[\"params\"]) as batches:\n for p, state, p_names in batches:\n ...\n \n\n Args:\n group: a parameter group, which is a list of parameters; should be\n one of self.param_groups.\n group_params_names: name for each parameter in group,\n which is List[str].\n \"\"\"\n batches = defaultdict(\n list\n ) # `batches` maps from tuple (dtype_as_str,*shape) to list of nn.Parameter\n batches_names = defaultdict(\n list\n ) # `batches` maps from tuple (dtype_as_str,*shape) to list of str\n\n assert len(param_group) == len(group_params_names), f\"len(param_group): {len(param_group)}, len(group_params_names): {len(group_params_names)}\"\n for p, named_p in zip(param_group, group_params_names):\n key = (str(p.dtype), *p.shape)\n batches[key].append(p)\n batches_names[key].append(named_p)\n\n batches_names_keys = list(batches_names.keys())\n sorted_idx = sorted(\n range(len(batches_names)), key=lambda i: batches_names_keys[i]\n )\n batches_names = [\n batches_names[batches_names_keys[idx]] for idx in sorted_idx\n ]\n batches = [batches[batches_names_keys[idx]] for idx in sorted_idx]\n\n stacked_params_dict = dict()\n\n # turn batches into a list, in deterministic order.\n # tuples will contain tuples of (stacked_param, state, stacked_params_names),\n # one for each batch in `batches`.\n tuples = []\n\n for batch, batch_names in zip(batches, batches_names):\n p = batch[0]\n # we arbitrarily store the state in the\n # state corresponding to the 1st parameter in the\n # group. class Optimizer will take care of saving/loading state.\n state = self.state[p]\n p_stacked = torch.stack(batch)\n grad = torch.stack(\n [\n torch.zeros_like(p) if p.grad is None else p.grad\n for p in batch\n ]\n )\n p_stacked.grad = grad\n stacked_params_dict[key] = p_stacked\n tuples.append((p_stacked, state, batch_names))\n\n yield tuples # <-- calling code will do the actual optimization here!\n\n for ((stacked_params, _state, _names), batch) in zip(tuples, batches):\n for i, p in enumerate(batch): # batch is list of Parameter\n p.copy_(stacked_params[i])\n\n\nclass ScaledAdam(BatchedOptimizer):\n \"\"\"\n Implements 'Scaled Adam', a variant of Adam where we scale each parameter's update\n proportional to the norm of that parameter; and also learn the scale of the parameter,\n in log space, subject to upper and lower limits (as if we had factored each parameter as\n param = underlying_param * log_scale.exp())\n\n\n Args:\n params: The parameters or param_groups to optimize (like other Optimizer subclasses)\n lr: The learning rate. We will typically use a learning rate schedule that starts\n at 0.03 and decreases over time, i.e. much higher than other common\n optimizers.\n clipping_scale: (e.g. 2.0)\n A scale for gradient-clipping: if specified, the normalized gradients\n over the whole model will be clipped to have 2-norm equal to\n `clipping_scale` times the median 2-norm over the most recent period\n of `clipping_update_period` minibatches. By \"normalized gradients\",\n we mean after multiplying by the rms parameter value for this tensor\n [for non-scalars]; this is appropriate because our update is scaled\n by this quantity.\n betas: beta1,beta2 are momentum constants for regular momentum, and moving sum-sq grad.\n Must satisfy 0 < beta <= beta2 < 1.\n scalar_lr_scale: A scaling factor on the learning rate, that we use to update the\n scale of each parameter tensor and scalar parameters of the mode..\n If each parameter were decomposed\n as p * p_scale.exp(), where (p**2).mean().sqrt() == 1.0, scalar_lr_scale\n would be a the scaling factor on the learning rate of p_scale.\n eps: A general-purpose epsilon to prevent division by zero\n param_min_rms: Minimum root-mean-square value of parameter tensor, for purposes of\n learning the scale on the parameters (we'll constrain the rms of each non-scalar\n parameter tensor to be >= this value)\n param_max_rms: Maximum root-mean-square value of parameter tensor, for purposes of\n learning the scale on the parameters (we'll constrain the rms of each non-scalar\n parameter tensor to be <= this value)\n scalar_max: Maximum absolute value for scalar parameters (applicable if your\n model has any parameters with numel() == 1).\n size_update_period: The periodicity, in steps, with which we update the size (scale)\n of the parameter tensor. This is provided to save a little time\n in the update.\n clipping_update_period: if clipping_scale is specified, this is the period\n \"\"\"\n\n def __init__(\n self,\n params,\n lr=3e-02,\n clipping_scale=None,\n betas=(0.9, 0.98),\n scalar_lr_scale=0.1,\n eps=1.0e-08,\n param_min_rms=1.0e-05,\n param_max_rms=3.0,\n scalar_max=10.0,\n size_update_period=4,\n clipping_update_period=100,\n parameters_names=None,\n show_dominant_parameters=True,\n ):\n\n assert parameters_names is not None, (\n \"Please prepare parameters_names,\"\n \"which is a List[List[str]]. Each List[str] is for a group\"\n \"and each str is for a parameter\"\n )\n defaults = dict(\n lr=lr,\n clipping_scale=clipping_scale,\n betas=betas,\n scalar_lr_scale=scalar_lr_scale,\n eps=eps,\n param_min_rms=param_min_rms,\n param_max_rms=param_max_rms,\n scalar_max=scalar_max,\n size_update_period=size_update_period,\n clipping_update_period=clipping_update_period,\n )\n\n super(ScaledAdam, self).__init__(params, defaults)\n assert len(self.param_groups) == len(parameters_names)\n self.parameters_names = parameters_names\n self.show_dominant_parameters = show_dominant_parameters\n\n def __setstate__(self, state):\n super(ScaledAdam, self).__setstate__(state)\n\n @torch.no_grad()\n def step(self, closure=None):\n \"\"\"Performs a single optimization step.\n\n Arguments:\n closure (callable, optional): A closure that reevaluates the model\n and returns the loss.\n \"\"\"\n loss = None\n if closure is not None:\n with torch.enable_grad():\n loss = closure()\n\n batch = True\n\n for group, group_params_names in zip(\n self.param_groups, self.parameters_names\n ):\n\n with self.batched_params(\n group[\"params\"], group_params_names\n ) as batches:\n\n # batches is list of pairs (stacked_param, state). stacked_param is like\n # a regular parameter, and will have a .grad, but the 1st dim corresponds to\n # a stacking dim, it is not a real dim.\n\n if (\n len(batches[0][1]) == 0\n ): # if len(first state) == 0: not yet initialized\n clipping_scale = 1\n else:\n clipping_scale = self._get_clipping_scale(group, batches)\n\n for p, state, _ in batches:\n # Perform optimization step.\n # grad is not going to be None, we handled that when creating the batches.\n grad = p.grad\n if grad.is_sparse:\n raise RuntimeError(\n \"ScaledAdam optimizer does not support sparse gradients\"\n )\n # State initialization\n if len(state) == 0:\n self._init_state(group, p, state)\n\n self._step_one_batch(group, p, state, clipping_scale)\n\n return loss\n\n def _init_state(self, group: dict, p: Tensor, state: dict):\n \"\"\"\n Initializes state dict for parameter 'p'. Assumes that dim 0 of tensor p\n is actually the batch dimension, corresponding to batched-together\n parameters of a given shape.\n\n\n Args:\n group: Dict to look up configuration values.\n p: The parameter that we are initializing the state for\n state: Dict from string to whatever state we are initializing\n \"\"\"\n size_update_period = group[\"size_update_period\"]\n\n state[\"step\"] = 0\n\n kwargs = {\"device\": p.device, \"dtype\": p.dtype}\n\n # 'delta' implements conventional momentum. There are\n # several different kinds of update going on, so rather than\n # compute \"exp_avg\" like in Adam, we store and decay a\n # parameter-change \"delta\", which combines all forms of\n # update. this is equivalent to how it's done in Adam,\n # except for the first few steps.\n state[\"delta\"] = torch.zeros_like(\n p, memory_format=torch.preserve_format\n )\n\n batch_size = p.shape[0]\n numel = p.numel() // batch_size\n numel = p.numel()\n\n if numel > 1:\n # \"param_rms\" just periodically records the scalar root-mean-square value of\n # the parameter tensor.\n # it has a shape like (batch_size, 1, 1, 1, 1)\n param_rms = (\n (p ** 2).mean(dim=list(range(1, p.ndim)), keepdim=True).sqrt()\n )\n state[\"param_rms\"] = param_rms\n\n state[\"scale_exp_avg_sq\"] = torch.zeros_like(param_rms)\n state[\"scale_grads\"] = torch.zeros(\n size_update_period, *param_rms.shape, **kwargs\n )\n\n # exp_avg_sq is the weighted sum of scaled gradients. as in Adam.\n state[\"exp_avg_sq\"] = torch.zeros_like(\n p, memory_format=torch.preserve_format\n )\n\n def _get_clipping_scale(\n self, group: dict, tuples: List[Tuple[Tensor, dict, List[str]]]\n ) -> float:\n \"\"\"\n Returns a scalar factor <= 1.0 that dictates gradient clipping, i.e. we will scale the gradients\n by this amount before applying the rest of the update.\n\n Args:\n group: the parameter group, an item in self.param_groups\n tuples: a list of tuples of (param, state, param_names)\n where param is a batched set of parameters,\n with a .grad (1st dim is batch dim)\n and state is the state-dict where optimization parameters are kept.\n param_names is a List[str] while each str is name for a parameter\n in batched set of parameters \"param\".\n \"\"\"\n assert len(tuples) >= 1\n clipping_scale = group[\"clipping_scale\"]\n (first_p, first_state, _) = tuples[0]\n step = first_state[\"step\"]\n if clipping_scale is None or step == 0:\n # no clipping. return early on step == 0 because the other\n # parameters' state won't have been initialized yet.\n return 1.0\n clipping_update_period = group[\"clipping_update_period\"]\n\n tot_sumsq = torch.tensor(0.0, device=first_p.device)\n for (p, state, param_names) in tuples:\n grad = p.grad\n if grad.is_sparse:\n raise RuntimeError(\n \"ScaledAdam optimizer does not support sparse gradients\"\n )\n if p.numel() == p.shape[0]: # a batch of scalars\n tot_sumsq += (\n grad ** 2\n ).sum() # sum() to change shape [1] to []\n else:\n tot_sumsq += ((grad * state[\"param_rms\"]) ** 2).sum()\n\n tot_norm = tot_sumsq.sqrt()\n if \"model_norms\" not in first_state:\n first_state[\"model_norms\"] = torch.zeros(\n clipping_update_period, device=p.device\n )\n first_state[\"model_norms\"][step % clipping_update_period] = tot_norm\n\n if step % clipping_update_period == 0:\n # Print some stats.\n # We don't reach here if step == 0 because we would have returned\n # above.\n sorted_norms = first_state[\"model_norms\"].sort()[0].to(\"cpu\")\n quartiles = []\n for n in range(0, 5):\n index = min(\n clipping_update_period - 1,\n (clipping_update_period // 4) * n,\n )\n quartiles.append(sorted_norms[index].item())\n\n median = quartiles[2]\n threshold = clipping_scale * median\n first_state[\"model_norm_threshold\"] = threshold\n percent_clipped = (\n first_state[\"num_clipped\"] * 100.0 / clipping_update_period\n if \"num_clipped\" in first_state\n else 0.0\n )\n first_state[\"num_clipped\"] = 0\n quartiles = \" \".join([\"%.3e\" % x for x in quartiles])\n logging.info(\n f\"Clipping_scale={clipping_scale}, grad-norm quartiles {quartiles}, \"\n f\"threshold={threshold:.3e}, percent-clipped={percent_clipped:.1f}\"\n )\n\n if step < clipping_update_period:\n return 1.0 # We have not yet estimated a norm to clip to.\n else:\n try:\n model_norm_threshold = first_state[\"model_norm_threshold\"]\n except KeyError:\n logging.info(\n \"Warning: model_norm_threshold not in state: possibly \"\n \"you changed config when restarting, adding clipping_scale option?\"\n )\n return 1.0\n ans = min(1.0, (model_norm_threshold / (tot_norm + 1.0e-20)).item())\n if ans < 1.0:\n first_state[\"num_clipped\"] += 1\n if ans < 0.1:\n logging.warn(\n f\"Scaling gradients by {ans}, model_norm_threshold={model_norm_threshold}\"\n )\n if self.show_dominant_parameters:\n assert p.shape[0] == len(param_names)\n self._show_gradient_dominating_parameter(tuples, tot_sumsq)\n return ans\n\n def _show_gradient_dominating_parameter(\n self, tuples: List[Tuple[Tensor, dict, List[str]]], tot_sumsq: Tensor\n ):\n \"\"\"\n Show information of parameter wihch dominanting tot_sumsq.\n\n Args:\n tuples: a list of tuples of (param, state, param_names)\n where param is a batched set of parameters,\n with a .grad (1st dim is batch dim)\n and state is the state-dict where optimization parameters are kept.\n param_names is a List[str] while each str is name for a parameter\n in batched set of parameters \"param\".\n tot_sumsq: sumsq of all parameters. Though it's could be calculated\n from tuples, we still pass it to save some time.\n \"\"\"\n all_sumsq_orig = {}\n for (p, state, batch_param_names) in tuples:\n # p is a stacked batch parameters.\n batch_grad = p.grad\n if p.numel() == p.shape[0]: # a batch of scalars\n batch_sumsq_orig = batch_grad ** 2\n # Dummpy values used by following `zip` statement.\n batch_rms_orig = torch.ones(p.shape[0])\n else:\n batch_rms_orig = state[\"param_rms\"]\n batch_sumsq_orig = ((batch_grad * batch_rms_orig) ** 2).sum(\n dim=list(range(1, batch_grad.ndim))\n )\n\n for name, sumsq_orig, rms, grad in zip(\n batch_param_names, batch_sumsq_orig, batch_rms_orig, batch_grad\n ):\n\n proportion_orig = sumsq_orig / tot_sumsq\n all_sumsq_orig[name] = (proportion_orig, sumsq_orig, rms, grad)\n\n assert torch.isclose(\n sum([value[0] for value in all_sumsq_orig.values()]).cpu(),\n torch.tensor(1.0),\n )\n sorted_by_proportion = {\n k: v\n for k, v in sorted(\n all_sumsq_orig.items(),\n key=lambda item: item[1][0],\n reverse=True,\n )\n }\n dominant_param_name = next(iter(sorted_by_proportion))\n (\n dominant_proportion,\n dominant_sumsq,\n dominant_rms,\n dominant_grad,\n ) = sorted_by_proportion[dominant_param_name]\n logging.info(\n f\"Parameter Dominanting tot_sumsq {dominant_param_name}\"\n f\" with proportion {dominant_proportion:.2f},\"\n f\" where dominant_sumsq=(grad_sumsq*orig_rms_sq)\"\n f\"={dominant_sumsq:.3e},\"\n f\" grad_sumsq = {(dominant_grad**2).sum():.3e},\"\n f\" orig_rms_sq={(dominant_rms**2).item():.3e}\"\n )\n\n def _step_one_batch(\n self, group: dict, p: Tensor, state: dict, clipping_scale: float\n ):\n \"\"\"\n Do the step for one parameter, which is actually going to be a batch of\n `real` parameters, with dim 0 as the batch dim.\n Args:\n group: dict to look up configuration values\n p: parameter to update (actually multiple parameters stacked together\n as a batch)\n state: state-dict for p, to look up the optimizer state\n \"\"\"\n lr = group[\"lr\"]\n size_update_period = group[\"size_update_period\"]\n beta1 = group[\"betas\"][0]\n\n grad = p.grad\n if clipping_scale != 1.0:\n grad = grad * clipping_scale\n step = state[\"step\"]\n delta = state[\"delta\"]\n\n delta.mul_(beta1)\n batch_size = p.shape[0]\n numel = p.numel() // batch_size\n if numel > 1:\n # Update the size/scale of p, and set param_rms\n scale_grads = state[\"scale_grads\"]\n scale_grads[step % size_update_period] = (p * grad).sum(\n dim=list(range(1, p.ndim)), keepdim=True\n )\n if step % size_update_period == size_update_period - 1:\n param_rms = state[\"param_rms\"] # shape: (batch_size, 1, 1, ..)\n param_rms.copy_(\n (p ** 2)\n .mean(dim=list(range(1, p.ndim)), keepdim=True)\n .sqrt()\n )\n if step > 0:\n # self._size_update() learns the overall scale on the\n # parameter, by shrinking or expanding it.\n self._size_update(group, scale_grads, p, state)\n\n if numel == 1:\n # For parameters with 1 element we just use regular Adam.\n # Updates delta.\n self._step_scalar(group, p, state)\n else:\n self._step(group, p, state)\n\n state[\"step\"] = step + 1\n\n def _size_update(\n self, group: dict, scale_grads: Tensor, p: Tensor, state: dict\n ) -> None:\n \"\"\"\n Called only where p.numel() > 1, this updates the scale of the parameter.\n If we imagine: p = underlying_param * scale.exp(), and we are doing\n gradient descent on underlying param and on scale, this function does the update\n on `scale`.\n\n Args:\n group: dict to look up configuration values\n scale_grads: a tensor of shape (size_update_period, batch_size, 1, 1,...) containing\n grads w.r.t. the scales.\n p: The parameter to update\n state: The state-dict of p\n \"\"\"\n\n param_rms = state[\"param_rms\"]\n beta1, beta2 = group[\"betas\"]\n size_lr = group[\"lr\"] * group[\"scalar_lr_scale\"]\n param_min_rms = group[\"param_min_rms\"]\n param_max_rms = group[\"param_max_rms\"]\n eps = group[\"eps\"]\n step = state[\"step\"]\n batch_size = p.shape[0]\n\n size_update_period = scale_grads.shape[0]\n # correct beta2 for the size update period: we will have\n # faster decay at this level.\n beta2_corr = beta2 ** size_update_period\n\n scale_exp_avg_sq = state[\n \"scale_exp_avg_sq\"\n ] # shape: (batch_size, 1, 1, ..)\n scale_exp_avg_sq.mul_(beta2_corr).add_(\n (scale_grads ** 2).mean(\n dim=0\n ), # mean over dim `size_update_period`\n alpha=1 - beta2_corr,\n ) # shape is (batch_size, 1, 1, ...)\n\n # The 1st time we reach here is when size_step == 1.\n size_step = (step + 1) // size_update_period\n bias_correction2 = 1 - beta2_corr ** size_step\n # we don't bother with bias_correction1; this will help prevent divergence\n # at the start of training.\n\n denom = scale_exp_avg_sq.sqrt() + eps\n\n scale_step = (\n -size_lr\n * (bias_correction2 ** 0.5)\n * scale_grads.sum(dim=0)\n / denom\n )\n\n is_too_small = param_rms < param_min_rms\n is_too_large = param_rms > param_max_rms\n\n # when the param gets too small, just don't shrink it any further.\n scale_step.masked_fill_(is_too_small, 0.0)\n # when it gets too large, stop it from getting any larger.\n scale_step.masked_fill_(is_too_large, -size_lr * size_update_period)\n delta = state[\"delta\"]\n # the factor of (1-beta1) relates to momentum.\n delta.add_(p * scale_step, alpha=(1 - beta1))\n\n def _step(self, group: dict, p: Tensor, state: dict):\n \"\"\"\n This function does the core update of self.step(), in the case where the members of\n the batch have more than 1 element.\n\n Args:\n group: A dict which will be used to look up configuration values\n p: The parameter to be updated\n grad: The grad of p\n state: The state-dict corresponding to parameter p\n\n This function modifies p.\n \"\"\"\n grad = p.grad\n lr = group[\"lr\"]\n beta1, beta2 = group[\"betas\"]\n eps = group[\"eps\"]\n param_min_rms = group[\"param_min_rms\"]\n step = state[\"step\"]\n\n exp_avg_sq = state[\"exp_avg_sq\"]\n exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=(1 - beta2))\n\n this_step = state[\"step\"] - (\n state[\"zero_step\"] if \"zero_step\" in state else 0\n )\n bias_correction2 = 1 - beta2 ** (this_step + 1)\n if bias_correction2 < 0.99:\n # note: not in-place.\n exp_avg_sq = exp_avg_sq * (1.0 / bias_correction2)\n\n denom = exp_avg_sq.sqrt()\n denom += eps\n grad = grad / denom\n\n alpha = -lr * (1 - beta1) * state[\"param_rms\"].clamp(min=param_min_rms)\n\n delta = state[\"delta\"]\n delta.add_(grad * alpha)\n p.add_(delta)\n\n def _step_scalar(self, group: dict, p: Tensor, state: dict):\n \"\"\"\n A simplified form of the core update for scalar tensors, where we cannot get a good\n estimate of the parameter rms.\n \"\"\"\n beta1, beta2 = group[\"betas\"]\n scalar_max = group[\"scalar_max\"]\n eps = group[\"eps\"]\n lr = group[\"lr\"] * group[\"scalar_lr_scale\"]\n grad = p.grad\n\n exp_avg_sq = state[\"exp_avg_sq\"] # shape: (batch_size,)\n exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)\n\n # bias_correction2 is like in Adam. Don't bother with bias_correction1;\n # slower update at the start will help stability anyway.\n bias_correction2 = 1 - beta2 ** (state[\"step\"] + 1)\n denom = (exp_avg_sq / bias_correction2).sqrt() + eps\n\n delta = state[\"delta\"]\n delta.add_(grad / denom, alpha=-lr * (1 - beta1))\n p.clamp_(min=-scalar_max, max=scalar_max)\n p.add_(delta)\n\n\nclass LRScheduler(object):\n \"\"\"\n Base-class for learning rate schedulers where the learning-rate depends on both the\n batch and the epoch.\n \"\"\"\n\n def __init__(self, optimizer: Optimizer, verbose: bool = False):\n # Attach optimizer\n if not isinstance(optimizer, Optimizer):\n raise TypeError(\n \"{} is not an Optimizer\".format(type(optimizer).__name__)\n )\n self.optimizer = optimizer\n self.verbose = verbose\n\n for group in optimizer.param_groups:\n group.setdefault(\"base_lr\", group[\"lr\"])\n\n self.base_lrs = [group[\"base_lr\"] for group in optimizer.param_groups]\n\n self.epoch = 0\n self.batch = 0\n\n def state_dict(self):\n \"\"\"Returns the state of the scheduler as a :class:`dict`.\n\n It contains an entry for every variable in self.__dict__ which\n is not the optimizer.\n \"\"\"\n return {\n \"base_lrs\": self.base_lrs,\n \"epoch\": self.epoch,\n \"batch\": self.batch,\n }\n\n def load_state_dict(self, state_dict):\n \"\"\"Loads the schedulers state.\n\n Args:\n state_dict (dict): scheduler state. Should be an object returned\n from a call to :meth:`state_dict`.\n \"\"\"\n self.__dict__.update(state_dict)\n\n def get_last_lr(self) -> List[float]:\n \"\"\"Return last computed learning rate by current scheduler. Will be a list of float.\"\"\"\n return self._last_lr\n\n def get_lr(self):\n # Compute list of learning rates from self.epoch and self.batch and\n # self.base_lrs; this must be overloaded by the user.\n # e.g. return [some_formula(self.batch, self.epoch, base_lr) for base_lr in self.base_lrs ]\n raise NotImplementedError\n\n def step_batch(self, batch: Optional[int] = None) -> None:\n # Step the batch index, or just set it. If `batch` is specified, it\n # must be the batch index from the start of training, i.e. summed over\n # all epochs.\n # You can call this in any order; if you don't provide 'batch', it should\n # of course be called once per batch.\n if batch is not None:\n self.batch = batch\n else:\n self.batch = self.batch + 1\n self._set_lrs()\n\n def step_epoch(self, epoch: Optional[int] = None):\n # Step the epoch index, or just set it. If you provide the 'epoch' arg,\n # you should call this at the start of the epoch; if you don't provide the 'epoch'\n # arg, you should call it at the end of the epoch.\n if epoch is not None:\n self.epoch = epoch\n else:\n self.epoch = self.epoch + 1\n self._set_lrs()\n\n def _set_lrs(self):\n values = self.get_lr()\n assert len(values) == len(self.optimizer.param_groups)\n\n for i, data in enumerate(zip(self.optimizer.param_groups, values)):\n param_group, lr = data\n param_group[\"lr\"] = lr\n self.print_lr(self.verbose, i, lr)\n self._last_lr = [group[\"lr\"] for group in self.optimizer.param_groups]\n\n def print_lr(self, is_verbose, group, lr):\n \"\"\"Display the current learning rate.\"\"\"\n if is_verbose:\n logging.info(\n f\"Epoch={self.epoch}, batch={self.batch}: adjusting learning rate\"\n f\" of group {group} to {lr:.4e}.\"\n )\n\n\nclass Eden(LRScheduler):\n \"\"\"\n Eden scheduler.\n The basic formula (before warmup) is:\n lr = base_lr * (((batch**2 + lr_batches**2) / lr_batches**2) ** -0.25 *\n (((epoch**2 + lr_epochs**2) / lr_epochs**2) ** -0.25)) * warmup\n where `warmup` increases from linearly 0.5 to 1 over `warmup_batches` batches\n and then stays constant at 1.\n\n\n E.g. suggest base_lr = 0.04 (passed to optimizer) if used with ScaledAdam\n\n Args:\n optimizer: the optimizer to change the learning rates on\n lr_batches: the number of batches after which we start significantly\n decreasing the learning rate, suggest 5000.\n lr_epochs: the number of epochs after which we start significantly\n decreasing the learning rate, suggest 6 if you plan to do e.g.\n 20 to 40 epochs, but may need smaller number if dataset is huge\n and you will do few epochs.\n \"\"\"\n\n def __init__(\n self,\n optimizer: Optimizer,\n lr_batches: Union[int, float],\n lr_epochs: Union[int, float],\n warmup_batches: Union[int, float] = 500.0,\n verbose: bool = False,\n ):\n super(Eden, self).__init__(optimizer, verbose)\n self.lr_batches = lr_batches\n self.lr_epochs = lr_epochs\n self.warmup_batches = warmup_batches\n\n def get_lr(self):\n factor = (\n (self.batch ** 2 + self.lr_batches ** 2) / self.lr_batches ** 2\n ) ** -0.25 * (\n ((self.epoch ** 2 + self.lr_epochs ** 2) / self.lr_epochs ** 2)\n ** -0.25\n )\n warmup_factor = (\n 1.0\n if self.batch >= self.warmup_batches\n else 0.5 + 0.5 * (self.batch / self.warmup_batches)\n )\n\n return [x * factor * warmup_factor for x in self.base_lrs]\n\n\ndef _test_eden():\n m = torch.nn.Linear(100, 100)\n optim = ScaledAdam(m.parameters(), lr=0.03)\n\n scheduler = Eden(optim, lr_batches=100, lr_epochs=2, verbose=True)\n\n for epoch in range(10):\n scheduler.step_epoch(epoch) # sets epoch to `epoch`\n\n for step in range(20):\n x = torch.randn(200, 100).detach()\n x.requires_grad = True\n y = m(x)\n dy = torch.randn(200, 100).detach()\n f = (y * dy).sum()\n f.backward()\n\n optim.step()\n scheduler.step_batch()\n optim.zero_grad()\n\n logging.info(f\"last lr = {scheduler.get_last_lr()}\")\n logging.info(f\"state dict = {scheduler.state_dict()}\")\n\n\n# This is included mostly as a baseline for ScaledAdam.\nclass Eve(Optimizer):\n \"\"\"\n Implements Eve algorithm. This is a modified version of AdamW with a special\n way of setting the weight-decay / shrinkage-factor, which is designed to make the\n rms of the parameters approach a particular target_rms (default: 0.1). This is\n for use with networks with 'scaled' versions of modules (see scaling.py), which\n will be close to invariant to the absolute scale on the parameter matrix.\n\n The original Adam algorithm was proposed in `Adam: A Method for Stochastic Optimization`_.\n The AdamW variant was proposed in `Decoupled Weight Decay Regularization`_.\n Eve is unpublished so far.\n\n Arguments:\n params (iterable): iterable of parameters to optimize or dicts defining\n parameter groups\n lr (float, optional): learning rate (default: 1e-3)\n betas (Tuple[float, float], optional): coefficients used for computing\n running averages of gradient and its square (default: (0.9, 0.999))\n eps (float, optional): term added to the denominator to improve\n numerical stability (default: 1e-8)\n weight_decay (float, optional): weight decay coefficient (default: 3e-4;\n this value means that the weight would decay significantly after\n about 3k minibatches. Is not multiplied by learning rate, but\n is conditional on RMS-value of parameter being > target_rms.\n target_rms (float, optional): target root-mean-square value of\n parameters, if they fall below this we will stop applying weight decay.\n\n\n .. _Adam: A Method for Stochastic Optimization:\n https://arxiv.org/abs/1412.6980\n .. _Decoupled Weight Decay Regularization:\n https://arxiv.org/abs/1711.05101\n .. _On the Convergence of Adam and Beyond:\n https://openreview.net/forum?id=ryQu7f-RZ\n \"\"\"\n\n def __init__(\n self,\n params,\n lr=1e-3,\n betas=(0.9, 0.98),\n eps=1e-8,\n weight_decay=1e-3,\n target_rms=0.1,\n ):\n if not 0.0 <= lr:\n raise ValueError(\"Invalid learning rate: {}\".format(lr))\n if not 0.0 <= eps:\n raise ValueError(\"Invalid epsilon value: {}\".format(eps))\n if not 0.0 <= betas[0] < 1.0:\n raise ValueError(\n \"Invalid beta parameter at index 0: {}\".format(betas[0])\n )\n if not 0.0 <= betas[1] < 1.0:\n raise ValueError(\n \"Invalid beta parameter at index 1: {}\".format(betas[1])\n )\n if not 0 <= weight_decay <= 0.1:\n raise ValueError(\n \"Invalid weight_decay value: {}\".format(weight_decay)\n )\n if not 0 < target_rms <= 10.0:\n raise ValueError(\"Invalid target_rms value: {}\".format(target_rms))\n defaults = dict(\n lr=lr,\n betas=betas,\n eps=eps,\n weight_decay=weight_decay,\n target_rms=target_rms,\n )\n super(Eve, self).__init__(params, defaults)\n\n def __setstate__(self, state):\n super(Eve, self).__setstate__(state)\n\n @torch.no_grad()\n def step(self, closure=None):\n \"\"\"Performs a single optimization step.\n\n Arguments:\n closure (callable, optional): A closure that reevaluates the model\n and returns the loss.\n \"\"\"\n loss = None\n if closure is not None:\n with torch.enable_grad():\n loss = closure()\n\n for group in self.param_groups:\n for p in group[\"params\"]:\n if p.grad is None:\n continue\n\n # Perform optimization step\n grad = p.grad\n if grad.is_sparse:\n raise RuntimeError(\n \"AdamW does not support sparse gradients\"\n )\n\n state = self.state[p]\n\n # State initialization\n if len(state) == 0:\n state[\"step\"] = 0\n # Exponential moving average of gradient values\n state[\"exp_avg\"] = torch.zeros_like(\n p, memory_format=torch.preserve_format\n )\n # Exponential moving average of squared gradient values\n state[\"exp_avg_sq\"] = torch.zeros_like(\n p, memory_format=torch.preserve_format\n )\n\n exp_avg, exp_avg_sq = state[\"exp_avg\"], state[\"exp_avg_sq\"]\n\n beta1, beta2 = group[\"betas\"]\n\n state[\"step\"] += 1\n bias_correction1 = 1 - beta1 ** state[\"step\"]\n bias_correction2 = 1 - beta2 ** state[\"step\"]\n\n # Decay the first and second moment running average coefficient\n exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1)\n exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)\n denom = (exp_avg_sq.sqrt() * (bias_correction2 ** -0.5)).add_(\n group[\"eps\"]\n )\n\n step_size = group[\"lr\"] / bias_correction1\n target_rms = group[\"target_rms\"]\n weight_decay = group[\"weight_decay\"]\n\n if p.numel() > 1:\n # avoid applying this weight-decay on \"scaling factors\"\n # (which are scalar).\n is_above_target_rms = p.norm() > (\n target_rms * (p.numel() ** 0.5)\n )\n p.mul_(1 - (weight_decay * is_above_target_rms))\n\n p.addcdiv_(exp_avg, denom, value=-step_size)\n\n # if random.random() < 0.0005:\n # step = (exp_avg / denom) * step_size\n # logging.info(\n # f\"Delta rms = {(step**2).mean().item()}, shape = {step.shape}\"\n # )\n\n return loss\n\ndef ScaledLinear(*args, initial_scale: float = 1.0, **kwargs) -> nn.Linear:\n \"\"\"\n Behaves like a constructor of a modified version of nn.Linear\n that gives an easy way to set the default initial parameter scale.\n\n Args:\n Accepts the standard args and kwargs that nn.Linear accepts\n e.g. in_features, out_features, bias=False.\n\n initial_scale: you can override this if you want to increase\n or decrease the initial magnitude of the module's output\n (affects the initialization of weight_scale and bias_scale).\n Another option, if you want to do something like this, is\n to re-initialize the parameters.\n \"\"\"\n ans = nn.Linear(*args, **kwargs)\n with torch.no_grad():\n ans.weight[:] *= initial_scale\n if ans.bias is not None:\n torch.nn.init.uniform_(\n ans.bias, -0.1 * initial_scale, 0.1 * initial_scale\n )\n return ans\ndef _test_scaled_adam(hidden_dim: int):\n import timeit\n\n E = 100\n B = 4\n T = 2\n logging.info(\"in test_eve_cain\")\n # device = torch.device('cuda')\n device = torch.device(\"cpu\")\n dtype = torch.float32\n\n # these input_magnitudes and output_magnitudes are to test that\n # Abel is working as we expect and is able to adjust scales of\n # different dims differently.\n input_magnitudes = (1.0 * torch.randn(E, dtype=dtype, device=device)).exp()\n output_magnitudes = (1.0 * torch.randn(E, dtype=dtype, device=device)).exp()\n\n for iter in [1, 0]:\n Linear = torch.nn.Linear if iter == 0 else ScaledLinear\n\n m = torch.nn.Sequential(\n Linear(E, hidden_dim),\n torch.nn.PReLU(),\n Linear(hidden_dim, hidden_dim),\n torch.nn.PReLU(),\n Linear(hidden_dim, E),\n ).to(device)\n\n train_pairs = [\n (\n 100.0\n * torch.randn(B, T, E, device=device, dtype=dtype)\n * input_magnitudes,\n torch.randn(B, T, E, device=device, dtype=dtype)\n * output_magnitudes,\n )\n for _ in range(20)\n ]\n\n if iter == 0:\n optim = Eve(m.parameters(), lr=0.003)\n elif iter == 1:\n optim = ScaledAdam(m.parameters(), lr=0.03, clipping_scale=2.0)\n scheduler = Eden(optim, lr_batches=200, lr_epochs=5, verbose=False)\n\n start = timeit.default_timer()\n avg_loss = 0.0\n for epoch in range(180):\n scheduler.step_epoch()\n # if epoch == 100 and iter in [2,3]:\n # optim.reset_speedup() # check it doesn't crash.\n\n # if epoch == 130:\n # opts = diagnostics.TensorDiagnosticOptions(\n # 2 ** 22\n # ) # allow 4 megabytes per sub-module\n # diagnostic = diagnostics.attach_diagnostics(m, opts)\n\n for n, (x, y) in enumerate(train_pairs):\n y_out = m(x)\n loss = ((y_out - y) ** 2).mean() * 100.0\n if epoch == 0 and n == 0:\n avg_loss = loss.item()\n else:\n avg_loss = 0.98 * avg_loss + 0.02 * loss.item()\n if n == 0 and epoch % 5 == 0:\n # norm1 = '%.2e' % (m[0].weight**2).mean().sqrt().item()\n # norm1b = '%.2e' % (m[0].bias**2).mean().sqrt().item()\n # norm2 = '%.2e' % (m[2].weight**2).mean().sqrt().item()\n # norm2b = '%.2e' % (m[2].bias**2).mean().sqrt().item()\n # scale1 = '%.2e' % (m[0].weight_scale.exp().item())\n # scale1b = '%.2e' % (m[0].bias_scale.exp().item())\n # scale2 = '%.2e' % (m[2].weight_scale.exp().item())\n # scale2b = '%.2e' % (m[2].bias_scale.exp().item())\n lr = scheduler.get_last_lr()[0]\n logging.info(\n f\"Iter {iter}, epoch {epoch}, batch {n}, avg_loss {avg_loss:.4g}, lr={lr:.4e}\"\n ) # , norms={norm1,norm1b,norm2,norm2b}\") # scales={scale1,scale1b,scale2,scale2b}\n loss.log().backward()\n optim.step()\n optim.zero_grad()\n scheduler.step_batch()\n\n # diagnostic.print_diagnostics()\n\n stop = timeit.default_timer()\n logging.info(f\"Iter={iter}, Time taken: {stop - start}\")\n\n logging.info(f\"last lr = {scheduler.get_last_lr()}\")\n # logging.info(\"state dict = \", scheduler.state_dict())\n # logging.info(\"optim state_dict = \", optim.state_dict())\n logging.info(f\"input_magnitudes = {input_magnitudes}\")\n logging.info(f\"output_magnitudes = {output_magnitudes}\")\n\n\nif __name__ == \"__main__\":\n torch.set_num_threads(1)\n torch.set_num_interop_threads(1)\n logging.getLogger().setLevel(logging.INFO)\n import subprocess\n\n s = subprocess.check_output(\n \"git status -uno .; git log -1; git diff HEAD .\", shell=True\n )\n logging.info(s)\n import sys\n\n if len(sys.argv) > 1:\n hidden_dim = int(sys.argv[1])\n else:\n hidden_dim = 200\n\n _test_scaled_adam(hidden_dim)\n _test_eden()\n" }, { "path": "steps/trainer.py", "content": "import time\nimport os, random\nimport torch\nimport math, pickle\nfrom tqdm import tqdm\nfrom torch.optim import AdamW\nfrom torch.optim.lr_scheduler import LambdaLR\nimport torch.nn as nn\nimport torch.distributed as dist\nfrom torch.utils.tensorboard import SummaryWriter\nimport numpy as np\nfrom torch.utils.data.distributed import DistributedSampler\nimport logging\nfrom data import gigaspeech\nfrom models import voicecraft\n\nfrom .trainer_utils import DistributedDynamicBatchSampler, StatefulDistributedSampler, AverageMeter, print_model_info\nfrom .optim import ScaledAdam, Eden\n\n\nclass Trainer:\n \n def __init__(self, args, world_size, rank):\n self.start_time = time.time()\n self.args = args\n self.world_size, self.rank = world_size, rank\n self.device = torch.device(f\"cuda:{rank}\" if torch.cuda.is_available() else \"cpu\")\n if self.rank == 0:\n self.writer = SummaryWriter(args.exp_dir)\n self.seed_everything(seed=self.args.seed)\n self.meters = self._setup_meters()\n\n self.progress, self.total_progress = self._setup_progress()\n\n self.model, self.trainables, self.optim_states, self.scheduler_states = self._setup_models()\n\n self.train_dataset_length, self.train_sampler, self.train_loader, self.valid_loader = self._setup_dataloader()\n if self.args.num_steps != None:\n self.total_step = self.args.num_steps\n self.args.num_epochs = math.ceil(self.total_step / math.floor(self.train_dataset_length / self.args.batch_size)) if not self.args.dynamic_batching else None\n else:\n self.total_step = int(math.floor(self.train_dataset_length / self.args.batch_size))*self.args.num_epochs\n\n self.optimizer, self.scheduler = self._setup_optimizer()\n self.scaler = torch.cuda.amp.GradScaler()\n self.model = torch.nn.parallel.DistributedDataParallel(self.model, device_ids=[self.rank], find_unused_parameters=False)\n \n if self.rank == 0:\n self.early_stop_accu_steps = 0\n if self.args.dynamic_batching:\n logging.info(f\"max number of tokens per GPU in a training batch: {self.args.max_num_tokens}, max number of tokens per GPU in a inference batch: {self.args.val_max_num_tokens}\")\n else:\n logging.info(f\"batch size (summed over all GPUs): {self.args.batch_size}\")\n\n def train(self):\n flag = True\n skip_flag = False\n data_start_time = time.time()\n while flag:\n self.train_sampler.set_epoch(self.progress['epoch'])\n for i, batch in enumerate(self.train_loader):\n data_end_time = time.time()\n self.model.train()\n if self.progress['step'] > self.total_step:\n flag = False\n self.validate_and_save()\n if self.rank == 0:\n self.writer.close()\n break\n if isinstance(self.scheduler, Eden):\n self.scheduler.step_epoch(self.progress['step']//self.args.pseudo_epoch_size + 1)\n if self.args.optimizer_name == \"ScaledAdam\":\n cur_lr = self.scheduler.get_last_lr()[0]\n else:\n lrs = [param_group['lr'] for param_group in self.optimizer.param_groups]\n assert lrs[0] == lrs[1]\n cur_lr = lrs[0]\n\n if self.rank == 0 and self.progress['step'] % self.args.tb_write_every_n_steps == 0:\n self.writer.add_scalar(\"train/lr\", cur_lr, self.progress['step'])\n\n all_inds = list(range(len(batch['y'])))\n sum_losses = 0\n sum_top10acc = 0\n sum_ntoken = 0\n sum_top10acc_cbi = [0 for _ in range(self.args.n_codebooks)]\n for j in range(self.args.gradient_accumulation_steps):\n cur_ind = all_inds[j::self.args.gradient_accumulation_steps]\n cur_batch = {key: batch[key][cur_ind] for key in batch}\n with torch.cuda.amp.autocast(dtype=torch.float16 if self.args.precision==\"float16\" else torch.float32):\n out = self.model(cur_batch)\n if out == None:\n continue\n\n record_loss = out['loss'].detach().to(self.rank) \n top10acc = out['top10acc'].to(self.rank)\n effective_ntoken = out['effective_ntoken'].to(self.rank)\n is_nan = torch.tensor(int(torch.isnan(record_loss).any()), dtype=torch.float32, device=self.rank)\n \n dist.all_reduce(record_loss, op=dist.ReduceOp.SUM)\n dist.all_reduce(top10acc, op=dist.ReduceOp.SUM)\n dist.all_reduce(effective_ntoken, op=dist.ReduceOp.SUM)\n dist.all_reduce(is_nan, op=dist.ReduceOp.SUM)\n \n # check if loss is nan\n if is_nan.item() > 0:\n logging.info(f\"loss at step {self.progress['step']} is nan, therefore skip this batch\")\n skip_flag = True\n continue\n\n sum_losses += record_loss.item()\n sum_top10acc += top10acc.item()\n sum_ntoken += effective_ntoken.item()\n\n if 'top10acc_by_codebook' in out:\n for cb in range(self.args.n_codebooks):\n top10acc_cbi = out['top10acc_by_codebook'][cb]\n dist.all_reduce(top10acc_cbi, op=dist.ReduceOp.SUM)\n sum_top10acc_cbi[cb] += top10acc_cbi.item()\n \n if self.rank == 0:\n average_loss = sum_losses / sum_ntoken\n average_top10acc = sum_top10acc / sum_ntoken\n self.meters['train_loss'].update(average_loss, batch['x'].shape[0]*self.world_size)\n self.meters['train_top10acc'].update(average_top10acc, batch['x'].shape[0]*self.world_size)\n self.meters['train_top10acc'].update(average_top10acc, batch['x'].shape[0]*self.world_size)\n average_top10acc_cbi = [sum_top10acc_cbi[cb] / sum_ntoken * self.args.n_codebooks for cb in range(self.args.n_codebooks)]\n for cb in range(self.args.n_codebooks):\n self.meters[f'train_top10acc_cb{cb+1}'].update(average_top10acc_cbi[cb], batch['x'].shape[0]*self.world_size)\n\n if self.progress['step'] % self.args.tb_write_every_n_steps == 0:\n self.writer.add_scalar('train/loss', average_loss, self.progress['step'])\n self.writer.add_scalar('train/top10acc', average_top10acc, self.progress['step'])\n self.writer.add_scalar(\"train/ntokens\", sum_ntoken, self.progress['step'])\n for cb in range(self.args.n_codebooks):\n self.writer.add_scalar(f'train/top10acc_cb{cb+1}', average_top10acc_cbi[cb], self.progress['step'])\n\n if self.args.optimizer_name == \"ScaledAdam\":\n self.scaler.scale(out['loss']).backward() \n else:\n self.scaler.scale(out['loss']/out['effective_ntoken']).backward()\n\n if skip_flag:\n self.optimizer.zero_grad()\n skip_flag = False\n continue\n\n if self.args.optimizer_name != \"ScaledAdam\":\n self.scaler.unscale_(self.optimizer)\n torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.args.gradient_clip_val)\n self.scaler.step(self.optimizer)\n self.scaler.update()\n\n self.optimizer.zero_grad()\n\n if self.args.optimizer_name == \"ScaledAdam\":\n self.scheduler.step_batch(self.progress['step'])\n else:\n self.scheduler.step()\n\n if self.rank == 0:\n self.meters['data_time'].update(data_end_time - data_start_time)\n self.meters['train_time'].update(time.time() - data_end_time)\n if self.progress['step'] % self.args.tb_write_every_n_steps == 0:\n self.writer.add_scalar(\"train/data_time\", data_end_time - data_start_time, self.progress['step'])\n self.writer.add_scalar(\"train/train_time\", time.time() - data_end_time, self.progress['step'])\n \n\n # logging\n if self.progress['step'] % self.args.print_every_n_steps == 0:\n log_out = {}\n log_out['cur_epoch'] = f\"{self.progress['epoch']}/{self.args.num_epochs}\" if self.args.num_epochs is not None else f\"{self.progress['epoch']}\"\n log_out['cur_step'] = f\"{int(self.progress['cur_step']+1)}\"\n log_out['total_step'] = f\"{self.progress['step']}/{self.args.num_steps}\"\n log_out['lr'] = f\"{cur_lr:.7f}\"\n log_out['ntokens'] = f\"{sum_ntoken}\"\n for key in self.meters:\n if self.meters[key].val != 0 or self.meters[key].avg != 0:\n log_out[key] = f\"{self.meters[key].val:.4f} ({self.meters[key].avg:.4f})\" if isinstance(self.meters[key].val, float) else f\"{self.meters[key].val}\"\n logging.info(log_out)\n if np.isnan(self.meters['train_loss'].avg):\n logging.warning(\"training diverged...\")\n raise RuntimeError(\"training diverged...\")\n\n # validation and save models\n if self.progress['step'] % self.args.val_every_n_steps == 0:\n dist.barrier()\n self.validate_and_save()\n\n self.progress['step'] += 1\n self.progress['cur_step'] += 1\n\n data_start_time = time.time()\n self.progress['epoch'] += 1\n self.progress['cur_step'] = 0 # reset cur_step to be 0\n dist.destroy_process_group()\n\n def validate_and_save(self):\n self.model.eval()\n \n score = self.validate(self.valid_loader)\n\n if self.rank == 0:\n if self.args.early_stop_threshold > 0:\n if self.progress['best_score'] - score < self.args.early_stop_threshold:\n self.early_stop_accu_steps += self.args.val_every_n_steps\n if self.early_stop_accu_steps >= self.args.early_stop_step-1:\n logging.info(f\"early stop based on self.args.early_stop_threshold: {self.args.early_stop_threshold}, and self.args.early_stop_step: {self.args.early_stop_step}\")\n logging.info(f\"best validation score at step: {self.progress['best_step']}, and the score is {self.progress['best_score']:.4f}\")\n dist.destroy_process_group()\n raise RuntimeError(\"early stop\")\n else:\n self.early_stop_accu_steps = 0\n\n if (score < self.progress['best_score']):\n self.progress['best_step'] = self.progress['step']\n self.progress['best_score'] = score\n save_path = os.path.join(self.args.exp_dir,\"best_bundle.pth\")\n torch.save(\n {\n \"model\": self.model.module.state_dict(),\n \"optimizer\": self.optimizer.state_dict(),\n \"scheduler\": self.scheduler.state_dict(),\n \"config\": self.args,\n \"phn2num\": self.train_loader.dataset.phn2num\n },save_path\n )\n logging.info(f\"save *best* models at {save_path} at global step {self.progress['step']}\")\n self._save_progress()\n save_path = os.path.join(self.args.exp_dir,\"bundle.pth\")\n torch.save(\n {\n \"model\": self.model.module.state_dict(),\n \"optimizer\": self.optimizer.state_dict(),\n \"scheduler\": self.scheduler.state_dict(),\n \"config\": self.args,\n \"phn2num\": self.train_loader.dataset.phn2num\n },save_path\n )\n logging.info(f\"save models, indices, acc and other statistics at {save_path} and {self.args.exp_dir}/progress.pkl at global step {self.progress['step']}\")\n\n dist.barrier()\n\n def validate(self, valid_loader=None, hide_progress=True):\n if valid_loader == None:\n valid_loader = self.valid_loader\n self.model.eval()\n\n start_val_time = time.time()\n sum_losses = 0\n sum_top10acc = 0\n sum_ntoken = 0\n sum_top10acc_cbi = [0 for _ in range(self.args.n_codebooks)]\n\n with torch.no_grad():\n for i, batch in enumerate(tqdm(valid_loader, disable=hide_progress)):\n out = self.model(batch)\n sum_losses += out['loss']\n sum_top10acc += out['top10acc']\n sum_ntoken += out['effective_ntoken']\n if 'top10acc_by_codebook' in out:\n for cb in range(self.args.n_codebooks):\n sum_top10acc_cbi[cb] += out['top10acc_by_codebook'][cb]\n \n dist.all_reduce(sum_losses, op=dist.ReduceOp.SUM)\n dist.all_reduce(sum_top10acc, op=dist.ReduceOp.SUM)\n dist.all_reduce(sum_ntoken, op=dist.ReduceOp.SUM)\n \n if 'top10acc_by_codebook' in out:\n for cb in range(self.args.n_codebooks):\n dist.all_reduce(sum_top10acc_cbi[cb], op=dist.ReduceOp.SUM)\n \n if self.rank == 0:\n val_loss = sum_losses / sum_ntoken\n val_top10acc = sum_top10acc / sum_ntoken\n # logging\n self.meters['val_loss'].update(val_loss)\n logging.info(f\"val loss: {val_loss:.5f}\")\n self.writer.add_scalar(\"val/loss\", val_loss, self.progress['step'])\n\n self.meters['val_top10acc'].update(val_top10acc)\n logging.info(f\"val top10acc: {val_top10acc:.5f}\")\n self.writer.add_scalar(\"val/top10acc\", val_top10acc, self.progress['step'])\n for cb in range(self.args.n_codebooks):\n average_top10acc_cbi = sum_top10acc_cbi[cb] / sum_ntoken * self.args.n_codebooks\n self.meters[f'val_top10acc_cb{cb+1}'].update(average_top10acc_cbi)\n self.writer.add_scalar(f'val/top10acc_cb{cb+1}', average_top10acc_cbi, self.progress['step'])\n\n logging.info(f\"validation takes: {time.time() - start_val_time:.2f}s\")\n logging.info(f\"Step [{self.progress['step']}/{self.total_step}]\\t Time elapsed {(time.time() - self.start_time)/3600.:.2f}h, Val Loss: {val_loss:.4f}, Val Top10Acc: {val_top10acc:.4f}\")\n return val_loss.item()\n else:\n return None\n\n def _setup_meters(self):\n meters = {}\n meter_names = ['train_loss', 'val_loss', 'train_top10acc', 'val_top10acc', 'data_time', 'train_time']\n meter_names += ['train_dur_loss', 'train_dur_acc', 'val_dur_loss', 'val_dur_acc']\n meter_names += [f'train_top10acc_cb{cb+1}' for cb in range(self.args.n_codebooks)]\n meter_names += [f'val_top10acc_cb{cb+1}' for cb in range(self.args.n_codebooks)]\n for name in meter_names:\n meters[name] = AverageMeter()\n return meters\n def _setup_progress(self):\n progress = {}\n progress['best_step'] = 1\n progress['best_score'] = np.inf # this records loss value\n progress['step'] = 1\n progress['epoch'] = 1\n progress['cur_step'] = 0 # step in the current epoch, for resuming the sampler\n total_progress = []\n # if self.args.resume or self.args.validate:\n if self.args.resume:\n progress_pkl = \"%s/progress.pkl\" % self.args.exp_dir\n with open(progress_pkl, \"rb\") as f:\n total_progress = pickle.load(f)\n progress['best_step'], progress['best_score'], progress['step'], progress['epoch'], progress['cur_step'], _ = total_progress[-1]\n if self.rank == 0:\n logging.info(\"\\nResume training from:\")\n logging.info(\" epoch = %s\" % progress['epoch'])\n logging.info(\" cur_step = %s\" % progress['cur_step'])\n logging.info(\" step = %s\" % progress['step'])\n logging.info(\" best_step = %s\" % progress['best_step'])\n logging.info(\" best_score = %s\" % progress['best_score'])\n return progress, total_progress\n \n def _save_progress(self):\n self.total_progress.append([self.progress['best_step'], self.progress['best_score'], int(self.progress['step']+1), self.progress['epoch'], int(self.progress['cur_step']+1), time.time() - self.start_time])\n with open(\"%s/progress.pkl\" % self.args.exp_dir, \"wb\") as f:\n pickle.dump(self.total_progress, f)\n\n def _setup_dataloader(self):\n assert self.args.dataset == 'gigaspeech', \"only gigaspeech is supported for now\"\n train_dataset, val_dataset = gigaspeech.dataset(self.args, 'train'), gigaspeech.dataset(self.args, 'validation')\n \n if self.args.dynamic_batching:\n train_sampler = DistributedDynamicBatchSampler(train_dataset, self.args, num_replicas=self.world_size, rank=self.rank, shuffle=True, seed=self.args.seed, drop_last=True, lengths_list=train_dataset.lengths_list, verbose=True, epoch=0)\n valid_sampler = DistributedDynamicBatchSampler(val_dataset, self.args, num_replicas=self.world_size, rank=self.rank, shuffle=True, seed=self.args.seed, drop_last=True, lengths_list=val_dataset.lengths_list, verbose=True, epoch=0)\n else:\n train_sampler = StatefulDistributedSampler(train_dataset, self.args.batch_size//self.world_size, num_replicas=self.world_size, rank=self.rank, shuffle=True, seed=self.args.seed, drop_last=True)\n valid_sampler = DistributedSampler(val_dataset, num_replicas=self.world_size, rank=self.rank, shuffle=False, seed=self.args.seed, drop_last=False)\n \n if self.progress['step'] > 1:\n train_sampler.set_epoch_resume(self.progress['epoch'], self.progress['cur_step'])\n\n if self.args.dynamic_batching:\n train_loader = torch.utils.data.DataLoader(train_dataset, \n batch_sampler=train_sampler, \n num_workers=self.args.num_workers//self.world_size,\n collate_fn=train_dataset.collate, persistent_workers=True\n )\n valid_loader = torch.utils.data.DataLoader(val_dataset, \n batch_sampler=valid_sampler, \n num_workers=self.args.num_workers//self.world_size,\n collate_fn=val_dataset.collate, persistent_workers=True\n )\n else:\n train_loader = torch.utils.data.DataLoader(train_dataset, \n batch_size=self.args.batch_size//self.world_size, sampler=train_sampler, num_workers=self.args.num_workers//self.world_size,\n collate_fn=train_dataset.collate, persistent_workers=True\n )\n valid_loader = torch.utils.data.DataLoader(val_dataset, \n batch_size=self.args.batch_size//self.world_size, sampler=valid_sampler,\n num_workers=self.args.num_workers//self.world_size,\n collate_fn=val_dataset.collate, persistent_workers=True\n )\n return len(train_dataset), train_sampler, train_loader, valid_loader\n \n\n \n def _setup_models(self):\n model = voicecraft.VoiceCraft(self.args)\n\n if self.rank == 0:\n logging.info(model)\n logging.info(\"model parameters\")\n print_model_info(model)\n\n if self.progress['step'] > 1:\n bundle = torch.load(os.path.join(self.args.exp_dir, \"bundle.pth\"), map_location=\"cpu\")\n model.load_state_dict(bundle['model'])\n optim_states = bundle['optimizer']\n scheduler_states = bundle['scheduler']\n if self.rank == 0:\n logging.info(\"loaded parameters and data indices from epoch %d, global step %d\" % (self.progress['epoch'], self.progress['step']))\n del bundle['model']\n else:\n optim_states = None\n scheduler_states = None\n\n if self.args.load_model_from != None and self.progress['step'] <= 1:\n sd = torch.load(self.args.load_model_from, map_location=\"cpu\")['model']\n model.load_state_dict(sd)\n del sd\n \n if self.args.optimizer_name == \"ScaledAdam\":\n trainables = [p for p in model.parameters() if p.requires_grad]\n else:\n no_decay = [\".bias\", \".audio_embeddings.weight\", \".text_embeddings.weight\", \".norm.weight\", \".norm1.weight\", \".norm2.weight\"]\n optimizer_grouped_parameters = [\n {\n \"params\": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay) and p.requires_grad],\n \"weight_decay\": self.args.weight_decay,\n },\n {\n \"params\": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay) and p.requires_grad],\n \"weight_decay\": 0.0,\n },\n ]\n if len(optimizer_grouped_parameters[1]['params']) == 0:\n logging.info(\"there is no embedding weights, bias, and layernorm parameters in the model, which should be True, check model parameter names\")\n trainables = optimizer_grouped_parameters[0]\n else:\n trainables = optimizer_grouped_parameters\n model.to(self.device)\n\n return model, trainables, optim_states, scheduler_states\n\n \n def _setup_optimizer(self):\n if self.args.optimizer_name == \"ScaledAdam\":\n parameters_names = []\n parameters_names.append([n for n,p in self.model.named_parameters() if p.requires_grad])\n optimizer = ScaledAdam(\n self.trainables,\n lr=self.args.lr,\n betas=(0.9, 0.95),\n clipping_scale=2.0,\n parameters_names=parameters_names,\n show_dominant_parameters=False,\n clipping_update_period=self.args.clipping_update_period,\n )\n scheduler = Eden(optimizer, self.args.reduce_lr_start_step, self.args.reduce_lr_start_epoch, warmup_batches=self.total_step * self.args.warmup_fraction)\n\n else:\n optimizer = AdamW(self.trainables, lr=self.args.lr)\n warmup_steps = self.total_step * self.args.warmup_fraction\n def lr_lambda(current_step: int):\n if current_step < warmup_steps:\n return float(current_step) / float(max(1, warmup_steps))\n return max(\n 0.0, float(self.total_step - current_step) / float(max(1, self.total_step - warmup_steps))\n )\n\n scheduler = LambdaLR(optimizer, lr_lambda, last_epoch=-1)\n \n # if resume\n if self.progress['step'] > 1:\n optimizer.load_state_dict(self.optim_states)\n for state in optimizer.state.values():\n for k, v in state.items():\n if isinstance(v, torch.Tensor):\n state[k] = v.cuda()\n del self.optim_states\n\n scheduler.load_state_dict(self.scheduler_states)\n\n optimizer.zero_grad()\n return optimizer, scheduler\n \n def seed_everything(self, seed=1):\n os.environ['PYTHONHASHSEED'] = str(seed)\n random.seed(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.benchmark = False\n torch.backends.cudnn.deterministic = True" }, { "path": "steps/trainer_utils.py", "content": "\nimport torch\nimport math\nimport torch.distributed as dist\nfrom torch.utils.data.sampler import Sampler\nimport copy\nimport numpy as np\nfrom typing import List\nfrom scipy.stats import lognorm\nimport logging\n\nclass StatefulDistributedSampler(Sampler[int]):\n def __init__(self, dataset, batch_size, num_replicas = None, rank = None, shuffle = True, seed = 0, drop_last = False):\n if num_replicas is None:\n if not dist.is_available():\n raise RuntimeError(\"Requires distributed package to be available\")\n num_replicas = dist.get_world_size()\n if rank is None:\n if not dist.is_available():\n raise RuntimeError(\"Requires distributed package to be available\")\n rank = dist.get_rank()\n if rank >= num_replicas or rank < 0:\n raise ValueError(\n \"Invalid rank {}, rank should be in the interval\"\n \" [0, {}]\".format(rank, num_replicas - 1))\n self.dataset = dataset\n self.batch_size = batch_size\n self.num_replicas = num_replicas\n self.rank = rank\n self.epoch = 0\n self.cur_epoch = 0\n self.drop_last = drop_last\n # If the dataset length is evenly divisible by # of replicas, then there\n # is no need to drop any data, since the dataset will be split equally.\n if self.drop_last and len(self.dataset) % self.num_replicas != 0: # type: ignore[arg-type]\n # Split to nearest available length that is evenly divisible.\n # This is to ensure each rank receives the same amount of data when\n # using this Sampler.\n self.num_samples = math.ceil(\n (len(self.dataset) - self.num_replicas) / self.num_replicas # type: ignore[arg-type]\n )\n else:\n self.num_samples = math.ceil(len(self.dataset) / self.num_replicas) # type: ignore[arg-type]\n self.total_size = self.num_samples * self.num_replicas\n self.shuffle = shuffle\n self.seed = seed\n self.continue_flag = False\n def __len__(self):\n return self.num_samples\n\n def set_epoch(self, epoch):\n r\"\"\"\n Sets the epoch for this sampler. When :attr:`shuffle=True`, this ensures all replicas\n use a different random ordering for each epoch. Otherwise, the next iteration of this\n sampler will yield the same ordering.\n\n Args:\n epoch (int): Epoch number.\n \"\"\"\n self.epoch = epoch\n\n if self.shuffle:\n # deterministically shuffle based on epoch and seed\n g = torch.Generator()\n g.manual_seed(self.seed + self.epoch)\n indices = torch.randperm(len(self.dataset), generator=g).tolist() # type: ignore[arg-type]\n else:\n indices = list(range(len(self.dataset))) # type: ignore[arg-type]\n\n if not self.drop_last:\n # add extra samples to make it evenly divisible\n padding_size = self.total_size - len(indices)\n if padding_size <= len(indices):\n indices += indices[:padding_size]\n else:\n indices += (indices * math.ceil(padding_size / len(indices)))[:padding_size]\n else:\n # remove tail of data to make it evenly divisible.\n indices = indices[:self.total_size]\n assert len(indices) == self.total_size\n\n # subsample\n indices = indices[self.rank:self.total_size:self.num_replicas]\n assert len(indices) == self.num_samples\n self.indices = indices\n\n if self.continue_flag:\n self.indices = self.indices[int(self.cur_step*self.batch_size):]\n self.num_samples = len(self.indices)\n self.continue_flag = False\n \n def __iter__(self):\n for idx in self.indices:\n yield idx \n\n def set_epoch_resume(self, epoch, cur_step):\n self.epoch = epoch\n self.cur_step = cur_step\n self.continue_flag = True\n\n\nclass StatefulSampler(Sampler):\n def __init__(self, data_source_length, batch_size, use_random=True, seed=1, epoch=0):\n self.use_random = use_random\n self.data_source_length = data_source_length\n self.num_samples = self.data_source_length\n self.batch_size = batch_size\n self.continue_flag = False\n self.seed = seed\n self.epoch = epoch\n self.cur_step = 0\n\n def __len__(self):\n return self.num_samples\n\n def __iter__(self):\n\n for idx in self.indices:\n yield idx\n\n def set_epoch(self, epoch):\n self.epoch = epoch\n if self.use_random:\n # deterministically shuffle based on epoch and seed\n g = torch.Generator()\n g.manual_seed(self.seed + self.epoch)\n self.indices = torch.randperm(self.data_source_length, generator=g).tolist() # type: ignore[arg-type]\n else:\n self.indices = list(range(self.data_source_length)) # type: ignore[arg-type]\n if self.continue_flag == True:\n self.continue_flag = False\n self.indices = self.indices[int(self.cur_step*self.batch_size):]\n \n self.num_samples = len(self.indices)\n \n def set_epoch_resume(self, epoch, cur_step):\n self.epoch = epoch\n self.cur_step = cur_step\n self.continue_flag = True\n\n\nclass AverageMeter:\n \"\"\"Computes and stores the average and current value\"\"\"\n def __init__(self):\n self.reset()\n\n def reset(self):\n self.val = 0\n self.avg = 0\n self.sum = 0\n self.count = 0\n\n def update(self, val, n=1):\n self.val = val\n self.sum += val * n\n self.count += n\n self.avg = self.sum / self.count\n\ndef print_model_info(model, print_model = False, print_params = True):\n if print_model:\n logging.info(model)\n if print_params:\n all_params = {}\n for name, p in model.named_parameters():\n name = name.split(\".\")[0]\n if name in all_params:\n all_params[name] += p.numel()\n else:\n all_params[name] = p.numel()\n logging.info(\"num of parameters of each components:\")\n for name in all_params:\n logging.info(f\"{name}: {all_params[name]/1000000.:.2f}m\")\n\n\nclass DistributedDynamicBatchSampler(Sampler):\n \"\"\"\n modified from SpeechBrian, https://github.com/speechbrain/speechbrain/blob/develop/speechbrain/dataio/sampler.py#L307\n This BatchSampler batches examples together by grouping them by their length.\n\n Every example in the batch have approximately the same length and\n thus padding is minimized.\n This enables faster training on datasets\n where length of examples can vary significantly (e.g Librispeech).\n Inspired by: https://www.tensorflow.org/api_docs/python/tf/data/experimental/bucket_by_sequence_length\n\n Dynamic batching is performed by specifying a max_batch_length which is the\n upper limit for the sum of the length of examples in a batch:\n e.g., if ex1 has length 4, ex2 length 5 and if max_batch_length is set to 6\n ex1 and ex2 will be placed, alone, in two distinct batches.\n\n Length for each example can be obtained in two manners.\n If the input dataset is a DynamicItemDataset it can be obtained by specifying a\n length_func. Default assumes a \"duration\" entry is in the annotation.\n Length for each example can also be passed to this class upon instantiation\n by specifying a list containing the length for each example and passing it to\n lengths_list.\n\n Examples are grouped together by defining a set of possible discrete intervals\n (buckets). Examples whose length fall into these intervals can be batched together.\n\n The number of buckets can be specified by using the arg num_buckets.\n There is usually an optimal range for the value of this argument.\n\n If num_buckets == 1, all examples can be batched together. You have maximum randomization\n but your training speed will be slower due to the fact that a large amount of the values will be padding\n as long and short examples can be batched together.\n As the number of buckets grows only examples with similar\n length can be grouped together.\n This trades-off speed with randomization.\n TLDR: Low number -> better randomization, High number -> faster training.\n NOTE THAT: if set too high the training speed will decrease. If num_buckets -> number of examples in the dataset the batch size\n will be small impacting training speed and possibly performance.\n\n The buckets can also be specified by passing a list to the bucket_boundaries\n argument instead of specifying a left_bucket_length and a bucket_length_multiplier.\n\n Example\n -------\n >>> import torch\n >>> import speechbrain as sb\n >>> from speechbrain.dataio.sampler import DynamicBatchSampler\n >>> from speechbrain.dataio.dataset import DynamicItemDataset\n >>> from speechbrain.dataio.dataloader import SaveableDataLoader\n >>> from speechbrain.dataio.batch import PaddedBatch\n >>> import numpy as np\n >>> item_lengths = sorted([np.random.randint(10, 100) for x in range(20)])\n >>> dataset = {\"ex_{}\".format(x) : {\"wav\" :torch.randn(x)} for x in item_lengths}\n >>> dataset = DynamicItemDataset(dataset)\n >>> dataset.set_output_keys([\"wav\"])\n >>> length_func = lambda x : len(x) # trivial in this example\n >>> bsampler = DynamicBatchSampler(dataset, 20, 4, length_func, shuffle=False, batch_ordering='descending')\n >>> dataloader = SaveableDataLoader(dataset, batch_sampler=bsampler, collate_fn=PaddedBatch)\n >>> for i, b in enumerate(dataloader):\n ... data, length = b[\"wav\"]\n >>> assert data.shape[-1] == max(item_lengths)\n\n Arguments\n ---------\n dataset : torch.utils.data.Dataset\n Pytorch Dataset from which elements will be sampled.\n max_batch_length : int\n Upper limit for the sum of the length of examples in a batch.\n Should be chosen based on your GPU memory.\n num_buckets : int\n Number of discrete buckets used to group examples together.\n If num_buckets == 1, all examples can be batched together. As the number of buckets grows only examples with similar\n length can be grouped together. This trades-off speed with randomization.\n Low number -> better randomization, High number -> faster training.\n However if set too high the training speed will decrease. If num_buckets -> number of examples in the dataset the batch size\n will be small impacting training speed and possibly performance.\n NOTE: you have either to specify manually the bucket_boundaries or the number of buckets.\n length_func : callable\n Function used to get length of each example from the dataset.\n This argument can be used only when the dataset is a Speechbrain DynamicItemDataset object.\n Can be anything: e.g. lambda x: x[\"duration\"]*16000 returns number of samples\n if duration key in the annotation is in seconds and the file has 16kHz sampling freq.\n shuffle : bool\n Whether or not shuffle examples between each epoch.\n batch_ordering : string\n If ``random``, batches are randomly permuted; otherwise ``ascending`` or ``descending`` sorted by length.\n max_batch_ex: int\n If set, it limits the maximum number of examples that can be in a batch superseeding max_batch_length\n in instances where the amount of examples will exceeed the value specified here.\n E.g. you have a lot of short examples and the batch size for those will be too high, you can use this argument\n to limit the batch size for these short examples.\n bucket_boundaries : list\n Overrides bucket_length_multiplier and left_bucket_length by specifying manually\n the buckets right boundaries.\n lengths_list: list\n Overrides length_func by passing a list containing the length of each example\n in the dataset. This argument must be set when the dataset is a plain\n Pytorch Dataset object and not a DynamicItemDataset object as length_func\n cannot be used on Pytorch Datasets.\n epoch : int\n The epoch to start at.\n drop_last : bool\n If ``True``, the sampler will drop the last examples which\n have not been grouped.\n verbose: bool\n If ``True``, log also the stats for each batch at the first epoch.\n \"\"\"\n\n def __init__(\n self,\n dataset,\n args,\n num_replicas = None, \n rank = None, \n shuffle = True, \n seed = 0, \n drop_last = False,\n length_func=lambda x: x[\"duration\"],\n batch_ordering: str = \"random\",\n max_batch_ex: int = None,\n bucket_boundaries: List[int] = [],\n lengths_list: List[int] = None,\n epoch: int = 0,\n verbose: bool = False,\n ):\n self.args = args\n if num_replicas is None:\n if not dist.is_available():\n raise RuntimeError(\"Requires distributed package to be available\")\n num_replicas = dist.get_world_size()\n if rank is None:\n if not dist.is_available():\n raise RuntimeError(\"Requires distributed package to be available\")\n rank = dist.get_rank()\n if rank >= num_replicas or rank < 0:\n raise ValueError(\n \"Invalid rank {}, rank should be in the interval\"\n \" [0, {}]\".format(rank, num_replicas - 1))\n self.num_replicas = num_replicas\n self.rank = rank\n max_batch_length = self.args.max_num_tokens if dataset.split == \"train\" else self.args.val_max_num_tokens\n logging.info(f\"max_num_tokens per GPU for {dataset.split} split: {max_batch_length}\")\n num_buckets = self.args.num_buckets\n #############\n \n\n\n\n self._dataset = dataset\n self._ex_lengths = {}\n # ex_ids = self._dataset.data_ids\n self.verbose = verbose\n\n # We do not put a default on num_buckets to encourage users to play with this parameter\n if num_buckets is None and len(bucket_boundaries) == 0:\n raise RuntimeError(\n \"Please specify either num_buckets or bucket boundaries.\"\n \"Check the docs, and/or the tutorial !\"\n )\n assert lengths_list != None\n max_len = int(self.args.audio_max_length * self.args.encodec_sr)\n lengths_list = [min(l, max_len) for l in lengths_list] # replace all utt whose length is longer than max_len to max_len, will also do this in __getitem__ in dataset\n for indx in range(len(lengths_list)):\n self._ex_lengths[str(indx)] = lengths_list[indx]\n # if lengths_list is not None:\n # # take length of examples from this argument and bypass length_key\n # for indx in range(len(lengths_list)):\n # self._ex_lengths[str(indx)] = lengths_list[indx]\n # else:\n # # use length func\n # if not isinstance(dataset, DynamicItemDataset):\n # raise NotImplementedError(\n # \"Dataset should be a Speechbrain DynamicItemDataset when using length function\"\n # )\n # for indx in range(len(self._dataset)):\n # self._ex_lengths[str(indx)] = length_func(\n # self._dataset.data[ex_ids[indx]]\n # )\n\n if len(bucket_boundaries) > 0:\n if not all([x >= 0 for x in bucket_boundaries]):\n raise ValueError(\n \"All elements in bucket boundaries should be non-negative (>= 0).\"\n )\n if not len(set(bucket_boundaries)) == len(bucket_boundaries):\n raise ValueError(\n \"Bucket_boundaries should not contain duplicates.\"\n )\n np.testing.assert_array_equal(\n np.array(bucket_boundaries),\n np.array(sorted(bucket_boundaries)),\n err_msg=\"The arg bucket_boundaries should be an ascending sorted list of non negative values values!\",\n )\n self._bucket_boundaries = np.array(sorted(bucket_boundaries))\n else:\n # use num_buckets\n self._bucket_boundaries = np.array(\n self._get_boundaries_through_warping(\n # max_batch_length=max_batch_length,\n max_batch_length=max(lengths_list),\n num_quantiles=num_buckets,\n )\n )\n\n self._max_batch_length = max_batch_length\n self._shuffle_ex = shuffle\n self._batch_ordering = batch_ordering\n self._seed = seed\n self._drop_last = drop_last\n if max_batch_ex is None:\n max_batch_ex = np.inf\n self._max_batch_ex = max_batch_ex\n # Calculate bucket lengths - how often does one bucket boundary fit into max_batch_length?\n self._bucket_lens = [\n max(1, int(max_batch_length / self._bucket_boundaries[i]))\n for i in range(len(self._bucket_boundaries))\n ] + [1]\n self._epoch = epoch\n self._cur_step = 0\n self.continue_flag = False\n self._generate_batches()\n self.num_samples = int(math.floor(len(self._batches) / self.num_replicas))\n self.total_size = int(self.num_samples * self.num_replicas)\n self._replica_batches = self._batches[self.rank:self.total_size:self.num_replicas]\n assert len(self._replica_batches) == self.num_samples, f\"len(self._batches): {len(self._batches)}, self.total_size: {self.total_size}, self.num_samples: {self.num_samples},len(self._replica_batches): {len(self._replica_batches)}\"\n logging.info(f\"len(self._batches): {len(self._batches)}\")\n logging.info(f\"self.num_replicas: {self.num_replicas}\")\n logging.info(f\"num of batches on each replica: {self.num_samples}\")\n\n def get_durations(self, batch):\n \"\"\"Gets durations of the elements in the batch.\"\"\"\n return [self._ex_lengths[str(idx)] for idx in batch]\n\n def _get_boundaries_through_warping(\n self, max_batch_length: int, num_quantiles: int,\n ) -> List[int]:\n\n # NOTE: the following lines do not cover that there is only one example in the dataset\n # warp frames (duration) distribution of train data\n logging.info(\"Batch quantisation in latent space\")\n # linspace set-up\n num_boundaries = num_quantiles + 1\n # create latent linearly equal spaced buckets\n latent_boundaries = np.linspace(\n 1 / num_boundaries, num_quantiles / num_boundaries, num_quantiles,\n )\n # get quantiles using lognormal distribution\n quantiles = lognorm.ppf(latent_boundaries, 1)\n # scale up to to max_batch_length\n bucket_boundaries = quantiles * max_batch_length / quantiles[-1]\n # compute resulting bucket length multipliers\n length_multipliers = [\n bucket_boundaries[x + 1] / bucket_boundaries[x]\n for x in range(num_quantiles - 1)\n ]\n # logging\n logging.debug(\n \"Latent bucket boundary - buckets: {} - length multipliers: {}\".format(\n list(map(\"{:.2f}\".format, bucket_boundaries)),\n list(map(\"{:.2f}\".format, length_multipliers)),\n )\n )\n return list(sorted(bucket_boundaries))\n\n def _permute_batches(self):\n\n if self._batch_ordering == \"random\":\n # deterministically shuffle based on epoch and seed\n g = torch.Generator()\n g.manual_seed(self._seed + self._epoch) # since the random seed is based on self._seed and self._epoch, it should be the same for different processes when using DDP, and therefore the generated order should be the same across different process, this is important, because each replica will only take a portion of it, we want to make sure they take a non-overlapping portion, and all of them constitute the entire dataset\n sampler = torch.randperm(\n len(self._batches), generator=g\n ).tolist() # type: ignore\n tmp = []\n for idx in sampler:\n tmp.append(self._batches[idx])\n self._batches = tmp\n\n elif self._batch_ordering == \"ascending\":\n self._batches = sorted(\n self._batches,\n key=lambda x: max([self._ex_lengths[str(idx)] for idx in x]),\n )\n elif self._batch_ordering == \"descending\":\n self._batches = sorted(\n self._batches,\n key=lambda x: max([self._ex_lengths[str(idx)] for idx in x]),\n reverse=True,\n )\n else:\n raise NotImplementedError\n\n def _generate_batches(self):\n logging.info(\"DynamicBatchSampler: Generating dynamic batches\")\n if self._shuffle_ex:\n # deterministically shuffle based on epoch and seed\n g = torch.Generator()\n g.manual_seed(self._seed + self._epoch) # since the random seed is based on self._seed and self._epoch, it should be the same for different processes when using DDP, and therefore the generated order should be the same across different process, this is important, because each replica will only take a portion of it, we want to make sure they take a non-overlapping portion, and all of them constitute the entire dataset\n sampler = torch.randperm(len(self._dataset), generator=g).tolist() # type: ignore \n # pyp note: this is actually randomly permoted indices\n else:\n # take examples as they are: e.g. they have been sorted\n sampler = range(len(self._dataset)) # type: ignore\n\n self._batches = []\n bucket_batches = [[] for i in self._bucket_lens]\n\n stats_tracker = [\n {\"min\": np.inf, \"max\": -np.inf, \"tot\": 0, \"n_ex\": 0}\n for i in self._bucket_lens\n ]\n\n for idx in sampler:\n # length of pre-sampled audio\n item_len = self._ex_lengths[str(idx)]\n # bucket to fill up most padding\n bucket_id = np.searchsorted(self._bucket_boundaries, item_len)\n # fill audio's duration into that bucket\n bucket_batches[bucket_id].append(idx)\n\n stats_tracker[bucket_id][\"min\"] = min(\n stats_tracker[bucket_id][\"min\"], item_len\n )\n stats_tracker[bucket_id][\"max\"] = max(\n stats_tracker[bucket_id][\"max\"], item_len\n )\n stats_tracker[bucket_id][\"tot\"] += item_len\n stats_tracker[bucket_id][\"n_ex\"] += 1\n # track #samples - why not duration/#frames; rounded up?\n # keep track of durations, if necessary\n\n if (\n len(bucket_batches[bucket_id]) >= self._bucket_lens[bucket_id]\n or len(bucket_batches[bucket_id]) >= self._max_batch_ex\n ):\n self._batches.append(bucket_batches[bucket_id])\n bucket_batches[bucket_id] = []\n # keep track of durations\n\n # Dump remaining batches\n if not self._drop_last:\n for batch in bucket_batches:\n if batch:\n self._batches.append(batch)\n\n self._permute_batches() # possibly reorder batches\n\n if self._epoch == 0: # only log at first epoch\n # frames per batch & their padding remaining\n boundaries = [0] + self._bucket_boundaries.tolist()\n\n for bucket_indx in range(len(self._bucket_boundaries)):\n try:\n num_batches = stats_tracker[bucket_indx][\"tot\"] // (\n self._max_batch_length\n )\n pad_factor = (\n stats_tracker[bucket_indx][\"max\"]\n - stats_tracker[bucket_indx][\"min\"]\n ) / (\n stats_tracker[bucket_indx][\"tot\"]\n / stats_tracker[bucket_indx][\"n_ex\"]\n )\n except ZeroDivisionError:\n num_batches = 0\n pad_factor = 0\n\n logging.debug(\n (\n \"DynamicBatchSampler: Bucket {} with boundary {:.1f}-{:.1f} and \"\n + \"batch_size {}: Num Examples {:.1f}, Num Full Batches {:.3f}, Pad Factor {:.3f}.\"\n ).format(\n bucket_indx,\n boundaries[bucket_indx],\n boundaries[bucket_indx + 1],\n self._bucket_lens[bucket_indx],\n stats_tracker[bucket_indx][\"n_ex\"],\n num_batches,\n pad_factor * 100,\n )\n )\n\n if self.verbose:\n batch_stats = {\n \"tot_frames\": [],\n \"tot_pad_frames\": [],\n \"pad_%\": [],\n }\n for batch in self._batches:\n tot_frames = sum(\n [self._ex_lengths[str(idx)] for idx in batch]\n )\n batch_stats[\"tot_frames\"].append(tot_frames)\n max_frames = max(\n [self._ex_lengths[str(idx)] for idx in batch]\n )\n tot_pad = sum(\n [\n max_frames - self._ex_lengths[str(idx)]\n for idx in batch\n ]\n )\n batch_stats[\"tot_pad_frames\"].append(tot_pad)\n batch_stats[\"pad_%\"].append(tot_pad / tot_frames * 100)\n\n padding_details = \"Batch {} with {:.1f} frames with {} files - {:.1f} padding, {:.2f} (%) of total.\"\n padding_details = \"DynamicBatchSampler: \" + padding_details\n for i in range(len(self._batches)):\n logging.debug(\n padding_details.format(\n i,\n batch_stats[\"tot_frames\"][i],\n len(self._batches[i]),\n batch_stats[\"tot_pad_frames\"][i],\n batch_stats[\"pad_%\"][i],\n )\n )\n\n def __iter__(self):\n\n for batch in self._replica_batches:\n yield batch\n\n\n # if self._shuffle_ex: # re-generate examples if ex_ordering == \"random\"\n # self._generate_batches()\n # if self._batch_ordering == \"random\":\n # # we randomly permute the batches only --> faster\n # self._permute_batches()\n\n def set_epoch(self, epoch):\n \"\"\"\n You can also just access self.epoch, but we maintain this interface\n to mirror torch.utils.data.distributed.DistributedSampler\n \"\"\"\n self._epoch = epoch\n self._generate_batches()\n self._replica_batches = self._batches[self.rank:self.total_size:self.num_replicas]\n self.num_samples = int(math.floor(len(self._batches) / self.num_replicas))\n assert len(self._replica_batches) == self.num_samples, f\"len(self._batches): {len(self._batches)}, self.total_size: {self.total_size}, self.num_samples: {self.num_samples},len(self._replica_batches): {len(self._replica_batches)}\"\n\n if self.continue_flag:\n self.continue_flag = False\n self._replica_batches = self._replica_batches[self._cur_step:]\n self.num_samples = len(self._replica_batches)\n \n\n def __len__(self):\n return self.num_samples\n \n def set_epoch_resume(self, epoch, cur_step):\n self.continue_flag = True\n self._epoch = epoch\n self._cur_step = cur_step\n" }, { "path": "tts_demo.py", "content": "\"\"\"\nThis script will allow you to run TTS inference with Voicecraft\nBefore getting started, be sure to follow the environment setup.\n\"\"\"\n\nfrom inference_tts_scale import inference_one_sample\nfrom models import voicecraft\nfrom data.tokenizer import (\n AudioTokenizer,\n TextTokenizer,\n)\nimport argparse\nimport random\nimport numpy as np\nimport torchaudio\nimport torch\nimport os\nos.environ[\"USER\"] = \"me\" # TODO change this to your username\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n\ndef parse_arguments():\n parser = argparse.ArgumentParser(\n description=\"VoiceCraft TTS Inference: see the script for more information on the options\")\n\n parser.add_argument(\"-m\", \"--model_name\", type=str, default=\"giga830M\", choices=[\n \"giga330M\", \"giga830M\", \"giga330M_TTSEnhanced\", \"giga830M_TTSEnhanced\"],\n help=\"VoiceCraft model to use\")\n parser.add_argument(\"-st\", \"--silence_tokens\", type=int, nargs=\"*\",\n default=[1388, 1898, 131], help=\"Silence token IDs\")\n parser.add_argument(\"-casr\", \"--codec_audio_sr\", type=int,\n default=16000, help=\"Codec audio sample rate.\")\n parser.add_argument(\"-csr\", \"--codec_sr\", type=int, default=50,\n help=\"Codec sample rate.\")\n\n parser.add_argument(\"-k\", \"--top_k\", type=float,\n default=0, help=\"Top k value.\")\n parser.add_argument(\"-p\", \"--top_p\", type=float,\n default=0.8, help=\"Top p value.\")\n parser.add_argument(\"-t\", \"--temperature\", type=float,\n default=1, help=\"Temperature value.\")\n parser.add_argument(\"-kv\", \"--kvcache\", type=float, choices=[0, 1],\n default=0, help=\"Kvcache value.\")\n parser.add_argument(\"-sr\", \"--stop_repetition\", type=int,\n default=-1, help=\"Stop repetition for generation\")\n parser.add_argument(\"--sample_batch_size\", type=int,\n default=3, help=\"Batch size for sampling\")\n parser.add_argument(\"-s\", \"--seed\", type=int,\n default=1, help=\"Seed value.\")\n parser.add_argument(\"-bs\", \"--beam_size\", type=int, default=50,\n help=\"beam size for MFA alignment\")\n parser.add_argument(\"-rbs\", \"--retry_beam_size\", type=int, default=200,\n help=\"retry beam size for MFA alignment\")\n parser.add_argument(\"--output_dir\", type=str, default=\"./generated_tts\",\n help=\"directory to save generated audio\")\n parser.add_argument(\"-oa\", \"--original_audio\", type=str,\n default=\"./demo/5895_34622_000026_000002.wav\", help=\"location of audio file\")\n parser.add_argument(\"-ot\", \"--original_transcript\", type=str,\n default=\"Gwynplaine had, besides, for his work and for his feats of strength, round his neck and over his shoulders, an esclavine of leather.\",\n help=\"original transcript\")\n parser.add_argument(\"-tt\", \"--target_transcript\", type=str,\n default=\"I cannot believe that the same model can also do text to speech synthesis too!\",\n help=\"target transcript\")\n parser.add_argument(\"-co\", \"--cut_off_sec\", type=float, default=3.6,\n help=\"cut off point in seconds for input prompt\")\n parser.add_argument(\"-ma\", \"--margin\", type=float, default=0.04,\n help=\"margin in seconds between the end of the cutoff words and the start of the next word. If the next word is not immediately following the cutoff word, the algorithm is more tolerant to word alignment errors\")\n parser.add_argument(\"-cuttol\", \"--cutoff_tolerance\", type=float, default=1, help=\"tolerance in seconds for the cutoff time, if given cut_off_sec plus the tolerance, we still are not able to find the next word, we will use the best cutoff time found, i.e. likely no margin or very small margin between the end of the cutoff word and the start of the next word\")\n\n args = parser.parse_args()\n return args\n\n\nargs = parse_arguments()\nvoicecraft_name = args.model_name\n# hyperparameters for inference\ncodec_audio_sr = args.codec_audio_sr\ncodec_sr = args.codec_sr\ntop_k = args.top_k\ntop_p = args.top_p # defaults to 0.9 can also try 0.8, but 0.9 seems to work better\ntemperature = args.temperature\nsilence_tokens = args.silence_tokens\nkvcache = args.kvcache # NOTE if OOM, change this to 0, or try the 330M model\n\n# NOTE adjust the below three arguments if the generation is not as good\n# NOTE if the model generate long silence, reduce the stop_repetition to 3, 2 or even 1\nstop_repetition = args.stop_repetition\n\n# NOTE: if the if there are long silence or unnaturally strecthed words,\n# increase sample_batch_size to 4 or higher. What this will do to the model is that the\n# model will run sample_batch_size examples of the same audio, and pick the one that's the shortest.\n# So if the speech rate of the generated is too fast change it to a smaller number.\nsample_batch_size = args.sample_batch_size\nseed = args.seed # change seed if you are still unhappy with the result\n\n# load the model\nif voicecraft_name == \"330M\":\n voicecraft_name = \"giga330M\"\nelif voicecraft_name == \"830M\":\n voicecraft_name = \"giga830M\"\nelif voicecraft_name == \"330M_TTSEnhanced\":\n voicecraft_name = \"330M_TTSEnhanced\"\nelif voicecraft_name == \"830M_TTSEnhanced\":\n voicecraft_name = \"830M_TTSEnhanced\"\nmodel = voicecraft.VoiceCraft.from_pretrained(\n f\"pyp1/VoiceCraft_{voicecraft_name.replace('.pth', '')}\")\nphn2num = model.args.phn2num\nconfig = vars(model.args)\nmodel.to(device)\n\nencodec_fn = \"./pretrained_models/encodec_4cb2048_giga.th\"\nif not os.path.exists(encodec_fn):\n os.system(\n f\"wget https://huggingface.co/pyp1/VoiceCraft/resolve/main/encodec_4cb2048_giga.th -O ./pretrained_models/encodec_4cb2048_giga.th\")\n# will also put the neural codec model on gpu\naudio_tokenizer = AudioTokenizer(signature=encodec_fn, device=device)\n\ntext_tokenizer = TextTokenizer(backend=\"espeak\")\n\n# Prepare your audio\n# point to the original audio whose speech you want to clone\n# write down the transcript for the file, or run whisper to get the transcript (and you can modify it if it's not accurate), save it as a .txt file\norig_audio = args.original_audio\norig_transcript = args.original_transcript\n\n# move the audio and transcript to temp folder\ntemp_folder = \"./demo/temp\"\nos.makedirs(temp_folder, exist_ok=True)\nos.system(f\"cp {orig_audio} {temp_folder}\")\nfilename = os.path.splitext(orig_audio.split(\"/\")[-1])[0]\nwith open(f\"{temp_folder}/{filename}.txt\", \"w\") as f:\n f.write(orig_transcript)\n# run MFA to get the alignment\nalign_temp = f\"{temp_folder}/mfa_alignments\"\nbeam_size = args.beam_size\nretry_beam_size = args.retry_beam_size\nalignments = f\"{temp_folder}/mfa_alignments/{filename}.csv\"\nif not os.path.isfile(alignments):\n os.system(f\"mfa align -v --clean -j 1 --output_format csv {temp_folder} \\\n english_us_arpa english_us_arpa {align_temp} --beam {beam_size} --retry_beam {retry_beam_size}\")\n# if the above fails, it could be because the audio is too hard for the alignment model,\n# increasing the beam_size and retry_beam_size usually solves the issue\n\ndef find_closest_word_boundary(alignments, cut_off_sec, margin, cutoff_tolerance = 1):\n with open(alignments, 'r') as file:\n # skip header\n next(file)\n cutoff_time = None\n cutoff_index = None\n cutoff_time_best = None\n cutoff_index_best = None\n lines = [l for l in file.readlines()]\n for i, line in enumerate(lines):\n end = float(line.strip().split(',')[1])\n if end >= cut_off_sec and cutoff_time == None:\n cutoff_time = end\n cutoff_index = i\n if end >= cut_off_sec and end < cut_off_sec + cutoff_tolerance and float(lines[i+1].strip().split(',')[0]) - end >= margin:\n cutoff_time_best = end + margin * 2 / 3\n cutoff_index_best = i\n break\n if cutoff_time_best != None:\n cutoff_time = cutoff_time_best\n cutoff_index = cutoff_index_best\n return cutoff_time, cutoff_index\n\n# take a look at demo/temp/mfa_alignment, decide which part of the audio to use as prompt\n# NOTE: according to forced-alignment file demo/temp/mfa_alignments/5895_34622_000026_000002.wav, the word \"strength\" stop as 3.561 sec, so we use first 3.6 sec as the prompt. this should be different for different audio\ncut_off_sec = args.cut_off_sec\nmargin = args.margin\naudio_fn = f\"{temp_folder}/{filename}.wav\"\n\ncut_off_sec, cut_off_word_idx = find_closest_word_boundary(alignments, cut_off_sec, margin, args.cutoff_tolerance)\ntarget_transcript = \" \".join(orig_transcript.split(\" \")[:cut_off_word_idx+1]) + \" \" + args.target_transcript\n# NOTE: 3 sec of reference is generally enough for high quality voice cloning, but longer is generally better, try e.g. 3~6 sec.\ninfo = torchaudio.info(audio_fn)\naudio_dur = info.num_frames / info.sample_rate\n\nassert cut_off_sec < audio_dur, f\"cut_off_sec {cut_off_sec} is larger than the audio duration {audio_dur}\"\nprompt_end_frame = int(cut_off_sec * info.sample_rate)\n\n\ndef seed_everything(seed):\n os.environ['PYTHONHASHSEED'] = str(seed)\n random.seed(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.benchmark = False\n torch.backends.cudnn.deterministic = True\n\n\nseed_everything(seed)\n\n# inference\ndecode_config = {'top_k': top_k, 'top_p': top_p, 'temperature': temperature, 'stop_repetition': stop_repetition, 'kvcache': kvcache,\n \"codec_audio_sr\": codec_audio_sr, \"codec_sr\": codec_sr, \"silence_tokens\": silence_tokens, \"sample_batch_size\": sample_batch_size}\nconcated_audio, gen_audio = inference_one_sample(model, argparse.Namespace(\n **config), phn2num, text_tokenizer, audio_tokenizer, audio_fn, target_transcript, device, decode_config, prompt_end_frame)\n\n# save segments for comparison\nconcated_audio, gen_audio = concated_audio[0].cpu(), gen_audio[0].cpu()\n# logging.info(f\"length of the resynthesize orig audio: {orig_audio.shape}\")\n\n# save the audio\n# output_dir\noutput_dir = args.output_dir\nos.makedirs(output_dir, exist_ok=True)\nseg_save_fn_gen = f\"{output_dir}/{os.path.basename(audio_fn)[:-4]}_gen_seed{seed}.wav\"\nseg_save_fn_concat = f\"{output_dir}/{os.path.basename(audio_fn)[:-4]}_concat_seed{seed}.wav\"\n\ntorchaudio.save(seg_save_fn_gen, gen_audio, codec_audio_sr)\ntorchaudio.save(seg_save_fn_concat, concated_audio, codec_audio_sr)\n\n# you might get warnings like WARNING:phonemizer:words count mismatch on 300.0% of the lines (3/1), this can be safely ignored\n" }, { "path": "voicecraft-gradio-colab.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"view-in-github\"\n },\n \"source\": [\n \"\\\"Open\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"Y87ixxsUVIhM\"\n },\n \"outputs\": [],\n \"source\": [\n \"!git clone https://github.com/jasonppy/VoiceCraft\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"-w3USR91XdxY\"\n },\n \"outputs\": [],\n \"source\": [\n \"!pip install tensorboard\\n\",\n \"!pip install phonemizer\\n\",\n \"!pip install datasets\\n\",\n \"!pip install torchmetrics\\n\",\n \"\\n\",\n \"!apt-get install -y espeak espeak-data libespeak1 libespeak-dev\\n\",\n \"!apt-get install -y festival*\\n\",\n \"!apt-get install -y build-essential\\n\",\n \"!apt-get install -y flac libasound2-dev libsndfile1-dev vorbis-tools\\n\",\n \"!apt-get install -y libxml2-dev libxslt-dev zlib1g-dev\\n\",\n \"\\n\",\n \"!pip install -e git+https://github.com/facebookresearch/audiocraft.git@c5157b5bf14bf83449c17ea1eeb66c19fb4bc7f0#egg=audiocraft\\n\",\n \"\\n\",\n \"!pip install -r \\\"/content/VoiceCraft/gradio_requirements.txt\\\"\\n\",\n \"!pip install typer==0.7.0\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"jNuzjrtmv2n1\"\n },\n \"source\": [\n \"# Let it restarted, it won't let your entire installation be aborted.\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"AnqGEwZ4NxtJ\"\n },\n \"source\": [\n \"# Note before launching the `gradio_app.py`\\n\",\n \"\\n\",\n \"***You will get JSON warning if you move anything beside `sample_batch_size`, `stop_repetition` and `seed`.*** Which for most advanced setting, `kvache` and `temperature` unable to set in different value.\\n\",\n \"\\n\",\n \"You will download a .file File when you download the output audio for some reason. You will need to **convert the file from .snd to .wav/.mp3 manually**. Or if you enable showing file type in the name in Windows or wherever you are, change the file name to \\\"xxx.wav\\\" or \\\"xxx.mp3\\\". (know the solution? pull request my repository)\\n\",\n \"\\n\",\n \"Frequency of VRAM spikes no longer exist as well in April 5 Update.\\n\",\n \"* Nevermind, I have observed some weird usage on Colab's GPU Memory Monitor. It can spike up to 13.5GB VRAM even in WhisperX mode. (April 11)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"dE0W76cMN3Si\"\n },\n \"source\": [\n \"Don't make your `prompt end time` too long, 6-9s is fine. Or else it will **either raise up JSON issue or cut off your generated audio**. This one is due to how VoiceCraft worked (so probably unfixable). It will add those text you want to get audio from at the end of the input audio transcript. It was way too much word for application or code to handle as it added up with original transcript. So please keep it short.\\n\",\n \"\\n\",\n \"Your total audio length (`prompt end time` + add-up audio) must not exceed 16 or 17s.\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"nnu2cY4t8P6X\"\n },\n \"source\": [\n \"For voice cloning, I suggest you to probably have a monotone input to feed the voice cloning. Of course you can always try input that have tons of tone variety, but I find that as per April 11 Update, it's much more easy to replicate in monotone rather than audio that have laugh, scream, crying inside.\\n\",\n \"\\n\",\n \"The inference speed is much stable. With sample text, T4 (Free Tier Colab GPU) can do 6-15s on 6s-8s of `prompt end time`.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"NDt4r4DiXAwG\"\n },\n \"outputs\": [],\n \"source\": [\n \"!python /content/VoiceCraft/gradio_app.py --demo-path=/content/VoiceCraft/demo --tmp-path=/content/VoiceCraft/demo/temp --models-path=/content/VoiceCraft/pretrained_models --share\"\n ]\n }\n ],\n \"metadata\": {\n \"accelerator\": \"GPU\",\n \"colab\": {\n \"authorship_tag\": \"ABX9TyPsqFhtOeQ18CXHnRkWAQSk\",\n \"gpuType\": \"T4\",\n \"include_colab_link\": true,\n \"provenance\": []\n },\n \"kernelspec\": {\n \"display_name\": \"Python 3\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"name\": \"python\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 0\n}\n" }, { "path": "z_scripts/e830M.sh", "content": "#!/bin/bash\nsource ~/miniconda3/etc/profile.d/conda.sh\nconda activate voicecraft\nexport CUDA_VISIBLE_DEVICES=0,1,2,3\nexport WORLD_SIZE=4\n\ndataset=gigaspeech\nmkdir -p ./logs/${dataset}\n\nexp_root=\"path/to/store/exp_results\"\nexp_name=e830M\ndataset_dir=\"path/to/stored_extracted_codes_and_phonemes/xl\" # xs if you only extracted xs in previous step\nencodec_codes_folder_name=\"encodec_16khz_4codebooks\"\n\n# export CUDA_LAUNCH_BLOCKING=1 # for debugging\n\ntorchrun --nnodes=1 --rdzv-backend=c10d --rdzv-endpoint=localhost:41977 --nproc_per_node=${WORLD_SIZE} \\\n../main.py \\\n--reduced_eog 1 \\\n--drop_long 1 \\\n--eos 2051 \\\n--n_special 4 \\\n--pad_x 0 \\\n--codebook_weight \"[5,1,0.5,0.1]\" \\\n--encodec_sr 50 \\\n--num_steps 50000 \\\n--lr 0.05 \\\n--warmup_fraction 0.01 \\\n--optimizer_name \"ScaledAdam\" \\\n--pseudo_epoch_size 3000 \\\n--reduce_lr_start_step 3000 \\\n--reduce_lr_start_epoch 4 \\\n--clipping_update_period 1000 \\\n--d_model 2048 \\\n--audio_embedding_dim 2048 \\\n--nhead 16 \\\n--num_decoder_layers 16 \\\n--max_num_tokens 100000 \\\n--gradient_accumulation_steps 26 \\\n--val_max_num_tokens 6000 \\\n--num_buckets 6 \\\n--audio_max_length 20 \\\n--audio_min_length 2 \\\n--text_max_length 400 \\\n--text_min_length 10 \\\n--mask_len_min 1 \\\n--mask_len_max 600 \\\n--tb_write_every_n_steps 10 \\\n--print_every_n_steps 400 \\\n--val_every_n_steps 1600 \\\n--text_vocab_size 100 \\\n--text_pad_token 100 \\\n--phn_folder_name \"phonemes\" \\\n--manifest_name \"manifest\" \\\n--encodec_folder_name ${encodec_codes_folder_name} \\\n--audio_vocab_size 2048 \\\n--empty_token 2048 \\\n--eog 2049 \\\n--audio_pad_token 2050 \\\n--n_codebooks 4 \\\n--max_n_spans 3 \\\n--shuffle_mask_embedding 0 \\\n--mask_sample_dist poisson1 \\\n--max_mask_portion 0.9 \\\n--min_gap 5 \\\n--num_workers 8 \\\n--dynamic_batching 1 \\\n--dataset $dataset \\\n--exp_dir \"${exp_root}/${dataset}/${exp_name}\" \\\n--dataset_dir ${dataset_dir}\n# >> ./logs/${dataset}/${exp_name}.log 2>&1" }, { "path": "z_scripts/e830M_ft.sh", "content": "#!/bin/bash\nsource ~/miniconda3/etc/profile.d/conda.sh\nconda activate voicecraft\nexport CUDA_VISIBLE_DEVICES=0,1,2,3\nexport WORLD_SIZE=4\n\ndataset=gigaspeech\nmkdir -p ./logs/${dataset}\n\nexp_root=\"path/to/store/exp_results\"\nexp_name=e830M_ft\ndataset_dir=\"path/to/stored_extracted_codes_and_phonemes/xl\" # xs if you only extracted xs in previous step\nencodec_codes_folder_name=\"encodec_16khz_4codebooks\"\nload_model_from=\"./pretrained_models/giga830M.pth\"\n\n# export CUDA_LAUNCH_BLOCKING=1 # for debugging\n\ntorchrun --nnodes=1 --rdzv-backend=c10d --rdzv-endpoint=localhost:41977 --nproc_per_node=${WORLD_SIZE} \\\n../main.py \\\n--load_model_from ${load_model_from} \\\n--reduced_eog 1 \\\n--drop_long 1 \\\n--eos 2051 \\\n--n_special 4 \\\n--pad_x 0 \\\n--codebook_weight \"[3,1,1,1]\" \\\n--encodec_sr 50 \\\n--num_steps 500000 \\\n--lr 0.00001 \\\n--warmup_fraction 0.1 \\\n--optimizer_name \"AdamW\" \\\n--d_model 2048 \\\n--audio_embedding_dim 2048 \\\n--nhead 16 \\\n--num_decoder_layers 16 \\\n--max_num_tokens 20000 \\\n--gradient_accumulation_steps 12 \\\n--val_max_num_tokens 6000 \\\n--num_buckets 6 \\\n--audio_max_length 20 \\\n--audio_min_length 2 \\\n--text_max_length 400 \\\n--text_min_length 10 \\\n--mask_len_min 1 \\\n--mask_len_max 600 \\\n--tb_write_every_n_steps 10 \\\n--print_every_n_steps 400 \\\n--val_every_n_steps 1600 \\\n--text_vocab_size 100 \\\n--text_pad_token 100 \\\n--phn_folder_name \"phonemes\" \\\n--manifest_name \"manifest\" \\\n--encodec_folder_name ${encodec_codes_folder_name} \\\n--audio_vocab_size 2048 \\\n--empty_token 2048 \\\n--eog 2049 \\\n--audio_pad_token 2050 \\\n--n_codebooks 4 \\\n--max_n_spans 3 \\\n--shuffle_mask_embedding 0 \\\n--mask_sample_dist poisson1 \\\n--max_mask_portion 0.9 \\\n--min_gap 5 \\\n--num_workers 8 \\\n--dynamic_batching 1 \\\n--dataset $dataset \\\n--exp_dir \"${exp_root}/${dataset}/${exp_name}\" \\\n--dataset_dir ${dataset_dir}\n# >> ./logs/${dataset}/${exp_name}.log 2>&1" } ]