[ { "path": ".gitignore", "content": "# Global\n.DS_Store\n.idea\n\n# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\npip-wheel-metadata/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n*.py,cover\n.hypothesis/\n.pytest_cache/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\ndb.sqlite3-journal\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# IPython\nprofile_default/\nipython_config.py\n\n# pyenv\n.python-version\n\n# pipenv\n# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.\n# However, in case of collaboration, if having platform-specific dependencies or dependencies\n# having no cross-platform support, pipenv may install dependencies that don't work, or not\n# install all needed dependencies.\n#Pipfile.lock\n\n# PEP 582; used by e.g. github.com/David-OConnor/pyflow\n__pypackages__/\n\n# Celery stuff\ncelerybeat-schedule\ncelerybeat.pid\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n.dmypy.json\ndmypy.json\n\n# Pyre type checker\n.pyre/" }, { "path": "LICENSE", "content": "Noncommercial Use License\n\nSoftware Copyright (c) 2020 OpenAI\n\nWe don’t claim ownership of the content you create with Jukebox.\nWe only ask that you use Jukebox responsibly and clearly indicate your content was created using OpenAI’s Jukebox.\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this software and associated\ndocumentation files (the \"Software\"), to deal in the Software, including without limitation the rights to use, copy,\nmodify, merge, publish, distribute, and/or sublicense copies of the Software, and to permit persons to whom the\nSoftware is furnished to do so, subject to the following conditions:\n\nNo portion of the Software, nor any content created with the Software, may be used for commercial purposes.\n\nThe above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n\nThe above copyright notice and this permission notice need not be included with content created by the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE\nWARRANTIES OF MERCHANTABILITY,FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR\nOTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE." }, { "path": "MANIFEST.in", "content": "recursive-include jukebox *.py\nrecursive-include jukebox *.txt\n" }, { "path": "README.md", "content": "**Status:** Archive (code is provided as-is, no updates expected)\n\n# Jukebox\nCode for \"Jukebox: A Generative Model for Music\"\n\n[Paper](https://arxiv.org/abs/2005.00341) \n[Blog](https://openai.com/blog/jukebox) \n[Explorer](http://jukebox.openai.com/) \n[Colab](https://colab.research.google.com/github/openai/jukebox/blob/master/jukebox/Interacting_with_Jukebox.ipynb) \n\n# Install\nInstall the conda package manager from https://docs.conda.io/en/latest/miniconda.html \n \n``` \n# Required: Sampling\nconda create --name jukebox python=3.7.5\nconda activate jukebox\nconda install mpi4py=3.0.3 # if this fails, try: pip install mpi4py==3.0.3\nconda install pytorch=1.4 torchvision=0.5 cudatoolkit=10.0 -c pytorch\ngit clone https://github.com/openai/jukebox.git\ncd jukebox\npip install -r requirements.txt\npip install -e .\n\n# Required: Training\nconda install av=7.0.01 -c conda-forge \npip install ./tensorboardX\n \n# Optional: Apex for faster training with fused_adam\nconda install pytorch=1.1 torchvision=0.3 cudatoolkit=10.0 -c pytorch\npip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ./apex\n```\n\n# Sampling\n## Sampling from scratch\nTo sample normally, run the following command. Model can be `5b`, `5b_lyrics`, `1b_lyrics`\n``` \npython jukebox/sample.py --model=5b_lyrics --name=sample_5b --levels=3 --sample_length_in_seconds=20 \\\n--total_sample_length_in_seconds=180 --sr=44100 --n_samples=6 --hop_fraction=0.5,0.5,0.125\n```\n``` \npython jukebox/sample.py --model=1b_lyrics --name=sample_1b --levels=3 --sample_length_in_seconds=20 \\\n--total_sample_length_in_seconds=180 --sr=44100 --n_samples=16 --hop_fraction=0.5,0.5,0.125\n```\nThe above generates the first `sample_length_in_seconds` seconds of audio from a song of total length `total_sample_length_in_seconds`.\nTo use multiple GPU's, launch the above scripts as `mpiexec -n {ngpus} python jukebox/sample.py ...` so they use `{ngpus}`\n\nThe samples decoded from each level are stored in `{name}/level_{level}`. \nYou can also view the samples as an html with the aligned lyrics under `{name}/level_{level}/index.html`. \nRun `python -m http.server` and open the html through the server to see the lyrics animate as the song plays. \nA summary of all sampling data including zs, x, labels and sampling_kwargs is stored in `{name}/level_{level}/data.pth.tar`.\n\nThe hps are for a V100 GPU with 16 GB GPU memory. The `1b_lyrics`, `5b`, and `5b_lyrics` top-level priors take up \n3.8 GB, 10.3 GB, and 11.5 GB, respectively. The peak memory usage to store transformer key, value cache is about 400 MB \nfor `1b_lyrics` and 1 GB for `5b_lyrics` per sample. If you are having trouble with CUDA OOM issues, try `1b_lyrics` or \ndecrease `max_batch_size` in sample.py, and `--n_samples` in the script call.\n\nOn a V100, it takes about 3 hrs to fully sample 20 seconds of music. Since this is a long time, it is recommended to use `n_samples > 1` so you can generate as many samples as possible in parallel. The 1B lyrics and upsamplers can process 16 samples at a time, while 5B can fit only up to 3. Since the vast majority of time is spent on upsampling, we recommend using a multiple of 3 less than 16 like `--n_samples 15` for `5b_lyrics`. This will make the top-level generate samples in groups of three while upsampling is done in one pass.\n\nTo continue sampling from already generated codes for a longer duration, you can run\n```\npython jukebox/sample.py --model=5b_lyrics --name=sample_5b --levels=3 --mode=continue \\\n--codes_file=sample_5b/level_0/data.pth.tar --sample_length_in_seconds=40 --total_sample_length_in_seconds=180 \\\n--sr=44100 --n_samples=6 --hop_fraction=0.5,0.5,0.125\n```\nHere, we take the 20 seconds samples saved from the first sampling run at `sample_5b/level_0/data.pth.tar` and continue by adding 20 more seconds. \n\nYou could also continue directly from the level 2 saved outputs, just pass `--codes_file=sample_5b/level_2/data.pth.tar`.\n Note this will upsample the full 40 seconds song at the end.\n\nIf you stopped sampling at only the first level and want to upsample the saved codes, you can run\n```\npython jukebox/sample.py --model=5b_lyrics --name=sample_5b --levels=3 --mode=upsample \\\n--codes_file=sample_5b/level_2/data.pth.tar --sample_length_in_seconds=20 --total_sample_length_in_seconds=180 \\\n--sr=44100 --n_samples=6 --hop_fraction=0.5,0.5,0.125\n```\nHere, we take the 20 seconds samples saved from the first sampling run at `sample_5b/level_2/data.pth.tar` and upsample the lower two levels.\n\n## Prompt with your own music\nIf you want to prompt the model with your own creative piece or any other music, first save them as wave files and run\n```\npython jukebox/sample.py --model=5b_lyrics --name=sample_5b_prompted --levels=3 --mode=primed \\\n--audio_file=path/to/recording.wav,awesome-mix.wav,fav-song.wav,etc.wav --prompt_length_in_seconds=12 \\\n--sample_length_in_seconds=20 --total_sample_length_in_seconds=180 --sr=44100 --n_samples=6 --hop_fraction=0.5,0.5,0.125\n```\nThis will load the four files, tile them to fill up to `n_samples` batch size, and prime the model with the first `prompt_length_in_seconds` seconds.\n\n# Training\n## VQVAE\nTo train a small vqvae, run\n```\nmpiexec -n {ngpus} python jukebox/train.py --hps=small_vqvae --name=small_vqvae --sample_length=262144 --bs=4 \\\n--audio_files_dir={audio_files_dir} --labels=False --train --aug_shift --aug_blend\n```\nHere, `{audio_files_dir}` is the directory in which you can put the audio files for your dataset, and `{ngpus}` is number of GPU's you want to use to train. \nThe above trains a two-level VQ-VAE with `downs_t = (5,3)`, and `strides_t = (2, 2)` meaning we downsample the audio by `2**5 = 32` to get the first level of codes, and `2**8 = 256` to get the second level codes. \nCheckpoints are stored in the `logs` folder. You can monitor the training by running Tensorboard\n```\ntensorboard --logdir logs\n```\n \n## Prior\n### Train prior or upsamplers\nOnce the VQ-VAE is trained, we can restore it from its saved checkpoint and train priors on the learnt codes. \nTo train the top-level prior, we can run\n\n```\nmpiexec -n {ngpus} python jukebox/train.py --hps=small_vqvae,small_prior,all_fp16,cpu_ema --name=small_prior \\\n--sample_length=2097152 --bs=4 --audio_files_dir={audio_files_dir} --labels=False --train --test --aug_shift --aug_blend \\\n--restore_vqvae=logs/small_vqvae/checkpoint_latest.pth.tar --prior --levels=2 --level=1 --weight_decay=0.01 --save_iters=1000\n```\n\nTo train the upsampler, we can run\n```\nmpiexec -n {ngpus} python jukebox/train.py --hps=small_vqvae,small_upsampler,all_fp16,cpu_ema --name=small_upsampler \\\n--sample_length=262144 --bs=4 --audio_files_dir={audio_files_dir} --labels=False --train --test --aug_shift --aug_blend \\\n--restore_vqvae=logs/small_vqvae/checkpoint_latest.pth.tar --prior --levels=2 --level=0 --weight_decay=0.01 --save_iters=1000\n```\nWe pass `sample_length = n_ctx * downsample_of_level` so that after downsampling the tokens match the n_ctx of the prior hps. \nHere, `n_ctx = 8192` and `downsamples = (32, 256)`, giving `sample_lengths = (8192 * 32, 8192 * 256) = (65536, 2097152)` respectively for the bottom and top level. \n\n### Learning rate annealing\nTo get the best sample quality anneal the learning rate to 0 near the end of training. To do so, continue training from the latest \ncheckpoint and run with\n```\n--restore_prior=\"path/to/checkpoint\" --lr_use_linear_decay --lr_start_linear_decay={already_trained_steps} --lr_decay={decay_steps_as_needed}\n```\n\n### Reuse pre-trained VQ-VAE and train top-level prior on new dataset from scratch.\n#### Train without labels\nOur pre-trained VQ-VAE can produce compressed codes for a wide variety of genres of music, and the pre-trained upsamplers \ncan upsample them back to audio that sound very similar to the original audio.\nTo re-use these for a new dataset of your choice, you can retrain just the top-level \n\nTo train top-level on a new dataset, run\n```\nmpiexec -n {ngpus} python jukebox/train.py --hps=vqvae,small_prior,all_fp16,cpu_ema --name=pretrained_vqvae_small_prior \\\n--sample_length=1048576 --bs=4 --aug_shift --aug_blend --audio_files_dir={audio_files_dir} \\\n--labels=False --train --test --prior --levels=3 --level=2 --weight_decay=0.01 --save_iters=1000\n```\nTraining the `small_prior` with a batch size of 2, 4, and 8 requires 6.7 GB, 9.3 GB, and 15.8 GB of GPU memory, respectively. A few days to a week of training typically yields reasonable samples when the dataset is homogeneous (e.g. all piano pieces, songs of the same style, etc).\n\nNear the end of training, follow [this](#learning-rate-annealing) to anneal the learning rate to 0\n\n#### Sample from new model\nYou can then run sample.py with the top-level of our models replaced by your new model. To do so,\n- Add an entry `my_model=(\"vqvae\", \"upsampler_level_0\", \"upsampler_level_1\", \"small_prior\")` in `MODELS` in `make_models.py`. \n- Update the `small_prior` dictionary in `hparams.py` to include `restore_prior='path/to/checkpoint'`. If you\nyou changed any hps directly in the command line script (eg:`heads`), make sure to update them in the dictionary too so \nthat `make_models` restores our checkpoint correctly.\n- Run sample.py as outlined in the sampling section, but now with `--model=my_model` \n\nFor example, let's say we trained `small_vqvae`, `small_prior`, and `small_upsampler` under `/path/to/jukebox/logs`. In `make_models.py`, we are going to declare a tuple of the new models as `my_model`.\n```\nMODELS = {\n '5b': (\"vqvae\", \"upsampler_level_0\", \"upsampler_level_1\", \"prior_5b\"),\n '5b_lyrics': (\"vqvae\", \"upsampler_level_0\", \"upsampler_level_1\", \"prior_5b_lyrics\"),\n '1b_lyrics': (\"vqvae\", \"upsampler_level_0\", \"upsampler_level_1\", \"prior_1b_lyrics\"),\n 'my_model': (\"my_small_vqvae\", \"my_small_upsampler\", \"my_small_prior\"),\n}\n```\n\nNext, in `hparams.py`, we add them to the registry with the corresponding `restore_`paths and any other command line options used during training. Another important note is that for top-level priors with lyric conditioning, we have to locate a self-attention layer that shows alignment between the lyric and music tokens. Look for layers where `prior.prior.transformer._attn_mods[layer].attn_func` is either 6 or 7. If your model is starting to sing along lyrics, it means some layer, head pair has learned alignment. Congrats!\n```\nmy_small_vqvae = Hyperparams(\n restore_vqvae='/path/to/jukebox/logs/small_vqvae/checkpoint_some_step.pth.tar',\n)\nmy_small_vqvae.update(small_vqvae)\nHPARAMS_REGISTRY[\"my_small_vqvae\"] = my_small_vqvae\n\nmy_small_prior = Hyperparams(\n restore_prior='/path/to/jukebox/logs/small_prior/checkpoint_latest.pth.tar',\n level=1,\n labels=False,\n # TODO For the two lines below, if `--labels` was used and the model is\n # trained with lyrics, find and enter the layer, head pair that has learned\n # alignment.\n alignment_layer=47,\n alignment_head=0,\n)\nmy_small_prior.update(small_prior)\nHPARAMS_REGISTRY[\"my_small_prior\"] = my_small_prior\n\nmy_small_upsampler = Hyperparams(\n restore_prior='/path/to/jukebox/logs/small_upsampler/checkpoint_latest.pth.tar',\n level=0,\n labels=False,\n)\nmy_small_upsampler.update(small_upsampler)\nHPARAMS_REGISTRY[\"my_small_upsampler\"] = my_small_upsampler\n```\n\n#### Train with labels \nTo train with you own metadata for your audio files, implement `get_metadata` in `data/files_dataset.py` to return the \n`artist`, `genre` and `lyrics` for a given audio file. For now, you can pass `''` for lyrics to not use any lyrics.\n\nFor training with labels, we'll use `small_labelled_prior` in `hparams.py`, and we set `labels=True,labels_v3=True`. \nWe use 2 kinds of labels information:\n- Artist/Genre: \n - For each file, we return an artist_id and a list of genre_ids. The reason we have a list and not a single genre_id \n is that in v2, we split genres like `blues_rock` into a bag of words `[blues, rock]`, and we pass atmost \n `max_bow_genre_size` of those, in `v3` we consider it as a single word and just set `max_bow_genre_size=1`.\n - Update the `v3_artist_ids` and `v3_genre_ids` to use ids from your new dataset. \n - In `small_labelled_prior`, set the hps `y_bins = (number_of_genres, number_of_artists)` and `max_bow_genre_size=1`. \n- Timing: \n - For each chunk of audio, we return the `total_length` of the song, the `offset` the current audio chunk is at and \n the `sample_length` of the audio chunk. We have three timing embeddings: total_length, our current position, and our \n current position as a fraction of the total length, and we divide the range of these values into `t_bins` discrete bins. \n - In `small_labelled_prior`, set the hps `min_duration` and `max_duration` to be the shortest/longest duration of audio \n files you want for your dataset, and `t_bins` for how many bins you want to discretize timing information into. Note \n `min_duration * sr` needs to be at least `sample_length` to have an audio chunk in it.\n\nAfter these modifications, to train a top-level with labels, run\n```\nmpiexec -n {ngpus} python jukebox/train.py --hps=vqvae,small_labelled_prior,all_fp16,cpu_ema --name=pretrained_vqvae_small_prior_labels \\\n--sample_length=1048576 --bs=4 --aug_shift --aug_blend --audio_files_dir={audio_files_dir} \\\n--labels=True --train --test --prior --levels=3 --level=2 --weight_decay=0.01 --save_iters=1000\n```\n\nFor sampling, follow same instructions as [above](#sample-from-new-model) but use `small_labelled_prior` instead of `small_prior`. \n\n#### Train with lyrics\nTo train in addition with lyrics, update `get_metadata` in `data/files_dataset.py` to return `lyrics` too.\nFor training with lyrics, we'll use `small_single_enc_dec_prior` in `hparams.py`. \n- Lyrics: \n - For each file, we linearly align the lyric characters to the audio, find the position in lyric that corresponds to \n the midpoint of our audio chunk, and pass a window of `n_tokens` lyric characters centred around that. \n - In `small_single_enc_dec_prior`, set the hps `use_tokens=True` and `n_tokens` to be the number of lyric characters \n to use for an audio chunk. Set it according to the `sample_length` you're training on so that its large enough that \n the lyrics for an audio chunk are almost always found inside a window of that size.\n - If you use a non-English vocabulary, update `text_processor.py` with your new vocab and set\n `n_vocab = number of characters in vocabulary` accordingly in `small_single_enc_dec_prior`. In v2, we had a `n_vocab=80` \n and in v3 we missed `+` and so `n_vocab=79` of characters. \n\nAfter these modifications, to train a top-level with labels and lyrics, run\n```\nmpiexec -n {ngpus} python jukebox/train.py --hps=vqvae,small_single_enc_dec_prior,all_fp16,cpu_ema --name=pretrained_vqvae_small_single_enc_dec_prior_labels \\\n--sample_length=786432 --bs=4 --aug_shift --aug_blend --audio_files_dir={audio_files_dir} \\\n--labels=True --train --test --prior --levels=3 --level=2 --weight_decay=0.01 --save_iters=1000\n```\nTo simplify hps choices, here we used a `single_enc_dec` model like the `1b_lyrics` model that combines both encoder and \ndecoder of the transformer into a single model. We do so by merging the lyric vocab and vq-vae vocab into a single \nlarger vocab, and flattening the lyric tokens and the vq-vae codes into a single sequence of length `n_ctx + n_tokens`. \nThis uses `attn_order=12` which includes `prime_attention` layers with keys/values from lyrics and queries from audio. \nIf you instead want to use a model with the usual encoder-decoder style transformer, use `small_sep_enc_dec_prior`.\n\nFor sampling, follow same instructions as [above](#sample-from-new-model) but use `small_single_enc_dec_prior` instead of \n`small_prior`. To also get the alignment between lyrics and samples in the saved html, you'll need to set `alignment_layer` \nand `alignment_head` in `small_single_enc_dec_prior`. To find which layer/head is best to use, run a forward pass on a training example,\nsave the attention weight tensors for all prime_attention layers, and pick the (layer, head) which has the best linear alignment \npattern between the lyrics keys and music queries. \n\n### Fine-tune pre-trained top-level prior to new style(s)\nPreviously, we showed how to train a small top-level prior from scratch. Assuming you have a GPU with at least 15 GB of memory and support for fp16, you could fine-tune from our pre-trained 1B top-level prior. Here are the steps:\n\n- Support `--labels=True` by implementing `get_metadata` in `jukebox/data/files_dataset.py` for your dataset.\n- Add new entries in `jukebox/data/ids`. We recommend replacing existing mappings (e.g. rename `\"unknown\"`, etc with styles of your choice). This uses the pre-trained style vectors as initialization and could potentially save some compute.\n\nAfter these modifications, run \n```\nmpiexec -n {ngpus} python jukebox/train.py --hps=vqvae,prior_1b_lyrics,all_fp16,cpu_ema --name=finetuned \\\n--sample_length=1048576 --bs=1 --aug_shift --aug_blend --audio_files_dir={audio_files_dir} \\\n--labels=True --train --test --prior --levels=3 --level=2 --weight_decay=0.01 --save_iters=1000\n```\nTo get the best sample quality, it is recommended to anneal the learning rate in the end. Training the 5B top-level requires GPipe which is not supported in this release.\n\n# Citation\n\nPlease cite using the following bibtex entry:\n\n```\n@article{dhariwal2020jukebox,\n title={Jukebox: A Generative Model for Music},\n author={Dhariwal, Prafulla and Jun, Heewoo and Payne, Christine and Kim, Jong Wook and Radford, Alec and Sutskever, Ilya},\n journal={arXiv preprint arXiv:2005.00341},\n year={2020}\n}\n```\n\n# License \n[Noncommercial Use License](./LICENSE) \n\nIt covers both released code and weights. \n\n" }, { "path": "apex/.gitignore", "content": "apex.egg-info\ndist\nbuild\ndocs/build\n*~" }, { "path": "apex/.nojekyll", "content": "" }, { "path": "apex/LICENSE", "content": "All rights reserved.\n\nRedistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:\n\n1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.\n\n2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.\n\n3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE." }, { "path": "apex/README.md", "content": "# Introduction\n\nThis repository holds NVIDIA-maintained utilities to streamline \nmixed precision and distributed training in Pytorch. \nSome of the code here will be included in upstream Pytorch eventually.\nThe intention of Apex is to make up-to-date utilities available to \nusers as quickly as possible.\n\n## Full API Documentation: [https://nvidia.github.io/apex](https://nvidia.github.io/apex)\n\n# Contents\n\n## 1. Amp: Automatic Mixed Precision\n\n`apex.amp` is a tool to enable mixed precision training by changing only 3 lines of your script.\nUsers can easily experiment with different pure and mixed precision training modes by supplying\ndifferent flags to `amp.initialize`.\n\n[Webinar introducing Amp](https://info.nvidia.com/webinar-mixed-precision-with-pytorch-reg-page.html)\n(The flag `cast_batchnorm` has been renamed to `keep_batchnorm_fp32`).\n\n[API Documentation](https://nvidia.github.io/apex/amp.html)\n\n[Comprehensive Imagenet example](https://github.com/NVIDIA/apex/tree/master/examples/imagenet)\n\n[DCGAN example coming soon...](https://github.com/NVIDIA/apex/tree/master/examples/dcgan)\n\n[Moving to the new Amp API](https://nvidia.github.io/apex/amp.html#transition-guide-for-old-api-users) (for users of the deprecated \"Amp\" and \"FP16_Optimizer\" APIs)\n\n## 2. Distributed Training\n\n`apex.parallel.DistributedDataParallel` is a module wrapper, similar to \n`torch.nn.parallel.DistributedDataParallel`. It enables convenient multiprocess distributed training,\noptimized for NVIDIA's NCCL communication library.\n\n[API Documentation](https://nvidia.github.io/apex/parallel.html)\n\n[Python Source](https://github.com/NVIDIA/apex/tree/master/apex/parallel)\n\n[Example/Walkthrough](https://github.com/NVIDIA/apex/tree/master/examples/simple/distributed)\n\nThe [Imagenet example](https://github.com/NVIDIA/apex/tree/master/examples/imagenet)\nshows use of `apex.parallel.DistributedDataParallel` along with `apex.amp`.\n\n### Synchronized Batch Normalization\n\n`apex.parallel.SyncBatchNorm` extends `torch.nn.modules.batchnorm._BatchNorm` to\nsupport synchronized BN.\nIt allreduces stats across processes during multiprocess (DistributedDataParallel) training.\nSynchronous BN has been used in cases where only a small\nlocal minibatch can fit on each GPU.\nAllreduced stats increase the effective batch size for the BN layer to the\nglobal batch size across all processes (which, technically, is the correct\nformulation).\nSynchronous BN has been observed to improve converged accuracy in some of our research models.\n\n# Requirements\n\nPython 3\n\nCUDA 9 or newer\n\nPyTorch 0.4 or newer. The CUDA and C++ extensions require pytorch 1.0 or newer.\n\nWe recommend the latest stable release, obtainable from\n[https://pytorch.org/](https://pytorch.org/). We also test against the latest master branch, obtainable from [https://github.com/pytorch/pytorch](https://github.com/pytorch/pytorch).\n\nIt's often convenient to use Apex in Docker containers. Compatible options include:\n* [NVIDIA Pytorch containers from NGC](https://ngc.nvidia.com/catalog/containers/nvidia%2Fpytorch), which come with Apex preinstalled. To use the latest Amp API, you may need to `pip uninstall apex` then reinstall Apex using the **Quick Start** commands below.\n* [official Pytorch -devel Dockerfiles](https://hub.docker.com/r/pytorch/pytorch/tags), e.g. `docker pull pytorch/pytorch:nightly-devel-cuda10.0-cudnn7`, in which you can install Apex using the **Quick Start** commands.\n\nSee the [Docker example folder](https://github.com/NVIDIA/apex/tree/master/examples/docker) for details.\n\n# Quick Start\n\n### Linux\n\nFor performance and full functionality, we recommend installing Apex with\nCUDA and C++ extensions via\n```\n$ git clone https://github.com/NVIDIA/apex\n$ cd apex\n$ pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" .\n```\n\nApex also supports a Python-only build (required with Pytorch 0.4) via\n```\n$ pip install -v --no-cache-dir .\n```\nA Python-only build omits:\n- Fused kernels required to use `apex.optimizers.FusedAdam`.\n- Fused kernels required to use `apex.normalization.FusedLayerNorm`.\n- Fused kernels that improve the performance and numerical stability of `apex.parallel.SyncBatchNorm`.\n- Fused kernels that improve the performance of `apex.parallel.DistributedDataParallel` and `apex.amp`.\n`DistributedDataParallel`, `amp`, and `SyncBatchNorm` will still be usable, but they may be slower.\n\n### Windows support\nWindows support is experimental, and Linux is recommended. `pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" .` may work if you were able to build Pytorch from source\non your system. `pip install -v --no-cache-dir .` (without CUDA/C++ extensions) is more likely to work. If you installed Pytorch in a Conda environment, make sure to install Apex in that same environment.\n" }, { "path": "apex/apex/RNN/README.md", "content": "Under construction...\n" }, { "path": "apex/apex/RNN/RNNBackend.py", "content": "import torch\nimport torch.nn as nn\nfrom torch.autograd import Variable\n\nimport torch.nn.functional as F\n\nimport math\n\n\ndef is_iterable(maybe_iterable):\n return isinstance(maybe_iterable, list) or isinstance(maybe_iterable, tuple)\n\n\ndef flatten_list(tens_list):\n \"\"\"\n flatten_list\n \"\"\"\n if not is_iterable(tens_list):\n return tens_list\n \n return torch.cat(tens_list, dim=0).view(len(tens_list), *tens_list[0].size() )\n\n \n#These modules always assumes batch_first\nclass bidirectionalRNN(nn.Module):\n \"\"\"\n bidirectionalRNN\n \"\"\"\n def __init__(self, inputRNN, num_layers=1, dropout = 0):\n super(bidirectionalRNN, self).__init__()\n self.dropout = dropout\n self.fwd = stackedRNN(inputRNN, num_layers=num_layers, dropout = dropout)\n self.bckwrd = stackedRNN(inputRNN.new_like(), num_layers=num_layers, dropout = dropout)\n self.rnns = nn.ModuleList([self.fwd, self.bckwrd])\n \n #collect hidden option will return all hidden/cell states from entire RNN\n def forward(self, input, collect_hidden=False):\n \"\"\"\n forward()\n \"\"\"\n seq_len = input.size(0)\n bsz = input.size(1)\n\n fwd_out, fwd_hiddens = list(self.fwd(input, collect_hidden = collect_hidden))\n bckwrd_out, bckwrd_hiddens = list(self.bckwrd(input, reverse=True, collect_hidden = collect_hidden))\n \n output = torch.cat( [fwd_out, bckwrd_out], -1 )\n hiddens = tuple( torch.cat(hidden, -1) for hidden in zip( fwd_hiddens, bckwrd_hiddens) )\n\n return output, hiddens\n\n def reset_parameters(self):\n \"\"\"\n reset_parameters()\n \"\"\"\n for rnn in self.rnns:\n rnn.reset_parameters()\n \n def init_hidden(self, bsz):\n \"\"\"\n init_hidden()\n \"\"\"\n for rnn in self.rnns:\n rnn.init_hidden(bsz)\n\n def detach_hidden(self):\n \"\"\"\n detach_hidden()\n \"\"\"\n for rnn in self.rnns:\n rnn.detachHidden()\n \n def reset_hidden(self, bsz):\n \"\"\"\n reset_hidden()\n \"\"\"\n for rnn in self.rnns:\n rnn.reset_hidden(bsz)\n\n def init_inference(self, bsz): \n \"\"\"\n init_inference()\n \"\"\"\n for rnn in self.rnns:\n rnn.init_inference(bsz)\n\n \n#assumes hidden_state[0] of inputRNN is output hidden state\n#constructor either takes an RNNCell or list of RNN layers\nclass stackedRNN(nn.Module): \n \"\"\"\n stackedRNN\n \"\"\"\n def __init__(self, inputRNN, num_layers=1, dropout=0):\n super(stackedRNN, self).__init__()\n \n self.dropout = dropout\n \n if isinstance(inputRNN, RNNCell):\n self.rnns = [inputRNN]\n for i in range(num_layers-1):\n self.rnns.append(inputRNN.new_like(inputRNN.output_size))\n elif isinstance(inputRNN, list):\n assert len(inputRNN) == num_layers, \"RNN list length must be equal to num_layers\"\n self.rnns=inputRNN\n else:\n raise RuntimeError()\n \n self.nLayers = len(self.rnns)\n \n self.rnns = nn.ModuleList(self.rnns)\n\n\n '''\n Returns output as hidden_state[0] Tensor([sequence steps][batch size][features])\n If collect hidden will also return Tuple(\n [n_hidden_states][sequence steps] Tensor([layer][batch size][features])\n )\n If not collect hidden will also return Tuple(\n [n_hidden_states] Tensor([layer][batch size][features])\n '''\n def forward(self, input, collect_hidden=False, reverse=False):\n \"\"\"\n forward()\n \"\"\"\n seq_len = input.size(0)\n bsz = input.size(1)\n inp_iter = reversed(range(seq_len)) if reverse else range(seq_len)\n\n hidden_states = [[] for i in range(self.nLayers)]\n outputs = []\n\n for seq in inp_iter:\n for layer in range(self.nLayers):\n\n if layer == 0:\n prev_out = input[seq]\n \n outs = self.rnns[layer](prev_out)\n\n if collect_hidden:\n hidden_states[layer].append(outs)\n elif seq == seq_len-1:\n hidden_states[layer].append(outs)\n \n prev_out = outs[0]\n\n outputs.append(prev_out)\n\n if reverse:\n outputs = list(reversed(outputs))\n '''\n At this point outputs is in format:\n list( [seq_length] x Tensor([bsz][features]) )\n need to convert it to:\n list( Tensor([seq_length][bsz][features]) )\n '''\n output = flatten_list(outputs)\n\n '''\n hidden_states at this point is in format:\n list( [layer][seq_length][hidden_states] x Tensor([bsz][features]) )\n need to convert it to:\n For not collect hidden:\n list( [hidden_states] x Tensor([layer][bsz][features]) )\n For collect hidden:\n list( [hidden_states][seq_length] x Tensor([layer][bsz][features]) )\n '''\n if not collect_hidden:\n seq_len = 1\n n_hid = self.rnns[0].n_hidden_states\n new_hidden = [ [ [ None for k in range(self.nLayers)] for j in range(seq_len) ] for i in range(n_hid) ]\n\n\n for i in range(n_hid):\n for j in range(seq_len):\n for k in range(self.nLayers):\n new_hidden[i][j][k] = hidden_states[k][j][i]\n\n hidden_states = new_hidden\n #Now in format list( [hidden_states][seq_length][layer] x Tensor([bsz][features]) )\n #Reverse seq_length if reverse\n if reverse:\n hidden_states = list( list(reversed(list(entry))) for entry in hidden_states)\n\n #flatten layer dimension into tensor\n hiddens = list( list(\n flatten_list(seq) for seq in hidden )\n for hidden in hidden_states )\n \n #Now in format list( [hidden_states][seq_length] x Tensor([layer][bsz][features]) )\n #Remove seq_length dimension if not collect_hidden\n if not collect_hidden:\n hidden_states = list( entry[0] for entry in hidden_states)\n return output, hidden_states\n \n def reset_parameters(self):\n \"\"\"\n reset_parameters()\n \"\"\"\n for rnn in self.rnns:\n rnn.reset_parameters()\n \n def init_hidden(self, bsz):\n \"\"\"\n init_hidden()\n \"\"\"\n for rnn in self.rnns:\n rnn.init_hidden(bsz)\n\n def detach_hidden(self):\n \"\"\"\n detach_hidden()\n \"\"\"\n for rnn in self.rnns:\n rnn.detach_hidden()\n \n def reset_hidden(self, bsz):\n \"\"\"\n reset_hidden()\n \"\"\"\n for rnn in self.rnns:\n rnn.reset_hidden(bsz)\n\n def init_inference(self, bsz): \n \"\"\" \n init_inference()\n \"\"\"\n for rnn in self.rnns:\n rnn.init_inference(bsz)\n\nclass RNNCell(nn.Module):\n \"\"\" \n RNNCell \n gate_multiplier is related to the architecture you're working with\n For LSTM-like it will be 4 and GRU-like will be 3.\n Always assumes input is NOT batch_first.\n Output size that's not hidden size will use output projection\n Hidden_states is number of hidden states that are needed for cell\n if one will go directly to cell as tensor, if more will go as list\n \"\"\"\n def __init__(self, gate_multiplier, input_size, hidden_size, cell, n_hidden_states = 2, bias = False, output_size = None):\n super(RNNCell, self).__init__()\n\n self.gate_multiplier = gate_multiplier\n self.input_size = input_size\n self.hidden_size = hidden_size\n self.cell = cell\n self.bias = bias\n self.output_size = output_size\n if output_size is None:\n self.output_size = hidden_size\n\n self.gate_size = gate_multiplier * self.hidden_size\n self.n_hidden_states = n_hidden_states\n\n self.w_ih = nn.Parameter(torch.Tensor(self.gate_size, self.input_size))\n self.w_hh = nn.Parameter(torch.Tensor(self.gate_size, self.output_size))\n\n #Check if there's recurrent projection\n if(self.output_size != self.hidden_size):\n self.w_ho = nn.Parameter(torch.Tensor(self.output_size, self.hidden_size))\n\n self.b_ih = self.b_hh = None\n if self.bias:\n self.b_ih = nn.Parameter(torch.Tensor(self.gate_size))\n self.b_hh = nn.Parameter(torch.Tensor(self.gate_size))\n \n #hidden states for forward\n self.hidden = [ None for states in range(self.n_hidden_states)]\n\n self.reset_parameters()\n\n def new_like(self, new_input_size=None):\n \"\"\"\n new_like()\n \"\"\"\n if new_input_size is None:\n new_input_size = self.input_size\n \n return type(self)(self.gate_multiplier,\n new_input_size,\n self.hidden_size,\n self.cell,\n self.n_hidden_states,\n self.bias,\n self.output_size)\n\n \n #Use xavier where we can (weights), otherwise use uniform (bias)\n def reset_parameters(self, gain=1):\n \"\"\"\n reset_parameters()\n \"\"\"\n stdev = 1.0 / math.sqrt(self.hidden_size)\n for param in self.parameters():\n param.data.uniform_(-stdev, stdev)\n '''\n Xavier reset:\n def reset_parameters(self, gain=1):\n stdv = 1.0 / math.sqrt(self.gate_size)\n\n for param in self.parameters():\n if (param.dim() > 1):\n torch.nn.init.xavier_normal(param, gain)\n else:\n param.data.uniform_(-stdv, stdv)\n '''\n def init_hidden(self, bsz):\n \"\"\"\n init_hidden()\n \"\"\"\n for param in self.parameters():\n if param is not None:\n a_param = param\n break\n\n for i, _ in enumerate(self.hidden):\n if(self.hidden[i] is None or self.hidden[i].data.size()[0] != bsz):\n\n if i==0:\n hidden_size = self.output_size\n else:\n hidden_size = self.hidden_size\n\n tens = a_param.data.new(bsz, hidden_size).zero_()\n self.hidden[i] = Variable(tens, requires_grad=False)\n \n \n def reset_hidden(self, bsz):\n \"\"\"\n reset_hidden()\n \"\"\"\n for i, _ in enumerate(self.hidden):\n self.hidden[i] = None\n self.init_hidden(bsz)\n\n def detach_hidden(self):\n \"\"\"\n detach_hidden()\n \"\"\"\n for i, _ in enumerate(self.hidden):\n if self.hidden[i] is None:\n raise RuntimeError(\"Must initialize hidden state before you can detach it\")\n for i, _ in enumerate(self.hidden):\n self.hidden[i] = self.hidden[i].detach()\n \n def forward(self, input):\n \"\"\"\n forward()\n if not inited or bsz has changed this will create hidden states\n \"\"\"\n self.init_hidden(input.size()[0])\n\n hidden_state = self.hidden[0] if self.n_hidden_states == 1 else self.hidden\n self.hidden = self.cell(input, hidden_state, self.w_ih, self.w_hh, b_ih=self.b_ih, b_hh=self.b_hh)\n if(self.n_hidden_states > 1):\n self.hidden = list(self.hidden)\n else:\n self.hidden=[self.hidden]\n\n if self.output_size != self.hidden_size:\n self.hidden[0] = F.linear(self.hidden[0], self.w_ho)\n\n return tuple(self.hidden)\n" }, { "path": "apex/apex/RNN/__init__.py", "content": "from .models import LSTM, GRU, ReLU, Tanh, mLSTM\n\n__all__ = ['models']\n" }, { "path": "apex/apex/RNN/cells.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom .RNNBackend import RNNCell\n\nfrom torch.nn._functions.thnn import rnnFusedPointwise as fusedBackend\n\nimport math \n\n\nclass mLSTMRNNCell(RNNCell):\n \"\"\"\n mLSTMRNNCell\n \"\"\"\n\n def __init__(self, input_size, hidden_size, bias = False, output_size = None):\n gate_multiplier = 4\n super(mLSTMRNNCell, self).__init__(gate_multiplier, input_size, hidden_size, mLSTMCell, n_hidden_states = 2, bias = bias, output_size = output_size)\n\n self.w_mih = nn.Parameter(torch.Tensor(self.output_size, self.input_size))\n self.w_mhh = nn.Parameter(torch.Tensor(self.output_size, self.output_size))\n\n self.reset_parameters()\n\n def forward(self, input):\n \"\"\"\n mLSTMRNNCell.forward()\n \"\"\"\n #if not inited or bsz has changed this will create hidden states\n self.init_hidden(input.size()[0])\n\n hidden_state = self.hidden[0] if self.n_hidden_states == 1 else self.hidden\n\n self.hidden = list(\n self.cell(input, hidden_state, self.w_ih, self.w_hh, self.w_mih, self.w_mhh,\n b_ih=self.b_ih, b_hh=self.b_hh)\n )\n \n if self.output_size != self.hidden_size:\n self.hidden[0] = F.linear(self.hidden[0], self.w_ho)\n return tuple(self.hidden)\n\n\n def new_like(self, new_input_size=None):\n if new_input_size is None:\n new_input_size = self.input_size\n \n return type(self)(\n new_input_size,\n self.hidden_size,\n self.bias,\n self.output_size)\n\ndef mLSTMCell(input, hidden, w_ih, w_hh, w_mih, w_mhh, b_ih=None, b_hh=None):\n \"\"\"\n mLSTMCell\n \"\"\"\n\n if input.is_cuda:\n igates = F.linear(input, w_ih)\n m = F.linear(input, w_mih) * F.linear(hidden[0], w_mhh)\n hgates = F.linear(m, w_hh)\n\n state = fusedBackend.LSTMFused.apply\n return state(igates, hgates, hidden[1], b_ih, b_hh)\n\n hx, cx = hidden\n \n m = F.linear(input, w_mih) * F.linear(hidden[0], w_mhh)\n gates = F.linear(input, w_ih, b_ih) + F.linear(m, w_hh, b_hh)\n\n ingate, forgetgate, cellgate, outgate = gates.chunk(4, 1)\n\n ingate = F.sigmoid(ingate)\n forgetgate = F.sigmoid(forgetgate)\n cellgate = F.tanh(cellgate)\n outgate = F.sigmoid(outgate)\n \n cy = (forgetgate * cx) + (ingate * cellgate)\n hy = outgate * F.tanh(cy)\n \n return hy, cy\n \n" }, { "path": "apex/apex/RNN/models.py", "content": "import torch\n\nfrom torch.nn._functions.rnn import LSTMCell, RNNReLUCell, RNNTanhCell, GRUCell\n\nfrom .RNNBackend import bidirectionalRNN, stackedRNN, RNNCell\nfrom .cells import mLSTMRNNCell, mLSTMCell\n\ndef toRNNBackend(inputRNN, num_layers, bidirectional=False, dropout = 0):\n \"\"\"\n :class:`toRNNBackend`\n \"\"\"\n\n if bidirectional:\n return bidirectionalRNN(inputRNN, num_layers, dropout = dropout)\n else:\n return stackedRNN(inputRNN, num_layers, dropout = dropout)\n\n\ndef LSTM(input_size, hidden_size, num_layers, bias=True, batch_first=False, dropout=0, bidirectional=False, output_size = None):\n \"\"\"\n :class:`LSTM`\n \"\"\"\n inputRNN = RNNCell(4, input_size, hidden_size, LSTMCell, 2, bias, output_size)\n return toRNNBackend(inputRNN, num_layers, bidirectional, dropout=dropout)\n\ndef GRU(input_size, hidden_size, num_layers, bias=True, batch_first=False, dropout=0, bidirectional=False, output_size = None):\n \"\"\"\n :class:`GRU`\n \"\"\"\n inputRNN = RNNCell(3, input_size, hidden_size, GRUCell, 1, bias, output_size)\n return toRNNBackend(inputRNN, num_layers, bidirectional, dropout=dropout)\n\ndef ReLU(input_size, hidden_size, num_layers, bias=True, batch_first=False, dropout=0, bidirectional=False, output_size = None):\n \"\"\"\n :class:`ReLU`\n \"\"\"\n inputRNN = RNNCell(1, input_size, hidden_size, RNNReLUCell, 1, bias, output_size)\n return toRNNBackend(inputRNN, num_layers, bidirectional, dropout=dropout)\n\ndef Tanh(input_size, hidden_size, num_layers, bias=True, batch_first=False, dropout=0, bidirectional=False, output_size = None):\n \"\"\"\n :class:`Tanh`\n \"\"\"\n inputRNN = RNNCell(1, input_size, hidden_size, RNNTanhCell, 1, bias, output_size)\n return toRNNBackend(inputRNN, num_layers, bidirectional, dropout=dropout)\n \ndef mLSTM(input_size, hidden_size, num_layers, bias=True, batch_first=False, dropout=0, bidirectional=False, output_size = None):\n \"\"\"\n :class:`mLSTM`\n \"\"\"\n inputRNN = mLSTMRNNCell(input_size, hidden_size, bias=bias, output_size=output_size)\n return toRNNBackend(inputRNN, num_layers, bidirectional, dropout=dropout)\n\n\n" }, { "path": "apex/apex/__init__.py", "content": "from . import parallel\nfrom . import amp\nfrom . import fp16_utils\n\n# For optimizers and normalization there is no Python fallback.\n# Absence of cuda backend is a hard error.\n# I would like the errors from importing fused_adam_cuda or fused_layer_norm_cuda\n# to be triggered lazily, because if someone has installed with --cpp_ext and --cuda_ext\n# so they expect those backends to be available, but for some reason they actually aren't\n# available (for example because they built improperly in a way that isn't revealed until\n# load time) the error message is timely and visible.\nfrom . import optimizers\nfrom . import normalization\n" }, { "path": "apex/apex/amp/README.md", "content": "# amp: Automatic Mixed Precision\n\n## Annotating User Functions\n\nNearly all PyTorch user code needs nothing more than the two steps\nabove to use amp. After all, custom layers are built out of simpler\nPyTorch components, and amp already can see those.\n\nHowever, any custom C++ or CUDA code is outside of amp's (default)\nview of things. For example, suppose I implemented a new recurrent\ncell called a \"forgetful recurrent unit\" that calls directly into a\nCUDA backend:\n\n```python\nfrom backend import FRUBackend\n\ndef fru(input, hidden, weight, bias):\n # call to CUDA code\n FRUBackend(input, hidden, weight, bias)\n```\n\nIn this case, it is possible to get a runtime type mismatch. For\nexample, you might have `input` in fp16, and `weight` in fp32, and amp\ndoesn't have the visibility to insert an appropriate cast.\n\namp exposes two ways to handle \"invisible\" backend code: function\nannotations and explicit registration.\n\n#### Function annotation\n\nThe first way to handle backend code is a set of function annotations:\n\n- `@amp.half_function`\n- `@amp.float_function`\n- `@amp.promote_function`\n\nThese correspond to:\n\n- Cast all arguments to fp16\n- Cast all argumnets fo fp32\n- If there are any type mismatches, cast everything to the widest type\n\nIn our example, we believe that the FRU unit is fp16-safe and will get\nperformance gains from casting its arguments to fp16, so we write:\n\n```python\n@amp.half_function\ndef fru(input, hidden, weight, bias):\n #...\n```\n\n#### Explicit registration\n\nThe other way to handle backend code is with explicit function\nregistration:\n\n- `amp.register_half_function(module, function_name)`\n- `amp.register_float_function(module, function_name)`\n- `amp.register_promote_function(module, function_name)`\n\nWhen using this API, `module` is the containing class or module for\nthe function, and `function_name` is the _string_ name of the\nfunction. Note that the function must be registered before the call to\n`amp.initalize()`.\n\nFor our FRU unit, we can register the backend function directly:\n\n```python\nimport backend\n\namp.register_half_function(backend, 'FRUBackend')\n```\n" }, { "path": "apex/apex/amp/__init__.py", "content": "from .amp import init, half_function, float_function, promote_function,\\\n register_half_function, register_float_function, register_promote_function\nfrom .handle import scale_loss, disable_casts\nfrom .frontend import initialize\nfrom ._amp_state import master_params, _amp_state\n" }, { "path": "apex/apex/amp/__version__.py", "content": "VERSION = (0, 1, 0)\n__version__ = '.'.join(map(str, VERSION))\n" }, { "path": "apex/apex/amp/_amp_state.py", "content": "# This is a \"header object\" that allows different amp modules to communicate.\n# I'm a C++ guy, not a python guy. I decided this approach because it seemed most C++-like. \n# But apparently it's ok:\n# http://effbot.org/pyfaq/how-do-i-share-global-variables-across-modules.htm\nimport os\nimport torch\n\nTORCH_MAJOR = int(torch.__version__.split('.')[0])\nTORCH_MINOR = int(torch.__version__.split('.')[1])\n\nif TORCH_MAJOR == 0:\n import collections.abc as container_abcs\nelse:\n from torch._six import container_abcs\n\n\nclass AmpState(object):\n def __init__(self):\n self.hard_override=False\n self.allow_incoming_model_not_fp32 = False\n self.verbosity=1\n\n\n# Attribute stash. Could also just stash things as global module attributes.\n_amp_state = AmpState()\n\n\ndef warn_or_err(msg):\n if _amp_state.hard_override:\n print(\"Warning: \" + msg)\n else:\n raise RuntimeError(msg)\n # I'm not sure if allowing hard_override is a good idea.\n # + \" If you're sure you know what you're doing, supply \" +\n # \"hard_override=True to amp.initialize.\")\n\n\ndistributed = False\nif 'WORLD_SIZE' in os.environ:\n distributed = int(os.environ['WORLD_SIZE']) > 1\n\n\ndef maybe_print(msg, rank0=False):\n if _amp_state.verbosity > 0:\n if rank0:\n if distributed:\n if torch.distributed.get_rank() == 0:\n print(msg)\n else:\n print(msg)\n else:\n print(msg)\n\n\n# def iter_params(param_groups):\n# for group in param_groups:\n# for p in group['params']:\n# yield p\n\n\ndef master_params(optimizer):\n \"\"\"\n Generator expression that iterates over the params owned by ``optimizer``.\n\n Args:\n optimizer: An optimizer previously returned from ``amp.initialize``.\n \"\"\"\n for group in optimizer.param_groups:\n for p in group['params']:\n yield p\n" }, { "path": "apex/apex/amp/_initialize.py", "content": "import torch\nfrom torch._six import string_classes\nimport functools\nimport numpy as np\nimport warnings\nfrom ._amp_state import _amp_state, warn_or_err, container_abcs\nfrom .handle import disable_casts\nfrom .scaler import LossScaler\nfrom ._process_optimizer import _process_optimizer\nfrom apex.fp16_utils import convert_network\nfrom ..fp16_utils import FP16_Optimizer as FP16_Optimizer_general\nfrom ..optimizers import FP16_Optimizer as FP16_Optimizer_for_fused\nfrom ..optimizers import FusedAdam\nfrom ..parallel import DistributedDataParallel as apex_DDP\nfrom ..parallel.LARC import LARC\n\n\ndef to_type(dtype, t):\n if isinstance(t, torch.Tensor):\n if not t.is_cuda:\n # This should not be a hard error, since it may be legitimate.\n warnings.warn(\"An input tensor was not cuda.\")\n # GANs require this.\n # if t.requires_grad:\n # warn_or_err(\"input data requires grad. Since input data is not a model parameter,\\n\"\n # \"its gradients will not be properly allreduced by DDP.\")\n if t.is_floating_point():\n return t.to(dtype)\n return t\n else:\n # Trust the user's custom batch type, that's all I can do here.\n return t.to(dtype)\n\n\n# Modified from torch.optim.optimizer.py. This is a bit more general than casted_args in utils.py.\ndef applier(value, fn):\n if isinstance(value, torch.Tensor):\n return fn(value)\n elif isinstance(value, string_classes):\n return value\n elif isinstance(value, np.ndarray):\n return value\n elif hasattr(value, \"to\"): # Allow handling of custom batch classes\n return fn(value)\n elif isinstance(value, container_abcs.Mapping):\n return {applier(k, fn) : applier(v, fn) for k, v in value.items()}\n elif isinstance(value, container_abcs.Iterable):\n return type(value)(applier(v, fn) for v in value)\n else:\n # Do I want this to fire off even if someone chooses to pass something ordinary like\n # an int or float? May be more annoying than it's worth.\n # print(\"Warning: unrecognized type in applier. If your input data is a custom class, \"\n # \"provide it with a .to(dtype) method which converts its floating-point Tensors to dtype. \"\n # \"Amp will check for your custom to() and invoke it to cast the batch's \"\n # \"floating-point Tensors to the appropriate type. \"\n # \"Also, if your data is a custom class, it is your responsibility to ensure that \"\n # \"any Tensors you want to be cuda are already cuda.\"\n return value\n\n\ndef check_models(models):\n for model in models:\n parallel_type = None\n if isinstance(model, torch.nn.parallel.DistributedDataParallel):\n parallel_type = \"torch.nn.parallel.DistributedDataParallel\"\n if isinstance(model, apex_DDP):\n parallel_type = \"apex.parallel.DistributedDataParallel\"\n if isinstance(model, torch.nn.parallel.DataParallel):\n parallel_type = \"torch.nn.parallel.DataParallel\"\n if parallel_type is not None:\n raise RuntimeError(\"Incoming model is an instance of {}. \".format(parallel_type) +\n \"Parallel wrappers should only be applied to the model(s) AFTER \\n\"\n \"the model(s) have been returned from amp.initialize.\")\n\n\ndef check_params_fp32(models):\n for model in models:\n for name, param in model.named_parameters():\n if param.is_floating_point():\n if 'Half' in param.type():\n warn_or_err(\"Found param {} with type {}, expected torch.cuda.FloatTensor.\\n\"\n \"When using amp.initialize, you do not need to call .half() on your model\\n\"\n \"before passing it, no matter what optimization level you choose.\".format(\n name, param.type()))\n elif not param.is_cuda:\n warn_or_err(\"Found param {} with type {}, expected torch.cuda.FloatTensor.\\n\"\n \"When using amp.initialize, you need to provide a model with parameters\\n\"\n \"located on a CUDA device before passing it no matter what optimization level\\n\"\n \"you chose. Use model.to('cuda') to use the default device.\".format(\n name, param.type()))\n\n # Backward compatibility for PyTorch 0.4\n if hasattr(model, 'named_buffers'):\n buf_iter = model.named_buffers()\n else:\n buf_iter = model._buffers\n for obj in buf_iter:\n if type(obj)==tuple:\n name, buf = obj\n else:\n name, buf = obj, buf_iter[obj]\n if buf.is_floating_point():\n if 'Half' in buf.type():\n warn_or_err(\"Found buffer {} with type {}, expected torch.cuda.FloatTensor.\\n\"\n \"When using amp.initialize, you do not need to call .half() on your model\\n\"\n \"before passing it, no matter what optimization level you choose.\".format(\n name, buf.type()))\n elif not buf.is_cuda:\n warn_or_err(\"Found buffer {} with type {}, expected torch.cuda.FloatTensor.\\n\"\n \"When using amp.initialize, you need to provide a model with buffers\\n\"\n \"located on a CUDA device before passing it no matter what optimization level\\n\"\n \"you chose. Use model.to('cuda') to use the default device.\".format(\n name, buf.type()))\n\n\ndef check_optimizers(optimizers):\n for optim in optimizers:\n bad_optim_type = None\n if isinstance(optim, FP16_Optimizer_general):\n bad_optim_type = \"apex.fp16_utils.FP16_Optimizer\"\n if isinstance(optim, FP16_Optimizer_for_fused):\n bad_optim_type = \"apex.optimizers.FP16_Optimizer\"\n if bad_optim_type is not None:\n raise RuntimeError(\"An incoming optimizer is an instance of {}. \".format(bad_optim_type) +\n \"The optimizer(s) passed to amp.initialize() must be bare \\n\"\n \"instances of either ordinary Pytorch optimizers, or Apex fused \\n\"\n \"optimizers (currently just FusedAdam, but FusedSGD will be added \\n\"\n \"soon). You should not manually wrap your optimizer in either \\n\"\n \"apex.fp16_utils.FP16_Optimizer or apex.optimizers.FP16_Optimizer. \\n\"\n \"amp.initialize will take care of that for you (if necessary) based \\n\"\n \"on the specified opt_level (and optional overridden properties).\")\n\n\ndef wrap_fused_adam(optimizer, properties):\n msg = 'Currently, the usage of FusedAdam is restricted to '\\\n 'amp.initialize(..., opt_level=\"O2\", keep_batchnorm_fp32=False, '\\\n 'loss_scale=float or \"dynamic\"). We are working on enabling more general usage.'\n\n assert properties.master_weights is True, msg\n assert properties.cast_model_type is torch.float16, msg\n assert (properties.keep_batchnorm_fp32 is False or\n properties.keep_batchnorm_fp32 is None), msg\n\n if properties.loss_scale == \"dynamic\":\n return FP16_Optimizer_for_fused(optimizer, dynamic_loss_scale=True)\n else:\n return FP16_Optimizer_for_fused(optimizer, static_loss_scale=properties.loss_scale)\n\n\ndef _initialize(models, optimizers, properties, num_losses=1, cast_model_outputs=None):\n from apex.parallel import DistributedDataParallel as apex_DDP\n from .amp import init as amp_init\n\n optimizers_was_list = False\n if isinstance(optimizers, torch.optim.Optimizer) or isinstance(optimizers, LARC):\n optimizers = [optimizers]\n elif optimizers is None:\n optimizers = []\n elif isinstance(optimizers, list):\n optimizers_was_list = True\n check_optimizers(optimizers)\n else:\n check_optimizers([optimizers])\n raise TypeError(\"optimizers must be either a single optimizer or a list of optimizers.\")\n\n if isinstance(models, torch.nn.Module):\n models_was_list = False\n models = [models]\n elif isinstance(models, list):\n models_was_list = True\n else:\n raise TypeError(\"models must be either a single model or a list of models.\")\n\n check_models(models)\n\n if not _amp_state.allow_incoming_model_not_fp32:\n check_params_fp32(models)\n\n\n # In the future, when FP16_Optimizer can be deprecated and master weights can\n # become an attribute, remember to stash master weights before casting the model.\n\n if properties.cast_model_type:\n if properties.keep_batchnorm_fp32:\n for model in models:\n convert_network(model, properties.cast_model_type)\n else:\n for model in models:\n model.to(properties.cast_model_type)\n\n input_caster = functools.partial(to_type, properties.cast_model_type)\n if cast_model_outputs is not None:\n output_caster = functools.partial(to_type, cast_model_outputs)\n else:\n output_caster = functools.partial(to_type, torch.float32)\n\n for model in models:\n # Patch the forward method to cast incoming data to the correct type, and\n # outgoing data to float32, so \"the user never needs to call .half().\"\n # I like writing things explicitly more than decorators.\n def patch_forward(old_fwd):\n def new_fwd(*args, **kwargs):\n output = old_fwd(*applier(args, input_caster),\n **applier(kwargs, input_caster))\n return applier(output, output_caster)\n return new_fwd\n\n model.forward = patch_forward(model.forward)\n\n # State dict trick to recast any preexisting per-param state tensors \n for optimizer in optimizers:\n optimizer.load_state_dict(optimizer.state_dict())\n elif cast_model_outputs is not None:\n output_caster = functools.partial(to_type, cast_model_outputs)\n\n for model in models:\n def patch_forward(old_fwd):\n def new_fwd(*args, **kwargs):\n output = old_fwd(*args, **kwargs)\n return applier(output, output_caster)\n return new_fwd\n\n model.forward = patch_forward(model.forward)\n\n for i, optimizer in enumerate(optimizers):\n # Still need to special case this for the first pass\n if isinstance(optimizer, FusedAdam):\n optimizers[i] = wrap_fused_adam(optimizer, properties)\n else:\n optimizers[i] = _process_optimizer(optimizer, properties)\n\n _amp_state.loss_scalers = []\n for _ in range(num_losses):\n _amp_state.loss_scalers.append(LossScaler(properties.loss_scale,\n min_loss_scale=_amp_state.min_loss_scale,\n max_loss_scale=_amp_state.max_loss_scale))\n\n if properties.patch_torch_functions:\n # handle is unused here. It's accessible later through a global value anyway.\n handle = amp_init(loss_scale=properties.loss_scale, verbose=(_amp_state.verbosity == 2))\n for optimizer in optimizers:\n # Disable Amp casting for the optimizer step, because it should only be\n # applied to FP32 master params anyway.\n def patch_step(old_step):\n def new_step(*args, **kwargs):\n with disable_casts():\n output = old_step(*args, **kwargs)\n return output\n return new_step\n\n optimizer.step = patch_step(optimizer.step)\n\n if optimizers_was_list:\n if models_was_list:\n return models, optimizers\n else:\n return models[0], optimizers\n else:\n if models_was_list:\n if len(optimizers) == 0:\n return models\n else:\n return models, optimizers[0]\n else:\n if len(optimizers) == 0:\n return models[0]\n else:\n return models[0], optimizers[0]\n" }, { "path": "apex/apex/amp/_process_optimizer.py", "content": "import types\nfrom ..fp16_utils import master_params_to_model_params\nfrom ..multi_tensor_apply import multi_tensor_applier\nfrom ._amp_state import maybe_print\nimport torch\n\n\nclass AmpOptimizerState(object):\n def __init__(self):\n pass\n\n\ndef lazy_init_with_master_weights(self):\n stash = self._amp_stash\n stash.fp16_groups = []\n stash.fp32_from_fp16_groups = []\n stash.fp32_from_fp32_groups = []\n for i, param_group in enumerate(self.param_groups):\n # maybe_print(\"FP16_Optimizer processing param group {}:\".format(i))\n fp16_params_this_group = []\n fp32_params_this_group = []\n fp32_from_fp16_params_this_group = []\n for i, param in enumerate(param_group['params']):\n if param.requires_grad:\n if param.type() == 'torch.cuda.HalfTensor':\n # maybe_print(\"FP16_Optimizer received torch.cuda.HalfTensor with {}\"\n # .format(param.size()))\n fp16_params_this_group.append(param)\n master_param = param.detach().clone().float()\n master_param.requires_grad = True\n param_group['params'][i] = master_param\n fp32_from_fp16_params_this_group.append(master_param)\n # Reset existing state dict key to the new master param.\n # We still need to recast per-param state tensors, if any, to FP32.\n if param in self.state:\n self.state[master_param] = self.state.pop(param)\n elif param.type() == 'torch.cuda.FloatTensor':\n # maybe_print(\"FP16_Optimizer received torch.cuda.FloatTensor with {}\"\n # .format(param.size()))\n fp32_params_this_group.append(param)\n param_group['params'][i] = param\n else:\n raise TypeError(\"Optimizer's parameters must be either \"\n \"torch.cuda.FloatTensor or torch.cuda.HalfTensor. \"\n \"Received {}\".format(param.type()))\n\n stash.fp16_groups.append(fp16_params_this_group)\n stash.fp32_from_fp16_groups.append(fp32_from_fp16_params_this_group)\n stash.fp32_from_fp32_groups.append(fp32_params_this_group)\n\n stash.all_fp16_params = []\n for group in stash.fp16_groups:\n stash.all_fp16_params += group\n\n stash.all_fp32_from_fp16_params = []\n for group in stash.fp32_from_fp16_groups:\n stash.all_fp32_from_fp16_params += group\n\n stash.all_fp32_from_fp32_params = []\n for group in stash.fp32_from_fp32_groups:\n stash.all_fp32_from_fp32_params += group\n\n # stash.all_fp32_from_fp16_grad_stash = [None for _ in stash.all_fp32_from_fp16_params]\n stash.all_fp32_from_fp32_grad_stash = [None for _ in stash.all_fp32_from_fp32_params]\n\n for param in stash.all_fp32_from_fp16_params:\n param.grad = None\n\n for param in stash.all_fp32_from_fp32_params:\n param.grad = None\n\n # Leverage state_dict() and load_state_dict() to recast preexisting per-param state tensors\n self.load_state_dict(self.state_dict())\n\n\ndef prepare_backward_with_master_weights(self):\n stash = self._amp_stash\n\n if not stash.lazy_init_called:\n self._lazy_init_maybe_master_weights()\n stash.lazy_init_called = True\n\n for i, param in enumerate(stash.all_fp16_params):\n # Set up to leverage grad copy elision:\n param.grad = None\n\n # for i, param in enumerate(stash.all_fp32_from_fp16_params):\n # stash.all_fp32_from_fp16_grad_stash[i] = param.grad\n\n for i, param in enumerate(stash.all_fp32_from_fp32_params):\n stash.all_fp32_from_fp32_grad_stash[i] = param.grad\n # Set up to leverage grad copy elision:\n param.grad = None\n\n\ndef post_backward_with_master_weights(self, scaler):\n stash = self._amp_stash\n\n # This is a lot of python overhead...\n fp16_grads_needing_unscale = []\n new_fp32_grads = []\n fp16_grads_needing_unscale_with_stash = []\n preexisting_fp32_grads = []\n for fp16_param, fp32_param in zip(stash.all_fp16_params,\n stash.all_fp32_from_fp16_params):\n if fp16_param.grad is None and fp32_param.grad is not None:\n continue\n elif fp16_param.grad is not None and fp32_param.grad is None:\n fp32_param.grad = torch.empty_like(fp32_param)\n fp16_grads_needing_unscale.append(fp16_param.grad)\n new_fp32_grads.append(fp32_param.grad)\n elif fp16_param.grad is not None and fp32_param.grad is not None:\n fp16_grads_needing_unscale_with_stash.append(fp16_param.grad)\n preexisting_fp32_grads.append(fp32_param.grad)\n else: # fp16_param.grad is None and fp32_param.grad is None:\n continue\n\n if len(fp16_grads_needing_unscale) > 0:\n scaler.unscale(\n fp16_grads_needing_unscale,\n new_fp32_grads,\n scaler.loss_scale(),\n models_are_masters=False)\n\n if len(fp16_grads_needing_unscale_with_stash) > 0:\n scaler.unscale_with_stashed(\n fp16_grads_needing_unscale_with_stash,\n preexisting_fp32_grads,\n preexisting_fp32_grads)\n\n # fp32 params can be treated as they would be in the \"no_master_weights\" case.\n grads_needing_unscale = []\n grads_needing_unscale_with_stash = []\n stashed = []\n for param, stashed_grad in zip(stash.all_fp32_from_fp32_params,\n stash.all_fp32_from_fp32_grad_stash):\n if param.grad is None and stashed_grad is not None:\n param.grad = stashed_grad\n elif param.grad is not None and stashed_grad is None:\n grads_needing_unscale.append(param.grad)\n elif param.grad is not None and stashed_grad is not None:\n grads_needing_unscale_with_stash.append(param.grad)\n stashed.append(stashed_grad)\n else: # param.grad is None and stashed_grad is None:\n continue\n\n if len(grads_needing_unscale) > 0:\n scaler.unscale(\n grads_needing_unscale,\n grads_needing_unscale,\n scaler.loss_scale(),\n models_are_masters=True)\n\n if len(grads_needing_unscale_with_stash) > 0:\n scaler.unscale_with_stashed(\n grads_needing_unscale_with_stash,\n stashed,\n grads_needing_unscale_with_stash)\n\n # Clear the stash.\n for i in range(len(stash.all_fp32_from_fp32_grad_stash)):\n stash.all_fp32_from_fp32_grad_stash[i] = None\n\n\ndef lazy_init_no_master_weights(self):\n stash = self._amp_stash\n stash.all_fp16_params = []\n stash.all_fp32_params = []\n for i, param_group in enumerate(self.param_groups):\n for i, param in enumerate(param_group['params']):\n if param.type() == 'torch.cuda.HalfTensor':\n stash.all_fp16_params.append(param)\n elif param.type() == 'torch.cuda.FloatTensor':\n stash.all_fp32_params.append(param)\n else:\n raise TypeError(\"Optimizer's parameters must be either \"\n \"torch.cuda.FloatTensor or torch.cuda.HalfTensor. \"\n \"Received {}\".format(param.type()))\n\n stash.all_fp16_grad_stash = [None for _ in stash.all_fp16_params]\n stash.all_fp32_grad_stash = [None for _ in stash.all_fp32_params]\n\n\ndef prepare_backward_no_master_weights(self):\n stash = self._amp_stash\n\n if not stash.lazy_init_called:\n self._lazy_init_maybe_master_weights()\n stash.lazy_init_called = True\n\n for i, param in enumerate(stash.all_fp16_params):\n stash.all_fp16_grad_stash[i] = param.grad\n # Set up to leverage grad copy elision:\n param.grad = None\n\n for i, param in enumerate(stash.all_fp32_params):\n stash.all_fp32_grad_stash[i] = param.grad\n # Set up to leverage grad copy elision:\n param.grad = None\n\n\ndef post_backward_no_master_weights(self, scaler):\n stash = self._amp_stash\n\n split_types = ((stash.all_fp16_params, stash.all_fp16_grad_stash),\n (stash.all_fp32_params, stash.all_fp32_grad_stash))\n\n for params, stashed_grads in split_types:\n # This is a lot of python overhead...\n grads_needing_unscale = []\n grads_needing_unscale_with_stash = []\n stashed = []\n for param, stashed_grad in zip(params, stashed_grads):\n if param.grad is None and stashed_grad is not None:\n param.grad = stashed_grad\n elif param.grad is not None and stashed_grad is None:\n grads_needing_unscale.append(param.grad)\n elif param.grad is not None and stashed_grad is not None:\n grads_needing_unscale_with_stash.append(param.grad)\n stashed.append(stashed_grad)\n else: # param.grad is None and stashed_grad is None\n continue\n\n if len(grads_needing_unscale) > 0:\n scaler.unscale(\n grads_needing_unscale,\n grads_needing_unscale,\n scaler.loss_scale(),\n models_are_masters=True)\n\n if len(grads_needing_unscale_with_stash) > 0:\n scaler.unscale_with_stashed(\n grads_needing_unscale_with_stash,\n stashed,\n grads_needing_unscale_with_stash)\n\n # Clear the stash.\n for i in range(len(stashed_grads)):\n stashed_grads[i] = None\n\n\ndef _master_params_to_model_params(self):\n stash = self._amp_stash\n if multi_tensor_applier.available:\n if len(stash.all_fp16_params) > 0:\n multi_tensor_applier(\n stash.multi_tensor_scale,\n stash.dummy_overflow_buf,\n [stash.all_fp32_from_fp16_params, stash.all_fp16_params],\n 1.0)\n else:\n for fp16_group, fp32_from_fp16_group in zip(stash.fp16_groups, stash.fp32_from_fp16_groups):\n master_params_to_model_params(fp16_group, fp32_from_fp16_group)\n\n\ndef _process_optimizer(optimizer, properties):\n if hasattr(optimizer, \"_amp_stash\"):\n raise RuntimeError(\"A given optimizer should only be passed through amp.initialize once.\")\n else:\n optimizer._amp_stash = AmpOptimizerState()\n\n optimizer._amp_stash.lazy_init_called = False\n optimizer._amp_stash.already_patched = False\n optimizer._amp_stash.params_have_scaled_gradients = False\n\n for name in (\"_lazy_init_maybe_master_weights\",\n \"_master_params_to_model_params\",\n \"_prepare_amp_backward\",\n \"_post_amp_backward\"):\n if hasattr(optimizer, name):\n raise RuntimeError(\"Incoming optimizer already has {} defined.\".format(name))\n\n # TODO: Centralize exposure and import error checking for the C backend.\n if multi_tensor_applier.available:\n import amp_C\n optimizer._amp_stash.multi_tensor_scale = amp_C.multi_tensor_scale\n optimizer._amp_stash.dummy_overflow_buf = torch.cuda.IntTensor([0]);\n\n if properties.master_weights:\n optimizer._lazy_init_maybe_master_weights = types.MethodType(\n lazy_init_with_master_weights, optimizer)\n\n optimizer._master_params_to_model_params = types.MethodType(\n _master_params_to_model_params, optimizer)\n\n old_step = optimizer.step\n def new_step(self, closure=None):\n if closure is not None:\n raise RuntimeError(\"Currently, Amp does not support closure use with optimizers.\")\n retval = old_step()\n self._master_params_to_model_params()\n # Clear the master grads that wouldn't be zeroed by model.zero_grad()\n for param in self._amp_stash.all_fp32_from_fp16_params:\n param.grad = None\n return retval\n optimizer.step = types.MethodType(new_step, optimizer)\n\n old_zero_grad = optimizer.zero_grad\n def new_zero_grad(self):\n stash = self._amp_stash\n if not stash.lazy_init_called:\n self._lazy_init_maybe_master_weights()\n stash.lazy_init_called = True\n # Zero the model grads.\n for param in stash.all_fp16_params:\n if param.grad is not None:\n param.grad.detach_()\n param.grad.zero_()\n for param in stash.all_fp32_from_fp32_params:\n if param.grad is not None:\n param.grad.detach_()\n param.grad.zero_()\n # Clear the master grads that are independent of model grads\n for param in self._amp_stash.all_fp32_from_fp16_params:\n param.grad = None\n optimizer.zero_grad = types.MethodType(new_zero_grad, optimizer)\n\n optimizer._prepare_amp_backward = types.MethodType(\n prepare_backward_with_master_weights, optimizer)\n\n optimizer._post_amp_backward = types.MethodType(\n post_backward_with_master_weights, optimizer)\n else:\n optimizer._lazy_init_maybe_master_weights = types.MethodType(\n lazy_init_no_master_weights, optimizer)\n\n optimizer._prepare_amp_backward = types.MethodType(\n prepare_backward_no_master_weights, optimizer)\n\n optimizer._post_amp_backward = types.MethodType(\n post_backward_no_master_weights, optimizer)\n\n old_add_param_group = optimizer.add_param_group\n\n def new_add_param_group(self, new_group):\n stash = self._amp_stash\n\n if not stash.lazy_init_called:\n self._lazy_init_maybe_master_weights()\n stash.lazy_init_called = True\n\n assert isinstance(new_group, dict), \"param group must be a dict\"\n\n new_params = new_group['params']\n if isinstance(new_params, torch.Tensor):\n new_group['params'] = [new_params]\n elif isinstance(new_params, set):\n raise TypeError('optimizer parameters need to be organized in ordered collections, but '\n 'the ordering of tensors in sets will change between runs. Please use a list instead.')\n else:\n new_group['params'] = list(new_params)\n\n if properties.master_weights:\n # Mutate new_group in-place to use FP32 master params\n fp16_params_this_group = []\n fp32_params_this_group = []\n fp32_from_fp16_params_this_group = []\n for i, param in enumerate(new_group['params']):\n if param.requires_grad:\n if param.type() == 'torch.cuda.HalfTensor':\n fp16_params_this_group.append(param)\n master_param = param.detach().clone().float()\n master_param.requires_grad = True\n new_group['params'][i] = master_param\n fp32_from_fp16_params_this_group.append(master_param)\n elif param.type() == 'torch.cuda.FloatTensor':\n fp32_params_this_group.append(param)\n new_group['params'][i] = param\n else:\n raise TypeError(\"Optimizer's parameters must be either \"\n \"torch.cuda.FloatTensor or torch.cuda.HalfTensor. \"\n \"Received {}\".format(param.type()))\n\n stash.fp16_groups.append(fp16_params_this_group)\n stash.fp32_from_fp16_groups.append(fp32_from_fp16_params_this_group)\n stash.fp32_from_fp32_groups.append(fp32_params_this_group)\n\n stash.all_fp16_params += fp16_params_this_group\n stash.all_fp32_from_fp16_params += fp32_from_fp16_params_this_group\n stash.all_fp32_from_fp32_params += fp32_params_this_group\n\n # stash.all_fp32_from_fp16_grad_stash = [None for _ in stash.all_fp32_from_fp16_params]\n stash.all_fp32_from_fp32_grad_stash += [None for _ in fp32_params_this_group]\n\n # It should be ok to let params be added with existing .grad attributes.\n # for param in fp16_params_this_group:\n # param.grad = None\n\n # for param in fp32_from_fp16_params_this_group:\n # param.grad = None\n\n # for param in stash.fp32_params_this_group:\n # param.grad = None\n else:\n for param in new_group['params']:\n if param.type() == 'torch.cuda.HalfTensor':\n stash.all_fp16_params.append(param)\n stash.all_fp16_grad_stash.append(None)\n elif param.type() == 'torch.cuda.FloatTensor':\n stash.all_fp32_params.append(param)\n stash.all_fp32_grad_stash.append(None)\n else:\n raise TypeError(\"Optimizer's parameters must be either \"\n \"torch.cuda.FloatTensor or torch.cuda.HalfTensor. \"\n \"Received {}\".format(param.type()))\n\n old_add_param_group(new_group)\n\n optimizer.add_param_group = types.MethodType(new_add_param_group, optimizer)\n\n return optimizer\n" }, { "path": "apex/apex/amp/amp.py", "content": "from . import compat, rnn_compat, utils, wrap\nfrom .handle import AmpHandle, NoOpHandle\nfrom .lists import functional_overrides, torch_overrides, tensor_overrides\nfrom ._amp_state import _amp_state\nfrom .frontend import *\n\nimport functools\nimport itertools\n\nimport torch\n\n\n_DECORATOR_HANDLE = None\n_USER_CAST_REGISTRY = set()\n_USER_PROMOTE_REGISTRY = set()\n\n\ndef _decorator_helper(orig_fn, cast_fn, wrap_fn):\n def wrapper(*args, **kwargs):\n handle = _DECORATOR_HANDLE\n if handle is None or not handle.is_active():\n return orig_fn(*args, **kwargs)\n inner_cast_fn = utils.verbosify(cast_fn, orig_fn.__name__,\n handle.verbose)\n return wrap_fn(orig_fn, inner_cast_fn, handle)(*args, **kwargs)\n return wrapper\n\n\n# Decorator form\ndef half_function(fn):\n wrap_fn = functools.partial(wrap.make_cast_wrapper, try_caching=True)\n return _decorator_helper(fn, utils.maybe_half, wrap_fn)\n\n\ndef float_function(fn):\n wrap_fn = functools.partial(wrap.make_cast_wrapper, try_caching=False)\n return _decorator_helper(fn, utils.maybe_float, wrap_fn)\n\n\ndef promote_function(fn):\n wrap_fn = functools.partial(wrap.make_promote_wrapper)\n return _decorator_helper(fn, utils.maybe_float, wrap_fn)\n\n\n# Registry form\ndef register_half_function(module, name):\n if not hasattr(module, name):\n raise ValueError('No function named {} in module {}.'.format(\n name, module))\n _USER_CAST_REGISTRY.add((module, name, utils.maybe_half))\n\n\ndef register_float_function(module, name):\n if not hasattr(module, name):\n raise ValueError('No function named {} in module {}.'.format(\n name, module))\n _USER_CAST_REGISTRY.add((module, name, utils.maybe_float))\n\n\ndef register_promote_function(module, name):\n if not hasattr(module, name):\n raise ValueError('No function named {} in module {}.'.format(\n name, module))\n _USER_PROMOTE_REGISTRY.add((module, name))\n\n\n# Top-level function to insert _all_ the hooks.\ndef init(enabled=True, loss_scale=\"dynamic\", enable_caching=True, verbose=False, allow_banned=False):\n global _DECORATOR_HANDLE\n\n if not enabled:\n handle = NoOpHandle()\n _DECORATOR_HANDLE = handle\n return handle\n\n handle = AmpHandle(loss_scale, enable_caching, verbose)\n\n # 0) Force-{fp16, fp32} for user-annotated functions\n for mod, fn, cast_fn in _USER_CAST_REGISTRY:\n try_caching = (cast_fn == utils.maybe_half)\n wrap.cached_cast(mod, fn, cast_fn, handle,\n try_caching, verbose)\n _USER_CAST_REGISTRY.clear()\n\n # 0.5) Force-promote for user-annotated functions\n for mod, fn in _USER_PROMOTE_REGISTRY:\n wrap.promote(mod, fn, handle, verbose)\n _USER_PROMOTE_REGISTRY.clear()\n\n # 1) Force-{fp16, fp32} on white- / black-list functions\n override_modules = [functional_overrides,\n torch_overrides,\n tensor_overrides]\n cast_table = [('FP16_FUNCS', utils.maybe_half),\n ('FP32_FUNCS', utils.maybe_float)]\n for module, (list_name, cast_fn) in itertools.product(override_modules,\n cast_table):\n for fn in getattr(module, list_name):\n try_caching = (cast_fn == utils.maybe_half)\n wrap.cached_cast(module.MODULE, fn, cast_fn, handle,\n try_caching, verbose)\n\n # 1.5) Pre-0.4, put the blacklist methods on HalfTensor and whitelist\n # methods on FloatTensor, since they're distinct types.\n if compat.tensor_is_float_tensor():\n for fn in tensor_overrides.FP16_FUNCS:\n wrap.cached_cast(torch.cuda.FloatTensor, fn, utils.maybe_half,\n handle, try_caching=True, verbose=verbose)\n for fn in tensor_overrides.FP32_FUNCS:\n wrap.cached_cast(torch.cuda.HalfTensor, fn, utils.maybe_float,\n handle, try_caching=False, verbose=verbose)\n\n # 2) Enable type-promotion on multi-arg functions and methods.\n # NB: special handling for sequence fns (e.g. `torch.cat`).\n promote_modules = [torch_overrides, tensor_overrides]\n promote_table = [('CASTS', wrap.promote),\n ('SEQUENCE_CASTS', wrap.sequence_promote)]\n for promote_mod, (list_name, promote_fn) in itertools.product(promote_modules,\n promote_table):\n for fn in getattr(promote_mod, list_name):\n promote_fn(promote_mod.MODULE, fn, handle, verbose)\n\n # 2.5) Pre-0.4, add blacklist methods directly to HalfTensor and FloatTensor types\n if compat.tensor_is_float_tensor():\n for cls, (list_name, promote_fn) in itertools.product([torch.cuda.FloatTensor,\n torch.cuda.HalfTensor],\n promote_table):\n for fn in getattr(tensor_overrides, list_name):\n promote_fn(cls, fn, handle, verbose)\n\n # 3) For any in-place version of a blacklist function, error if any input is fp16.\n # NB: this is overly conservative.\n for fn in utils.as_inplace(torch_overrides.FP32_FUNCS):\n wrap.err_if_any_half(torch_overrides.MODULE, fn, handle)\n\n # 3.5) For any in-place blacklist method, error if called on fp16 tensor\n for fn in utils.as_inplace(tensor_overrides.FP32_FUNCS):\n wrap.err_if_arg0_half(tensor_overrides.MODULE, fn, handle, verbose)\n if compat.tensor_is_float_tensor():\n wrap.err_if_arg0_half(torch.cuda.HalfTensor, fn, handle, verbose)\n\n # 4) For other in-place methods, match the type of self tensor\n for fn in utils.as_inplace(itertools.chain(\n tensor_overrides.FP16_FUNCS,\n tensor_overrides.CASTS)):\n wrap.promote_match_arg0(tensor_overrides.MODULE, fn, handle, verbose)\n if compat.tensor_is_float_tensor():\n wrap.promote_match_arg0(torch.cuda.HalfTensor, fn, handle, verbose)\n wrap.promote_match_arg0(torch.cuda.FloatTensor, fn, handle, verbose)\n\n # 5) RNNs + RNN cells are whitelisted specially\n if rnn_compat.has_old_rnns():\n wrap.rnn_cast(torch.nn.backends.thnn.backend, 'RNN', handle, verbose)\n if not rnn_compat.has_old_rnns():\n # Patch in our own indirection of `_VF` in modules/rnn s.t. it is mutable.\n torch.nn.modules.rnn._VF = rnn_compat.VariableFunctionsShim()\n # Wrap all the rnns\n for x in rnn_compat.RNN_NAMES:\n wrap.new_rnn_cast(x.upper(), handle, verbose)\n\n # Wrap all the RNN cells\n rnn_compat.whitelist_rnn_cells(handle, verbose)\n\n # 6) Place error+print message on banned functions.\n # Or, if allow_banned, then cast to FP32.\n for fn, err_msg in functional_overrides.BANNED_FUNCS:\n if allow_banned:\n wrap.cached_cast(functional_overrides.MODULE, fn, utils.maybe_float,\n handle, try_caching=True, verbose=verbose)\n else:\n wrap.err_if_any_half(functional_overrides.MODULE, fn, handle, err_msg)\n\n _DECORATOR_HANDLE = handle\n\n _amp_state.handle = handle\n\n return handle\n" }, { "path": "apex/apex/amp/compat.py", "content": "import torch\n\n# True for post-0.4, when Variables/Tensors merged.\ndef variable_is_tensor():\n v = torch.autograd.Variable()\n return isinstance(v, torch.Tensor)\n\ndef tensor_is_variable():\n x = torch.Tensor()\n return type(x) == torch.autograd.Variable\n\n# False for post-0.4\ndef tensor_is_float_tensor():\n x = torch.Tensor()\n return type(x) == torch.FloatTensor\n\n# Akin to `torch.is_tensor`, but returns True for Variable\n# objects in pre-0.4.\ndef is_tensor_like(x):\n return torch.is_tensor(x) or isinstance(x, torch.autograd.Variable)\n\n# Wraps `torch.is_floating_point` if present, otherwise checks\n# the suffix of `x.type()`.\ndef is_floating_point(x):\n if hasattr(torch, 'is_floating_point'):\n return torch.is_floating_point(x)\n try:\n torch_type = x.type()\n return torch_type.endswith('FloatTensor') or \\\n torch_type.endswith('HalfTensor') or \\\n torch_type.endswith('DoubleTensor')\n except AttributeError:\n return False\n\ndef scalar_python_val(x):\n if hasattr(x, 'item'):\n return x.item()\n else:\n if isinstance(x, torch.autograd.Variable):\n return x.data[0]\n else:\n return x[0]\n" }, { "path": "apex/apex/amp/frontend.py", "content": "import torch\nfrom ._initialize import _initialize\nfrom ._amp_state import _amp_state, warn_or_err, maybe_print\n\n\nclass Properties(object):\n \"\"\"\n This class has two purposes: to establish a set of default properties,\n and to route setting of these attributes through __setattr__ so that (in theory)\n they can be checked for consistency with other existing args.\n \"\"\"\n def __init__(self):\n self.options = {\n \"enabled\" : False,\n \"opt_level\" : None,\n \"cast_model_type\" : None,\n \"patch_torch_functions\" : False,\n \"keep_batchnorm_fp32\" : None,\n \"master_weights\" : None,\n \"loss_scale\" : 1.0,\n # Reserved for future functionality\n # \"fused_optimizer\" : False,\n # \"enable_ddp_interop\" : False,\n }\n\n \"\"\"\n This function allows updating several options at a time without routing through\n __setattr__ checks, to avoid \"you can't get there from here\" scenarios.\n Currently not intended to be exposed; users are expected to select an opt_level\n and apply consistent modifications.\n \"\"\"\n def _update_options_dict(new_options):\n for k, v in new_options:\n if k in self.options:\n self.options[k] = v\n else:\n raise ValueError(\"Tried to set unexpected option {}\".format(k))\n \"\"\"\n The members of \"options\" are not direct attributes of self, so access attempts\n will roll down to __getattr__. This borrows from the logic in torch.nn.Module.\n \"\"\"\n def __getattr__(self, name):\n if \"options\" in self.__dict__:\n options = self.__dict__[\"options\"]\n if name in options:\n return options[name]\n raise AttributeError(\"'{}' object has no attribute '{}'\".format(\n type(self).__name__, name))\n\n def __setattr__(self, name, value):\n if \"options\" in self.__dict__:\n if name in self.options:\n # print(\"setting {} {}\".format(name, value))\n if name == \"cast_model_type\":\n if self.opt_level == \"O1\" and value is not None:\n if value is not False:\n if value is not torch.float32:\n warn_or_err(\"O1 inserts casts around Torch functions rather than \"\n \"model weights, so with O1, the model weights themselves \"\n \"should remain FP32. If you wish to cast the model to a \"\n \"different type, use opt_level='O2' or 'O3'. \" +\n \"cast_model_type was {}\".format(value))\n self.options[name] = value\n elif name == \"patch_torch_functions\":\n if self.opt_level != \"O1\" and value:\n warn_or_err(\"Currently, patch_torch_functions=True should only be set by \"\n \"selecting opt_level='O1'.\")\n self.options[name] = value\n elif name == \"keep_batchnorm_fp32\":\n if self.opt_level == \"O1\" and value is not None:\n warn_or_err(\"With opt_level O1, batchnorm functions are automatically patched \"\n \"to run in FP32, so keep_batchnorm_fp32 should be None.\" +\n \" keep_batchnorm_fp32 was {}\".format(value))\n if value == \"False\":\n self.options[name] = False\n elif value == \"True\":\n self.options[name] = True\n else:\n assert (value is True or value is False or value is None),\\\n \"keep_batchnorm_fp32 must be a boolean, the string 'True' or 'False', \"\\\n \"or None, found keep_batchnorm_fp32={}\".format(value)\n self.options[name] = value\n elif name == \"master_weights\":\n if self.opt_level == \"O1\" and value is not None:\n warn_or_err(\"It doesn't make sense to use master_weights with O1. \"\n \"With O1, your model weights themselves should be FP32.\")\n self.options[name] = value\n elif name == \"loss_scale\":\n if value == \"dynamic\":\n self.options[name] = value\n else:\n self.options[name] = float(value)\n else:\n self.options[name] = value\n else:\n super(Properties, self).__setattr__(name, value)\n\n\n\"\"\" O0-O3 are convenience wrappers to establish defaults for typically used mixed precision options. \"\"\"\n\nclass O3:\n brief = \"O3: Pure FP16 training.\"\n more = \"Calls .half() on your model, converting the entire model to FP16.\\n\"\\\n \"A casting operation is also inserted to cast incoming Tensors to FP16,\\n\"\\\n \"so you don't need to change your data pipeline.\\n\"\\\n \"This mode is useful for establishing a performance ceiling.\\n\"\\\n \"It's also possible training may 'just work' in this mode.\\n\"\\\n \"If not, try other optimization levels.\"\n\n def __call__(self, properties):\n properties.enabled = True\n properties.opt_level = \"O3\"\n properties.cast_model_type = torch.float16\n properties.patch_torch_functions = False\n properties.keep_batchnorm_fp32 = False\n properties.master_weights = False\n properties.loss_scale = 1.0\n # properties.fused_optimizer = False\n # properties.enable_ddp_interop = False\n return properties # modified in place so this isn't really necessary\n\n\nclass O2:\n brief = \"O2: FP16 training with FP32 batchnorm and FP32 master weights.\\n\"\n more = \"Calls .half() on your model, converting the entire model (except for batchnorms)\\n\"\\\n \"to FP16. Batchnorms are retained in FP32 for additional stability.\\n\"\\\n \"The forward pass is patched to cast incoming Tensors to FP16, so you don't need to change\\n\"\\\n \"your data pipeline.\\n\"\\\n \"O2 creates FP32 master weights outside the model and patches any optimizers to update\\n\"\\\n \"these master weights, then copy the master weights into the FP16 model weights.\\n\"\\\n \"Master weights can also improve convergence and stability.\"\n\n def __call__(self, properties):\n properties.enabled = True\n properties.opt_level = \"O2\"\n properties.cast_model_type = torch.float16\n properties.patch_torch_functions = False\n properties.keep_batchnorm_fp32 = True\n properties.master_weights = True\n properties.loss_scale = \"dynamic\"\n # properties.fused_optimizer = False\n # properties.enable_ddp_interop = False\n return properties # modified in place so this isn't really necessary\n\n\nclass O1:\n brief = \"O1: Insert automatic casts around Pytorch functions and Tensor methods.\\n\"\n more = \"The type of your model's weights is not altered. However, internally,\\n\"\\\n \"Pytorch functions are patched to cast any Tensor Core-friendly ops to FP16 for speed,\\n\"\\\n \"while operations that might benefit from the additional stability of FP32 are patched\\n\"\\\n \"to cast their inputs to fp32.\\n\"\\\n \"O1 is the safest way to try mixed precision training, and is recommended when\\n\"\\\n \"trying mixed precision training for the first time.\"\n\n def __call__(self, properties):\n properties.enabled = True\n properties.opt_level = \"O1\"\n properties.cast_model_type = None\n properties.patch_torch_functions = True\n properties.keep_batchnorm_fp32 = None\n properties.master_weights = None\n properties.loss_scale = \"dynamic\"\n # properties.fused_optimizer = False\n # properties.enable_ddp_interop = False\n return properties # modified in place so this isn't really necessary\n\n\nclass O0:\n brief = \"O0: Pure FP32 training.\\n\"\n more = \"Your models are checked to make sure parameters are FP32, but otherwise the\\n\"\\\n \"types of weights and internal Pytorch operations are not altered. This mode disables any\\n\"\\\n \"FP16 arithmetic, although other optimizations like DDP interop may still be requested.\\n\"\n\n def __call__(self, properties):\n properties.enabled = True\n properties.opt_level = \"O0\"\n properties.cast_model_type = torch.float32\n properties.patch_torch_functions = False\n properties.keep_batchnorm_fp32 = None\n properties.master_weights = False\n properties.loss_scale = 1.0\n # properties.fused_optimizer = False\n # properties.enable_ddp_interop = False\n return properties # modified in place so this isn't really necessary\n\n\nopt_levels = {\"O3\": O3(),\n \"O2\": O2(),\n \"O1\": O1(),\n \"O0\": O0()}\n\n\n# allow user to directly pass Properties struct as well?\ndef initialize(\n models,\n optimizers=None,\n enabled=True,\n opt_level=\"O1\",\n cast_model_type=None,\n patch_torch_functions=None,\n keep_batchnorm_fp32=None,\n master_weights=None,\n loss_scale=None,\n cast_model_outputs=None,\n num_losses=1,\n verbosity=1,\n min_loss_scale=None,\n max_loss_scale=2.**24\n ):\n \"\"\"\n Initialize your models, optimizers, and the Torch tensor and functional namespace according to the\n chosen ``opt_level`` and overridden properties, if any.\n\n ``amp.initialize`` should be called **after** you have finished\n constructing your model(s) and\n optimizer(s), but **before** you send your model through any DistributedDataParallel wrapper.\n See `Distributed training`_ in the Imagenet example.\n\n Currently, ``amp.initialize`` should only be called **once**,\n although it can process an arbitrary number of\n models and optimizers (see the corresponding `Advanced Amp Usage topic`_).\n If you think your use case requires ``amp.initialize`` to be called more than once,\n `let us know`_.\n\n Any property keyword argument that is not ``None`` will be interpreted as a manual override.\n\n To prevent having to rewrite anything else in your script, name the returned models/optimizers\n to replace the passed models/optimizers, as in the code sample below.\n\n Args:\n models (torch.nn.Module or list of torch.nn.Modules): Models to modify/cast.\n optimizers (optional, torch.optim.Optimizer or list of torch.optim.Optimizers): Optimizers to modify/cast.\n REQUIRED for training, optional for inference.\n enabled (bool, optional, default=True): If False, renders all Amp calls no-ops, so your script\n should run as if Amp were not present.\n opt_level (str, optional, default=\"O1\"): Pure or mixed precision optimization level. Accepted values are\n \"O0\", \"O1\", \"O2\", and \"O3\", explained in detail above.\n cast_model_type (``torch.dtype``, optional, default=None): Optional property override, see\n above.\n patch_torch_functions (bool, optional, default=None): Optional property override.\n keep_batchnorm_fp32 (bool or str, optional, default=None): Optional property override. If\n passed as a string, must be the string \"True\" or \"False\".\n master_weights (bool, optional, default=None): Optional property override.\n loss_scale (float or str, optional, default=None): Optional property override. If passed as a string,\n must be a string representing a number, e.g., \"128.0\", or the string \"dynamic\".\n cast_model_outputs (torch.dtype, optional, default=None): Option to ensure that the outputs\n of your model(s) are always cast to a particular type regardless of ``opt_level``.\n num_losses (int, optional, default=1): Option to tell Amp in advance how many losses/backward\n passes you plan to use. When used in conjunction with the ``loss_id`` argument to\n ``amp.scale_loss``, enables Amp to use a different loss scale per loss/backward pass,\n which can improve stability. See \"Multiple models/optimizers/losses\"\n under `Advanced Amp Usage`_ for examples. If ``num_losses`` is left to 1, Amp will still\n support multiple losses/backward passes, but use a single global loss scale\n for all of them.\n verbosity (int, default=1): Set to 0 to suppress Amp-related output.\n min_loss_scale (float, default=None): Sets a floor for the loss scale values that can be chosen by dynamic\n loss scaling. The default value of None means that no floor is imposed.\n If dynamic loss scaling is not used, `min_loss_scale` is ignored.\n max_loss_scale (float, default=2.**24): Sets a ceiling for the loss scale values that can be chosen by\n dynamic loss scaling. If dynamic loss scaling is not used, `max_loss_scale` is ignored.\n\n Returns:\n Model(s) and optimizer(s) modified according to the ``opt_level``.\n If either the ``models`` or ``optimizers`` args were lists, the corresponding return value will\n also be a list.\n\n Permissible invocations::\n\n model, optim = amp.initialize(model, optim,...)\n model, [optim1, optim2] = amp.initialize(model, [optim1, optim2],...)\n [model1, model2], optim = amp.initialize([model1, model2], optim,...)\n [model1, model2], [optim1, optim2] = amp.initialize([model1, model2], [optim1, optim2],...)\n\n # This is not an exhaustive list of the cross product of options that are possible,\n # just a set of examples.\n model, optim = amp.initialize(model, optim, opt_level=\"O0\")\n model, optim = amp.initialize(model, optim, opt_level=\"O0\", loss_scale=\"dynamic\"|128.0|\"128.0\")\n\n model, optim = amp.initialize(model, optim, opt_level=\"O1\") # uses \"loss_scale=\"dynamic\" default\n model, optim = amp.initialize(model, optim, opt_level=\"O1\", loss_scale=128.0|\"128.0\")\n\n model, optim = amp.initialize(model, optim, opt_level=\"O2\") # uses \"loss_scale=\"dynamic\" default\n model, optim = amp.initialize(model, optim, opt_level=\"O2\", loss_scale=128.0|\"128.0\")\n model, optim = amp.initialize(model, optim, opt_level=\"O2\", keep_batchnorm_fp32=True|False|\"True\"|\"False\")\n\n model, optim = amp.initialize(model, optim, opt_level=\"O3\") # uses loss_scale=1.0 default\n model, optim = amp.initialize(model, optim, opt_level=\"O3\", loss_scale=\"dynamic\"|128.0|\"128.0\")\n model, optim = amp.initialize(model, optim, opt_level=\"O3\", keep_batchnorm_fp32=True|False|\"True\"|\"False\")\n\n The `Imagenet example`_ demonstrates live use of various opt_levels and overrides.\n\n .. _`Distributed training`:\n https://github.com/NVIDIA/apex/tree/master/examples/imagenet#distributed-training\n\n .. _`Imagenet example`:\n https://github.com/NVIDIA/apex/tree/master/examples/imagenet\n\n .. _`Advanced Amp Usage`:\n https://nvidia.github.io/apex/advanced.html\n\n .. _`Advanced Amp Usage topic`:\n https://nvidia.github.io/apex/advanced.html#multiple-models-optimizers-losses\n\n .. _`let us know`:\n https://github.com/NVIDIA/apex/issues\n \"\"\"\n _amp_state.opt_properties = Properties()\n _amp_state.verbosity = verbosity\n\n if not enabled:\n if optimizers is None:\n return models\n else:\n return models, optimizers\n\n if not torch.backends.cudnn.enabled:\n raise RuntimeError(\n \"Amp requires torch.backends.cudnn.enabled = True\")\n\n if opt_level not in opt_levels:\n raise RuntimeError(\n \"Unexpected optimization level {}. \".format(opt_level) +\n \"Options are 'O0', 'O1', 'O2', 'O3'. Note that in `O0`, `O1`, etc., the prefix O is the letter O, \" +\n \"not the number zero.\")\n else:\n _amp_state.opt_properties = opt_levels[opt_level](_amp_state.opt_properties)\n maybe_print(\"Selected optimization level {}\".format(opt_levels[opt_level].brief), True)\n maybe_print(\"Defaults for this optimization level are:\", True)\n for k, v in _amp_state.opt_properties.options.items():\n maybe_print(\"{:22} : {}\".format(k, v), True)\n\n _amp_state.min_loss_scale = min_loss_scale\n _amp_state.max_loss_scale = max_loss_scale\n\n maybe_print(\"Processing user overrides (additional kwargs that are not None)...\", True)\n # I chose to have the keyword arguments listed directly in the argument list,\n # instead of **kwargs, so I can't use kwargs.items() here.\n if enabled is not None:\n _amp_state.opt_properties.enabled = enabled\n if opt_level is not None:\n _amp_state.opt_properties.opt_level = opt_level\n if cast_model_type is not None:\n _amp_state.opt_properties.cast_model_type = cast_model_type\n if patch_torch_functions is not None:\n _amp_state.opt_properties.patch_torch_functions = patch_torch_functions\n if keep_batchnorm_fp32 is not None:\n _amp_state.opt_properties.keep_batchnorm_fp32 = keep_batchnorm_fp32\n if master_weights is not None:\n _amp_state.opt_properties.master_weights = master_weights\n if loss_scale is not None:\n _amp_state.opt_properties.loss_scale = loss_scale\n\n maybe_print(\"After processing overrides, optimization options are:\", True)\n for k, v in _amp_state.opt_properties.options.items():\n maybe_print(\"{:22} : {}\".format(k, v), True)\n\n return _initialize(models, optimizers, _amp_state.opt_properties, num_losses, cast_model_outputs)\n\n\n# TODO: is this necessary/useful?\n# def check_option_consistency(enabled=True,\n# opt_level=None,\n# cast_model_type=None,\n# patch_torch_functions=None,\n# keep_batchnorm_fp32=None,\n# master_weights=None,\n# loss_scale=None,\n# enable_ddp_interop=None,\n# hard_override=False):\n# \"\"\"\n# Utility function that enables users to quickly check if the option combination they intend\n# to use is permitted. ``check_option_consistency`` does not require models or optimizers\n# to be constructed, and can be called at any point in the script. ``check_option_consistency``\n# is totally self-contained; it does not set any amp global state or affect anything outside\n# of itself.\n# \"\"\"\n#\n# if not enabled:\n# return\n#\n# if opt_level not in opt_levels:\n# raise RuntimeError(\"Unexpected optimization level. Options are 'O0', 'O1', 'O2', 'O3'.\")\n# else:\n# opt_properties = opt_levels[opt_level](Properties())\n# print(\"Selected optimization level {}\", opt_levels[opt_level].brief)\n# print(\"Defaults for this optimization level are:\")\n# for k, v in opt_properties.options:\n# print(\"{:22} : {}\".format(k, v))\n#\n# print(\"Processing user overrides (additional kwargs that are not None)...\")\n# for k, v in kwargs:\n# if k not in _amp_state.opt_properties.options:\n# raise RuntimeError(\"Unexpected kwarg {}\".format(k))\n# if v is not None:\n# setattr(opt_properties, k, v)\n#\n# print(\"After processing overrides, optimization options are:\")\n# for k, v in opt_properties.options:\n# print(\"{:22} : {}\".format(k, v))\n" }, { "path": "apex/apex/amp/handle.py", "content": "import contextlib\nimport warnings\nimport torch\n\nfrom . import utils\nfrom .opt import OptimWrapper\nfrom .scaler import LossScaler\nfrom ._amp_state import _amp_state, master_params, maybe_print\nfrom ..fp16_utils import FP16_Optimizer as FP16_Optimizer_general\nfrom ..optimizers import FP16_Optimizer as FP16_Optimizer_for_fused\nfrom ..parallel.LARC import LARC\n\n\n# There's no reason to expose the notion of a \"handle\". Everything can happen through amp.* calls.\n@contextlib.contextmanager\ndef scale_loss(loss,\n optimizers,\n loss_id=0,\n model=None,\n delay_unscale=False,\n delay_overflow_check=False):\n \"\"\"\n On context manager entrance, creates ``scaled_loss = (loss.float())*current loss scale``.\n ``scaled_loss`` is yielded so that the user can call ``scaled_loss.backward()``::\n\n with amp.scale_loss(loss, optimizer) as scaled_loss:\n scaled_loss.backward()\n\n On context manager exit (if ``delay_unscale=False``), the gradients are checked for infs/NaNs\n and unscaled, so that ``optimizer.step()`` can be called.\n\n .. note::\n If Amp is using explicit FP32 master params (which is the default for ``opt_level=O2``, and\n can also be manually enabled by supplying ``master_weights=True`` to ``amp.initialize``)\n any FP16 gradients are copied to FP32 master gradients before being unscaled.\n ``optimizer.step()`` will then apply the unscaled master gradients to the master params.\n\n .. warning::\n If Amp is using explicit FP32 master params, only the FP32 master gradients will be\n unscaled. The direct ``.grad`` attributes of any FP16\n model params will remain scaled after context manager exit.\n This subtlety affects gradient clipping. See \"Gradient clipping\" under\n `Advanced Amp Usage`_ for best practices.\n\n Args:\n loss(Tensor): Typically a scalar Tensor. The ``scaled_loss`` that the context\n manager yields is simply ``loss.float()*loss_scale``, so in principle\n ``loss`` could have more than one element, as long as you call\n ``backward()`` on ``scaled_loss`` appropriately within the context manager body.\n optimizers: All optimizer(s) for which the current backward pass is creating gradients.\n Must be an optimizer or list of optimizers returned from an earlier call\n to ``amp.initialize``. For example use with multiple optimizers, see\n \"Multiple models/optimizers/losses\" under `Advanced Amp Usage`_.\n loss_id(int, optional, default=0): When used in conjunction with the ``num_losses`` argument\n to ``amp.initialize``, enables Amp to use a different loss scale per loss. ``loss_id``\n must be an integer between 0 and ``num_losses`` that tells Amp which loss is\n being used for the current backward pass. See \"Multiple models/optimizers/losses\"\n under `Advanced Amp Usage`_ for examples. If ``loss_id`` is left unspecified, Amp\n will use the default global loss scaler for this backward pass.\n model(torch.nn.Module, optional, default=None): Currently unused, reserved to enable future\n optimizations.\n delay_unscale(bool, optional, default=False): ``delay_unscale`` is never necessary, and\n the default value of ``False`` is strongly recommended.\n If ``True``, Amp will not unscale the gradients or perform model->master\n gradient copies on context manager exit.\n ``delay_unscale=True`` is a minor ninja performance optimization and can result\n in weird gotchas (especially with multiple models/optimizers/losses),\n so only use it if you know what you're doing.\n \"Gradient accumulation across iterations\" under `Advanced Amp Usage`_\n illustrates a situation where this CAN (but does not need to) be used.\n\n .. warning::\n If ``delay_unscale`` is ``True`` for a given backward pass, ``optimizer.step()`` cannot be\n called yet after context manager exit, and must wait for another, later backward context\n manager invocation with ``delay_unscale`` left to False.\n\n .. _`Advanced Amp Usage`:\n https://nvidia.github.io/apex/advanced.html\n \"\"\"\n if not hasattr(_amp_state, \"opt_properties\"):\n raise RuntimeError(\"Invoked 'with amp.scale_loss`, but internal Amp state has not been initialized. \"\n \"model, optimizer = amp.initialize(model, optimizer, opt_level=...) must be called \"\n \"before `with amp.scale_loss`.\")\n\n if not _amp_state.opt_properties.enabled:\n yield loss\n return\n\n if isinstance(optimizers, torch.optim.Optimizer) or isinstance(optimizers, LARC):\n optimizers = [optimizers]\n\n # this is what happens when i have to support tools from different sources under the same API...\n # TODO: Rewrite FusedAdam to use multi-tensor apply and the same loss scaler.\n if isinstance(optimizers, FP16_Optimizer_for_fused):\n loss_scale = optimizers.cur_scale\n else:\n loss_scaler = _amp_state.loss_scalers[loss_id]\n loss_scale = loss_scaler.loss_scale()\n\n if ((not _amp_state.opt_properties.master_weights)\n and (not loss_scaler.dynamic)\n and loss_scale == 1.0):\n yield loss.float()\n # Needing to drop the cache here as well is an ugly gotcha.\n # But for now I think it's necessary to short-circuit.\n # Probably ok to skip this if not delay_unscale\n if _amp_state.opt_properties.patch_torch_functions:\n _amp_state.handle._clear_cache()\n return\n\n if not delay_unscale:\n if isinstance(optimizers, list):\n for optimizer in optimizers:\n if not optimizer._amp_stash.params_have_scaled_gradients:\n optimizer._prepare_amp_backward()\n\n yield (loss.float())*loss_scale\n\n if delay_unscale:\n for optimizer in optimizers:\n optimizer._amp_stash.params_have_scaled_gradients = True\n else:\n # FusedAdam and FusedSGD will take care of unscaling as part of their step() methods.\n if not isinstance(optimizers, FP16_Optimizer_for_fused):\n loss_scaler.clear_overflow_state()\n for optimizer in optimizers:\n optimizer._post_amp_backward(loss_scaler)\n optimizer._amp_stash.params_have_scaled_gradients = False\n # For future fused optimizers that enable sync-free dynamic loss scaling,\n # should_skip will always be False.\n should_skip = False if delay_overflow_check else loss_scaler.update_scale()\n if should_skip:\n for optimizer in optimizers:\n if not optimizer._amp_stash.already_patched:\n # Close on loss_scaler and loss_id as well, to be safe. Probably not\n # necessary because amp.scale_loss is already creating a temporary scope.\n def patch_step(opt, loss_scaler, loss_id):\n opt_step = opt.step\n def skip_step(closure=None):\n if closure is not None:\n raise RuntimeError(\"Currently, Amp does not support closure use with optimizers.\")\n maybe_print((\"Gradient overflow. Skipping step, loss scaler \" +\n \"{} reducing loss scale to {}\").format(loss_id,\n loss_scaler.loss_scale()))\n if hasattr(opt._amp_stash, \"all_fp32_from_fp16_params\"):\n # Clear the master grads that wouldn't be zeroed by model.zero_grad()\n for param in opt._amp_stash.all_fp32_from_fp16_params:\n param.grad = None\n opt.step = opt_step\n opt._amp_stash.already_patched = False\n return skip_step\n optimizer.step = patch_step(optimizer, loss_scaler, loss_id)\n optimizer._amp_stash.already_patched = True\n\n # Probably ok to skip this if not delay_unscale\n if _amp_state.opt_properties.patch_torch_functions:\n _amp_state.handle._clear_cache()\n\n\n# Free function version of AmpHandle.disable_casts, another step on the\n# path to removing the concept of \"AmpHandle\"\n@contextlib.contextmanager\ndef disable_casts():\n _amp_state.handle._is_active = False\n yield\n _amp_state.handle._is_active = True\n\n\nclass AmpHandle(object):\n def __init__(self, loss_scale=\"dynamic\", enable_caching=True, verbose=False):\n self._enable_caching = enable_caching\n self._verbose = verbose\n self._cache = dict()\n self._default_scaler = LossScaler(loss_scale)\n self._is_active = True\n self._all_wrappers = []\n\n def is_active(self):\n return self._is_active\n\n @contextlib.contextmanager\n def _disable_casts(self):\n self._is_active = False\n yield\n self._is_active = True\n\n def wrap_optimizer(self, optimizer, num_loss=1):\n self._default_scaler = None\n return OptimWrapper(optimizer, self, num_loss)\n\n @contextlib.contextmanager\n def scale_loss(self, loss, optimizer):\n raise RuntimeError(\"The old Amp API is no longer supported. Please move to the new API, \"\n \"documented here: https://nvidia.github.io/apex/amp.html. Transition guide: \"\n \"https://nvidia.github.io/apex/amp.html#transition-guide-for-old-api-users\")\n\n if not self.is_active():\n yield loss\n return\n\n if self._default_scaler is None:\n raise RuntimeError(\n 'After calling `handle.wrap_optimizer()`, you must explicitly ' +\n 'use `optimizer.scale_loss(loss)`.')\n\n # TODO: this code block is duplicated here and `opt.py`. Unify.\n loss_scale = self._default_scaler.loss_scale()\n yield loss * loss_scale\n\n self._default_scaler.clear_overflow_state()\n self._default_scaler.unscale(\n master_params(optimizer),\n master_params(optimizer),\n loss_scale)\n should_skip = self._default_scaler.update_scale()\n if should_skip:\n optimizer_step = optimizer.step\n def skip_step():\n maybe_print('Gradient overflow, skipping update')\n optimizer.step = optimizer_step\n optimizer.step = skip_step\n\n self._clear_cache()\n\n def _clear_cache(self):\n self._cache.clear()\n\n # Experimental support for saving / restoring uncasted versions of functions\n def _save_func(self, mod, fn, func):\n self._all_wrappers.append((mod, fn, func))\n\n def _deactivate(self):\n for mod, fn, func in self._all_wrappers:\n utils.set_func(mod, fn, func)\n self._all_wrappers = []\n\n @property\n def has_cache(self):\n return self._enable_caching\n\n @property\n def cache(self):\n return self._cache\n\n def remove_cache(self, param):\n if self.has_cache and param in self.cache:\n del self.cache[param]\n\n @property\n def verbose(self):\n return self._verbose\n\nclass NoOpHandle(object):\n def is_active(self):\n return False\n\n @contextlib.contextmanager\n def _disable_casts(self):\n yield\n\n def wrap_optimizer(self, optimizer, num_loss=1):\n return OptimWrapper(optimizer, self, num_loss)\n\n @contextlib.contextmanager\n def scale_loss(self, loss, optimizer):\n yield loss\n\n @property\n def has_cache(self):\n return False\n\n @property\n def verbose(self):\n return False\n\n def _clear_cache(self):\n pass\n\n def _deactivate(self):\n pass\n" }, { "path": "apex/apex/amp/lists/__init__.py", "content": "" }, { "path": "apex/apex/amp/lists/functional_overrides.py", "content": "\n# TODO: think about the following two. They do weird things.\n# - torch.nn.utils.clip_grad (but it should always be fp32 anyway)\n# - torch.nn.utils.weight_norm\n\n# Notes:\n# F.instance_norm uses batch_norm internally. Which correctly handles\n# fp16 in/out with fp32 weights. So we shouldn't do anything for\n# either of these.\n# F.normalize calls `input.norm()` internally, so it's redundant, but\n# kept here in case impl. changes.\n# F.cosine_similarity is same: calls `x.norm()` internally.\n\nimport torch.nn.functional\n\nMODULE = torch.nn.functional\n\nFP16_FUNCS = [\n 'conv1d',\n 'conv2d',\n 'conv3d',\n 'conv_transpose1d',\n 'conv_transpose2d',\n 'conv_transpose3d',\n 'conv_tbc', # Undocumented / maybe new?\n 'linear',\n]\n\nFP32_FUNCS = [\n\n # Interpolation/Upsampling\n 'interpolate',\n\n # Pointwise\n 'softplus',\n 'softmin',\n 'log_softmax',\n 'softmax',\n\n # Normalization\n 'layer_norm',\n 'group_norm',\n 'local_response_norm',\n 'normalize',\n 'cosine_similarity',\n\n # Loss functions\n # TODO: which of these can be fp16?\n 'poisson_nll_loss',\n 'cosine_embedding_loss',\n 'cross_entropy',\n 'hinge_embedding_loss',\n 'kl_div',\n 'l1_loss',\n 'mse_loss',\n 'margin_ranking_loss',\n 'multilabel_margin_loss',\n 'multilabel_soft_margin_loss',\n 'multi_margin_loss',\n 'nll_loss',\n 'binary_cross_entropy_with_logits',\n 'smooth_l1_loss',\n 'soft_margin_loss',\n 'triplet_margin_loss'\n]\n\nBANNED_FUNCS = [\n ('binary_cross_entropy',\n (\"\\namp does not work out-of-the-box with `F.binary_cross_entropy` or `torch.nn.BCELoss.` \"\n \"It requires that the output of the previous function be already a FloatTensor. \\n\\n\"\n \"Most models have a Sigmoid right before BCELoss. In that case, you can use\\n\"\n \" torch.nn.BCEWithLogitsLoss\\nto combine Sigmoid+BCELoss into a single layer \"\n \"that is compatible with amp.\\nAnother option is to add\\n\"\n \" amp.register_float_function(torch, 'sigmoid')\\nbefore calling `amp.init()`.\\n\"\n \"If you _really_ know what you are doing, you can disable this warning by passing \"\n \"allow_banned=True to `amp.init()`.\"))\n]\n" }, { "path": "apex/apex/amp/lists/tensor_overrides.py", "content": "from .. import compat\nfrom . import torch_overrides\n\nimport importlib\n\nimport torch\n\nif compat.variable_is_tensor() and not compat.tensor_is_variable():\n MODULE = torch.Tensor\nelse:\n MODULE = torch.autograd.Variable\n\n\nFP16_FUNCS = [\n '__matmul__',\n]\n\nFP32_FUNCS = [\n '__ipow__',\n '__pow__',\n '__rpow__',\n\n # Cast to fp32 before transfer to CPU\n 'cpu',\n]\n\nCASTS = [\n '__add__',\n '__div__',\n '__eq__',\n '__ge__',\n '__gt__',\n '__iadd__',\n '__idiv__',\n '__imul__',\n '__isub__',\n '__itruediv__',\n '__le__',\n '__lt__',\n '__mul__',\n '__ne__',\n '__radd__',\n '__rdiv__',\n '__rmul__',\n '__rsub__',\n '__rtruediv__',\n '__sub__',\n '__truediv__',\n]\n\n# None of these, but here to make code cleaner.\nSEQUENCE_CASTS = []\n\n# We need to grab all the methods from torch_overrides and add them to\n# the Tensor lists as well, as almost all methods are duplicated\n# between `torch` and `torch.Tensor` (and check with `hasattr`,\n# because a few random ones aren't defined on Tensor)\n_self_mod = importlib.import_module(__name__)\nfor attrname in ['FP16_FUNCS', 'FP32_FUNCS', 'CASTS', 'SEQUENCE_CASTS']:\n lst = getattr(_self_mod, attrname)\n for fn in getattr(torch_overrides, attrname):\n if hasattr(MODULE, fn):\n lst.append(fn)\n" }, { "path": "apex/apex/amp/lists/torch_overrides.py", "content": "import torch\n\nfrom .. import utils\n\nMODULE = torch\n\nFP16_FUNCS = [\n # Low level functions wrapped by torch.nn layers.\n # The wrapper layers contain the weights which are then passed in as a parameter\n # to these functions.\n 'conv1d',\n 'conv2d',\n 'conv3d',\n 'conv_transpose1d',\n 'conv_transpose2d',\n 'conv_transpose3d',\n 'conv_tbc',\n 'prelu',\n\n # BLAS\n 'addmm',\n 'addmv',\n 'addr',\n 'matmul',\n 'mm',\n 'mv',\n]\n\nFP32_FUNCS = [\n # Pointwise\n 'acos',\n 'asin',\n 'cosh',\n 'erfinv',\n 'exp',\n 'expm1',\n 'log',\n 'log10',\n 'log2',\n 'reciprocal',\n 'rsqrt',\n 'sinh',\n 'tan',\n\n # Other math\n 'pow',\n\n # Reduction\n 'cumprod',\n 'cumsum',\n 'dist',\n 'mean',\n 'norm',\n 'prod',\n 'std',\n 'sum',\n 'var',\n\n # Misc\n 'renorm'\n]\n\n# Before CUDA 9.1, batched matmul was missing fast FP16 kernels. We\n# check the CUDA version -- if at least 9.1, then put the bmm\n# functions on the fp16 list. Otherwise, put them on the fp32 list.\n_bmms = ['addbmm',\n 'baddbmm',\n 'bmm']\nif utils.get_cuda_version() >= (9, 1, 0):\n FP16_FUNCS.extend(_bmms)\nelse:\n FP32_FUNCS.extend(_bmms)\n\n# Multi-tensor fns that may need type promotion\nCASTS = [\n # Multi-tensor math\n 'addcdiv',\n 'addcmul',\n 'atan2',\n 'cross',\n 'bilinear',\n\n # Element-wise _or_ tensor-wise math\n 'add',\n 'div',\n 'mul',\n\n # Comparison\n 'eq',\n 'equal',\n 'ge',\n 'gt',\n 'le',\n 'lt',\n 'ne'\n]\n\n# Functions that take sequence arguments. We need to inspect the whole\n# sequence and cast to the widest type.\nSEQUENCE_CASTS = [\n 'cat',\n 'stack'\n]\n" }, { "path": "apex/apex/amp/opt.py", "content": "import contextlib\nimport warnings\n\nfrom .scaler import LossScaler, master_params\nfrom ._amp_state import maybe_print\n\nimport numpy as np\n\nclass OptimWrapper(object):\n def __init__(self, optimizer, amp_handle, num_loss):\n self._optimizer = optimizer\n self._amp_handle = amp_handle\n self._num_loss = num_loss\n self._loss_idx = 0\n self._skip_next = [False] * num_loss\n self._loss_scaler = [LossScaler('dynamic') for _ in range(num_loss)]\n\n @contextlib.contextmanager\n def scale_loss(self, loss):\n if not self._amp_handle.is_active():\n yield loss\n return\n\n # When there are multiple losses per-optimizer, we need\n # to save out current grad accumulation, since we won't be\n # able to unscale this particulare loss once the grads are\n # all mixed together.\n cached_grads = []\n if self._loss_idx > 0:\n for p in master_params(self._optimizer):\n if p.grad is not None:\n cached_grads.append(p.grad.data.detach().clone())\n else:\n cached_grads.append(None)\n self._optimizer.zero_grad()\n\n loss_scale = self._cur_loss_scaler().loss_scale()\n yield loss * loss_scale\n\n self._cur_loss_scaler().clear_overflow_state()\n self._cur_loss_scaler().unscale(\n master_params(self._optimizer),\n master_params(self._optimizer),\n loss_scale)\n self._skip_next[self._loss_idx] = self._cur_loss_scaler().update_scale()\n self._loss_idx += 1\n\n if len(cached_grads) > 0:\n for p, cached_grad in zip(master_params(self._optimizer),\n cached_grads):\n if cached_grad is not None:\n p.grad.data.add_(cached_grad)\n cached_grads = []\n\n def _cur_loss_scaler(self):\n assert 0 <= self._loss_idx < self._num_loss\n return self._loss_scaler[self._loss_idx]\n\n def step(self, closure=None):\n if not self._amp_handle.is_active():\n return self._optimizer.step(closure=closure)\n\n self._loss_idx = 0\n\n for group in self._optimizer.param_groups:\n for p in group['params']:\n self._amp_handle.remove_cache(p)\n\n if closure is not None:\n raise NotImplementedError(\n 'The `closure` argument is unsupported by the amp ' +\n 'optimizer wrapper.')\n if any(self._skip_next):\n maybe_print('Gradient overflow, skipping update')\n self._skip_next = [False] * self._num_loss\n else:\n return self._optimizer.step(closure=closure)\n\n # Forward any attribute lookups\n def __getattr__(self, attr):\n return getattr(self._optimizer, attr)\n\n # Forward all torch.optim.Optimizer methods\n def __getstate__(self):\n return self._optimizer.__getstate__()\n\n def __setstate__(self):\n return self._optimizer.__setstate__()\n\n def __repr__(self):\n return self._optimizer.__repr__()\n\n def state_dict(self):\n return self._optimizer.state_dict()\n\n def load_state_dict(self, state_dict):\n return self._optimizer.load_state_dict(state_dict)\n\n def zero_grad(self):\n return self._optimizer.zero_grad()\n\n def add_param_group(self, param_group):\n return self._optimizer.add_param_group(param_group)\n" }, { "path": "apex/apex/amp/rnn_compat.py", "content": "from . import utils, wrap\n\nimport torch\n_VF = torch._C._VariableFunctions\nRNN_NAMES = ['rnn_relu', 'rnn_tanh', 'gru', 'lstm']\n\ndef _gen_VF_wrapper(name):\n def wrapper(*args, **kwargs):\n return getattr(_VF, name)(*args, **kwargs)\n return wrapper\n\n# Some python magic to generate an object that has the rnn cell functions\n# defined on it, all of which call into corresponding _VF version.\n# Intended to patch torch.nn.modules.rnn._VF (aka, the ref named \"_VF\"\n# imported at module scope within torch.nn.modules.rnn). This should\n# not affect third-party importers of _VF.py.\nclass VariableFunctionsShim(object):\n def __init__(self):\n for name in RNN_NAMES:\n for suffix in ['', '_cell']:\n fn_name = name + suffix\n setattr(self, fn_name, _gen_VF_wrapper(fn_name))\n\ndef has_old_rnns():\n try:\n torch.nn.backends.thnn.backend.LSTMCell\n return True\n except:\n return False\n\ndef whitelist_rnn_cells(handle, verbose):\n # Different module + function names in old/new RNN cases\n if has_old_rnns():\n fn_names = ['RNNReLUCell', 'RNNTanhCell', 'LSTMCell', 'GRUCell']\n mod = torch.nn.backends.thnn.backend\n else:\n fn_names = [x + '_cell' for x in RNN_NAMES]\n mod = torch.nn.modules.rnn._VF\n assert isinstance(mod, VariableFunctionsShim)\n\n # Insert casts on cell functions\n for fn in fn_names:\n wrap.cached_cast(mod, fn, utils.maybe_half, handle,\n try_caching=True, verbose=verbose)\n\n if has_old_rnns():\n # Special handling of `backward` for fused gru / lstm:\n # The `backward` method calls Tensor.sum() (blacklist) internally,\n # and then the resulting grad_input has the wrong type.\n # TODO: where else is this a problem?\n for rnn_type in ['GRUFused', 'LSTMFused']:\n mod = getattr(torch.nn._functions.thnn.rnnFusedPointwise, rnn_type)\n wrap.disable_casts(mod, 'backward', handle)\n" }, { "path": "apex/apex/amp/scaler.py", "content": "import torch\nfrom ..multi_tensor_apply import multi_tensor_applier\nfrom ._amp_state import _amp_state, master_params, maybe_print\nfrom itertools import product\n\ndef scale_check_overflow_python(model_grad, master_grad, scale, check_overflow=False):\n # Exception handling for 18.04 compatibility\n if check_overflow:\n cpu_sum = float(model_grad.float().sum())\n if cpu_sum == float('inf') or cpu_sum == -float('inf') or cpu_sum != cpu_sum:\n return True\n\n if master_grad is not model_grad: # copy_ probably internally short-circuits this\n master_grad.copy_(model_grad)\n if scale != 1.0:\n master_grad.mul_(scale)\n return False\n\ndef axpby_check_overflow_python(model_grad, stashed_grad, master_grad, scale, check_overflow=False):\n # Exception handling for 18.04 compatibility\n if check_overflow:\n cpu_sum = float(model_grad.float().sum())\n if cpu_sum == float('inf') or cpu_sum == -float('inf') or cpu_sum != cpu_sum:\n return True\n\n # if master_grad is not model_grad: # copy_ probably internally short-circuits this\n # master_grad.copy_(model_grad)\n assert stashed_grad.dtype == master_grad.dtype\n converted_model_grad = model_grad.to(master_grad.dtype)\n stashed_grad.add_(scale, converted_model_grad)\n master_grad.data = stashed_grad.data\n return False\n\nclass LossScaler(object):\n warned_no_fused_kernel = False\n warned_unscaling_non_fp32_grad = False\n has_fused_kernel = False\n\n def __init__(self,\n loss_scale,\n init_scale=2.**16,\n scale_factor=2.,\n scale_window=2000,\n min_loss_scale=None,\n max_loss_scale=2.**24):\n if loss_scale == \"dynamic\":\n self.dynamic = True\n self._loss_scale = init_scale\n else:\n self.dynamic = False\n self._loss_scale = loss_scale\n self._max_loss_scale = max_loss_scale\n self._min_loss_scale = min_loss_scale\n self._scale_seq_len = scale_window\n self._unskipped = 0\n self._has_overflow = False\n self._overflow_buf = torch.cuda.IntTensor([0])\n if multi_tensor_applier.available:\n import amp_C\n LossScaler.has_fused_kernel = multi_tensor_applier.available\n LossScaler.multi_tensor_scale_cuda = amp_C.multi_tensor_scale\n LossScaler.multi_tensor_axpby_cuda = amp_C.multi_tensor_axpby\n else:\n if not LossScaler.warned_no_fused_kernel:\n maybe_print(\n \"Warning: multi_tensor_applier fused unscale kernel is unavailable, \"\n \"possibly because apex was installed without --cuda_ext --cpp_ext. \"\n \"Using Python fallback. Original ImportError was: \" +\n repr(multi_tensor_applier.import_err),\n True)\n LossScaler.has_fused_kernel = False\n LossScaler.warned_no_fused_kernel = True\n\n def loss_scale(self):\n return self._loss_scale\n\n def unscale_python(self, model_grads, master_grads, scale):\n for model, master in zip(model_grads, master_grads):\n if model is not None:\n if not LossScaler.warned_unscaling_non_fp32_grad:\n if master.dtype != torch.float32:\n maybe_print(\n \"Attempting to unscale a grad with type {} \".format(master.type()) +\n \"Unscaling non-fp32 grads may indicate an error. \"\n \"When using Amp, you don't need to call .half() on your model.\")\n LossScaler.warned_unscaling_non_fp32_grad = True\n self._has_overflow = scale_check_overflow_python(model,\n master,\n 1./scale,\n self.dynamic)\n if self._has_overflow and self.dynamic:\n break\n\n # unused_scale keeps some of the old API alive for hopefully a short time.\n def unscale(self, model_grads, master_grads, unused_scale, models_are_masters=False):\n if self._has_overflow:\n return\n\n scale = self._loss_scale\n\n if scale == 1.0 and models_are_masters and not self.dynamic:\n return\n\n if LossScaler.has_fused_kernel:\n # if (not LossScaler.warned_unscaling_non_fp32_grad\n # and master_grads[0].dtype == torch.float16):\n # print(\"Warning: unscaling grads that are not FP32. \"\n # \"Unscaling non-fp32 grads may indicate an error. \"\n # \"When using Amp, you don't need to call .half() on your model.\")\n # # Setting this to True unconditionally allows the possibility of an escape\n # # if never-before-seen non-fp32 grads are created in some later iteration.\n # LossScaler.warned_unscaling_non_fp32_grad = True\n multi_tensor_applier(LossScaler.multi_tensor_scale_cuda,\n self._overflow_buf,\n [model_grads, master_grads],\n 1./scale)\n else:\n self.unscale_python(model_grads, master_grads, scale)\n\n # Defer to update_scale\n # If the fused kernel is available, we only need one D2H memcopy and sync.\n # if LossScaler.has_fused_kernel and self.dynamic and not self._has_overflow:\n # self._has_overflow = self._overflow_buf.item()\n\n def unscale_with_stashed_python(self,\n model_grads,\n stashed_master_grads,\n master_grads,\n scale):\n for model, stashed, master in zip(model_grads, stashed_master_grads, master_grads):\n if model is None and stashed is None:\n continue\n else:\n if not LossScaler.warned_unscaling_non_fp32_grad:\n if master.dtype != torch.float32:\n maybe_print(\n \"Attempting to unscale a grad with type {} \".format(master.type()) +\n \"Unscaling non-fp32 grads may indicate an error. \"\n \"When using Amp, you don't need to call .half() on your model.\")\n LossScaler.warned_unscaling_non_fp32_grad = True\n self._has_overflow = axpby_check_overflow_python(model,\n stashed,\n master,\n 1./scale,\n self.dynamic)\n if self._has_overflow and self.dynamic:\n break\n\n def unscale_with_stashed(self,\n model_grads,\n stashed_master_grads,\n master_grads):\n if self._has_overflow:\n return\n\n scale = self._loss_scale\n\n if LossScaler.has_fused_kernel:\n if (not LossScaler.warned_unscaling_non_fp32_grad\n and master_grads[0].dtype == torch.float16):\n print(\"Warning: unscaling grads that are not FP32. \"\n \"Unscaling non-fp32 grads may indicate an error. \"\n \"When using Amp, you don't need to call .half() on your model.\")\n # Setting this to True unconditionally allows the possibility of an escape\n # if never-before-seen non-fp32 grads are created in some later iteration.\n LossScaler.warned_unscaling_non_fp32_grad = True\n multi_tensor_applier(LossScaler.multi_tensor_axpby_cuda,\n self._overflow_buf,\n [model_grads, stashed_master_grads, master_grads],\n 1./scale,\n 1.0,\n 0) # check only arg 0, aka the incoming model grads, for infs\n else:\n self.unscale_with_stashed_python(model_grads,\n stashed_master_grads,\n master_grads,\n scale)\n\n # Defer to update_scale\n # If the fused kernel is available, we only need one D2H memcopy and sync.\n # if LossScaler.has_fused_kernel and self.dynamic and not self._has_overflow:\n # self._has_overflow = self._overflow_buf.item()\n\n def clear_overflow_state(self):\n self._has_overflow = False\n if self.has_fused_kernel:\n self._overflow_buf.zero_()\n\n # Separate so unscale() can be called more that once before updating.\n def update_scale(self):\n # If the fused kernel is available, we only need one D2H memcopy and sync.\n if LossScaler.has_fused_kernel and self.dynamic and not self._has_overflow:\n self._has_overflow = self._overflow_buf.item()\n\n if self._has_overflow and self.dynamic:\n should_skip = True\n if(self._min_loss_scale):\n self._loss_scale = max(self._min_loss_scale, self._loss_scale/2.)\n else:\n self._loss_scale = self._loss_scale/2.\n self._unskipped = 0\n else:\n should_skip = False\n self._unskipped += 1\n\n if self._unskipped == self._scale_seq_len and self.dynamic:\n self._loss_scale = min(self._max_loss_scale, self._loss_scale*2.)\n self._unskipped = 0\n\n return should_skip\n" }, { "path": "apex/apex/amp/utils.py", "content": "from . import compat\n\nimport functools\nimport itertools\n\nimport torch\n\ndef get_cuda_version():\n return tuple(int(x) for x in torch.version.cuda.split('.'))\n\ndef is_fp_tensor(x):\n if is_nested(x):\n # Fast-fail version of all(is_fp_tensor)\n for y in x:\n if not is_fp_tensor(y):\n return False\n return True\n return compat.is_tensor_like(x) and compat.is_floating_point(x)\n\ndef is_nested(x):\n return isinstance(x, tuple) or isinstance(x, list)\n\ndef should_cache(x):\n if is_nested(x):\n # Fast-fail version of all(should_cache)\n for y in x:\n if not should_cache(y):\n return False\n return True\n return isinstance(x, torch.nn.parameter.Parameter) and \\\n type_string(x) == 'FloatTensor'\n\ndef collect_fp_tensor_types(args, kwargs):\n def collect_types(x, types):\n if is_nested(x):\n for y in x:\n collect_types(y, types)\n else:\n types.add(type_string(x))\n\n all_args = itertools.chain(args, kwargs.values())\n types = set()\n for x in all_args:\n if is_fp_tensor(x):\n collect_types(x, types)\n return types\n\ndef type_string(x):\n return x.type().split('.')[-1]\n\ndef maybe_half(x, name='', verbose=False):\n if is_nested(x):\n return type(x)([maybe_half(y) for y in x])\n\n if not x.is_cuda or type_string(x) == 'HalfTensor':\n return x\n else:\n if verbose:\n print('Float->Half ({})'.format(name))\n return x.half()\n\ndef maybe_float(x, name='', verbose=False):\n if is_nested(x):\n return type(x)([maybe_float(y) for y in x])\n\n if not x.is_cuda or type_string(x) == 'FloatTensor':\n return x\n else:\n if verbose:\n print('Half->Float ({})'.format(name))\n return x.float()\n\n# NB: returneds casted `args`, mutates `kwargs` in-place\ndef casted_args(cast_fn, args, kwargs):\n new_args = []\n for x in args:\n if is_fp_tensor(x):\n new_args.append(cast_fn(x))\n else:\n new_args.append(x)\n for k in kwargs:\n val = kwargs[k]\n if is_fp_tensor(val):\n kwargs[k] = cast_fn(val)\n return new_args\n\ndef cached_cast(cast_fn, x, cache):\n if is_nested(x):\n return type(x)([cached_cast(y) for y in x])\n if x in cache:\n cached_x = cache[x]\n if x.requires_grad and cached_x.requires_grad:\n # Make sure x is actually cached_x's autograd parent.\n if cached_x.grad_fn.next_functions[1][0].variable is not x:\n raise RuntimeError(\"x and cache[x] both require grad, but x is not \"\n \"cache[x]'s parent. This is likely an error.\")\n # During eval, it's possible to end up caching casted weights with\n # requires_grad=False. On the next training iter, if cached_x is found\n # and reused from the cache, it will not actually have x as its parent.\n # Therefore, we choose to invalidate the cache (and force refreshing the cast)\n # if x.requires_grad and cached_x.requires_grad do not match.\n #\n # During eval (i.e. running under with torch.no_grad()) the invalidation\n # check would cause the cached value to be dropped every time, because\n # cached_x would always be created with requires_grad=False, while x would\n # still have requires_grad=True. This would render the cache effectively\n # useless during eval. Therefore, if we are running under the no_grad()\n # context manager (torch.is_grad_enabled=False) we elide the invalidation\n # check, and use the cached value even though its requires_grad flag doesn't\n # match. During eval, we don't care that there's no autograd-graph\n # connection between x and cached_x.\n if torch.is_grad_enabled() and x.requires_grad != cached_x.requires_grad:\n del cache[x]\n else:\n return cached_x\n\n casted_x = cast_fn(x)\n cache[x] = casted_x\n return casted_x\n\ndef verbosify(cast_fn, fn_name, verbose):\n if verbose:\n return functools.partial(cast_fn, name=fn_name, verbose=verbose)\n else:\n return cast_fn\n\ndef as_inplace(fns):\n for x in fns:\n yield x + '_'\n\ndef has_func(mod, fn):\n if isinstance(mod, torch.nn.backends.backend.FunctionBackend):\n return fn in mod.function_classes\n elif isinstance(mod, dict):\n return fn in mod\n else:\n return hasattr(mod, fn)\n\ndef get_func(mod, fn):\n if isinstance(mod, torch.nn.backends.backend.FunctionBackend):\n return mod.function_classes[fn]\n elif isinstance(mod, dict):\n return mod[fn]\n else:\n return getattr(mod, fn)\n\ndef set_func(mod, fn, new_fn):\n if isinstance(mod, torch.nn.backends.backend.FunctionBackend):\n mod.function_classes[fn] = new_fn\n elif isinstance(mod, dict):\n mod[fn] = new_fn\n else:\n setattr(mod, fn, new_fn)\n\ndef set_func_save(handle, mod, fn, new_fn):\n cur_fn = get_func(mod, fn)\n handle._save_func(mod, fn, cur_fn)\n set_func(mod, fn, new_fn)\n\n# A couple problems get solved here:\n# - The flat_weight buffer is disconnected from autograd graph,\n# so the fp16 weights need to be derived from the input weights\n# to this forward call, not the flat buffer.\n# - The ordering of weights in the flat buffer is...idiosyncratic.\n# First problem is solved with combination of set_ (to set up\n# correct storage) and copy_ (so the fp16 weight derives from the\n# fp32 one in autograd.\n# Second is solved by doing ptr arithmetic on the fp32 weights\n# to derive the correct offset.\n#\n# TODO: maybe this should actually use\n# `torch._cudnn_rnn_flatten_weight`? But then I need to call\n# on first iter and cache the right offsets. Ugh.\ndef synthesize_flattened_rnn_weights(fp32_weights,\n fp16_flat_tensor,\n rnn_fn='',\n verbose=False):\n fp16_weights = []\n fp32_base_ptr = fp32_weights[0][0].data_ptr()\n for layer_weights in fp32_weights:\n fp16_layer_weights = []\n for w_fp32 in layer_weights:\n w_fp16 = w_fp32.new().half()\n offset = (w_fp32.data_ptr() - fp32_base_ptr) // w_fp32.element_size()\n w_fp16.set_(fp16_flat_tensor.storage(),\n offset,\n w_fp32.shape)\n w_fp16.copy_(w_fp32)\n if verbose:\n print('Float->Half ({})'.format(rnn_fn))\n fp16_layer_weights.append(w_fp16)\n fp16_weights.append(fp16_layer_weights)\n return fp16_weights\n\n# Roughly same as above, just the `fp32_weights` aren't nested.\n# Code kept separate for readability.\ndef new_synthesize_flattened_rnn_weights(fp32_weights,\n fp16_flat_tensor,\n rnn_fn='',\n verbose=False):\n fp16_weights = []\n fp32_base_ptr = fp32_weights[0].data_ptr()\n for w_fp32 in fp32_weights:\n w_fp16 = w_fp32.new().half()\n offset = (w_fp32.data_ptr() - fp32_base_ptr) // w_fp32.element_size()\n w_fp16.set_(fp16_flat_tensor.storage(),\n offset,\n w_fp32.shape)\n w_fp16.copy_(w_fp32)\n if verbose:\n print('Float->Half ({})'.format(rnn_fn))\n fp16_weights.append(w_fp16)\n return fp16_weights\n" }, { "path": "apex/apex/amp/wrap.py", "content": "from . import compat\nfrom . import utils\nfrom ._amp_state import _amp_state\nfrom . import rnn_compat\n\nimport functools\n\nimport torch\n\ndef make_cast_wrapper(orig_fn, cast_fn, handle,\n try_caching=False):\n @functools.wraps(orig_fn)\n def wrapper(*args, **kwargs):\n if not handle.is_active():\n return orig_fn(*args, **kwargs)\n\n if try_caching and handle.has_cache:\n args = list(args)\n for i in range(len(args)):\n if utils.should_cache(args[i]):\n args[i] = utils.cached_cast(cast_fn, args[i], handle.cache)\n for k in kwargs:\n if utils.should_cache(kwargs[k]):\n kwargs[k] = utils.cached_cast(cast_fn, kwargs[k], handle.cache)\n new_args = utils.casted_args(cast_fn,\n args,\n kwargs)\n return orig_fn(*new_args, **kwargs)\n return wrapper\n\ndef cached_cast(mod, fn, cast_fn, handle,\n try_caching=False, verbose=False):\n if not utils.has_func(mod, fn):\n return\n\n orig_fn = utils.get_func(mod, fn)\n cast_fn = utils.verbosify(cast_fn, fn, verbose)\n wrapper = make_cast_wrapper(orig_fn, cast_fn, handle, try_caching)\n utils.set_func_save(handle, mod, fn, wrapper)\n\n# `handle` arg is unused, but simplifies API to make `make_cast_wrapper`\n# Annoyingly, make_promote_wrapper still uses the global handle. Once everyone\n# is on the new API and I am free to get rid of handle, I can clean this up.\ndef make_promote_wrapper(orig_fn, cast_fn, handle=None):\n @functools.wraps(orig_fn)\n def wrapper(*args, **kwargs):\n if not _amp_state.handle.is_active():\n return orig_fn(*args, **kwargs)\n\n types = utils.collect_fp_tensor_types(args, kwargs)\n\n if len(types) <= 1:\n return orig_fn(*args, **kwargs)\n elif len(types) == 2 and types == set(['HalfTensor', 'FloatTensor']):\n new_args = utils.casted_args(cast_fn,\n args,\n kwargs)\n return orig_fn(*new_args, **kwargs)\n else:\n raise NotImplementedError('Do not know how to handle ' +\n 'these types to promote: {}'\n .format(types))\n return wrapper\n\ndef promote(mod, fn, handle, verbose=False):\n orig_fn = utils.get_func(mod, fn)\n maybe_float = utils.verbosify(utils.maybe_float, fn, verbose)\n wrapper = make_promote_wrapper(orig_fn, maybe_float)\n utils.set_func_save(handle, mod, fn, wrapper)\n\ndef sequence_promote(mod, fn, handle, verbose=False):\n orig_fn = utils.get_func(mod, fn)\n maybe_float = utils.verbosify(utils.maybe_float, fn, verbose)\n @functools.wraps(orig_fn)\n def wrapper(seq, *args, **kwargs):\n if not _amp_state.handle.is_active():\n return orig_fn(seq, *args, **kwargs)\n\n types = set([utils.type_string(x) for x in seq])\n if len(types) <= 1:\n return orig_fn(seq, *args, **kwargs)\n elif types == set(['HalfTensor', 'FloatTensor']):\n cast_seq = utils.casted_args(maybe_float,\n seq, {})\n return orig_fn(cast_seq, *args, **kwargs)\n else:\n # TODO: other mixed-type cases aren't due to amp.\n # Just pass through?\n return orig_fn(seq, *args, **kwargs)\n utils.set_func_save(handle, mod, fn, wrapper)\n\ndef promote_match_arg0(mod, fn, handle, verbose=False):\n if not utils.has_func(mod, fn):\n return\n\n orig_fn = utils.get_func(mod, fn)\n @functools.wraps(orig_fn)\n def wrapper(arg0, *args, **kwargs):\n assert compat.is_tensor_like(arg0)\n if not _amp_state.handle.is_active():\n return orig_fn(arg0, *args, **kwargs)\n\n if utils.type_string(arg0) == 'HalfTensor':\n cast_fn = utils.maybe_half\n elif utils.type_string(arg0) == 'FloatTensor':\n cast_fn = utils.maybe_float\n else:\n return orig_fn(arg0, *args, **kwargs)\n cast_fn = utils.verbosify(cast_fn, fn, verbose)\n new_args = utils.casted_args(cast_fn, args, kwargs)\n return orig_fn(arg0, *new_args, **kwargs)\n utils.set_func_save(handle, mod, fn, wrapper)\n\ndef err_if_any_half(mod, fn, handle, custom_err_msg=None):\n if not utils.has_func(mod, fn):\n return\n\n orig_fn = utils.get_func(mod, fn)\n @functools.wraps(orig_fn)\n def wrapper(*args, **kwargs):\n types = utils.collect_fp_tensor_types(args, kwargs)\n if 'HalfTensor' in types:\n if custom_err_msg:\n raise NotImplementedError(custom_err_msg)\n else:\n raise NotImplementedError('Cannot call in-place function ' +\n '{} with fp16 arguments.'.format(fn))\n else:\n return orig_fn(*args, **kwargs)\n utils.set_func_save(handle, mod, fn, wrapper)\n\ndef err_if_arg0_half(mod, fn, handle, verbose=False):\n if not utils.has_func(mod, fn):\n return\n\n orig_fn = utils.get_func(mod, fn)\n @functools.wraps(orig_fn)\n def wrapper(arg0, *args, **kwargs):\n assert compat.is_tensor_like(arg0)\n if utils.type_string(arg0) == 'HalfTensor':\n raise NotImplementedError('Cannot call in-place method ' +\n '{} on fp16 Tensors.'.format(fn))\n else:\n cast_fn = utils.verbosify(utils.maybe_float, fn, verbose)\n new_args = utils.casted_args(cast_fn, args, kwargs)\n return orig_fn(arg0, *new_args, **kwargs)\n utils.set_func_save(handle, mod, fn, wrapper)\n\n# Current RNN approach:\n# - Wrap top-level `RNN` function in thnn backend\n# - Will call into either CudnnRNN or AutogradRNN\n# - Each of these are factory functions that return a per-iter\n# `forward` function\n# - We interpose on the factory function to:\n# 1) Interpose on the actual forward function and put in casts\n# 2) Insert an fp16 `flat_weight` if necessary\ndef rnn_cast(backend, fn, handle, verbose=False):\n orig_rnn = utils.get_func(backend, fn)\n @functools.wraps(orig_rnn)\n def rnn_wrapper(*args, **kwargs):\n flat_weight = kwargs.get('flat_weight')\n if flat_weight is not None:\n # We replace `flat_weight` with an uninitialized fp16\n # Tensor. The \"actual\" weight tensors (provided in `forward`),\n # will then be set up as ptrs into the buffer and have the\n # corresponding fp32 values copied in.\n # We need to call `copy` on the \"actual\" weights so that the\n # autograd graph correctly backprops from the wgrads computed\n # inside cuDNN (on fp16 weights) into the fp32 weights.\n assert utils.type_string(flat_weight) == 'FloatTensor'\n if compat.tensor_is_float_tensor() or compat.tensor_is_variable():\n # Pre-0.4. A little slower, since it zeros out memory.\n flat_weight_fp16 = flat_weight.new().half().resize_(flat_weight.shape)\n else:\n flat_weight_fp16 = torch.empty_like(flat_weight,\n dtype=torch.float16)\n kwargs['flat_weight'] = flat_weight_fp16\n else:\n flat_weight_fp16 = None\n\n forward = orig_rnn(*args, **kwargs)\n @functools.wraps(forward)\n def fwd_wrapper(*fargs, **fkwargs):\n assert len(fargs) == 3 or len(fargs) == 4\n inputs, weights, hiddens = fargs[:3]\n assert utils.is_fp_tensor(inputs)\n assert isinstance(weights, list)\n cast_fn = utils.verbosify(utils.maybe_half,\n fn,\n verbose)\n new_args = []\n\n # 0) Inputs\n new_args.append(cast_fn(inputs))\n\n # 1) Weights\n if flat_weight_fp16 is not None:\n fp16_weights = utils.synthesize_flattened_rnn_weights(\n weights, flat_weight_fp16, fn, verbose)\n else:\n fp16_weights = [[cast_fn(w) for w in layer]\n for layer in weights]\n new_args.append(fp16_weights)\n\n # 2) Inputs: either a tuple (for LSTM) or single tensor\n if isinstance(hiddens, tuple):\n new_args.append(tuple(cast_fn(x) for x in hiddens))\n elif utils.is_fp_tensor(hiddens):\n new_args.append(cast_fn(hiddens))\n else:\n # Hiddens can, in principle, be `None` -- pass through\n new_args.append(hiddens)\n\n # 3) Batch sizes (0.4 or later only)\n if len(fargs) == 4:\n new_args.append(fargs[3])\n\n return forward(*new_args, **fkwargs)\n return fwd_wrapper\n utils.set_func_save(handle, backend, fn, rnn_wrapper)\n\ndef new_rnn_cast(fn, handle, verbose=False):\n # Forward+backward compatibility around https://github.com/pytorch/pytorch/pull/15744\n # For rnn backend calls that route through _rnn_impls, we must patch the ref\n # that _rnn_impls stashed. For rnn backend calls that directly invoke\n # _VF., e.g. _VF.lstm, we can patch onto VariableFunctionsShim,\n # which in turn has patched the ref named \"_VF\" in torch.nn.modules.rnn.\n if utils.has_func(torch.nn.modules.rnn._rnn_impls, fn):\n mod = torch.nn.modules.rnn._rnn_impls\n else:\n mod = torch.nn.modules.rnn._VF\n assert isinstance(mod, rnn_compat.VariableFunctionsShim)\n fn = fn.lower()\n orig_fn = utils.get_func(mod, fn)\n cast_fn = utils.verbosify(utils.maybe_half, fn, verbose)\n @functools.wraps(orig_fn)\n def wrapper(*args, **kwargs):\n # Exact call signature from modules/rnn.py\n assert len(args) == 9\n assert len(kwargs) == 0\n\n if not _amp_state.handle.is_active():\n return orig_fn(*args, **kwargs)\n\n if isinstance(args[6], bool):\n params_idx = 2 # Not PackedSequence case\n else:\n params_idx = 3 # PackedSequence case\n\n new_args = []\n for i, arg in enumerate(args):\n if i == params_idx:\n num_params = sum([x.numel() for x in arg])\n fp16_weight_buf = args[0].new_empty((num_params,),\n dtype=torch.half)\n casted_weights = utils.new_synthesize_flattened_rnn_weights(\n arg, fp16_weight_buf, fn, verbose)\n new_args.append(casted_weights)\n elif utils.is_fp_tensor(arg):\n new_args.append(cast_fn(arg))\n else:\n new_args.append(arg)\n\n return orig_fn(*new_args)\n utils.set_func_save(handle, mod, fn, wrapper)\n\ndef disable_casts(mod, fn, handle):\n if not utils.has_func(mod, fn):\n return\n\n orig_fn = utils.get_func(mod, fn)\n @functools.wraps(orig_fn)\n def wrapper(*args, **kwargs):\n with handle._disable_casts():\n return orig_fn(*args, **kwargs)\n utils.set_func_save(handle, mod, fn, wrapper)\n" }, { "path": "apex/apex/fp16_utils/README.md", "content": "fp16_optimizer.py contains `FP16_Optimizer`, a Python class designed to wrap an existing Pytorch optimizer and automatically enable master parameters and loss scaling in a manner transparent to the user. To use `FP16_Optimizer`, only two lines of one's Python model need to change.\n\n#### [FP16_Optimizer API documentation](https://nvidia.github.io/apex/fp16_utils.html#automatic-management-of-master-params-loss-scaling)\n\n#### [Simple examples with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/FP16_Optimizer_simple)\n\n#### [Imagenet with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/imagenet)\n\n#### [word_language_model with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/word_language_model)\n\n\nfp16_util.py contains a number of utilities to manually manage master parameters and loss scaling, if the user chooses. \n\n#### [Manual management documentation](https://nvidia.github.io/apex/fp16_utils.html#manual-master-parameter-management)\n\nThe [Imagenet with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/imagenet) and [word_language_model with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/word_language_model) directories also contain `main.py` files that demonstrate manual management of master parameters and static loss scaling. These examples illustrate what sort of operations `FP16_Optimizer` is performing automatically.\n" }, { "path": "apex/apex/fp16_utils/__init__.py", "content": "from .fp16util import (\n BN_convert_float,\n network_to_half,\n prep_param_lists,\n model_grads_to_master_grads,\n master_params_to_model_params,\n tofp16,\n to_python_float,\n clip_grad_norm,\n convert_module,\n convert_network,\n FP16Model,\n)\n\nfrom .fp16_optimizer import FP16_Optimizer\nfrom .loss_scaler import LossScaler, DynamicLossScaler\n" }, { "path": "apex/apex/fp16_utils/fp16_optimizer.py", "content": "import torch\r\nfrom torch import nn\r\nfrom torch.autograd import Variable\r\nfrom torch.nn.parameter import Parameter\r\nfrom torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors\r\n\r\nfrom ..amp._amp_state import _amp_state, maybe_print\r\nfrom ..amp.scaler import LossScaler\r\nfrom ..multi_tensor_apply import multi_tensor_applier\r\nfrom .fp16util import model_grads_to_master_grads, master_params_to_model_params, clip_grad_norm\r\n\r\n# TODO: Update overflow check + downscale to use Carl's fused kernel.\r\nclass FP16_Optimizer(object):\r\n \"\"\"\r\n :class:`FP16_Optimizer` is designed to wrap an existing PyTorch optimizer, \r\n and manage static or dynamic loss scaling and master weights in a manner transparent to the user.\r\n For standard use, only two lines must be changed: creating the :class:`FP16_Optimizer` instance,\r\n and changing the call to ``backward``.\r\n\r\n Example::\r\n\r\n model = torch.nn.Linear(D_in, D_out).cuda().half()\r\n optimizer = torch.optim.SGD(model.parameters(), lr=1e-3)\r\n # Name the FP16_Optimizer instance to replace the existing optimizer\r\n # (recommended but not required):\r\n optimizer = FP16_Optimizer(optimizer, static_loss_scale = 128.0)\r\n ...\r\n # loss.backward() becomes:\r\n optimizer.backward(loss)\r\n ...\r\n\r\n Example with dynamic loss scaling::\r\n\r\n ...\r\n optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)\r\n # optional arg to control dynamic loss scaling behavior\r\n # dynamic_loss_args={'scale_window' : 500})\r\n # Usually, dynamic_loss_args is not necessary. \r\n\r\n Args:\r\n init_optimizer (torch.optim.optimizer): Existing optimizer created with the parameters to optimize. Internally, :class:`FP16_Optimizer` replaces the passed optimizer's fp16 parameters, if any, with fp32 master parameters copied from the original ones. :class:`FP16_Optimizer` also stores references to the original fp16 parameters, and updates these fp16 parameters from the master fp32 copy at the end of each :attr:`step`. \r\n static_loss_scale (float, optional, default=1.0): Loss scale used internally to scale gradients computed by the model. Any fp16 gradients will be copied to fp32, then downscaled before being applied to the fp32 master params, so ``static_loss_scale`` should not affect learning rate.\r\n dynamic_loss_scale (bool, optional, default=False): Use dynamic loss scaling. If True, this will override any ``static_loss_scale`` option.\r\n dynamic_loss_args (dict, optional, default=None): Dict of kwargs that will be forwarded to the internal :class:`LossScaler` instance's constructor. Keys of this dict must match kwargs accepted by :class:`LossScaler`'s constructor. If ``dynamic_loss_args`` is unspecified, :class:`LossScaler`'s defaults will be used.\r\n verbose (bool, optional, default=True): By default, FP16_Optimizer's constructor prints out the parameters and parameter groups it is ingesting, as a sanity check. If this becomes annoying (e.g. for large models), it can be disabled by passing ``verbose=False``. ``verbose=False`` will not disable printing when the loss scale is readjusted during dynamic loss scaling.\r\n\r\n ``init_optimizer`` is expected to have been constructed in the ordinary way. \r\n It is recommended (although not required) that the newly constructed :class:`FP16_Optimizer` instance be \r\n named to replace ``init_optimizer``, for two reasons: \r\n First, it means that references to the same name\r\n later in the file will not have to change. \r\n Second, :class:`FP16_Optimizer` reserves the right (as an implementation detail) to \r\n modify ``init_optimizer``. If you do choose a unique name for the new\r\n :class:`FP16_Optimizer` instance, you should only work with this new instance,\r\n because the preexisting optimizer might no longer behave as expected.\r\n\r\n ``init_optimizer`` may be any Pytorch optimizer. \r\n It may contain a mixture of fp16 and fp32 parameters organized into any number of \r\n ``param_groups`` with different hyperparameters. The :class:`FP16_Optimizer` constructor will \r\n ingest these ``param_groups`` and remember them. \r\n\r\n Calls to ::\r\n\r\n loss.backward() \r\n\r\n must be replaced with ::\r\n\r\n optimizer.backward(loss) \r\n\r\n because :class:`FP16_Optimizer` requires ownership of the backward pass to implement \r\n loss scaling and copies to master gradients.\r\n\r\n .. note::\r\n Loss scaling, either static or dynamic, is orthogonal to learning rate, because gradients\r\n are downscaled before being applied. This means that adjusting the loss scale, or using\r\n dynamic loss scaling, should not require retuning the learning rate or any other \r\n hyperparameters.\r\n\r\n\r\n **Advanced options**\r\n\r\n **Closures**: :class:`FP16_Optimizer` can wrap a Pytorch optimizer that receives a closure.\r\n See docstring for :attr:`step`.\r\n\r\n **Gradient clipping**: Use :attr:`clip_master_grads`.\r\n \r\n **Multiple losses**: If your model accumulates gradients from multiple losses,\r\n this can be made more efficient by supplying ``update_master_grads=False``\r\n to :attr:`backward`. See docstring for :attr:`backward`.\r\n\r\n **Manually adjusting loss scale**: The current loss scale can be retrieved or set via ::\r\n\r\n print(optimizer.loss_scale)\r\n optimizer.loss_scale = new_loss_scale\r\n\r\n For static loss scaling, manually adjusting the loss scale over time is a reasonable\r\n thing to do. During later epochs, gradients may become smaller, and a \r\n higher loss scale may be required, analogous to scheduling the learning rate. Dynamic loss\r\n scaling is more subtle (see :class:`DynamicLossScaler`) and in this case, manually adjusting \r\n the loss scale is not recommended.\r\n\r\n **Multi_GPU training**: If the wrapped ``init_optimizer`` was created from a model wrapped in\r\n Pytorch DistributedDataParallel or Apex DistributedDataParallel, :class:`FP16_Optimizer` \r\n should still work as intended.\r\n \"\"\"\r\n\r\n def __init__(self, \r\n init_optimizer, \r\n static_loss_scale=1.0, \r\n dynamic_loss_scale=False,\r\n dynamic_loss_args=None,\r\n verbose=True):\r\n if not torch.cuda.is_available:\r\n raise SystemError(\"Cannot use fp16 without CUDA.\")\r\n\r\n self.verbose = verbose\r\n\r\n self.optimizer = init_optimizer\r\n # init_state_dict sets up an alternative way to cast per-param state tensors.\r\n # Stashing here in case https://github.com/pytorch/pytorch/issues/7733 makes it necessary.\r\n # init_state_dict = init_optimizer.state_dict()\r\n\r\n self.fp16_groups = []\r\n self.fp32_from_fp16_groups = []\r\n self.fp32_from_fp32_groups = []\r\n for i, param_group in enumerate(self.optimizer.param_groups):\r\n self.maybe_print(\"FP16_Optimizer processing param group {}:\".format(i))\r\n fp16_params_this_group = []\r\n fp32_params_this_group = []\r\n fp32_from_fp16_params_this_group = []\r\n for i, param in enumerate(param_group['params']):\r\n if param.requires_grad:\r\n if param.type() == 'torch.cuda.HalfTensor':\r\n self.maybe_print(\"FP16_Optimizer received torch.cuda.HalfTensor with {}\"\r\n .format(param.size()))\r\n fp16_params_this_group.append(param)\r\n master_param = param.detach().clone().float()\r\n master_param.requires_grad = True\r\n param_group['params'][i] = master_param\r\n fp32_from_fp16_params_this_group.append(master_param)\r\n # Reset existing state dict key to the new master param.\r\n # We still need to recast per-param state tensors, if any, to FP32.\r\n if param in self.optimizer.state:\r\n self.optimizer.state[master_param] = self.optimizer.state.pop(param) \r\n elif param.type() == 'torch.cuda.FloatTensor':\r\n self.maybe_print(\"FP16_Optimizer received torch.cuda.FloatTensor with {}\"\r\n .format(param.size()))\r\n fp32_params_this_group.append(param)\r\n param_group['params'][i] = param\r\n else:\r\n raise TypeError(\"Wrapped parameters must be either \"\r\n \"torch.cuda.FloatTensor or torch.cuda.HalfTensor. \" \r\n \"Received {}\".format(param.type()))\r\n \r\n self.fp16_groups.append(fp16_params_this_group)\r\n self.fp32_from_fp16_groups.append(fp32_from_fp16_params_this_group)\r\n self.fp32_from_fp32_groups.append(fp32_params_this_group)\r\n\r\n self.all_fp16_params = []\r\n for group in self.fp16_groups:\r\n self.all_fp16_params += group\r\n\r\n self.all_fp32_from_fp16_params = []\r\n for group in self.fp32_from_fp16_groups:\r\n self.all_fp32_from_fp16_params += group\r\n\r\n self.all_fp32_from_fp32_params = []\r\n for group in self.fp32_from_fp32_groups:\r\n self.all_fp32_from_fp32_params += group\r\n\r\n # Leverage state_dict() and load_state_dict() to recast preexisting per-param state tensors\r\n self.optimizer.load_state_dict(self.optimizer.state_dict())\r\n # alternative way to cast per-param state tensors:\r\n # self.optimizer.load_state_dict(init_state_dict)\r\n\r\n if dynamic_loss_scale:\r\n self.dynamic_loss_scale = True\r\n if dynamic_loss_args is not None:\r\n self.loss_scaler = LossScaler(\"dynamic\", **dynamic_loss_args)\r\n else:\r\n self.loss_scaler = LossScaler(\"dynamic\")\r\n else:\r\n self.dynamic_loss_scale = False\r\n self.loss_scaler = LossScaler(static_loss_scale)\r\n\r\n self.overflow = False\r\n self.first_closure_call_this_step = True\r\n\r\n self.clip_grad_norm = clip_grad_norm\r\n\r\n # TODO: Centralize exposure and import error checking for the C backend.\r\n if multi_tensor_applier.available:\r\n import amp_C\r\n self.multi_tensor_scale = amp_C.multi_tensor_scale\r\n self._dummy_overflow_buf = torch.cuda.IntTensor([0]);\r\n\r\n # Having self.maybe_print distinct from _amp_state.maybe_print is another artifact\r\n # of having to support FP16_Optimizer separately, for the time being.\r\n def maybe_print(self, msg):\r\n if self.verbose:\r\n print(msg)\r\n \r\n def __getstate__(self):\r\n raise RuntimeError(\"FP16_Optimizer should be serialized using state_dict().\")\r\n\r\n def __setstate__(self, state):\r\n raise RuntimeError(\"FP16_Optimizer should be deserialized using load_state_dict().\")\r\n\r\n def zero_grad(self, set_grads_to_None=False):\r\n \"\"\"\r\n Zero fp32 and fp16 parameter grads.\r\n \"\"\"\r\n # In principle, only the .grad attributes of the model params need to be zeroed,\r\n # because gradients are copied into the FP32 master params. However, we zero\r\n # all gradients owned by the optimizer, just to be safe:\r\n for group in self.optimizer.param_groups:\r\n for p in group['params']:\r\n if set_grads_to_None:\r\n p.grad = None\r\n else:\r\n if p.grad is not None:\r\n p.grad.detach_()\r\n p.grad.zero_()\r\n\r\n # Zero fp16 gradients owned by the model:\r\n for fp16_group in self.fp16_groups:\r\n for param in fp16_group:\r\n if set_grads_to_None:\r\n param.grad = None\r\n else:\r\n if param.grad is not None:\r\n param.grad.detach_() # as in torch.optim.optimizer.zero_grad()\r\n param.grad.zero_()\r\n\r\n # Should not be used anymore.\r\n # def _check_overflow(self):\r\n # params = []\r\n # for group in self.fp16_groups:\r\n # for param in group:\r\n # params.append(param)\r\n # for group in self.fp32_from_fp32_groups:\r\n # for param in group:\r\n # params.append(param)\r\n # self.overflow = self.loss_scaler.has_overflow(params)\r\n\r\n # def _update_scale(self, has_overflow=False):\r\n # self.loss_scaler.update_scale(has_overflow)\r\n\r\n def _master_params_to_model_params(self):\r\n if multi_tensor_applier.available:\r\n if len(self.all_fp16_params) > 0:\r\n multi_tensor_applier(\r\n self.multi_tensor_scale,\r\n self._dummy_overflow_buf,\r\n [self.all_fp32_from_fp16_params, self.all_fp16_params],\r\n 1.0)\r\n else:\r\n for fp16_group, fp32_from_fp16_group in zip(self.fp16_groups, self.fp32_from_fp16_groups):\r\n master_params_to_model_params(fp16_group, fp32_from_fp16_group)\r\n\r\n # To consider: Integrate distributed with this wrapper by registering a hook on each variable\r\n # that does the overflow check, gradient copy + downscale, and fp32 allreduce in a different stream.\r\n # def _model_grads_to_master_grads(self):\r\n # for fp16_group, fp32_from_fp16_group in zip(self.fp16_groups, self.fp32_from_fp16_groups):\r\n # model_grads_to_master_grads(fp16_group, fp32_from_fp16_group)\r\n\r\n # def _downscale_master(self):\r\n # if self.loss_scale != 1.0:\r\n # for group in self.optimizer.param_groups:\r\n # for param in group['params']:\r\n # if param.grad is not None:\r\n # param.grad.data.mul_(1./self.loss_scale)\r\n\r\n def clip_master_grads(self, max_norm, norm_type=2):\r\n \"\"\"\r\n Clips fp32 master gradients via ``torch.nn.utils.clip_grad_norm``.\r\n\r\n Args:\r\n max_norm (float or int): max norm of the gradients\r\n norm_type (float or int): type of the used p-norm. Can be ``'inf'`` for\r\n infinity norm.\r\n\r\n Returns:\r\n Total norm of the current fp32 gradients (viewed as a single vector).\r\n\r\n .. warning::\r\n Returns -1 if the most recently computed fp16 gradients overflowed (that is, if ``self.overflow`` is ``True``).\r\n \"\"\"\r\n if not self.overflow:\r\n fp32_params = []\r\n for param_group in self.optimizer.param_groups:\r\n for param in param_group['params']:\r\n fp32_params.append(param)\r\n return self.clip_grad_norm(fp32_params, max_norm, norm_type)\r\n else:\r\n return -1\r\n\r\n def state_dict(self):\r\n \"\"\"\r\n Returns a dict containing the current state of this :class:`FP16_Optimizer` instance.\r\n This dict contains attributes of :class:`FP16_Optimizer`, as well as the state_dict\r\n of the contained Pytorch optimizer.\r\n Example::\r\n\r\n checkpoint = {}\r\n checkpoint['model'] = model.state_dict()\r\n checkpoint['optimizer'] = optimizer.state_dict()\r\n torch.save(checkpoint, \"saved.pth\")\r\n \"\"\"\r\n state_dict = {}\r\n state_dict['loss_scaler'] = self.loss_scaler\r\n state_dict['dynamic_loss_scale'] = self.dynamic_loss_scale\r\n state_dict['overflow'] = self.overflow\r\n state_dict['first_closure_call_this_step'] = self.first_closure_call_this_step\r\n state_dict['optimizer_state_dict'] = self.optimizer.state_dict()\r\n state_dict['fp32_from_fp16'] = self.fp32_from_fp16_groups\r\n return state_dict\r\n\r\n def load_state_dict(self, state_dict):\r\n \"\"\"\r\n Loads a state_dict created by an earlier call to state_dict(). \r\n If ``fp16_optimizer_instance`` was constructed from some ``init_optimizer``, \r\n whose parameters in turn came from ``model``, it is expected that the user \r\n will call ``model.load_state_dict()`` before\r\n ``fp16_optimizer_instance.load_state_dict()`` is called.\r\n\r\n Example::\r\n\r\n model = torch.nn.Linear(D_in, D_out).cuda().half()\r\n optimizer = torch.optim.SGD(model.parameters(), lr=1e-3)\r\n optimizer = FP16_Optimizer(optimizer, static_loss_scale = 128.0)\r\n ...\r\n checkpoint = torch.load(\"saved.pth\")\r\n model.load_state_dict(checkpoint['model'])\r\n optimizer.load_state_dict(checkpoint['optimizer'])\r\n \"\"\"\r\n # I think it should actually be ok to reload the optimizer before the model.\r\n self.loss_scaler = state_dict['loss_scaler']\r\n self.dynamic_loss_scale = state_dict['dynamic_loss_scale']\r\n self.overflow = state_dict['overflow']\r\n self.first_closure_call_this_step = state_dict['first_closure_call_this_step']\r\n self.optimizer.load_state_dict(state_dict['optimizer_state_dict'])\r\n # At this point, the optimizer's references to the model's fp32 parameters are up to date.\r\n # The optimizer's hyperparameters and internal buffers are also up to date. \r\n # However, the fp32 master copies of the model's fp16 params stored by the optimizer are still\r\n # out of date. There are two options. \r\n # 1: Refresh the master params from the model's fp16 params. \r\n # This requires less storage but incurs precision loss.\r\n # 2: Save and restore the fp32 master copies separately.\r\n # We choose option 2.\r\n # \r\n # Pytorch Optimizer.load_state_dict casts saved buffers (e.g. momentum) to the type and device \r\n # of their associated parameters, because it's possible those buffers might not exist yet in \r\n # the current optimizer instance. In our case, as long as the current FP16_Optimizer has been \r\n # constructed in the same way as the one whose state_dict we are loading, the same master params\r\n # are guaranteed to exist, so we can just copy_() from the saved master params.\r\n for current_group, saved_group in zip(self.fp32_from_fp16_groups, state_dict['fp32_from_fp16']):\r\n for current, saved in zip(current_group, saved_group):\r\n current.data.copy_(saved.data)\r\n\r\n def step(self, closure=None): # could add clip option.\r\n \"\"\"\r\n If no closure is supplied, :attr:`step` should be called after \r\n ``fp16_optimizer_obj.backward(loss)``.\r\n :attr:`step` updates the fp32 master copy of parameters using the optimizer supplied to\r\n :class:`FP16_Optimizer`'s constructor, then copies the updated fp32 params into the fp16 params\r\n originally referenced by :class:`FP16_Optimizer`'s constructor, so the user may immediately run\r\n another forward pass using their model.\r\n\r\n If a closure is supplied, :attr:`step` may be called without a prior call to \r\n :attr:`backward(loss)`.\r\n This control flow is identical to `ordinary Pytorch optimizer use`_ with closures.\r\n However, the user should take care that any ``loss.backward()`` call within the closure\r\n has been replaced by ``fp16_optimizer_obj.backward(loss)``.\r\n\r\n Args:\r\n closure (optional): Closure that will be supplied to the underlying optimizer originally passed to :class:`FP16_Optimizer`'s constructor. closure should call :attr:`zero_grad()` on the :class:`FP16_Optimizer` object, compute the loss, call :attr:`backward(loss)`, and return the loss.\r\n\r\n Example with closure::\r\n\r\n # optimizer is assumed to be an FP16_Optimizer object, previously constructed from an \r\n # existing pytorch optimizer.\r\n for input, target in dataset:\r\n def closure():\r\n optimizer.zero_grad()\r\n output = model(input)\r\n loss = loss_fn(output, target)\r\n # loss.backward() becomes:\r\n optimizer.backward(loss)\r\n return loss\r\n optimizer.step(closure)\r\n\r\n .. warning::\r\n Currently, calling :attr:`step` with a closure is not compatible with dynamic loss scaling.\r\n\r\n .. _`ordinary Pytorch optimizer use`:\r\n http://pytorch.org/docs/master/optim.html#optimizer-step-closure\r\n \"\"\"\r\n\r\n scale = self.loss_scaler.loss_scale()\r\n # To consider: Should this be in step(), or update_master_grads? It works either way,\r\n # but I should make it consistent with the Amp control flow, which updates the scale\r\n # during backward context manager exit.\r\n # self._update_scale(self.overflow)\r\n\r\n if self.overflow:\r\n # Using _amp_state.maybe_print instead of self.print here is intentional.\r\n maybe_print(\"Gradient overflow. Skipping step, reducing \" +\r\n \"loss scale to {}\".format(self.loss_scaler.loss_scale()))\r\n return\r\n \r\n if closure is not None:\r\n retval = self._step_with_closure(closure)\r\n else:\r\n # torch.cuda.nvtx.range_push(\"pytorch optimizer step\")\r\n retval = self.optimizer.step()\r\n # torch.cuda.nvtx.range_pop()\r\n\r\n self._master_params_to_model_params()\r\n\r\n return retval\r\n\r\n def _step_with_closure(self, closure):\r\n def wrapped_closure():\r\n # helpful for debugging\r\n # print(\"Calling wrapped_closure, first_closure_call_this_step = {}\"\r\n # .format(self.first_closure_call_this_step))\r\n if self.first_closure_call_this_step:\r\n # We expect that the fp16 params are initially fresh on entering self.step(),\r\n # so _master_params_to_model_params() is unnecessary the first time wrapped_closure()\r\n # is called within self.optimizer.step().\r\n self.first_closure_call_this_step = False\r\n else:\r\n # If self.optimizer.step() internally calls wrapped_closure more than once,\r\n # it may update the fp32 params after each call. However, self.optimizer \r\n # doesn't know about the fp16 params at all. If the fp32 params get updated,\r\n # we can't rely on self.optimizer to refresh the fp16 params. We need\r\n # to handle that manually:\r\n self._master_params_to_model_params()\r\n # Our API expects the user to give us ownership of the backward() call by\r\n # replacing all calls to loss.backward() with optimizer.backward(loss).\r\n # This requirement holds whether or not the call to backward() is made within a closure.\r\n # If the user is properly calling optimizer.backward(loss) within \"closure,\" \r\n # calling closure() here will give the fp32 master params fresh gradients\r\n # for the optimizer to play with, so all wrapped_closure needs to do is call \r\n # closure() and return the loss.\r\n temp_loss = closure() \r\n while(self.overflow):\r\n scale = self.loss_scaler.loss_scale()\r\n # self._update_scale(self.overflow) # now done at the end of backward\r\n print(\"OVERFLOW within closure! Skipping step, reducing loss scale to {}\".format(\r\n self.loss_scaler.loss_scale()))\r\n temp_loss = closure()\r\n return temp_loss\r\n\r\n retval = self.optimizer.step(wrapped_closure)\r\n\r\n self.first_closure_call_this_step = True\r\n\r\n return retval\r\n\r\n def backward(self, loss, update_master_grads=True, retain_graph=False):\r\n \"\"\" \r\n :attr:`backward` performs the following conceptual steps:\r\n\r\n 1. fp32_loss = loss.float() (see first Note below)\r\n 2. scaled_loss = fp32_loss*loss_scale\r\n 3. scaled_loss.backward(), which accumulates scaled gradients into the ``.grad`` attributes of the model's leaves (which may be fp16, fp32, or a mixture, depending how your model was defined).\r\n 4. fp16 grads are then copied to the master params' ``.grad`` attributes (see second Note), which are guaranteed to be fp32.\r\n 5. Finally, master grads are divided by loss_scale.\r\n\r\n In this way, after :attr:`backward`, the master params have fresh gradients,\r\n and :attr:`step` may be called.\r\n\r\n .. note::\r\n :attr:`backward` internally converts the loss to fp32 before applying the loss scale.\r\n This provides some additional safety against overflow if the user has supplied an \r\n fp16 loss value. \r\n However, for maximum overflow safety, the user should\r\n compute the loss criterion (MSE, cross entropy, etc) in fp32 before supplying it to \r\n :attr:`backward`.\r\n\r\n .. warning::\r\n The gradients found in a model's leaves after the call to \r\n :attr:`backward` should not be regarded as valid in general, \r\n because it's possible \r\n they have been scaled (and in the case of dynamic loss scaling, \r\n the scale factor may change over time). \r\n If the user wants to inspect gradients after a call to :attr:`backward`, \r\n only the master gradients should be regarded as valid. These can be retrieved via\r\n :attr:`inspect_master_grad_data()`.\r\n\r\n Args:\r\n loss: The loss output by the user's model. loss may be either float or half (but see first Note above).\r\n update_master_grads (bool, optional, default=True): Option to copy fp16 grads to fp32 grads on this call. By setting this to False, the user can delay the copy, which is useful to eliminate redundant fp16->fp32 grad copies if :attr:`backward` is being called on multiple losses in one iteration. If set to False, the user becomes responsible for calling :attr:`update_master_grads` before calling :attr:`step`.\r\n retain_graph (bool, optional, default=False): Forwards the usual ``retain_graph=True`` option to the internal call to ``loss.backward``. If ``retain_graph`` is being used to accumulate gradient values from multiple backward passes before calling ``optimizer.step``, passing ``update_master_grads=False`` is also recommended (see Example below).\r\n\r\n Example::\r\n\r\n # Ordinary operation:\r\n optimizer.backward(loss)\r\n\r\n # Naive operation with multiple losses (technically valid, but less efficient):\r\n # fp32 grads will be correct after the second call, but \r\n # the first call incurs an unnecessary fp16->fp32 grad copy.\r\n optimizer.backward(loss1)\r\n optimizer.backward(loss2)\r\n\r\n # More efficient way to handle multiple losses:\r\n # The fp16->fp32 grad copy is delayed until fp16 grads from all \r\n # losses have been accumulated.\r\n optimizer.backward(loss1, update_master_grads=False)\r\n optimizer.backward(loss2, update_master_grads=False)\r\n optimizer.update_master_grads()\r\n \"\"\" \r\n # To consider: try multiple backward passes using retain_grad=True to find \r\n # a loss scale that works. After you find a loss scale that works, do a final dummy\r\n # backward pass with retain_graph=False to tear down the graph. Doing this would avoid \r\n # discarding the iteration, but probably wouldn't improve overall efficiency. \r\n scaled_loss = loss.float()*self.loss_scaler.loss_scale()\r\n scaled_loss.backward(retain_graph=retain_graph)\r\n if update_master_grads:\r\n self.update_master_grads()\r\n\r\n def update_master_grads(self):\r\n # torch.cuda.nvtx.range_push(\"update_master_grads\")\r\n \"\"\"\r\n Copy the ``.grad`` attribute from stored references to fp16 parameters to \r\n the ``.grad`` attribute of the fp32 master parameters that are directly \r\n updated by the optimizer. :attr:`update_master_grads` only needs to be called if\r\n ``fp16_optimizer_obj.backward`` was called with ``update_master_grads=False``.\r\n \"\"\"\r\n # if self.dynamic_loss_scale:\r\n # self._check_overflow()\r\n # if self.overflow: return\r\n # self._model_grads_to_master_grads()\r\n # self._downscale_master()\r\n # Use the one-shot multi-tensor apply kernel\r\n self.loss_scaler.clear_overflow_state()\r\n if len(self.all_fp16_params) > 0:\r\n # print(\"Model grads before\")\r\n # print([param.grad.data for param in self.all_fp16_params])\r\n # I'm ONLY writing this as an incremental way to make some tests pass until\r\n # I can refactor the tests as well.\r\n # FP16_Optimizer should not be used by anyone.\r\n model_grads = []\r\n master_grads = []\r\n for model_param, master_param in zip(self.all_fp16_params,\r\n self.all_fp32_from_fp16_params):\r\n if model_param.grad is not None:\r\n model_grads.append(model_param.grad)\r\n if master_param.grad is None:\r\n master_param.grad = torch.empty_like(master_param)\r\n master_grads.append(master_param.grad)\r\n self.loss_scaler.unscale(\r\n model_grads,\r\n master_grads,\r\n self.loss_scaler.loss_scale())\r\n # print(\"Master grads after\")\r\n # print([param.grad.data for param in self.all_fp32_from_fp16_params])\r\n if len(self.all_fp32_from_fp32_params) > 0:\r\n model_grads = []\r\n master_grads = []\r\n for model_param, master_param in zip(self.all_fp32_from_fp32_params,\r\n self.all_fp32_from_fp32_params):\r\n if model_param.grad is not None:\r\n model_grads.append(model_param.grad)\r\n master_grads.append(master_param.grad)\r\n # print(\"Model grads before\")\r\n # print([param.grad.data for param in self.all_fp32_from_fp32_params])\r\n self.loss_scaler.unscale(\r\n model_grads,\r\n master_grads,\r\n self.loss_scaler.loss_scale())\r\n # print(\"Master grads after\")\r\n # print([param.grad.data for param in self.all_fp32_from_fp32_params])\r\n # quit()\r\n self.overflow = self.loss_scaler.update_scale()\r\n # torch.cuda.nvtx.range_pop()\r\n\r\n\r\n def inspect_master_grad_data(self):\r\n \"\"\"\r\n When running with :class:`FP16_Optimizer`, \r\n ``.grad`` attributes of a model's fp16 leaves should not be\r\n regarded as truthful, because they might be scaled. \r\n After a call to :attr:`fp16_optimizer_obj.backward(loss)`, if no overflow was encountered,\r\n the fp32 master params' ``.grad``\r\n attributes will contain valid gradients properly divided by the loss scale. However, \r\n because :class:`FP16_Optimizer` flattens some parameters, accessing them may be \r\n nonintuitive. :attr:`inspect_master_grad_data`\r\n allows those gradients to be viewed with shapes corresponding to their associated model leaves.\r\n\r\n Returns:\r\n List of lists (one list for each parameter group). The list for each parameter group\r\n is a list of the ``.grad.data`` attributes of the fp32 master params belonging to that group. \r\n \"\"\"\r\n if self.overflow:\r\n print(\"Warning: calling FP16_Optimizer.inspect_master_grad_data while in an overflow state. \"\r\n \"Gradients are currently invalid (may be inf, nan, or stale). Returning None.\")\r\n return None\r\n else:\r\n # The optimizer owns only references to master params.\r\n master_grads_data = []\r\n for param_group in self.optimizer.param_groups:\r\n master_grads_this_group = []\r\n for param in param_group['params']:\r\n if param.grad is not None:\r\n master_grads_this_group.append(param.grad.data)\r\n else:\r\n master_grads_this_group.append(None)\r\n master_grads_data.append(master_grads_this_group)\r\n return master_grads_data\r\n\r\n\r\n # Promote loss scale so it can be retrieved or set via \"fp16_optimizer_instance.loss_scale\"\r\n def _get_loss_scale(self):\r\n return self.loss_scaler.loss_scale()\r\n\r\n def _set_loss_scale(self, value):\r\n self.loss_scaler._loss_scale = value\r\n\r\n loss_scale = property(_get_loss_scale, _set_loss_scale)\r\n\r\n # Promote state so it can be retrieved or set via \"fp16_optimizer_instance.state\"\r\n def _get_state(self):\r\n return self.optimizer.state\r\n\r\n def _set_state(self, value):\r\n self.optimizer.state = value\r\n\r\n state = property(_get_state, _set_state)\r\n\r\n # Promote param_groups so it can be retrieved or set via \"fp16_optimizer_instance.param_groups\"\r\n # (for example, to adjust the learning rate)\r\n def _get_param_groups(self):\r\n return self.optimizer.param_groups\r\n\r\n def _set_param_groups(self, value):\r\n self.optimizer.param_groups = value\r\n\r\n param_groups = property(_get_param_groups, _set_param_groups)\r\n\r\n" }, { "path": "apex/apex/fp16_utils/fp16util.py", "content": "import torch\nimport torch.nn as nn\nfrom torch.autograd import Variable\nfrom torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors\n\n\nclass tofp16(nn.Module):\n \"\"\"\n Utility module that implements::\n\n def forward(self, input):\n return input.half()\n \"\"\"\n\n def __init__(self):\n super(tofp16, self).__init__()\n\n def forward(self, input):\n return input.half()\n\n\ndef BN_convert_float(module):\n \"\"\"\n Utility function for network_to_half().\n\n Retained for legacy purposes.\n \"\"\"\n if isinstance(module, torch.nn.modules.batchnorm._BatchNorm) and module.affine is True:\n module.float()\n for child in module.children():\n BN_convert_float(child)\n return module\n\n\ndef network_to_half(network):\n \"\"\"\n Convert model to half precision in a batchnorm-safe way.\n\n Retained for legacy purposes. It is recommended to use FP16Model.\n \"\"\"\n return nn.Sequential(tofp16(), BN_convert_float(network.half()))\n\n\ndef convert_module(module, dtype):\n \"\"\"\n Converts a module's immediate parameters and buffers to dtype.\n \"\"\"\n for param in module.parameters(recurse=False):\n if param is not None:\n if param.data.dtype.is_floating_point:\n param.data = param.data.to(dtype=dtype)\n if param._grad is not None and param._grad.data.dtype.is_floating_point:\n param._grad.data = param._grad.data.to(dtype=dtype)\n\n for buf in module.buffers(recurse=False):\n if buf is not None and buf.data.dtype.is_floating_point:\n buf.data = buf.data.to(dtype=dtype)\n\n\ndef convert_network(network, dtype):\n \"\"\"\n Converts a network's parameters and buffers to dtype.\n \"\"\"\n for module in network.modules():\n if isinstance(module, torch.nn.modules.batchnorm._BatchNorm) and module.affine is True:\n continue\n convert_module(module, dtype)\n if isinstance(module, torch.nn.RNNBase) or isinstance(module, torch.nn.modules.rnn.RNNBase):\n module.flatten_parameters()\n return network\n\n\nclass FP16Model(nn.Module):\n \"\"\"\n Convert model to half precision in a batchnorm-safe way.\n \"\"\"\n\n def __init__(self, network):\n super(FP16Model, self).__init__()\n self.network = convert_network(network, dtype=torch.half)\n\n def forward(self, *inputs):\n inputs = tuple(t.half() for t in inputs)\n return self.network(*inputs)\n\n\ndef backwards_debug_hook(grad):\n raise RuntimeError(\"master_params recieved a gradient in the backward pass!\")\n\ndef prep_param_lists(model, flat_master=False):\n \"\"\"\n Creates a list of FP32 master parameters for a given model, as in\n `Training Neural Networks with Mixed Precision: Real Examples`_.\n\n Args:\n model (torch.nn.Module): Existing Pytorch model\n flat_master (bool, optional, default=False): Flatten the master parameters into a single tensor, as a performance optimization.\n Returns:\n A tuple (``model_params``, ``master_params``). ``model_params`` is a list of the model's parameters for later use with :func:`model_grads_to_master_grads` and :func:`master_params_to_model_params`. ``master_params`` is a list of FP32 master gradients. If ``flat_master=True``, ``master_params`` will be a list with one element.\n\n Example::\n\n model_params, master_params = prep_param_lists(model)\n\n .. warning::\n Currently, if ``flat_master=True``, all the model's parameters must be the same type. If the model has parameters of different types, use ``flat_master=False``, or use :class:`FP16_Optimizer`.\n\n .. _`Training Neural Networks with Mixed Precision: Real Examples`:\n http://on-demand.gputechconf.com/gtc/2018/video/S81012/\n \"\"\"\n model_params = [param for param in model.parameters() if param.requires_grad]\n\n if flat_master:\n # Give the user some more useful error messages\n try:\n # flatten_dense_tensors returns a contiguous flat array.\n # http://pytorch.org/docs/master/_modules/torch/_utils.html\n master_params = _flatten_dense_tensors([param.data for param in model_params]).float()\n except:\n print(\"Error in prep_param_lists: model may contain a mixture of parameters \"\n \"of different types. Use flat_master=False, or use F16_Optimizer.\")\n raise\n master_params = torch.nn.Parameter(master_params)\n master_params.requires_grad = True\n # master_params.register_hook(backwards_debug_hook)\n if master_params.grad is None:\n master_params.grad = master_params.new(*master_params.size())\n return model_params, [master_params]\n else:\n master_params = [param.clone().float().detach() for param in model_params]\n for param in master_params:\n param.requires_grad = True\n return model_params, master_params\n\n\ndef model_grads_to_master_grads(model_params, master_params, flat_master=False):\n \"\"\"\n Copy model gradients to master gradients. \n\n Args:\n model_params: List of model parameters created by :func:`prep_param_lists`.\n master_params: List of FP32 master parameters created by :func:`prep_param_lists`. If ``master_params`` was created with ``flat_master=True``, ``flat_master=True`` should also be supplied to :func:`model_grads_to_master_grads`.\n \"\"\"\n if flat_master:\n # The flattening may incur one more deep copy than is necessary.\n master_params[0].grad.data.copy_(\n _flatten_dense_tensors([p.grad.data for p in model_params]))\n else:\n for model, master in zip(model_params, master_params):\n if model.grad is not None:\n if master.grad is None:\n master.grad = Variable(master.data.new(*master.data.size()))\n master.grad.data.copy_(model.grad.data)\n else:\n master.grad = None\n\n\ndef master_params_to_model_params(model_params, master_params, flat_master=False):\n \"\"\"\n Copy master parameters to model parameters.\n\n Args:\n model_params: List of model parameters created by :func:`prep_param_lists`.\n master_params: List of FP32 master parameters created by :func:`prep_param_lists`. If ``master_params`` was created with ``flat_master=True``, ``flat_master=True`` should also be supplied to :func:`master_params_to_model_params`.\n \"\"\"\n if flat_master:\n for model, master in zip(model_params, \n _unflatten_dense_tensors(master_params[0].data, model_params)):\n model.data.copy_(master)\n else:\n for model, master in zip(model_params, master_params):\n model.data.copy_(master.data)\n\n# Backward compatibility fixes\n\ndef to_python_float(t):\n if hasattr(t, 'item'):\n return t.item()\n else:\n return t[0]\n\nTORCH_MAJOR = int(torch.__version__.split('.')[0])\nTORCH_MINOR = int(torch.__version__.split('.')[1])\nif TORCH_MAJOR == 0 and TORCH_MINOR <= 4:\n clip_grad_norm = torch.nn.utils.clip_grad_norm\nelse:\n clip_grad_norm = torch.nn.utils.clip_grad_norm_\n" }, { "path": "apex/apex/fp16_utils/loss_scaler.py", "content": "import torch\n\n# item() is a recent addition, so this helps with backward compatibility.\ndef to_python_float(t):\n if hasattr(t, 'item'):\n return t.item()\n else:\n return t[0]\n\nclass LossScaler:\n \"\"\"\n Class that manages a static loss scale. This class is intended to interact with\n :class:`FP16_Optimizer`, and should not be directly manipulated by the user.\n\n Use of :class:`LossScaler` is enabled via the ``static_loss_scale`` argument to \n :class:`FP16_Optimizer`'s constructor.\n\n Args:\n scale (float, optional, default=1.0): The loss scale.\n \"\"\"\n\n def __init__(self, scale=1):\n self.cur_scale = scale\n\n # `params` is a list / generator of torch.Variable\n def has_overflow(self, params):\n return False\n\n # `x` is a torch.Tensor\n def _has_inf_or_nan(x):\n return False\n\n def update_scale(self, overflow):\n pass\n\n @property\n def loss_scale(self):\n return self.cur_scale\n\n def scale_gradient(self, module, grad_in, grad_out):\n return tuple(self.loss_scale * g for g in grad_in)\n\n def backward(self, loss, retain_graph=False):\n scaled_loss = loss*self.loss_scale\n scaled_loss.backward(retain_graph=retain_graph)\n\nclass DynamicLossScaler:\n \"\"\"\n Class that manages dynamic loss scaling. It is recommended to use :class:`DynamicLossScaler`\n indirectly, by supplying ``dynamic_loss_scale=True`` to the constructor of \n :class:`FP16_Optimizer`. However, it's important to understand how :class:`DynamicLossScaler`\n operates, because the default options can be changed using the\n the ``dynamic_loss_args`` argument to :class:`FP16_Optimizer`'s constructor.\n\n Loss scaling is designed to combat the problem of underflowing gradients encountered at long\n times when training fp16 networks. Dynamic loss scaling begins by attempting a very high loss\n scale. Ironically, this may result in OVERflowing gradients. If overflowing gradients are\n encountered, :class:`DynamicLossScaler` informs :class:`FP16_Optimizer` that an overflow has \n occurred.\n :class:`FP16_Optimizer` then skips the update step for this particular iteration/minibatch,\n and :class:`DynamicLossScaler` adjusts the loss scale to a lower value. \n If a certain number of iterations occur without overflowing gradients detected,\n :class:`DynamicLossScaler` increases the loss scale once more.\n In this way :class:`DynamicLossScaler` attempts to \"ride the edge\" of \n always using the highest loss scale possible without incurring overflow.\n\n Args:\n init_scale (float, optional, default=2**32): Initial loss scale attempted by :class:`DynamicLossScaler.`\n scale_factor (float, optional, default=2.0): Factor used when adjusting the loss scale. If an overflow is encountered, the loss scale is readjusted to loss scale/``scale_factor``. If ``scale_window`` consecutive iterations take place without an overflow, the loss scale is readjusted to loss_scale*``scale_factor``. \n scale_window (int, optional, default=1000): Number of consecutive iterations without an overflow to wait before increasing the loss scale.\n \"\"\"\n\n def __init__(self,\n init_scale=2**32,\n scale_factor=2.,\n scale_window=1000):\n self.cur_scale = init_scale\n self.cur_iter = 0\n self.last_overflow_iter = -1\n self.scale_factor = scale_factor\n self.scale_window = scale_window\n\n # `params` is a list / generator of torch.Variable\n def has_overflow(self, params):\n for p in params:\n if p.grad is not None and DynamicLossScaler._has_inf_or_nan(p.grad.data):\n return True\n\n return False\n\n # `x` is a torch.Tensor\n def _has_inf_or_nan(x):\n try:\n # if x is half, the .float() incurs an additional deep copy, but it's necessary if \n # Pytorch's .sum() creates a one-element tensor of the same type as x \n # (which is true for some recent version of pytorch).\n cpu_sum = float(x.float().sum())\n # More efficient version that can be used if .sum() returns a Python scalar\n # cpu_sum = float(x.sum())\n except RuntimeError as instance:\n # We want to check if inst is actually an overflow exception.\n # RuntimeError could come from a different error.\n # If so, we still want the exception to propagate.\n if \"value cannot be converted\" not in instance.args[0]:\n raise\n return True\n else:\n if cpu_sum == float('inf') or cpu_sum == -float('inf') or cpu_sum != cpu_sum:\n return True\n return False\n\n # `overflow` is boolean indicating whether the gradient overflowed\n def update_scale(self, overflow):\n if overflow:\n # self.cur_scale /= self.scale_factor\n self.cur_scale = max(self.cur_scale/self.scale_factor, 1)\n self.last_overflow_iter = self.cur_iter\n else:\n if (self.cur_iter - self.last_overflow_iter) % self.scale_window == 0:\n self.cur_scale *= self.scale_factor\n self.cur_iter += 1\n\n @property\n def loss_scale(self):\n return self.cur_scale\n\n def scale_gradient(self, module, grad_in, grad_out):\n return tuple(self.loss_scale * g for g in grad_in)\n\n def backward(self, loss, retain_graph=False):\n scaled_loss = loss*self.loss_scale\n scaled_loss.backward(retain_graph=retain_graph)\n \n############################################################## \n# Example usage below here -- assuming it's in a separate file\n##############################################################\n\"\"\"\nTO-DO separate out into an example.\nif __name__ == \"__main__\":\n import torch\n from torch.autograd import Variable\n from dynamic_loss_scaler import DynamicLossScaler\n\n # N is batch size; D_in is input dimension;\n # H is hidden dimension; D_out is output dimension.\n N, D_in, H, D_out = 64, 1000, 100, 10\n\n # Create random Tensors to hold inputs and outputs, and wrap them in Variables.\n x = Variable(torch.randn(N, D_in), requires_grad=False)\n y = Variable(torch.randn(N, D_out), requires_grad=False)\n\n w1 = Variable(torch.randn(D_in, H), requires_grad=True)\n w2 = Variable(torch.randn(H, D_out), requires_grad=True)\n parameters = [w1, w2]\n\n learning_rate = 1e-6\n optimizer = torch.optim.SGD(parameters, lr=learning_rate)\n loss_scaler = DynamicLossScaler()\n\n for t in range(500):\n y_pred = x.mm(w1).clamp(min=0).mm(w2)\n loss = (y_pred - y).pow(2).sum() * loss_scaler.loss_scale\n print('Iter {} loss scale: {}'.format(t, loss_scaler.loss_scale))\n print('Iter {} scaled loss: {}'.format(t, loss.data[0]))\n print('Iter {} unscaled loss: {}'.format(t, loss.data[0] / loss_scaler.loss_scale))\n\n # Run backprop\n optimizer.zero_grad()\n loss.backward()\n \n # Check for overflow\n has_overflow = DynamicLossScaler.has_overflow(parameters)\n \n # If no overflow, unscale grad and update as usual\n if not has_overflow:\n for param in parameters:\n param.grad.data.mul_(1. / loss_scaler.loss_scale)\n optimizer.step()\n # Otherwise, don't do anything -- ie, skip iteration\n else:\n print('OVERFLOW!')\n\n # Update loss scale for next iteration\n loss_scaler.update_scale(has_overflow)\n\n\"\"\"\n" }, { "path": "apex/apex/multi_tensor_apply/__init__.py", "content": "from .multi_tensor_apply import MultiTensorApply\n\nmulti_tensor_applier = MultiTensorApply(2048*32)\n\n" }, { "path": "apex/apex/multi_tensor_apply/multi_tensor_apply.py", "content": "import torch\n\nclass MultiTensorApply(object):\n available = False\n warned = False\n\n def __init__(self, chunk_size):\n try:\n import amp_C\n MultiTensorApply.available = True\n self.chunk_size = chunk_size\n except ImportError as err:\n MultiTensorApply.available = False\n MultiTensorApply.import_err = err\n\n def check_avail(self):\n if MultiTensorApply.available == False:\n raise RuntimeError(\n \"Attempted to call MultiTensorApply method, but MultiTensorApply \"\n \"is not available, possibly because Apex was installed without \"\n \"--cpp_ext --cuda_ext. Original import error message:\",\n MultiTensorApply.import_err)\n\n def __call__(self, op, noop_flag_buffer, tensor_lists, *args):\n self.check_avail()\n\n return op(self.chunk_size,\n noop_flag_buffer,\n tensor_lists,\n *args)\n" }, { "path": "apex/apex/normalization/__init__.py", "content": "from .fused_layer_norm import FusedLayerNorm\n" }, { "path": "apex/apex/normalization/fused_layer_norm.py", "content": "import math\nimport torch\nimport numbers\nfrom torch.nn.parameter import Parameter\nfrom torch.nn import init\nfrom torch.nn import functional as F\nimport importlib\n\nclass FusedLayerNormAffineFunction(torch.autograd.Function):\n def __init__(self, normalized_shape, eps=1e-6):\n global fused_layer_norm_cuda\n fused_layer_norm_cuda = importlib.import_module(\"fused_layer_norm_cuda\")\n\n self.normalized_shape = normalized_shape\n self.eps = eps\n\n def forward(self, input, weight, bias):\n input_ = input.contiguous()\n weight_ = weight.contiguous()\n bias_ = bias.contiguous()\n output, mean, invvar = fused_layer_norm_cuda.forward_affine(\n input_, self.normalized_shape, weight_, bias_, self.eps)\n self.save_for_backward(input_, weight_, bias_, mean, invvar)\n return output\n\n def backward(self, grad_output):\n input_, weight_, bias_, mean, invvar = self.saved_tensors\n grad_input = grad_weight = grad_bias = None\n grad_input, grad_weight, grad_bias = fused_layer_norm_cuda.backward_affine(\n grad_output.contiguous(), mean, invvar,\n input_, self.normalized_shape, \n weight_, bias_, self.eps)\n return grad_input, grad_weight, grad_bias;\n \nclass FusedLayerNormFunction(torch.autograd.Function):\n def __init__(self, normalized_shape, eps=1e-6):\n global fused_layer_norm_cuda\n fused_layer_norm_cuda = importlib.import_module(\"fused_layer_norm_cuda\")\n self.normalized_shape = normalized_shape\n self.eps = eps\n\n def forward(self, input):\n input_ = input.contiguous()\n output, mean, invvar = fused_layer_norm_cuda.forward(\n input_, self.normalized_shape, self.eps)\n self.save_for_backward(input_, mean, invvar)\n return output\n\n def backward(self, grad_output):\n input_, mean, invvar = self.saved_tensors\n grad_input = None\n grad_input = fused_layer_norm_cuda.backward(\n grad_output.contiguous(), mean, invvar,\n input_, self.normalized_shape,\n self.eps)\n return grad_input\n\ndef fused_layer_norm_affine(input, normalized_shape, weight, bias, eps=1e-6):\n return FusedLayerNormAffineFunction(normalized_shape,eps)(input, weight, bias)\n\ndef fused_layer_norm(input, normalized_shape, eps=1e-6):\n return FusedLayerNormFunction(normalized_shape,eps)(input)\n\nclass FusedLayerNorm(torch.nn.Module):\n r\"\"\"Applies Layer Normalization over a mini-batch of inputs as described in\n the paper `Layer Normalization`_ .\n\n Currently only runs on cuda() tensors.\n\n .. math::\n y = \\frac{x - \\mathrm{E}[x]}{ \\sqrt{\\mathrm{Var}[x] + \\epsilon}} * \\gamma + \\beta\n\n The mean and standard-deviation are calculated separately over the last\n certain number dimensions which have to be of the shape specified by\n :attr:`normalized_shape`.\n :math:`\\gamma` and :math:`\\beta` are learnable affine transform parameters of\n :attr:`normalized_shape` if :attr:`elementwise_affine` is ``True``.\n\n .. note::\n Unlike Batch Normalization and Instance Normalization, which applies\n scalar scale and bias for each entire channel/plane with the\n :attr:`affine` option, Layer Normalization applies per-element scale and\n bias with :attr:`elementwise_affine`.\n\n This layer uses statistics computed from input data in both training and\n evaluation modes.\n\n Args:\n normalized_shape (int or list or torch.Size): input shape from an expected input\n of size\n\n .. math::\n [* \\times \\text{normalized}\\_\\text{shape}[0] \\times \\text{normalized}\\_\\text{shape}[1]\n \\times \\ldots \\times \\text{normalized}\\_\\text{shape}[-1]]\n\n If a single integer is used, it is treated as a singleton list, and this module will\n normalize over the last dimension which is expected to be of that specific size.\n eps: a value added to the denominator for numerical stability. Default: 1e-5\n elementwise_affine: a boolean value that when set to ``True``, this module\n has learnable per-element affine parameters initialized to ones (for weights)\n and zeros (for biases). Default: ``True``.\n\n Shape:\n - Input: :math:`(N, *)`\n - Output: :math:`(N, *)` (same shape as input)\n\n Examples::\n\n >>> input = torch.randn(20, 5, 10, 10)\n >>> # With Learnable Parameters\n >>> m = apex.normalization.FusedLayerNorm(input.size()[1:])\n >>> # Without Learnable Parameters\n >>> m = apex.normalization.FusedLayerNorm(input.size()[1:], elementwise_affine=False)\n >>> # Normalize over last two dimensions\n >>> m = apex.normalization.FusedLayerNorm([10, 10])\n >>> # Normalize over last dimension of size 10\n >>> m = apex.normalization.FusedLayerNorm(10)\n >>> # Activating the module\n >>> output = m(input)\n\n .. _`Layer Normalization`: https://arxiv.org/abs/1607.06450\n \"\"\"\n def __init__(self, normalized_shape, eps=1e-5, elementwise_affine=True):\n super(FusedLayerNorm, self).__init__()\n\n global fused_layer_norm_cuda\n fused_layer_norm_cuda = importlib.import_module(\"fused_layer_norm_cuda\")\n\n if isinstance(normalized_shape, numbers.Integral):\n normalized_shape = (normalized_shape,)\n self.normalized_shape = torch.Size(normalized_shape)\n self.eps = eps\n self.elementwise_affine = elementwise_affine\n if self.elementwise_affine:\n self.weight = Parameter(torch.Tensor(*normalized_shape))\n self.bias = Parameter(torch.Tensor(*normalized_shape))\n else:\n self.register_parameter('weight', None)\n self.register_parameter('bias', None)\n self.reset_parameters()\n\n def reset_parameters(self):\n if self.elementwise_affine:\n init.ones_(self.weight)\n init.zeros_(self.bias)\n\n def forward(self, input):\n if not input.is_cuda:\n return F.layer_norm(\n input, self.normalized_shape, self.weight, self.bias, self.eps)\n if self.elementwise_affine:\n return FusedLayerNormAffineFunction(self.normalized_shape,self.eps)(\n input, self.weight, self.bias)\n else:\n return FusedLayerNormFunction(self.normalized_shape,self.eps)(\n input)\n\n def extra_repr(self):\n return '{normalized_shape}, eps={eps}, ' \\\n 'elementwise_affine={elementwise_affine}'.format(**self.__dict__)\n" }, { "path": "apex/apex/optimizers/__init__.py", "content": "from .fused_adam import FusedAdam\nfrom .fp16_optimizer import FP16_Optimizer\n" }, { "path": "apex/apex/optimizers/fp16_optimizer.py", "content": "import torch\nfrom torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors\n\nclass FP16_Optimizer(object):\n \"\"\"\n :class:`FP16_Optimizer` A cutdown version of apex.fp16_utils.FP16_Optimizer.\n Designed only to wrap apex.optimizers.FusedAdam.\n Refer to apex.fp16_utils documents for more information.\n\n Example::\n\n model = torch.nn.Linear(D_in, D_out).cuda().half()\n optimizer = apex.optimizers.FusedAdam(model.parameters())\n # Name the FP16_Optimizer instance to replace the existing optimizer\n # (recommended but not required):\n optimizer = FP16_Optimizer(optimizer, static_loss_scale = 128.0)\n ...\n # loss.backward() becomes:\n optimizer.backward(loss)\n ...\n\n Example with dynamic loss scaling::\n\n ...\n optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)\n # optional arg to control dynamic loss scaling behavior\n # dynamic_loss_args={'scale_window' : 500})\n # Usually, dynamic_loss_args is not necessary.\n \"\"\"\n\n def __init__(self,\n init_optimizer,\n static_loss_scale=1.0,\n dynamic_loss_scale=False,\n dynamic_loss_args=None,\n verbose=True):\n\n # The fused optimizer does all the work. We need this layer for two reason:\n # 1. maintain same user API from apex.fp16_utils\n # 2. keep common stuff here in case we need to add new fused optimizer later\n\n # differences from apex.fp16_utils:\n # - assume all model params in fp16\n # - assume all params requires grad\n # - flat by groups, not keeping state. TODO: remove state explicitly?\n # - master gard and unflat master weight never exist. TODO: a way to save out unflat master?\n if not torch.cuda.is_available:\n raise SystemError(\"Cannot use fp16 without CUDA.\")\n self.optimizer = init_optimizer\n\n # param flattened by groups\n self.fp16_groups = []\n self.fp16_groups_flat = []\n self.fp32_groups_flat = []\n\n # loop to deal with groups\n for i, param_group in enumerate(self.optimizer.param_groups):\n # push this group to list before modify\n self.fp16_groups.append(param_group['params'])\n # init fp16 weight buffer, flattened\n self.fp16_groups_flat.append(_flatten_dense_tensors([p.clone().detach() for p in self.fp16_groups[i]]))\n # set model fp16 weight to slices of flattened buffer\n updated_params = _unflatten_dense_tensors(self.fp16_groups_flat[i], self.fp16_groups[i])\n for p,q in zip(self.fp16_groups[i], updated_params):\n p.data = q.data\n # init master weight, flattened\n self.fp32_groups_flat.append(self.fp16_groups_flat[i].clone().float().detach())\n # modify optimizer of have flat master weight\n self.fp32_groups_flat[i].requires_grad = True # keep this in case internal optimizer uses it\n param_group['params'] = [self.fp32_groups_flat[i]]\n\n # we may have a way of fusing dynamic scale. Do not support for now\n if dynamic_loss_scale:\n if dynamic_loss_args is not None:\n raise SystemError(\"Do not support dynamic loss scale args for now.\")\n self.dynamic_loss_scale = True\n self.cur_scale = 2**16\n self.cur_iter = 0\n self.last_overflow_iter = -1\n self.scale_factor = 2\n self.scale_window = 1000\n else:\n self.dynamic_loss_scale = False\n self.cur_iter = 0\n self.cur_scale = static_loss_scale\n self.verbose = verbose\n\n def zero_grad(self, set_grads_to_None=True):\n \"\"\"\n Zero FP16 parameter grads.\n \"\"\"\n # FP32 grad should never exist.\n # For speed, set model fp16 grad to None by default\n for group in self.fp16_groups:\n for p in group:\n if set_grads_to_None:\n p.grad = None\n else:\n if p.grad is not None:\n p.grad.detach_()\n p.grad.zero_()\n\n def _compute_grad_norm(self, fp16_grads_flat, norm_type=2):\n \"\"\"\n Compute fp16 grad norm for later clipping(fused with update).\n Internal accumulated in fp32.\n Also fused in NaN check. Possibly other reduction needed for grad.\n\n Args:\n fp16_grads_flat (tensor): fp16 grad flattened\n norm_type (float or int): type of the used p-norm. Can be ``'inf'`` for\n infinity norm.\n\n Returns:\n Total norm of the current fp16 gradients (viewed as a single vector).\n Returns -1 if the most recently computed fp16 gradients overflowed\n \"\"\"\n # TODO: Not most efficient with copy to cpu and sync\n # only support 2-norm now\n # for torch version <= 1.0.1, torch.norm with dtype will fail and fall back to cast\n try:\n norm = float(torch.norm(fp16_grads_flat, 2.0, dtype=torch.float32))\n except TypeError as err:\n norm = float(torch.norm(fp16_grads_flat.float(), 2.0))\n if norm == float('inf') or norm == -float('inf') or norm != norm:\n return -1\n else:\n return norm\n\n def step(self, closure=None):\n \"\"\"\n Not supporting closure.\n \"\"\"\n # First compute norm for all group so we know if there is overflow\n grads_groups_flat = []\n norm_groups = []\n skip = False\n for i, group in enumerate(self.fp16_groups):\n grads_groups_flat.append(_flatten_dense_tensors([p.grad for p in group]))\n norm_groups.append(self._compute_grad_norm(grads_groups_flat[i]))\n if norm_groups[i] == -1: #TODO: early break\n skip = True\n\n if skip:\n self._update_scale(skip)\n return\n\n # norm is in fact norm*cur_scale\n self.optimizer.step(grads=[[g] for g in grads_groups_flat],\n output_params=[[p] for p in self.fp16_groups_flat],\n scale=self.cur_scale,\n grad_norms=norm_groups)\n\n # TODO: we probably don't need this? just to be safe\n for i in range(len(norm_groups)):\n updated_params = _unflatten_dense_tensors(self.fp16_groups_flat[i], self.fp16_groups[i])\n for p,q in zip(self.fp16_groups[i], updated_params):\n p.data = q.data\n\n self._update_scale(False)\n return\n\n def backward(self, loss):\n \"\"\"\n :attr:`backward` performs the following steps:\n\n 1. fp32_loss = loss.float()\n 2. scaled_loss = fp32_loss*loss_scale\n 3. scaled_loss.backward(), which accumulates scaled gradients into the ``.grad`` attributes of the model's fp16 leaves\n \"\"\"\n scaled_loss = (loss.float()) * self.cur_scale\n scaled_loss.backward()\n\n def _update_scale(self, skip):\n if self.dynamic_loss_scale:\n if skip:\n if self.verbose:\n print(\"\\nGrad overflow on iteration\", self.cur_iter)\n print(\"Using dynamic loss scale of\", self.cur_scale)\n self.cur_scale = max(self.cur_scale/self.scale_factor, 1)\n self.last_overflow_iter = self.cur_iter\n else:\n if (self.cur_iter - self.last_overflow_iter) % self.scale_window == 0:\n self.cur_scale *= self.scale_factor\n else:\n if skip:\n print(\"\\nGrad overflow on iteration\", self.cur_iter)\n print(\"Using static loss scale of\", self.cur_scale)\n self.cur_iter +=1\n return\n\n # Promote state so it can be retrieved or set via \"fp16_optimizer_instance.state\"\n def _get_state(self):\n return self.optimizer.state\n\n def _set_state(self, value):\n self.optimizer.state = value\n\n state = property(_get_state, _set_state)\n\n # Promote param_groups so it can be retrieved or set via \"fp16_optimizer_instance.param_groups\"\n # (for example, to adjust the learning rate)\n def _get_param_groups(self):\n return self.optimizer.param_groups\n\n def _set_param_groups(self, value):\n self.optimizer.param_groups = value\n\n param_groups = property(_get_param_groups, _set_param_groups)\n\n def state_dict(self):\n \"\"\"\n Returns a dict containing the current state of this :class:`FP16_Optimizer` instance.\n This dict contains attributes of :class:`FP16_Optimizer`, as well as the state_dict\n of the contained Pytorch optimizer.\n Example::\n checkpoint = {}\n checkpoint['model'] = model.state_dict()\n checkpoint['optimizer'] = optimizer.state_dict()\n torch.save(checkpoint, \"saved.pth\")\n \"\"\"\n state_dict = {}\n state_dict['dynamic_loss_scale'] = self.dynamic_loss_scale\n state_dict['cur_scale'] = self.cur_scale\n state_dict['cur_iter'] = self.cur_iter\n if state_dict['dynamic_loss_scale']:\n state_dict['last_overflow_iter'] = self.last_overflow_iter\n state_dict['scale_factor'] = self.scale_factor\n state_dict['scale_window'] = self.scale_window\n state_dict['optimizer_state_dict'] = self.optimizer.state_dict()\n state_dict['fp32_groups_flat'] = self.fp32_groups_flat\n return state_dict\n\n def load_state_dict(self, state_dict):\n \"\"\"\n Loads a state_dict created by an earlier call to state_dict().\n If ``fp16_optimizer_instance`` was constructed from some ``init_optimizer``,\n whose parameters in turn came from ``model``, it is expected that the user\n will call ``model.load_state_dict()`` before\n ``fp16_optimizer_instance.load_state_dict()`` is called.\n Example::\n model = torch.nn.Linear(D_in, D_out).cuda().half()\n optimizer = torch.optim.SGD(model.parameters(), lr=1e-3)\n optimizer = FP16_Optimizer(optimizer, static_loss_scale = 128.0)\n ...\n checkpoint = torch.load(\"saved.pth\")\n model.load_state_dict(checkpoint['model'])\n optimizer.load_state_dict(checkpoint['optimizer'])\n \"\"\"\n # I think it should actually be ok to reload the optimizer before the model.\n self.dynamic_loss_scale = state_dict['dynamic_loss_scale']\n self.cur_scale = state_dict['cur_scale']\n self.cur_iter = state_dict['cur_iter']\n if state_dict['dynamic_loss_scale']:\n self.last_overflow_iter = state_dict['last_overflow_iter']\n self.scale_factor = state_dict['scale_factor']\n self.scale_window = state_dict['scale_window']\n self.optimizer.load_state_dict(state_dict['optimizer_state_dict'])\n # At this point, the optimizer's references to the model's fp32 parameters are up to date.\n # The optimizer's hyperparameters and internal buffers are also up to date.\n # However, the fp32 master copies of the model's fp16 params stored by the optimizer are still\n # out of date. There are two options.\n # 1: Refresh the master params from the model's fp16 params.\n # This requires less storage but incurs precision loss.\n # 2: Save and restore the fp32 master copies separately.\n # We choose option 2.\n #\n # Pytorch Optimizer.load_state_dict casts saved buffers (e.g. momentum) to the type and device\n # of their associated parameters, because it's possible those buffers might not exist yet in\n # the current optimizer instance. In our case, as long as the current FP16_Optimizer has been\n # constructed in the same way as the one whose state_dict we are loading, the same master params\n # are guaranteed to exist, so we can just copy_() from the saved master params.\n for current, saved in zip(self.fp32_groups_flat, state_dict['fp32_groups_flat']):\n current.data.copy_(saved.data)\n" }, { "path": "apex/apex/optimizers/fused_adam.py", "content": "import types\nimport torch\nimport importlib\n\nclass FusedAdam(torch.optim.Optimizer):\n\n \"\"\"Implements Adam algorithm. Currently GPU-only. Requires Apex to be installed via\n ``python setup.py install --cuda_ext --cpp_ext``.\n\n It has been proposed in `Adam: A Method for Stochastic Optimization`_.\n\n Arguments:\n params (iterable): iterable of parameters to optimize or dicts defining\n parameter groups.\n lr (float, optional): learning rate. (default: 1e-3)\n betas (Tuple[float, float], optional): coefficients used for computing\n running averages of gradient and its square. (default: (0.9, 0.999))\n eps (float, optional): term added to the denominator to improve\n numerical stability. (default: 1e-8)\n weight_decay (float, optional): weight decay (L2 penalty) (default: 0)\n amsgrad (boolean, optional): whether to use the AMSGrad variant of this\n algorithm from the paper `On the Convergence of Adam and Beyond`_\n (default: False) NOT SUPPORTED in FusedAdam!\n eps_inside_sqrt (boolean, optional): in the 'update parameters' step,\n adds eps to the bias-corrected second moment estimate before\n evaluating square root instead of adding it to the square root of\n second moment estimate as in the original paper. (default: False)\n\n .. _Adam\\: A Method for Stochastic Optimization:\n https://arxiv.org/abs/1412.6980\n .. _On the Convergence of Adam and Beyond:\n https://openreview.net/forum?id=ryQu7f-RZ\n \"\"\"\n\n def __init__(self, params,\n lr=1e-3, bias_correction = True,\n betas=(0.9, 0.999), eps=1e-8, eps_inside_sqrt = False,\n weight_decay=0., max_grad_norm=0., amsgrad=False):\n global fused_adam_cuda\n fused_adam_cuda = importlib.import_module(\"fused_adam_cuda\")\n\n if amsgrad:\n raise RuntimeError('FusedAdam does not support the AMSGrad variant.')\n defaults = dict(lr=lr, bias_correction=bias_correction,\n betas=betas, eps=eps, weight_decay=weight_decay,\n max_grad_norm=max_grad_norm)\n super(FusedAdam, self).__init__(params, defaults)\n self.eps_mode = 0 if eps_inside_sqrt else 1\n\n def step(self, closure=None, grads=None, output_params=None, scale=1., grad_norms=None):\n \"\"\"Performs a single optimization step.\n\n Arguments:\n closure (callable, optional): A closure that reevaluates the model\n and returns the loss.\n grads (list of tensors, optional): weight gradient to use for the\n optimizer update. If gradients have type torch.half, parameters\n are expected to be in type torch.float. (default: None)\n output params (list of tensors, optional): A reduced precision copy\n of the updated weights written out in addition to the regular\n updated weights. Have to be of same type as gradients. (default: None)\n scale (float, optional): factor to divide gradient tensor values\n by before applying to weights. (default: 1)\n \"\"\"\n loss = None\n if closure is not None:\n loss = closure()\n\n if grads is None:\n grads_group = [None]*len(self.param_groups)\n # backward compatibility\n # assuming a list/generator of parameter means single group\n elif isinstance(grads, types.GeneratorType):\n grads_group = [grads]\n elif type(grads[0])!=list:\n grads_group = [grads]\n else:\n grads_group = grads\n\n if output_params is None:\n output_params_group = [None]*len(self.param_groups)\n elif isinstance(output_params, types.GeneratorType):\n output_params_group = [output_params]\n elif type(output_params[0])!=list:\n output_params_group = [output_params]\n else:\n output_params_group = output_params\n\n if grad_norms is None:\n grad_norms = [None]*len(self.param_groups)\n\n for group, grads_this_group, output_params_this_group, grad_norm in zip(self.param_groups, grads_group, output_params_group, grad_norms):\n if grads_this_group is None:\n grads_this_group = [None]*len(group['params'])\n if output_params_this_group is None:\n output_params_this_group = [None]*len(group['params'])\n\n # compute combined scale factor for this group\n combined_scale = scale\n if group['max_grad_norm'] > 0:\n # norm is in fact norm*scale\n clip = ((grad_norm / scale) + 1e-6) / group['max_grad_norm']\n if clip > 1:\n combined_scale = clip * scale\n\n bias_correction = 1 if group['bias_correction'] else 0\n\n for p, grad, output_param in zip(group['params'], grads_this_group, output_params_this_group):\n #note: p.grad should not ever be set for correct operation of mixed precision optimizer that sometimes sends None gradients\n if p.grad is None and grad is None:\n continue\n if grad is None:\n grad = p.grad.data\n if grad.is_sparse:\n raise RuntimeError('FusedAdam does not support sparse gradients, please consider SparseAdam instead')\n\n state = self.state[p]\n\n # State initialization\n if len(state) == 0:\n state['step'] = 0\n # Exponential moving average of gradient values\n state['exp_avg'] = torch.zeros_like(p.data)\n # Exponential moving average of squared gradient values\n state['exp_avg_sq'] = torch.zeros_like(p.data)\n\n exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n beta1, beta2 = group['betas']\n\n state['step'] += 1\n\n out_p = torch.tensor([], dtype = torch.float) if output_param is None else output_param\n fused_adam_cuda.adam(p.data,\n out_p,\n exp_avg,\n exp_avg_sq,\n grad,\n group['lr'],\n beta1,\n beta2,\n group['eps'],\n combined_scale,\n state['step'],\n self.eps_mode,\n bias_correction,\n group['weight_decay'])\n return loss\n" }, { "path": "apex/apex/parallel/LARC.py", "content": "import torch\nfrom torch import nn\nfrom torch.autograd import Variable\nfrom torch.nn.parameter import Parameter\n\nclass LARC(object):\n \"\"\"\n :class:`LARC` is a pytorch implementation of both the scaling and clipping variants of LARC,\n in which the ratio between gradient and parameter magnitudes is used to calculate an adaptive \n local learning rate for each individual parameter. The algorithm is designed to improve\n convergence of large batch training.\n \n See https://arxiv.org/abs/1708.03888 for calculation of the local learning rate.\n\n In practice it modifies the gradients of parameters as a proxy for modifying the learning rate\n of the parameters. This design allows it to be used as a wrapper around any torch.optim Optimizer.\n\n ```\n model = ...\n optim = torch.optim.Adam(model.parameters(), lr=...)\n optim = LARC(optim)\n ```\n\n It can even be used in conjunction with apex.fp16_utils.FP16_optimizer.\n\n ```\n model = ...\n optim = torch.optim.Adam(model.parameters(), lr=...)\n optim = LARC(optim)\n optim = apex.fp16_utils.FP16_Optimizer(optim)\n ```\n\n Args:\n optimizer: Pytorch optimizer to wrap and modify learning rate for.\n trust_coefficient: Trust coefficient for calculating the lr. See https://arxiv.org/abs/1708.03888\n clip: Decides between clipping or scaling mode of LARC. If `clip=True` the learning rate is set to `min(optimizer_lr, local_lr)` for each parameter. If `clip=False` the learning rate is set to `local_lr*optimizer_lr`.\n eps: epsilon kludge to help with numerical stability while calculating adaptive_lr\n \"\"\"\n\n def __init__(self, optimizer, trust_coefficient=0.02, clip=True, eps=1e-8):\n self.param_groups = optimizer.param_groups\n self.optim = optimizer\n self.trust_coefficient = trust_coefficient\n self.eps = eps\n self.clip = clip\n\n def __getstate__(self):\n return self.optim.__getstate__()\n\n def __setstate__(self, state):\n self.optim.__setstate__(state)\n\n def __repr__(self):\n return self.optim.__repr__()\n\n def state_dict(self):\n return self.optim.state_dict()\n\n def load_state_dict(self, state_dict):\n self.optim.load_state_dict(state_dict)\n\n def zero_grad(self):\n self.optim.zero_grad()\n\n def add_param_group(self, param_group):\n self.optim.add_param_group( param_group)\n\n def step(self):\n with torch.no_grad():\n weight_decays = []\n for group in self.optim.param_groups:\n # absorb weight decay control from optimizer\n weight_decay = group['weight_decay'] if 'weight_decay' in group else 0\n weight_decays.append(weight_decay)\n group['weight_decay'] = 0\n for p in group['params']:\n if p.grad is None:\n continue\n param_norm = torch.norm(p.data)\n grad_norm = torch.norm(p.grad.data)\n\n if param_norm != 0 and grad_norm != 0:\n # calculate adaptive lr + weight decay\n adaptive_lr = self.trust_coefficient * (param_norm) / (grad_norm + param_norm * weight_decay + self.eps)\n\n # clip learning rate for LARC\n if self.clip:\n # calculation of adaptive_lr so that when multiplied by lr it equals `min(adaptive_lr, lr)`\n adaptive_lr = min(adaptive_lr/group['lr'], 1)\n\n p.grad.data += weight_decay * p.data\n p.grad.data *= adaptive_lr\n\n self.optim.step()\n # return weight decay control to optimizer\n for i, group in enumerate(self.optim.param_groups):\n group['weight_decay'] = weight_decays[i]\n" }, { "path": "apex/apex/parallel/README.md", "content": "## Distributed Data Parallel\n\ndistributed.py contains the source code for `apex.parallel.DistributedDataParallel`, a module wrapper that enables multi-process multi-GPU data parallel training optimized for NVIDIA's NCCL communication library.\n\n`apex.parallel.DistributedDataParallel` achieves high performance by overlapping communication with\ncomputation in the backward pass and bucketing smaller transfers to reduce the total number of\ntransfers required.\n\nmultiproc.py contains the source code for `apex.parallel.multiproc`, a launch utility that places one process on each of the node's available GPUs.\n\n#### [API Documentation](https://nvidia.github.io/apex/parallel.html)\n\n#### [Example/Walkthrough](https://github.com/NVIDIA/apex/tree/master/examples/distributed)\n\n#### [Imagenet example with Mixed Precision](https://github.com/NVIDIA/apex/tree/master/examples/imagenet)\n\n#### [Simple example with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/FP16_Optimizer_simple/distributed_apex)\n\n### Synchronized Batch Normalization\n\n`apex.parallel.SyncBatchNorm` has similar APIs as with `torch.nn.BatchNorm*N*d`.\nIt reduces stats on the first (channel) dimension of the Tensor and accepts\narbitrary spatial dimensions.\n\n#### Installation\n\nApex provides two sync BN implementation:\n\n1. There is the Python-only implementation, which is the default implementation\nwhen install with `python setup.py install`.\nIt uses PyTorch primitive operations and distributed communication package from\n`torch.distributed`.\n\n - _Python-only implementation requires input tensor to be of same data type as\nlayer_\n\n2. We also provide implementation with kernels through CUDA/C++ extension with\nimproved performance. We are experimenting with Welford and Kahan for reduction\nhoping to get better accuracy.\n To use the kernel implementation, user need to install Apex with CUDA extension\nenabled `python setup.py install --cuda_ext`.\n\n - _Custom kernel implementation supports fp16 input with fp32 layer as cudnn.\nThis is required to run imagenet example in fp16._\n\n - _Currently kernel implementation only supports GPU._\n\n#### HowTo\n\n1. User could use `apex.parallel.SyncBatchNorm` by building their module with\nthe layer explicitly.\n\n```\nimport apex\ninput_t = torch.randn(3, 5, 20).cuda()\nsbn = apex.parallel.SyncBatchNorm(5).cuda()\noutput_t = sbn(input)\n```\n\n2. User could also take a constructed `torch.nn.Model` and replace all its `torch.nn.BatchNorm*N*d` modules with `apex.parallel.SyncBatchNorm` through utility function `apex.parallel.convert_syncbn_model`.\n\n```\n# model is an instance of torch.nn.Module\nimport apex\nsync_bn_model = apex.parallel.convert_syncbn_model(model)\n```\n" }, { "path": "apex/apex/parallel/__init__.py", "content": "import torch\n\nif hasattr(torch.distributed, 'ReduceOp'):\n ReduceOp = torch.distributed.ReduceOp\nelif hasattr(torch.distributed, 'reduce_op'):\n ReduceOp = torch.distributed.reduce_op\nelse:\n ReduceOp = torch.distributed.deprecated.reduce_op\n\nfrom .distributed import DistributedDataParallel, Reducer\n# This is tricky because I'd like SyncBatchNorm to be exposed the same way\n# for both the cuda-enabled and python-fallback versions, and I don't want\n# to suppress the error information.\ntry:\n import syncbn\n from .optimized_sync_batchnorm import SyncBatchNorm\nexcept ImportError as err:\n from .sync_batchnorm import SyncBatchNorm\n SyncBatchNorm.syncbn_import_error = err\n\ndef convert_syncbn_model(module, process_group=None, channel_last=False):\n '''\n Recursively traverse module and its children to replace all instances of\n ``torch.nn.modules.batchnorm._BatchNorm`` with :class:`apex.parallel.SyncBatchNorm`.\n\n All ``torch.nn.BatchNorm*N*d`` wrap around\n ``torch.nn.modules.batchnorm._BatchNorm``, so this function lets you easily switch\n to use sync BN.\n\n Args:\n module (torch.nn.Module): input module\n\n Example::\n\n >>> # model is an instance of torch.nn.Module\n >>> import apex\n >>> sync_bn_model = apex.parallel.convert_syncbn_model(model)\n '''\n mod = module\n if isinstance(module, torch.nn.modules.batchnorm._BatchNorm):\n mod = SyncBatchNorm(module.num_features, module.eps, module.momentum, module.affine, module.track_running_stats, process_group, channel_last=channel_last)\n mod.running_mean = module.running_mean\n mod.running_var = module.running_var\n if module.affine:\n mod.weight.data = module.weight.data.clone().detach()\n mod.bias.data = module.bias.data.clone().detach()\n for name, child in module.named_children():\n mod.add_module(name, convert_syncbn_model(child,\n process_group=process_group,\n channel_last=channel_last))\n # TODO(jie) should I delete model explicitly?\n del module\n return mod\n\ndef create_syncbn_process_group(group_size):\n '''\n Creates process groups to be used for syncbn of a give ``group_size`` and returns\n process group that current GPU participates in.\n\n ``group_size`` must divide the total number of GPUs (world_size).\n\n ``group_size`` of 0 would be considered as =world_size. In this case ``None`` will be returned.\n\n ``group_size`` of 1 would be equivalent to using non-sync bn, but will still carry the overhead.\n\n Args:\n group_size (int): number of GPU's to collaborate for sync bn\n\n Example::\n\n >>> # model is an instance of torch.nn.Module\n >>> import apex\n >>> group = apex.parallel.create_syncbn_process_group(group_size)\n '''\n\n if group_size==0:\n return None\n\n world_size = torch.distributed.get_world_size()\n assert(world_size >= group_size)\n assert(world_size % group_size == 0)\n\n group=None\n for group_num in (range(world_size//group_size)):\n group_ids = range(group_num*group_size, (group_num+1)*group_size)\n cur_group = torch.distributed.new_group(ranks=group_ids)\n if (torch.distributed.get_rank()//group_size == group_num):\n group = cur_group\n #can not drop out and return here, every process must go through creation of all subgroups\n\n assert(group is not None)\n return group\n" }, { "path": "apex/apex/parallel/distributed.py", "content": "import torch\nimport torch.distributed as dist\nfrom torch.nn.modules import Module\nfrom torch.autograd import Variable\nfrom collections import OrderedDict\nfrom itertools import chain\nimport copy\nimport importlib\nfrom ..multi_tensor_apply import multi_tensor_applier\n\nimported_flatten_impl = False\n\ndef import_flatten_impl():\n global flatten_impl, unflatten_impl, imported_flatten_impl\n try:\n import apex_C\n flatten_impl = apex_C.flatten\n unflatten_impl = apex_C.unflatten\n except ImportError:\n print(\"Warning: apex was installed without --cpp_ext. Falling back to Python flatten and unflatten.\")\n flatten_impl = torch._utils._flatten_dense_tensors\n unflatten_impl = torch._utils._unflatten_dense_tensors\n imported_flatten_impl = True\n\ndef flatten(bucket):\n if not imported_flatten_impl:\n import_flatten_impl()\n return flatten_impl(bucket)\n\ndef unflatten(coalesced, bucket):\n if not imported_flatten_impl:\n import_flatten_impl()\n return unflatten_impl(coalesced, bucket)\n\n# apply_dist_call requires that tensors in 'bucket' are all the same type.\ndef apply_flat_dist_call(bucket, call, extra_args=None):\n\n coalesced = flatten(bucket)\n\n if extra_args is not None:\n call(coalesced, *extra_args)\n else:\n call(coalesced)\n\n if call is dist.all_reduce:\n coalesced /= dist.get_world_size()\n \n for buf, synced in zip(bucket, unflatten(coalesced, bucket)):\n buf.copy_(synced)\n\ndef split_half_float_double(tensors):\n dtypes = [\"torch.cuda.HalfTensor\", \"torch.cuda.FloatTensor\", \"torch.cuda.DoubleTensor\"]\n buckets = []\n for i, dtype in enumerate(dtypes):\n bucket = [t for t in tensors if t.type() == dtype]\n if bucket:\n buckets.append(bucket) \n return buckets\n\ndef split_by_type(tensors):\n buckets = OrderedDict()\n for tensor in tensors:\n tp = tensor.type()\n if tp not in buckets:\n buckets[tp] = []\n buckets[tp].append(tensor)\n return buckets\n\n# flat_dist_call organizes 'tensors' by type.\ndef flat_dist_call(tensors, call, extra_args=None):\n buckets = split_by_type(tensors)\n \n for tp in buckets:\n bucket = buckets[tp]\n apply_flat_dist_call(bucket, call, extra_args)\n\n \ndef extract_tensors(maybe_tensor, tensor_list):\n if torch.is_tensor(maybe_tensor):\n tensor_list.append(maybe_tensor)\n else:\n try:\n for item in maybe_tensor:\n extract_tensors(item, tensor_list)\n except TypeError:\n return\n\n \nclass Reducer(object):\n \"\"\"\n :class:`apex.parallel.Reducer` is a simple class that helps allreduce a module's parameters\n across processes. :class:`Reducer` is intended to give the user additional control:\n Unlike :class:`DistributedDataParallel`, :class:`Reducer` will not automatically allreduce\n parameters during ``backward()``.\n Instead, :class:`Reducer` waits for the user to call ``.reduce()`` manually.\n This enables, for example, delaying the allreduce to be carried out every \n several iterations instead of every single iteration.\n\n Like :class:`DistributedDataParallel`, :class:`Reducer` averages any tensors it allreduces \n over the number of participating processes.\n\n :class:`Reducer` is designed to work with the upstream launch utility script \n ``torch.distributed.launch`` with ``--nproc_per_node <= number of gpus per node``.\n When used with this launcher, :class:`Reducer` assumes 1:1 mapping of processes to GPUs.\n It also assumes that your script calls ``torch.cuda.set_device(args.rank)`` before creating the model.\n\n Args:\n module_or_grads_list: Either a network definition (module) being run in multi-gpu/distributed mode, or an iterable of gradients to be reduced. If a module is passed in, the Reducer constructor will sync the parameters across processes (broadcasting from rank 0) to make sure they're all initialized with the same values. If a list of gradients (that came from some module) is passed in, the user is responsible for manually syncing that module's parameters at the beginning of training.\n \"\"\"\n \n def __init__(self, module_or_grads_list):\n if isinstance(module_or_grads_list, Module):\n self.module = module_or_grads_list\n flat_dist_call([param.data for param in self.module.parameters()], dist.broadcast, (0,) )\n\n else:\n self.module = None\n self.grads = []\n extract_tensors(module_or_grads_list, self.grads)\n \n def reduce(self):\n if self.module:\n grads = [param.grad.data for param in self.module.parameters() if param.grad is not None]\n flat_dist_call(grads, dist.all_reduce)\n else:\n flat_dist_call(self.grads, dist.all_reduce)\n \n \nclass DistributedDataParallel(Module):\n \"\"\"\n :class:`apex.parallel.DistributedDataParallel` is a module wrapper that enables\n easy multiprocess distributed data parallel training, similar to ``torch.nn.parallel.DistributedDataParallel``. Parameters are broadcast across participating processes on initialization, and gradients are\n allreduced and averaged over processes during ``backward()``.\n\n :class:`DistributedDataParallel` is optimized for use with NCCL. It achieves high performance by \n overlapping communication with computation during ``backward()`` and bucketing smaller gradient\n transfers to reduce the total number of transfers required.\n\n :class:`DistributedDataParallel` is designed to work with the upstream launch utility script \n ``torch.distributed.launch`` with ``--nproc_per_node <= number of gpus per node``.\n When used with this launcher, :class:`DistributedDataParallel` assumes 1:1 mapping of processes to GPUs.\n It also assumes that your script calls ``torch.cuda.set_device(args.rank)`` before creating the model.\n\n https://github.com/NVIDIA/apex/tree/master/examples/simple/distributed shows detailed usage.\n https://github.com/NVIDIA/apex/tree/master/examples/imagenet shows another example\n that combines :class:`DistributedDataParallel` with mixed precision training.\n\n Args:\n module: Network definition to be run in multi-gpu/distributed mode.\n message_size (int, default=1e7): Minimum number of elements in a communication bucket.\n delay_allreduce (bool, default=False): Delay all communication to the end of the backward pass. This disables overlapping communication with computation.\n allreduce_trigger_params (list, optional, default=None): If supplied, should contain a list of parameters drawn from the model. Allreduces will be kicked off whenever one of these parameters receives its gradient (as opposed to when a bucket of size message_size is full). At the end of backward(), a cleanup allreduce to catch any remaining gradients will also be performed automatically. If allreduce_trigger_params is supplied, the message_size argument will be ignored.\n allreduce_always_fp32 (bool, default=False): Convert any FP16 gradients to FP32 before allreducing. This can improve stability for widely scaled-out runs.\n gradient_average (bool, default=True): Option to toggle whether or not DDP averages the allreduced gradients over processes. For proper scaling, the default value of True is recommended.\n gradient_predivide_factor (float, default=1.0): Allows perfoming the average of gradients over processes partially before and partially after the allreduce. Before allreduce: ``grads.mul_(1.0/gradient_predivide_factor)``. After allreduce: ``grads.mul_(gradient_predivide_factor/world size)``. This can reduce the stress on the dynamic range of FP16 allreduces for widely scaled-out runs.\n\n .. warning::\n If ``gradient_average=False``, the pre-allreduce division (``grads.mul_(1.0/gradient_predivide_factor)``) will still be applied, but the post-allreduce gradient averaging (``grads.mul_(gradient_predivide_factor/world size)``) will be omitted.\n\n \"\"\"\n\n def __init__(self, \n module, \n message_size=10000000, \n delay_allreduce=False, \n shared_param=None,\n allreduce_trigger_params=None,\n retain_allreduce_buffers=False,\n allreduce_always_fp32=False,\n gradient_average=True,\n gradient_predivide_factor=1.0):\n super(DistributedDataParallel, self).__init__()\n\n # Backward/forward compatibility around \n # https://github.com/pytorch/pytorch/commit/540ef9b1fc5506369a48491af8a285a686689b36 and\n # https://github.com/pytorch/pytorch/commit/044d00516ccd6572c0d6ab6d54587155b02a3b86\n if hasattr(dist, \"get_backend\"):\n self._backend = dist.get_backend()\n if hasattr(dist, \"DistBackend\"):\n self.backend_enum_holder = dist.DistBackend\n else:\n self.backend_enum_holder = dist.Backend\n else:\n self._backend = dist._backend \n self.backend_enum_holder = dist.dist_backend\n\n self.warn_on_half = True if self._backend == self.backend_enum_holder.GLOO else False\n\n if shared_param is not None:\n raise ValueError(\"shared_param is no longer supported as an option. It was misleadingly named from the start. It turns out overlapping communication with computation should work fine with shared parameters. If you still wish to delay communication to the end of the backward pass, use delay_allreduce=True|False instead.\") \n\n self.world_size = float(dist.get_world_size())\n\n self.retain_allreduce_buffers = retain_allreduce_buffers\n self.allreduce_always_fp32 = allreduce_always_fp32\n self.gradient_average = gradient_average\n self.gradient_predivide_factor = gradient_predivide_factor\n\n self.custom_allreduce_triggers = False\n if allreduce_trigger_params is not None:\n if delay_allreduce:\n raise ValueError(\"Setting allreduce_trigger_params is only valid if delay_allreduce=False.\") \n self.custom_allreduce_triggers = True\n self.allreduce_trigger_params = set([id(param) for param in allreduce_trigger_params])\n\n self.delay_allreduce = delay_allreduce\n self.message_size = message_size\n\n self.reduction_stream = torch.cuda.Stream()\n self.reduction_event = torch.cuda.Event(enable_timing=False, blocking=False) \n \n self.module = module\n\n self._disable_allreduce = False\n \n if self._backend == self.backend_enum_holder.NCCL:\n for param in self.module.parameters():\n assert param.is_cuda, \"NCCL backend only supports model parameters to be on GPU.\"\n\n self.active_params = []\n\n self.param_type_to_tmp_i = {\"torch.cuda.HalfTensor\" : 0, \n \"torch.cuda.FloatTensor\" : 1,\n \"torch.cuda.DoubleTensor\" : 2}\n\n if multi_tensor_applier.available:\n # TODO: I really need to centralize the C++ backed imports\n import amp_C\n self.multi_tensor_scale = amp_C.multi_tensor_scale\n self._overflow_buf = torch.cuda.IntTensor([0])\n\n self.create_hooks()\n\n flat_dist_call([param.data for param in self.module.parameters()], dist.broadcast, (0,) )\n\n\n def __setstate__(self, state):\n super(DistributedDataParallel, self).__setstate__(state)\n self.reduction_stream = torch.cuda.Stream()\n self.reduction_event = torch.cuda.Event(enable_timing=False, blocking=False) \n\n\n def __getstate__(self):\n attrs = copy.copy(self.__dict__)\n if self._backend != self.backend_enum_holder.NCCL:\n del attrs['self.reduction_stream']\n del attrs['self.reduction_event']\n return attrs\n\n def enable_allreduce(self):\n self._disable_allreduce = False\n\n def disable_allreduce(self):\n self._disable_allreduce = True\n \n # Broadcast rank 0's bucket structure across all processes, and have all processes \n # regenerate their bucket structures to match. \n def sync_bucket_structure(self):\n # Append leftover buckets\n for tmp_bucket in self.tmp_buckets:\n if len(tmp_bucket) > 0:\n self.active_i_buckets.append(tmp_bucket)\n\n self.num_buckets = len(self.active_i_buckets)\n self.bucket_sizes = [len(bucket) for bucket in self.active_i_buckets]\n\n info_tensor = torch.cuda.IntTensor([self.num_buckets] + \n self.bucket_sizes + \n list(chain(*self.active_i_buckets)))\n\n dist.broadcast(info_tensor, 0)\n\n info = [int(entry) for entry in info_tensor]\n\n self.num_buckets = info[0]\n self.bucket_sizes = info[1:self.num_buckets + 1] \n self.buckets = [[None for _ in range(self.bucket_sizes[i])] \n for i in range(self.num_buckets)] \n # Technically, active_i_buckets' work is done. But the information is still useful to\n # keep around. Therefore, refresh active_i_buckets based on rank 0 as well.\n self.active_i_buckets = [[None for _ in range(self.bucket_sizes[i])] \n for i in range(self.num_buckets)] \n \n flattened_buckets = info[self.num_buckets + 1:]\n flat_i = 0\n for bucket_idx in range(self.num_buckets): \n for bucket_loc in range(self.bucket_sizes[bucket_idx]):\n param_i = flattened_buckets[flat_i]\n self.active_i_buckets[bucket_idx][bucket_loc] = param_i \n self.param_id_to_bucket[id(self.active_params[param_i])] = (bucket_idx, bucket_loc)\n flat_i += 1 \n \n \n def create_hooks(self):\n # Fallback hook that's only called at the end of backward.\n # Used if you deliberately want to delay allreduces to the end, or to refresh the \n # bucket structure that will be used to overlap communication with computation in later\n # iterations.\n def allreduce_params():\n # Bucket record refresh\n if not self.delay_allreduce:\n if self.needs_refresh:\n self.sync_bucket_structure()\n\n self.needs_refresh = False\n\n self.allreduce_fallback()\n\n\n def overlapping_backward_epilogue():\n self.reduction_stream.record_event(self.reduction_event)\n torch.cuda.current_stream().wait_event(self.reduction_event)\n \n # Sanity checks that all the buckets were kicked off\n if self.next_bucket != self.num_buckets:\n raise RuntimeError(\"In epilogue, next_bucket ({}) != num_buckets ({}). \".format(\n self.next_bucket, self.num_buckets),\n \"This probably indicates some buckets were not allreduced.\")\n\n for actual, expected in zip(self.buckets_ready_size, self.bucket_sizes):\n if actual != expected:\n raise RuntimeError(\"Some param buckets were not allreduced.\")\n \n\n self.grad_accs = []\n for param in self.module.parameters():\n if param.requires_grad:\n def wrapper(param):\n param_tmp = param.expand_as(param)\n grad_acc = param_tmp.grad_fn.next_functions[0][0]\n\n def allreduce_hook(*unused):\n if not self._disable_allreduce:\n if self.delay_allreduce or self.needs_refresh:\n # TODO: How do we want to handle multiple backward passes between\n # each forward, e.g., backward passes with retain_graph=True?\n # needs_refresh and callback_queued are both vulnerable states.\n if not self.delay_allreduce and self.needs_refresh:\n # Use the backward pass to build the bucket structure on the fly.\n active_i = self.param_id_to_active_i[id(param)]\n\n # Float, half, and double tensors are grouped into buckets separately.\n current_type = self.param_type_to_tmp_i[param.type()]\n \n self.tmp_buckets[current_type].append(active_i) \n\n ship_tmp_bucket = False\n if self.custom_allreduce_triggers:\n if id(param) in self.allreduce_trigger_params:\n ship_tmp_bucket = True\n else:\n self.tmp_numels[current_type] += param.numel()\n if self.tmp_numels[current_type] >= self.message_size:\n ship_tmp_bucket = True\n\n # To consider: If custom_allreduce_triggers are in use, ship all\n # tmp_buckets, not just tmp_buckets[current_type].\n if ship_tmp_bucket:\n self.active_i_buckets.append(self.tmp_buckets[current_type])\n self.tmp_buckets[current_type] = []\n self.tmp_numels[current_type] = 0\n \n if not self.callback_queued:\n Variable._execution_engine.queue_callback(allreduce_params)\n self.callback_queued = True\n else:\n if not self.callback_queued:\n Variable._execution_engine.queue_callback(overlapping_backward_epilogue)\n self.callback_queued = True \n\n self.comm_ready_buckets(param)\n \n grad_acc.register_hook(allreduce_hook)\n self.grad_accs.append(grad_acc)\n\n wrapper(param)\n\n def allreduce_bucket(self, bucket):\n tensor = flatten(bucket)\n\n tensor_to_allreduce = tensor \n\n if self.allreduce_always_fp32:\n tensor_to_allreduce = tensor.float() \n\n if self.gradient_predivide_factor != 1.0:\n tensor_to_allreduce.mul_(1./self.gradient_predivide_factor)\n\n dist.all_reduce(tensor_to_allreduce)\n\n if self.gradient_average:\n if self.gradient_predivide_factor != self.world_size:\n tensor_to_allreduce.mul_(self.gradient_predivide_factor/self.world_size)\n\n if self.allreduce_always_fp32 and tensor is not tensor_to_allreduce:\n tensor.copy_(tensor_to_allreduce)\n \n return tensor\n \n\n def allreduce_maybe_retain(self, bucket, bucket_idx=-1):\n allreduced = self.allreduce_bucket(bucket)\n if self.retain_allreduce_buffers:\n if self.allreduce_buffers[bucket_idx] is not None:\n raise RuntimeError(\"The backward pass is attempting to replace an already-filled \"\n \"allreduce buffer. This is almost certainly an error.\")\n self.allreduce_buffers[bucket_idx] = allreduced\n else:\n if multi_tensor_applier.available:\n multi_tensor_applier(\n self.multi_tensor_scale,\n self._overflow_buf,\n [unflatten(allreduced, bucket), bucket],\n 1.0)\n else:\n for buf, synced in zip(bucket, unflatten(allreduced, bucket)):\n buf.copy_(synced)\n\n\n def allreduce_fallback(self):\n grads = [param.grad.data for param in self.module.parameters() if param.grad is not None]\n\n split_buckets = split_half_float_double(grads)\n\n # If retain_allreduce_buffers is True and delay_allreduce is False,\n # this will only be done during the first backward pass, ignored by the \n # training script, and overwritten in the next forward pass. So it's harmless. \n if self.retain_allreduce_buffers:\n self.allreduce_buffers = [None for _ in range(len(split_buckets))]\n \n for i, bucket in enumerate(split_buckets):\n allreduced = self.allreduce_maybe_retain(bucket, i)\n\n\n def comm_ready_buckets(self, param):\n # Need to do this in every hook for compatibility with Ruberry's streaming backward PR.\n # self.reduction_stream.wait_stream(torch.cuda.current_stream())\n\n bucket_idx, bucket_loc = self.param_id_to_bucket[id(param)]\n\n if self.buckets[bucket_idx][bucket_loc] is not None:\n raise RuntimeError(\"The backward pass is attempting to replace an already-filled \"\n \"bucket slot. This is almost certainly an error.\")\n\n self.buckets[bucket_idx][bucket_loc] = param.grad.data\n self.buckets_ready_size[bucket_idx] += 1\n\n if self.buckets_ready_size[bucket_idx] == self.bucket_sizes[bucket_idx]:\n if bucket_idx == self.next_bucket:\n torch.cuda.current_stream().record_event(self.reduction_event)\n self.reduction_stream.wait_event(self.reduction_event)\n with torch.cuda.stream(self.reduction_stream):\n self.allreduce_maybe_retain(self.buckets[bucket_idx], bucket_idx)\n\n self.next_bucket += 1\n\n # Reversing upstream's logic here, because we constructed our buckets based on\n # the order things were received during backward.\n if len(self.ready_buckets_not_reduced) > 0:\n sorted_todo = sorted(self.ready_buckets_not_reduced)\n for i in sorted_todo:\n # Nothing can be reduced now\n if i > self.next_bucket:\n break\n elif i == self.next_bucket:\n self.allreduce_maybe_retain(self.buckets[i], i)\n self.ready_buckets_not_reduced.remove(i)\n self.next_bucket += 1 \n else:\n raise ValueError(\"i should always be >= next_bucket\")\n else:\n self.ready_buckets_not_reduced.add(bucket_idx)\n\n \n def forward(self, *inputs, **kwargs):\n result = self.module(*inputs, **kwargs)\n \n if not self._disable_allreduce:\n if not self.delay_allreduce:\n param_list = [param for param in self.module.parameters() if param.requires_grad]\n\n # Conditions under which to refresh self.record\n # Forward has the authority to set needs_refresh to True, but only allreduce_params\n # in backward has the authority to set needs_refresh to False.\n # Parentheses are not necessary for correct order of operations, but make the intent clearer.\n if ((not self.active_params) or \n (len(param_list) != len(self.active_params)) or\n any([param1 is not param2 for param1, param2 in zip(param_list, self.active_params)])):\n self.needs_refresh = True\n\n if self.needs_refresh:\n self.active_i_buckets = []\n self.buckets = []\n self.tmp_buckets = [[], [], []] # [running half, float, double buckets]\n self.tmp_numels = [0, 0, 0]\n self.bucket_sizes = []\n self.param_id_to_active_i = {id(param) : i for i, param in enumerate(param_list)} \n self.param_id_to_bucket = {}\n else:\n self.buckets = [[None for _ in range(self.bucket_sizes[i])] \n for i in range(self.num_buckets)] \n self.buckets_ready_size = [0 for i in range(self.num_buckets)]\n if(self.retain_allreduce_buffers):\n self.allreduce_buffers = [None for _ in range(self.num_buckets)]\n self.next_bucket = 0\n self.ready_buckets_not_reduced = set()\n \n self.active_params = param_list\n\n self.callback_queued = False\n \n return result\n" }, { "path": "apex/apex/parallel/multiproc.py", "content": "import torch\nimport sys\nimport subprocess\n\ndef docstring_hack():\n \"\"\"\n Multiproc file which will launch a set of processes locally for multi-gpu\n usage: python -m apex.parallel.multiproc main.py ...\n \"\"\"\n pass\n\nargslist = list(sys.argv)[1:]\nworld_size = torch.cuda.device_count()\n\nif '--world-size' in argslist:\n world_size = int(argslist[argslist.index('--world-size')+1])\nelse:\n argslist.append('--world-size')\n argslist.append(str(world_size))\n\nworkers = []\n\nfor i in range(world_size):\n if '--rank' in argslist:\n argslist[argslist.index('--rank')+1] = str(i)\n else:\n argslist.append('--rank')\n argslist.append(str(i))\n stdout = None if i == 0 else open(\"GPU_\"+str(i)+\".log\", \"w\")\n print(argslist)\n p = subprocess.Popen([str(sys.executable)]+argslist, stdout=stdout)\n workers.append(p)\n\nfor p in workers:\n p.wait()\n" }, { "path": "apex/apex/parallel/optimized_sync_batchnorm.py", "content": "import torch\nfrom torch.nn.modules.batchnorm import _BatchNorm\nfrom torch.nn import functional as F\n\nimport syncbn\nfrom .optimized_sync_batchnorm_kernel import SyncBatchnormFunction\n\n\nclass SyncBatchNorm(_BatchNorm):\n \"\"\"\n synchronized batch normalization module extented from `torch.nn.BatchNormNd`\n with the added stats reduction across multiple processes.\n :class:`apex.parallel.SyncBatchNorm` is designed to work with\n `DistributedDataParallel`.\n\n When running in training mode, the layer reduces stats across all processes\n to increase the effective batchsize for normalization layer. This is useful\n in applications where batch size is small on a given process that would\n diminish converged accuracy of the model. The model uses collective\n communication package from `torch.distributed`.\n\n When running in evaluation mode, the layer falls back to\n `torch.nn.functional.batch_norm`\n\n Args:\n num_features: :math:`C` from an expected input of size\n :math:`(N, C, L)` or :math:`L` from input of size :math:`(N, L)`\n eps: a value added to the denominator for numerical stability.\n Default: 1e-5\n momentum: the value used for the running_mean and running_var\n computation. Can be set to ``None`` for cumulative moving average\n (i.e. simple average). Default: 0.1\n affine: a boolean value that when set to ``True``, this module has\n learnable affine parameters. Default: ``True``\n track_running_stats: a boolean value that when set to ``True``, this\n module tracks the running mean and variance, and when set to ``False``,\n this module does not track such statistics and always uses batch\n statistics in both training and eval modes. Default: ``True``\n process_group: pass in a process group within which the stats of the\n mini-batch is being synchronized. ``None`` for using default process\n group\n channel_last: a boolean value that when set to ``True``, this module\n take the last dimension of the input tensor to be the channel\n dimension. Default: False\n\n Examples::\n >>> # channel first tensor\n >>> sbn = apex.parallel.SyncBatchNorm(100).cuda()\n >>> inp = torch.randn(10, 100, 14, 14).cuda()\n >>> out = sbn(inp)\n >>> inp = torch.randn(3, 100, 20).cuda()\n >>> out = sbn(inp)\n >>> # channel last tensor\n >>> sbn = apex.parallel.SyncBatchNorm(100, channel_last=True).cuda()\n >>> inp = torch.randn(10, 14, 14, 100).cuda()\n \"\"\"\n\n def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True, track_running_stats=True, process_group=None, channel_last=False):\n super(SyncBatchNorm, self).__init__(num_features, eps=eps, momentum=momentum, affine=affine, track_running_stats=track_running_stats)\n self.process_group = process_group\n self.channel_last = channel_last\n\n def _specify_process_group(self, process_group):\n self.process_group = process_group\n\n def _specify_channel_last(self, channel_last):\n self.channel_last = channel_last\n\n def forward(self, input):\n # if input.dim() == 2, we switch to channel_last for efficient memory accessing\n channel_last = self.channel_last if input.dim() != 2 else True\n\n if not self.training and self.track_running_stats and not channel_last:\n # fall back to pytorch implementation for inference\n return F.batch_norm(input, self.running_mean, self.running_var, self.weight, self.bias, False, 0.0, self.eps)\n else:\n exponential_average_factor = 0.0\n if self.training and self.track_running_stats:\n self.num_batches_tracked += 1\n if self.momentum is None:\n exponential_average_factor = 1.0 / float(self.num_batches_tracked)\n else:\n exponential_average_factor = self.momentum\n return SyncBatchnormFunction.apply(input, self.weight, self.bias, self.running_mean, self.running_var, self.eps, self.training or not self.track_running_stats, exponential_average_factor, self.process_group, channel_last)\n" }, { "path": "apex/apex/parallel/optimized_sync_batchnorm_kernel.py", "content": "import torch\nfrom torch.autograd.function import Function\n\nimport syncbn\nfrom apex.parallel import ReduceOp\n\nclass SyncBatchnormFunction(Function):\n\n @staticmethod\n def forward(ctx, input, weight, bias, running_mean, running_variance, eps, track_running_stats = True, momentum = 1.0, process_group = None, channel_last = False):\n torch.cuda.nvtx.range_push(\"sync_BN_fw\")\n input = input.contiguous()\n world_size = 0\n\n mean = None\n var_biased = None\n inv_std = None\n var = None\n out = None\n count = None\n if track_running_stats:\n if channel_last:\n count = int(input.numel()/input.size(-1))\n mean, var_biased = syncbn.welford_mean_var_c_last(input)\n else:\n count = int(input.numel()/input.size(1))\n mean, var_biased = syncbn.welford_mean_var(input)\n\n if torch.distributed.is_initialized():\n if not process_group:\n process_group = torch.distributed.group.WORLD\n world_size = torch.distributed.get_world_size(process_group)\n mean_all = torch.empty(world_size, mean.size(0), dtype=mean.dtype, device=mean.device)\n var_all = torch.empty(world_size, var_biased.size(0), dtype=var_biased.dtype, device=var_biased.device)\n mean_l = [mean_all.narrow(0, i, 1) for i in range(world_size)]\n var_l = [var_all.narrow(0, i, 1) for i in range(world_size)]\n torch.distributed.all_gather(mean_l, mean, process_group)\n torch.distributed.all_gather(var_l, var_biased, process_group)\n mean, var, inv_std = syncbn.welford_parallel(mean_all, var_all, count, eps)\n # TODO(Jie): should do fp32 math instead!\n else:\n inv_std = 1.0 / torch.sqrt(var_biased + eps)\n var = var_biased * (count) / (count-1) \n\n if count == 1 and world_size < 2:\n raise ValueError('Expected more than 1 value per channel when training, got input size{}'.format(input.size()))\n\n r_m_inc = mean if running_mean.dtype != torch.float16 else mean.half()\n r_v_inc = var if running_variance.dtype != torch.float16 else var.half()\n running_mean.data = running_mean.data * (1-momentum) + momentum*r_m_inc\n running_variance.data = running_variance.data * (1-momentum) + momentum*r_v_inc\n else:\n mean = running_mean.data\n inv_std = 1.0 / torch.sqrt(running_variance.data + eps)\n\n ctx.save_for_backward(input, weight, mean, inv_std)\n ctx.process_group = process_group\n ctx.channel_last = channel_last\n ctx.world_size = world_size\n\n if channel_last:\n out = syncbn.batchnorm_forward_c_last(input, mean, inv_std, weight, bias)\n else:\n out = syncbn.batchnorm_forward(input, mean, inv_std, weight, bias)\n\n torch.cuda.nvtx.range_pop()\n return out\n\n @staticmethod\n def backward(ctx, grad_output):\n grad_output = grad_output.contiguous()\n torch.cuda.nvtx.range_push(\"sync_BN_bw\")\n # mini batch mean & var are calculated by forward path.\n # mu = 1./N*np.sum(h, axis = 0)\n # var = 1./N*np.sum((h-mu)**2, axis = 0)\n saved_input, weight, mean, inv_std = ctx.saved_tensors\n process_group = ctx.process_group\n channel_last = ctx.channel_last\n world_size = ctx.world_size\n grad_input = grad_weight = grad_bias = None\n\n # TODO(jie): why do I have to clone here? life time of grad_output?\n if channel_last:\n mean_dy, mean_dy_xmu, grad_weight, grad_bias = syncbn.reduce_bn_c_last(grad_output, saved_input, mean, inv_std, weight)\n else:\n mean_dy, mean_dy_xmu, grad_weight, grad_bias = syncbn.reduce_bn(grad_output, saved_input, mean, inv_std, weight)\n\n # calculate grad_input\n if ctx.needs_input_grad[0]:\n\n if torch.distributed.is_initialized():\n torch.distributed.all_reduce(\n mean_dy, ReduceOp.SUM, process_group)\n mean_dy = mean_dy / world_size\n torch.distributed.all_reduce(\n mean_dy_xmu, ReduceOp.SUM, process_group)\n mean_dy_xmu = mean_dy_xmu / world_size\n if channel_last:\n grad_input = syncbn.batchnorm_backward_c_last(grad_output, saved_input, mean, inv_std, weight, mean_dy, mean_dy_xmu)\n else:\n grad_input = syncbn.batchnorm_backward(grad_output, saved_input, mean, inv_std, weight, mean_dy, mean_dy_xmu)\n\n if weight is None or not ctx.needs_input_grad[1]:\n grad_weight = None\n\n if weight is None or not ctx.needs_input_grad[2]:\n grad_bias = None\n\n torch.cuda.nvtx.range_pop()\n return grad_input, grad_weight, grad_bias, None, None, None, None, None, None, None\n" }, { "path": "apex/apex/parallel/sync_batchnorm.py", "content": "import torch\nfrom torch.nn.modules.batchnorm import _BatchNorm\nfrom torch.nn import functional as F\n\nfrom .sync_batchnorm_kernel import SyncBatchnormFunction\nfrom apex.parallel import ReduceOp\n\n\nclass SyncBatchNorm(_BatchNorm):\n \"\"\"\n synchronized batch normalization module extented from ``torch.nn.BatchNormNd``\n with the added stats reduction across multiple processes.\n :class:`apex.parallel.SyncBatchNorm` is designed to work with\n ``DistributedDataParallel``.\n\n When running in training mode, the layer reduces stats across all processes\n to increase the effective batchsize for normalization layer. This is useful\n in applications where batch size is small on a given process that would\n diminish converged accuracy of the model. The model uses collective\n communication package from ``torch.distributed``.\n\n When running in evaluation mode, the layer falls back to\n ``torch.nn.functional.batch_norm``.\n\n Args:\n num_features: :math:`C` from an expected input of size\n :math:`(N, C, L)` or :math:`L` from input of size :math:`(N, L)`\n eps: a value added to the denominator for numerical stability.\n Default: 1e-5\n momentum: the value used for the running_mean and running_var\n computation. Can be set to ``None`` for cumulative moving average\n (i.e. simple average). Default: 0.1\n affine: a boolean value that when set to ``True``, this module has\n learnable affine parameters. Default: ``True``\n track_running_stats: a boolean value that when set to ``True``, this\n module tracks the running mean and variance, and when set to ``False``,\n this module does not track such statistics and always uses batch\n statistics in both training and eval modes. Default: ``True``\n\n Example::\n\n >>> sbn = apex.parallel.SyncBatchNorm(100).cuda()\n >>> inp = torch.randn(10, 100, 14, 14).cuda()\n >>> out = sbn(inp)\n >>> inp = torch.randn(3, 100, 20).cuda()\n >>> out = sbn(inp)\n \"\"\"\n\n warned = False\n\n def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True, track_running_stats=True, process_group=None, channel_last=False):\n if channel_last == True:\n raise AttributeError(\"channel_last is not supported by primitive SyncBatchNorm implementation. Try install apex with `--cuda_ext` if channel_last is desired.\")\n\n if not SyncBatchNorm.warned:\n print(\"Warning: using Python fallback for SyncBatchNorm, possibly because apex was installed without --cuda_ext. The exception raised when attempting to import the cuda backend was: \", self.syncbn_import_error)\n SyncBatchNorm.warned = True\n\n super(SyncBatchNorm, self).__init__(num_features, eps=eps, momentum=momentum, affine=affine, track_running_stats=track_running_stats)\n self.process_group = process_group\n\n def _specify_process_group(self, process_group):\n self.process_group = process_group\n\n def forward(self, input):\n torch.cuda.nvtx.range_push(\"sync_bn_fw_with_mean_var\")\n mean = None\n var = None\n cast = None\n out = None\n\n # casting to handle mismatch input type to layer type\n if self.running_mean is not None:\n if self.running_mean.dtype != input.dtype:\n input = input.to(self.running_mean.dtype)\n cast = input.dtype\n elif self.weight is not None:\n if self.weight.dtype != input.dtype:\n input = input.to(self.weight.dtype)\n cast = input.dtype\n\n if not self.training and self.track_running_stats:\n # fall back to pytorch implementation for inference\n torch.cuda.nvtx.range_pop()\n out = F.batch_norm(input, self.running_mean, self.running_var, self.weight, self.bias, False, 0.0, self.eps)\n else:\n process_group = self.process_group\n world_size = 1\n if not self.process_group:\n process_group = torch.distributed.group.WORLD\n self.num_batches_tracked += 1\n with torch.no_grad():\n channel_first_input = input.transpose(0, 1).contiguous()\n squashed_input_tensor_view = channel_first_input.view(\n channel_first_input.size(0), -1)\n # total number of data points for each variance entry. Used to calculate unbiased variance estimate\n m = None\n local_m = float(squashed_input_tensor_view.size()[1])\n local_mean = torch.mean(squashed_input_tensor_view, 1)\n local_sqr_mean = torch.pow(\n squashed_input_tensor_view, 2).mean(1)\n if torch.distributed.is_initialized():\n world_size = torch.distributed.get_world_size(process_group)\n torch.distributed.all_reduce(\n local_mean, ReduceOp.SUM, process_group)\n mean = local_mean / world_size\n torch.distributed.all_reduce(\n local_sqr_mean, ReduceOp.SUM, process_group)\n sqr_mean = local_sqr_mean / world_size\n m = local_m * world_size\n else:\n m = local_m\n mean = local_mean\n sqr_mean = local_sqr_mean\n # var(x) = E (( x - mean_x ) ** 2)\n # = 1 / N * sum ( x - mean_x ) ** 2\n # = 1 / N * sum (x**2) - mean_x**2\n var = sqr_mean - mean.pow(2)\n\n if self.running_mean is not None:\n self.running_mean = self.momentum * mean + \\\n (1 - self.momentum) * self.running_mean\n if self.running_var is not None:\n # as noted by the paper, we used unbiased variance estimate of the mini-batch\n # Var[x] = m / (m-1) * Eb (sample_variance)\n self.running_var = m / \\\n (m-1) * self.momentum * var + \\\n (1 - self.momentum) * self.running_var\n torch.cuda.nvtx.range_pop()\n out = SyncBatchnormFunction.apply(input, self.weight, self.bias, mean, var, self.eps, process_group, world_size)\n out = out.to(cast)\n" }, { "path": "apex/apex/parallel/sync_batchnorm_kernel.py", "content": "import torch\nfrom torch.autograd.function import Function\n\nfrom apex.parallel import ReduceOp\n\n\nclass SyncBatchnormFunction(Function):\n\n @staticmethod\n def forward(ctx, input, weight, bias, running_mean, running_variance, eps, process_group, world_size):\n torch.cuda.nvtx.range_push(\"sync_BN_fw\")\n # transpose it to channel last to support broadcasting for input with different rank\n c_last_input = input.transpose(1, -1).contiguous().clone()\n\n ctx.save_for_backward(c_last_input, weight, bias,\n running_mean, running_variance)\n ctx.eps = eps\n ctx.process_group = process_group\n ctx.world_size = world_size\n\n c_last_input = (c_last_input - running_mean) / \\\n torch.sqrt(running_variance + eps)\n\n if weight is not None:\n c_last_input = c_last_input * weight\n if bias is not None:\n c_last_input = c_last_input + bias\n\n torch.cuda.nvtx.range_pop()\n return c_last_input.transpose(1, -1).contiguous().clone()\n\n @staticmethod\n def backward(ctx, grad_output):\n torch.cuda.nvtx.range_push(\"sync_BN_bw\")\n # mini batch mean & var are calculated by forward path.\n # mu = 1./N*np.sum(h, axis = 0)\n # var = 1./N*np.sum((h-mu)**2, axis = 0)\n c_last_input, weight, bias, running_mean, running_variance = ctx.saved_tensors\n\n eps = ctx.eps\n process_group = ctx.process_group\n world_size = ctx.world_size\n grad_input = grad_weight = grad_bias = None\n num_features = running_mean.size()[0]\n\n # transpose it to channel last to support broadcasting for input with different rank\n torch.cuda.nvtx.range_push(\"carilli field\")\n c_last_grad = grad_output.transpose(1, -1).contiguous()\n # squash non-channel dimension so we can easily calculate mean\n c_grad = c_last_grad.view(-1, num_features).contiguous()\n torch.cuda.nvtx.range_pop()\n\n # calculate grad_input\n if ctx.needs_input_grad[0]:\n # dh = gamma * (var + eps)**(-1. / 2.) * (dy - np.mean(dy, axis=0)\n # - (h - mu) * (var + eps)**(-1.0) * np.mean(dy * (h - mu), axis=0))\n mean_dy = c_grad.mean(0)\n mean_dy_xmu = (c_last_grad * (c_last_input -\n running_mean)).view(-1, num_features).mean(0)\n if torch.distributed.is_initialized():\n torch.distributed.all_reduce(\n mean_dy, ReduceOp.SUM, process_group)\n mean_dy = mean_dy / world_size\n torch.distributed.all_reduce(\n mean_dy_xmu, ReduceOp.SUM, process_group)\n mean_dy_xmu = mean_dy_xmu / world_size\n c_last_grad_input = (c_last_grad - mean_dy - (c_last_input - running_mean) / (\n running_variance + eps) * mean_dy_xmu) / torch.sqrt(running_variance + eps)\n if weight is not None:\n c_last_grad_input.mul_(weight)\n grad_input = c_last_grad_input.transpose(1, -1).contiguous()\n\n # calculate grad_weight\n grad_weight = None\n if weight is not None and ctx.needs_input_grad[1]:\n # dgamma = np.sum((h - mu) * (var + eps)**(-1. / 2.) * dy, axis=0)\n grad_weight = ((c_last_input - running_mean) / torch.sqrt(\n running_variance + eps) * c_last_grad).view(-1, num_features).sum(0)\n\n # calculate grad_bias\n grad_bias = None\n if bias is not None and ctx.needs_input_grad[2]:\n # dbeta = np.sum(dy, axis=0)\n grad_bias = c_grad.sum(0)\n\n torch.cuda.nvtx.range_pop()\n return grad_input, grad_weight, grad_bias, None, None, None, None, None\n" }, { "path": "apex/apex/reparameterization/README.md", "content": "Under construction...\n" }, { "path": "apex/apex/reparameterization/__init__.py", "content": "from .weight_norm import WeightNorm\r\nfrom .reparameterization import Reparameterization\r\n\r\ndef apply_weight_norm(module, name='', dim=0, hook_child=True):\r\n \"\"\"\r\n Applies weight normalization to a parameter in the given module.\r\n If no parameter is provided, applies weight normalization to all\r\n parameters in model (except 1-d vectors and scalars).\r\n\r\n .. math::\r\n \\mathbf{w} = g \\dfrac{\\mathbf{v}}{\\|\\mathbf{v}\\|}\r\n\r\n Weight normalization is a reparameterization that decouples the magnitude\r\n of a weight tensor from its direction. This replaces the parameter specified\r\n by `name` (e.g. \"weight\") with two parameters: one specifying the magnitude\r\n (e.g. \"weight_g\") and one specifying the direction (e.g. \"weight_v\").\r\n Weight normalization is implemented via a hook that recomputes the weight\r\n tensor from the magnitude and direction before every :meth:`~Module.forward`\r\n call.\r\n\r\n By default, with `dim=0`, the norm is computed independently per output\r\n channel/plane. To compute a norm over the entire weight tensor, use\r\n `dim=None`.\r\n\r\n See https://arxiv.org/abs/1602.07868\r\n\r\n Args:\r\n module (nn.Module): containing module\r\n name (str, optional): name of weight parameter\r\n dim (int, optional): dimension over which to compute the norm\r\n hook_child (boolean, optional): adds reparameterization hook to direct parent of the \r\n parameters. If False, it's added to `module` instead. Default: True\r\n\r\n Returns:\r\n The original module with the weight norm hook\r\n\r\n Example::\r\n\r\n >>> m = apply_weight_norm(nn.Linear(20, 40), name='weight')\r\n Linear (20 -> 40)\r\n >>> m.weight_g.size()\r\n torch.Size([40, 1])\r\n >>> m.weight_v.size()\r\n torch.Size([40, 20])\r\n\r\n \"\"\"\r\n return apply_reparameterization(module, reparameterization=WeightNorm, hook_child=hook_child,\r\n name=name, dim=dim)\r\n\r\ndef remove_weight_norm(module, name='', remove_all=False):\r\n \"\"\"\r\n Removes the weight normalization reparameterization of a parameter from a module.\r\n If no parameter is supplied then all weight norm parameterizations are removed.\r\n Args:\r\n module (nn.Module): containing module\r\n name (str, optional): name of weight parameter\r\n Example:\r\n >>> m = apply_weight_norm(nn.Linear(20, 40))\r\n >>> remove_weight_norm(m)\r\n \"\"\"\r\n return remove_reparameterization(module, reparameterization=WeightNorm,\r\n name=name, remove_all=remove_all)\r\n\r\ndef apply_reparameterization(module, reparameterization=None, name='', dim=0, hook_child=True):\r\n \"\"\"\r\n Applies a given weight reparameterization (such as weight normalization) to\r\n a parameter in the given module. If no parameter is given, applies the reparameterization\r\n to all parameters in model (except 1-d vectors and scalars).\r\n\r\n Args:\r\n module (nn.Module): containing module\r\n reparameterization (Reparameterization): reparamaterization class to apply\r\n name (str, optional): name of weight parameter\r\n dim (int, optional): dimension over which to perform reparameterization op\r\n hook_child (boolean, optional): adds reparameterization hook to direct parent of the \r\n parameters. If False, it's added to `module` instead. Default: True\r\n\r\n Returns:\r\n The original module with the reparameterization hook\r\n\r\n Example::\r\n\r\n >>> m = apply_reparameterization(nn.Linear(20, 40), WeightNorm)\r\n Linear (20 -> 40)\r\n\r\n \"\"\"\r\n assert reparameterization is not None\r\n if name != '':\r\n Reparameterization.apply(module, name, dim, reparameterization, hook_child)\r\n else:\r\n names = list(module.state_dict().keys())\r\n for name in names:\r\n apply_reparameterization(module, reparameterization, name, dim, hook_child)\r\n return module\r\n\r\ndef remove_reparameterization(module, reparameterization=Reparameterization,\r\n name='', remove_all=False):\r\n \"\"\"\r\n Removes the given reparameterization of a parameter from a module.\r\n If no parameter is supplied then all reparameterizations are removed.\r\n Args:\r\n module (nn.Module): containing module\r\n reparameterization (Reparameterization): reparamaterization class to apply\r\n name (str, optional): name of weight parameter\r\n remove_all (bool, optional): if True, remove all reparamaterizations of given type. Default: False\r\n Example:\r\n >>> m = apply_reparameterization(nn.Linear(20, 40),WeightNorm)\r\n >>> remove_reparameterization(m)\r\n \"\"\"\r\n if name != '' or remove_all:\r\n to_remove = []\r\n for k, hook in module._forward_pre_hooks.items():\r\n if isinstance(hook, reparameterization) and (hook.name == name or remove_all):\r\n hook.remove(module)\r\n to_remove.append(k)\r\n if len(to_remove) > 0:\r\n for k in to_remove:\r\n del module._forward_pre_hooks[k]\r\n return module\r\n if not remove_all:\r\n raise ValueError(\"reparameterization of '{}' not found in {}\"\r\n .format(name, module))\r\n else:\r\n modules = [module]+[x for x in module.modules()]\r\n for m in modules:\r\n remove_reparameterization(m, reparameterization=reparameterization, remove_all=True)\r\n return module\r\n" }, { "path": "apex/apex/reparameterization/reparameterization.py", "content": "import torch\r\nfrom torch.nn.parameter import Parameter\r\nimport sys\r\nclass Reparameterization(object):\r\n \"\"\"\r\n Class interface for performing weight reparameterizations\r\n Arguments:\r\n name (str): name of weight parameter\r\n dim (int): dimension over which to compute the norm\r\n module (nn.Module): parent module to which param `name` is registered to\r\n retain_forward (bool, optional): if False deletes weight on call to \r\n module.backward. Used to avoid memory leaks with DataParallel Default: True\r\n Attributes:\r\n reparameterization_names (list, str): contains names of all parameters \r\n needed to compute reparameterization.\r\n backward_hook_key (int): torch.utils.hooks.RemovableHandle.id for hook used in module backward pass.\r\n \"\"\"\r\n\r\n def __init__(self, name, dim, module, retain_forward=True):\r\n self.name = name\r\n self.dim = dim\r\n self.evaluated = False\r\n self.retain_forward = retain_forward\r\n self.reparameterization_names = []\r\n self.backward_hook_key = None\r\n self.module = module\r\n\r\n def compute_weight(self, module=None, name=None):\r\n \"\"\"\r\n Computes reparameterized weight value to assign value to module attribute\r\n with name `name`.\r\n See WeightNorm class for example.\r\n Arguments:\r\n module (nn.Module): module with weight we'd like to reparameterize\r\n Returns:\r\n w (Tensor): Tensor object containing value of reparameterized weight\r\n \"\"\"\r\n raise NotImplementedError\r\n\r\n def reparameterize(self, name, weight, dim):\r\n \"\"\"\r\n Creates Parameters to be used for reparameterization and creates names that\r\n for attributes for the module these Parameters will correspond to.\r\n The parameters will be registered according to the names provided.\r\n See WeightNorm class for example.\r\n Arguments:\r\n module (nn.Module): module with weight we'd like to reparameterize\r\n name (str, optional): name of weight parameter\r\n dim (int, optional): dimension over which to compute parameterization\r\n Returns:\r\n names (list, str): names of Parameters to be used for reparameterization\r\n params (list, Parameter): Parameters to be used for reparameterization\r\n \"\"\"\r\n raise NotImplementedError\r\n\r\n @staticmethod\r\n def apply(module, name, dim, reparameterization=None, hook_child=True):\r\n \"\"\"\r\n Applies reparametrization to module's `name` parameter and modifies instance attributes as appropriate.\r\n `hook_child` adds reparameterization hook to direct parent of the parameters. If False, it's added to `module` instead.\r\n \"\"\"\r\n if reparameterization is None:\r\n reparameterization = Reparameterization\r\n module2use, name2use = Reparameterization.get_module_and_name(module, name)\r\n # does not work on sparse\r\n if name2use is None or isinstance(module2use, (torch.nn.Embedding, torch.nn.EmbeddingBag)):\r\n return\r\n\r\n if hook_child:\r\n fn = reparameterization(name2use, dim, module2use)\r\n else:\r\n fn = reparameterization(name, dim, module)\r\n\r\n weight = getattr(module2use, name2use)\r\n if weight.dim() <= 1:\r\n return\r\n\r\n # remove weight from parameter list\r\n del module2use._parameters[name2use]\r\n\r\n # add parameters of reparameterization of parameter to module\r\n names, params = fn.reparameterize(name2use, weight, dim)\r\n for n, p in zip(names, params):\r\n module2use.register_parameter(n, p)\r\n\r\n # add parameters to reparameterization so they can be removed later\r\n fn.reparameterization_names = names\r\n\r\n setattr(module2use, name2use, None)\r\n\r\n hook_module = module2use\r\n if not hook_child:\r\n hook_module = module\r\n # recompute weight before every forward()\r\n hook_module.register_forward_pre_hook(fn)\r\n\r\n # remove weight during backward\r\n handle = hook_module.register_backward_hook(fn.backward_hook)\r\n # get hook key so we can delete it later\r\n fn.backward_hook_key = handle.id\r\n\r\n return fn\r\n\r\n @staticmethod\r\n def get_module_and_name(module, name):\r\n \"\"\"\r\n recursively fetches (possible) child module and name of weight to be reparameterized\r\n \"\"\"\r\n name2use = None\r\n module2use = None\r\n names = name.split('.')\r\n if len(names) == 1 and names[0] != '':\r\n name2use = names[0]\r\n module2use = module\r\n elif len(names) > 1:\r\n module2use = module\r\n name2use = names[0]\r\n for i in range(len(names)-1):\r\n module2use = getattr(module2use, name2use)\r\n name2use = names[i+1]\r\n return module2use, name2use\r\n\r\n def get_params(self, module):\r\n \"\"\"gets params of reparameterization based on known attribute names\"\"\"\r\n return [getattr(module, n) for n in self.reparameterization_names]\r\n\r\n def remove(self, module):\r\n \"\"\"removes reparameterization and backward hook (does not remove forward hook)\"\"\"\r\n module2use, name2use = Reparameterization.get_module_and_name(module, self.name)\r\n for p in self.get_params(module2use):\r\n p.requires_grad = False\r\n weight = self.compute_weight(module2use, name2use)\r\n delattr(module2use, name2use)\r\n for n in self.reparameterization_names:\r\n del module2use._parameters[n]\r\n module2use.register_parameter(name2use, Parameter(weight.data))\r\n del module._backward_hooks[self.backward_hook_key]\r\n\r\n def __call__(self, module, inputs):\r\n \"\"\"callable hook for forward pass\"\"\"\r\n module2use, name2use = Reparameterization.get_module_and_name(module, self.name)\r\n _w = getattr(module2use, name2use)\r\n if not self.evaluated or _w is None:\r\n setattr(module2use, name2use, self.compute_weight(module2use, name2use))\r\n self.evaluated = True\r\n\r\n def backward_hook(self, module, grad_input, grad_output):\r\n \"\"\"callable hook for backward pass\"\"\"\r\n module2use, name2use = Reparameterization.get_module_and_name(module, self.name)\r\n wn = getattr(module2use, name2use)\r\n self.evaluated = False\r\n" }, { "path": "apex/apex/reparameterization/weight_norm.py", "content": "import torch\r\nfrom torch.nn.parameter import Parameter\r\nfrom ..fp16_utils import Fused_Weight_Norm\r\nimport time\r\n\r\nfrom .reparameterization import Reparameterization\r\n\r\ndef _norm(p, dim):\r\n \"\"\"Computes the norm over all dimensions except dim\"\"\"\r\n if dim is None:\r\n return p.norm()\r\n elif dim == 0:\r\n output_size = (p.size(0),) + (1,) * (p.dim() - 1)\r\n return p.contiguous().view(p.size(0), -1).norm(dim=1).view(*output_size)\r\n elif dim == p.dim() - 1:\r\n output_size = (1,) * (p.dim() - 1) + (p.size(-1),)\r\n return p.contiguous().view(-1, p.size(-1)).norm(dim=0).view(*output_size)\r\n return _norm(p.transpose(0, dim), 0).transpose(0, dim)\r\n\r\nHALF_TYPES = (torch.cuda.HalfTensor, torch.HalfTensor)\r\n\r\nclass WeightNorm(Reparameterization):\r\n \"\"\"\r\n Weight normalization is a reparameterization that decouples the magnitude\r\n of a weight tensor from its direction. This replaces the parameter specified\r\n by `name` (e.g. \"weight\") with two parameters: one specifying the magnitude\r\n (e.g. \"weight_g\") and one specifying the direction (e.g. \"weight_v\").\r\n Weight normalization is implemented via a hook that recomputes the weight\r\n tensor from the magnitude and direction before every :meth:`~Module.forward`\r\n call.\r\n\r\n .. math::\r\n \\mathbf{w} = g \\dfrac{\\mathbf{v}}{\\|\\mathbf{v}\\|}\r\n\r\n By default, with `dim=0`, the norm is computed independently per output\r\n channel/plane. To compute a norm over the entire weight tensor, use\r\n `dim=None`.\r\n \"\"\"\r\n def compute_weight(self, module=None, name=None):\r\n \"\"\"\r\n Computes weight normalized weight value to assign value to module attribute\r\n with name `name`.\r\n Arguments:\r\n module (nn.Module): module with weight we'd like to reparameterize\r\n Returns:\r\n w (Tensor): Tensor object containing value of reparameterized weight\r\n \"\"\"\r\n if module is None:\r\n module = self.module\r\n if name is None:\r\n name = self.name\r\n module, name = Reparameterization.get_module_and_name(module, name)\r\n g = getattr(module, name + '_g')\r\n v = getattr(module, name + '_v')\r\n\r\n fused_weight_norm = Fused_Weight_Norm.apply\r\n v = v.contiguous()\r\n w = fused_weight_norm(v, g, self.dim)\r\n\r\n return w\r\n\r\n def reparameterize(self, name, weight, dim):\r\n \"\"\"\r\n Creates Parameters v and gto be used for weight normalization\r\n and creates names that for attributes for the module these Parameters\r\n will correspond to. The parameters will be registered according to the names\r\n provided.\r\n Arguments:\r\n module (nn.Module): module with weight we'd like to reparameterize\r\n name (str, optional): name of weight parameter\r\n dim (int, optional): dimension over which to compute parameterization\r\n Returns:\r\n names (list, str): names of Parameters to be used for reparameterization\r\n params (list, Parameter): Parameters to be used for reparameterization\r\n \"\"\"\r\n names = [name + '_g', name + '_v']\r\n params = [Parameter(_norm(weight, dim).data), Parameter(weight.data)]\r\n return names, params\r\n" }, { "path": "apex/apex.patch", "content": "diff --git a/csrc/fused_adam_cuda_kernel.cu b/csrc/fused_adam_cuda_kernel.cu\nindex 34f7aa2..95581d1 100644\n--- a/csrc/fused_adam_cuda_kernel.cu\n+++ b/csrc/fused_adam_cuda_kernel.cu\n@@ -19,8 +19,8 @@ typedef enum{\n \n template \n __global__ void adam_cuda_kernel(\n- T* __restrict__ p,\n- GRAD_T* __restrict__ p_copy, // For mixed precision training, pass NULL if not needed\n+ GRAD_T* __restrict__ p,\n+ T* __restrict__ p_copy, // For mixed precision training, pass NULL if not needed\n T* __restrict__ m,\n T* __restrict__ v,\n const GRAD_T * __restrict__ g,\n@@ -50,7 +50,7 @@ __global__ void adam_cuda_kernel(\n else // Mode 1\n denom = sqrtf(v[j]) + eps;\n float update = (m[j]/denom) + (decay*p[j]);\n- p[j] = p[j] - (step_size*update);\n+ p[j] = (GRAD_T) (p[j] - (step_size*update));\n if (p_copy != NULL) p_copy[j] = (GRAD_T) p[j];\n }\n }\n@@ -93,14 +93,14 @@ void fused_adam_cuda(\n \n if (g.scalar_type() == at::ScalarType::Half) {\n //all other values should be fp32 for half gradients\n- AT_ASSERTM(p.scalar_type() == at::ScalarType::Float, \"expected parameter to be of float type\");\n+// AT_ASSERTM(p.scalar_type() == at::ScalarType::Float, \"expected parameter to be of float type\");\n //dispatch is done on the gradient type\n using namespace at; // prevents \"toString is undefined\" errors\n DISPATCH_FLOAT_AND_HALF(g.scalar_type(), 0, \"adam_cuda_kernel\", \n using accscalar_t = at::acc_type;\n adam_cuda_kernel<<>>(\n- p.data(),\n- p_copy.numel() ? p_copy.data() : NULL,\n+ p.data(),\n+ NULL, //don't output p_copy for fp32, it's wasted write\n m.data(),\n v.data(),\n g.data(),\n" }, { "path": "apex/csrc/amp_C_frontend.cpp", "content": "#include \n\nvoid multi_tensor_scale_cuda(\n int chunk_size,\n at::Tensor noop_flag,\n std::vector> tensor_lists,\n float scale);\n\nvoid multi_tensor_axpby_cuda(\n int chunk_size,\n at::Tensor noop_flag,\n std::vector> tensor_lists,\n float a,\n float b,\n int arg_to_check);\n\nstd::tuple multi_tensor_l2norm_cuda(\n int chunk_size,\n at::Tensor noop_flag,\n std::vector> tensor_lists,\n at::optional per_tensor_python);\n\nvoid multi_tensor_lamb_stage1_cuda(\n int chunk_size,\n at::Tensor noop_flag,\n std::vector> tensor_lists,\n at::Tensor per_tensor_decay,\n const int step,\n const float beta1,\n const float beta2,\n const float epsilon,\n const float global_grad_norm,\n const float max_global_grad_norm);\n\nvoid multi_tensor_lamb_stage2_cuda(\n int chunk_size,\n at::Tensor noop_flag,\n std::vector> tensor_lists,\n at::Tensor per_tensor_param_norm,\n at::Tensor per_tensor_update_norm,\n const float step_size);\n\nPYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {\n m.def(\"multi_tensor_scale\", &multi_tensor_scale_cuda,\n \"Fused overflow check + scale for a list of contiguous tensors\");\n m.def(\"multi_tensor_axpby\", &multi_tensor_axpby_cuda,\n \"out = a*x + b*y for a list of contiguous tensors\");\n m.def(\"multi_tensor_l2norm\", &multi_tensor_l2norm_cuda,\n \"Computes L2 norm for a list of contiguous tensors\");\n m.def(\"multi_tensor_lamb_stage1_cuda\", &multi_tensor_lamb_stage1_cuda,\n \"Computes update part of LAMB optimizer\");\n m.def(\"multi_tensor_lamb_stage2_cuda\", &multi_tensor_lamb_stage2_cuda,\n \"Completes application of gradient to parameters for LAMB optimizer\");\n}\n" }, { "path": "apex/csrc/flatten_unflatten.cpp", "content": "#include \n#include \n// https://github.com/pytorch/pytorch/blob/master/torch/csrc/utils/tensor_flatten.h\n\nat::Tensor flatten(std::vector tensors)\n{\n return torch::utils::flatten_dense_tensors(tensors);\n}\n\nstd::vector unflatten(at::Tensor flat, std::vector tensors)\n{\n return torch::utils::unflatten_dense_tensors(flat, tensors);\n}\n\nPYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {\n m.def(\"flatten\", &flatten, \"Flatten dense tensors\");\n m.def(\"unflatten\", &unflatten, \"Unflatten dense tensors\");\n}\n" }, { "path": "apex/csrc/fused_adam_cuda.cpp", "content": "#include \n\n// CUDA forward declaration\nvoid fused_adam_cuda(at::Tensor & p, at::Tensor & p_copy, at::Tensor & m, at::Tensor & v, at::Tensor & g, float lr, float beta1, float beta2, float eps, float grad_scale, int step, int mode, int bias_correction, float decay);\n\n#define CHECK_CUDA(x) AT_ASSERTM(x.type().is_cuda(), #x \" must be a CUDA tensor\")\n#define CHECK_CONTIGUOUS(x) AT_ASSERTM(x.is_contiguous(), #x \" must be contiguous\")\n#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x)\n\n// C++ interface\nvoid adam(at::Tensor & p, at::Tensor & p_copy, at::Tensor & m, at::Tensor & v, at::Tensor & g, float lr, float beta1, float beta2, float eps, float grad_scale, int step, int mode, int bias_correction, float decay) {\n CHECK_INPUT(p)\n if (p_copy.numel() > 0) CHECK_INPUT(p_copy);\n CHECK_INPUT(m);\n CHECK_INPUT(v);\n CHECK_INPUT(g);\n int64_t num_elem = p.numel();\n AT_ASSERTM(m.numel() == num_elem, \"number of elements in m and p tensors should be equal\");\n AT_ASSERTM(v.numel() == num_elem, \"number of elements in v and p tensors should be equal\");\n AT_ASSERTM(g.numel() == num_elem, \"number of elements in g and p tensors should be equal\");\n AT_ASSERTM(p_copy.numel() == num_elem || p_copy.numel() == 0, \"number of elements in p_copy and p tensors should be equal, or p_copy should be empty\");\n\n fused_adam_cuda(p, p_copy, m, v, g, lr, beta1, beta2, eps, grad_scale, step, mode, bias_correction, decay);\n}\n\nPYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {\n m.def(\"adam\", &adam, \"Adam optimized CUDA implementation.\");\n}\n" }, { "path": "apex/csrc/fused_adam_cuda_kernel.cu", "content": "#include \"ATen/ATen.h\"\n#include \"ATen/cuda/CUDAContext.h\"\n#include \"ATen/cuda/detail/IndexUtils.cuh\"\n#include \n#include \n#include \n#include \n#include \"ATen/TensorUtils.h\"\n#include \"ATen/Type.h\"\n#include \"ATen/AccumulateType.h\"\n#include \n\n#include \"type_shim.h\"\n\ntypedef enum{\n ADAM_MODE_0 =0, // eps under square root\n ADAM_MODE_1 =1 // eps outside square root\n} adamMode_t;\n\ntemplate \n__global__ void adam_cuda_kernel(\n GRAD_T* __restrict__ p,\n T* __restrict__ p_copy, // For mixed precision training, pass NULL if not needed\n T* __restrict__ m,\n T* __restrict__ v,\n const GRAD_T * __restrict__ g,\n const float b1,\n const float b2,\n const float eps,\n const float grad_scale,\n const float step_size,\n const size_t tsize,\n adamMode_t mode,\n const float decay)\n{\n //Assuming 2D grids and 2D blocks\n const int blockId = gridDim.x * blockIdx.y + blockIdx.x;\n const int threadsPerBlock = blockDim.x * blockDim.y;\n const int threadIdInBlock = threadIdx.y * blockDim.x + threadIdx.x;\n const int i = (blockId * threadsPerBlock + threadIdInBlock);\n const int totThreads = gridDim.x*gridDim.y*threadsPerBlock;\n\n for (int j = i; j < tsize; j+=totThreads) {\n T scaled_grad = g[j]/grad_scale;\n m[j] = b1*m[j] + (1-b1)*scaled_grad;\n v[j] = b2*v[j] + (1-b2)*scaled_grad*scaled_grad;\n float denom;\n if (mode == ADAM_MODE_0)\n denom = sqrtf(v[j] + eps);\n else // Mode 1\n denom = sqrtf(v[j]) + eps;\n float update = (m[j]/denom) + (decay*p[j]);\n p[j] = (GRAD_T) (p[j] - (step_size*update));\n if (p_copy != NULL) p_copy[j] = (GRAD_T) p[j];\n }\n}\n\nvoid fused_adam_cuda(\n at::Tensor & p,\n at::Tensor & p_copy,\n at::Tensor & m,\n at::Tensor & v,\n at::Tensor & g,\n float lr,\n float beta1,\n float beta2,\n float eps,\n float grad_scale,\n int step,\n int mode,\n int bias_correction,\n float decay)\n{\n// using namespace at;\n\n //Get tensor size\n int tsize = p.numel();\n //Determine #threads and #blocks\n const int threadsPerBlock = 512;\n const dim3 blocks((tsize+threadsPerBlock-1)/threadsPerBlock);\n AT_ASSERTM(at::cuda::detail::canUse32BitIndexMath(p), \"parameter tensor is too large to be indexed with int32\");\n //Constants\n float step_size = 0;\n if (bias_correction == 1) {\n const float bias_correction1 = 1 - std::pow(beta1, step);\n const float bias_correction2 = 1 - std::pow(beta2, step);\n step_size = lr * std::sqrt(bias_correction2)/bias_correction1;\n }\n else {\n step_size = lr;\n }\n cudaStream_t stream = at::cuda::getCurrentCUDAStream();\n\n if (g.scalar_type() == at::ScalarType::Half) {\n//all other values should be fp32 for half gradients\n// AT_ASSERTM(p.scalar_type() == at::ScalarType::Float, \"expected parameter to be of float type\");\n//dispatch is done on the gradient type\n using namespace at; // prevents \"toString is undefined\" errors\n DISPATCH_FLOAT_AND_HALF(g.scalar_type(), 0, \"adam_cuda_kernel\", \n using accscalar_t = at::acc_type;\n adam_cuda_kernel<<>>(\n p.data(),\n NULL, //don't output p_copy for fp32, it's wasted write\n m.data(),\n v.data(),\n g.data(),\n beta1,\n beta2,\n eps,\n grad_scale,\n step_size,\n tsize,\n (adamMode_t) mode,\n decay);\n )\n } else {\n using namespace at;\n DISPATCH_DOUBLE_AND_FLOAT(g.scalar_type(), 0, \"adam_cuda_kernel\",\n adam_cuda_kernel<<>>(\n p.data(),\n NULL, //don't output p_copy for fp32, it's wasted write\n m.data(),\n v.data(),\n g.data(),\n beta1,\n beta2,\n eps,\n grad_scale,\n step_size,\n tsize,\n (adamMode_t) mode,\n decay);\n );\n }\n THCudaCheck(cudaGetLastError());\n\n}\n" }, { "path": "apex/csrc/layer_norm_cuda.cpp", "content": "#include \n#include \n#include \n\nnamespace {\nvoid compute_n1_n2(\n at::Tensor input,\n #ifdef VERSION_GE_1_1\n at::IntArrayRef normalized_shape,\n #else\n at::IntList normalized_shape,\n #endif\n int& n1,\n int& n2)\n{\n int idiff = input.ndimension() - normalized_shape.size();\n n2 = 1;\n for (int i = 0; i < (int)normalized_shape.size(); ++i) {\n\t assert( input.sizes()[i+idiff] == normalized_shape[i] );\n\t n2 *= normalized_shape[i];\n }\n n1 = 1;\n for (int i = 0; i < idiff; ++i) {\n\t n1 *= input.sizes()[i];\n }\n}\n\nvoid check_args(\n #ifdef VERSION_GE_1_1\n at::IntArrayRef normalized_shape,\n #else\n at::IntList normalized_shape,\n #endif\n at::Tensor gamma,\n at::Tensor beta\n )\n{\n AT_CHECK(!gamma.defined() || gamma.sizes().equals(normalized_shape));\n AT_CHECK(!beta.defined() || beta.sizes().equals(normalized_shape));\n}\n\nvoid check_args(\n at::Tensor input,\n #ifdef VERSION_GE_1_1\n at::IntArrayRef normalized_shape,\n #else\n at::IntList normalized_shape,\n #endif\n int& n1,\n int& n2\n )\n{\n int64_t normalized_ndim = normalized_shape.size();\n\n if (normalized_ndim < 1) {\n std::stringstream ss;\n ss << \"Expected normalized_shape to be at least 1-dimensional, i.e., \"\n << \"containing at least one element, but got normalized_shape=\"\n << normalized_shape;\n throw std::runtime_error(ss.str());\n }\n\n auto input_shape = input.sizes();\n auto input_ndim = input.dim();\n\n if (input_ndim < normalized_ndim ||\n !input_shape.slice(input_ndim - normalized_ndim).equals(normalized_shape)) {\n std::stringstream ss;\n ss << \"Given normalized_shape=\" << normalized_shape\n << \", expected input with shape [*\";\n for (auto size : normalized_shape) {\n ss << \", \" << size;\n }\n ss << \"], but got input of size\" << input_shape;\n throw std::runtime_error(ss.str());\n }\n\n compute_n1_n2(input,normalized_shape,n1,n2);\n}\n\n\nvoid check_args(\n at::Tensor input,\n #ifdef VERSION_GE_1_1\n at::IntArrayRef normalized_shape,\n #else\n at::IntList normalized_shape,\n #endif\n at::Tensor gamma,\n at::Tensor beta,\n int& n1,\n int& n2\n )\n{\n check_args(input,normalized_shape,n1,n2);\n check_args(normalized_shape,gamma,beta);\n}\n}\n\nvoid cuda_layer_norm(\n at::Tensor* output,\n at::Tensor* mean,\n at::Tensor* invvar,\n at::Tensor* input,\n int n1,\n int n2,\n #ifdef VERSION_GE_1_1\n at::IntArrayRef normalized_shape,\n #else\n at::IntList normalized_shape,\n #endif\n at::Tensor* gamma,\n at::Tensor* beta,\n double epsilon);\n\n#define CHECK_CUDA(x) AT_CHECK(x.type().is_cuda(), #x \" must be a CUDA tensor\")\n#define CHECK_CONTIGUOUS(x) AT_CHECK(x.is_contiguous(), #x \" must be contiguous\")\n#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x)\n\nstd::vector layer_norm(\n at::Tensor input,\n #ifdef VERSION_GE_1_1\n at::IntArrayRef normalized_shape,\n #else\n at::IntList normalized_shape,\n #endif\n double epsilon) {\n CHECK_INPUT(input);\n int n1,n2;\n check_args(input,normalized_shape,n1,n2);\n at::Tensor output = at::empty_like(input);\n at::Tensor mean = at::empty({n1}, input.options().dtype(input.scalar_type()==at::ScalarType::Half ? at::ScalarType::Float : input.scalar_type()));\n at::Tensor invvar = at::empty_like(mean);\n cuda_layer_norm(&output,&mean,&invvar,&input,n1,n2,\n normalized_shape,NULL,NULL,epsilon);\n return {output, mean, invvar};\n}\nstd::vector layer_norm_affine(\n at::Tensor input,\n #ifdef VERSION_GE_1_1\n at::IntArrayRef normalized_shape,\n #else\n at::IntList normalized_shape,\n #endif\n at::Tensor gamma,\n at::Tensor beta,\n double epsilon) {\n CHECK_INPUT(input);\n CHECK_INPUT(gamma);\n CHECK_INPUT(beta);\n int n1,n2;\n check_args(input,normalized_shape,gamma,beta,n1,n2);\n at::Tensor output = at::empty_like(input);\n at::Tensor mean = at::empty({n1}, input.options().dtype(input.scalar_type()==at::ScalarType::Half ? at::ScalarType::Float : input.scalar_type()));\n at::Tensor invvar = at::empty_like(mean);\n cuda_layer_norm(&output,&mean,&invvar,&input,n1,n2,\n normalized_shape,&gamma,&beta,epsilon);\n return {output, mean, invvar};\n}\n\nvoid cuda_layer_norm_gradient(\n at::Tensor* dout,\n at::Tensor* mean,\n at::Tensor* invvar,\n at::Tensor* input,\n int n1,\n int n2,\n #ifdef VERSION_GE_1_1\n at::IntArrayRef normalized_shape,\n #else\n at::IntList normalized_shape,\n #endif\n at::Tensor* gamma,\n at::Tensor* beta,\n double epsilon,\n at::Tensor* grad_input,\n at::Tensor* grad_gamma,\n at::Tensor* grad_beta\n );\n\nat::Tensor layer_norm_gradient(\n at::Tensor dout,\n at::Tensor mean,\n at::Tensor invvar,\n at::Tensor input,\n #ifdef VERSION_GE_1_1\n at::IntArrayRef normalized_shape,\n #else\n at::IntList normalized_shape,\n #endif\n double epsilon) {\n CHECK_INPUT(dout);\n CHECK_INPUT(mean);\n CHECK_INPUT(invvar);\n CHECK_INPUT(input);\n int n1,n2;\n check_args(input,normalized_shape,n1,n2);\n at::Tensor grad_input = at::empty_like(input);\n cuda_layer_norm_gradient(&dout,&mean,&invvar,&input,n1,n2,\n normalized_shape,NULL,NULL,epsilon,\n &grad_input,NULL,NULL);\n return grad_input;\n}\nstd::vector layer_norm_gradient_affine(\n at::Tensor dout,\n at::Tensor mean,\n at::Tensor invvar,\n at::Tensor input,\n #ifdef VERSION_GE_1_1\n at::IntArrayRef normalized_shape,\n #else\n at::IntList normalized_shape,\n #endif\n at::Tensor gamma,\n at::Tensor beta,\n double epsilon) {\n CHECK_INPUT(dout);\n CHECK_INPUT(mean);\n CHECK_INPUT(invvar);\n CHECK_INPUT(input);\n CHECK_INPUT(gamma);\n CHECK_INPUT(beta);\n int n1,n2;\n check_args(input,normalized_shape,gamma,beta,n1,n2);\n at::Tensor grad_input = at::empty_like(input);\n at::Tensor grad_gamma = at::empty_like(gamma);\n at::Tensor grad_beta = at::empty_like(beta);\n cuda_layer_norm_gradient(&dout,&mean,&invvar,&input,n1,n2,\n normalized_shape,&gamma,&beta,epsilon,\n &grad_input,&grad_gamma,&grad_beta);\n return {grad_input, grad_gamma, grad_beta};\n}\n\nPYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {\n m.def(\"forward_affine\", &layer_norm_affine, \"LayerNorm forward (CUDA)\");\n m.def(\"forward\", &layer_norm, \"LayerNorm forward (CUDA)\");\n m.def(\"backward_affine\", &layer_norm_gradient_affine, \"LayerNorm backward (CUDA)\");\n m.def(\"backward\", &layer_norm_gradient, \"LayerNorm backward (CUDA)\");\n}\n\n" }, { "path": "apex/csrc/layer_norm_cuda_kernel.cu", "content": "#include \"ATen/ATen.h\"\n#include \"ATen/AccumulateType.h\"\n#include \"ATen/cuda/CUDAContext.h\"\n#include \n\n#include \n#include \n\n#include \"type_shim.h\"\n\ntemplate __device__\nvoid cuWelfordOnlineSum(\n const U curr,\n U& mu,\n U& sigma2,\n U& count)\n{\n count = count + U(1);\n U delta = curr - mu;\n U lmean = mu + delta / count;\n mu = lmean;\n U delta2 = curr - lmean;\n sigma2 = sigma2 + delta * delta2;\n}\n\ntemplate __device__\nvoid cuChanOnlineSum(\n const U muB,\n const U sigma2B,\n const U countB,\n U& mu,\n U& sigma2,\n U& count)\n{\n U delta = muB - mu;\n U nA = count;\n U nB = countB;\n count = count + countB;\n U nX = count;\n if (nX > U(0)) {\n nA = nA / nX;\n nB = nB / nX;\n mu = nA*mu + nB*muB;\n sigma2 = sigma2 + sigma2B + delta * delta * nA * nB * nX;\n } else {\n mu = U(0);\n sigma2 = U(0);\n }\n}\n\ntemplate __device__\nvoid cuWelfordMuSigma2(\n const T* __restrict__ vals,\n const int n1,\n const int n2,\n U& mu,\n U& sigma2,\n U* buf) \n{\n // Assumptions:\n // 1) blockDim.x == warpSize\n // 2) Tensor is contiguous\n // 3) 2*blockDim.y*sizeof(U)+blockDim.y*sizeof(int) shared memory available.\n //\n // compute variance and mean over n2\n U count = U(0);\n mu= U(0);\n sigma2 = U(0);\n int i1 = blockIdx.y;\n if (i1 < n1) {\n // one warp normalizes one n1 index,\n // synchronization is implicit\n // initialize with standard Welford algorithm\n const int numx = blockDim.x * blockDim.y;\n const int thrx = threadIdx.x + threadIdx.y * blockDim.x;\n const T* lvals = vals + i1*n2;\n int l = 4*thrx;\n for (; l+3 < n2; l+=4*numx) {\n for (int k = 0; k < 4; ++k) {\n U curr = static_cast(lvals[l+k]);\n cuWelfordOnlineSum(curr,mu,sigma2,count);\n }\n }\n for (; l < n2; ++l) {\n U curr = static_cast(lvals[l]);\n cuWelfordOnlineSum(curr,mu,sigma2,count);\n }\n // intra-warp reductions\n for (int l = 0; l <= 4; ++l) {\n int srcLaneB = (threadIdx.x+(1<(muB,sigma2B,countB,mu,sigma2,count);\n }\n // threadIdx.x == 0 has correct values for each warp\n // inter-warp reductions\n if (blockDim.y > 1) {\n U* ubuf = (U*)buf;\n U* ibuf = (U*)(ubuf + blockDim.y);\n for (int offset = blockDim.y/2; offset > 0; offset /= 2) {\n // upper half of warps write to shared\n if (threadIdx.x == 0 && threadIdx.y >= offset && threadIdx.y < 2*offset) {\n const int wrt_y = threadIdx.y - offset;\n ubuf[2*wrt_y] = mu;\n ubuf[2*wrt_y+1] = sigma2;\n ibuf[wrt_y] = count;\n }\n __syncthreads();\n // lower half merges\n if (threadIdx.x == 0 && threadIdx.y < offset) {\n U muB = ubuf[2*threadIdx.y];\n U sigma2B = ubuf[2*threadIdx.y+1];\n U countB = ibuf[threadIdx.y];\n cuChanOnlineSum(muB,sigma2B,countB,mu,sigma2,count);\n }\n __syncthreads();\n }\n // threadIdx.x = 0 && threadIdx.y == 0 only thread that has correct values\n if (threadIdx.x == 0 && threadIdx.y == 0) {\n ubuf[0] = mu;\n ubuf[1] = sigma2;\n }\n __syncthreads();\n mu = ubuf[0];\n sigma2 = ubuf[1]/U(n2);\n // don't care about final value of count, we know count == n2\n } else {\n mu = WARP_SHFL(mu, 0);\n sigma2 = WARP_SHFL(sigma2/U(n2), 0);\n }\n }\n}\n\ntemplate<> __device__\nvoid cuWelfordMuSigma2(\n const at::Half* __restrict__ vals,\n const int n1,\n const int n2,\n float& mu,\n float& sigma2,\n float* buf) \n{\n // Assumptions:\n // 1) blockDim.x == warpSize\n // 2) Tensor is contiguous\n // 3) 2*blockDim.y*sizeof(U)+blockDim.y*sizeof(int) shared memory available.\n //\n // compute variance and mean over n2\n float count = 0.0f;\n mu= float(0);\n sigma2 = float(0);\n int i1 = blockIdx.y;\n if (i1 < n1) {\n // one warp normalizes one n1 index,\n // synchronization is implicit\n // initialize with standard Welford algorithm\n const int numx = blockDim.x * blockDim.y;\n const int thrx = threadIdx.x + threadIdx.y * blockDim.x;\n const at::Half* lvals = vals + i1*n2;\n int l = 8*thrx;\n if ((((size_t)lvals)&3) != 0) {\n // 16 bit alignment\n // first thread consumes first point\n if (thrx == 0) {\n float curr = static_cast(lvals[0]);\n cuWelfordOnlineSum(curr,mu,sigma2,count);\n }\n ++l;\n }\n // at this point, lvals[l] are 32 bit aligned for all threads.\n for (; l+7 < n2; l+=8*numx) {\n for (int k = 0; k < 8; k+=2) {\n float2 curr = __half22float2(*((__half2*)(lvals+l+k)));\n cuWelfordOnlineSum(curr.x,mu,sigma2,count);\n\tcuWelfordOnlineSum(curr.y,mu,sigma2,count);\n }\n }\n for (; l < n2; ++l) {\n float curr = static_cast(lvals[l]);\n cuWelfordOnlineSum(curr,mu,sigma2,count);\n }\n // intra-warp reductions\n for (int l = 0; l <= 4; ++l) {\n int srcLaneB = (threadIdx.x+(1< 1) {\n float* ubuf = (float*)buf;\n float* ibuf = (float*)(ubuf + blockDim.y);\n for (int offset = blockDim.y/2; offset > 0; offset /= 2) {\n // upper half of warps write to shared\n if (threadIdx.x == 0 && threadIdx.y >= offset && threadIdx.y < 2*offset) {\n const int wrt_y = threadIdx.y - offset;\n ubuf[2*wrt_y] = mu;\n ubuf[2*wrt_y+1] = sigma2;\n ibuf[wrt_y] = count;\n }\n __syncthreads();\n // lower half merges\n if (threadIdx.x == 0 && threadIdx.y < offset) {\n float muB = ubuf[2*threadIdx.y];\n float sigma2B = ubuf[2*threadIdx.y+1];\n float countB = ibuf[threadIdx.y];\n cuChanOnlineSum(muB,sigma2B,countB,mu,sigma2,count);\n }\n __syncthreads();\n }\n // threadIdx.x = 0 && threadIdx.y == 0 only thread that has correct values\n if (threadIdx.x == 0 && threadIdx.y == 0) {\n ubuf[0] = mu;\n ubuf[1] = sigma2;\n }\n __syncthreads();\n mu = ubuf[0];\n sigma2 = ubuf[1]/float(n2);\n // don't care about final value of count, we know count == n2\n } else {\n mu = WARP_SHFL(mu, 0);\n sigma2 = WARP_SHFL(sigma2/float(n2), 0);\n }\n }\n}\n\ntemplate U rsqrt(U v) {\n return U(1) / sqrt(v);\n}\ntemplate<> float rsqrt(float v) {\n return rsqrtf(v);\n}\ntemplate<> double rsqrt(double v) {\n return rsqrt(v);\n}\n\nnamespace {\n// This is the un-specialized struct. Note that we prevent instantiation of this\n// struct by putting an undefined symbol in the function body so it won't compile.\n// template \n// struct SharedMemory\n// {\n// // Ensure that we won't compile any un-specialized types\n// __device__ T *getPointer()\n// {\n// extern __device__ void error(void);\n// error();\n// return NULL;\n// }\n// };\n// https://github.com/NVIDIA/apex/issues/246\ntemplate \nstruct SharedMemory;\n\ntemplate <>\nstruct SharedMemory \n{\n __device__ float *getPointer()\n {\n extern __shared__ float s_float[];\n return s_float;\n }\n};\n\ntemplate <>\nstruct SharedMemory \n{\n __device__ double *getPointer()\n {\n extern __shared__ double s_double[];\n return s_double;\n }\n};\n}\n\ntemplate __global__\nvoid cuApplyLayerNorm(\n T* __restrict__ output_vals,\n U* __restrict__ mean,\n U* __restrict__ invvar,\n const T* __restrict__ vals,\n const int n1,\n const int n2,\n const U epsilon,\n const T* __restrict__ gamma,\n const T* __restrict__ beta\n ) \n{\n // Assumptions:\n // 1) blockDim.x == warpSize\n // 2) Tensors are contiguous\n //\n int i1 = blockIdx.y;\n if (i1 < n1) {\n SharedMemory shared;\n U* buf = shared.getPointer();\n U mu,sigma2;\n cuWelfordMuSigma2(vals,n1,n2,mu,sigma2,buf);\n const T* lvals = vals + i1*n2;\n T* ovals = output_vals + i1*n2;\n U c_invvar = rsqrt(sigma2 + epsilon);\n const int numx = blockDim.x * blockDim.y;\n const int thrx = threadIdx.x + threadIdx.y * blockDim.x;\n if (gamma != NULL && beta != NULL) {\n for (int i = thrx; i < n2; i+=numx) {\n U curr = static_cast(lvals[i]);\n ovals[i] = gamma[i] * static_cast(c_invvar * (curr - mu)) + beta[i];\n }\n } else {\n for (int i = thrx; i < n2; i+=numx) {\n U curr = static_cast(lvals[i]);\n ovals[i] = static_cast(c_invvar * (curr - mu));\n }\n }\n if (threadIdx.x == 0 && threadIdx.y == 0) {\n mean[i1] = mu;\n invvar[i1] = c_invvar;\n }\n }\n}\n\ntemplate __device__\nvoid cuLoadWriteStridedInputs(\n const int i1_block,\n const int thr_load_row_off,\n const int thr_load_col_off,\n const int i2_off,\n const int row_stride,\n U* warp_buf1,\n U* warp_buf2,\n const T* input,\n const T* dout,\n const int i1_end,\n const int n2,\n const U* __restrict__ mean,\n const U* __restrict__ invvar\n )\n{\n int i1 = i1_block+thr_load_row_off;\n if (i1 < i1_end) {\n U curr_mean = mean[i1];\n U curr_invvar = invvar[i1];\n for (int k = 0; k < blockDim.y; ++k) {\n int i2 = i2_off + k;\n int load_idx = i1*n2+i2;\n int write_idx = thr_load_row_off*row_stride+thr_load_col_off+k;\n if (i2(input[load_idx]);\n\tU curr_dout = static_cast(dout[load_idx]);\n\twarp_buf1[write_idx] = curr_dout;\n\twarp_buf2[write_idx] = curr_dout * (curr_input - curr_mean) * curr_invvar;\n } else {\n warp_buf1[write_idx] = U(0);\n warp_buf2[write_idx] = U(0);\n }\n }\n } else {\n for (int k = 0; k < blockDim.y; ++k) {\n int write_idx = thr_load_row_off*row_stride+thr_load_col_off+k;\n warp_buf1[write_idx] = U(0);\n warp_buf2[write_idx] = U(0);\n }\n }\n}\n\ntemplate __device__\nvoid cuLoadAddStridedInputs(\n const int i1_block,\n const int thr_load_row_off,\n const int thr_load_col_off,\n const int i2_off,\n const int row_stride,\n U* warp_buf1,\n U* warp_buf2,\n const T* input,\n const T* dout,\n const int i1_end,\n const int n2,\n const U* __restrict__ mean,\n const U* __restrict__ invvar\n )\n{\n int i1 = i1_block+thr_load_row_off;\n if (i1 < i1_end) {\n U curr_mean = mean[i1];\n U curr_invvar = invvar[i1];\n for (int k = 0; k < blockDim.y; ++k) {\n int i2 = i2_off + k;\n int load_idx = i1*n2+i2;\n int write_idx = thr_load_row_off*row_stride+thr_load_col_off+k;\n if (i2(input[load_idx]);\n\tU curr_dout = static_cast(dout[load_idx]);\n\twarp_buf1[write_idx] += curr_dout;\n\twarp_buf2[write_idx] += curr_dout * (curr_input - curr_mean) * curr_invvar;\n }\n }\n }\n}\n\ntemplate __global__\nvoid cuComputePartGradGammaBeta(\n const T* __restrict__ dout,\n const T* __restrict__ input,\n const int n1,\n const int n2,\n const U* __restrict__ mean,\n const U* __restrict__ invvar,\n U epsilon,\n U* part_grad_gamma,\n U* part_grad_beta)\n{\n const int numsegs_n1 = (n1+blockDim.y*blockDim.y-1) / (blockDim.y*blockDim.y);\n const int segs_per_block = (numsegs_n1 + gridDim.y - 1) / gridDim.y;\n const int i1_beg = blockIdx.y * segs_per_block * blockDim.y*blockDim.y;\n const int i1_beg_plus_one = (blockIdx.y+1) * segs_per_block * blockDim.y*blockDim.y;\n const int i1_end = i1_beg_plus_one < n1 ? i1_beg_plus_one : n1;\n const int row_stride = blockDim.x+1;\n const int thr_load_col_off = (threadIdx.x*blockDim.y)&(blockDim.x-1);\n const int thr_load_row_off = (threadIdx.x*blockDim.y)/blockDim.x + threadIdx.y*blockDim.y;\n const int i2_off = blockIdx.x * blockDim.x + thr_load_col_off;\n SharedMemory shared;\n U* buf = shared.getPointer(); // buf has at least blockDim.x * blockDim.y * blockDim.y + (blockDim.y - 1)*(blockDim.x/blockDim.y) elements\n U* warp_buf1 = (U*)buf;\n U* warp_buf2 = warp_buf1 + blockDim.y * blockDim.y * row_stride;\n // compute partial sums from strided inputs\n // do this to increase number of loads in flight\n cuLoadWriteStridedInputs(i1_beg,thr_load_row_off,thr_load_col_off,i2_off,row_stride,warp_buf1,warp_buf2,input,dout,i1_end,n2,mean,invvar);\n for (int i1_block = i1_beg+blockDim.y*blockDim.y; i1_block < i1_end; i1_block+=blockDim.y*blockDim.y) {\n cuLoadAddStridedInputs(i1_block,thr_load_row_off,thr_load_col_off,i2_off,row_stride,warp_buf1,warp_buf2,input,dout,i1_end,n2,mean,invvar);\n }\n __syncthreads();\n // inter-warp reductions\n // sum within each warp\n U acc1 = U(0);\n U acc2 = U(0);\n for (int k = 0; k < blockDim.y; ++k) {\n int row1 = threadIdx.y + k*blockDim.y;\n int idx1 = row1*row_stride + threadIdx.x;\n acc1 += warp_buf1[idx1];\n acc2 += warp_buf2[idx1];\n }\n warp_buf1[threadIdx.y*row_stride+threadIdx.x] = acc1;\n warp_buf2[threadIdx.y*row_stride+threadIdx.x] = acc2;\n __syncthreads();\n // sum all warps\n for (int offset = blockDim.y/2; offset > 1; offset /= 2) {\n if (threadIdx.y < offset) {\n int row1 = threadIdx.y;\n\tint row2 = threadIdx.y + offset;\n\tint idx1 = row1*row_stride + threadIdx.x;\n\tint idx2 = row2*row_stride + threadIdx.x;\n\twarp_buf1[idx1] += warp_buf1[idx2];\n\twarp_buf2[idx1] += warp_buf2[idx2];\n }\n __syncthreads();\n }\n int i2 = blockIdx.x * blockDim.x + threadIdx.x;\n if (threadIdx.y == 0 && i2 < n2) {\n int row1 = threadIdx.y;\n int row2 = threadIdx.y + 1;\n int idx1 = row1*row_stride + threadIdx.x;\n int idx2 = row2*row_stride + threadIdx.x;\n part_grad_beta[blockIdx.y*n2+i2] = warp_buf1[idx1] + warp_buf1[idx2];\n part_grad_gamma[blockIdx.y*n2+i2] = warp_buf2[idx1] + warp_buf2[idx2];\n }\n}\n\ntemplate __global__\nvoid cuComputeGradGammaBeta(\n const U* part_grad_gamma,\n const U* part_grad_beta,\n const int part_size,\n const int n1,\n const int n2,\n T* grad_gamma,\n T* grad_beta)\n{\n // sum partial gradients for gamma and beta\n SharedMemory shared;\n U* buf = shared.getPointer(); \n int i2 = blockIdx.x * blockDim.x + threadIdx.x;\n if (i2 < n2) {\n // each warp does sequential reductions until reduced part_size is num_warps\n int num_warp_reductions = part_size / blockDim.y;\n U sum_gamma = U(0);\n U sum_beta = U(0);\n const U* part_grad_gamma_ptr = part_grad_gamma + threadIdx.y * num_warp_reductions * n2 + i2;\n const U* part_grad_beta_ptr = part_grad_beta + threadIdx.y * num_warp_reductions * n2 + i2;\n for (int warp_offset = 0; warp_offset < num_warp_reductions; ++warp_offset) {\n sum_gamma += part_grad_gamma_ptr[warp_offset*n2];\n sum_beta += part_grad_beta_ptr[warp_offset*n2];\n }\n // inter-warp reductions\n const int nbsize3 = blockDim.x * blockDim.y / 2;\n for (int offset = blockDim.y/2; offset >= 1; offset /= 2) {\n // top half write to shared memory\n if (threadIdx.y >= offset && threadIdx.y < 2*offset) {\n const int write_idx = (threadIdx.y - offset) * blockDim.x + threadIdx.x;\n buf[write_idx] = sum_gamma;\n buf[write_idx+nbsize3] = sum_beta;\n }\n __syncthreads();\n // bottom half sums\n if (threadIdx.y < offset) {\n const int read_idx = threadIdx.y * blockDim.x + threadIdx.x;\n sum_gamma += buf[read_idx];\n sum_beta += buf[read_idx+nbsize3];\n }\n __syncthreads();\n }\n // write out fully summed gradients\n if (threadIdx.y == 0) {\n grad_gamma[i2] = sum_gamma;\n grad_beta[i2] = sum_beta;\n }\n }\n}\n\ntemplate __global__\nvoid cuComputeGradInput(\n const T* __restrict__ dout,\n const T* __restrict__ input,\n const int n1,\n const int n2,\n const U* __restrict__ mean,\n const U* __restrict__ invvar,\n U epsilon,\n const T* gamma,\n T* grad_input)\n{\n int i1 = blockIdx.y;\n if (i1 < n1) {\n U sum_loss1 = U(0);\n U sum_loss2 = U(0);\n const U c_mean = mean[i1];\n const U c_invvar = invvar[i1];\n const T* k_input = input + i1*n2;\n const T* k_dout = dout + i1*n2;\n const int numx = blockDim.x * blockDim.y;\n const int thrx = threadIdx.x + threadIdx.y * blockDim.x;\n if (gamma != NULL) {\n int l = 4*thrx;\n for (; l+3 < n2; l+=4*numx) {\n for (int k = 0; k < 4; ++k) {\n const U c_h = static_cast(k_input[l+k]);\n const U c_loss = static_cast(k_dout[l+k]);\n sum_loss1 += c_loss * gamma[l+k];\n sum_loss2 += c_loss * gamma[l+k] * (c_h - c_mean) * c_invvar;\n }\n }\n for (; l < n2; ++l) {\n const U c_h = static_cast(k_input[l]);\n const U c_loss = static_cast(k_dout[l]);\n sum_loss1 += c_loss * gamma[l];\n sum_loss2 += c_loss * gamma[l] * (c_h - c_mean) * c_invvar;\n }\n } else {\n int l = 4*thrx;\n for (; l+3 < n2; l+=4*numx) {\n for (int k = 0; k < 4; ++k) {\n const U c_h = static_cast(k_input[l+k]);\n const U c_loss = static_cast(k_dout[l+k]);\n sum_loss1 += c_loss;\n sum_loss2 += c_loss * (c_h - c_mean) * c_invvar;\n }\n }\n for (; l < n2; ++l) {\n const U c_h = static_cast(k_input[l]);\n const U c_loss = static_cast(k_dout[l]);\n sum_loss1 += c_loss;\n sum_loss2 += c_loss * (c_h - c_mean) * c_invvar;\n }\n }\n // intra-warp reductions\n for (int mask = blockDim.x/2; mask > 0; mask /= 2) {\n sum_loss1 += WARP_SHFL_XOR(sum_loss1, mask);\n sum_loss2 += WARP_SHFL_XOR(sum_loss2, mask);\n }\n // inter-warp reductions\n if (blockDim.y > 1) {\n SharedMemory shared;\n U* buf = shared.getPointer(); \n for (int offset = blockDim.y/2; offset > 0; offset /= 2) {\n // upper half of warps write to shared\n if (threadIdx.y >= offset && threadIdx.y < 2*offset) {\n const int wrt_i = (threadIdx.y - offset) * blockDim.x + threadIdx.x;\n buf[2*wrt_i] = sum_loss1;\n buf[2*wrt_i+1] = sum_loss2;\n }\n __syncthreads();\n // lower half merges\n if (threadIdx.y < offset) {\n const int read_i = threadIdx.y * blockDim.x + threadIdx.x;\n sum_loss1 += buf[2*read_i];\n sum_loss2 += buf[2*read_i+1];\n }\n __syncthreads();\n }\n if (threadIdx.y == 0) {\n buf[2*threadIdx.x] = sum_loss1;\n buf[2*threadIdx.x+1] = sum_loss2;\n }\n __syncthreads();\n if (threadIdx.y !=0) {\n sum_loss1 = buf[2*threadIdx.x];\n sum_loss2 = buf[2*threadIdx.x+1];\n } \n }\n // all threads now have the two sums over l\n U fH = (U)n2;\n U term1 = (U(1) / fH) * c_invvar;\n T* k_grad_input = grad_input + i1*n2;\n if (gamma != NULL) {\n for (int l = thrx; l < n2; l+=numx) {\n const U c_h = static_cast(k_input[l]);\n const U c_loss = static_cast(k_dout[l]);\n U f_grad_input = fH * c_loss * gamma[l];\n f_grad_input -= sum_loss1;\n f_grad_input -= (c_h - c_mean) * c_invvar * sum_loss2;\n f_grad_input *= term1;\n k_grad_input[l] = static_cast(f_grad_input);\n }\n } else {\n for (int l = thrx; l < n2; l+=numx) {\n const U c_h = static_cast(k_input[l]);\n const U c_loss = static_cast(k_dout[l]);\n U f_grad_input = fH * c_loss;\n f_grad_input -= sum_loss1;\n f_grad_input -= (c_h - c_mean) * c_invvar * sum_loss2;\n f_grad_input *= term1;\n k_grad_input[l] = static_cast(f_grad_input);\n }\n }\n }\n}\n\ntemplate \nvoid HostApplyLayerNorm(\n T* output,\n U* mean,\n U* invvar,\n const T* input,\n int n1,\n int n2,\n double epsilon,\n const T* gamma,\n const T* beta\n )\n{\n auto stream = at::cuda::getCurrentCUDAStream().stream();\n const dim3 threads(32,4,1);\n const dim3 blocks(1,n1,1);\n int nshared = \n threads.y > 1 ? \n\t threads.y*sizeof(U)+(threads.y/2)*sizeof(U) : \n\t 0;\n cuApplyLayerNorm<<>>(\n\t\t output,\n\t\t mean,\n\t\t invvar,\n\t\t input,\n\t\t n1,n2,\n\t\t U(epsilon),\n gamma,beta);\n}\n\nvoid cuda_layer_norm(\n at::Tensor* output,\n at::Tensor* mean,\n at::Tensor* invvar,\n at::Tensor* input,\n int n1,\n int n2,\n #ifdef VERSION_GE_1_1\n at::IntArrayRef normalized_shape,\n #else\n at::IntList normalized_shape,\n #endif\n at::Tensor* gamma,\n at::Tensor* beta,\n double epsilon)\n{\n using namespace at;\n DISPATCH_DOUBLE_FLOAT_AND_HALF(input->scalar_type(), 0, \"layer_norm_cuda_kernel\",\n using accscalar_t = at::acc_type;\n HostApplyLayerNorm(\n output->data(),\n\t mean->data(),\n\t invvar->data(),\n\t input->data(),\n\t n1,n2,\n\t epsilon,\n\t gamma != NULL ? gamma->data() : NULL,\n\t beta != NULL ? beta->data() : NULL);\n )\n}\n\ntemplate \nvoid HostLayerNormGradient(\n const T* dout,\n const U* mean,\n const U* invvar,\n at::Tensor* input,\n int n1,\n int n2,\n const T* gamma,\n const T* beta,\n double epsilon,\n T* grad_input,\n T* grad_gamma,\n T* grad_beta\n )\n{\n auto stream = at::cuda::getCurrentCUDAStream().stream();\n\n if (gamma != NULL && beta != NULL) {\n // compute grad_gamma(j) and grad_beta(j)\n const int part_size = 16;\n const dim3 threads2(32,4,1);\n const dim3 blocks2((n2+threads2.x-1)/threads2.x,part_size,1);\n const int nshared2_a = 2 * sizeof(U) * threads2.y * threads2.y * (threads2.x + 1);\n const int nshared2_b = threads2.x * threads2.y * sizeof(U);\n const int nshared2 = nshared2_a > nshared2_b ? nshared2_a : nshared2_b;\n at::Tensor part_grad_gamma = at::empty({part_size,n2}, input->options().dtype(input->scalar_type()==at::ScalarType::Half ? at::ScalarType::Float : input->scalar_type()));\n at::Tensor part_grad_beta = at::empty_like(part_grad_gamma);\n cuComputePartGradGammaBeta<<>>(\n\t\t dout,\n\t\t input->data(),\n\t\t n1,n2,\n\t\t mean,\n\t\t invvar,\n\t\t U(epsilon),\n\t\t part_grad_gamma.data(),\n\t\t part_grad_beta.data());\n\n const dim3 threads3(32,8,1);\n const dim3 blocks3((n2+threads2.x-1)/threads2.x,1,1);\n const int nshared3 = threads3.x * threads3.y * sizeof(U);\n cuComputeGradGammaBeta<<>>(\n\t\t part_grad_gamma.data(),\n\t\t part_grad_beta.data(),\n\t\t part_size,\n\t\t n1,n2,\n\t\t grad_gamma,\n\t\t grad_beta);\n }\n\n // compute grad_input\n const dim3 threads1(32,4,1);\n const dim3 blocks1(1,n1,1);\n int nshared =\n\t threads1.y > 1 ?\n\t threads1.y*threads1.x*sizeof(U) :\n\t 0;\n cuComputeGradInput<<>>(\n dout,\n input->data(),\n n1,n2,\n mean,\n invvar,\n U(epsilon),\n gamma,\n grad_input);\n}\n\nvoid cuda_layer_norm_gradient(\n at::Tensor* dout,\n at::Tensor* mean,\n at::Tensor* invvar,\n at::Tensor* input,\n int n1,\n int n2,\n #ifdef VERSION_GE_1_1\n at::IntArrayRef normalized_shape,\n #else\n at::IntList normalized_shape,\n #endif\n at::Tensor* gamma,\n at::Tensor* beta,\n double epsilon,\n at::Tensor* grad_input,\n at::Tensor* grad_gamma,\n at::Tensor* grad_beta)\n{\n using namespace at;\n DISPATCH_FLOAT_AND_HALF(input->scalar_type(), 0, \"cuComputeGradInput\",\n using accscalar_t = at::acc_type;\n HostLayerNormGradient(\n\t dout->data(),\n\t mean->data(),\n\t invvar->data(),\n\t input,\n\t n1,n2,\n\t gamma->data(),\n\t beta->data(),\n\t epsilon,\n\t grad_input->data(),\n\t grad_gamma->data(),\n\t grad_beta->data());\n )\n}\n" }, { "path": "apex/csrc/multi_tensor_apply.cuh", "content": "#include \n#include \n#include \n#include \n\n#include \n\n// #include \n\n// This header is the one-stop shop for all your multi-tensor apply needs.\n\n\n// TODO: Kernel arg size limit may be <4KB for some other cards (ie Jetson)\nconstexpr int depth_to_max_tensors[5] = {110, 64, 48, 36, 30};\nconstexpr int depth_to_max_blocks[5] = {320, 320, 320, 320, 320};\n\ntemplate struct TensorListMetadata\n{\n void* addresses[n][depth_to_max_tensors[n-1]];\n int sizes[depth_to_max_tensors[n-1]];\n unsigned char block_to_tensor[depth_to_max_blocks[n-1]];\n int block_to_chunk[depth_to_max_blocks[n-1]]; // I fear this needs to be a full int.\n int start_tensor_this_launch;\n};\n\n\ntemplate\n__global__ void multi_tensor_apply_kernel(\n int chunk_size,\n volatile int* noop_flag,\n T tl,\n U callable,\n ArgTypes... args)\n{\n // Hand the chunk information to the user-supplied functor to process however it likes.\n callable(chunk_size, noop_flag, tl, args...); \n}\n\ntemplate\nvoid multi_tensor_apply(\n int block_size,\n int chunk_size,\n const at::Tensor& noop_flag,\n const std::vector>& tensor_lists,\n T callable,\n ArgTypes... args)\n{\n AT_CHECK(tensor_lists.size() == depth, \"tensor_lists.size() != depth\");\n int len0 = tensor_lists[0].size();\n AT_CHECK(len0 > 0, \"tensor_lists[0].size() is not > 0\");\n\n for(int l = 0; l < tensor_lists.size(); l++) // No range-based for because I need indices\n {\n AT_CHECK(tensor_lists[l].size() == len0, \"Size mismatch among tensor lists\");\n for(int t = 0; t < tensor_lists[l].size(); t++)\n {\n // TODO: Print which tensor fails.\n AT_CHECK(tensor_lists[l][t].is_contiguous(), \"A tensor was not contiguous.\");\n AT_CHECK(tensor_lists[l][t].is_cuda(), \"A tensor was not cuda.\");\n AT_CHECK(tensor_lists[l][t].numel() == tensor_lists[0][t].numel(), \"Size mismatch\");\n }\n }\n\n int ntensors = tensor_lists[0].size();\n\n TensorListMetadata tl;\n\n auto stream = at::cuda::getCurrentCUDAStream();\n \n tl.start_tensor_this_launch = 0;\n int loc_block_info = 0;\n int loc_tensor_info = 0;\n for(int t = 0; t < ntensors; t++)\n {\n tl.sizes[loc_tensor_info] = tensor_lists[0][t].numel();\n for(int d = 0; d < depth; d++)\n tl.addresses[d][loc_tensor_info] = tensor_lists[d][t].data_ptr();\n loc_tensor_info++;\n\n int chunks_this_tensor = (tensor_lists[0][t].numel() + chunk_size - 1)/chunk_size;\n\n for(int chunk = 0; chunk < chunks_this_tensor; chunk++)\n {\n // std::cout << chunks_this_tensor << std::endl;\n tl.block_to_tensor[loc_block_info] = loc_tensor_info - 1;\n tl.block_to_chunk[loc_block_info] = chunk;\n loc_block_info++;\n \n bool tensors_full = (loc_tensor_info == depth_to_max_tensors[depth-1] &&\n chunk == chunks_this_tensor - 1);\n bool blocks_full = (loc_block_info == depth_to_max_blocks[depth-1]);\n bool last_chunk = (t == ntensors - 1 && chunk == chunks_this_tensor - 1);\n if(tensors_full || blocks_full || last_chunk)\n {\n // using accscalar_t = acc_type;\n multi_tensor_apply_kernel<<>>(\n chunk_size,\n noop_flag.data(),\n tl,\n callable,\n args...);\n\n AT_CUDA_CHECK(cudaGetLastError());\n\n // Reset. The control flow possibilities here make my brain hurt.\n loc_block_info = 0;\n if(chunk == chunks_this_tensor - 1)\n {\n // std::cout << \"Hit case 1 \" << cond1 << \" \" << cond2 << \" \" << cond3 << std::endl;\n loc_tensor_info = 0; \n tl.start_tensor_this_launch = t + 1;\n }\n else\n {\n // std::cout << \"Hit case 2 \" << cond1 << \" \" << cond2 << \" \" << cond3 << std::endl;\n tl.sizes[0] = tl.sizes[loc_tensor_info-1];\n for(int d = 0; d < depth; d++)\n tl.addresses[d][0] = tl.addresses[d][loc_tensor_info-1];\n loc_tensor_info = 1;\n tl.start_tensor_this_launch = t;\n }\n }\n }\n }\n}\n" }, { "path": "apex/csrc/multi_tensor_axpby_kernel.cu", "content": "#include \n#include \n#include \n#include \n// Another possibility:\n// #include \n\n#include \n\n#include \"type_shim.h\"\n#include \"multi_tensor_apply.cuh\"\n\n#define BLOCK_SIZE 512\n#define ILP 4\n\ntemplate\nstruct AxpbyFunctor\n{\n __device__ __forceinline__ void operator()(\n int chunk_size,\n volatile int* noop_gmem,\n TensorListMetadata<3>& tl,\n float a,\n float b,\n int arg_to_check)\n {\n // I'd like this kernel to propagate infs/nans.\n // if(*noop_gmem == 1)\n // return;\n\n int tensor_loc = tl.block_to_tensor[blockIdx.x];\n int chunk_idx = tl.block_to_chunk[blockIdx.x];\n int n = tl.sizes[tensor_loc];\n\n x_t* x = (x_t*)tl.addresses[0][tensor_loc];\n x += chunk_idx*chunk_size;\n\n y_t* y = (y_t*)tl.addresses[1][tensor_loc];\n y += chunk_idx*chunk_size;\n\n out_t* out = (out_t*)tl.addresses[2][tensor_loc];\n out += chunk_idx*chunk_size;\n\n n -= chunk_idx*chunk_size;\n\n // Non-divergent exit condition for __syncthreads, not necessary here\n float xs[ILP];\n float ys[ILP];\n for(int i_start = 0;\n i_start < n && i_start < chunk_size;\n i_start += blockDim.x*ILP)\n {\n #pragma unroll\n for(int ii = 0; ii < ILP; ii++)\n {\n xs[ii] = 0;\n ys[ii] = 0;\n int i = i_start + threadIdx.x + ii*blockDim.x;\n if(i < n && i < chunk_size)\n {\n xs[ii] = static_cast(x[i]);\n ys[ii] = static_cast(y[i]);\n }\n }\n\n // see note in multi_tensor_scale_kernel.cu\n #pragma unroll\n for(int ii = 0; ii < ILP; ii++)\n {\n int i = i_start + threadIdx.x + ii*blockDim.x;\n if(i < n && i < chunk_size)\n {\n out[i] = static_cast(a*xs[ii] + b*ys[ii]);\n bool finite = true;\n if(arg_to_check == -1)\n finite = (isfinite(xs[ii]) && isfinite(ys[ii]));\n if(arg_to_check == 0)\n finite = isfinite(xs[ii]);\n if(arg_to_check == 1)\n finite = isfinite(ys[ii]);\n if(!finite)\n *noop_gmem = 1; // Blindly fire off a write. These will race but that's ok.\n }\n }\n }\n }\n};\n\nvoid multi_tensor_axpby_cuda(\n int chunk_size,\n at::Tensor noop_flag,\n std::vector> tensor_lists,\n float a,\n float b,\n int arg_to_check)\n{\n using namespace at;\n // The output (downscaled) type is always float.\n // If build times suffer, think about where to put this dispatch,\n // and what logic should be moved out of multi_tensor_apply.\n\n DISPATCH_FLOAT_AND_HALF(tensor_lists[0][0].scalar_type(), 0, \"multi_tensor_axpby_cuda\",\n DISPATCH_FLOAT_AND_HALF(tensor_lists[1][0].scalar_type(), 1, \"multi_tensor_axpby_cuda\",\n DISPATCH_FLOAT_AND_HALF(tensor_lists[2][0].scalar_type(), 2, \"multi_tensor_axpby_cuda\",\n multi_tensor_apply<3>(\n BLOCK_SIZE,\n chunk_size,\n noop_flag,\n tensor_lists,\n AxpbyFunctor(),\n a,\n b,\n arg_to_check); )))\n\n AT_CUDA_CHECK(cudaGetLastError());\n\n // AT_CUDA_CHECK(cudaDeviceSynchronize());\n}\n" }, { "path": "apex/csrc/multi_tensor_l2norm_kernel.cu", "content": "#include \n#include \n#include \n#include \n// Another possibility:\n// #include \n\n#include \n\n#include \"type_shim.h\"\n#include \"multi_tensor_apply.cuh\"\n\n#define BLOCK_SIZE 512\n#define ILP 4\n\ntemplate\nstruct L2NormFunctor\n{\n __device__ __forceinline__ void operator()(\n int chunk_size,\n volatile int* noop_gmem,\n TensorListMetadata<1>& tl,\n float* output,\n float* output_per_tensor,\n bool per_tensor,\n int max_chunks_per_tensor)\n {\n // I'd like this kernel to propagate infs/nans.\n // if(*noop_gmem == 1)\n // return;\n\n int tensor_loc = tl.block_to_tensor[blockIdx.x];\n int chunk_idx = tl.block_to_chunk[blockIdx.x];\n int n = tl.sizes[tensor_loc];\n\n x_t* x = (x_t*)tl.addresses[0][tensor_loc];\n x += chunk_idx*chunk_size;\n\n n -= chunk_idx*chunk_size;\n\n __shared__ float s_vals[512];\n\n float vals[ILP]; // = {0}; // this probably works too but I want to be sure...\n for(int i = 0; i < ILP; i++)\n vals[i] = 0.f;\n\n for(int i_start = 0; i_start < n && i_start < chunk_size; i_start += blockDim.x*ILP)\n {\n #pragma unroll\n for(int ii = 0; ii < ILP; ii++)\n {\n int i = i_start + threadIdx.x + ii*blockDim.x;\n if(i < n && i < chunk_size)\n {\n float next = static_cast(x[i]);\n vals[ii] += next*next;\n }\n }\n }\n\n float val = 0.f;\n for(int i = 0; i < ILP; i++)\n val += vals[i];\n\n float final = reduce_block_into_lanes(s_vals, val);\n\n if(threadIdx.x == 0)\n {\n if(!isfinite(final))\n *noop_gmem = 1; // Blindly fire off a write. These will race but that's ok.\n output[blockIdx.x] += final;\n if(per_tensor)\n output_per_tensor[(tl.start_tensor_this_launch + tensor_loc)*max_chunks_per_tensor + chunk_idx] = final;\n }\n }\n};\n\n\n__global__ void cleanup(\n float* output,\n float* output_per_tensor,\n float* ret,\n float* ret_per_tensor,\n bool per_tensor,\n int max_chunks_per_tensor)\n{\n __shared__ float vals[512];\n\n if(blockIdx.x == 0)\n {\n float val = 0;\n if(threadIdx.x < 320)\n val = output[threadIdx.x];\n\n float final = reduce_block_into_lanes(vals, val);\n\n if(threadIdx.x == 0)\n *ret = sqrt(final);\n }\n\n if(per_tensor)\n {\n float* output_this_tensor = output_per_tensor + blockIdx.x*max_chunks_per_tensor;\n\n float val = 0;\n for(int i = threadIdx.x; i < max_chunks_per_tensor; i += blockDim.x)\n val += output_this_tensor[i];\n\n float final = reduce_block_into_lanes(vals, val);\n\n if(threadIdx.x == 0)\n ret_per_tensor[blockIdx.x] = sqrt(final);\n }\n}\n\n\nstd::tuple multi_tensor_l2norm_cuda(\n int chunk_size,\n at::Tensor noop_flag,\n std::vector> tensor_lists,\n at::optional per_tensor_python)\n{\n bool per_tensor = per_tensor_python.has_value() ? per_tensor_python.value() : false;\n\n auto float_options = tensor_lists[0][0].options().dtype(at::kFloat);\n auto output = at::zeros({320}, float_options);\n\n at::Tensor output_per_tensor;\n at::Tensor ret_per_tensor;\n\n int ntensors = tensor_lists[0].size();\n int max_chunks_per_tensor = -1;\n\n if(per_tensor)\n {\n for(int t = 0; t < ntensors; t++)\n {\n int max_chunks_this_tensor = (tensor_lists[0][t].numel() + chunk_size - 1)/chunk_size;\n if(max_chunks_this_tensor > max_chunks_per_tensor)\n max_chunks_per_tensor = max_chunks_this_tensor;\n }\n output_per_tensor = at::zeros({ntensors*max_chunks_per_tensor}, float_options);\n ret_per_tensor = at::empty({ntensors}, float_options);\n }\n else\n {\n ret_per_tensor = at::empty({0}, float_options);\n }\n\n DISPATCH_FLOAT_AND_HALF(tensor_lists[0][0].scalar_type(), 0, \"multi_tensor_l2norm_cuda\",\n multi_tensor_apply<1>(\n BLOCK_SIZE,\n chunk_size,\n noop_flag,\n tensor_lists,\n L2NormFunctor(),\n output.data(),\n per_tensor ? output_per_tensor.data() : nullptr,\n per_tensor,\n max_chunks_per_tensor);)\n\n AT_CUDA_CHECK(cudaGetLastError());\n\n // AT_CUDA_CHECK(cudaDeviceSynchronize());\n\n // This involves one more small kernel launches, but will be negligible end to end.\n // I could get rid of these by hacking the functor + multi tensor harness with persistence\n // logic, but keeping it simple for now\n auto ret = at::empty({1}, output.options());\n auto stream = at::cuda::getCurrentCUDAStream();\n cleanup<<>>(\n output.data(),\n per_tensor ? output_per_tensor.data() : nullptr,\n ret.data(),\n per_tensor ? ret_per_tensor.data() : nullptr,\n per_tensor,\n max_chunks_per_tensor);\n\n return std::tuple(ret, ret_per_tensor);\n}\n" }, { "path": "apex/csrc/multi_tensor_lamb_stage_1.cu", "content": "#include \n#include \n#include \n#include \n// Another possibility:\n// #include \n\n#include \n\n#include \"type_shim.h\"\n#include \"multi_tensor_apply.cuh\"\n\n#define BLOCK_SIZE 512\n#define ILP 4\n\n// Step 1 computes the 'update' value of regular Adam optimizer.\ntemplate\nstruct LAMBStage1Functor\n{\n __device__ __forceinline__ void operator()(\n int chunk_size,\n volatile int* noop_gmem,\n TensorListMetadata<5>& tl,\n const float* per_tensor_decay,\n const float beta1,\n const float beta2,\n const float beta1_correction,\n const float beta2_correction,\n const float epsilon,\n const float clipped_global_grad_norm)\n {\n // I'd like this kernel to propagate infs/nans.\n // if(*noop_gmem == 1)\n // return;\n\n int tensor_loc = tl.block_to_tensor[blockIdx.x];\n int tensor_num = tl.start_tensor_this_launch + tensor_loc;\n int chunk_idx = tl.block_to_chunk[blockIdx.x];\n int n = tl.sizes[tensor_loc];\n\n float decay = per_tensor_decay[tensor_num];\n\n GRAD_T* g = (GRAD_T*)tl.addresses[0][tensor_loc];\n g += chunk_idx*chunk_size;\n\n T* p = (T*)tl.addresses[1][tensor_loc];\n p += chunk_idx*chunk_size;\n\n T* m = (T*)tl.addresses[2][tensor_loc];\n m += chunk_idx*chunk_size;\n\n T* v = (T*)tl.addresses[3][tensor_loc];\n v += chunk_idx*chunk_size;\n\n T* update = (T*)tl.addresses[4][tensor_loc];\n update += chunk_idx*chunk_size;\n\n n -= chunk_idx*chunk_size;\n\n // see note in multi_tensor_scale_kernel.cu\n for(int i_start = 0;\n i_start < n && i_start < chunk_size;\n i_start += blockDim.x*ILP)\n {\n GRAD_T r_g[ILP];\n T r_p[ILP];\n T r_m[ILP];\n T r_v[ILP];\n#pragma unroll\n for(int ii = 0; ii < ILP; ii++)\n {\n int i = i_start + threadIdx.x + ii*blockDim.x;\n if(i < n && i < chunk_size)\n {\n r_g[ii] = g[i];\n r_p[ii] = p[i];\n r_m[ii] = m[i];\n r_v[ii] = v[i];\n } else {\n r_g[ii] = GRAD_T(0);\n r_p[ii] = T(0);\n r_m[ii] = T(0);\n r_v[ii] = T(0);\n }\n }\n#pragma unroll\n for(int ii = 0; ii < ILP; ii++)\n {\n T scaled_grad = r_g[ii] / clipped_global_grad_norm;\n r_m[ii] = r_m[ii] * beta1 + (1-beta1) * scaled_grad;\n r_v[ii] = r_v[ii] * beta2 + (1-beta2) * scaled_grad * scaled_grad;\n T next_m_unbiased = r_m[ii] / beta1_correction;\n T next_v_unbiased = r_v[ii] / beta2_correction;\n T denom = std::sqrt(next_v_unbiased) + epsilon;\n r_p[ii] = (next_m_unbiased/denom) + (decay*r_p[ii]);\n }\n#pragma unroll\n for(int ii = 0; ii < ILP; ii++)\n {\n int i = i_start + threadIdx.x + ii*blockDim.x;\n if(i < n && i < chunk_size)\n {\n update[i] = r_p[ii];\n m[i] = r_m[ii];\n v[i] = r_v[ii];\n }\n }\n }\n }\n};\n\nvoid multi_tensor_lamb_stage1_cuda(\n int chunk_size,\n at::Tensor noop_flag,\n std::vector> tensor_lists,\n at::Tensor per_tensor_decay,\n const int step,\n const float beta1,\n const float beta2,\n const float epsilon,\n const float global_grad_norm,\n const float max_global_grad_norm)\n{\n using namespace at;\n\n float clipped_global_grad_norm = global_grad_norm > max_global_grad_norm ? global_grad_norm / max_global_grad_norm : 1.0f;\n float next_step = float(step+1);\n float beta1_correction = 1.0f - std::pow(beta1, next_step);\n float beta2_correction = 1.0f - std::pow(beta2, next_step);\n DISPATCH_FLOAT_AND_HALF(tensor_lists[0][0].scalar_type(), 0, \"lamb_stage_1\",\n DISPATCH_FLOAT_AND_HALF(tensor_lists[1][0].scalar_type(), 1, \"lamb_stage_1\",\n multi_tensor_apply<5>(\n BLOCK_SIZE,\n chunk_size,\n noop_flag,\n tensor_lists,\n LAMBStage1Functor(),\n per_tensor_decay.data(),\n beta1,\n beta2,\n beta1_correction,\n beta2_correction,\n epsilon,\n clipped_global_grad_norm); ))\n\n AT_CUDA_CHECK(cudaGetLastError());\n\n // AT_CUDA_CHECK(cudaDeviceSynchronize());\n}\n" }, { "path": "apex/csrc/multi_tensor_lamb_stage_2.cu", "content": "#include \n#include \n#include \n#include \n// Another possibility:\n// #include \n\n#include \n\n#include \"type_shim.h\"\n#include \"multi_tensor_apply.cuh\"\n\n#define BLOCK_SIZE 512\n#define ILP 4\n\n// Step 2 reads in 'update' value and per-tensor param_norm and update_norm.\n// It computes new parameter value.\ntemplate\nstruct LAMBStage2Functor\n{\n __device__ __forceinline__ void operator()(\n int chunk_size,\n volatile int* noop_gmem,\n TensorListMetadata<2>& tl,\n const float* per_tensor_param_norm,\n const float* per_tensor_update_norm,\n const float learning_rate)\n {\n // I'd like this kernel to propagate infs/nans.\n // if(*noop_gmem == 1)\n // return;\n\n int tensor_loc = tl.block_to_tensor[blockIdx.x];\n int tensor_num = tl.start_tensor_this_launch + tensor_loc;\n int chunk_idx = tl.block_to_chunk[blockIdx.x];\n int n = tl.sizes[tensor_loc];\n\n float param_norm = per_tensor_param_norm[tensor_num];\n float update_norm = per_tensor_update_norm[tensor_num];\n T ratio = (update_norm != 0.0f && param_norm != 0.0f) ? learning_rate * (param_norm / update_norm) : learning_rate;\n\n T* p = (T*)tl.addresses[0][tensor_loc];\n p += chunk_idx*chunk_size;\n\n T* update = (T*)tl.addresses[1][tensor_loc];\n update += chunk_idx*chunk_size;\n\n n -= chunk_idx*chunk_size;\n\n for(int i_start = 0;\n i_start < n && i_start < chunk_size;\n i_start += blockDim.x*ILP)\n {\n T r_p[ILP];\n T r_update[ILP];\n#pragma unroll\n for(int ii = 0; ii < ILP; ii++)\n {\n int i = i_start + threadIdx.x + ii*blockDim.x;\n if(i < n && i < chunk_size)\n {\n r_p[ii] = p[i];\n r_update[ii] = update[i];\n }\n }\n#pragma unroll\n for(int ii = 0; ii < ILP; ii++)\n {\n r_p[ii] = r_p[ii] - (ratio*r_update[ii]);\n }\n#pragma unroll\n for(int ii = 0; ii < ILP; ii++)\n {\n int i = i_start + threadIdx.x + ii*blockDim.x;\n if(i < n && i < chunk_size)\n {\n p[i] = r_p[ii];\n }\n }\n }\n }\n};\n\nvoid multi_tensor_lamb_stage2_cuda(\n int chunk_size,\n at::Tensor noop_flag,\n std::vector> tensor_lists,\n at::Tensor per_tensor_param_norm,\n at::Tensor per_tensor_update_norm,\n const float learning_rate)\n{\n using namespace at;\n\n DISPATCH_FLOAT_AND_HALF(tensor_lists[0][0].scalar_type(), 0, \"lamb_stage_2\",\n multi_tensor_apply<2>(\n BLOCK_SIZE,\n chunk_size,\n noop_flag,\n tensor_lists,\n LAMBStage2Functor(),\n per_tensor_param_norm.data(),\n per_tensor_update_norm.data(),\n learning_rate); )\n\n AT_CUDA_CHECK(cudaGetLastError());\n\n // AT_CUDA_CHECK(cudaDeviceSynchronize());\n}\n" }, { "path": "apex/csrc/multi_tensor_scale_kernel.cu", "content": "#include \n#include \n#include \n#include \n// Another possibility:\n// #include \n\n#include \n// Stringstream is a big hammer, but I want to rely on operator<< for dtype.\n#include \n\n#include \"type_shim.h\"\n#include \"multi_tensor_apply.cuh\"\n\n#define BLOCK_SIZE 512\n#define ILP 4\n\ntemplate\nstruct ScaleFunctor\n{\n __device__ __forceinline__ void operator()(\n int chunk_size,\n volatile int* noop_gmem,\n TensorListMetadata<2>& tl,\n float scale)\n {\n // I'd like this kernel to propagate infs/nans.\n // if(*noop_gmem == 1)\n // return;\n\n int tensor_loc = tl.block_to_tensor[blockIdx.x];\n int chunk_idx = tl.block_to_chunk[blockIdx.x];\n int n = tl.sizes[tensor_loc];\n\n in_t* in = (in_t*)tl.addresses[0][tensor_loc];\n in += chunk_idx*chunk_size;\n \n out_t* out = (out_t*)tl.addresses[1][tensor_loc];\n out += chunk_idx*chunk_size;\n\n n -= chunk_idx*chunk_size;\n\n // Non-divergent exit condition for __syncthreads, not necessary here\n float incoming_vals[ILP];\n for(int i_start = 0;\n i_start < n && i_start < chunk_size;\n i_start += blockDim.x*ILP)\n {\n #pragma unroll\n for(int ii = 0; ii < ILP; ii++)\n {\n incoming_vals[ii] = 0;\n int i = i_start + threadIdx.x + ii*blockDim.x;\n if(i < n && i < chunk_size)\n incoming_vals[ii] = static_cast(in[i]);\n }\n\n // note for clarification to future michael:\n // From a pure memory dependency perspective, there's likely no point unrolling\n // the write loop, since writes just fire off once their LDGs arrive.\n // Put another way, the STGs are dependent on the LDGs, but not on each other.\n // There is still compute ILP benefit from unrolling the loop though.\n #pragma unroll\n for(int ii = 0; ii < ILP; ii++)\n {\n int i = i_start + threadIdx.x + ii*blockDim.x;\n if(i < n && i < chunk_size)\n {\n out[i] = static_cast(incoming_vals[ii]*scale);\n if(!isfinite(incoming_vals[ii]))\n *noop_gmem = 1; // Blindly fire off a write. These will race but that's ok.\n }\n }\n }\n }\n};\n\nvoid multi_tensor_scale_cuda(\n int chunk_size,\n at::Tensor noop_flag,\n std::vector> tensor_lists,\n float scale)\n{\n using namespace at;\n // The output (downscaled) type is always float.\n // If build times suffer, think about where to put this dispatch,\n // and what logic should be moved out of multi_tensor_apply.\n\n DISPATCH_FLOAT_AND_HALF(tensor_lists[0][0].scalar_type(), 0, \"multi_tensor_scale_cuda\",\n DISPATCH_FLOAT_AND_HALF(tensor_lists[1][0].scalar_type(), 1, \"multi_tensor_scale_cuda\",\n multi_tensor_apply<2>(\n BLOCK_SIZE,\n chunk_size,\n noop_flag,\n tensor_lists,\n ScaleFunctor(),\n scale); ))\n AT_CUDA_CHECK(cudaGetLastError());\n\n // AT_CUDA_CHECK(cudaDeviceSynchronize());\n}\n" }, { "path": "apex/csrc/syncbn.cpp", "content": "#include \n#include \n\n#include \n\n// returns {mean,biased_var}\n// implemented using welford \nstd::vector welford_mean_var_CUDA(const at::Tensor input);\n\n// reduces array of mean/var across processes\n// returns global {mean,inv_std,biased_var}\n// implemented using welford \nstd::vector welford_parallel_CUDA(const at::Tensor mean_feature_nodes,\n const at::Tensor var_biased_feature_nodes,\n int numel,\n const float eps);\n\n// elementwise BN operation, returns output\n// input/weight/shift should have identical data type;\n// mean/inv_std have promoted data type (dtype==fp16?fp32:dtype)\nat::Tensor batchnorm_forward_CUDA(const at::Tensor input,\n const at::Tensor mean,\n const at::Tensor inv_std,\n const at::optional weight,\n const at::optional shift);\n\n// backward BN operation, returns {mean_dy, mean_dy_xmu, grad_weight, grad_bias}\n// grad_output/input should have identical data type;\n// mean/inv_std have promoted data type (dtype==fp16?fp32:dtype)\n// implemented using kahan summation\nstd::vector reduce_bn_CUDA(const at::Tensor grad_output,\n const at::Tensor input,\n const at::Tensor mean,\n const at::Tensor inv_std,\n const at::optional weight);\n\n// elementwise backward BN operation, returns grad_input\n// grad_output/input/weight precision could be fp16/fp32;\n// mean/inv_std/mean_dy/mean_dy_xmu precision is fp32\nat::Tensor batchnorm_backward_CUDA(const at::Tensor grad_output,\n const at::Tensor input,\n const at::Tensor mean,\n const at::Tensor inv_std,\n const at::optional weight,\n const at::Tensor mean_dy,\n const at::Tensor mean_dy_xmu);\n\n// returns {mean, biased_var}\n// implemented using welford \n// expect data to be in n+c format (channel last) and applies CUDNN_BATCHNORM_SPATIAL\nstd::vector welford_mean_var_c_last_CUDA(const at::Tensor input);\n\n// elementwise BN operation, returns output\n// input/weight/shift should have identical data type;\n// mean/inv_std have promoted data type (dtype==fp16?fp32:dtype)\n// expect data to be in n+c format (channel last) and applies CUDNN_BATCHNORM_SPATIAL\nat::Tensor batchnorm_forward_c_last_CUDA(const at::Tensor input,\n const at::Tensor mean,\n const at::Tensor inv_std,\n const at::optional weight,\n const at::optional shift);\n\n// backward BN operation, returns {mean_dy, mean_dy_xmu, grad_weight, grad_bias}\n// grad_output/input should have identical data type;\n// mean/inv_std have promoted data type (dtype==fp16?fp32:dtype)\n// expect data to be in n+c format (channel last) and applies CUDNN_BATCHNORM_SPATIAL\nstd::vector reduce_bn_c_last_CUDA(const at::Tensor grad_output,\n const at::Tensor input,\n const at::Tensor mean,\n const at::Tensor inv_std,\n const at::optional weight);\n\n// elementwise backward BN operation, returns grad_input\n// grad_output/input/weight precision could be fp16/fp32;\n// mean/inv_std/mean_dy/mean_dy_xmu precision is fp32\n// expect data to be in n+c format (channel last) and applies CUDNN_BATCHNORM_SPATIAL\nat::Tensor batchnorm_backward_c_last_CUDA(const at::Tensor grad_output,\n const at::Tensor input,\n const at::Tensor mean,\n const at::Tensor inv_std,\n const at::optional weight,\n const at::Tensor mean_dy,\n const at::Tensor mean_dy_xmu);\n\nPYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {\n m.def(\"welford_mean_var\", &welford_mean_var_CUDA, \"welford mean variance\");\n m.def(\"welford_parallel\", &welford_parallel_CUDA, \"welford parallel reduce mean variance\");\n m.def(\"batchnorm_forward\", &batchnorm_forward_CUDA, \"batchnorm forward\");\n m.def(\"reduce_bn\", &reduce_bn_CUDA, \"batchnorm backward reduce grad sum and bias/weight grad\");\n m.def(\"batchnorm_backward\", &batchnorm_backward_CUDA, \"batchnorm backward dgrad\");\n m.def(\"welford_mean_var_c_last\", &welford_mean_var_c_last_CUDA, \"welford mean variance nhwc\");\n m.def(\"batchnorm_forward_c_last\", &batchnorm_forward_c_last_CUDA, \"batchnorm forward nhwc\");\n m.def(\"reduce_bn_c_last\", &reduce_bn_c_last_CUDA, \"batchnorm backwards reduce grad sum and bias/weight grad nhwc\");\n m.def(\"batchnorm_backward_c_last\", &batchnorm_backward_c_last_CUDA, \"batchnorm backward dgrad nhwc\");\n}\n" }, { "path": "apex/csrc/type_shim.h", "content": "#include \n\n// Forward/backward compatiblity hack around\n// https://github.com/pytorch/pytorch/commit/3aeb78079bcd68282fe9117088e138b77318e288\n// pending more future-proof guidance from upstream.\n// struct TypeShim\n// {\n// const at::Type& payload;\n// TypeShim(const at::Type& type) : payload(type) {}\n// // Enable trivial conversion to a const at::Type& for pre-3aeb78\n// operator const at::Type&(){ return payload; };\n// // Enable dispatch switch statements to take *this directly for post-3aeb78\n// //operator at::ScalarType(){ return payload.; };\n// };\n\n#define DISPATCH_FLOAT_AND_HALF(TYPE, LEVEL, NAME, ...) \\\n switch(TYPE) \\\n { \\\n case at::ScalarType::Float: \\\n { \\\n using scalar_t_##LEVEL = float; \\\n __VA_ARGS__; \\\n break; \\\n } \\\n case at::ScalarType::Half: \\\n { \\\n using scalar_t_##LEVEL = at::Half; \\\n __VA_ARGS__; \\\n break; \\\n } \\\n default: \\\n AT_ERROR(#NAME, \" not implemented for '\", toString(TYPE), \"'\"); \\\n }\n\n\n#define DISPATCH_DOUBLE_FLOAT_AND_HALF(TYPE, LEVEL, NAME, ...) \\\n switch(TYPE) \\\n { \\\n case at::ScalarType::Double: \\\n { \\\n using scalar_t_##LEVEL = double; \\\n __VA_ARGS__; \\\n break; \\\n } \\\n case at::ScalarType::Float: \\\n { \\\n using scalar_t_##LEVEL = float; \\\n __VA_ARGS__; \\\n break; \\\n } \\\n case at::ScalarType::Half: \\\n { \\\n using scalar_t_##LEVEL = at::Half; \\\n __VA_ARGS__; \\\n break; \\\n } \\\n default: \\\n AT_ERROR(#NAME, \" not implemented for '\", toString(TYPE), \"'\"); \\\n }\n\n\n #define DISPATCH_DOUBLE_AND_FLOAT(TYPE, LEVEL, NAME, ...) \\\n switch(TYPE) \\\n { \\\n case at::ScalarType::Double: \\\n { \\\n using scalar_t_##LEVEL = double; \\\n __VA_ARGS__; \\\n break; \\\n } \\\n case at::ScalarType::Float: \\\n { \\\n using scalar_t_##LEVEL = float; \\\n __VA_ARGS__; \\\n break; \\\n } \\\n default: \\\n AT_ERROR(#NAME, \" not implemented for '\", toString(TYPE), \"'\"); \\\n }\n\n\ntemplate\n__device__ __forceinline__ T reduce_block_into_lanes\n (T *x,\n T val,\n int lanes=1,\n bool share_result=false) // lanes is intended to be <= 32.\n{\n int tid = threadIdx.x + threadIdx.y*blockDim.x;\n int blockSize = blockDim.x*blockDim.y; // blockSize is intended to be a multiple of 32.\n\n if(blockSize >= 64)\n {\n x[tid] = val;\n __syncthreads();\n }\n\n #pragma unroll\n for(int i = (blockSize >> 1); i >= 64; i >>= 1)\n {\n if(tid < i)\n x[tid] = x[tid] + x[tid+i];\n __syncthreads();\n }\n\n T final;\n\n if(tid < 32)\n {\n if(blockSize >= 64)\n final = x[tid] + x[tid+32];\n else\n final = val;\n // __SYNCWARP();\n\n #pragma unroll\n for(int i = 16; i >= lanes; i >>= 1)\n final = final + __shfl_down_sync(0xffffffff, final, i);\n }\n\n if(share_result)\n {\n if(tid < lanes)\n x[tid] = final; // EpilogueOp\n // Make sure the smem result is visible to all warps.\n __syncthreads();\n }\n\n return final;\n}\n" }, { "path": "apex/csrc/welford.cu", "content": "#include \n#include \n#include \n#include \n\n#include \n#include \n\n#include \n\n#include \"type_shim.h\"\n\n\n__device__ __forceinline__ int lastpow2(int n)\n{\n int out = 1 << (31 - __clz(n));\n if(n == out)\n out >>= 1;\n return out;\n}\n\n__host__ __forceinline__ int h_next_pow2(unsigned int n) {\n n--;\n n |= (n >> 1);\n n |= (n >> 2);\n n |= (n >> 4);\n n |= (n >> 8);\n n |= (n >> 16);\n return ++n;\n}\n\n__host__ __forceinline__ int h_last_pow2(unsigned int n) {\n n |= (n >> 1);\n n |= (n >> 2);\n n |= (n >> 4);\n n |= (n >> 8);\n n |= (n >> 16);\n return n - (n >> 1);\n}\n\n\n#define WARP_SIZE 32\n\ntemplate\n__device__ __forceinline__ T warp_reduce_sum(T val)\n{\n #pragma unroll\n for(int i = WARP_SIZE/2; i > 0; i >>= 1)\n val = val + __shfl_down_sync(0xffffffff, val, i);\n return val;\n}\n\ntemplate\n__device__ __forceinline__ T reduce_block(T *x, T val)\n{\n int tid = threadIdx.y*blockDim.x + threadIdx.x;\n int blockSize = blockDim.x * blockDim.y;\n\n if (blockSize > 32) {\n val = warp_reduce_sum(val);\n if (tid % WARP_SIZE == 0)\n x[tid/WARP_SIZE] = val;\n\n __syncthreads();\n\n val = (tid < blockSize / WARP_SIZE? x[tid%WARP_SIZE] : T(0));\n }\n\n if(tid/WARP_SIZE==0) val = warp_reduce_sum(val);\n\n return val;\n}\n\n#define ELEMENTS_PER_ITER 4 // enables concurrency within each thread to hide latency\n#define ELEMENTS_PER_THREAD 16\n#define OPTIMAL_TILE_W 32\n#define MAX_H_BLOCK 128\n#define MAX_BLOCK_SIZE 512\n\n__host__ int div_ru(int x, int y) {\n return h_last_pow2(1 + (x-1)/y);\n}\n\n__host__ void flexible_launch_configs(\n const int reduction,\n const int stride,\n dim3 &block,\n dim3 &grid,\n const bool coop_flag = false) {\n int block_x = std::min(h_last_pow2(stride), OPTIMAL_TILE_W);\n int block_y = std::min(h_last_pow2(div_ru(reduction , ELEMENTS_PER_THREAD)),\n MAX_BLOCK_SIZE / block_x);\n if (block_x * block_y != MAX_BLOCK_SIZE) {\n block_x = std::min(h_last_pow2(stride), MAX_BLOCK_SIZE / block_y);\n }\n\n int grid_x = div_ru(stride, block_x);\n int grid_y = std::min(div_ru(reduction, block_y * ELEMENTS_PER_THREAD), MAX_H_BLOCK);\n if (coop_flag) {\n // it's not worth having a grid reduction if the reduction dimension is not big enough\n grid_y = grid_y < 8 ? 1 : grid_y;\n }\n\n block.x = block_x;\n block.y = block_y;\n block.z = 1;\n grid.x = grid_x;\n grid.y = grid_y;\n grid.z = 1;\n}\n\ntemplate\n__device__ __forceinline__ void welford_merge_element(C& count,\n T& mean,\n T& m2n,\n const C& num_new,\n const T& mean_new,\n const T& m2n_new) {\n T factor = T(1.0) / max(1, (count + num_new));\n T delta0 = mean - mean_new;\n mean = (mean_new * num_new + mean * count) * factor;\n m2n += m2n_new + delta0 * delta0 * num_new * count * factor;\n count += num_new;\n}\n\ntemplate\n__device__ __forceinline__ void warp_reduce_mean_m2n(T &mean, T &m2n, int &num)\n{\n #pragma unroll\n for(int i = WARP_SIZE/2; i > 0; i >>= 1) {\n auto num_new = __shfl_down_sync(0xffffffff, num, i);\n auto mean_new = __shfl_down_sync(0xffffffff, mean, i);\n auto m2n_new = __shfl_down_sync(0xffffffff, m2n, i);\n welford_merge_element(num, mean, m2n, num_new, mean_new, m2n_new);\n }\n}\n\ntemplate \n__device__ void welford_reduce_mean_m2n(\n T* __restrict__ x,\n int* __restrict__ count,\n T &mean,\n T &m2n,\n int &num,\n int block_size,\n int thread_id)\n{\n int lane = thread_id % WARP_SIZE;\n int wid = thread_id / WARP_SIZE;\n\n if (block_size > 32) {\n warp_reduce_mean_m2n(mean, m2n, num);\n if (lane == 0) {\n x[wid*2] = mean;\n x[wid*2+1] = m2n;\n count[wid] = num;\n }\n __syncthreads();\n\n if (wid == 0) {\n mean = (thread_id < block_size / WARP_SIZE)? x[lane*2] : T(0);\n m2n = (thread_id < block_size / WARP_SIZE)? x[lane*2+1] : T(0);\n num = (thread_id < block_size / WARP_SIZE)? count[lane] : int(0);\n }\n }\n\n if (wid==0) warp_reduce_mean_m2n(mean, m2n, num);\n\n return;\n}\n\n// return spatial size for NC+ Tensors\n__host__ int get_tensor_spatial_size(const at::Tensor& input)\n{\n auto space_size = input.size(2);\n for (int i = 3; i < input.ndimension(); i++) {\n space_size *= input.size(i);\n }\n return space_size;\n}\n\n// promote accumulation scalar type. promote half to float.\n__host__ at::ScalarType promote_scalartype(const at::Tensor& input)\n{\n return input.scalar_type() == at::ScalarType::Half ?\n at::ScalarType::Float : input.scalar_type();\n}\n\n// return single element size, optional accumulation type promotion.\n__host__ size_t get_element_data_size(const at::Tensor& input, bool accumulation = false)\n{\n auto scalar_type = accumulation ? promote_scalartype(input) : input.scalar_type();\n return at::elementSize(scalar_type);\n}\n\ntemplate\n__device__ __forceinline__ void welford_merge_block_vertical(C& count,\n T& mean,\n T& m2n,\n C* shmem_count,\n T* shmem_mean,\n T* shmem_m2n) {\n // write to shared memory\n auto address_base = threadIdx.x + threadIdx.y * blockDim.x;\n shmem_mean[address_base] = mean;\n shmem_m2n[address_base] = m2n;\n shmem_count[address_base] = count;\n\n#pragma unroll\n for (int offset = blockDim.y/2; offset > 0; offset >>= 1) {\n __syncthreads();\n if (threadIdx.y < offset && threadIdx.y + offset < blockDim.y) {\n auto address = address_base + offset * blockDim.x;\n // read shared memory back to register for reduction\n auto num_new = shmem_count[address];\n auto mean_new = shmem_mean[address];\n auto m2n_new = shmem_m2n[address];\n\n welford_merge_element(count, mean, m2n, num_new, mean_new, m2n_new);\n\n // last write is not necessary\n shmem_mean[address_base] = mean;\n shmem_m2n[address_base] = m2n;\n shmem_count[address_base] = count;\n }\n }\n}\n\ntemplate\n__device__ __forceinline__ void merge_block_vertical(T& sum_dy,\n T& sum_dy_xmu,\n T* shmem_sum_dy,\n T* shmem_sum_dy_xmu) {\n // write to shared memory\n auto address_base = threadIdx.x + threadIdx.y * blockDim.x;\n shmem_sum_dy[address_base] = sum_dy;\n shmem_sum_dy_xmu[address_base] = sum_dy_xmu;\n\n#pragma unroll\n for (int offset = blockDim.y/2; offset > 0; offset >>= 1) {\n __syncthreads();\n if (threadIdx.y < offset && threadIdx.y + offset < blockDim.y) {\n auto address = address_base + offset * blockDim.x;\n\n sum_dy += shmem_sum_dy[address];\n sum_dy_xmu += shmem_sum_dy_xmu[address];\n\n // last write is not necessary\n shmem_sum_dy[address_base] = sum_dy;\n shmem_sum_dy_xmu[address_base] = sum_dy_xmu;\n }\n }\n}\n\n\n// welford kernel calculating mean/biased_variance/unbiased_variance\ntemplate \n__global__ void welford_kernel(\n const scalar_t* __restrict__ input,\n outscalar_t* __restrict__ out_mean,\n outscalar_t* __restrict__ out_var_biased,\n const int bs,\n const int fs,\n const int ss) {\n int block_size = blockDim.x * blockDim.y;\n int count = 0;\n accscalar_t x_mean = accscalar_t(0);\n accscalar_t m_2_n = accscalar_t(0);\n\n int thread_id = threadIdx.y*blockDim.x + threadIdx.x;\n\n for (int batch_id = threadIdx.y; batch_id < bs; batch_id += blockDim.y) {\n int input_base = blockIdx.x*ss + batch_id*ss*fs;\n // sequential welford\n for (int offset = threadIdx.x; offset < ss ; offset += blockDim.x) {\n count++;\n auto x_n = static_cast(input[offset+input_base]);\n auto d = x_n - x_mean;\n x_mean += d / count;\n m_2_n += d * (x_n - x_mean);\n }\n }\n\n static __shared__ int s_mem[160];\n accscalar_t* s_mem_ac = (accscalar_t*) &s_mem[32];\n\n welford_reduce_mean_m2n(s_mem_ac, s_mem, x_mean, m_2_n, count, block_size, thread_id);\n\n if (thread_id == 0) {\n out_mean[blockIdx.x] = static_cast(x_mean);\n out_var_biased[blockIdx.x] = static_cast(m_2_n/count);\n }\n}\n\n// elementwise BN kernel\ntemplate \n__global__ void batchnorm_forward_kernel(\n const scalar_t* __restrict__ input,\n const accscalar_t* __restrict__ mean,\n const accscalar_t* __restrict__ inv_std,\n const layerscalar_t* __restrict__ weight,\n const layerscalar_t* __restrict__ shift,\n scalar_t* __restrict__ out,\n const int ss,\n const int bs) {\n auto m_c = mean[blockIdx.x];\n auto inv_std_c = inv_std[blockIdx.x];\n auto w_c = weight == NULL ? accscalar_t(1.0) : static_cast(weight[blockIdx.x]);\n auto s_c = shift == NULL ? accscalar_t(0.0) : static_cast(shift[blockIdx.x]);\n\n for (int batch_offset = blockIdx.y*blockDim.y + threadIdx.y; batch_offset < bs; batch_offset += gridDim.y*blockDim.y) {\n int address_base = blockIdx.x*ss + batch_offset*gridDim.x*ss;\n for (int offset = threadIdx.x + blockIdx.z*blockDim.x; offset < ss ; offset+= gridDim.z*blockDim.x) {\n out[address_base+offset] = static_cast(w_c * (static_cast(input[address_base+offset]) - m_c ) * inv_std_c + s_c);\n }\n }\n}\n\n// Backward BN kernel, calculates grad_bias, grad_weight as well as intermediate\n// results to calculating grad_input.\n// Breaking the grad_input to two step to support sync BN, which requires all\n// reduce of the intermediate results across processes.\ntemplate \n__global__ void reduce_bn_kernel(\n const scalar_t* __restrict__ input,\n const scalar_t* __restrict__ grad_output,\n const accscalar_t* __restrict__ mean,\n const accscalar_t* __restrict__ inv_std,\n accscalar_t* __restrict__ mean_dy,\n accscalar_t* __restrict__ mean_dy_xmu,\n layerscalar_t* __restrict__ grad_weight,\n layerscalar_t* __restrict__ grad_bias,\n const int bs,\n const int fs,\n const int ss) {\n static __shared__ int s_mem[64];\n int total_item_num = bs * ss;\n\n int thread_id = threadIdx.y*blockDim.x + threadIdx.x;\n\n auto r_mean = mean[blockIdx.x];\n auto factor = inv_std[blockIdx.x];\n\n // Kahan sum\n accscalar_t sum_dy = 0.0;\n accscalar_t sum_dy_xmu = 0.0;\n accscalar_t sum_dy_c = 0.0;\n accscalar_t sum_dy_xmu_c = 0.0;\n for (int batch_id = threadIdx.y; batch_id < bs; batch_id += blockDim.y) {\n int input_base = blockIdx.x*ss + batch_id*ss*fs;\n for (int offset = threadIdx.x; offset < ss ; offset += blockDim.x) {\n auto e_grad = static_cast(grad_output[offset+input_base]);\n auto e_input = static_cast(input[offset+input_base]);\n // calculating sum_dy\n auto sum_dy_y = e_grad - sum_dy_c;\n auto sum_dy_t = sum_dy + sum_dy_y;\n sum_dy_c = (sum_dy_t - sum_dy) - sum_dy_y;\n sum_dy = sum_dy_t;\n\n // calculating sum_dy_xmu\n auto sum_dy_xmu_y = e_grad * (e_input - r_mean) - sum_dy_xmu_c;\n auto sum_dy_xmu_t = sum_dy_xmu + sum_dy_xmu_y;\n sum_dy_xmu_c = (sum_dy_xmu_t - sum_dy_xmu) - sum_dy_xmu_y;\n sum_dy_xmu = sum_dy_xmu_t;\n }\n }\n\n sum_dy = reduce_block((accscalar_t*)s_mem, sum_dy);\n __syncthreads();\n sum_dy_xmu = reduce_block((accscalar_t*)s_mem, sum_dy_xmu);\n\n if (thread_id == 0) {\n if (grad_bias != NULL) {\n grad_bias[blockIdx.x] = static_cast(sum_dy);\n }\n if (grad_weight != NULL) {\n grad_weight[blockIdx.x] = static_cast(sum_dy_xmu * factor);\n }\n mean_dy[blockIdx.x] = sum_dy / total_item_num;\n mean_dy_xmu[blockIdx.x] = sum_dy_xmu / total_item_num;\n }\n}\n\n// elementwise backward BN kernel\ntemplate \n__global__ void batchnorm_backward_kernel(\n const scalar_t* __restrict__ grad_output,\n const scalar_t* __restrict__ input,\n const accscalar_t* __restrict__ mean,\n const accscalar_t* __restrict__ inv_std,\n const layerscalar_t* __restrict__ weight,\n const accscalar_t* __restrict__ mean_dy,\n const accscalar_t* __restrict__ mean_dy_xmu,\n scalar_t* __restrict__ grad_input,\n const int ss,\n const int bs) {\n auto m_c = static_cast(mean[blockIdx.x]);\n auto m_dy_c = static_cast(mean_dy[blockIdx.x]);\n auto factor_1_c = inv_std[blockIdx.x];\n auto factor_2_c = (weight == NULL ? accscalar_t(1.0) : static_cast(weight[blockIdx.x])) * factor_1_c;\n factor_1_c = factor_1_c * factor_1_c * mean_dy_xmu[blockIdx.x];\n\n for (int batch_offset = blockIdx.y*blockDim.y+threadIdx.y; batch_offset < bs; batch_offset += gridDim.y*blockDim.y) {\n int address_base = blockIdx.x*ss + batch_offset*gridDim.x*ss;\n for (int offset = threadIdx.x + blockIdx.z*blockDim.x; offset < ss ; offset+= gridDim.z*blockDim.x) {\n grad_input[address_base+offset] = (static_cast(grad_output[address_base+offset]) - m_dy_c - (static_cast(input[address_base+offset]) - m_c) * factor_1_c) * factor_2_c;\n }\n }\n}\n\n// welford kernel for c last tensor calculating mean/biased_variance/unbiased_variance\ntemplate\n \n__global__ void\nwelford_kernel_c_last(\n const scalar_t* __restrict__ input,\n outscalar_t* __restrict__ out_mean,\n outscalar_t* __restrict__ out_var_biased,\n volatile accscalar_t* staging_data,\n int* semaphores,\n const int reduction_size,\n const int stride) {\n // hide latency with concurrency\n accscalar_t x_mean[PARALLEL_LOADS];\n accscalar_t m_2_n[PARALLEL_LOADS];\n int count[PARALLEL_LOADS];\n\n#pragma unroll\n for (int i = 0; i < PARALLEL_LOADS; i++) {\n x_mean[i] = accscalar_t(0);\n m_2_n[i] = accscalar_t(0);\n count[i] = accscalar_t(0);\n }\n // tensor dimension (m,c)\n\n // loop along m dimension\n int inner_loop_stride = blockDim.y * gridDim.y;\n\n // offset along m dimension\n int m_offset = blockIdx.y * blockDim.y + threadIdx.y;\n int c_offset = blockIdx.x * blockDim.x + threadIdx.x;\n\n int loop_count = 1 + (reduction_size - 1) / (inner_loop_stride * PARALLEL_LOADS);\n int address_base = m_offset * stride + c_offset;\n int address_increment = inner_loop_stride * stride;\n\n for (int i = 0; i < loop_count; i++) {\n accscalar_t x_math[PARALLEL_LOADS];\n accscalar_t x_count_inv[PARALLEL_LOADS];\n accscalar_t is_valid[PARALLEL_LOADS];\n\n // load multiple data in\n#pragma unroll\n for (int j = 0; j < PARALLEL_LOADS; j++) {\n if (c_offset < stride && m_offset < reduction_size) {\n x_math[j] = input[address_base];\n count[j]++;\n x_count_inv[j] = accscalar_t(1) / count[j];\n is_valid[j] = accscalar_t(1);\n } else {\n x_math[j] = accscalar_t(0);\n x_count_inv[j] = accscalar_t(0);\n is_valid[j] = accscalar_t(0);\n }\n m_offset += inner_loop_stride;\n address_base += address_increment;\n }\n\n // calculate mean/m2n with welford\n#pragma unroll\n for (int j = 0; j < PARALLEL_LOADS; j++) {\n accscalar_t delta0 = x_math[j] - x_mean[j];\n x_mean[j] += delta0 * x_count_inv[j];\n accscalar_t delta1 = x_math[j] - x_mean[j];\n m_2_n[j] += delta0 * delta1 * is_valid[j];\n }\n }\n\n // thread reduction to accumulate mean/m_2_n/count between PARALLEL_LOADS\n#pragma unroll\n for (int j = 1; j < PARALLEL_LOADS; j++) {\n welford_merge_element(count[0], x_mean[0], m_2_n[0], count[j], x_mean[j], m_2_n[j]);\n }\n\n // release x_mean / m_2_n\n auto mean_th = x_mean[0];\n auto m2_th = m_2_n[0];\n auto count_th = count[0];\n\n // block-wise reduction with shared memory (since reduction cannot be done within a warp)\n static __shared__ accscalar_t shmem_mean[MAX_BLOCK_SIZE];\n static __shared__ accscalar_t shmem_m2n[MAX_BLOCK_SIZE];\n static __shared__ int shmem_count[MAX_BLOCK_SIZE];\n\n welford_merge_block_vertical(count_th, mean_th, m2_th, shmem_count, shmem_mean, shmem_m2n);\n\n // grid reduction if needed (coop launch used at the first place)\n if (gridDim.y > 1) {\n volatile accscalar_t* staging_mean = staging_data;\n volatile accscalar_t* staging_m2n = &staging_data[stride*gridDim.y];\n volatile int* staging_count = reinterpret_cast(&staging_m2n[stride*gridDim.y]);\n\n address_base = c_offset + blockIdx.y * stride;\n // write data to staging_data;\n if (threadIdx.y == 0 && c_offset < stride) {\n staging_mean[address_base] = mean_th;\n staging_m2n[address_base] = m2_th;\n staging_count[address_base] = count_th;\n }\n\n __threadfence();\n __syncthreads(); // ensuring writes to staging_ is visible to all blocks\n\n __shared__ bool is_last_block_done;\n // mark block done\n if (threadIdx.x == 0 && threadIdx.y == 0) {\n int old = atomicAdd(&semaphores[blockIdx.x], 1);\n is_last_block_done = (old == (gridDim.y-1));\n }\n\n __syncthreads();\n\n // check that all data is now available in global memory\n if (is_last_block_done) {\n count_th = 0;\n mean_th = accscalar_t(0.0);\n m2_th = accscalar_t(0.0);\n\n for (int y = threadIdx.y; y < gridDim.y; y += blockDim.y) {\n address_base = c_offset + y * stride;\n int num_new = c_offset < stride ? staging_count[address_base] : 0;\n accscalar_t mean_new = c_offset < stride ? staging_mean[address_base] : accscalar_t(0.0);\n accscalar_t m2n_new = c_offset < stride ? staging_m2n[address_base] : accscalar_t(0.0);\n\n welford_merge_element(count_th, mean_th, m2_th, num_new, mean_new, m2n_new);\n }\n\n welford_merge_block_vertical(count_th, mean_th, m2_th, shmem_count, shmem_mean, shmem_m2n);\n if (threadIdx.y == 0 && c_offset < stride) {\n out_mean[c_offset] = static_cast(mean_th);\n out_var_biased[c_offset] = static_cast(m2_th / count_th);\n }\n }\n } else {\n if (blockIdx.y == 0 && threadIdx.y == 0 && c_offset < stride) {\n out_mean[c_offset] = static_cast(mean_th);\n out_var_biased[c_offset] = static_cast(m2_th / count_th);\n }\n }\n}\n\n// parallel welford kernel to further reduce mean / biased_var\n// into mean / unbiased_var / inv_std across multiple processes.\ntemplate \n__global__ void welford_kernel_parallel(\n const scalar_t* __restrict__ mean,\n const scalar_t* __restrict__ var_biased,\n scalar_t* __restrict__ out_mean,\n scalar_t* __restrict__ out_var,\n scalar_t* __restrict__ inv_std,\n const int world_size,\n const int feature_size,\n const float eps,\n const int numel) {\n\n for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < feature_size; i += gridDim.x * blockDim.x) {\n // load data;\n int address = i;\n scalar_t x_mean = 0;\n scalar_t m_2_n = 0;\n int count = 0;\n for (int j = 0; j < world_size; j++) {\n welford_merge_element(count, x_mean, m_2_n, numel, mean[address], var_biased[address]*numel);\n address += feature_size;\n }\n out_mean[i] = x_mean;\n out_var[i] = m_2_n/ (count - 1);\n inv_std[i] = scalar_t(1) / sqrt(m_2_n/count + eps);\n }\n}\n\n// elementwise BN kernel\ntemplate <\n typename scalar_t,\n typename accscalar_t,\n typename layerscalar_t,\n int PARALLEL_LOADS>\n__global__ void batchnorm_forward_c_last_kernel(\n const scalar_t* __restrict__ input,\n const accscalar_t* __restrict__ mean,\n const accscalar_t* __restrict__ inv_std,\n const layerscalar_t* __restrict__ weight,\n const layerscalar_t* __restrict__ shift,\n scalar_t* __restrict__ out,\n const int reduction_size,\n const int stride) {\n // tensor dimension (m,c)\n // loop along m dimension\n int inner_loop_stride = blockDim.y * gridDim.y;\n\n // offset along m dimension\n int m_offset = blockIdx.y * blockDim.y + threadIdx.y;\n int c_offset = blockIdx.x * blockDim.x + threadIdx.x;\n\n auto m_c = mean[c_offset];\n auto inv_std_c = static_cast(inv_std[c_offset]);\n auto w_c = weight == NULL ? accscalar_t(1.0) : static_cast(weight[c_offset]);\n auto s_c = shift == NULL ? accscalar_t(0.0) : static_cast(shift[c_offset]);\n\n int loop_count = 1 + (reduction_size - 1) / (inner_loop_stride * PARALLEL_LOADS);\n int address_base = m_offset * stride + c_offset;\n int address_increment = inner_loop_stride * stride;\n\n for (int i = 0; i < loop_count; i++) {\n#pragma unroll\n for (int j = 0; j < PARALLEL_LOADS; j++) {\n if (c_offset < stride && m_offset < reduction_size) {\n out[address_base] = static_cast(\n w_c * (static_cast(input[address_base]) - m_c ) * inv_std_c + s_c\n );\n }\n m_offset += inner_loop_stride;\n address_base += address_increment;\n }\n }\n}\n\n// batchnorm backward kernel for c last tensor\ntemplate\n \n__global__ void reduce_bn_c_last_kernel(\n const scalar_t* __restrict__ input,\n const scalar_t* __restrict__ grad_output,\n const accscalar_t* __restrict__ mean,\n const accscalar_t* __restrict__ inv_std,\n accscalar_t* __restrict__ mean_dy,\n accscalar_t* __restrict__ mean_dy_xmu,\n layerscalar_t* __restrict__ grad_weight,\n layerscalar_t* __restrict__ grad_bias,\n volatile accscalar_t* staging_data,\n int* semaphores,\n const int reduction_size,\n const int stride) {\n\n // hide latency with concurrency\n accscalar_t sum_dy[PARALLEL_LOADS];\n accscalar_t sum_dy_xmu[PARALLEL_LOADS];\n\n#pragma unroll\n for (int i = 0; i < PARALLEL_LOADS; i++) {\n sum_dy[i] = accscalar_t(0);\n sum_dy_xmu[i] = accscalar_t(0);\n }\n // tensor dimension (m,c)\n\n // loop along m dimension\n int inner_loop_stride = blockDim.y * gridDim.y;\n\n // offset along m dimension\n int m_offset = blockIdx.y * blockDim.y + threadIdx.y;\n int c_offset = blockIdx.x * blockDim.x + threadIdx.x;\n\n int loop_count = 1 + (reduction_size - 1) / (inner_loop_stride * PARALLEL_LOADS);\n int address_base = m_offset * stride + c_offset;\n int address_increment = inner_loop_stride * stride;\n\n auto r_mean = mean[c_offset];\n auto factor = inv_std[c_offset];\n\n for (int i = 0; i < loop_count; i++) {\n accscalar_t x_input[PARALLEL_LOADS];\n accscalar_t x_grad_output[PARALLEL_LOADS];\n\n // load multiple data in\n#pragma unroll\n for (int j = 0; j < PARALLEL_LOADS; j++) {\n if (c_offset < stride && m_offset < reduction_size) {\n x_input[j] = input[address_base];\n x_grad_output[j] = grad_output[address_base];\n } else {\n x_input[j] = accscalar_t(0);\n x_grad_output[j] = accscalar_t(0);\n }\n m_offset += inner_loop_stride;\n address_base += address_increment;\n }\n\n // calculate sum_dy / sum_dy_xmu\n#pragma unroll\n for (int j = 0; j < PARALLEL_LOADS; j++) {\n sum_dy[j] += x_grad_output[j];\n sum_dy_xmu[j] += x_grad_output[j] * (x_input[j] - r_mean);\n }\n }\n\n // thread reduction to accumulate sum_dy / sum_dy_xmu between PARALLEL_LOADS\n#pragma unroll\n for (int j = 1; j < PARALLEL_LOADS; j++) {\n sum_dy[0] += sum_dy[j];\n sum_dy_xmu[0] += sum_dy_xmu[j];\n }\n\n // release array of registers\n auto sum_dy_th = sum_dy[0];\n auto sum_dy_xmu_th = sum_dy_xmu[0];\n\n // block-wise reduction with shared memory (since reduction cannot be done within a warp)\n static __shared__ accscalar_t shmem_sum_dy[MAX_BLOCK_SIZE];\n static __shared__ accscalar_t shmem_sum_dy_xmu[MAX_BLOCK_SIZE];\n\n merge_block_vertical(sum_dy_th, sum_dy_xmu_th, shmem_sum_dy, shmem_sum_dy_xmu);\n\n // grid reduction if needed (coop launch used at the first place)\n if (gridDim.y > 1) {\n volatile accscalar_t* staging_sum_dy = staging_data;\n volatile accscalar_t* staging_sum_dy_xmu = &staging_data[stride*gridDim.y];\n\n address_base = c_offset + blockIdx.y * stride;\n // write data to staging_data;\n if (threadIdx.y == 0 && c_offset < stride) {\n staging_sum_dy[address_base] = sum_dy_th;\n staging_sum_dy_xmu[address_base] = sum_dy_xmu_th;\n }\n\n __threadfence();\n __syncthreads(); // ensuring writes to staging_ is visible to all blocks\n\n __shared__ bool is_last_block_done;\n // mark block done\n if (threadIdx.x == 0 && threadIdx.y == 0) {\n int old = atomicAdd(&semaphores[blockIdx.x], 1);\n is_last_block_done = (old == (gridDim.y-1));\n }\n\n __syncthreads();\n\n // check that all data is now available in global memory\n if (is_last_block_done) {\n sum_dy_th = accscalar_t(0.0);\n sum_dy_xmu_th = accscalar_t(0.0);\n\n for (int y = threadIdx.y; y < gridDim.y; y += blockDim.y) {\n address_base = c_offset + y * stride;\n sum_dy_th += (c_offset < stride ? staging_sum_dy[address_base] : accscalar_t(0.0));\n sum_dy_xmu_th += (c_offset < stride ? staging_sum_dy_xmu[address_base] : accscalar_t(0.0));\n }\n\n merge_block_vertical(sum_dy_th, sum_dy_xmu_th, shmem_sum_dy, shmem_sum_dy_xmu);\n if (threadIdx.y == 0 && c_offset < stride) {\n if (grad_bias != NULL) {\n grad_bias[c_offset] = static_cast(sum_dy_th);\n }\n if (grad_weight != NULL) {\n grad_weight[c_offset] = static_cast(sum_dy_xmu_th * factor);\n }\n mean_dy[c_offset] = sum_dy_th / reduction_size;\n mean_dy_xmu[c_offset] = sum_dy_xmu_th / reduction_size;\n }\n }\n } else {\n if (blockIdx.y == 0 && threadIdx.y == 0 && c_offset < stride) {\n if (grad_bias != NULL) {\n grad_bias[c_offset] = static_cast(sum_dy_th);\n }\n if (grad_weight != NULL) {\n grad_weight[c_offset] = static_cast(sum_dy_xmu_th * factor);\n }\n mean_dy[c_offset] = sum_dy_th / reduction_size;\n mean_dy_xmu[c_offset] = sum_dy_xmu_th / reduction_size;\n }\n }\n}\n\n// elementwise BN kernel\ntemplate <\n typename scalar_t,\n typename accscalar_t,\n typename layerscalar_t,\n int PARALLEL_LOADS>\n__global__ void batchnorm_backward_c_last_kernel(\n const scalar_t* __restrict__ grad_output,\n const scalar_t* __restrict__ input,\n const accscalar_t* __restrict__ mean,\n const accscalar_t* __restrict__ inv_std,\n const layerscalar_t* __restrict__ weight,\n const accscalar_t* __restrict__ mean_dy,\n const accscalar_t* __restrict__ mean_dy_xmu,\n scalar_t* __restrict__ grad_input,\n const int reduction_size,\n const int stride) {\n // tensor dimension (m,c)\n // loop along m dimension\n int inner_loop_stride = blockDim.y * gridDim.y;\n\n // offset along m dimension\n int m_offset = blockIdx.y * blockDim.y + threadIdx.y;\n int c_offset = blockIdx.x * blockDim.x + threadIdx.x;\n\n auto m_c = mean[c_offset];\n auto m_dy_c = mean_dy[c_offset];\n auto factor_1_c = inv_std[c_offset];\n auto factor_2_c = (weight == NULL? accscalar_t(1.0) : static_cast(weight[c_offset])) * factor_1_c;\n factor_1_c = factor_1_c * factor_1_c * mean_dy_xmu[c_offset];\n\n int loop_count = 1 + (reduction_size - 1) / (inner_loop_stride * PARALLEL_LOADS);\n int address_base = m_offset * stride + c_offset;\n int address_increment = inner_loop_stride * stride;\n\n for (int i = 0; i < loop_count; i++) {\n#pragma unroll\n for (int j = 0; j < PARALLEL_LOADS; j++) {\n if (c_offset < stride && m_offset < reduction_size) {\n grad_input[address_base] = static_cast(\n (static_cast(grad_output[address_base]) - m_dy_c -\n (static_cast(input[address_base]) - m_c) * factor_1_c)\n * factor_2_c);\n }\n m_offset += inner_loop_stride;\n address_base += address_increment;\n }\n }\n}\n\nstd::vector welford_mean_var_CUDA(const at::Tensor input) {\n const auto batch_size = input.size(0);\n const auto feature_size = input.size(1);\n\n auto space_size = get_tensor_spatial_size(input);\n auto scalar_type = promote_scalartype(input);\n\n at::Tensor out_var_biased = at::empty({feature_size}, input.options().dtype(scalar_type));\n at::Tensor out_mean = at::empty({feature_size}, input.options().dtype(scalar_type));\n\n int block_y = min(h_last_pow2(batch_size), int(MAX_BLOCK_SIZE / 32));\n int block_x = max(1, min(MAX_BLOCK_SIZE / block_y, h_last_pow2(space_size)));\n const dim3 block(block_x, block_y);\n const dim3 grid(feature_size);\n\n auto stream = at::cuda::getCurrentCUDAStream();\n\n {\n using namespace at;\n DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, \"welford_mean_var_kernel\",\n using accscalar_t = at::acc_type;\n welford_kernel<<>>(\n input.data(),\n out_mean.data(),\n out_var_biased.data(),\n batch_size,\n feature_size,\n space_size);\n );\n }\n\n return {out_mean, out_var_biased};\n}\n\nat::Tensor batchnorm_forward_CUDA(\n const at::Tensor input,\n const at::Tensor mean,\n const at::Tensor inv_std,\n const at::optional weight,\n const at::optional shift) {\n const auto batch_size = input.size(0);\n const auto feature_size = input.size(1);\n at::Tensor out = at::empty_like(input);\n\n auto space_size = get_tensor_spatial_size(input);\n\n int block_x = max(32, min(MAX_BLOCK_SIZE, h_last_pow2(space_size)/4));\n int block_y = max(1, min(MAX_BLOCK_SIZE/block_x, h_last_pow2(batch_size)/4));\n const dim3 block(block_x, block_y);\n int grid_z = max(1, min(65535, h_last_pow2(space_size)/4/block_x));\n int batch_group_size = max(1, min(65535, h_last_pow2(batch_size)/block_y));\n const dim3 grid(feature_size, batch_group_size, grid_z);\n auto stream = at::cuda::getCurrentCUDAStream();\n\n if (input.scalar_type() == at::ScalarType::Half\n && weight.has_value() &&\n weight.value().scalar_type() == at::ScalarType::Float) {\n using namespace at;\n DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, \"batchnorm_forward\",\n using accscalar_t = at::acc_type;\n batchnorm_forward_kernel<<>>(\n input.data(),\n mean.data(),\n inv_std.data(),\n weight.has_value() ? weight.value().data() : NULL,\n shift.has_value() ? shift.value().data() : NULL,\n out.data(),\n space_size,\n batch_size);\n );\n } else {\n if (weight.has_value()) {\n AT_CHECK(input.scalar_type() == weight.value().scalar_type(),\n \"input.scalar_type() is not supported with weight.scalar_type()\");\n }\n using namespace at;\n DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, \"batchnorm_forward\",\n using accscalar_t = at::acc_type;\n batchnorm_forward_kernel<<>>(\n input.data(),\n mean.data(),\n inv_std.data(),\n weight.has_value() ? weight.value().data() : NULL,\n shift.has_value() ? shift.value().data() : NULL,\n out.data(),\n space_size,\n batch_size);\n );\n }\n return out;\n}\n\nstd::vector reduce_bn_CUDA(\n const at::Tensor grad_output,\n const at::Tensor input,\n const at::Tensor mean,\n const at::Tensor inv_std,\n const at::optional weight)\n{\n const auto batch_size = input.size(0);\n const auto feature_size = input.size(1);\n\n auto scalar_type = promote_scalartype(input);\n\n at::Tensor mean_dy = at::empty({feature_size}, mean.options());\n at::Tensor mean_dy_xmu = at::empty({feature_size}, mean.options());\n\n at::Tensor grad_weight;\n at::Tensor grad_bias;\n if (weight.has_value()) {\n grad_weight = at::empty({feature_size}, weight.value().options());\n grad_bias = at::empty({feature_size}, weight.value().options());\n } else {\n grad_weight = at::empty({0}, mean.options());\n grad_bias = at::empty({0}, mean.options());\n }\n\n auto space_size = get_tensor_spatial_size(input);\n\n int block_y = min(h_last_pow2(batch_size), int(MAX_BLOCK_SIZE/ 32));\n int block_x = max(1, min(MAX_BLOCK_SIZE/ block_y, h_last_pow2(space_size)));\n const dim3 block(block_x, block_y);\n const dim3 grid(feature_size);\n auto stream = at::cuda::getCurrentCUDAStream();\n\n if (input.scalar_type() == at::ScalarType::Half\n && weight.has_value() &&\n weight.value().scalar_type() == at::ScalarType::Float) {\n using namespace at;\n DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, \"batchnorm_backward_reduce\",\n using accscalar_t = at::acc_type;\n reduce_bn_kernel<<>>(\n input.data(),\n grad_output.data(),\n mean.data(),\n inv_std.data(),\n mean_dy.data(),\n mean_dy_xmu.data(),\n weight.has_value() ? grad_weight.data() : NULL,\n weight.has_value() ? grad_bias.data() : NULL,\n batch_size,\n feature_size,\n space_size);\n );\n } else {\n if (weight.has_value()) {\n AT_CHECK(input.scalar_type() == weight.value().scalar_type(),\n \"input.scalar_type() is not supported with weight.scalar_type()\");\n }\n using namespace at;\n DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, \"batchnorm_backward_reduce\",\n using accscalar_t = at::acc_type;\n reduce_bn_kernel<<>>(\n input.data(),\n grad_output.data(),\n mean.data(),\n inv_std.data(),\n mean_dy.data(),\n mean_dy_xmu.data(),\n weight.has_value() ? grad_weight.data() : NULL,\n weight.has_value() ? grad_bias.data() : NULL,\n batch_size,\n feature_size,\n space_size);\n );\n }\n\n return {mean_dy, mean_dy_xmu, grad_weight, grad_bias};\n}\n\nat::Tensor batchnorm_backward_CUDA(\n const at::Tensor grad_output,\n const at::Tensor input,\n const at::Tensor mean,\n const at::Tensor inv_std,\n const at::optional weight,\n const at::Tensor mean_dy,\n const at::Tensor mean_dy_xmu) {\n const auto batch_size = input.size(0);\n const auto feature_size = input.size(1);\n\n at::Tensor grad_input = at::empty_like(input);\n\n auto space_size = get_tensor_spatial_size(input);\n\n int block_x = max(32, min(MAX_BLOCK_SIZE, h_last_pow2(space_size)/4));\n int block_y = max(1, min(MAX_BLOCK_SIZE/block_x, h_last_pow2(batch_size)/4));\n const dim3 block(block_x, block_y);\n int grid_z = max(1, min(65535, h_last_pow2(space_size)/4/block_x));\n int batch_group_size = max(1, min(65535, h_last_pow2(batch_size)/block_y));\n const dim3 grid(feature_size, batch_group_size, grid_z);\n\n auto stream = at::cuda::getCurrentCUDAStream();\n\n if (input.scalar_type() == at::ScalarType::Half\n && weight.has_value() &&\n weight.value().scalar_type() == at::ScalarType::Float) {\n using namespace at;\n DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, \"batchnorm_backward\",\n using accscalar_t = at::acc_type;\n batchnorm_backward_kernel<<>>(\n grad_output.data(),\n input.data(),\n mean.data(),\n inv_std.data(),\n weight.has_value() ? weight.value().data() : NULL,\n mean_dy.data(),\n mean_dy_xmu.data(),\n grad_input.data(),\n space_size,\n batch_size);\n );\n } else {\n if (weight.has_value()) {\n AT_CHECK(input.scalar_type() == weight.value().scalar_type(),\n \"input.scalar_type() is not supported with weight.scalar_type()\");\n }\n using namespace at;\n DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, \"batchnorm_backward\",\n using accscalar_t = at::acc_type;\n batchnorm_backward_kernel<<>>(\n grad_output.data(),\n input.data(),\n mean.data(),\n inv_std.data(),\n weight.has_value() ? weight.value().data() : NULL,\n mean_dy.data(),\n mean_dy_xmu.data(),\n grad_input.data(),\n space_size,\n batch_size);\n );\n }\n\n return grad_input;\n}\n\nstd::vector welford_parallel_CUDA(const at::Tensor mean_feature_nodes,\n const at::Tensor var_biased,\n int numel,\n const float eps) {\n const auto world_size = mean_feature_nodes.size(0);\n const auto feature_size = mean_feature_nodes.size(1);\n\n at::Tensor out_var = at::empty({feature_size}, var_biased.options());\n at::Tensor inv_std = at::empty_like(out_var);\n at::Tensor out_mean = at::empty_like(out_var);\n\n // TODO(jie): tile this for memory coalescing!\n const int block = std::min(h_last_pow2(feature_size), MAX_BLOCK_SIZE);\n const int grid = std::max(1, feature_size / block);\n\n auto stream = at::cuda::getCurrentCUDAStream();\n\n {\n using namespace at;\n DISPATCH_FLOAT_AND_HALF(mean_feature_nodes.scalar_type(), 0, \"welford_parallel_kernel\",\n welford_kernel_parallel<<>>(\n mean_feature_nodes.data(),\n var_biased.data(),\n out_mean.data(),\n out_var.data(),\n inv_std.data(),\n world_size,\n feature_size,\n eps,\n numel);\n );\n }\n\n return {out_mean, out_var, inv_std};\n}\n\nstd::vector welford_mean_var_c_last_CUDA(const at::Tensor input) {\n const auto stride = input.size(input.ndimension()-1);\n const auto reduction_size = input.numel() / stride;\n\n auto scalar_type = promote_scalartype(input);\n auto option = input.options().dtype(scalar_type);\n\n at::Tensor out_var_biased = at::empty({stride}, option);\n at::Tensor out_mean = at::empty({stride}, option);\n\n dim3 block;\n dim3 grid;\n flexible_launch_configs(reduction_size, stride, block, grid, true);\n\n at::Tensor staging_data;\n at::Tensor semaphores;\n if (grid.y > 1) {\n staging_data = at::empty({4*stride*grid.y}, option);\n semaphores = at::zeros({grid.x}, input.options().dtype(at::kInt));\n }\n\n auto stream = at::cuda::getCurrentCUDAStream();\n\n {\n using namespace at;\n DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, \"welford_mean_var_c_last\",\n using accscalar_t = at::acc_type;\n accscalar_t* staging_data_ptr = grid.y > 1 ? staging_data.data() : nullptr;\n int* semaphores_ptr = grid.y > 1 ? semaphores.data() : nullptr;\n welford_kernel_c_last\n <<>>(\n input.data(),\n out_mean.data(),\n out_var_biased.data(),\n staging_data_ptr,\n semaphores_ptr,\n reduction_size,\n stride);\n );\n }\n\n return {out_mean, out_var_biased};\n}\n\nat::Tensor batchnorm_forward_c_last_CUDA(\n const at::Tensor input,\n const at::Tensor mean,\n const at::Tensor inv_std,\n const at::optional weight,\n const at::optional shift) {\n const auto stride = input.size(input.ndimension()-1);\n const auto reduction_size = input.numel() / stride;\n\n at::Tensor out = at::empty_like(input);\n\n dim3 block;\n dim3 grid;\n flexible_launch_configs(reduction_size, stride, block, grid);\n\n auto stream = at::cuda::getCurrentCUDAStream();\n\n if (input.scalar_type() == at::ScalarType::Half\n && weight.has_value() && weight.value().scalar_type() == at::ScalarType::Float) {\n using namespace at;\n DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, \"batchnorm_forward\",\n using accscalar_t = at::acc_type;\n batchnorm_forward_c_last_kernel\n <<>>(\n input.data(),\n mean.data(),\n inv_std.data(),\n weight.has_value() ? weight.value().data() : NULL,\n shift.has_value() ? shift.value().data(): NULL,\n out.data(),\n reduction_size,\n stride);\n );\n } else {\n if (weight.has_value()) {\n AT_CHECK(input.scalar_type() == weight.value().scalar_type(),\n \"input.scalar_type() is not supported with weight.scalar_type()\");\n }\n using namespace at;\n DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, \"batchnorm_forward\",\n using accscalar_t = at::acc_type;\n batchnorm_forward_c_last_kernel\n <<>>(\n input.data(),\n mean.data(),\n inv_std.data(),\n weight.has_value() ? weight.value().data() : NULL,\n shift.has_value() ? shift.value().data(): NULL,\n out.data(),\n reduction_size,\n stride);\n );\n }\n return out;\n}\n\nstd::vector reduce_bn_c_last_CUDA(\n const at::Tensor grad_output,\n const at::Tensor input,\n const at::Tensor mean,\n const at::Tensor inv_std,\n const at::optional weight) {\n const auto stride = input.size(input.ndimension()-1);\n const auto reduction_size = input.numel() / stride;\n\n at::Tensor mean_dy = at::empty({stride}, mean.options());\n at::Tensor mean_dy_xmu = at::empty({stride}, mean.options());\n\n at::Tensor grad_weight;\n at::Tensor grad_bias;\n if (weight.has_value()) {\n grad_weight = at::empty({stride}, weight.value().options());\n grad_bias = at::empty({stride}, weight.value().options());\n } else {\n // because I cannot return an uninitialized at::Tensor\n grad_weight = at::empty({0}, mean.options());\n grad_bias = at::empty({0}, mean.options());\n }\n\n dim3 block;\n dim3 grid;\n flexible_launch_configs(reduction_size, stride, block, grid, true);\n\n at::Tensor staging_data;\n at::Tensor semaphores;\n if (grid.y > 1) {\n staging_data = at::empty({2*stride*grid.y}, mean.options());\n semaphores = at::zeros({grid.x}, input.options().dtype(at::kInt));\n }\n auto stream = at::cuda::getCurrentCUDAStream();\n\n if (input.scalar_type() == at::ScalarType::Half\n && weight.has_value()\n && weight.value().scalar_type() == at::ScalarType::Float) {\n using namespace at;\n DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, \"batchnorm_backward_reduce\",\n using accscalar_t = at::acc_type;\n accscalar_t* staging_data_ptr = grid.y > 1 ? staging_data.data() : nullptr;\n int* semaphores_ptr = grid.y > 1 ? semaphores.data() : nullptr;\n reduce_bn_c_last_kernel\n <<>>(\n input.data(),\n grad_output.data(),\n mean.data(),\n inv_std.data(),\n mean_dy.data(),\n mean_dy_xmu.data(),\n weight.has_value() ? grad_weight.data() : NULL,\n weight.has_value() ?grad_bias.data() : NULL,\n staging_data_ptr,\n semaphores_ptr,\n reduction_size,\n stride);\n );\n } else {\n if (weight.has_value()) {\n AT_CHECK(input.scalar_type() == weight.value().scalar_type(),\n \"input.scalar_type() is not supported with weight.scalar_type()\");\n }\n using namespace at;\n DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, \"batchnorm_backward_reduce\",\n using accscalar_t = at::acc_type;\n accscalar_t* staging_data_ptr = grid.y > 1 ? staging_data.data() : nullptr;\n int* semaphores_ptr = grid.y > 1 ? semaphores.data() : nullptr;\n reduce_bn_c_last_kernel\n <<>>(\n input.data(),\n grad_output.data(),\n mean.data(),\n inv_std.data(),\n mean_dy.data(),\n mean_dy_xmu.data(),\n weight.has_value() ? grad_weight.data() : NULL,\n weight.has_value() ?grad_bias.data() : NULL,\n staging_data_ptr,\n semaphores_ptr,\n reduction_size,\n stride);\n );\n }\n\n return {mean_dy, mean_dy_xmu, grad_weight, grad_bias};\n}\n\nat::Tensor batchnorm_backward_c_last_CUDA(\n const at::Tensor grad_output,\n const at::Tensor input,\n const at::Tensor mean,\n const at::Tensor inv_std,\n const at::optional weight,\n const at::Tensor mean_dy,\n const at::Tensor mean_dy_xmu) {\n const auto stride = input.size(input.ndimension()-1);\n const auto reduction_size = input.numel() / stride;\n\n at::Tensor grad_input = at::empty_like(input);\n\n dim3 block;\n dim3 grid;\n flexible_launch_configs(reduction_size, stride, block, grid);\n\n auto stream = at::cuda::getCurrentCUDAStream();\n\n if (input.scalar_type() == at::ScalarType::Half\n && weight.has_value() && weight.value().scalar_type() == at::ScalarType::Float) {\n using namespace at;\n DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, \"batchnorm_forward\",\n using accscalar_t = at::acc_type;\n batchnorm_backward_c_last_kernel\n <<>>(\n grad_output.data(),\n input.data(),\n mean.data(),\n inv_std.data(),\n weight.has_value() ? weight.value().data() : NULL,\n mean_dy.data(),\n mean_dy_xmu.data(),\n grad_input.data(),\n reduction_size,\n stride);\n );\n } else {\n if (weight.has_value()) {\n AT_CHECK(input.scalar_type() == weight.value().scalar_type(),\n \"input.scalar_type() is not supported with weight.scalar_type()\");\n }\n using namespace at;\n DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, \"batchnorm_forward\",\n using accscalar_t = at::acc_type;\n batchnorm_backward_c_last_kernel\n <<>>(\n grad_output.data(),\n input.data(),\n mean.data(),\n inv_std.data(),\n weight.has_value() ? weight.value().data() : NULL,\n mean_dy.data(),\n mean_dy_xmu.data(),\n grad_input.data(),\n reduction_size,\n stride);\n );\n }\n \n return grad_input;\n}\n" }, { "path": "apex/docs/Makefile", "content": "# Minimal makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHINXBUILD = sphinx-build\nSPHINXPROJ = NVIDIAAPEX\nSOURCEDIR = source\nBUILDDIR = build\n\n# Put it first so that \"make\" without argument is like \"make help\".\nhelp:\n\t@$(SPHINXBUILD) -M help \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)\n\ngh-pages:\n\tgit checkout gh-pages\n\trm -rf build\n\trm -rf source\n\tgit checkout master -- .\n\tmake html\n\trm -rf ../_modules ../_sources ../_static\n\tmv -fv build/html/* ../\n\trm -rf build\n\tgit add -A\n\tgit commit -m \"Generated gh-pages for `git log master -1 --pretty=short --abbrev-commit`\" && git push origin gh-pages ; git checkout master\n\n.PHONY: help Makefile\n\n# Catch-all target: route all unknown targets to Sphinx using the new\n# \"make mode\" option. $(O) is meant as a shortcut for $(SPHINXOPTS).\n%: Makefile\n\t@$(SPHINXBUILD) -M $@ \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)\n" }, { "path": "apex/docs/source/_static/css/pytorch_theme.css", "content": "body {\n font-family: \"Lato\",\"proxima-nova\",\"Helvetica Neue\",Arial,sans-serif;\n}\n\n/* Default header fonts are ugly */\nh1, h2, .rst-content .toctree-wrapper p.caption, h3, h4, h5, h6, legend, p.caption {\n font-family: \"Lato\",\"proxima-nova\",\"Helvetica Neue\",Arial,sans-serif;\n}\n\n/* Use white for docs background */\n.wy-side-nav-search {\n background-color: #fff;\n}\n\n.wy-nav-content-wrap, .wy-menu li.current > a {\n background-color: #fff;\n}\n\n@media screen and (min-width: 1400px) {\n .wy-nav-content-wrap {\n background-color: rgba(0, 0, 0, 0.0470588);\n }\n\n .wy-nav-content {\n background-color: #fff;\n }\n}\n\n/* Fixes for mobile */\n.wy-nav-top {\n background-color: #fff;\n background-image: url('../img/apex.jpg');\n background-repeat: no-repeat;\n background-position: center;\n padding: 0;\n margin: 0.4045em 0.809em;\n color: #333;\n}\n\n.wy-nav-top > a {\n display: none;\n}\n\n@media screen and (max-width: 768px) {\n .wy-side-nav-search>a img.logo {\n height: 60px;\n }\n}\n\n/* This is needed to ensure that logo above search scales properly */\n.wy-side-nav-search a {\n display: block;\n}\n\n/* This ensures that multiple constructors will remain in separate lines. */\n.rst-content dl:not(.docutils) dt {\n display: table;\n}\n\n/* Use our red for literals (it's very similar to the original color) */\n.rst-content tt.literal, .rst-content tt.literal, .rst-content code.literal {\n color: #F05732;\n}\n\n.rst-content tt.xref, a .rst-content tt, .rst-content tt.xref,\n.rst-content code.xref, a .rst-content tt, a .rst-content code {\n color: #404040;\n}\n\n/* Change link colors (except for the menu) */\n\na {\n color: #F05732;\n}\n\na:hover {\n color: #F05732;\n}\n\n\na:visited {\n color: #D44D2C;\n}\n\n.wy-menu a {\n color: #b3b3b3;\n}\n\n.wy-menu a:hover {\n color: #b3b3b3;\n}\n\n/* Default footer text is quite big */\nfooter {\n font-size: 80%;\n}\n\nfooter .rst-footer-buttons {\n font-size: 125%; /* revert footer settings - 1/80% = 125% */\n}\n\nfooter p {\n font-size: 100%;\n}\n\n/* For hidden headers that appear in TOC tree */\n/* see http://stackoverflow.com/a/32363545/3343043 */\n.rst-content .hidden-section {\n display: none;\n}\n\nnav .hidden-section {\n display: inherit;\n}\n\n.wy-side-nav-search>div.version {\n color: #000;\n}\n" }, { "path": "apex/docs/source/_templates/layout.html", "content": "{% extends \"!layout.html\" %}\n {% block sidebartitle %} {{ super() }}\n\n \n {% endblock %}\n\n {% block footer %} {{ super() }}\n\n \n {% endblock %}\n" }, { "path": "apex/docs/source/advanced.rst", "content": ".. role:: hidden\n :class: hidden-section\n\nAdvanced Amp Usage\n===================================\n\nGANs\n----\n\nGANs are an interesting synthesis of several topics below. A `comprehensive example`_\nis under construction.\n\n.. _`comprehensive example`:\n https://github.com/NVIDIA/apex/tree/master/examples/dcgan\n\nGradient clipping\n-----------------\nAmp calls the params owned directly by the optimizer's ``param_groups`` the \"master params.\"\n\nThese master params may be fully or partially distinct from ``model.parameters()``.\nFor example, with `opt_level=\"O2\"`_, ``amp.initialize`` casts most model params to FP16,\ncreates an FP32 master param outside the model for each newly-FP16 model param,\nand updates the optimizer's ``param_groups`` to point to these FP32 params.\n\nThe master params owned by the optimizer's ``param_groups`` may also fully coincide with the\nmodel params, which is typically true for ``opt_level``\\s ``O0``, ``O1``, and ``O3``.\n\nIn all cases, correct practice is to clip the gradients of the params that are guaranteed to be\nowned **by the optimizer's** ``param_groups``, instead of those retrieved via ``model.parameters()``.\n\nAlso, if Amp uses loss scaling, gradients must be clipped after they have been unscaled\n(which occurs during exit from the ``amp.scale_loss`` context manager).\n\nThe following pattern should be correct for any ``opt_level``::\n\n with amp.scale_loss(loss, optimizer) as scaled_loss:\n scaled_loss.backward()\n # Gradients are unscaled during context manager exit.\n # Now it's safe to clip. Replace\n # torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm)\n # with\n torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), max_norm)\n # or\n torch.nn.utils.clip_grad_value_(amp.master_params(optimizer), max_)\n\nNote the use of the utility function ``amp.master_params(optimizer)``,\nwhich returns a generator-expression that iterates over the\nparams in the optimizer's ``param_groups``.\n\nAlso note that ``clip_grad_norm_(amp.master_params(optimizer), max_norm)`` is invoked\n*instead of*, not *in addition to*, ``clip_grad_norm_(model.parameters(), max_norm)``.\n\n.. _`opt_level=\"O2\"`:\n https://nvidia.github.io/apex/amp.html#o2-fast-mixed-precision\n\nCustom/user-defined autograd functions\n--------------------------------------\n\nThe old Amp API for `registering user functions`_ is still considered correct. Functions must\nbe registered before calling ``amp.initialize``.\n\n.. _`registering user functions`:\n https://github.com/NVIDIA/apex/tree/master/apex/amp#annotating-user-functions\n\nForcing particular layers/functions to a desired type\n-----------------------------------------------------\n\nI'm still working on a generalizable exposure for this that won't require user-side code divergence\nacross different ``opt-level``\\ s.\n\nMultiple models/optimizers/losses\n---------------------------------\n\nInitialization with multiple models/optimizers\n**********************************************\n\n``amp.initialize``'s optimizer argument may be a single optimizer or a list of optimizers,\nas long as the output you accept has the same type.\nSimilarly, the ``model`` argument may be a single model or a list of models, as long as the accepted\noutput matches. The following calls are all legal::\n\n model, optim = amp.initialize(model, optim,...)\n model, [optim0, optim1] = amp.initialize(model, [optim0, optim1],...)\n [model0, model1], optim = amp.initialize([model0, model1], optim,...)\n [model0, model1], [optim0, optim1] = amp.initialize([model0, model1], [optim0, optim1],...)\n\nBackward passes with multiple optimizers\n****************************************\n\nWhenever you invoke a backward pass, the ``amp.scale_loss`` context manager must receive\n**all the optimizers that own any params for which the current backward pass is creating gradients.**\nThis is true even if each optimizer owns only some, but not all, of the params that are about to\nreceive gradients.\n\nIf, for a given backward pass, there's only one optimizer whose params are about to receive gradients,\nyou may pass that optimizer directly to ``amp.scale_loss``. Otherwise, you must pass the\nlist of optimizers whose params are about to receive gradients::\n\n # loss0 accumulates gradients only into params owned by optim0:\n with amp.scale_loss(loss0, optim0) as scaled_loss:\n scaled_loss.backward()\n\n # loss1 accumulates gradients only into params owned by optim1:\n with amp.scale_loss(loss1, optim1) as scaled_loss:\n scaled_loss.backward()\n\n # loss2 accumulates gradients into some params owned by optim0\n # and some params owned by optim1\n with amp.scale_loss(loss2, [optim0, optim1]) as scaled_loss:\n scaled_loss.backward()\n\nOptionally have Amp use a different loss scaler per-loss\n********************************************************\n\nBy default, Amp maintains a single global loss scaler that will be used for all backward passes\n(all invocations of ``with amp.scale_loss(...)``). No additional arguments to ``amp.initialize``\nor ``amp.scale_loss`` are required to use the global loss scaler. The code snippets above with\nmultiple optimizers/backward passes use the single global loss scaler under the hood,\nand they should \"just work.\"\n\nHowever, you can optionally tell Amp to maintain a loss scaler per-loss, which gives Amp increased\nnumerical flexibility. This is accomplished by supplying the ``num_losses`` argument to\n``amp.initialize`` (which tells Amp how many backward passes you plan to invoke, and therefore\nhow many loss scalers Amp should create), then supplying the ``loss_id`` argument to each of your\nbackward passes (which tells Amp the loss scaler to use for this particular backward pass)::\n\n model, [optim0, optim1] = amp.initialize(model, [optim0, optim1], ..., num_losses=3)\n\n with amp.scale_loss(loss0, optim0, loss_id=0) as scaled_loss:\n scaled_loss.backward()\n\n with amp.scale_loss(loss1, optim1, loss_id=1) as scaled_loss:\n scaled_loss.backward()\n\n with amp.scale_loss(loss2, [optim0, optim1], loss_id=2) as scaled_loss:\n scaled_loss.backward()\n\n``num_losses`` and ``loss_id``\\ s should be specified purely based on the set of\nlosses/backward passes. The use of multiple optimizers, or association of single or\nmultiple optimizers with each backward pass, is unrelated.\n\nGradient accumulation across iterations\n---------------------------------------\n\nThe following should \"just work,\" and properly accommodate multiple models/optimizers/losses, as well as\ngradient clipping via the `instructions above`_::\n\n if iter%iters_to_accumulate == 0:\n # Every iters_to_accumulate iterations, unscale and step\n with amp.scale_loss(loss, optimizer) as scaled_loss:\n scaled_loss.backward()\n # Gradient clipping if desired:\n # torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), max_norm)\n optimizer.step()\n optimizer.zero_grad()\n else:\n # Otherwise, accumulate gradients, don't unscale or step.\n with amp.scale_loss(loss, optimizer) as scaled_loss:\n scaled_loss.backward()\n\nAs a minor performance optimization, you can pass ``delay_unscale=True``\nto ``amp.scale_loss`` until you're ready to ``step()``. You should only attempt ``delay_unscale=True``\nif you're sure you know what you're doing, because the interaction with gradient clipping and\nmultiple models/optimizers/losses can become tricky.::\n\n if iter%iters_to_accumulate == 0:\n # Every iters_to_accumulate iterations, unscale and step\n with amp.scale_loss(loss, optimizer) as scaled_loss:\n scaled_loss.backward()\n optimizer.step()\n optimizer.zero_grad()\n else:\n # Otherwise, accumulate gradients, don't unscale or step.\n with amp.scale_loss(loss, optimizer, delay_unscale=True) as scaled_loss:\n scaled_loss.backward()\n\n.. _`instructions above`:\n https://nvidia.github.io/apex/advanced.html#gradient-clipping\n\nCustom data batch types\n-----------------------\n\nThe intention of Amp is that you never need to cast your input data manually, regardless of\n``opt_level``. Amp accomplishes this by patching any models' ``forward`` methods to cast\nincoming data appropriately for the ``opt_level``. But to cast incoming data,\nAmp needs to know how. The patched ``forward`` will recognize and cast floating-point Tensors\n(non-floating-point Tensors like IntTensors are not touched) and\nPython containers of floating-point Tensors. However, if you wrap your Tensors in a custom class,\nthe casting logic doesn't know how to drill\nthrough the tough custom shell to access and cast the juicy Tensor meat within. You need to tell\nAmp how to cast your custom batch class, by assigning it a ``to`` method that accepts a ``torch.dtype``\n(e.g., ``torch.float16`` or ``torch.float32``) and returns an instance of the custom batch cast to\n``dtype``. The patched ``forward`` checks for the presence of your ``to`` method, and will\ninvoke it with the correct type for the ``opt_level``.\n\nExample::\n\n class CustomData(object):\n def __init__(self):\n self.tensor = torch.cuda.FloatTensor([1,2,3])\n\n def to(self, dtype):\n self.tensor = self.tensor.to(dtype)\n return self\n\n.. warning::\n\n Amp also forwards numpy ndarrays without casting them. If you send input data as a raw, unwrapped\n ndarray, then later use it to create a Tensor within your ``model.forward``, this Tensor's type will\n not depend on the ``opt_level``, and may or may not be correct. Users are encouraged to pass\n castable data inputs (Tensors, collections of Tensors, or custom classes with a ``to`` method)\n wherever possible.\n\n.. note::\n\n Amp does not call ``.cuda()`` on any Tensors for you. Amp assumes that your original script\n is already set up to move Tensors from the host to the device as needed.\n" }, { "path": "apex/docs/source/amp.rst", "content": ".. role:: hidden\n :class: hidden-section\n\napex.amp\n===================================\n\nThis page documents the updated API for Amp (Automatic Mixed Precision),\na tool to enable Tensor Core-accelerated training in only 3 lines of Python.\n\nA `runnable, comprehensive Imagenet example`_ demonstrating good practices can be found\non the Github page.\n\nGANs are a tricky case that many people have requested. A `comprehensive DCGAN example`_\nis under construction.\n\nIf you already implemented Amp based on the instructions below, but it isn't behaving as expected,\nplease review `Advanced Amp Usage`_ to see if any topics match your use case. If that doesn't help,\n`file an issue`_.\n\n.. _`file an issue`:\n https://github.com/NVIDIA/apex/issues\n\n``opt_level``\\ s and Properties\n-------------------------------\n\nAmp allows users to easily experiment with different pure and mixed precision modes.\nCommonly-used default modes are chosen by\nselecting an \"optimization level\" or ``opt_level``; each ``opt_level`` establishes a set of\nproperties that govern Amp's implementation of pure or mixed precision training.\nFiner-grained control of how a given ``opt_level`` behaves can be achieved by passing values for\nparticular properties directly to ``amp.initialize``. These manually specified values\noverride the defaults established by the ``opt_level``.\n\nExample::\n\n # Declare model and optimizer as usual, with default (FP32) precision\n model = torch.nn.Linear(D_in, D_out).cuda()\n optimizer = torch.optim.SGD(model.parameters(), lr=1e-3)\n\n # Allow Amp to perform casts as required by the opt_level\n model, optimizer = amp.initialize(model, optimizer, opt_level=\"O1\")\n ...\n # loss.backward() becomes:\n with amp.scale_loss(loss, optimizer) as scaled_loss:\n scaled_loss.backward()\n ...\n\nUsers **should not** manually cast their model or data to ``.half()``, regardless of what ``opt_level``\nor properties are chosen. Amp intends that users start with an existing default (FP32) script,\nadd the three lines corresponding to the Amp API, and begin training with mixed precision.\nAmp can also be disabled, in which case the original script will behave exactly as it used to.\nIn this way, there's no risk adhering to the Amp API, and a lot of potential performance benefit.\n\n.. note::\n Because it's never necessary to manually cast your model (aside from the call ``amp.initialize``)\n or input data, a script that adheres to the new API\n can switch between different ``opt-level``\\ s without having to make any other changes.\n\n.. _`runnable, comprehensive Imagenet example`:\n https://github.com/NVIDIA/apex/tree/master/examples/imagenet\n\n.. _`comprehensive DCGAN example`:\n https://github.com/NVIDIA/apex/tree/master/examples/dcgan\n\n.. _`Advanced Amp Usage`:\n https://nvidia.github.io/apex/advanced.html\n\nProperties\n**********\n\nCurrently, the under-the-hood properties that govern pure or mixed precision training are the following:\n\n- ``cast_model_type``: Casts your model's parameters and buffers to the desired type.\n- ``patch_torch_functions``: Patch all Torch functions and Tensor methods to perform Tensor Core-friendly ops like GEMMs and convolutions in FP16, and any ops that benefit from FP32 precision in FP32.\n- ``keep_batchnorm_fp32``: To enhance precision and enable cudnn batchnorm (which improves performance), it's often beneficial to keep batchnorm weights in FP32 even if the rest of the model is FP16.\n- ``master_weights``: Maintain FP32 master weights to accompany any FP16 model weights. FP32 master weights are stepped by the optimizer to enhance precision and capture small gradients.\n- ``loss_scale``: If ``loss_scale`` is a float value, use this value as the static (fixed) loss scale. If ``loss_scale`` is the string ``\"dynamic\"``, adaptively adjust the loss scale over time. Dynamic loss scale adjustments are performed by Amp automatically.\n\nAgain, you often don't need to specify these properties by hand. Instead, select an ``opt_level``,\nwhich will set them up for you. After selecting an ``opt_level``, you can optionally pass property\nkwargs as manual overrides.\n\nIf you attempt to override a property that does not make sense for the selected ``opt_level``,\nAmp will raise an error with an explanation. For example, selecting ``opt_level=\"O1\"`` combined with\nthe override ``master_weights=True`` does not make sense. ``O1`` inserts casts\naround Torch functions rather than model weights. Data, activations, and weights are recast\nout-of-place on the fly as they flow through patched functions. Therefore, the model weights themselves\ncan (and should) remain FP32, and there is no need to maintain separate FP32 master weights.\n\n``opt_level``\\ s\n****************\n\nRecognized ``opt_level``\\ s are ``\"O0\"``, ``\"O1\"``, ``\"O2\"``, and ``\"O3\"``.\n\n``O0`` and ``O3`` are not true mixed precision, but they are useful for establishing accuracy and\nspeed baselines, respectively.\n\n``O1`` and ``O2`` are different implementations of mixed precision. Try both, and see\nwhat gives the best speedup and accuracy for your model.\n\n``O0``: FP32 training\n^^^^^^^^^^^^^^^^^^^^^^\nYour incoming model should be FP32 already, so this is likely a no-op.\n``O0`` can be useful to establish an accuracy baseline.\n\n| Default properties set by ``O0``:\n| ``cast_model_type=torch.float32``\n| ``patch_torch_functions=False``\n| ``keep_batchnorm_fp32=None`` (effectively, \"not applicable,\" everything is FP32)\n| ``master_weights=False``\n| ``loss_scale=1.0``\n|\n|\n\n``O1``: Mixed Precision (recommended for typical use)\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nPatch all Torch functions and Tensor methods to cast their inputs according to a whitelist-blacklist\nmodel. Whitelist ops (for example, Tensor Core-friendly ops like GEMMs and convolutions) are performed\nin FP16. Blacklist ops that benefit from FP32 precision (for example, softmax)\nare performed in FP32. ``O1`` also uses dynamic loss scaling, unless overridden.\n\n| Default properties set by ``O1``:\n| ``cast_model_type=None`` (not applicable)\n| ``patch_torch_functions=True``\n| ``keep_batchnorm_fp32=None`` (again, not applicable, all model weights remain FP32)\n| ``master_weights=None`` (not applicable, model weights remain FP32)\n| ``loss_scale=\"dynamic\"``\n|\n|\n\n``O2``: \"Almost FP16\" Mixed Precision\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n``O2`` casts the model weights to FP16,\npatches the model's ``forward`` method to cast input\ndata to FP16, keeps batchnorms in FP32, maintains FP32 master weights,\nupdates the optimizer's ``param_groups`` so that the ``optimizer.step()``\nacts directly on the FP32 weights (followed by FP32 master weight->FP16 model weight\ncopies if necessary),\nand implements dynamic loss scaling (unless overridden).\nUnlike ``O1``, ``O2`` does not patch Torch functions or Tensor methods.\n\n| Default properties set by ``O2``:\n| ``cast_model_type=torch.float16``\n| ``patch_torch_functions=False``\n| ``keep_batchnorm_fp32=True``\n| ``master_weights=True``\n| ``loss_scale=\"dynamic\"``\n|\n|\n\n``O3``: FP16 training\n^^^^^^^^^^^^^^^^^^^^^^\n``O3`` may not achieve the stability of the true mixed precision options ``O1`` and ``O2``.\nHowever, it can be useful to establish a speed baseline for your model, against which\nthe performance of ``O1`` and ``O2`` can be compared. If your model uses batch normalization,\nto establish \"speed of light\" you can try ``O3`` with the additional property override\n``keep_batchnorm_fp32=True`` (which enables cudnn batchnorm, as stated earlier).\n\n| Default properties set by ``O3``:\n| ``cast_model_type=torch.float16``\n| ``patch_torch_functions=False``\n| ``keep_batchnorm_fp32=False``\n| ``master_weights=False``\n| ``loss_scale=1.0``\n|\n|\n\nUnified API\n-----------\n\n.. automodule:: apex.amp\n.. currentmodule:: apex.amp\n\n.. autofunction:: initialize\n\n.. autofunction:: scale_loss\n\n.. autofunction:: master_params\n\nAdvanced use cases\n------------------\n\nThe unified Amp API supports gradient accumulation across iterations,\nmultiple backward passes per iteration, multiple models/optimizers,\ncustom/user-defined autograd functions, and custom data batch classes. Gradient clipping and GANs also\nrequire special treatment, but this treatment does not need to change\nfor different ``opt_level``\\ s. Further details can be found here:\n\n.. toctree::\n :maxdepth: 1\n\n advanced\n\nTransition guide for old API users\n----------------------------------\n\nWe strongly encourage moving to the new Amp API, because it's more versatile, easier to use, and future proof. The original :class:`FP16_Optimizer` and the old \"Amp\" API are deprecated, and subject to removal at at any time.\n\nFor users of the old \"Amp\" API\n******************************\n\nIn the new API, ``opt-level O1`` performs the same patching of the Torch namespace as the old thing\ncalled \"Amp.\"\nHowever, the new API allows static or dynamic loss scaling, while the old API only allowed dynamic loss scaling.\n\nIn the new API, the old call to ``amp_handle = amp.init()``, and the returned ``amp_handle``, are no\nlonger exposed or necessary. The new ``amp.initialize()`` does the duty of ``amp.init()`` (and more).\nTherefore, any existing calls to ``amp_handle = amp.init()`` should be deleted.\n\nThe functions formerly exposed through ``amp_handle`` are now free\nfunctions accessible through the ``amp`` module.\n\nThe backward context manager must be changed accordingly::\n\n # old API\n with amp_handle.scale_loss(loss, optimizer) as scaled_loss:\n scaled_loss.backward()\n ->\n # new API\n with amp.scale_loss(loss, optimizer) as scaled_loss:\n scaled_loss.backward()\n\nFor now, the deprecated \"Amp\" API documentation can still be found on the Github README: https://github.com/NVIDIA/apex/tree/master/apex/amp. The old API calls that `annotate user functions`_ to run\nwith a particular precision are still honored by the new API.\n\n.. _`annotate user functions`:\n https://github.com/NVIDIA/apex/tree/master/apex/amp#annotating-user-functions\n\n\nFor users of the old FP16_Optimizer\n***********************************\n\n``opt-level O2`` is equivalent to :class:`FP16_Optimizer` with ``dynamic_loss_scale=True``.\nOnce again, the backward pass must be changed to the unified version::\n\n optimizer.backward(loss)\n ->\n with amp.scale_loss(loss, optimizer) as scaled_loss:\n scaled_loss.backward()\n\nOne annoying aspect of FP16_Optimizer was that the user had to manually convert their model to half\n(either by calling ``.half()`` on it, or using a function or module wrapper from\n``apex.fp16_utils``), and also manually call ``.half()`` on input data. **Neither of these are\nnecessary in the new API. No matter what --opt-level\nyou choose, you can and should simply build your model and pass input data in the default FP32 format.**\nThe new Amp API will perform the right conversions during\n``model, optimizer = amp.initialize(model, optimizer, opt_level=....)`` based on the ``--opt-level``\nand any overridden flags. Floating point input data may be FP32 or FP16, but you may as well just\nlet it be FP16, because the ``model`` returned by ``amp.initialize`` will have its ``forward``\nmethod patched to cast the input data appropriately.\n" }, { "path": "apex/docs/source/conf.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# PyTorch documentation build configuration file, created by\n# sphinx-quickstart on Fri Dec 23 13:31:47 2016.\n#\n# This file is execfile()d with the current directory set to its\n# containing dir.\n#\n# Note that not all possible configuration values are present in this\n# autogenerated file.\n#\n# All configuration values have a default; values that are commented out\n# serve to show the default.\n\n# If extensions (or modules to document with autodoc) are in another directory,\n# add these directories to sys.path here. If the directory is relative to the\n# documentation root, use os.path.abspath to make it absolute, like shown here.\n#\nimport os\nimport sys\nsys.path.insert(0, os.path.abspath('.'))\n# sys.path.insert(0, os.path.abspath('../../apex/parallel/'))\nimport apex\n# import multiproc\nimport sphinx_rtd_theme\n\n\n# -- General configuration ------------------------------------------------\n\n# If your documentation needs a minimal Sphinx version, state it here.\n#\n# needs_sphinx = '1.0'\n\n# Add any Sphinx extension module names here, as strings. They can be\n# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom\n# ones.\nextensions = [\n 'sphinx.ext.autodoc',\n 'sphinx.ext.autosummary',\n 'sphinx.ext.doctest',\n 'sphinx.ext.intersphinx',\n 'sphinx.ext.todo',\n 'sphinx.ext.coverage',\n 'sphinx.ext.mathjax',\n 'sphinx.ext.napoleon',\n 'sphinx.ext.viewcode',\n 'sphinx.ext.extlinks',\n]\n\nnapoleon_use_ivar = True\n\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = ['_templates']\n\n# The suffix(es) of source filenames.\n# You can specify multiple suffix as a list of string:\n#\n# source_suffix = ['.rst', '.md']\nsource_suffix = '.rst'\n\n# The master toctree document.\nmaster_doc = 'index'\n\n# General information about the project.\nproject = 'Apex'\ncopyright = '2018'\nauthor = 'Christian Sarofeen, Natalia Gimelshein, Michael Carilli, Raul Puri'\n\n# The version info for the project you're documenting, acts as replacement for\n# |version| and |release|, also used in various other places throughout the\n# built documents.\n#\n# The short X.Y version.\n# TODO: change to [:2] at v1.0\n# version = 'master (' + torch.__version__ + ' )'\nversion = '0.1'\n# The full version, including alpha/beta/rc tags.\n# TODO: verify this works as expected\nrelease = '0.1.0'\n\n# The language for content autogenerated by Sphinx. Refer to documentation\n# for a list of supported languages.\n#\n# This is also used if you do content translation via gettext catalogs.\n# Usually you set \"language\" from the command line for these cases.\nlanguage = None\n\n# List of patterns, relative to source directory, that match files and\n# directories to ignore when looking for source files.\n# This pattern also affects html_static_path and html_extra_path\nexclude_patterns = []\n\n# The name of the Pygments (syntax highlighting) style to use.\npygments_style = 'sphinx'\n\n# If true, `todo` and `todoList` produce output, else they produce nothing.\ntodo_include_todos = True\n\n\n# -- Options for HTML output -------------------------------------------------\n\n# The theme to use for HTML and HTML Help pages. See the documentation for\n# a list of builtin themes.\n#\nhtml_theme = 'sphinx_rtd_theme'\nhtml_theme_path = [sphinx_rtd_theme.get_html_theme_path()]\n\n# Theme options are theme-specific and customize the look and feel of a theme\n# further. For a list of options available for each theme, see the\n# documentation.\n#\nhtml_theme_options = {\n 'collapse_navigation': False,\n 'display_version': True,\n 'logo_only': True,\n}\n\n# html_logo = '_static/img/nv-pytorch2.png'\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\nhtml_static_path = ['_static']\n\n# html_style_path = 'css/pytorch_theme.css'\nhtml_context = {\n 'css_files': [\n 'https://fonts.googleapis.com/css?family=Lato',\n '_static/css/pytorch_theme.css'\n ],\n}\n\n\n# -- Options for HTMLHelp output ---------------------------------------------\n\n# Output file base name for HTML help builder.\nhtmlhelp_basename = 'PyTorchdoc'\n\n\n# -- Options for LaTeX output ------------------------------------------------\n\nlatex_elements = {\n # The paper size ('letterpaper' or 'a4paper').\n #\n # 'papersize': 'letterpaper',\n\n # The font size ('10pt', '11pt' or '12pt').\n #\n # 'pointsize': '10pt',\n\n # Additional stuff for the LaTeX preamble.\n #\n # 'preamble': '',\n\n # Latex figure (float) alignment\n #\n # 'figure_align': 'htbp',\n}\n\n# Grouping the document tree into LaTeX files. List of tuples\n# (source start file, target name, title,\n# author, documentclass [howto, manual, or own class]).\nlatex_documents = [\n (master_doc, 'apex.tex', 'Apex Documentation',\n 'Torch Contributors', 'manual'),\n]\n\n\n# -- Options for manual page output ------------------------------------------\n\n# One entry per manual page. List of tuples\n# (source start file, name, description, authors, manual section).\nman_pages = [\n (master_doc, 'Apex', 'Apex Documentation',\n [author], 1)\n]\n\n\n# -- Options for Texinfo output ----------------------------------------------\n\n# Grouping the document tree into Texinfo files. List of tuples\n# (source start file, target name, title, author,\n# dir menu entry, description, category)\ntexinfo_documents = [\n (master_doc, 'Apex', 'Apex Documentation',\n author, 'Apex', 'One line description of project.',\n 'Miscellaneous'),\n]\n\n\n# Example configuration for intersphinx: refer to the Python standard library.\nintersphinx_mapping = {\n 'python': ('https://docs.python.org/', None),\n 'numpy': ('http://docs.scipy.org/doc/numpy/', None),\n}\n\n# -- A patch that prevents Sphinx from cross-referencing ivar tags -------\n# See http://stackoverflow.com/a/41184353/3343043\n\nfrom docutils import nodes\nfrom sphinx.util.docfields import TypedField\nfrom sphinx import addnodes\n\n\ndef patched_make_field(self, types, domain, items, **kw):\n # `kw` catches `env=None` needed for newer sphinx while maintaining\n # backwards compatibility when passed along further down!\n\n # type: (List, unicode, Tuple) -> nodes.field\n def handle_item(fieldarg, content):\n par = nodes.paragraph()\n par += addnodes.literal_strong('', fieldarg) # Patch: this line added\n # par.extend(self.make_xrefs(self.rolename, domain, fieldarg,\n # addnodes.literal_strong))\n if fieldarg in types:\n par += nodes.Text(' (')\n # NOTE: using .pop() here to prevent a single type node to be\n # inserted twice into the doctree, which leads to\n # inconsistencies later when references are resolved\n fieldtype = types.pop(fieldarg)\n if len(fieldtype) == 1 and isinstance(fieldtype[0], nodes.Text):\n typename = u''.join(n.astext() for n in fieldtype)\n typename = typename.replace('int', 'python:int')\n typename = typename.replace('long', 'python:long')\n typename = typename.replace('float', 'python:float')\n typename = typename.replace('type', 'python:type')\n par.extend(self.make_xrefs(self.typerolename, domain, typename,\n addnodes.literal_emphasis, **kw))\n else:\n par += fieldtype\n par += nodes.Text(')')\n par += nodes.Text(' -- ')\n par += content\n return par\n\n fieldname = nodes.field_name('', self.label)\n if len(items) == 1 and self.can_collapse:\n fieldarg, content = items[0]\n bodynode = handle_item(fieldarg, content)\n else:\n bodynode = self.list_type()\n for fieldarg, content in items:\n bodynode += nodes.list_item('', handle_item(fieldarg, content))\n fieldbody = nodes.field_body('', bodynode)\n return nodes.field('', fieldname, fieldbody)\n\nTypedField.make_field = patched_make_field\n" }, { "path": "apex/docs/source/fp16_utils.rst", "content": ".. role:: hidden\n :class: hidden-section\n\napex.fp16_utils\n===================================\n\nThis submodule contains utilities designed to streamline the mixed precision training recipe \npresented by NVIDIA `on Parallel Forall`_ and in GTC 2018 Sessions \n`Training Neural Networks with Mixed Precision: Theory and Practice`_ and \n`Training Neural Networks with Mixed Precision: Real Examples`_.\nFor Pytorch users, Real Examples in particular is recommended.\n\nFull runnable Python scripts demonstrating ``apex.fp16_utils`` \ncan be found on the Github page:\n\n| `Simple FP16_Optimizer demos`_\n|\n| `Distributed Mixed Precision Training with imagenet`_\n|\n| `Mixed Precision Training with word_language_model`_\n|\n|\n\n.. _`on Parallel Forall`:\n https://devblogs.nvidia.com/mixed-precision-training-deep-neural-networks/\n.. _`Training Neural Networks with Mixed Precision: Theory and Practice`:\n http://on-demand.gputechconf.com/gtc/2018/video/S8923/\n.. _`Training Neural Networks with Mixed Precision: Real Examples`:\n http://on-demand.gputechconf.com/gtc/2018/video/S81012/\n.. _`Simple FP16_Optimizer demos`:\n https://github.com/NVIDIA/apex/tree/master/examples/FP16_Optimizer_simple\n.. _`Distributed Mixed Precision Training with imagenet`:\n https://github.com/NVIDIA/apex/tree/master/examples/imagenet\n.. _`Mixed Precision Training with word_language_model`:\n https://github.com/NVIDIA/apex/tree/master/examples/word_language_model\n\n.. automodule:: apex.fp16_utils\n.. currentmodule:: apex.fp16_utils\n\nAutomatic management of master params + loss scaling\n----------------------------------------------------\n\n.. autoclass:: FP16_Optimizer\n :members:\n\n.. autoclass:: LossScaler\n :members:\n\n.. autoclass:: DynamicLossScaler\n :members:\n\nManual master parameter management\n----------------------------------\n\n.. autofunction:: prep_param_lists\n\n.. autofunction:: master_params_to_model_params\n\n.. autofunction:: model_grads_to_master_grads\n" }, { "path": "apex/docs/source/index.rst", "content": ".. PyTorch documentation master file, created by\n sphinx-quickstart on Fri Dec 23 13:31:47 2016.\n You can adapt this file completely to your liking, but it should at least\n contain the root `toctree` directive.\n\n:github_url: https://github.com/nvidia/apex\n\nApex (A PyTorch Extension)\n===================================\n\nThis site contains the API documentation for Apex (https://github.com/nvidia/apex),\na Pytorch extension with NVIDIA-maintained utilities to streamline mixed precision and distributed training. Some of the code here will be included in upstream Pytorch eventually. The intention of Apex is to make up-to-date utilities available to users as quickly as possible.\n\nInstallation instructions can be found here: https://github.com/NVIDIA/apex#quick-start.\n\n.. toctree::\n :maxdepth: 1\n :caption: AMP: Automatic Mixed Precision\n\n amp\n\n.. toctree::\n :maxdepth: 1\n :caption: Distributed Training\n\n parallel\n\n.. toctree::\n :maxdepth: 1\n :caption: Fused Optimizers\n\n optimizers\n\n.. toctree::\n :maxdepth: 1\n :caption: Fused Layer Norm\n\n layernorm\n\n.. .. toctree::\n :maxdepth: 1\n :caption: Deprecated mixed precision API\n fp16_util\n\n.. reparameterization\n.. RNN\n \nIndices and tables\n==================\n\n* :ref:`genindex`\n* :ref:`modindex`\n" }, { "path": "apex/docs/source/layernorm.rst", "content": ".. role:: hidden\n :class: hidden-section\n\napex.normalization.fused_layer_norm\n===================================\n\n.. automodule:: apex.normalization\n.. currentmodule:: apex.normalization\n\n.. FusedAdam\n ----------\n\n.. autoclass:: FusedLayerNorm\n :members:\n" }, { "path": "apex/docs/source/optimizers.rst", "content": ".. role:: hidden\n :class: hidden-section\n\napex.optimizers\n===================================\n\n.. automodule:: apex.optimizers\n.. currentmodule:: apex.optimizers\n\n.. FusedAdam\n ----------\n\n.. autoclass:: FusedAdam\n :members:\n" }, { "path": "apex/docs/source/parallel.rst", "content": ".. role:: hidden\n :class: hidden-section\n\napex.parallel\n===================================\n\n.. automodule:: apex.parallel\n.. currentmodule:: apex.parallel\n\n.. DistributedDataParallel\n ----------\n\n.. autoclass:: DistributedDataParallel\n :members:\n\n.. autoclass:: Reducer\n :members:\n\n.. autoclass:: SyncBatchNorm\n :members:\n\nUtility functions\n----------------------------------\n\n.. autofunction:: convert_syncbn_model\n" }, { "path": "apex/examples/README.md", "content": "This directory contains examples illustrating Apex mixed precision and distributed tools.\n\n**Note for users of the pre-unification API**:\n`deprecated_api` contains examples illustrating the old (pre-unified) APIs. These APIs will be removed soon, and users are strongly encouraged to switch. The separate mixed precision tools called `Amp` and `FP16_Optimizer` in the old API are exposed via different flags/optimization levels in the new API.\n" }, { "path": "apex/examples/dcgan/README.md", "content": "Under construction...\n" }, { "path": "apex/examples/docker/Dockerfile", "content": "# Base image must at least have pytorch and CUDA installed.\nARG BASE_IMAGE=nvcr.io/nvidia/pytorch:19.03-py3\nFROM $BASE_IMAGE\nARG BASE_IMAGE\nRUN echo \"Installing Apex on top of ${BASE_IMAGE}\"\n# make sure we don't overwrite some existing directory called \"apex\"\nWORKDIR /tmp/unique_for_apex\n# uninstall Apex if present, twice to make absolutely sure :)\nRUN pip uninstall -y apex || :\nRUN pip uninstall -y apex || :\n# SHA is something the user can touch to force recreation of this Docker layer,\n# and therefore force cloning of the latest version of Apex\nRUN SHA=ToUcHMe git clone https://github.com/NVIDIA/apex.git\nWORKDIR /tmp/unique_for_apex/apex\nRUN pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" .\nWORKDIR /workspace\n" }, { "path": "apex/examples/docker/README.md", "content": "## Option 1: Create a new container with Apex\n\n**Dockerfile** installs the latest Apex on top of an existing image. Run\n```\ndocker build -t new_image_with_apex .\n```\nBy default, **Dockerfile** uses NVIDIA's Pytorch container as the base image,\nwhich requires an NVIDIA GPU Cloud (NGC) account. If you don't have an NGC account, you can sign up for free by following the instructions [here](https://docs.nvidia.com/ngc/ngc-getting-started-guide/index.html#generating-api-key).\n\nAlternatively, you can supply your own base image via the `BASE_IMAGE` build-arg.\n`BASE_IMAGE` must have Pytorch and Cuda installed. For example, any\n`-devel` image for Pytorch 1.0 and later from the\n[official Pytorch Dockerhub](https://hub.docker.com/r/pytorch/pytorch) may be used:\n```\ndocker build --build-arg BASE_IMAGE=pytorch/pytorch:nightly-devel-cuda10.0-cudnn7 -t new_image_with_apex .\n```\n\nIf you want to rebuild your image, and force the latest Apex to be cloned and installed, make any small change to the `SHA` variable in **Dockerfile**.\n\n**Warning:**\nCurrently, the non-`-devel` images on Pytorch Dockerhub do not contain the Cuda compiler `nvcc`. Therefore,\nimages whose name does not contain `-devel` are not eligible candidates for `BASE_IMAGE`.\n\n### Running your Apex container\n\nLike any Cuda-enabled Pytorch container, a container with Apex should be run via [nvidia-docker](https://github.com/NVIDIA/nvidia-docker), for example:\n```\ndocker run --runtime=nvidia -it --rm --ipc=host new_image_with_apex\n```\n\n## Option 2: Install Apex in a running container\n\nInstead of building a new container, it is also a viable option to `git clone https://github.com/NVIDIA/apex.git` on bare metal, mount the Apex repo into your container at launch by running, for example,\n```\ndocker run --runtime=nvidia -it --rm --ipc=host -v /bare/metal/apex:/apex/in/container \n```\nthen go to /apex/in/container within the running container and\n```\npip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" .\n```\n" }, { "path": "apex/examples/imagenet/README.md", "content": "# Mixed Precision ImageNet Training in PyTorch\n\n`main_amp.py` is based on [https://github.com/pytorch/examples/tree/master/imagenet](https://github.com/pytorch/examples/tree/master/imagenet).\nIt implements Automatic Mixed Precision (Amp) training of popular model architectures, such as ResNet, AlexNet, and VGG, on the ImageNet dataset. Command-line flags forwarded to `amp.initialize` are used to easily manipulate and switch between various pure and mixed precision \"optimization levels\" or `opt_level`s. For a detailed explanation of `opt_level`s, see the [updated API guide](https://nvidia.github.io/apex/amp.html).\n\nThree lines enable Amp:\n```\n# Added after model and optimizer construction\nmodel, optimizer = amp.initialize(model, optimizer, flags...)\n...\n# loss.backward() changed to:\nwith amp.scale_loss(loss, optimizer) as scaled_loss:\n scaled_loss.backward()\n```\n\nWith the new Amp API **you never need to explicitly convert your model, or the input data, to half().**\n\n## Requirements\n\n- Download the ImageNet dataset and move validation images to labeled subfolders\n - The following script may be helpful: https://raw.githubusercontent.com/soumith/imagenetloader.torch/master/valprep.sh\n\n## Training\n\nTo train a model, create softlinks to the Imagenet dataset, then run `main.py` with the desired model architecture, as shown in `Example commands` below.\n\nThe default learning rate schedule is set for ResNet50. `main_amp.py` script rescales the learning rate according to the global batch size (number of distributed processes \\* per-process minibatch size).\n\n## Example commands\n\n**Note:** batch size `--b 224` assumes your GPUs have >=16GB of onboard memory. You may be able to increase this to 256, but that's cutting it close, so it may out-of-memory for different Pytorch versions.\n\n**Note:** All of the following use 4 dataloader subprocesses (`--workers 4`) to reduce potential\nCPU data loading bottlenecks.\n\n**Note:** `--opt-level` `O1` and `O2` both use dynamic loss scaling by default unless manually overridden.\n`--opt-level` `O0` and `O3` (the \"pure\" training modes) do not use loss scaling by default.\n`O0` and `O3` can be told to use loss scaling via manual overrides, but using loss scaling with `O0`\n(pure FP32 training) does not really make sense, and will trigger a warning.\n\nSoftlink training and validation datasets into the current directory:\n```\n$ ln -sf /data/imagenet/train-jpeg/ train\n$ ln -sf /data/imagenet/val-jpeg/ val\n```\n\n### Summary\n\nAmp allows easy experimentation with various pure and mixed precision options.\n```\n$ python main_amp.py -a resnet50 --b 128 --workers 4 --opt-level O0 ./\n$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O3 ./\n$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O3 --keep-batchnorm-fp32 True ./\n$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O1 ./\n$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O1 --loss-scale 128.0 ./\n$ python -m torch.distributed.launch --nproc_per_node=2 main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O1 ./\n$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O2 ./\n$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O2 --loss-scale 128.0 ./\n$ python -m torch.distributed.launch --nproc_per_node=2 main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O2 ./\n```\nOptions are explained below. Again, the [updated API guide](https://nvidia.github.io/apex/amp.html) provides more detail.\n\n#### `--opt-level O0` (FP32 training) and `O3` (FP16 training)\n\n\"Pure FP32\" training:\n```\n$ python main_amp.py -a resnet50 --b 128 --workers 4 --opt-level O0 ./\n```\n\"Pure FP16\" training:\n```\n$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O3 ./\n```\nFP16 training with FP32 batchnorm:\n```\n$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O3 --keep-batchnorm-fp32 True ./\n```\nKeeping the batchnorms in FP32 improves stability and allows Pytorch\nto use cudnn batchnorms, which significantly increases speed in Resnet50.\n\nThe `O3` options might not converge, because they are not true mixed precision.\nHowever, they can be useful to establish \"speed of light\" performance for\nyour model, which provides a baseline for comparison with `O1` and `O2`.\nFor Resnet50 in particular, `--opt-level O3 --keep-batchnorm-fp32 True` establishes\nthe \"speed of light.\" (Without `--keep-batchnorm-fp32`, it's slower, because it does\nnot use cudnn batchnorm.)\n\n#### `--opt-level O1` (\"conservative mixed precision\")\n\n`O1` patches Torch functions to cast inputs according to a whitelist-blacklist model.\nFP16-friendly (Tensor Core) ops like gemms and convolutions run in FP16, while ops\nthat benefit from FP32, like batchnorm and softmax, run in FP32.\nAlso, dynamic loss scaling is used by default.\n```\n$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O1 ./\n```\n`O1` overridden to use static loss scaling:\n```\n$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O1 --loss-scale 128.0\n```\nDistributed training with 2 processes (1 GPU per process, see **Distributed training** below\nfor more detail)\n```\n$ python -m torch.distributed.launch --nproc_per_node=2 main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O1 ./\n```\nFor best performance, set `--nproc_per_node` equal to the total number of GPUs on the node\nto use all available resources.\n\n#### `--opt-level O2` (\"fast mixed precision\")\n\n`O2` casts the model to FP16, keeps batchnorms in FP32,\nmaintains master weights in FP32, and implements\ndynamic loss scaling by default. (Unlike --opt-level O1, --opt-level O2\ndoes not patch Torch functions.)\n```\n$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O2 ./\n```\n\"Fast mixed precision\" overridden to use static loss scaling:\n```\n$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O2 --loss-scale 128.0 ./\n```\nDistributed training with 2 processes (1 GPU per process)\n```\n$ python -m torch.distributed.launch --nproc_per_node=2 main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O2 ./\n```\n\n## Distributed training\n\n`main_amp.py` optionally uses `apex.parallel.DistributedDataParallel` (DDP) for multiprocess training with one GPU per process.\n```\nmodel = apex.parallel.DistributedDataParallel(model)\n```\nis a drop-in replacement for\n```\nmodel = torch.nn.parallel.DistributedDataParallel(model,\n device_ids=[arg.local_rank],\n output_device=arg.local_rank)\n```\n(because Torch DDP permits multiple GPUs per process, with Torch DDP you are required to\nmanually specify the device to run on and the output device.\nWith Apex DDP, it uses only the current device by default).\n\nThe choice of DDP wrapper (Torch or Apex) is orthogonal to the use of Amp and other Apex tools. It is safe to use `apex.amp` with either `torch.nn.parallel.DistributedDataParallel` or `apex.parallel.DistributedDataParallel`. In the future, I may add some features that permit optional tighter integration between `Amp` and `apex.parallel.DistributedDataParallel` for marginal performance benefits, but currently, there's no compelling reason to use Apex DDP versus Torch DDP for most models.\n\nTo use DDP with `apex.amp`, the only gotcha is that\n```\nmodel, optimizer = amp.initialize(model, optimizer, flags...)\n```\nmust precede\n```\nmodel = DDP(model)\n```\nIf DDP wrapping occurs before `amp.initialize`, `amp.initialize` will raise an error.\n\nWith both Apex DDP and Torch DDP, you must also call `torch.cuda.set_device(args.local_rank)` within\neach process prior to initializing your model or any other tensors.\nMore information can be found in the docs for the\nPytorch multiprocess launcher module [torch.distributed.launch](https://pytorch.org/docs/stable/distributed.html#launch-utility).\n\n`main_amp.py` is written to interact with \n[torch.distributed.launch](https://pytorch.org/docs/master/distributed.html#launch-utility),\nwhich spawns multiprocess jobs using the following syntax:\n```\npython -m torch.distributed.launch --nproc_per_node=NUM_GPUS main_amp.py args...\n```\n`NUM_GPUS` should be less than or equal to the number of visible GPU devices on the node. The use of `torch.distributed.launch` is unrelated to the choice of DDP wrapper. It is safe to use either apex DDP or torch DDP with `torch.distributed.launch`.\n\nOptionally, one can run imagenet with synchronized batch normalization across processes by adding\n`--sync_bn` to the `args...`\n\n## Deterministic training (for debugging purposes)\n\nRunning with the `--deterministic` flag should produce bitwise identical outputs run-to-run,\nregardless of what other options are used (see [Pytorch docs on reproducibility](https://pytorch.org/docs/stable/notes/randomness.html)).\nSince `--deterministic` disables `torch.backends.cudnn.benchmark`, `--deterministic` may\ncause a modest performance decrease.\n" }, { "path": "apex/examples/imagenet/main_amp.py", "content": "import argparse\nimport os\nimport shutil\nimport time\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.parallel\nimport torch.backends.cudnn as cudnn\nimport torch.distributed as dist\nimport torch.optim\nimport torch.utils.data\nimport torch.utils.data.distributed\nimport torchvision.transforms as transforms\nimport torchvision.datasets as datasets\nimport torchvision.models as models\n\nimport numpy as np\n\ntry:\n from apex.parallel import DistributedDataParallel as DDP\n from apex.fp16_utils import *\n from apex import amp, optimizers\n from apex.multi_tensor_apply import multi_tensor_applier\nexcept ImportError:\n raise ImportError(\"Please install apex from https://www.github.com/nvidia/apex to run this example.\")\n\nmodel_names = sorted(name for name in models.__dict__\n if name.islower() and not name.startswith(\"__\")\n and callable(models.__dict__[name]))\n\nparser = argparse.ArgumentParser(description='PyTorch ImageNet Training')\nparser.add_argument('data', metavar='DIR',\n help='path to dataset')\nparser.add_argument('--arch', '-a', metavar='ARCH', default='resnet18',\n choices=model_names,\n help='model architecture: ' +\n ' | '.join(model_names) +\n ' (default: resnet18)')\nparser.add_argument('-j', '--workers', default=4, type=int, metavar='N',\n help='number of data loading workers (default: 4)')\nparser.add_argument('--epochs', default=90, type=int, metavar='N',\n help='number of total epochs to run')\nparser.add_argument('--start-epoch', default=0, type=int, metavar='N',\n help='manual epoch number (useful on restarts)')\nparser.add_argument('-b', '--batch-size', default=256, type=int,\n metavar='N', help='mini-batch size per process (default: 256)')\nparser.add_argument('--lr', '--learning-rate', default=0.1, type=float,\n metavar='LR', help='Initial learning rate. Will be scaled by /256: args.lr = args.lr*float(args.batch_size*args.world_size)/256. A warmup schedule will also be applied over the first 5 epochs.')\nparser.add_argument('--momentum', default=0.9, type=float, metavar='M',\n help='momentum')\nparser.add_argument('--weight-decay', '--wd', default=1e-4, type=float,\n metavar='W', help='weight decay (default: 1e-4)')\nparser.add_argument('--print-freq', '-p', default=10, type=int,\n metavar='N', help='print frequency (default: 10)')\nparser.add_argument('--resume', default='', type=str, metavar='PATH',\n help='path to latest checkpoint (default: none)')\nparser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true',\n help='evaluate model on validation set')\nparser.add_argument('--pretrained', dest='pretrained', action='store_true',\n help='use pre-trained model')\n\nparser.add_argument('--prof', dest='prof', action='store_true',\n help='Only run 10 iterations for profiling.')\nparser.add_argument('--deterministic', action='store_true')\n\nparser.add_argument(\"--local_rank\", default=0, type=int)\nparser.add_argument('--sync_bn', action='store_true',\n help='enabling apex sync BN.')\n\nparser.add_argument('--opt-level', type=str)\nparser.add_argument('--keep-batchnorm-fp32', type=str, default=None)\nparser.add_argument('--loss-scale', type=str, default=None)\n\ncudnn.benchmark = True\n\ndef fast_collate(batch):\n imgs = [img[0] for img in batch]\n targets = torch.tensor([target[1] for target in batch], dtype=torch.int64)\n w = imgs[0].size[0]\n h = imgs[0].size[1]\n tensor = torch.zeros( (len(imgs), 3, h, w), dtype=torch.uint8 )\n for i, img in enumerate(imgs):\n nump_array = np.asarray(img, dtype=np.uint8)\n if(nump_array.ndim < 3):\n nump_array = np.expand_dims(nump_array, axis=-1)\n nump_array = np.rollaxis(nump_array, 2)\n\n tensor[i] += torch.from_numpy(nump_array)\n \n return tensor, targets\n\nbest_prec1 = 0\nargs = parser.parse_args()\n\nprint(\"opt_level = {}\".format(args.opt_level))\nprint(\"keep_batchnorm_fp32 = {}\".format(args.keep_batchnorm_fp32), type(args.keep_batchnorm_fp32))\nprint(\"loss_scale = {}\".format(args.loss_scale), type(args.loss_scale))\n\nprint(\"\\nCUDNN VERSION: {}\\n\".format(torch.backends.cudnn.version()))\n\nif args.deterministic:\n cudnn.benchmark = False\n cudnn.deterministic = True\n torch.manual_seed(args.local_rank)\n torch.set_printoptions(precision=10)\n\ndef main():\n global best_prec1, args\n\n args.distributed = False\n if 'WORLD_SIZE' in os.environ:\n args.distributed = int(os.environ['WORLD_SIZE']) > 1\n\n args.gpu = 0\n args.world_size = 1\n\n if args.distributed:\n args.gpu = args.local_rank\n torch.cuda.set_device(args.gpu)\n torch.distributed.init_process_group(backend='nccl',\n init_method='env://')\n args.world_size = torch.distributed.get_world_size()\n\n assert torch.backends.cudnn.enabled, \"Amp requires cudnn backend to be enabled.\"\n\n # create model\n if args.pretrained:\n print(\"=> using pre-trained model '{}'\".format(args.arch))\n model = models.__dict__[args.arch](pretrained=True)\n else:\n print(\"=> creating model '{}'\".format(args.arch))\n model = models.__dict__[args.arch]()\n\n if args.sync_bn:\n import apex\n print(\"using apex synced BN\")\n model = apex.parallel.convert_syncbn_model(model)\n\n model = model.cuda()\n\n # Scale learning rate based on global batch size\n args.lr = args.lr*float(args.batch_size*args.world_size)/256. \n optimizer = torch.optim.SGD(model.parameters(), args.lr,\n momentum=args.momentum,\n weight_decay=args.weight_decay)\n\n # Initialize Amp. Amp accepts either values or strings for the optional override arguments,\n # for convenient interoperation with argparse.\n model, optimizer = amp.initialize(model, optimizer,\n opt_level=args.opt_level,\n keep_batchnorm_fp32=args.keep_batchnorm_fp32,\n loss_scale=args.loss_scale\n )\n\n # For distributed training, wrap the model with apex.parallel.DistributedDataParallel.\n # This must be done AFTER the call to amp.initialize. If model = DDP(model) is called\n # before model, ... = amp.initialize(model, ...), the call to amp.initialize may alter\n # the types of model's parameters in a way that disrupts or destroys DDP's allreduce hooks.\n if args.distributed:\n # By default, apex.parallel.DistributedDataParallel overlaps communication with \n # computation in the backward pass.\n # model = DDP(model)\n # delay_allreduce delays all communication to the end of the backward pass.\n model = DDP(model, delay_allreduce=True)\n\n # define loss function (criterion) and optimizer\n criterion = nn.CrossEntropyLoss().cuda()\n\n # Optionally resume from a checkpoint\n if args.resume:\n # Use a local scope to avoid dangling references\n def resume():\n if os.path.isfile(args.resume):\n print(\"=> loading checkpoint '{}'\".format(args.resume))\n checkpoint = torch.load(args.resume, map_location = lambda storage, loc: storage.cuda(args.gpu))\n args.start_epoch = checkpoint['epoch']\n best_prec1 = checkpoint['best_prec1']\n model.load_state_dict(checkpoint['state_dict'])\n optimizer.load_state_dict(checkpoint['optimizer'])\n print(\"=> loaded checkpoint '{}' (epoch {})\"\n .format(args.resume, checkpoint['epoch']))\n else:\n print(\"=> no checkpoint found at '{}'\".format(args.resume))\n resume()\n\n # Data loading code\n traindir = os.path.join(args.data, 'train')\n valdir = os.path.join(args.data, 'val')\n\n if(args.arch == \"inception_v3\"):\n raise RuntimeError(\"Currently, inception_v3 is not supported by this example.\")\n # crop_size = 299\n # val_size = 320 # I chose this value arbitrarily, we can adjust.\n else:\n crop_size = 224\n val_size = 256\n\n train_dataset = datasets.ImageFolder(\n traindir,\n transforms.Compose([\n transforms.RandomResizedCrop(crop_size),\n transforms.RandomHorizontalFlip(),\n # transforms.ToTensor(), Too slow\n # normalize,\n ]))\n val_dataset = datasets.ImageFolder(valdir, transforms.Compose([\n transforms.Resize(val_size),\n transforms.CenterCrop(crop_size),\n ]))\n\n train_sampler = None\n val_sampler = None\n if args.distributed:\n train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset)\n val_sampler = torch.utils.data.distributed.DistributedSampler(val_dataset)\n\n train_loader = torch.utils.data.DataLoader(\n train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None),\n num_workers=args.workers, pin_memory=True, sampler=train_sampler, collate_fn=fast_collate)\n\n val_loader = torch.utils.data.DataLoader(\n val_dataset,\n batch_size=args.batch_size, shuffle=False,\n num_workers=args.workers, pin_memory=True,\n sampler=val_sampler,\n collate_fn=fast_collate)\n\n if args.evaluate:\n validate(val_loader, model, criterion)\n return\n\n for epoch in range(args.start_epoch, args.epochs):\n if args.distributed:\n train_sampler.set_epoch(epoch)\n\n # train for one epoch\n train(train_loader, model, criterion, optimizer, epoch)\n if args.prof:\n break\n # evaluate on validation set\n prec1 = validate(val_loader, model, criterion)\n\n # remember best prec@1 and save checkpoint\n if args.local_rank == 0:\n is_best = prec1 > best_prec1\n best_prec1 = max(prec1, best_prec1)\n save_checkpoint({\n 'epoch': epoch + 1,\n 'arch': args.arch,\n 'state_dict': model.state_dict(),\n 'best_prec1': best_prec1,\n 'optimizer' : optimizer.state_dict(),\n }, is_best)\n\nclass data_prefetcher():\n def __init__(self, loader):\n self.loader = iter(loader)\n self.stream = torch.cuda.Stream()\n self.mean = torch.tensor([0.485 * 255, 0.456 * 255, 0.406 * 255]).cuda().view(1,3,1,1)\n self.std = torch.tensor([0.229 * 255, 0.224 * 255, 0.225 * 255]).cuda().view(1,3,1,1)\n # With Amp, it isn't necessary to manually convert data to half.\n # if args.fp16:\n # self.mean = self.mean.half()\n # self.std = self.std.half()\n self.preload()\n\n def preload(self):\n try:\n self.next_input, self.next_target = next(self.loader)\n except StopIteration:\n self.next_input = None\n self.next_target = None\n return\n # if record_stream() doesn't work, another option is to make sure device inputs are created\n # on the main stream.\n # self.next_input_gpu = torch.empty_like(self.next_input, device='cuda')\n # self.next_target_gpu = torch.empty_like(self.next_target, device='cuda')\n # Need to make sure the memory allocated for next_* is not still in use by the main stream\n # at the time we start copying to next_*:\n # self.stream.wait_stream(torch.cuda.current_stream())\n with torch.cuda.stream(self.stream):\n self.next_input = self.next_input.cuda(non_blocking=True)\n self.next_target = self.next_target.cuda(non_blocking=True)\n # more code for the alternative if record_stream() doesn't work:\n # copy_ will record the use of the pinned source tensor in this side stream.\n # self.next_input_gpu.copy_(self.next_input, non_blocking=True)\n # self.next_target_gpu.copy_(self.next_target, non_blocking=True)\n # self.next_input = self.next_input_gpu\n # self.next_target = self.next_target_gpu\n\n # With Amp, it isn't necessary to manually convert data to half.\n # if args.fp16:\n # self.next_input = self.next_input.half()\n # else:\n self.next_input = self.next_input.float()\n self.next_input = self.next_input.sub_(self.mean).div_(self.std)\n \n def next(self):\n torch.cuda.current_stream().wait_stream(self.stream)\n input = self.next_input\n target = self.next_target\n input.record_stream(torch.cuda.current_stream())\n target.record_stream(torch.cuda.current_stream())\n self.preload()\n return input, target\n\n\ndef train(train_loader, model, criterion, optimizer, epoch):\n batch_time = AverageMeter()\n losses = AverageMeter()\n top1 = AverageMeter()\n top5 = AverageMeter()\n\n # switch to train mode\n model.train()\n end = time.time()\n\n prefetcher = data_prefetcher(train_loader)\n input, target = prefetcher.next()\n i = 0\n while input is not None:\n i += 1\n\n adjust_learning_rate(optimizer, epoch, i, len(train_loader))\n\n if args.prof:\n if i > 10:\n break\n\n # compute output\n if args.prof: torch.cuda.nvtx.range_push(\"forward\")\n output = model(input)\n if args.prof: torch.cuda.nvtx.range_pop()\n loss = criterion(output, target)\n\n # compute gradient and do SGD step\n optimizer.zero_grad()\n\n if args.prof: torch.cuda.nvtx.range_push(\"backward\")\n with amp.scale_loss(loss, optimizer) as scaled_loss:\n scaled_loss.backward()\n if args.prof: torch.cuda.nvtx.range_pop()\n\n # for param in model.parameters():\n # print(param.data.double().sum().item(), param.grad.data.double().sum().item())\n\n if args.prof: torch.cuda.nvtx.range_push(\"step\")\n optimizer.step()\n if args.prof: torch.cuda.nvtx.range_pop()\n\n if i%args.print_freq == 0:\n # Every print_freq iterations, check the loss, accuracy, and speed.\n # For best performance, it doesn't make sense to print these metrics every\n # iteration, since they incur an allreduce and some host<->device syncs.\n\n # Measure accuracy\n prec1, prec5 = accuracy(output.data, target, topk=(1, 5))\n \n # Average loss and accuracy across processes for logging \n if args.distributed:\n reduced_loss = reduce_tensor(loss.data)\n prec1 = reduce_tensor(prec1)\n prec5 = reduce_tensor(prec5)\n else:\n reduced_loss = loss.data\n \n # to_python_float incurs a host<->device sync\n losses.update(to_python_float(reduced_loss), input.size(0))\n top1.update(to_python_float(prec1), input.size(0))\n top5.update(to_python_float(prec5), input.size(0))\n \n torch.cuda.synchronize()\n batch_time.update((time.time() - end)/args.print_freq)\n end = time.time()\n\n if args.local_rank == 0:\n print('Epoch: [{0}][{1}/{2}]\\t'\n 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\\t'\n 'Speed {3:.3f} ({4:.3f})\\t'\n 'Loss {loss.val:.10f} ({loss.avg:.4f})\\t'\n 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\\t'\n 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(\n epoch, i, len(train_loader),\n args.world_size*args.batch_size/batch_time.val,\n args.world_size*args.batch_size/batch_time.avg,\n batch_time=batch_time,\n loss=losses, top1=top1, top5=top5))\n\n input, target = prefetcher.next()\n\n\ndef validate(val_loader, model, criterion):\n batch_time = AverageMeter()\n losses = AverageMeter()\n top1 = AverageMeter()\n top5 = AverageMeter()\n\n # switch to evaluate mode\n model.eval()\n\n end = time.time()\n\n prefetcher = data_prefetcher(val_loader)\n input, target = prefetcher.next()\n i = 0\n while input is not None:\n i += 1\n\n # compute output\n with torch.no_grad():\n output = model(input)\n loss = criterion(output, target)\n\n # measure accuracy and record loss\n prec1, prec5 = accuracy(output.data, target, topk=(1, 5))\n\n if args.distributed:\n reduced_loss = reduce_tensor(loss.data)\n prec1 = reduce_tensor(prec1)\n prec5 = reduce_tensor(prec5)\n else:\n reduced_loss = loss.data\n\n losses.update(to_python_float(reduced_loss), input.size(0))\n top1.update(to_python_float(prec1), input.size(0))\n top5.update(to_python_float(prec5), input.size(0))\n\n # measure elapsed time\n batch_time.update(time.time() - end)\n end = time.time()\n\n # TODO: Change timings to mirror train().\n if args.local_rank == 0 and i % args.print_freq == 0:\n print('Test: [{0}/{1}]\\t'\n 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\\t'\n 'Speed {2:.3f} ({3:.3f})\\t'\n 'Loss {loss.val:.4f} ({loss.avg:.4f})\\t'\n 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\\t'\n 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(\n i, len(val_loader),\n args.world_size * args.batch_size / batch_time.val,\n args.world_size * args.batch_size / batch_time.avg,\n batch_time=batch_time, loss=losses,\n top1=top1, top5=top5))\n\n input, target = prefetcher.next()\n\n print(' * Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}'\n .format(top1=top1, top5=top5))\n\n return top1.avg\n\n\ndef save_checkpoint(state, is_best, filename='checkpoint.pth.tar'):\n torch.save(state, filename)\n if is_best:\n shutil.copyfile(filename, 'model_best.pth.tar')\n\n\nclass AverageMeter(object):\n \"\"\"Computes and stores the average and current value\"\"\"\n def __init__(self):\n self.reset()\n\n def reset(self):\n self.val = 0\n self.avg = 0\n self.sum = 0\n self.count = 0\n\n def update(self, val, n=1):\n self.val = val\n self.sum += val * n\n self.count += n\n self.avg = self.sum / self.count\n\n\ndef adjust_learning_rate(optimizer, epoch, step, len_epoch):\n \"\"\"LR schedule that should yield 76% converged accuracy with batch size 256\"\"\"\n factor = epoch // 30\n\n if epoch >= 80:\n factor = factor + 1\n\n lr = args.lr*(0.1**factor)\n\n \"\"\"Warmup\"\"\"\n if epoch < 5:\n lr = lr*float(1 + step + epoch*len_epoch)/(5.*len_epoch)\n\n # if(args.local_rank == 0):\n # print(\"epoch = {}, step = {}, lr = {}\".format(epoch, step, lr))\n\n for param_group in optimizer.param_groups:\n param_group['lr'] = lr\n\n\ndef accuracy(output, target, topk=(1,)):\n \"\"\"Computes the precision@k for the specified values of k\"\"\"\n maxk = max(topk)\n batch_size = target.size(0)\n\n _, pred = output.topk(maxk, 1, True, True)\n pred = pred.t()\n correct = pred.eq(target.view(1, -1).expand_as(pred))\n\n res = []\n for k in topk:\n correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)\n res.append(correct_k.mul_(100.0 / batch_size))\n return res\n\n\ndef reduce_tensor(tensor):\n rt = tensor.clone()\n dist.all_reduce(rt, op=dist.reduce_op.SUM)\n rt /= args.world_size\n return rt\n\nif __name__ == '__main__':\n main()\n" }, { "path": "apex/examples/simple/distributed/README.md", "content": "**distributed_data_parallel.py** and **run.sh** show an example using Amp with\n[apex.parallel.DistributedDataParallel](https://nvidia.github.io/apex/parallel.html) or\n[torch.nn.parallel.DistributedDataParallel](https://pytorch.org/docs/stable/nn.html#distributeddataparallel)\nand the Pytorch multiprocess launcher script,\n[torch.distributed.launch](https://pytorch.org/docs/master/distributed.html#launch-utility).\nThe use of `Amp` with DistributedDataParallel does not need to change from ordinary \nsingle-process use. The only gotcha is that wrapping your model with `DistributedDataParallel` must\ncome after the call to `amp.initialize`. Test via\n```bash\nbash run.sh\n```\n\n**This is intended purely as an instructional example, not a performance showcase.**\n" }, { "path": "apex/examples/simple/distributed/distributed_data_parallel.py", "content": "import torch\nimport argparse\nimport os\nfrom apex import amp\n# FOR DISTRIBUTED: (can also use torch.nn.parallel.DistributedDataParallel instead)\nfrom apex.parallel import DistributedDataParallel\n\nparser = argparse.ArgumentParser()\n# FOR DISTRIBUTED: Parse for the local_rank argument, which will be supplied\n# automatically by torch.distributed.launch.\nparser.add_argument(\"--local_rank\", default=0, type=int)\nargs = parser.parse_args()\n\n# FOR DISTRIBUTED: If we are running under torch.distributed.launch,\n# the 'WORLD_SIZE' environment variable will also be set automatically.\nargs.distributed = False\nif 'WORLD_SIZE' in os.environ:\n args.distributed = int(os.environ['WORLD_SIZE']) > 1\n\nif args.distributed:\n # FOR DISTRIBUTED: Set the device according to local_rank.\n torch.cuda.set_device(args.local_rank)\n\n # FOR DISTRIBUTED: Initialize the backend. torch.distributed.launch will provide\n # environment variables, and requires that you use init_method=`env://`.\n torch.distributed.init_process_group(backend='nccl',\n init_method='env://')\n\ntorch.backends.cudnn.benchmark = True\n\nN, D_in, D_out = 64, 1024, 16\n\n# Each process receives its own batch of \"fake input data\" and \"fake target data.\"\n# The \"training loop\" in each process just uses this fake batch over and over.\n# https://github.com/NVIDIA/apex/tree/master/examples/imagenet provides a more realistic\n# example of distributed data sampling for both training and validation.\nx = torch.randn(N, D_in, device='cuda')\ny = torch.randn(N, D_out, device='cuda')\n\nmodel = torch.nn.Linear(D_in, D_out).cuda()\noptimizer = torch.optim.SGD(model.parameters(), lr=1e-3)\n\nmodel, optimizer = amp.initialize(model, optimizer, opt_level=\"O1\")\n\nif args.distributed:\n # FOR DISTRIBUTED: After amp.initialize, wrap the model with\n # apex.parallel.DistributedDataParallel.\n model = DistributedDataParallel(model)\n # torch.nn.parallel.DistributedDataParallel is also fine, with some added args:\n # model = torch.nn.parallel.DistributedDataParallel(model,\n # device_ids=[args.local_rank],\n # output_device=args.local_rank)\n\nloss_fn = torch.nn.MSELoss()\n\nfor t in range(500):\n optimizer.zero_grad()\n y_pred = model(x)\n loss = loss_fn(y_pred, y)\n with amp.scale_loss(loss, optimizer) as scaled_loss:\n scaled_loss.backward()\n optimizer.step()\n\nif args.local_rank == 0:\n print(\"final loss = \", loss)\n" }, { "path": "apex/examples/simple/distributed/run.sh", "content": "#!/bin/bash\npython -m torch.distributed.launch --nproc_per_node=2 distributed_data_parallel.py\n" }, { "path": "apex/setup.py", "content": "import torch\nfrom setuptools import setup, find_packages\nimport subprocess\n\nimport sys\n\nif not torch.cuda.is_available():\n print(\"\\nWarning: Torch did not find available GPUs on this system.\\n\",\n \"If your intention is to cross-compile, this is not an error.\\n\")\n\nprint(\"torch.__version__ = \", torch.__version__)\nTORCH_MAJOR = int(torch.__version__.split('.')[0])\nTORCH_MINOR = int(torch.__version__.split('.')[1])\n\nif TORCH_MAJOR == 0 and TORCH_MINOR < 4:\n raise RuntimeError(\"Apex requires Pytorch 0.4 or newer.\\n\" +\n \"The latest stable release can be obtained from https://pytorch.org/\")\n\ncmdclass = {}\next_modules = []\n\nif \"--cpp_ext\" in sys.argv or \"--cuda_ext\" in sys.argv:\n if TORCH_MAJOR == 0:\n raise RuntimeError(\"--cpp_ext requires Pytorch 1.0 or later, \"\n \"found torch.__version__ = {}\".format(torch.__version__))\n from torch.utils.cpp_extension import BuildExtension\n cmdclass['build_ext'] = BuildExtension\n\nif \"--cpp_ext\" in sys.argv:\n from torch.utils.cpp_extension import CppExtension\n sys.argv.remove(\"--cpp_ext\")\n ext_modules.append(\n CppExtension('apex_C',\n ['csrc/flatten_unflatten.cpp',]))\n\ndef check_cuda_torch_binary_vs_bare_metal(cuda_dir):\n raw_output = subprocess.check_output([cuda_dir + \"/bin/nvcc\", \"-V\"], universal_newlines=True)\n output = raw_output.split()\n release_idx = output.index(\"release\") + 1\n release = output[release_idx].split(\".\")\n bare_metal_major = release[0]\n bare_metal_minor = release[1][0]\n torch_binary_major = torch.version.cuda.split(\".\")[0]\n torch_binary_minor = torch.version.cuda.split(\".\")[1]\n\n print(\"\\nCompiling cuda extensions with\")\n print(raw_output + \"from \" + cuda_dir + \"/bin\\n\")\n\n if (bare_metal_major != torch_binary_major) or (bare_metal_minor != torch_binary_minor):\n raise RuntimeError(\"Cuda extensions are being compiled with a version of Cuda that does \" +\n \"not match the version used to compile Pytorch binaries. \" +\n \"Pytorch binaries were compiled with Cuda {}.\\n\".format(torch.version.cuda) +\n \"In some cases, a minor-version mismatch will not cause later errors: \" +\n \"https://github.com/NVIDIA/apex/pull/323#discussion_r287021798. \"\n \"You can try commenting out this check (at your own risk).\")\n\nif \"--cuda_ext\" in sys.argv:\n from torch.utils.cpp_extension import CUDAExtension\n sys.argv.remove(\"--cuda_ext\")\n\n if torch.utils.cpp_extension.CUDA_HOME is None:\n raise RuntimeError(\"--cuda_ext was requested, but nvcc was not found. Are you sure your environment has nvcc available? If you're installing within a container from https://hub.docker.com/r/pytorch/pytorch, only images whose names contain 'devel' will provide nvcc.\")\n else:\n check_cuda_torch_binary_vs_bare_metal(torch.utils.cpp_extension.CUDA_HOME)\n\n # Set up macros for forward/backward compatibility hack around\n # https://github.com/pytorch/pytorch/commit/4404762d7dd955383acee92e6f06b48144a0742e\n version_ge_1_1 = []\n if (TORCH_MAJOR > 1) or (TORCH_MAJOR == 1 and TORCH_MINOR > 0):\n version_ge_1_1 = ['-DVERSION_GE_1_1']\n\n ext_modules.append(\n CUDAExtension(name='amp_C',\n sources=['csrc/amp_C_frontend.cpp',\n 'csrc/multi_tensor_scale_kernel.cu',\n 'csrc/multi_tensor_axpby_kernel.cu',\n 'csrc/multi_tensor_l2norm_kernel.cu',\n 'csrc/multi_tensor_lamb_stage_1.cu',\n 'csrc/multi_tensor_lamb_stage_2.cu'],\n extra_compile_args={'cxx': ['-O3'],\n 'nvcc':['-lineinfo',\n '-O3',\n # '--resource-usage',\n '--use_fast_math']}))\n ext_modules.append(\n CUDAExtension(name='fused_adam_cuda',\n sources=['csrc/fused_adam_cuda.cpp',\n 'csrc/fused_adam_cuda_kernel.cu'],\n extra_compile_args={'cxx': ['-O3',],\n 'nvcc':['-O3',\n '--use_fast_math']}))\n ext_modules.append(\n CUDAExtension(name='syncbn',\n sources=['csrc/syncbn.cpp',\n 'csrc/welford.cu']))\n ext_modules.append(\n CUDAExtension(name='fused_layer_norm_cuda',\n sources=['csrc/layer_norm_cuda.cpp',\n 'csrc/layer_norm_cuda_kernel.cu'],\n extra_compile_args={'cxx': ['-O3'] + version_ge_1_1,\n 'nvcc':['-maxrregcount=50',\n '-O3',\n '--use_fast_math'] + version_ge_1_1}))\n\nsetup(\n name='apex',\n version='0.1',\n packages=find_packages(exclude=('build',\n 'csrc',\n 'include',\n 'tests',\n 'dist',\n 'docs',\n 'tests',\n 'examples',\n 'apex.egg-info',)),\n description='PyTorch Extensions written by NVIDIA',\n ext_modules=ext_modules,\n cmdclass=cmdclass,\n)\n" }, { "path": "apex/tests/L0/run_amp/__init__.py", "content": "" }, { "path": "apex/tests/L0/run_amp/test_add_param_group.py", "content": "import unittest\n\nimport functools as ft\nimport itertools as it\n\nfrom apex import amp\nfrom apex.amp import _amp_state\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torch.nn import Parameter\n\nfrom utils import common_init, HALF, FLOAT,\\\n ALWAYS_HALF, ALWAYS_FLOAT, MATCH_INPUT\n\nclass MyModel(torch.nn.Module):\n def __init__(self, unique):\n super(MyModel, self).__init__()\n self.weight0 = Parameter(unique +\n torch.arange(2, device='cuda', dtype=torch.float32))\n self.weight1 = Parameter(1. + unique + torch.arange(2, device='cuda', dtype=torch.float16))\n\n @staticmethod\n def ops(input, weight0, weight1):\n return ((input*(weight0.float()))*(weight1.float())).sum()\n\n def forward(self, input):\n return self.ops(input, self.weight0, self.weight1)\n\n\n# Abandon all hope, ye who enter here.\n\n\nclass TestAddParamGroup(unittest.TestCase):\n def setUp(self):\n self.x = torch.ones((2), device='cuda', dtype=torch.float32)\n common_init(self)\n\n def tearDown(self):\n pass\n\n def zero_grad(self, models, optimizer, how_to_zero):\n if how_to_zero == \"none\":\n for model in models:\n for param in model.parameters():\n param.grad = None\n elif how_to_zero == \"model\":\n for model in models:\n model.zero_grad()\n elif how_to_zero == \"optimizer\":\n optimizer.zero_grad()\n\n def test_add_param_group(self):\n for opt_level in (\"O0\", \"O1\", \"O2\", \"O3\"):\n for zero_before_add in (True, False):\n for try_accumulation in (True, False):\n model0 = MyModel(1)\n model1 = MyModel(2)\n\n optimizer = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}],\n momentum=0.125)\n\n optimizer.zero_grad()\n loss = model0(self.x)\n loss.backward()\n optimizer.step()\n\n if zero_before_add:\n optimizer.zero_grad()\n optimizer.add_param_group({'params' : model1.parameters(), 'lr' : 0.5})\n if not zero_before_add:\n optimizer.zero_grad()\n\n loss = model0(self.x) + model1(self.x)\n loss.backward(retain_graph=try_accumulation)\n if try_accumulation:\n loss.backward()\n optimizer.step()\n\n # Once more to make sure the new params pick up momemtums properly\n optimizer.zero_grad()\n loss = model0(self.x) + model1(self.x)\n loss.backward(retain_graph=try_accumulation)\n if try_accumulation:\n loss.backward()\n optimizer.step()\n\n reference_params = [param.data.clone() for param in model0.parameters()] + \\\n [param.data.clone() for param in model1.parameters()]\n\n for how_to_zero in \"none\", \"model\", \"optimizer\":\n model0 = MyModel(1)\n model1 = MyModel(2)\n\n optimizer = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}],\n momentum=0.125)\n\n _amp_state.allow_incoming_model_not_fp32 = True\n [model0, model1], optimizer = amp.initialize([model0, model1],\n optimizer,\n opt_level=opt_level,\n verbosity=0,\n cast_model_type=False)\n _amp_state.allow_incoming_model_not_fp32 = False\n\n _amp_state.loss_scalers[0]._loss_scale = 4.0\n\n self.zero_grad([model0, model1], optimizer, how_to_zero)\n loss = model0(self.x)\n with amp.scale_loss(loss, optimizer) as scaled_loss:\n scaled_loss.backward()\n optimizer.step()\n\n if zero_before_add:\n self.zero_grad([model0, model1], optimizer, how_to_zero)\n optimizer.add_param_group({'params' : model1.parameters(), 'lr' : 0.5})\n if not zero_before_add:\n self.zero_grad([model0, model1], optimizer, how_to_zero)\n\n loss = model0(self.x) + model1(self.x)\n with amp.scale_loss(loss, optimizer) as scaled_loss:\n scaled_loss.backward(retain_graph=try_accumulation)\n if try_accumulation:\n with amp.scale_loss(loss, optimizer) as scaled_loss:\n scaled_loss.backward()\n optimizer.step()\n\n # Once more to make sure the new params pick up momentums properly\n self.zero_grad([model0, model1], optimizer, how_to_zero)\n loss = model0(self.x) + model1(self.x)\n with amp.scale_loss(loss, optimizer) as scaled_loss:\n scaled_loss.backward(retain_graph=try_accumulation)\n if try_accumulation:\n with amp.scale_loss(loss, optimizer) as scaled_loss:\n scaled_loss.backward()\n optimizer.step()\n\n final_params = [param.data.clone() for param in model0.parameters()] + \\\n [param.data.clone() for param in model1.parameters()]\n\n for reference, final in zip(reference_params, final_params):\n self.assertTrue(torch.allclose(reference.to(final.dtype), final),\n \"opt_level = {}, how_to_zero = {}, zero_before_add = {}\".format(\n opt_level, how_to_zero, zero_before_add))\n\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "apex/tests/L0/run_amp/test_basic_casts.py", "content": "import unittest\n\nimport functools as ft\nimport itertools as it\n\nfrom apex import amp\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\n\nfrom utils import common_init, HALF, FLOAT,\\\n ALWAYS_HALF, ALWAYS_FLOAT, MATCH_INPUT\n\ndef run_layer_test(test_case, fns, expected, input_shape, test_backward=True):\n for fn, typ in it.product(fns, expected.keys()):\n x = torch.randn(input_shape, dtype=typ).requires_grad_()\n y = fn(x)\n test_case.assertEqual(y.type(), expected[typ])\n if test_backward:\n y.float().sum().backward()\n test_case.assertEqual(x.grad.type(), MATCH_INPUT[typ])\n\nclass TestBasicCasts(unittest.TestCase):\n def setUp(self):\n self.handle = amp.init(enabled=True)\n common_init(self)\n\n def tearDown(self):\n self.handle._deactivate()\n\n def test_linear_is_half(self):\n m = nn.Linear(self.h, self.h)\n f = ft.partial(F.linear, weight=m.weight, bias=m.bias)\n run_layer_test(self, [m, f], ALWAYS_HALF, (self.b, self.h))\n\n def test_conv2d_is_half(self):\n m = nn.Conv2d(self.c, self.c, self.k)\n f = ft.partial(F.conv2d, weight=m.weight, bias=m.bias)\n run_layer_test(self, [m, f], ALWAYS_HALF, (self.b, self.c, self.h, self.h))\n\n def test_softmax_is_float(self):\n m = nn.Softmax(dim=1)\n f = ft.partial(F.softmax, dim=1)\n run_layer_test(self, [m, f], ALWAYS_FLOAT, (self.b, self.h))\n\n def test_group_norm_is_float(self):\n m = nn.GroupNorm(num_groups=4, num_channels=self.c)\n run_layer_test(self, [m], ALWAYS_FLOAT, (self.b, self.c, self.h, self.h))\n\n def test_mse_loss_is_float(self):\n shape = (self.b, self.h)\n target = torch.randn(shape)\n mod = nn.MSELoss()\n m = lambda x: mod(x, target)\n f = ft.partial(F.mse_loss, target=target)\n run_layer_test(self, [m], ALWAYS_FLOAT, shape)\n\n def test_relu_is_match(self):\n run_layer_test(self, [nn.ReLU(), F.relu], MATCH_INPUT, (self.b, self.h))\n\n def test_batch_norm_is_match(self):\n m = nn.BatchNorm2d(num_features=self.c)\n f = ft.partial(F.batch_norm, running_mean=m.running_mean, running_var=m.running_var,\n weight=m.weight, bias=m.bias, training=True)\n run_layer_test(self, [m], MATCH_INPUT, (self.b, self.c, self.h, self.h))\n\n # Test forward-only for BN inference\n m.eval()\n f = ft.partial(F.batch_norm, running_mean=m.running_mean, running_var=m.running_var,\n weight=m.weight, bias=m.bias, training=False)\n run_layer_test(self, [m, f], MATCH_INPUT, (self.b, self.c, self.h, self.h),\n test_backward=False)\n\nclass TestBannedMethods(unittest.TestCase):\n def setUp(self):\n self.handle = amp.init(enabled=True)\n common_init(self)\n\n def tearDown(self):\n self.handle._deactivate()\n\n def bce_common(self, assertion):\n shape = (self.b, self.h)\n target = torch.rand(shape)\n mod = nn.BCELoss()\n m = lambda x: mod(x, target)\n f = ft.partial(F.binary_cross_entropy, target=target)\n for fn in [m, f]:\n x = torch.rand(shape, dtype=torch.half)\n assertion(fn, x)\n\n def test_bce_raises_by_default(self):\n assertion = lambda fn, x: self.assertRaises(NotImplementedError, fn, x)\n self.bce_common(assertion)\n\n def test_bce_is_float_with_allow_banned(self):\n self.handle._deactivate()\n self.handle = amp.init(enabled=True, allow_banned=True)\n assertion = lambda fn, x: self.assertEqual(fn(x).type(), FLOAT)\n self.bce_common(assertion)\n\nclass TestTensorCasts(unittest.TestCase):\n def setUp(self):\n self.handle = amp.init(enabled=True)\n common_init(self)\n\n def tearDown(self):\n self.handle._deactivate()\n\n def test_matmul_method_is_half(self):\n other = torch.randn(self.h, self.h)\n lhs = lambda x: x.matmul(other)\n rhs = lambda x: other.matmul(x)\n run_layer_test(self, [lhs, rhs], ALWAYS_HALF, (self.h, self.h))\n\n def test_matmul_op_is_half(self):\n other = torch.randn(self.h, self.h)\n lhs = lambda x: x @ other\n rhs = lambda x: other @ x\n run_layer_test(self, [lhs, rhs], ALWAYS_HALF, (self.h, self.h))\n\n def test_pow_method_is_float(self):\n fn = lambda x: x.pow(2.)\n run_layer_test(self, [fn], ALWAYS_FLOAT, (self.b, self.h))\n\n def test_pow_op_is_float(self):\n fn = lambda x: x ** 2.\n run_layer_test(self, [fn], ALWAYS_FLOAT, (self.b, self.h))\n\n def test_cpu_is_float(self):\n fn = lambda x: x.cpu()\n always_cpu_float = {torch.float: 'torch.FloatTensor',\n torch.half: 'torch.FloatTensor'}\n run_layer_test(self, [fn], always_cpu_float, (self.b, self.h))\n\n def test_sum_is_float(self):\n fn = lambda x: x.sum()\n run_layer_test(self, [fn], ALWAYS_FLOAT, (self.b, self.h))\n\nclass TestDisabledCasts(unittest.TestCase):\n def setUp(self):\n self.handle = amp.init(enabled=False)\n common_init(self)\n\n def test_disabled_linear(self):\n m = nn.Linear(self.h, self.h)\n f = ft.partial(F.linear, weight=m.weight, bias=m.bias)\n input_shape = (self.b, self.h)\n\n for fn in [m, f]:\n x = torch.randn(input_shape, dtype=torch.float).requires_grad_()\n y = fn(x)\n self.assertEqual(y.type(), FLOAT)\n y.sum().backward()\n self.assertEqual(x.grad.type(), FLOAT)\n\n x = torch.randn(input_shape, dtype=torch.half).requires_grad_()\n self.assertRaises(RuntimeError, fn, x)\n\n # TODO: maybe more tests on disabled casting?\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "apex/tests/L0/run_amp/test_cache.py", "content": "import unittest\n\nimport functools as ft\nimport itertools as it\n\nfrom apex import amp\nfrom apex.amp import _amp_state\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\n\nfrom utils import common_init, HALF, FLOAT,\\\n ALWAYS_HALF, ALWAYS_FLOAT, MATCH_INPUT\n\ndef get_reference_grad(i, w, ops):\n # Creating new tensors ensures, among other things, that the new tensors are not in the cache.\n # In fact, they are guaranteed not to use the cache because they are not torch.nn.Parameters.\n fp32_i = i.detach().clone().float()\n fp32_w = w.detach().clone().float().requires_grad_()\n loss = ops(fp32_i, fp32_w)\n loss.backward()\n return fp32_w.grad\n\nclass WhitelistModule(torch.nn.Module):\n def __init__(self, dtype):\n super(WhitelistModule, self).__init__()\n self.weight = torch.nn.Parameter(torch.arange(8*8, device='cuda', dtype=dtype).view(8,8))\n\n @staticmethod\n def ops(input, weight):\n return (input.mm(weight)).mm(weight).sum()\n\n def forward(self, input):\n return self.ops(input, self.weight)\n\n\nclass BlacklistModule(torch.nn.Module):\n def __init__(self, dtype):\n super(BlacklistModule, self).__init__()\n self.weight = torch.nn.Parameter(torch.arange(2*8, device='cuda', dtype=dtype).view(2,8))\n\n @staticmethod\n def ops(input, weight):\n return (input + torch.pow(weight, 2) + torch.pow(weight, 2)).sum()\n\n def forward(self, input):\n return self.ops(input, self.weight)\n\n\nclass PromoteModule(torch.nn.Module):\n def __init__(self, dtype):\n super(PromoteModule, self).__init__()\n self.weight = torch.nn.Parameter(torch.arange(2*8, device='cuda', dtype=dtype).view(2,8))\n\n @staticmethod\n def ops(input, weight):\n return ((input*weight)*weight).sum()\n\n def forward(self, input):\n return self.ops(input, self.weight)\n\nclass TestCache(unittest.TestCase):\n def setUp(self):\n self.x = torch.ones((2, 8), device='cuda', dtype=torch.float32)\n common_init(self)\n\n def tearDown(self):\n pass\n\n def train_eval_train_test(self, module, t):\n model = module(t).cuda()\n optimizer = torch.optim.SGD(model.parameters(), lr=1.0)\n\n _amp_state.allow_incoming_model_not_fp32 = True\n model, optimizer = amp.initialize(model, optimizer, opt_level=\"O1\", verbosity=0)\n _amp_state.allow_incoming_model_not_fp32 = False\n \n def training_step():\n for param in model.parameters():\n param.grad = None\n \n loss = model(self.x).sum()\n _amp_state.loss_scalers[0]._loss_scale = 4.0\n with amp.scale_loss(loss, optimizer) as scaled_loss:\n scaled_loss.backward()\n \n self.assertEqual(len([p.grad for p in model.parameters() if p.grad is not None]), 1)\n self.assertEqual(model.weight.grad.type(), model.weight.type())\n \n reference_grad = get_reference_grad(self.x, model.weight, model.ops)\n \n # Currently there's no difference in the allclose calls, so no need for branching,\n # but I'm keeping this in case we want different tolerances for fp16 and fp32 checks. \n if model.weight.grad.type() == \"torch.cuda.HalfTensor\":\n self.assertTrue(torch.allclose(model.weight.grad.float(), reference_grad))\n elif model.weight.grad.type() == \"torch.cuda.FloatTensor\":\n self.assertTrue(torch.allclose(model.weight.grad.float(), reference_grad))\n else:\n raise RuntimeError(\"model.weight.grad.type = {}\".format(model.weight.grad.type()))\n\n model.weight.data -= 1.\n \n # Simulates first epoch\n training_step()\n \n # Simulates eval\n with torch.no_grad():\n loss = model(self.x).sum()\n \n # Simulates resuming training after eval\n training_step()\n\n _amp_state.handle._deactivate()\n \n # I could easily have these as a set of for loops in a single test,\n # instead of going for granularity.\n def test_whitelist_module_fp16_weight(self):\n self.train_eval_train_test(WhitelistModule, torch.float16)\n\n def test_whitelist_module_fp32_weight(self):\n self.train_eval_train_test(WhitelistModule, torch.float32)\n\n def test_blacklist_module_fp16_weight(self):\n self.train_eval_train_test(BlacklistModule, torch.float16)\n\n def test_blacklist_module_fp32_weight(self):\n self.train_eval_train_test(BlacklistModule, torch.float32)\n\n def test_promote_module_fp16_weight(self):\n self.train_eval_train_test(PromoteModule, torch.float16)\n\n def test_promote_module_fp32_weight(self):\n self.train_eval_train_test(PromoteModule, torch.float32)\n\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "apex/tests/L0/run_amp/test_multi_tensor_axpby.py", "content": "import unittest\n\nimport functools as ft\nimport itertools as it\n\nfrom apex import amp\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\n\nfrom utils import common_init, HALF, FLOAT,\\\n ALWAYS_HALF, ALWAYS_FLOAT, MATCH_INPUT\n\ntry:\n import amp_C\n from amp_C import multi_tensor_axpby\n from apex.multi_tensor_apply import MultiTensorApply\n disabled = False\nexcept ImportError as err:\n print(\"amp_C fused kernels unavailable, disabling TestMultiTensorApply. ImportError was \", err)\n disabled = True\n\n\nclass TestMultiTensorAxpby(unittest.TestCase):\n\n def setUp(self):\n common_init(self)\n\n self.a = 2.0\n self.b = 8.0\n self.xval = 4.0\n self.yval = 16.0\n self.overflow_buf = torch.cuda.IntTensor(1).zero_()\n self.ref = torch.cuda.FloatTensor([136.0])\n\n def tearDown(self):\n pass\n\n # The tensor creation here is written for convenience, not speed.\n def axpby(self, sizea, sizeb, applier, repeat_tensors,\n x_type, y_type, out_type, inplace=False):\n self.overflow_buf.zero_()\n t1 = torch.cuda.FloatTensor(sizea).fill_(1.0)\n t2 = torch.cuda.FloatTensor(sizeb).fill_(1.0)\n\n y_list = []\n for i in range(repeat_tensors):\n y_list += [t1.clone().to(y_type)*self.yval, t2.clone().to(y_type)*self.yval]\n\n x_list = [x.clone().to(x_type)*(self.xval/self.yval) for x in y_list]\n\n if inplace:\n out_list = y_list\n else:\n out_list = [out.clone().to(out_type)*3.0 for out in y_list]\n\n applier(multi_tensor_axpby, self.overflow_buf, [x_list, y_list, out_list], self.a, self.b, -1)\n\n self.assertTrue(all([torch.allclose(out, self.ref.to(out_type)) for out in out_list]),\n msg=\"{} {} {} {} {} {} {}\".format(sizea, sizeb, repeat_tensors,\n x_type, y_type, out_type, inplace))\n self.assertTrue(self.overflow_buf.item() == 0,\n msg=\"{} {} {} {} {} {} {}\".format(sizea, sizeb, repeat_tensors,\n x_type, y_type, out_type, inplace))\n\n # def find_inf(self, sizea, sizeb, applier, repeat_tensors, in_type, out_type, t, ind, val, inplace=False):\n # self.overflow_buf.zero_()\n # a = torch.cuda.FloatTensor(sizea).fill_(self.scale)\n # b = torch.cuda.FloatTensor(sizeb).fill_(self.scale)\n\n # out_list = []\n # for i in range(repeat_tensors):\n # out_list += [a.clone().to(out_type), b.clone().to(out_type)]\n\n # if inplace:\n # in_list = out_list\n # else:\n # in_list = [out.clone().to(in_type) for out in out_list]\n\n # applier(multi_tensor_scale, self.overflow_buf, [in_list, out_list], 1./self.scale)\n\n # self.overflow_buf.zero_()\n # in_list[t][ind] = val\n # applier(multi_tensor_scale, self.overflow_buf, [in_list, out_list], 1./self.scale)\n # self.assertTrue(self.overflow_buf.item())\n\n @unittest.skipIf(disabled, \"amp_C is unavailable\")\n def test_fuzz(self):\n input_size_pairs = (\n (7777*77, 555*555),\n (777, 555),\n (555, 2048*32+1),\n (2048*32+1, 555),\n (555, 2048*32),\n (2048*32, 555),\n (33333, 555),\n (555, 33333))\n appliers = (\n MultiTensorApply(2048*32),\n MultiTensorApply(333),\n MultiTensorApply(33333))\n repeat_tensors = (\n 1,\n 55)\n\n for sizea, sizeb in input_size_pairs:\n for applier in appliers:\n for repeat in repeat_tensors:\n for x_type in (torch.float32, torch.float16):\n for y_type in (torch.float32, torch.float16):\n for out_type in (torch.float32, torch.float16):\n for inplace in (True, False):\n if inplace is True and (y_type is not out_type):\n continue\n else:\n self.axpby(sizea, sizeb, applier, repeat,\n x_type, y_type, out_type, inplace=inplace)\n # self.find_inf(sizea, sizeb, applier, repeat, in_type, out_type,\n # 0, 0, float('nan'), inplace=inplace)\n # self.find_inf(sizea, sizeb, applier, repeat, in_type, out_type,\n # 2*repeat-1, sizeb-1, float('inf'), inplace=inplace)\n # self.find_inf(sizea, sizeb, applier, repeat, in_type, out_type,\n # 2*(repeat//2), sizea//2, float('inf'), inplace=inplace)\n\n\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "apex/tests/L0/run_amp/test_multi_tensor_l2norm.py", "content": "import unittest\n\nimport functools as ft\nimport itertools as it\n\nfrom apex import amp\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\n\nfrom utils import common_init, HALF, FLOAT,\\\n ALWAYS_HALF, ALWAYS_FLOAT, MATCH_INPUT\n\ntry:\n import amp_C\n from amp_C import multi_tensor_l2norm\n from apex.multi_tensor_apply import MultiTensorApply\n disabled = False\nexcept ImportError as err:\n print(\"amp_C fused kernels unavailable, disabling TestMultiTensorApply. ImportError was \", err)\n disabled = True\n\n\nclass TestMultiTensorL2Norm(unittest.TestCase):\n\n def setUp(self):\n common_init(self)\n self.val = 4.0\n self.overflow_buf = torch.cuda.IntTensor(1).zero_()\n\n def tearDown(self):\n pass\n\n # The tensor creation here is written for convenience, not speed.\n def l2norm(self, sizea, sizeb, applier, repeat_tensors, in_type, per_tensor):\n self.overflow_buf.zero_()\n a = torch.cuda.FloatTensor(sizea).fill_(self.val)\n b = torch.cuda.FloatTensor(sizeb).fill_(self.val)\n\n in_list = []\n for i in range(repeat_tensors):\n in_list += [a.clone().to(in_type), b.clone().to(in_type)]\n\n if per_tensor:\n norm, norm_per_tensor = applier(multi_tensor_l2norm, self.overflow_buf, [in_list], True)\n normab = torch.cat((a.norm().view(1), b.norm().view(1)))\n norm_per_tensor = norm_per_tensor.view(-1, 2)\n else:\n norm, _ = applier(multi_tensor_l2norm, self.overflow_buf, [in_list], True)\n\n reference = torch.cuda.FloatTensor((sizea + sizeb)*repeat_tensors).fill_(self.val).norm()\n\n self.assertTrue(torch.allclose(norm, reference))\n if per_tensor:\n self.assertTrue(torch.allclose(norm_per_tensor, normab))\n self.assertTrue(self.overflow_buf.item() == 0)\n\n @unittest.skipIf(disabled, \"amp_C is unavailable\")\n def test_fuzz(self):\n input_size_pairs = (\n (7777*77, 555*555),\n (777, 555),\n (555, 2048*32+1),\n (2048*32+1, 555),\n (555, 2048*32),\n (2048*32, 555),\n (33333, 555),\n (555, 33333))\n appliers = (\n MultiTensorApply(2048*32), \n MultiTensorApply(333),\n MultiTensorApply(33333))\n repeat_tensors = (\n 1,\n 55)\n\n for sizea, sizeb in input_size_pairs:\n for applier in appliers:\n for repeat in repeat_tensors:\n for in_type in (torch.float32, torch.float16):\n for per_tensor in (False, True):\n self.l2norm(sizea, sizeb, applier, repeat, in_type, per_tensor)\n\n\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "apex/tests/L0/run_amp/test_multi_tensor_scale.py", "content": "import unittest\n\nimport functools as ft\nimport itertools as it\n\nfrom apex import amp\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\n\nfrom utils import common_init, HALF, FLOAT,\\\n ALWAYS_HALF, ALWAYS_FLOAT, MATCH_INPUT\n\ntry:\n import amp_C\n from amp_C import multi_tensor_scale \n from apex.multi_tensor_apply import MultiTensorApply\n disabled = False\nexcept ImportError as err:\n print(\"amp_C fused kernels unavailable, disabling TestMultiTensorApply. ImportError was \", err)\n disabled = True\n\n\nclass TestMultiTensorScale(unittest.TestCase):\n\n def setUp(self):\n common_init(self)\n self.scale = 4.0\n self.overflow_buf = torch.cuda.IntTensor(1).zero_()\n self.ref = torch.cuda.FloatTensor([1.0])\n\n def tearDown(self):\n pass\n\n # The tensor creation here is written for convenience, not speed.\n def downscale(self, sizea, sizeb, applier, repeat_tensors, in_type, out_type, inplace=False):\n self.overflow_buf.zero_()\n a = torch.cuda.FloatTensor(sizea).fill_(self.scale)\n b = torch.cuda.FloatTensor(sizeb).fill_(self.scale)\n\n out_list = []\n for i in range(repeat_tensors):\n out_list += [a.clone().to(out_type), b.clone().to(out_type)]\n\n if inplace:\n in_list = out_list\n else:\n in_list = [out.clone().to(in_type) for out in out_list]\n\n applier(multi_tensor_scale, self.overflow_buf, [in_list, out_list], 1./self.scale)\n\n self.assertTrue(all([torch.allclose(out, self.ref.to(out_type)) for out in out_list]))\n self.assertTrue(self.overflow_buf.item() == 0)\n \n def find_inf(self, sizea, sizeb, applier, repeat_tensors, in_type, out_type, t, ind, val, inplace=False):\n self.overflow_buf.zero_()\n a = torch.cuda.FloatTensor(sizea).fill_(self.scale)\n b = torch.cuda.FloatTensor(sizeb).fill_(self.scale)\n\n out_list = []\n for i in range(repeat_tensors):\n out_list += [a.clone().to(out_type), b.clone().to(out_type)]\n\n if inplace:\n in_list = out_list\n else:\n in_list = [out.clone().to(in_type) for out in out_list]\n\n applier(multi_tensor_scale, self.overflow_buf, [in_list, out_list], 1./self.scale)\n\n self.overflow_buf.zero_()\n in_list[t][ind] = val\n applier(multi_tensor_scale, self.overflow_buf, [in_list, out_list], 1./self.scale)\n self.assertTrue(self.overflow_buf.item())\n\n # Currently, the fused kernel gives a hard error if you attempt to downscale\n # into fp16 output, which imo is the desired behavior. Maybe someday we\n # will learn otherwise.\n # @unittest.skipIf(disabled, \"amp_C is unavailable\")\n # def test_fp16_to_fp16(self):\n # self.downscale(self.fp16, self.fp16, self.fp16_ref)\n # \n # @unittest.skipIf(disabled, \"amp_C is unavailable\")\n # def test_fp32_to_fp16(self):\n # self.downscale(self.fp32, self.fp16, self.fp16_ref)\n\n @unittest.skipIf(disabled, \"amp_C is unavailable\")\n def test_fuzz(self):\n input_size_pairs = (\n (7777*77, 555*555),\n (777, 555),\n (555, 2048*32+1),\n (2048*32+1, 555),\n (555, 2048*32),\n (2048*32, 555),\n (33333, 555),\n (555, 33333))\n appliers = (\n MultiTensorApply(2048*32), \n MultiTensorApply(333),\n MultiTensorApply(33333))\n repeat_tensors = (\n 1,\n 55)\n\n for sizea, sizeb in input_size_pairs:\n for applier in appliers:\n for repeat in repeat_tensors:\n for in_type in (torch.float32, torch.float16):\n for out_type in (torch.float32, torch.float16):\n for inplace in (True, False):\n if inplace is True and (out_type is not in_type):\n continue\n else:\n self.downscale(sizea, sizeb, applier, repeat, in_type, out_type, inplace=inplace)\n self.find_inf(sizea, sizeb, applier, repeat, in_type, out_type,\n 0, 0, float('nan'), inplace=inplace)\n self.find_inf(sizea, sizeb, applier, repeat, in_type, out_type,\n 2*repeat-1, sizeb-1, float('inf'), inplace=inplace)\n self.find_inf(sizea, sizeb, applier, repeat, in_type, out_type,\n 2*(repeat//2), sizea//2, float('inf'), inplace=inplace)\n\n\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "apex/tests/L0/run_amp/test_multiple_models_optimizers_losses.py", "content": "import unittest\n\nimport functools as ft\nimport itertools as it\n\nfrom apex import amp\nfrom apex.amp import _amp_state\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torch.nn import Parameter\n\nfrom utils import common_init, HALF, FLOAT,\\\n ALWAYS_HALF, ALWAYS_FLOAT, MATCH_INPUT\n\nclass MyModel(torch.nn.Module):\n def __init__(self, unique):\n super(MyModel, self).__init__()\n self.weight0 = Parameter(unique +\n torch.arange(2, device='cuda', dtype=torch.float32))\n self.weight1 = Parameter(1. + unique + torch.arange(2, device='cuda', dtype=torch.float16))\n\n @staticmethod\n def ops(input, weight0, weight1):\n return ((input*(weight0.float()))*(weight1.float())).sum()\n\n def forward(self, input):\n return self.ops(input, self.weight0, self.weight1)\n\n# Abandon all hope, ye who enter here.\n\n# This is hands down the ugliest code I have ever written, but it succeeds in testing\n# multiple models/optimizers/losses fairly thoroughly. Many of the different test cases\n# require slightly divergent code in a way that seems near-impossible to genericize into a simple\n# cross product or nested loops.\n\nclass TestMultipleModelsOptimizersLosses(unittest.TestCase):\n def setUp(self):\n self.x = torch.ones((2), device='cuda', dtype=torch.float32)\n common_init(self)\n\n def tearDown(self):\n pass\n\n def test_2models2losses1optimizer(self):\n model0 = MyModel(1)\n model1 = MyModel(2)\n\n optimizer = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25},\n {'params' : model1.parameters(), 'lr' : 0.5}],\n momentum=0.125)\n\n reference_grads = []\n for i in range(2):\n optimizer.zero_grad()\n loss0 = model0(self.x)\n loss1 = model1(self.x)\n loss0.backward()\n loss1.backward()\n\n reference_grads.append([param.grad.data.clone() for param in model0.parameters()] +\n [param.grad.data.clone() for param in model1.parameters()])\n\n optimizer.step()\n\n final_params = [param.data.clone() for param in model0.parameters()] + \\\n [param.data.clone() for param in model1.parameters()]\n\n for opt_level in (\"O0\", \"O1\", \"O2\", \"O3\"):\n for how_to_zero in (\"none\", \"model\", \"optimizer\"):\n for use_multiple_loss_scalers in (True, False):\n if opt_level == \"O1\" or opt_level == \"O2\":\n inject_inf_iters = (-1, 0, 1)\n else:\n inject_inf_iters = (-1,)\n\n for inject_inf in inject_inf_iters:\n if inject_inf >= 0:\n inject_inf_locs = (\"fp16\", \"fp32\")\n which_backwards = (0, 1)\n else:\n inject_inf_locs = (\"fdsa\",)\n which_backwards = (None,)\n\n for inject_inf_loc in inject_inf_locs:\n for which_backward in which_backwards:\n if use_multiple_loss_scalers:\n num_losses = 2\n loss_ids = [0, 1]\n else:\n num_losses = 1\n loss_ids = [0, 0]\n\n if inject_inf >= 0:\n iters = 3\n else:\n iters = 2\n\n model0 = MyModel(1)\n model1 = MyModel(2)\n\n models = [model0, model1]\n\n optimizer = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25},\n {'params' : model1.parameters(), 'lr' : 0.5}],\n momentum=0.125)\n\n _amp_state.allow_incoming_model_not_fp32 = True\n [model0, model1], optimizer = amp.initialize(\n [model0, model1],\n optimizer,\n opt_level=opt_level,\n verbosity=0,\n cast_model_type=False,\n num_losses=num_losses)\n _amp_state.allow_incoming_model_not_fp32 = False\n\n _amp_state.loss_scalers[0]._loss_scale = 4.0\n if use_multiple_loss_scalers:\n _amp_state.loss_scalers[1]._loss_scale = 16.0\n\n unskipped = 0\n for i in range(iters):\n if how_to_zero == \"none\":\n for model in models:\n for param in model.parameters():\n param.grad = None\n elif how_to_zero == \"model\":\n for model in models:\n model.zero_grad()\n else:\n optimizer.zero_grad()\n\n loss0 = model0(self.x)\n loss1 = model1(self.x)\n\n with amp.scale_loss(loss0, optimizer, loss_id=loss_ids[0]) as scaled_loss:\n scaled_loss.backward()\n if i == inject_inf and which_backward == 0:\n if inject_inf_loc == \"fp32\":\n model0.weight0.grad[0] = float('inf')\n elif inject_inf_loc == \"fp16\":\n model0.weight1.grad[0] = float('inf')\n with amp.scale_loss(loss1, optimizer, loss_id=loss_ids[1]) as scaled_loss:\n scaled_loss.backward()\n if i == inject_inf and which_backward == 1:\n if inject_inf_loc == \"fp32\":\n model1.weight0.grad[0] = float('inf')\n elif inject_inf_loc == \"fp16\":\n model1.weight1.grad[0] = float('inf')\n\n if i != inject_inf:\n for param, reference_grad in zip(amp.master_params(optimizer),\n reference_grads[unskipped]):\n self.assertTrue(torch.allclose(param.grad.float(), reference_grad.float()))\n unskipped += 1\n optimizer.step()\n\n model_params = [p for p in model0.parameters()] + [p for p in model1.parameters()]\n for model, master, reference in zip(\n model_params,\n amp.master_params(optimizer),\n final_params):\n self.assertTrue(torch.allclose(model, reference))\n self.assertTrue(torch.allclose(model, master.to(model.dtype)))\n\n if opt_level == \"O1\":\n _amp_state.handle._deactivate()\n\n def test_3models2losses1optimizer(self):\n\n model0 = MyModel(1)\n model1 = MyModel(2)\n model2 = MyModel(3)\n\n optimizer = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25},\n {'params' : model1.parameters(), 'lr' : 0.5},\n {'params' : model2.parameters(), 'lr' : 0.125}],\n momentum=0.125)\n\n reference_grads = []\n for i in range(2):\n optimizer.zero_grad()\n loss0 = model0(self.x) + model2(self.x)\n loss1 = model1(self.x) + model2(self.x)\n loss0.backward()\n loss1.backward()\n\n reference_grads.append([param.grad.data.clone() for param in model0.parameters()] +\n [param.grad.data.clone() for param in model1.parameters()] +\n [param.grad.data.clone() for param in model2.parameters()])\n\n optimizer.step()\n\n\n final_params = [param.data.clone() for param in model0.parameters()] + \\\n [param.data.clone() for param in model1.parameters()] + \\\n [param.data.clone() for param in model2.parameters()]\n\n for opt_level in (\"O0\", \"O1\", \"O2\", \"O3\"):\n for how_to_zero in (\"none\", \"model\", \"optimizer\"):\n for use_multiple_loss_scalers in (True, False):\n if opt_level == \"O1\" or opt_level == \"O2\":\n inject_inf_iters = (-1, 0, 1)\n else:\n inject_inf_iters = (-1,)\n\n for inject_inf in inject_inf_iters:\n if inject_inf >= 0:\n inject_inf_locs = (\"fp16\", \"fp32\")\n which_backwards = (0, 1)\n else:\n inject_inf_locs = (\"fdsa\",)\n which_backwards = (None,)\n\n for inject_inf_loc in inject_inf_locs:\n for which_backward in which_backwards:\n if use_multiple_loss_scalers:\n num_losses = 2\n loss_ids = [0, 1]\n else:\n num_losses = 1\n loss_ids = [0, 0]\n\n if inject_inf >= 0:\n iters = 3\n if which_backward == 0:\n which_models = (0, 2)\n elif which_backward == 1:\n which_models = (1, 2)\n else:\n iters = 2\n which_models = (None,)\n\n for which_model in which_models:\n model0 = MyModel(1)\n model1 = MyModel(2)\n model2 = MyModel(3)\n\n models = [model0, model1, model2]\n\n optimizer = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25},\n {'params' : model1.parameters(), 'lr' : 0.5},\n {'params' : model2.parameters(), 'lr' : 0.125}],\n momentum=0.125)\n\n _amp_state.allow_incoming_model_not_fp32 = True\n [model0, model1, model2], optimizer = amp.initialize(\n [model0, model1, model2],\n optimizer,\n opt_level=opt_level,\n verbosity=0,\n cast_model_type=False,\n num_losses=num_losses)\n _amp_state.allow_incoming_model_not_fp32 = False\n\n _amp_state.loss_scalers[0]._loss_scale = 4.0\n if use_multiple_loss_scalers:\n _amp_state.loss_scalers[1]._loss_scale = 16.0\n\n unskipped = 0\n for i in range(iters):\n if how_to_zero == \"none\":\n for model in models:\n for param in model.parameters():\n param.grad = None\n elif how_to_zero == \"model\":\n for model in models:\n model.zero_grad()\n else:\n optimizer.zero_grad()\n\n # print(\"opt_level {} i {} inject_inf {} which_backward {} inject_inf_loc {} which_model {} use_multiple_loss_scalers {}\".format(opt_level, i, inject_inf, which_backward, inject_inf_loc, which_model, use_multiple_loss_scalers))\n\n loss0 = model0(self.x) + model2(self.x)\n loss1 = model1(self.x) + model2(self.x)\n\n with amp.scale_loss(loss0, optimizer, loss_id=loss_ids[0]) as scaled_loss:\n scaled_loss.backward()\n if i == inject_inf and which_backward == 0:\n if which_model == 0:\n inj_model = model0\n elif which_model == 2:\n inj_model = model2\n else:\n raise RuntimeError(which_model + \" invalid for loss 0\")\n if inject_inf_loc == \"fp32\":\n inj_model.weight0.grad[0] = float('inf')\n elif inject_inf_loc == \"fp16\":\n inj_model.weight1.grad[0] = float('inf')\n with amp.scale_loss(loss1, optimizer, loss_id=loss_ids[1]) as scaled_loss:\n scaled_loss.backward()\n if i == inject_inf and which_backward == 1:\n if which_model == 1:\n inj_model = model1\n elif which_model == 2:\n inj_model = model2\n else:\n raise RuntimeError(which_model + \" invalid for loss 1 \")\n if inject_inf_loc == \"fp32\":\n inj_model.weight0.grad[0] = float('inf')\n elif inject_inf_loc == \"fp16\":\n inj_model.weight1.grad[0] = float('inf')\n\n if i != inject_inf:\n for param, reference_grad in zip(amp.master_params(optimizer),\n reference_grads[unskipped]):\n self.assertTrue(torch.allclose(param.grad.float(), reference_grad.float()))\n unskipped += 1\n\n optimizer.step()\n\n model_params = [p for p in model0.parameters()] + \\\n [p for p in model1.parameters()] + \\\n [p for p in model2.parameters()]\n for model, master, reference in zip(\n model_params,\n amp.master_params(optimizer),\n final_params):\n self.assertTrue(torch.allclose(model, reference))\n self.assertTrue(torch.allclose(model, master.to(model.dtype)))\n\n if opt_level == \"O1\":\n _amp_state.handle._deactivate()\n\n def test_2models2losses2optimizers(self):\n model0 = MyModel(1)\n model1 = MyModel(2)\n\n optimizer0 = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}],\n momentum=0.125)\n optimizer1 = torch.optim.SGD([{'params' : model1.parameters(), 'lr' : 0.5}],\n momentum=0.25)\n\n # Don't do it like this: reference_grads = [[]]*5\n # because then it creates a list of 5 references to the same \"[]\" and appending\n # to any of them effectively makes you append to all of them, which multiplies\n # the resulting size of reference_grads by 5x and needless to say makes the test fail.\n reference_grads = [[], [], [], [], []]\n final_params = [None, None, None, None, None]\n for i in range(2):\n optimizer0.zero_grad()\n optimizer1.zero_grad()\n loss0 = model0(self.x)\n loss1 = model1(self.x)\n loss0.backward()\n loss1.backward()\n\n reference_grads[0].append([param.grad.data.clone() for param in model0.parameters()] +\n [param.grad.data.clone() for param in model1.parameters()])\n\n optimizer0.step()\n optimizer1.step()\n\n final_params[0] = [param.data.clone() for param in model0.parameters()] + \\\n [param.data.clone() for param in model1.parameters()]\n\n def what_got_skipped(which_iter, which_backward):\n if which_iter == 0 and which_backward == 0:\n return 1\n if which_iter == 0 and which_backward == 1:\n return 2\n if which_iter == 1 and which_backward == 0:\n return 3\n if which_iter == 1 and which_backward == 1:\n return 4\n return 0\n\n for which_iter in (0,1):\n for which_backward in (0,1):\n model0 = MyModel(1)\n model1 = MyModel(2)\n\n optimizer0 = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}],\n momentum=0.125)\n optimizer1 = torch.optim.SGD([{'params' : model1.parameters(), 'lr' : 0.5}],\n momentum=0.25)\n\n for i in range(3):\n optimizer0.zero_grad()\n optimizer1.zero_grad()\n loss0 = model0(self.x)\n loss1 = model1(self.x)\n loss0.backward()\n loss1.backward()\n\n if i != which_iter:\n reference_grads[what_got_skipped(which_iter, which_backward)].append(\n [param.grad.data.clone() for param in model0.parameters()] +\n [param.grad.data.clone() for param in model1.parameters()])\n\n if i == which_iter:\n if which_backward == 0:\n optimizer1.step()\n else:\n optimizer0.step()\n else:\n optimizer0.step()\n optimizer1.step()\n\n final_params[what_got_skipped(which_iter, which_backward)] = \\\n [param.data.clone() for param in model0.parameters()] + \\\n [param.data.clone() for param in model1.parameters()]\n\n for opt_level in (\"O0\", \"O1\", \"O2\", \"O3\"):\n for how_to_zero in (\"none\", \"model\", \"optimizer\"):\n for use_multiple_loss_scalers in (True, False):\n if opt_level == \"O1\" or opt_level == \"O2\":\n inject_inf_iters = (-1, 0, 1)\n else:\n inject_inf_iters = (-1,)\n\n for inject_inf in inject_inf_iters:\n if inject_inf >= 0:\n inject_inf_locs = (\"fp16\", \"fp32\")\n which_backwards = (0, 1)\n else:\n inject_inf_locs = (\"fdsa\",)\n which_backwards = (None,)\n\n for inject_inf_loc in inject_inf_locs:\n for which_backward in which_backwards:\n if use_multiple_loss_scalers:\n num_losses = 2\n loss_ids = [0, 1]\n else:\n num_losses = 1\n loss_ids = [0, 0]\n\n if inject_inf >= 0:\n iters = 3\n else:\n iters = 2\n\n model0 = MyModel(1)\n model1 = MyModel(2)\n\n models = [model0, model1]\n\n optimizer0 = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}],\n momentum=0.125)\n optimizer1 = torch.optim.SGD([{'params' : model1.parameters(), 'lr' : 0.5}],\n momentum=0.25)\n\n _amp_state.allow_incoming_model_not_fp32 = True\n [model0, model1], [optimizer0, optimizer1] = amp.initialize(\n [model0, model1],\n [optimizer0, optimizer1],\n opt_level=opt_level,\n verbosity=0,\n cast_model_type=False,\n num_losses=num_losses)\n _amp_state.allow_incoming_model_not_fp32 = False\n\n _amp_state.loss_scalers[0]._loss_scale = 4.0\n if use_multiple_loss_scalers:\n _amp_state.loss_scalers[1]._loss_scale = 16.0\n\n unskipped = 0\n for i in range(iters):\n if how_to_zero == \"none\":\n for model in models:\n for param in model.parameters():\n param.grad = None\n elif how_to_zero == \"model\":\n for model in models:\n model.zero_grad()\n else:\n optimizer0.zero_grad()\n optimizer1.zero_grad()\n\n loss0 = model0(self.x)\n loss1 = model1(self.x)\n\n with amp.scale_loss(loss0, optimizer0, loss_id=loss_ids[0]) as scaled_loss:\n scaled_loss.backward()\n if i == inject_inf and which_backward == 0:\n if inject_inf_loc == \"fp32\":\n model0.weight0.grad[0] = float('inf')\n elif inject_inf_loc == \"fp16\":\n model0.weight1.grad[0] = float('inf')\n with amp.scale_loss(loss1, optimizer1, loss_id=loss_ids[1]) as scaled_loss:\n scaled_loss.backward()\n if i == inject_inf and which_backward == 1:\n if inject_inf_loc == \"fp32\":\n model1.weight0.grad[0] = float('inf')\n elif inject_inf_loc == \"fp16\":\n model1.weight1.grad[0] = float('inf')\n\n # print(\"opt_level {} i {} inject_inf {} which_backward {} inject_inf_loc {} use_multiple_loss_scalers {}\".format(opt_level, i, inject_inf, which_backward, inject_inf_loc, use_multiple_loss_scalers))\n\n if i != inject_inf:\n master_params = list(amp.master_params(optimizer0)) + \\\n list(amp.master_params(optimizer1))\n for param, reference_grad in zip(master_params,\n reference_grads[what_got_skipped(inject_inf, which_backward)][unskipped]):\n self.assertTrue(torch.allclose(param.grad.float(), reference_grad.float()))\n unskipped += 1\n\n optimizer0.step()\n optimizer1.step()\n\n model_params = [p for p in model0.parameters()] + [p for p in model1.parameters()]\n master_params = [p for p in amp.master_params(optimizer0)] + \\\n [p for p in amp.master_params(optimizer1)]\n for model, master, reference in zip(\n model_params,\n master_params,\n final_params[what_got_skipped(inject_inf, which_backward)]):\n self.assertTrue(torch.allclose(model, reference))\n self.assertTrue(torch.allclose(model, master.to(model.dtype)))\n\n if opt_level == \"O1\":\n _amp_state.handle._deactivate()\n\n def test_3models2losses2optimizers(self):\n model0 = MyModel(1)\n model1 = MyModel(2)\n model2 = MyModel(3)\n\n optimizer0 = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25},\n {'params' : model1.parameters(), 'lr' : 1.0}],\n momentum=0.5)\n optimizer1 = torch.optim.SGD([{'params' : model2.parameters(), 'lr' : 0.5}],\n momentum=0.25)\n\n # Again, can't do this: reference_grads = [[]]*9\n reference_grads = [[], [], [], [], [], [], [], [], []]\n final_params = [None, None, None, None, None, None, None, None, None]\n for i in range(2):\n optimizer0.zero_grad()\n optimizer1.zero_grad()\n loss0 = model0(self.x) + model1(self.x)\n loss1 = model2(self.x) + model1(self.x)\n loss0.backward()\n loss1.backward()\n\n reference_grads[0].append([param.grad.data.clone() for param in model0.parameters()] +\n [param.grad.data.clone() for param in model1.parameters()])\n\n optimizer0.step()\n optimizer1.step()\n\n final_params[0] = \\\n [param.data.clone() for param in model0.parameters()] + \\\n [param.data.clone() for param in model1.parameters()] + \\\n [param.data.clone() for param in model2.parameters()]\n\n def what_got_skipped(which_iter, which_backward, which_model):\n if which_iter == 0:\n if which_backward == 0:\n if which_model == 0:\n return 1\n if which_model == 1:\n return 2\n if which_backward == 1:\n if which_model == 2:\n return 3\n if which_model == 1:\n return 4\n if which_iter == 1:\n if which_backward == 0:\n if which_model == 0:\n return 5\n if which_model == 1:\n return 6\n if which_backward == 1:\n if which_model == 2:\n return 7\n if which_model == 1:\n return 8\n return 0\n\n for which_iter in (0,1):\n for which_backward in (0,1):\n if which_backward == 0:\n which_models = (0,1)\n if which_backward == 1:\n which_models = (2,1)\n for which_model in which_models:\n\n model0 = MyModel(1)\n model1 = MyModel(2)\n model2 = MyModel(3)\n\n optimizer0 = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25},\n {'params' : model1.parameters(), 'lr' : 1.0}],\n momentum=0.5)\n optimizer1 = torch.optim.SGD([{'params' : model2.parameters(), 'lr' : 0.5}],\n momentum=0.25)\n\n for i in range(3):\n optimizer0.zero_grad()\n optimizer1.zero_grad()\n loss0 = model0(self.x) + model1(self.x)\n loss1 = model2(self.x) + model1(self.x)\n loss0.backward()\n loss1.backward()\n\n if i != which_iter:\n reference_grads[what_got_skipped(which_iter,\n which_backward, which_model)].append(\n [param.grad.data.clone() for param in model0.parameters()] +\n [param.grad.data.clone() for param in model1.parameters()])\n\n if i == which_iter:\n if which_backward == 0:\n # if which_model == 0:\n optimizer1.step()\n # if which_model == 1:\n # optimizer1.step()\n if which_backward == 1:\n # if which_model == 2:\n # optimizer0.step()\n # if which_model == 1:\n continue\n else:\n optimizer0.step()\n optimizer1.step()\n\n final_params[what_got_skipped(which_iter, which_backward, which_model)] = \\\n [param.data.clone() for param in model0.parameters()] + \\\n [param.data.clone() for param in model1.parameters()] + \\\n [param.data.clone() for param in model2.parameters()]\n\n for opt_level in (\"O0\", \"O1\", \"O2\", \"O3\"):\n for how_to_zero in (\"none\", \"model\", \"optimizer\"):\n for use_multiple_loss_scalers in (True, False):\n if opt_level == \"O1\" or opt_level == \"O2\":\n inject_inf_iters = (-1, 0, 1)\n else:\n inject_inf_iters = (-1,)\n\n for inject_inf in inject_inf_iters:\n if inject_inf >= 0:\n inject_inf_locs = (\"fp16\", \"fp32\")\n which_backwards = (0, 1)\n else:\n inject_inf_locs = (\"fdsa\",)\n which_backwards = (None,)\n\n for inject_inf_loc in inject_inf_locs:\n for which_backward in which_backwards:\n if use_multiple_loss_scalers:\n num_losses = 2\n loss_ids = [0, 1]\n else:\n num_losses = 1\n loss_ids = [0, 0]\n\n if inject_inf >= 0:\n iters = 3\n if which_backward == 0:\n which_models = (0, 1)\n elif which_backward == 1:\n which_models = (2, 1)\n else:\n iters = 2\n which_models = (None,)\n\n for which_model in which_models:\n model0 = MyModel(1)\n model1 = MyModel(2)\n model2 = MyModel(3)\n\n models = [model0, model1, model2]\n\n optimizer0 = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25},\n {'params' : model1.parameters(), 'lr' : 1.0}],\n momentum=0.5)\n optimizer1 = torch.optim.SGD([{'params' : model2.parameters(), 'lr' : 0.5}],\n momentum=0.25)\n\n _amp_state.allow_incoming_model_not_fp32 = True\n [model0, model1, model2], [optimizer0, optimizer1] = amp.initialize(\n [model0, model1, model2],\n [optimizer0, optimizer1],\n opt_level=opt_level,\n verbosity=0,\n cast_model_type=False,\n num_losses=num_losses)\n _amp_state.allow_incoming_model_not_fp32 = False\n\n _amp_state.loss_scalers[0]._loss_scale = 4.0\n if use_multiple_loss_scalers:\n _amp_state.loss_scalers[1]._loss_scale = 16.0\n\n unskipped = 0\n for i in range(iters):\n if how_to_zero == \"none\":\n for model in models:\n for param in model.parameters():\n param.grad = None\n elif how_to_zero == \"model\":\n for model in models:\n model.zero_grad()\n else:\n optimizer0.zero_grad()\n optimizer1.zero_grad()\n\n loss0 = model0(self.x) + model1(self.x)\n loss1 = model2(self.x) + model1(self.x)\n\n with amp.scale_loss(loss0, optimizer0, loss_id=loss_ids[0]) as scaled_loss:\n scaled_loss.backward()\n if i == inject_inf and which_backward == 0:\n if which_model == 0:\n inj_model = model0\n elif which_model == 1:\n inj_model = model1\n else:\n raise RuntimeError(which_model + \" invalid for loss 0\")\n if inject_inf_loc == \"fp32\":\n inj_model.weight0.grad[0] = float('inf')\n elif inject_inf_loc == \"fp16\":\n inj_model.weight1.grad[0] = float('inf')\n with amp.scale_loss(loss1, [optimizer0, optimizer1], loss_id=loss_ids[1]) as scaled_loss:\n scaled_loss.backward()\n if i == inject_inf and which_backward == 1:\n if which_model == 2:\n inj_model = model2\n elif which_model == 1:\n inj_model = model1\n else:\n raise RuntimeError(which_model + \" invalid for loss 1 \")\n if inject_inf_loc == \"fp32\":\n inj_model.weight0.grad[0] = float('inf')\n elif inject_inf_loc == \"fp16\":\n inj_model.weight1.grad[0] = float('inf')\n\n if i != inject_inf:\n master_params = list(amp.master_params(optimizer0)) + \\\n list(amp.master_params(optimizer1))\n for param, reference_grad in zip(master_params,\n reference_grads[what_got_skipped(inject_inf,\n which_backward, which_model)][unskipped]):\n self.assertTrue(torch.allclose(param.grad.float(), reference_grad.float()))\n unskipped += 1\n\n optimizer0.step()\n optimizer1.step()\n\n model_params = [p for p in model0.parameters()] + \\\n [p for p in model1.parameters()] + \\\n [p for p in model2.parameters()]\n master_params = [p for p in amp.master_params(optimizer0)] + \\\n [p for p in amp.master_params(optimizer1)]\n\n # print(\"opt_level {} i {} inject_inf {} which_backward {} inject_inf_loc {} use_multiple_loss_scalers {} which_model {}\".format(opt_level, i, inject_inf, which_backward, inject_inf_loc, use_multiple_loss_scalers, which_model))\n\n for model, master, reference in zip(\n model_params,\n master_params,\n final_params[what_got_skipped(inject_inf, which_backward, which_model)]):\n self.assertTrue(torch.allclose(model, reference))\n self.assertTrue(torch.allclose(model, master.to(model.dtype)))\n\n if opt_level == \"O1\":\n _amp_state.handle._deactivate()\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "apex/tests/L0/run_amp/test_promotion.py", "content": "import unittest\n\nimport itertools as it\n\nfrom apex import amp\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\n\nfrom utils import common_init, HALF, FLOAT, DTYPES\n\nclass TestPromotion(unittest.TestCase):\n def setUp(self):\n self.handle = amp.init(enabled=True)\n common_init(self)\n\n def tearDown(self):\n self.handle._deactivate()\n\n def run_binary_promote_test(self, fns, input_shape, x_inplace=False):\n type_pairs = it.product(DTYPES, DTYPES)\n for fn, (xtype, ytype) in it.product(fns, type_pairs):\n x = torch.randn(input_shape, dtype=xtype).requires_grad_()\n x_leaf = x\n if x_inplace:\n # We need a non-leaf to call in place on\n x = x.clone()\n y = torch.randn(input_shape, dtype=ytype)\n out = fn(x, y)\n if x_inplace:\n # In place: always match xtype\n self.assertEqual(out.type(), x.type())\n else:\n # Out of place: match widest type\n if xtype == torch.float or ytype == torch.float:\n self.assertEqual(out.type(), FLOAT)\n else:\n self.assertEqual(out.type(), HALF)\n out.float().sum().backward()\n self.assertEqual(x_leaf.grad.dtype, xtype)\n\n def test_atan2_matches_widest(self):\n fns = [lambda x, y : torch.atan2(x, y),\n lambda x, y : x.atan2(y)]\n self.run_binary_promote_test(fns, (self.b,))\n\n def test_mul_matches_widest(self):\n fns = [lambda x, y : torch.mul(x, y),\n lambda x, y: x.mul(y)]\n self.run_binary_promote_test(fns, (self.b,))\n\n def test_cat_matches_widest(self):\n shape = self.b\n ys = [torch.randn(shape, dtype=torch.half) for _ in range(5)]\n x_float = torch.randn(shape)\n out = torch.cat(ys + [x_float])\n self.assertEqual(out.type(), FLOAT)\n x_half = torch.randn(shape, dtype=torch.half)\n out = torch.cat(ys + [x_half])\n self.assertEqual(out.type(), HALF)\n\n def test_inplace_exp_is_error_for_half(self):\n xs = torch.randn(self.b)\n xs.exp_()\n self.assertEqual(xs.type(), FLOAT)\n xs = torch.randn(self.b, dtype=torch.half)\n with self.assertRaises(NotImplementedError):\n xs.exp_()\n\n def test_inplace_add_matches_self(self):\n fn = lambda x, y: x.add_(y)\n self.run_binary_promote_test([fn], (self.b,), x_inplace=True)\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "apex/tests/L0/run_amp/test_rnn.py", "content": "import unittest\n\nfrom apex import amp\nimport random\nimport torch\nfrom torch import nn\n\nfrom utils import common_init, HALF\n\nclass TestRnnCells(unittest.TestCase):\n def setUp(self):\n self.handle = amp.init(enabled=True)\n common_init(self)\n\n def tearDown(self):\n self.handle._deactivate()\n\n def run_cell_test(self, cell, state_tuple=False):\n shape = (self.b, self.h)\n for typ in [torch.float, torch.half]:\n xs = [torch.randn(shape, dtype=typ).requires_grad_()\n for _ in range(self.t)]\n hidden_fn = lambda: torch.zeros(shape, dtype=typ)\n if state_tuple:\n hidden = (hidden_fn(), hidden_fn())\n else:\n hidden = hidden_fn()\n outputs = []\n for i in range(self.t):\n hidden = cell(xs[i], hidden)\n if state_tuple:\n output = hidden[0]\n else:\n output = hidden\n outputs.append(output)\n for y in outputs:\n self.assertEqual(y.type(), HALF)\n outputs[-1].float().sum().backward()\n for i, x in enumerate(xs):\n self.assertEqual(x.grad.dtype, x.dtype)\n\n def test_rnn_cell_is_half(self):\n cell = nn.RNNCell(self.h, self.h)\n self.run_cell_test(cell)\n\n def test_gru_cell_is_half(self):\n cell = nn.GRUCell(self.h, self.h)\n self.run_cell_test(cell)\n\n def test_lstm_cell_is_half(self):\n cell = nn.LSTMCell(self.h, self.h)\n self.run_cell_test(cell, state_tuple=True)\n\nclass TestRnns(unittest.TestCase):\n def setUp(self):\n self.handle = amp.init(enabled=True)\n common_init(self)\n\n def tearDown(self):\n self.handle._deactivate()\n\n def run_rnn_test(self, rnn, layers, bidir, state_tuple=False):\n for typ in [torch.float, torch.half]:\n x = torch.randn((self.t, self.b, self.h), dtype=typ).requires_grad_()\n hidden_fn = lambda: torch.zeros((layers + (layers * bidir),\n self.b, self.h), dtype=typ)\n if state_tuple:\n hidden = (hidden_fn(), hidden_fn())\n else:\n hidden = hidden_fn()\n output, _ = rnn(x, hidden)\n self.assertEqual(output.type(), HALF)\n output[-1, :, :].float().sum().backward()\n self.assertEqual(x.grad.dtype, x.dtype)\n\n def test_rnn_is_half(self):\n configs = [(1, False), (2, False), (2, True)]\n for layers, bidir in configs:\n rnn = nn.RNN(input_size=self.h, hidden_size=self.h, num_layers=layers,\n nonlinearity='relu', bidirectional=bidir)\n self.run_rnn_test(rnn, layers, bidir)\n\n def test_gru_is_half(self):\n configs = [(1, False), (2, False), (2, True)]\n for layers, bidir in configs:\n rnn = nn.GRU(input_size=self.h, hidden_size=self.h, num_layers=layers,\n bidirectional=bidir)\n self.run_rnn_test(rnn, layers, bidir)\n\n def test_lstm_is_half(self):\n configs = [(1, False), (2, False), (2, True)]\n for layers, bidir in configs:\n rnn = nn.LSTM(input_size=self.h, hidden_size=self.h, num_layers=layers,\n bidirectional=bidir)\n self.run_rnn_test(rnn, layers, bidir, state_tuple=True)\n\n def test_rnn_packed_sequence(self):\n num_layers = 2\n rnn = nn.RNN(input_size=self.h, hidden_size=self.h, num_layers=num_layers)\n for typ in [torch.float, torch.half]:\n x = torch.randn((self.t, self.b, self.h), dtype=typ).requires_grad_()\n lens = sorted([random.randint(self.t // 2, self.t) for _ in range(self.b)],\n reverse=True)\n # `pack_padded_sequence` breaks if default tensor type is non-CPU\n torch.set_default_tensor_type(torch.FloatTensor)\n lens = torch.tensor(lens, dtype=torch.int64, device=torch.device('cpu'))\n packed_seq = nn.utils.rnn.pack_padded_sequence(x, lens)\n torch.set_default_tensor_type(torch.cuda.FloatTensor)\n hidden = torch.zeros((num_layers, self.b, self.h), dtype=typ)\n output, _ = rnn(packed_seq, hidden)\n self.assertEqual(output.data.type(), HALF)\n output.data.float().sum().backward()\n self.assertEqual(x.grad.dtype, x.dtype)\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "apex/tests/L0/run_amp/utils.py", "content": "import torch\n\nHALF = 'torch.cuda.HalfTensor'\nFLOAT = 'torch.cuda.FloatTensor'\n\nDTYPES = [torch.half, torch.float]\n\nALWAYS_HALF = {torch.float: HALF,\n torch.half: HALF}\nALWAYS_FLOAT = {torch.float: FLOAT,\n torch.half: FLOAT}\nMATCH_INPUT = {torch.float: FLOAT,\n torch.half: HALF}\n\ndef common_init(test_case):\n test_case.h = 64\n test_case.b = 16\n test_case.c = 16\n test_case.k = 3\n test_case.t = 10\n torch.set_default_tensor_type(torch.cuda.FloatTensor)\n" }, { "path": "apex/tests/L0/run_fp16util/__init__.py", "content": "" }, { "path": "apex/tests/L0/run_fp16util/test_fp16util.py", "content": "import unittest\n\nimport torch\nimport torch.nn as nn\n\nfrom apex.fp16_utils import FP16Model\n\n\nclass DummyBlock(nn.Module):\n def __init__(self):\n super(DummyBlock, self).__init__()\n\n self.conv = nn.Conv2d(10, 10, 2)\n self.bn = nn.BatchNorm2d(10, affine=True)\n\n def forward(self, x):\n return self.conv(self.bn(x))\n\n\nclass DummyNet(nn.Module):\n def __init__(self):\n super(DummyNet, self).__init__()\n\n self.conv1 = nn.Conv2d(3, 10, 2)\n self.bn1 = nn.BatchNorm2d(10, affine=False)\n self.db1 = DummyBlock()\n self.db2 = DummyBlock()\n\n def forward(self, x):\n out = x\n out = self.conv1(out)\n out = self.bn1(out)\n out = self.db1(out)\n out = self.db2(out)\n return out\n\n\nclass DummyNetWrapper(nn.Module):\n def __init__(self):\n super(DummyNetWrapper, self).__init__()\n\n self.bn = nn.BatchNorm2d(3, affine=True)\n self.dn = DummyNet()\n\n def forward(self, x):\n return self.dn(self.bn(x))\n\n\nclass TestFP16Model(unittest.TestCase):\n def setUp(self):\n self.N = 64\n self.C_in = 3\n self.H_in = 16\n self.W_in = 32\n self.in_tensor = torch.randn((self.N, self.C_in, self.H_in, self.W_in)).cuda()\n self.orig_model = DummyNetWrapper().cuda()\n self.fp16_model = FP16Model(self.orig_model)\n\n def test_params_and_buffers(self):\n exempted_modules = [\n self.fp16_model.network.bn,\n self.fp16_model.network.dn.db1.bn,\n self.fp16_model.network.dn.db2.bn,\n ]\n for m in self.fp16_model.modules():\n expected_dtype = torch.float if (m in exempted_modules) else torch.half\n for p in m.parameters(recurse=False):\n assert p.dtype == expected_dtype\n for b in m.buffers(recurse=False):\n assert b.dtype in (expected_dtype, torch.int64)\n\n def test_output_is_half(self):\n out_tensor = self.fp16_model(self.in_tensor)\n assert out_tensor.dtype == torch.half\n\n" }, { "path": "apex/tests/L0/run_fused_layer_norm/test_fused_layer_norm.py", "content": "import unittest\nimport os\nimport random\n\nimport torch\nimport apex\n\n \nclass TestFusedLayerNorm(unittest.TestCase):\n def setUp(self):\n self.module = apex.normalization.FusedLayerNorm(normalized_shape=[32, 64], elementwise_affine=False)\n self.input_ = torch.randn(16, 32, 64)\n torch.cuda.manual_seed(42)\n \n def forward_cpu(self, input_):\n self.module.cpu()\n return self.module(input_.cpu())\n \n def forward_cuda(self, input_):\n self.module.cuda()\n return self.module(input_.cuda())\n \n def test_forward_cuda(self):\n out_ = self.forward_cuda(self.input_)\n assert out_.is_cuda == True\n \n def test_forward_cpu(self):\n out_ = self.forward_cpu(self.input_)\n assert out_.is_cuda == False\n \n def test_same_output(self):\n out_cpu = self.forward_cpu(self.input_)\n out_cuda = self.forward_cuda(self.input_)\n torch.testing.assert_allclose(out_cpu, out_cuda.cpu())\n \n \nclass TestFusedLayerNormElemWise(TestFusedLayerNorm):\n def setUp(self):\n self.module = apex.normalization.FusedLayerNorm(normalized_shape=[32, 64], elementwise_affine=True)\n self.input_ = torch.randn(16, 32, 64)\n torch.cuda.manual_seed(42)" }, { "path": "apex/tests/L0/run_mixed_adam/__init__.py", "content": "" }, { "path": "apex/tests/L0/run_mixed_adam/test_fp16_optimizer.py", "content": "import unittest\nimport torch\nimport apex\n\nclass TestFP16Optimizer(unittest.TestCase):\n def setUp(self, max_abs_diff=1e-3, max_rel_diff=1, iters=7):\n self.max_abs_diff = max_abs_diff\n self.max_rel_diff = max_rel_diff\n self.iters = iters\n torch.cuda.manual_seed(13337)\n\n N, D_in, D_out = 64, 1024, 16\n self.N = N\n self.D_in = D_in\n self.D_out = D_out\n self.x = torch.randn((N, D_in), dtype=torch.float16, device='cuda')\n self.ref_model = torch.nn.Linear(D_in, D_out).cuda().half()\n self.tst_model = torch.nn.Linear(D_in, D_out).cuda().half()\n for p,q in zip(self.tst_model.parameters(), self.ref_model.parameters()):\n p.data.copy_(q.data)\n\n def get_max_diff(self, ref_param, tst_param):\n max_abs_diff = max_rel_diff = 0\n for p_ref, p_tst in zip(ref_param, tst_param):\n max_abs_diff_p = (p_ref - p_tst).abs().max().item()\n max_rel_diff_p = ((p_ref - p_tst) / p_ref).abs().max().item()\n\n if max_abs_diff_p > max_abs_diff: max_abs_diff = max_abs_diff_p\n if max_rel_diff_p > max_rel_diff: max_rel_diff = max_rel_diff_p\n\n return max_abs_diff, max_rel_diff\n\n def test_fp16_optimizer(self):\n\n ref_optim = torch.optim.Adam(self.ref_model.parameters())\n ref_optim = apex.fp16_utils.FP16_Optimizer(ref_optim, verbose=False)\n\n tst_optim = apex.optimizers.FusedAdam(self.tst_model.parameters())\n tst_optim = apex.optimizers.FP16_Optimizer(tst_optim)\n\n for i in range(self.iters):\n ref_loss = self.ref_model(self.x).sum()\n ref_optim.backward(ref_loss)\n ref_optim.step()\n\n tst_loss = self.tst_model(self.x).sum()\n tst_optim.backward(tst_loss)\n tst_optim.step()\n\n max_abs_diff, max_rel_diff = self.get_max_diff(self.ref_model.parameters(), self.tst_model.parameters())\n self.assertLessEqual(max_abs_diff, self.max_abs_diff)\n self.assertLessEqual(max_rel_diff, self.max_rel_diff)\n\n\n def test_loss_scaling(self):\n\n ref_optim = torch.optim.Adam(self.ref_model.parameters())\n ref_optim = apex.fp16_utils.FP16_Optimizer(ref_optim, static_loss_scale=128.0, verbose=False)\n\n tst_optim = apex.optimizers.FusedAdam(self.tst_model.parameters())\n tst_optim = apex.optimizers.FP16_Optimizer(tst_optim, static_loss_scale=128.0)\n\n for i in range(self.iters):\n ref_loss = self.ref_model(self.x).sum()\n ref_optim.backward(ref_loss)\n ref_optim.step()\n\n tst_loss = self.tst_model(self.x).sum()\n tst_optim.backward(tst_loss)\n tst_optim.step()\n\n max_abs_diff, max_rel_diff = self.get_max_diff(self.ref_model.parameters(), self.tst_model.parameters())\n self.assertLessEqual(max_abs_diff, self.max_abs_diff)\n self.assertLessEqual(max_rel_diff, self.max_rel_diff)\n\n def test_parameter_groups(self):\n\n ref_groups = [{'params': [self.ref_model.weight]},{'params': [self.ref_model.bias]}]\n ref_optim = torch.optim.Adam(ref_groups)\n ref_optim = apex.fp16_utils.FP16_Optimizer(ref_optim, verbose=False)\n\n tst_groups = [{'params': [self.tst_model.weight]},{'params': [self.tst_model.bias]}]\n tst_optim = apex.optimizers.FusedAdam(tst_groups)\n tst_optim = apex.optimizers.FP16_Optimizer(tst_optim)\n\n for i in range(self.iters):\n ref_loss = self.ref_model(self.x).sum()\n ref_optim.backward(ref_loss)\n ref_optim.step()\n\n tst_loss = self.tst_model(self.x).sum()\n tst_optim.backward(tst_loss)\n tst_optim.step()\n\n max_abs_diff, max_rel_diff = self.get_max_diff(self.ref_model.parameters(), self.tst_model.parameters())\n self.assertLessEqual(max_abs_diff, self.max_abs_diff)\n self.assertLessEqual(max_rel_diff, self.max_rel_diff)\n\n def test_grad_clip(self):\n ref_optim = torch.optim.Adam(self.ref_model.parameters())\n ref_optim = apex.fp16_utils.FP16_Optimizer(ref_optim, verbose=False)\n\n tst_optim = apex.optimizers.FusedAdam(self.tst_model.parameters(), max_grad_norm=0.01)\n tst_optim = apex.optimizers.FP16_Optimizer(tst_optim)\n\n for i in range(self.iters):\n ref_loss = self.ref_model(self.x).sum()\n ref_optim.backward(ref_loss)\n ref_optim.clip_master_grads(0.01)\n ref_optim.step()\n\n tst_loss = self.tst_model(self.x).sum()\n tst_optim.backward(tst_loss)\n tst_optim.step()\n\n max_abs_diff, max_rel_diff = self.get_max_diff(self.ref_model.parameters(), self.tst_model.parameters())\n self.assertLessEqual(max_abs_diff, self.max_abs_diff)\n self.assertLessEqual(max_rel_diff, self.max_rel_diff)\n\n @unittest.skip('Not support grad being None')\n def test_grad_None(self):\n self.fail()\n\n @unittest.skip('Not support same weight decay as pytorch')\n def test_weight_decay(self):\n self.fail()\n\n @unittest.skip('Not support empty parameter groups')\n def test_group_empty(self):\n self.fail()\n\nif __name__ == '__main__':\n script_path = os.path.dirname(os.path.realpath(__file__))\n unittest.main()\n" }, { "path": "apex/tests/L0/run_mixed_adam/test_mixed_adam.py", "content": "import unittest\nimport os\nimport random\n\nimport torch\nimport apex\n\nclass TestFusedAdam(unittest.TestCase):\n def setUp(self, max_abs_diff=1e-3, max_rel_diff=1, iters=7):\n self.max_abs_diff = max_abs_diff\n self.max_rel_diff = max_rel_diff\n self.iters = iters\n torch.cuda.manual_seed(9876)\n\n def tearDown(self):\n pass\n\n def gen_param_optim(self, tensors, adam_option):\n ref_param = []\n tst_param = []\n for tensor in tensors:\n ref_param.append(torch.nn.Parameter(tensor.clone()))\n tst_param.append(torch.nn.Parameter(tensor.clone()))\n\n ref_optim = torch.optim.Adam(ref_param, **adam_option)\n tst_optim = apex.optimizers.FusedAdam(tst_param, **adam_option)\n \n return (ref_param, tst_param, ref_optim, tst_optim)\n\n def gen_grad(self, ref_param, tst_param):\n for p_ref, p_tst in zip(ref_param, tst_param):\n p_ref.grad = torch.rand_like(p_ref)\n p_tst.grad = p_ref.grad\n\n def gen_mixed_grad(self, ref_param, tst_param, scale=1.0):\n half_grads = []\n for p_ref, p_tst in zip(ref_param, tst_param):\n half_grads.append(torch.rand_like(p_ref).half())\n p_ref.grad = half_grads[-1].float() / scale\n return half_grads\n\n def get_max_diff(self, ref_param, tst_param):\n max_abs_diff = max_rel_diff = 0\n for p_ref, p_tst in zip(ref_param, tst_param):\n max_abs_diff_p = (p_ref - p_tst).abs().max().item()\n max_rel_diff_p = ((p_ref - p_tst) / p_ref).abs().max().item()\n\n if max_abs_diff_p > max_abs_diff: max_abs_diff = max_abs_diff_p\n if max_rel_diff_p > max_rel_diff: max_rel_diff = max_rel_diff_p\n\n return max_abs_diff, max_rel_diff\n\n def gen_single_type_test(self, param_type=torch.float):\n nelem = 278011\n adam_option = {'lr':5e-4, 'betas':(0.9, 0.999), 'eps':1e-08,\n 'weight_decay':0, 'amsgrad':False}\n\n tensor = torch.rand(nelem, dtype=param_type, device='cuda')\n ref_param, tst_param, ref_optim, tst_optim = \\\n self.gen_param_optim([tensor], adam_option)\n\n for i in range(self.iters):\n self.gen_grad(ref_param, tst_param)\n ref_optim.step()\n tst_optim.step()\n max_abs_diff, max_rel_diff = self.get_max_diff(ref_param, tst_param)\n\n self.assertLessEqual(max_abs_diff, self.max_abs_diff)\n self.assertLessEqual(max_rel_diff, self.max_rel_diff)\n\n def test_double(self):\n self.gen_single_type_test(param_type=torch.double)\n\n def test_float(self):\n self.gen_single_type_test(param_type=torch.float)\n\n def test_half(self):\n nelem = 278011\n adam_option = {'lr':5e-4, 'betas':(0.9, 0.999), 'eps':1e-08,\n 'weight_decay':0, 'amsgrad':False}\n\n tensor = torch.rand(nelem, dtype=torch.float, device='cuda')\n ref_param, tst_param, ref_optim, tst_optim = \\\n self.gen_param_optim([tensor], adam_option)\n\n for i in range(self.iters):\n half_grads = self.gen_mixed_grad(ref_param, tst_param)\n ref_optim.step()\n tst_optim.step(grads=half_grads)\n max_abs_diff, max_rel_diff = self.get_max_diff(ref_param, tst_param)\n\n self.assertLessEqual(max_abs_diff, self.max_abs_diff)\n self.assertLessEqual(max_rel_diff, self.max_rel_diff)\n\n def test_multi_params(self):\n sizes = [[4096, 1024], [4096], [4096, 2048], [32320, 1024], [1]]\n adam_option = {'lr':5e-4, 'betas':(0.9, 0.999), 'eps':1e-08,\n 'weight_decay':0, 'amsgrad':False}\n\n tensors = []\n for size in sizes:\n tensors.append(torch.rand(size, dtype=torch.float, device='cuda'))\n ref_param, tst_param, ref_optim, tst_optim = \\\n self.gen_param_optim(tensors, adam_option)\n\n for i in range(self.iters):\n half_grads = self.gen_mixed_grad(ref_param, tst_param)\n ref_optim.step()\n tst_optim.step(grads=half_grads)\n max_abs_diff, max_rel_diff = self.get_max_diff(ref_param, tst_param)\n self.assertLessEqual(max_abs_diff, self.max_abs_diff)\n self.assertLessEqual(max_rel_diff, self.max_rel_diff)\n\n def test_scale(self):\n nelem = 278011\n adam_option = {'lr':5e-4, 'betas':(0.9, 0.999), 'eps':1e-08,\n 'weight_decay':0, 'amsgrad':False}\n\n tensor = torch.rand(nelem, dtype=torch.float, device='cuda')\n ref_param, tst_param, ref_optim, tst_optim = \\\n self.gen_param_optim([tensor], adam_option)\n\n for i in range(self.iters):\n scale = random.random() * 1000\n half_grads = self.gen_mixed_grad(ref_param, tst_param, scale)\n ref_optim.step()\n tst_optim.step(grads=half_grads, scale=scale)\n max_abs_diff, max_rel_diff = self.get_max_diff(ref_param, tst_param)\n\n self.assertLessEqual(max_abs_diff, self.max_abs_diff)\n self.assertLessEqual(max_rel_diff, self.max_rel_diff)\n\n def test_fp16_output(self):\n nelem = 278011\n adam_option = {'lr':5e-4, 'betas':(0.9, 0.999), 'eps':1e-08,\n 'weight_decay':0, 'amsgrad':False}\n\n tensor = torch.rand(nelem, dtype=torch.float, device='cuda')\n ref_param, tst_param, ref_optim, tst_optim = \\\n self.gen_param_optim([tensor], adam_option)\n\n fp16_param = torch.nn.Parameter(tensor.clone().half())\n\n for i in range(self.iters):\n half_grads = self.gen_mixed_grad(ref_param, tst_param)\n ref_optim.step()\n tst_optim.step(grads=half_grads, output_params=[fp16_param])\n\n max_abs_diff, max_rel_diff = self.get_max_diff(ref_param, tst_param)\n self.assertLessEqual(max_abs_diff, self.max_abs_diff)\n self.assertLessEqual(max_rel_diff, self.max_rel_diff)\n\n max_abs_diff, max_rel_diff = self.get_max_diff(tst_param, \\\n [fp16_param.float()])\n self.assertLessEqual(max_abs_diff, self.max_abs_diff)\n self.assertLessEqual(max_rel_diff, self.max_rel_diff)\n\n def test_adam_option(self):\n nelem = 1\n adam_option = {'lr':0.01, 'betas':(0.6, 0.9), 'eps':3e-06,\n 'weight_decay':0, 'amsgrad':False}\n\n tensor = torch.rand(nelem, dtype=torch.float, device='cuda')\n ref_param, tst_param, ref_optim, tst_optim = \\\n self.gen_param_optim([tensor], adam_option)\n\n for i in range(self.iters):\n self.gen_grad(ref_param, tst_param)\n ref_optim.step()\n tst_optim.step()\n max_abs_diff, max_rel_diff = self.get_max_diff(ref_param, tst_param)\n\n self.assertLessEqual(max_abs_diff, self.max_abs_diff)\n self.assertLessEqual(max_rel_diff, self.max_rel_diff)\n\n\nif __name__ == '__main__':\n script_path = os.path.dirname(os.path.realpath(__file__))\n unittest.main()\n" }, { "path": "apex/tests/L0/run_test.py", "content": "import unittest\nimport sys\n\ntest_dirs = [\"run_amp\", \"run_fp16util\", \"run_mixed_adam\", \"run_fused_layer_norm\"]\n\nrunner = unittest.TextTestRunner(verbosity=2)\n\nerrcode = 0\n\nfor test_dir in test_dirs:\n suite = unittest.TestLoader().discover(test_dir)\n\n print(\"\\nExecuting tests from \" + test_dir)\n\n result = runner.run(suite)\n\n if not result.wasSuccessful():\n errcode = 1\n\nsys.exit(errcode)\n" }, { "path": "apex/tests/L1/common/compare.py", "content": "import argparse\nimport torch\n\nparser = argparse.ArgumentParser(description='Compare')\nparser.add_argument('--opt-level', type=str)\nparser.add_argument('--keep-batchnorm-fp32', type=str, default=None)\nparser.add_argument('--loss-scale', type=str, default=None)\nparser.add_argument('--fused-adam', action='store_true')\nparser.add_argument('--use_baseline', action='store_true')\nargs = parser.parse_args()\n\nbase_file = str(args.opt_level) + \"_\" +\\\n str(args.loss_scale) + \"_\" +\\\n str(args.keep_batchnorm_fp32) + \"_\" +\\\n str(args.fused_adam)\n\nfile_e = \"True_\" + base_file\nfile_p = \"False_\" + base_file\nif args.use_baseline:\n file_b = \"baselines/True_\" + base_file\n\ndict_e = torch.load(file_e)\ndict_p = torch.load(file_p)\nif args.use_baseline:\n dict_b = torch.load(file_b)\n\ntorch.set_printoptions(precision=10)\n\nprint(file_e)\nprint(file_p)\nif args.use_baseline:\n print(file_b)\n\n# ugly duplication here...\nif not args.use_baseline:\n for n, (i_e, i_p) in enumerate(zip(dict_e[\"Iteration\"], dict_p[\"Iteration\"])):\n assert i_e == i_p, \"i_e = {}, i_p = {}\".format(i_e, i_p)\n\n loss_e = dict_e[\"Loss\"][n]\n loss_p = dict_p[\"Loss\"][n]\n assert loss_e == loss_p, \"Iteration {}, loss_e = {}, loss_p = {}\".format(i_e, loss_e, loss_p)\n print(\"{:4} {:15.10f} {:15.10f} {:15.10f} {:15.10f}\".format(\n i_e,\n loss_e,\n loss_p,\n dict_e[\"Speed\"][n],\n dict_p[\"Speed\"][n]))\nelse:\n for n, (i_e, i_p) in enumerate(zip(dict_e[\"Iteration\"], dict_p[\"Iteration\"])):\n assert i_e == i_p, \"i_e = {}, i_p = {}\".format(i_e, i_p)\n\n loss_e = dict_e[\"Loss\"][n]\n loss_p = dict_p[\"Loss\"][n]\n loss_b = dict_b[\"Loss\"][n]\n assert loss_e == loss_p, \"Iteration {}, loss_e = {}, loss_p = {}\".format(i_e, loss_e, loss_p)\n assert loss_e == loss_b, \"Iteration {}, loss_e = {}, loss_b = {}\".format(i_e, loss_e, loss_b)\n print(\"{:4} {:15.10f} {:15.10f} {:15.10f} {:15.10f} {:15.10f} {:15.10f}\".format(\n i_e,\n loss_b,\n loss_e,\n loss_p,\n dict_b[\"Speed\"][n],\n dict_e[\"Speed\"][n],\n dict_p[\"Speed\"][n]))\n" }, { "path": "apex/tests/L1/common/main_amp.py", "content": "import argparse\nimport os\nimport shutil\nimport time\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.parallel\nimport torch.backends.cudnn as cudnn\nimport torch.distributed as dist\nimport torch.optim\nimport torch.utils.data\nimport torch.utils.data.distributed\nimport torchvision.transforms as transforms\nimport torchvision.datasets as datasets\nimport torchvision.models as models\n\nimport numpy as np\n\ntry:\n from apex.parallel import DistributedDataParallel as DDP\n from apex.fp16_utils import *\n from apex import amp, optimizers\n from apex.multi_tensor_apply import multi_tensor_applier\nexcept ImportError:\n raise ImportError(\"Please install apex from https://www.github.com/nvidia/apex to run this example.\")\n\nmodel_names = sorted(name for name in models.__dict__\n if name.islower() and not name.startswith(\"__\")\n and callable(models.__dict__[name]))\n\nparser = argparse.ArgumentParser(description='PyTorch ImageNet Training')\nparser.add_argument('data', metavar='DIR',\n help='path to dataset')\nparser.add_argument('--arch', '-a', metavar='ARCH', default='resnet18',\n choices=model_names,\n help='model architecture: ' +\n ' | '.join(model_names) +\n ' (default: resnet18)')\nparser.add_argument('-j', '--workers', default=4, type=int, metavar='N',\n help='number of data loading workers (default: 4)')\nparser.add_argument('--epochs', default=90, type=int, metavar='N',\n help='number of total epochs to run')\nparser.add_argument('--start-epoch', default=0, type=int, metavar='N',\n help='manual epoch number (useful on restarts)')\nparser.add_argument('-b', '--batch-size', default=256, type=int,\n metavar='N', help='mini-batch size per process (default: 256)')\nparser.add_argument('--lr', '--learning-rate', default=0.1, type=float,\n metavar='LR', help='Initial learning rate. Will be scaled by /256: args.lr = args.lr*float(args.batch_size*args.world_size)/256. A warmup schedule will also be applied over the first 5 epochs.')\nparser.add_argument('--momentum', default=0.9, type=float, metavar='M',\n help='momentum')\nparser.add_argument('--weight-decay', '--wd', default=1e-4, type=float,\n metavar='W', help='weight decay (default: 1e-4)')\nparser.add_argument('--print-freq', '-p', default=10, type=int,\n metavar='N', help='print frequency (default: 10)')\nparser.add_argument('--resume', default='', type=str, metavar='PATH',\n help='path to latest checkpoint (default: none)')\nparser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true',\n help='evaluate model on validation set')\nparser.add_argument('--pretrained', dest='pretrained', action='store_true',\n help='use pre-trained model')\n\nparser.add_argument('--prof', dest='prof', action='store_true',\n help='Only run 10 iterations for profiling.')\nparser.add_argument('--deterministic', action='store_true')\n\nparser.add_argument(\"--local_rank\", default=0, type=int)\nparser.add_argument('--sync_bn', action='store_true',\n help='enabling apex sync BN.')\n\nparser.add_argument('--has-ext', action='store_true')\nparser.add_argument('--opt-level', type=str)\nparser.add_argument('--keep-batchnorm-fp32', type=str, default=None)\nparser.add_argument('--loss-scale', type=str, default=None)\nparser.add_argument('--fused-adam', action='store_true')\n\nparser.add_argument('--prints-to-process', type=int, default=10)\n\ncudnn.benchmark = True\n\ndef fast_collate(batch):\n imgs = [img[0] for img in batch]\n targets = torch.tensor([target[1] for target in batch], dtype=torch.int64)\n w = imgs[0].size[0]\n h = imgs[0].size[1]\n tensor = torch.zeros( (len(imgs), 3, h, w), dtype=torch.uint8 )\n for i, img in enumerate(imgs):\n nump_array = np.asarray(img, dtype=np.uint8)\n if(nump_array.ndim < 3):\n nump_array = np.expand_dims(nump_array, axis=-1)\n nump_array = np.rollaxis(nump_array, 2)\n\n tensor[i] += torch.from_numpy(nump_array)\n \n return tensor, targets\n\nbest_prec1 = 0\nargs = parser.parse_args()\n\n# Let multi_tensor_applier be the canary in the coalmine\n# that verifies if the backend is what we think it is\nassert multi_tensor_applier.available == args.has_ext \n\nprint(\"opt_level = {}\".format(args.opt_level))\nprint(\"keep_batchnorm_fp32 = {}\".format(args.keep_batchnorm_fp32), type(args.keep_batchnorm_fp32))\nprint(\"loss_scale = {}\".format(args.loss_scale), type(args.loss_scale))\n\n\nprint(\"\\nCUDNN VERSION: {}\\n\".format(torch.backends.cudnn.version()))\n\nif args.deterministic:\n cudnn.benchmark = False\n cudnn.deterministic = True\n torch.manual_seed(args.local_rank)\n torch.set_printoptions(precision=10)\n\ndef main():\n global best_prec1, args\n\n args.distributed = False\n if 'WORLD_SIZE' in os.environ:\n args.distributed = int(os.environ['WORLD_SIZE']) > 1\n\n args.gpu = 0\n args.world_size = 1\n\n if args.distributed:\n args.gpu = args.local_rank % torch.cuda.device_count()\n torch.cuda.set_device(args.gpu)\n torch.distributed.init_process_group(backend='nccl',\n init_method='env://')\n args.world_size = torch.distributed.get_world_size()\n\n assert torch.backends.cudnn.enabled, \"Amp requires cudnn backend to be enabled.\"\n\n # create model\n if args.pretrained:\n print(\"=> using pre-trained model '{}'\".format(args.arch))\n model = models.__dict__[args.arch](pretrained=True)\n else:\n print(\"=> creating model '{}'\".format(args.arch))\n model = models.__dict__[args.arch]()\n\n if args.sync_bn:\n import apex\n print(\"using apex synced BN\")\n model = apex.parallel.convert_syncbn_model(model)\n\n model = model.cuda()\n\n # Scale learning rate based on global batch size\n args.lr = args.lr*float(args.batch_size*args.world_size)/256. \n if args.fused_adam:\n optimizer = optimizers.FusedAdam(model.parameters())\n else:\n optimizer = torch.optim.SGD(model.parameters(), args.lr,\n momentum=args.momentum,\n weight_decay=args.weight_decay)\n\n model, optimizer = amp.initialize(\n model, optimizer,\n # enabled=False,\n opt_level=args.opt_level,\n keep_batchnorm_fp32=args.keep_batchnorm_fp32,\n loss_scale=args.loss_scale\n )\n\n if args.distributed:\n # By default, apex.parallel.DistributedDataParallel overlaps communication with \n # computation in the backward pass.\n # model = DDP(model)\n # delay_allreduce delays all communication to the end of the backward pass.\n model = DDP(model, delay_allreduce=True)\n\n # define loss function (criterion) and optimizer\n criterion = nn.CrossEntropyLoss().cuda()\n\n # Optionally resume from a checkpoint\n if args.resume:\n # Use a local scope to avoid dangling references\n def resume():\n if os.path.isfile(args.resume):\n print(\"=> loading checkpoint '{}'\".format(args.resume))\n checkpoint = torch.load(args.resume, map_location = lambda storage, loc: storage.cuda(args.gpu))\n args.start_epoch = checkpoint['epoch']\n best_prec1 = checkpoint['best_prec1']\n model.load_state_dict(checkpoint['state_dict'])\n optimizer.load_state_dict(checkpoint['optimizer'])\n print(\"=> loaded checkpoint '{}' (epoch {})\"\n .format(args.resume, checkpoint['epoch']))\n else:\n print(\"=> no checkpoint found at '{}'\".format(args.resume))\n resume()\n\n # Data loading code\n traindir = os.path.join(args.data, 'train')\n valdir = os.path.join(args.data, 'val')\n\n if(args.arch == \"inception_v3\"):\n crop_size = 299\n val_size = 320 # I chose this value arbitrarily, we can adjust.\n else:\n crop_size = 224\n val_size = 256\n\n train_dataset = datasets.ImageFolder(\n traindir,\n transforms.Compose([\n transforms.RandomResizedCrop(crop_size),\n transforms.RandomHorizontalFlip(),\n # transforms.ToTensor(), Too slow\n # normalize,\n ]))\n val_dataset = datasets.ImageFolder(valdir, transforms.Compose([\n transforms.Resize(val_size),\n transforms.CenterCrop(crop_size),\n ]))\n\n train_sampler = None\n val_sampler = None\n if args.distributed:\n train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset)\n val_sampler = torch.utils.data.distributed.DistributedSampler(val_dataset)\n\n train_loader = torch.utils.data.DataLoader(\n train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None),\n num_workers=args.workers, pin_memory=True, sampler=train_sampler, collate_fn=fast_collate)\n\n val_loader = torch.utils.data.DataLoader(\n val_dataset,\n batch_size=args.batch_size, shuffle=False,\n num_workers=args.workers, pin_memory=True,\n sampler=val_sampler,\n collate_fn=fast_collate)\n\n if args.evaluate:\n validate(val_loader, model, criterion)\n return\n\n for epoch in range(args.start_epoch, args.epochs):\n if args.distributed:\n train_sampler.set_epoch(epoch)\n\n # train for one epoch\n train(train_loader, model, criterion, optimizer, epoch)\n if args.prof:\n break\n # evaluate on validation set\n prec1 = validate(val_loader, model, criterion)\n\n # remember best prec@1 and save checkpoint\n if args.local_rank == 0:\n is_best = prec1 > best_prec1\n best_prec1 = max(prec1, best_prec1)\n save_checkpoint({\n 'epoch': epoch + 1,\n 'arch': args.arch,\n 'state_dict': model.state_dict(),\n 'best_prec1': best_prec1,\n 'optimizer' : optimizer.state_dict(),\n }, is_best)\n\nclass data_prefetcher():\n def __init__(self, loader):\n self.loader = iter(loader)\n self.stream = torch.cuda.Stream()\n self.mean = torch.tensor([0.485 * 255, 0.456 * 255, 0.406 * 255]).cuda().view(1,3,1,1)\n self.std = torch.tensor([0.229 * 255, 0.224 * 255, 0.225 * 255]).cuda().view(1,3,1,1)\n # With Amp, it isn't necessary to manually convert data to half.\n # if args.fp16:\n # self.mean = self.mean.half()\n # self.std = self.std.half()\n self.preload()\n\n def preload(self):\n try:\n self.next_input, self.next_target = next(self.loader)\n except StopIteration:\n self.next_input = None\n self.next_target = None\n return\n with torch.cuda.stream(self.stream):\n self.next_input = self.next_input.cuda(non_blocking=True)\n self.next_target = self.next_target.cuda(non_blocking=True)\n # With Amp, it isn't necessary to manually convert data to half.\n # if args.fp16:\n # self.next_input = self.next_input.half()\n # else:\n self.next_input = self.next_input.float()\n self.next_input = self.next_input.sub_(self.mean).div_(self.std)\n \n def next(self):\n torch.cuda.current_stream().wait_stream(self.stream)\n input = self.next_input\n target = self.next_target\n self.preload()\n return input, target\n\n\ndef train(train_loader, model, criterion, optimizer, epoch):\n batch_time = AverageMeter()\n data_time = AverageMeter()\n losses = AverageMeter()\n top1 = AverageMeter()\n top5 = AverageMeter()\n\n # switch to train mode\n model.train()\n end = time.time()\n\n run_info_dict = {\"Iteration\" : [],\n \"Loss\" : [],\n \"Speed\" : []}\n\n prefetcher = data_prefetcher(train_loader)\n input, target = prefetcher.next()\n i = -1\n while input is not None:\n i += 1\n\n # No learning rate warmup for this test, to expose bitwise inaccuracies more quickly\n # adjust_learning_rate(optimizer, epoch, i, len(train_loader))\n\n if args.prof:\n if i > 10:\n break\n # measure data loading time\n data_time.update(time.time() - end)\n\n # compute output\n output = model(input)\n loss = criterion(output, target)\n\n # measure accuracy and record loss\n prec1, prec5 = accuracy(output.data, target, topk=(1, 5))\n\n if args.distributed:\n reduced_loss = reduce_tensor(loss.data)\n prec1 = reduce_tensor(prec1)\n prec5 = reduce_tensor(prec5)\n else:\n reduced_loss = loss.data\n\n losses.update(to_python_float(reduced_loss), input.size(0))\n top1.update(to_python_float(prec1), input.size(0))\n top5.update(to_python_float(prec5), input.size(0))\n\n # compute gradient and do SGD step\n optimizer.zero_grad()\n\n with amp.scale_loss(loss, optimizer) as scaled_loss:\n scaled_loss.backward()\n\n # for param in model.parameters():\n # print(param.data.double().sum().item(), param.grad.data.double().sum().item())\n\n # torch.cuda.synchronize()\n torch.cuda.nvtx.range_push(\"step\")\n optimizer.step()\n torch.cuda.nvtx.range_pop()\n\n torch.cuda.synchronize()\n # measure elapsed time\n batch_time.update(time.time() - end)\n\n end = time.time()\n\n # If you decide to refactor this test, like examples/imagenet, to sample the loss every\n # print_freq iterations, make sure to move this prefetching below the accuracy calculation.\n input, target = prefetcher.next()\n\n if i % args.print_freq == 0 and i > 1:\n if args.local_rank == 0:\n print('Epoch: [{0}][{1}/{2}]\\t'\n 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\\t'\n 'Speed {3:.3f} ({4:.3f})\\t'\n 'Data {data_time.val:.3f} ({data_time.avg:.3f})\\t'\n 'Loss {loss.val:.10f} ({loss.avg:.4f})\\t'\n 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\\t'\n 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(\n epoch, i, len(train_loader),\n args.world_size * args.batch_size / batch_time.val,\n args.world_size * args.batch_size / batch_time.avg,\n batch_time=batch_time,\n data_time=data_time, loss=losses, top1=top1, top5=top5))\n run_info_dict[\"Iteration\"].append(i)\n run_info_dict[\"Loss\"].append(losses.val)\n run_info_dict[\"Speed\"].append(args.world_size * args.batch_size / batch_time.val)\n if len(run_info_dict[\"Loss\"]) == args.prints_to_process:\n if args.local_rank == 0:\n torch.save(run_info_dict,\n str(args.has_ext) + \"_\" + str(args.opt_level) + \"_\" +\n str(args.loss_scale) + \"_\" + str(args.keep_batchnorm_fp32) + \"_\" +\n str(args.fused_adam))\n quit()\n\n\ndef validate(val_loader, model, criterion):\n batch_time = AverageMeter()\n losses = AverageMeter()\n top1 = AverageMeter()\n top5 = AverageMeter()\n\n # switch to evaluate mode\n model.eval()\n\n end = time.time()\n\n prefetcher = data_prefetcher(val_loader)\n input, target = prefetcher.next()\n i = -1\n while input is not None:\n i += 1\n\n # compute output\n with torch.no_grad():\n output = model(input)\n loss = criterion(output, target)\n\n # measure accuracy and record loss\n prec1, prec5 = accuracy(output.data, target, topk=(1, 5))\n\n if args.distributed:\n reduced_loss = reduce_tensor(loss.data)\n prec1 = reduce_tensor(prec1)\n prec5 = reduce_tensor(prec5)\n else:\n reduced_loss = loss.data\n\n losses.update(to_python_float(reduced_loss), input.size(0))\n top1.update(to_python_float(prec1), input.size(0))\n top5.update(to_python_float(prec5), input.size(0))\n\n # measure elapsed time\n batch_time.update(time.time() - end)\n end = time.time()\n\n if args.local_rank == 0 and i % args.print_freq == 0:\n print('Test: [{0}/{1}]\\t'\n 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\\t'\n 'Speed {2:.3f} ({3:.3f})\\t'\n 'Loss {loss.val:.4f} ({loss.avg:.4f})\\t'\n 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\\t'\n 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(\n i, len(val_loader),\n args.world_size * args.batch_size / batch_time.val,\n args.world_size * args.batch_size / batch_time.avg,\n batch_time=batch_time, loss=losses,\n top1=top1, top5=top5))\n\n input, target = prefetcher.next()\n\n print(' * Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}'\n .format(top1=top1, top5=top5))\n\n return top1.avg\n\n\ndef save_checkpoint(state, is_best, filename='checkpoint.pth.tar'):\n torch.save(state, filename)\n if is_best:\n shutil.copyfile(filename, 'model_best.pth.tar')\n\n\nclass AverageMeter(object):\n \"\"\"Computes and stores the average and current value\"\"\"\n def __init__(self):\n self.reset()\n\n def reset(self):\n self.val = 0\n self.avg = 0\n self.sum = 0\n self.count = 0\n\n def update(self, val, n=1):\n self.val = val\n self.sum += val * n\n self.count += n\n self.avg = self.sum / self.count\n\n\ndef adjust_learning_rate(optimizer, epoch, step, len_epoch):\n \"\"\"LR schedule that should yield 76% converged accuracy with batch size 256\"\"\"\n factor = epoch // 30\n\n if epoch >= 80:\n factor = factor + 1\n\n lr = args.lr*(0.1**factor)\n\n \"\"\"Warmup\"\"\"\n if epoch < 5:\n lr = lr*float(1 + step + epoch*len_epoch)/(5.*len_epoch)\n\n # if(args.local_rank == 0):\n # print(\"epoch = {}, step = {}, lr = {}\".format(epoch, step, lr))\n\n for param_group in optimizer.param_groups:\n param_group['lr'] = lr\n\n\ndef accuracy(output, target, topk=(1,)):\n \"\"\"Computes the precision@k for the specified values of k\"\"\"\n maxk = max(topk)\n batch_size = target.size(0)\n\n _, pred = output.topk(maxk, 1, True, True)\n pred = pred.t()\n correct = pred.eq(target.view(1, -1).expand_as(pred))\n\n res = []\n for k in topk:\n correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)\n res.append(correct_k.mul_(100.0 / batch_size))\n return res\n\n\ndef reduce_tensor(tensor):\n rt = tensor.clone()\n dist.all_reduce(rt, op=dist.reduce_op.SUM)\n rt /= args.world_size\n return rt\n\nif __name__ == '__main__':\n main()\n" }, { "path": "apex/tests/L1/common/run_test.sh", "content": "#!/bin/bash\n\nprint_banner() {\n printf \"\\n\\n\\n\\e[30m\\e[42m$1\\e[0m\\n\\n\\n\\n\"\n}\n\nprint_banner \"Distributed status: $1\"\n\necho $2\nDATADIR=$2\n\nif [ -n \"$3\" ]\nthen\n USE_BASELINE=\"\"\nelse\n USE_BASELINE=\"--use_baseline\"\nfi\n\nif [ \"$1\" == \"single_gpu\" ]\nthen\n BASE_CMD=\"python main_amp.py -a resnet50 --b 128 --workers 4 --deterministic --prints-to-process 5\"\nfi\n\nif [ \"$1\" == \"distributed\" ]\nthen\n BASE_CMD=\"python -m torch.distributed.launch --nproc_per_node=2 main_amp.py -a resnet50 --b 128 --workers 4 --deterministic --prints-to-process 5\"\nfi\n\nADAM_ARGS=\"--opt-level O2 --keep-batchnorm-fp32 False --fused-adam\"\n\nkeep_batchnorms=(\n\"\"\n\"--keep-batchnorm-fp32 True\"\n\"--keep-batchnorm-fp32 False\"\n)\n\nloss_scales=(\n\"\"\n\"--loss-scale 1.0\"\n\"--loss-scale 128.0\"\n\"--loss-scale dynamic\"\n)\n\nopt_levels=(\n\"O0\"\n\"O1\"\n\"O2\"\n\"O3\"\n)\n\nrm True*\nrm False*\n\nset -e\n\nprint_banner \"Installing Apex with --cuda_ext and --cpp_ext\"\n\npushd ../../..\npip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" .\npopd\n\nfor opt_level in \"${opt_levels[@]}\"\ndo\n for loss_scale in \"${loss_scales[@]}\"\n do\n for keep_batchnorm in \"${keep_batchnorms[@]}\"\n do\n if [ \"$opt_level\" == \"O1\" ] && [ -n \"${keep_batchnorm}\" ]\n then\n print_banner \"Skipping ${opt_level} ${loss_scale} ${keep_batchnorm}\"\n continue\n fi\n print_banner \"${BASE_CMD} --opt-level ${opt_level} ${loss_scale} ${keep_batchnorm} --has-ext $DATADIR\"\n set -x\n ${BASE_CMD} --opt-level ${opt_level} ${loss_scale} ${keep_batchnorm} --has-ext $DATADIR\n set +x\n done\n done\ndone\n\n# Handle FusedAdam separately due to limited support.\n# FusedAdam will not be tested for bitwise accuracy against the Python implementation.\n# The L0 tests already do so. These tests are here to ensure that it actually runs,\n# and get an idea of performance.\nfor loss_scale in \"${loss_scales[@]}\"\ndo\n print_banner \"${BASE_CMD} ${ADAM_ARGS} ${loss_scale} --has-ext $DATADIR\"\n set -x\n ${BASE_CMD} ${ADAM_ARGS} ${loss_scale} --has-ext $DATADIR\n set +x\ndone\n\nprint_banner \"Reinstalling apex without extensions\"\n\npushd ../../..\npip install -v --no-cache-dir .\npopd\n\nfor opt_level in \"${opt_levels[@]}\"\ndo\n for loss_scale in \"${loss_scales[@]}\"\n do\n for keep_batchnorm in \"${keep_batchnorms[@]}\"\n do\n if [ \"$opt_level\" == \"O1\" ] && [ -n \"${keep_batchnorm}\" ]\n then\n print_banner \"Skipping ${opt_level} ${loss_scale} ${keep_batchnorm}\"\n continue\n fi\n print_banner \"${BASE_CMD} --opt-level ${opt_level} ${loss_scale} ${keep_batchnorm} $DATADIR\"\n set -x\n ${BASE_CMD} --opt-level ${opt_level} ${loss_scale} ${keep_batchnorm} $DATADIR\n set +x\n done\n done\ndone\n\nprint_banner \"Checking for bitwise accuracy between Python-only and cpp/cuda extension installs\"\n\nfor opt_level in \"${opt_levels[@]}\"\ndo\n for loss_scale in \"${loss_scales[@]}\"\n do\n for keep_batchnorm in \"${keep_batchnorms[@]}\"\n do\n echo \"\"\n if [ \"$opt_level\" == \"O1\" ] && [ -n \"${keep_batchnorm}\" ]\n then\n echo \"Skipping ${opt_level} ${loss_scale} ${keep_batchnorm}\"\n continue\n fi\n echo \"${BASE_CMD} --opt-level ${opt_level} ${loss_scale} ${keep_batchnorm} [--has-ext] $DATADIR\"\n set -x\n python compare.py --opt-level ${opt_level} ${loss_scale} ${keep_batchnorm} --use_baseline\n set +x\n done\n done\ndone\n\nprint_banner \"Reinstalling Apex with --cuda_ext and --cpp_ext\"\n\npushd ../../..\npip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" .\npopd\n" }, { "path": "apex/tests/L1/cross_product/run.sh", "content": "#!/bin/bash\n\nDATADIR=\"/home/mcarilli/Desktop/pt18data/apex_stale/examples/imagenet/bare_metal_train_val/\"\n# DATADIR=\"/opt/home/apex/examples/imagenet/\"\ncp ../common/* .\nbash run_test.sh single_gpu $1 $DATADIR yes\n" }, { "path": "apex/tests/L1/cross_product_distributed/run.sh", "content": "#!/bin/bash\n\ncp ../common/* .\nbash run_test.sh distributed $1\n" }, { "path": "apex/tests/distributed/DDP/ddp_race_condition_test.py", "content": "import torch\nimport torch.distributed as dist\nfrom torch.nn import Parameter\nfrom torch.nn import Module\nfrom apex.parallel import DistributedDataParallel as DDP\nimport argparse\nimport os\n\n\nparser = argparse.ArgumentParser(description='allreduce hook example')\nparser.add_argument(\"--local_rank\", default=0, type=int)\nargs = parser.parse_args()\n\nargs.distributed = False\nif 'WORLD_SIZE' in os.environ:\n args.distributed = int(os.environ['WORLD_SIZE']) > 1\n\nif args.distributed:\n args.gpu = args.local_rank % torch.cuda.device_count()\n torch.cuda.set_device(args.gpu)\n torch.distributed.init_process_group(backend='nccl',\n init_method='env://')\n args.world_size = torch.distributed.get_world_size()\n\ntorch.set_printoptions(precision=10)\ntorch.manual_seed(args.local_rank)\n\nclass Model(Module):\n def __init__(self):\n super(Model, self).__init__()\n self.a = Parameter(torch.cuda.FloatTensor(4096*4096).fill_(1.0))\n self.b = Parameter(torch.cuda.FloatTensor(4096*4096).fill_(2.0))\n def forward(self, input):\n return (input*self.a)*self.b\n\nmodel = Model()\n# model = DDP(model, message_size=1, gradient_predivide_factor=8.0)\nmodel = DDP(model, delay_allreduce=True)\n# model = DDP(model, message_size=1, allreduce_trigger_params=[model.b])\n\nx = torch.cuda.FloatTensor(4096*4096)\n\npassed = True\ntorch.cuda.cudart().cudaProfilerStart()\nfor i in range(10):\n x.fill_(i + args.local_rank) # fill x with new values every iteration for sanity\n model.zero_grad()\n out = model(x)\n loss = out.sum()\n # torch.cuda.nvtx.range_push(\"backward\")\n loss.backward()\n # torch.cuda.nvtx.range_pop()\n \n # torch.cuda.nvtx.range_push(\"synchronize() + info\")\n # torch.cuda.synchronize()\n print(\"i = {}\".format(i))\n def info(name, param, val):\n expected = val*4096*4096*(2.*i+1)/2.\n actual = param.grad.data.sum().item()\n print(name+\": grad.data_ptr() = {}, expected sum {}, got {}\".format(\n param.grad.data_ptr(), expected, actual))\n return (expected == actual)\n if not info(\"model.a\", model.module.a, 2.): passed = False\n if not info(\"model.b\", model.module.b, 1.): passed = False\n # torch.cuda.nvtx.range_pop()\ntorch.cuda.cudart().cudaProfilerStop()\n\nprint(\"passed = \", passed)\n" }, { "path": "apex/tests/distributed/DDP/run_race_test.sh", "content": "#!/bin/bash\n\nCUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch --nproc_per_node=2 ddp_race_condition_test.py\n" }, { "path": "apex/tests/distributed/amp_master_params/amp_master_params.py", "content": "import torch\nimport argparse\nimport os\nfrom apex import amp\n# FOR DISTRIBUTED: (can also use torch.nn.parallel.DistributedDataParallel instead)\nfrom apex.parallel import DistributedDataParallel\n\nparser = argparse.ArgumentParser()\n# FOR DISTRIBUTED: Parse for the local_rank argument, which will be supplied\n# automatically by torch.distributed.launch.\nparser.add_argument(\"--local_rank\", default=0, type=int)\nargs = parser.parse_args()\n\n# FOR DISTRIBUTED: If we are running under torch.distributed.launch,\n# the 'WORLD_SIZE' environment variable will also be set automatically.\nargs.distributed = False\nif 'WORLD_SIZE' in os.environ:\n args.distributed = int(os.environ['WORLD_SIZE']) > 1\n\nif args.distributed:\n # FOR DISTRIBUTED: Set the device according to local_rank.\n torch.cuda.set_device(args.local_rank)\n\n # FOR DISTRIBUTED: Initialize the backend. torch.distributed.launch will provide\n # environment variables, and requires that you use init_method=`env://`.\n torch.distributed.init_process_group(backend='nccl',\n init_method='env://')\n\n torch.manual_seed(torch.distributed.get_rank())\n\ntorch.backends.cudnn.benchmark = True\n\nN, D_in, D_out = 64, 1024, 16\n\n# Each process receives its own batch of \"fake input data\" and \"fake target data.\"\n# The \"training loop\" in each process just uses this fake batch over and over.\n# https://github.com/NVIDIA/apex/tree/master/examples/imagenet provides a more realistic\n# example of distributed data sampling for both training and validation.\nx = torch.randn(N, D_in, device='cuda')\ny = torch.randn(N, D_out, device='cuda')\n\nmodel = torch.nn.Linear(D_in, D_out).cuda()\noptimizer = torch.optim.SGD(model.parameters(), lr=1e-3)\n\nmodel, optimizer = amp.initialize(model, optimizer, opt_level=\"O2\")\n\nif args.distributed:\n # FOR DISTRIBUTED: After amp.initialize, wrap the model with\n # apex.parallel.DistributedDataParallel.\n model = DistributedDataParallel(model)\n # torch.nn.parallel.DistributedDataParallel is also fine, with some added args:\n # model = torch.nn.parallel.DistributedDataParallel(model,\n # device_ids=[args.local_rank],\n # output_device=args.local_rank)\n\nloss_fn = torch.nn.MSELoss()\n\nfor t in range(500):\n optimizer.zero_grad()\n y_pred = model(x)\n loss = loss_fn(y_pred, y)\n with amp.scale_loss(loss, optimizer) as scaled_loss:\n scaled_loss.backward()\n optimizer.step()\n\nif args.local_rank == 0:\n print(\"final loss = \", loss)\n\ntorch.save(list(model.parameters()), \"rank{}model.pth\".format(torch.distributed.get_rank()))\ntorch.save(list(amp.master_params(optimizer)), \"rank{}master.pth\".format(torch.distributed.get_rank()))\n" }, { "path": "apex/tests/distributed/amp_master_params/compare.py", "content": "import torch\n\nmodel_params_rank0 = torch.load(\"rank0model.pth\",\n map_location = lambda storage, loc: storage.cuda(0))\nmodel_params_rank1 = torch.load(\"rank1model.pth\",\n map_location = lambda storage, loc: storage.cuda(0))\nmaster_params_rank0 = torch.load(\"rank0master.pth\",\n map_location = lambda storage, loc: storage.cuda(0))\nmaster_params_rank1 = torch.load(\"rank1master.pth\",\n map_location = lambda storage, loc: storage.cuda(0))\n\nfor model_rank0, model_rank1, master_rank0, master_rank1 in zip(\n model_params_rank0,\n model_params_rank1,\n master_params_rank0,\n master_params_rank1):\n assert torch.allclose(model_rank0, model_rank1), \"Model param mismatch\"\n assert torch.allclose(master_rank0, master_rank1), \"Master param mismatch\"\n # Some debugging/investigation assistance code:\n # maxval, maxind = torch.max(((torch.abs(model_rank0).float())/torch.abs(master_rank0)).view(-1), 0)\n # offending_val_half = model_rank0.view(-1)[maxind.item()]\n # offending_val_float = master_rank0.view(-1)[maxind.item()]\n # print(maxval.item(), maxind.item(), offending_val_half.item(), offending_val_float.item(),\n # offending_val_float.half().item())\n # rtol needs to be > 2^-11 because of denormals...\n assert torch.allclose(model_rank0, master_rank0.half(), rtol=.005), \"Model-master mismatch\"\n\nprint(\"OK: Model and master params match across ranks.\")\n" }, { "path": "apex/tests/distributed/amp_master_params/run.sh", "content": "#!/bin/bash\npython -m torch.distributed.launch --nproc_per_node=2 amp_master_params.py\n\npython compare.py\n" }, { "path": "apex/tests/distributed/synced_batchnorm/single_gpu_unit_test.py", "content": "import torch\nimport numpy as np\nimport apex\nif True:\n print(\"using setup tools\")\n import syncbn\nelse:\n print(\"using jit\")\n from torch.utils.cpp_extension import load\n syncbn = load(name='syncbn', sources=['../../csrc/syncbn.cpp', '../../csrc/welford.cu'])\n\ndef compare(desc, inp1, inp2, error):\n a = inp1.clone().detach().cpu().numpy()\n b = inp2.clone().detach().cpu().numpy()\n close = np.allclose(a,b, error, error)\n if not close:\n print(desc, close)\n z = a - b\n index = (np.abs(z) >= error + error * np.abs(b)).nonzero()\n print(\"dif : \", z[index])\n print(\"inp1 : \", a[index])\n print(\"inp2 : \", b[index])\n return close\n\nfeature_size = 10\nspace_size = 16\nbatch_size = 5\n\n\nerror = 1e-5\n\nnp.random.seed(1)\ndtype = np.float32\ninp = (np.random.randn(batch_size, feature_size, space_size, space_size)).astype(dtype)\ngrad = (np.random.randn(batch_size, feature_size, space_size, space_size)).astype(dtype)\nweight = (np.random.randn(feature_size)).astype(dtype)\nbias = (np.random.randn(feature_size)).astype(dtype)\n\ntype_tensor = torch.cuda.FloatTensor\nref_tensor = torch.cuda.DoubleTensor\n\ninp_t = type_tensor(inp)\nweight_t = type_tensor(weight)\nbias_t = type_tensor(bias)\n\ninp_r = ref_tensor(inp.transpose(1, 0, 2, 3).reshape(feature_size, -1))\ninp2_r = ref_tensor(inp)\nweight_r = ref_tensor(weight).view(-1, 1, 1)\nbias_r = ref_tensor(bias).view(-1, 1, 1)\n\ngrad_output_t = type_tensor(grad)\n\nm = inp_r.mean(1)\nb_v = inp_r.var(1, unbiased=False)\nunb_v = inp_r.var(1, unbiased=True)\n\neps = 1e-5\n\n#mean, var, var_biased = syncbn.welford_mean_var(inp_t)\nmean, var_biased = syncbn.welford_mean_var(inp_t)\ninv_std = 1.0 / torch.sqrt(var_biased + eps)\n\nbn = torch.nn.BatchNorm2d(feature_size).cuda()\nbn.momentum = 1.0\nbn.weight.data = weight_t.clone()\nbn.bias.data = bias_t.clone()\ninp_bn = inp_t.clone().requires_grad_()\ngrad_bn = grad_output_t.clone().detach()\nout_bn = bn(inp_bn)\nout_bn.backward(grad_bn)\n\nsbn = apex.parallel.SyncBatchNorm(feature_size).cuda()\nsbn.momentum = 1.0\nsbn.weight.data = weight_t.clone()\nsbn.bias.data = bias_t.clone()\ninp_sbn = inp_t.clone().requires_grad_()\ngrad_sbn = grad_output_t.clone().detach()\nout_sbn = sbn(inp_sbn)\nout_sbn.backward(grad_sbn)\n\nsbn_c_last = apex.parallel.SyncBatchNorm(feature_size, channel_last=True).cuda()\nsbn_c_last.momentum = 1.0\nsbn_c_last.weight.data = weight_t.clone()\nsbn_c_last.bias.data = bias_t.clone()\ninp_sbn_c_last = inp_t.clone().transpose(-1, 1).contiguous().requires_grad_()\ngrad_sbn_c_last = grad_output_t.clone().transpose(-1, 1).contiguous().detach()\nout_sbn_c_last = sbn_c_last(inp_sbn_c_last)\nout_sbn_c_last.backward(grad_sbn_c_last)\n\nsbn_result = True\nsbn_result_c_last = True\nbn_result = True\n\nsbn_result = compare(\"comparing mean: \", mean, m, error) and sbn_result\n#sbn_result = compare(\"comparing variance: \", var, unb_v, error) and sbn_result\nsbn_result = compare(\"comparing biased variance: \", var_biased, b_v, error) and sbn_result\n\n\nout = syncbn.batchnorm_forward(inp_t, mean, inv_std, weight_t, bias_t)\nout_r = weight_r * (inp2_r - m.view(-1, 1, 1)) * torch.rsqrt(b_v.view(-1,1,1) + eps) + bias_r\n\nsbn_result = compare(\"comparing output: \", out, out_r, error) and sbn_result\ncompare(\"comparing bn output: \", out_bn, out_r, error)\n\ngrad_output_t = type_tensor(grad)\n\ngrad_output_r = ref_tensor(grad.transpose(1, 0, 2, 3).reshape(feature_size, -1))\ngrad_output2_r = ref_tensor(grad)\n\ngrad_bias_r = grad_output_r.sum(1)\ngrad_weight_r = ((inp2_r - m.view(-1, 1, 1)) * torch.rsqrt(b_v.view(-1,1,1) + eps) * grad_output2_r).transpose(1,0).contiguous().view(feature_size, -1).sum(1)\n\nmean_dy_r = grad_output_r.mean(1)\nmean_dy_xmu_r = ((inp2_r - m.view(-1, 1, 1)) * grad_output2_r).transpose(1,0).contiguous().view(feature_size, -1).mean(1)\n\ngrad_input_r = (grad_output2_r - mean_dy_r.view(-1, 1, 1) - (inp2_r - m.view(-1, 1, 1)) / (b_v.view(-1,1,1) + eps) * mean_dy_xmu_r.view(-1, 1, 1) ) * torch.rsqrt(b_v.view(-1,1,1) + eps) * weight_r.view(-1,1,1)\n\nmean_dy, mean_dy_xmu, grad_weight, grad_bias = syncbn.reduce_bn(grad_output_t, inp_t, mean, inv_std, weight_t)\ngrad_input = syncbn.batchnorm_backward(grad_output_t, inp_t, mean, inv_std, weight_t, mean_dy, mean_dy_xmu)\nsbn_result = compare(\"comparing bias grad: \", grad_bias, grad_bias_r, error) and sbn_result\nsbn_result = compare(\"comparing weight grad: \", grad_weight, grad_weight_r, error) and sbn_result\nsbn_result = compare(\"comparing mean_dy grad: \", mean_dy, mean_dy_r, error) and sbn_result\nsbn_result = compare(\"comparing mean_dy_xmu grad: \", mean_dy_xmu, mean_dy_xmu_r, error) and sbn_result\nsbn_result = compare(\"comparing input grad: \", grad_input, grad_input_r, error) and sbn_result\ncompare(\"comparing bn input grad: \", inp_bn.grad, grad_input_r, error)\nsbn_result = compare(\"comparing sbn input grad: \", inp_sbn.grad, grad_input_r, error) and sbn_result\n\ncompare(\"comparing bn/sbn output: \", out_bn, out_sbn, error)\nsbn_result = compare(\"comparing running_mean: \", bn.running_mean.data, sbn.running_mean.data, error) and sbn_result\nsbn_result = compare(\"comparing running_variance: \", bn.running_var.data, sbn.running_var.data, error) and sbn_result\ncompare(\"comparing grad_input: \", inp_bn.grad, inp_sbn.grad, error)\ncompare(\"comparing grad_bias: \", bn.bias.grad, sbn.bias.grad, error)\ncompare(\"comparing grad_bias bn to ref: \", bn.bias.grad, grad_bias_r, error)\nsbn_result = compare(\"comparing grad_bias sbn to ref: \", sbn.bias.grad, grad_bias_r, error) and sbn_result\ncompare(\"comparing grad_weight: \", bn.weight.grad, sbn.weight.grad, error)\ncompare(\"comparing grad_weight bn to ref: \", bn.weight.grad, grad_weight_r, error)\nsbn_result = compare(\"comparing grad_weight sbn to ref: \", sbn.weight.grad, grad_weight_r, error) and sbn_result\n\ncompare(\"comparing channel last bn/sbn output: \", out_bn, out_sbn_c_last.transpose(-1, 1).contiguous(), error)\nsbn_result_c_last = compare(\"comparing channel last running_mean: \", bn.running_mean.data, sbn_c_last.running_mean.data, error) and sbn_result_c_last\nsbn_result_c_last = compare(\"comparing channel last running_variance: \", bn.running_var.data, sbn_c_last.running_var.data, error) and sbn_result_c_last\ncompare(\"comparing channel last grad_input: \", inp_bn.grad, inp_sbn_c_last.grad.transpose(-1, 1).contiguous(), error)\ncompare(\"comparing channel last grad_bias: \", bn.bias.grad, sbn_c_last.bias.grad, error)\nsbn_result_c_last = compare(\"comparing channel last grad_bias sbn to ref: \", sbn_c_last.bias.grad, grad_bias_r, error) and sbn_result_c_last\ncompare(\"comparing channel last grad_weight: \", bn.weight.grad, sbn_c_last.weight.grad, error)\nsbn_result_c_last = compare(\"comparing channel last grad_weight sbn to ref: \", sbn_c_last.weight.grad, grad_weight_r, error) and sbn_result_c_last\n\nif sbn_result:\n print(\"====SBN single gpu passed tests\")\nelse:\n print(\"*SBN single gpu failed*\")\n\nif sbn_result_c_last:\n print(\"====SBN channel last single gpu passed tests\")\nelse:\n print(\"*SBN channel last single gpu failed*\")\n" }, { "path": "apex/tests/distributed/synced_batchnorm/test_groups.py", "content": "import torch\nimport numpy as np\nimport apex\nimport syncbn\nimport os\nimport argparse\nimport torch.optim as optim\n\ndef compare(desc, inp1, inp2, error):\n a = inp1.clone().detach().cpu().numpy()\n b = inp2.clone().detach().cpu().numpy()\n close = np.allclose(a,b, error, error)\n if not close:\n print(desc, close)\n z = a - b\n index = (np.abs(z) >= error + error * np.abs(b)).nonzero()\n print(\"dif : \", z[index])\n print(\"inp1 : \", a[index])\n print(\"inp2 : \", b[index])\n return close\n\nfeature_size = 10\nspace_size = 40\nbatch_size = 32\n\n\nfrom apex.parallel import DistributedDataParallel as DDP\nparser = argparse.ArgumentParser()\nparser.add_argument(\"--local_rank\", default=0, type=int)\nparser.add_argument(\"--fp16\", action='store_true', default=False)\nparser.add_argument(\"--fp64\", action='store_true', default=False)\nparser.add_argument(\"--group_size\", default=0, type=int)\nargs = parser.parse_args()\n\ntry:\n args.world_size = int(os.environ['WORLD_SIZE'])\nexcept:\n print(\"This is a multi-gpu test. To run it please use 'python -m torch.distributed.launch --nproc_per_node= test_groups.py '\")\n exit(1)\n\ntorch.cuda.set_device(args.local_rank)\ntorch.distributed.init_process_group(backend='nccl', init_method='env://')\n\nstart = (args.local_rank%args.group_size) * batch_size//args.group_size\nfinish = (args.local_rank%args.group_size + 1) * batch_size//args.group_size\n\nerror = 1e-5\ndtype = np.float32\nif args.fp16:\n error = 1e-3\n dtype = np.float16\nelif args.fp64:\n error = 1e-8\n dtype = np.float64\n\n\nnp.random.seed(18 + args.local_rank//args.group_size)\n\ninp = np.random.randn(batch_size, feature_size, space_size, space_size).astype(dtype)\ngrad = np.random.randn(batch_size, feature_size, space_size, space_size).astype(dtype)\nweight = np.random.randn(feature_size).astype(dtype)\nbias = np.random.randn(feature_size).astype(dtype)\n\n\ntype_tensor = torch.cuda.FloatTensor\nif args.fp16:\n type_tensor = torch.cuda.HalfTensor\nif args.fp64:\n type_tensor = torch.cuda.DoubleTensor\n\nref_tensor = torch.cuda.DoubleTensor\n\ninp_t = type_tensor(inp)\nweight_t = type_tensor(weight)\nbias_t = type_tensor(bias)\n\ninp_r = ref_tensor(inp.transpose(1, 0, 2, 3).reshape(feature_size, -1))\ninp2_r = ref_tensor(inp)\nweight_r = ref_tensor(weight).view(-1, 1, 1)\nbias_r = ref_tensor(bias).view(-1, 1, 1)\n\ngrad_output_t = type_tensor(grad)\n\nm = inp_r.mean(1)\nb_v = inp_r.var(1, unbiased=False)\nunb_v = inp_r.var(1, unbiased=True)\n\neps = 1e-5\n\nmean, var_biased = syncbn.welford_mean_var(inp_t)\ninv_std = 1.0 / torch.sqrt(var_biased + eps)\n\nbn = torch.nn.BatchNorm2d(feature_size).cuda()\nbn.momentum = 1.0\nbn.weight.data = weight_t.clone()\nbn.bias.data = bias_t.clone()\nif args.fp16:\n bn.half()\nif args.fp64:\n bn.double()\nbn = DDP(bn)\ninp_bn = inp_t.clone().requires_grad_()\ngrad_bn = grad_output_t.clone().detach()\nout_bn = bn(inp_bn)\nout_bn.backward(grad_bn)\n# compensating the averaging over processes done by DDP\n# in order to produce mathematically equivalent result\n# https://github.com/NVIDIA/apex/issues/134#issuecomment-458307368\nfor param in bn.parameters():\n param.grad = param.grad / args.group_size\nbn_opt = optim.SGD(bn.parameters(), lr=1.0)\n\nsbn = apex.parallel.SyncBatchNorm(feature_size, process_group=apex.parallel.create_syncbn_process_group(args.group_size)).cuda()\nsbn.momentum = 1.0\nsbn.weight.data = weight_t.clone()\nsbn.bias.data = bias_t.clone()\nif args.fp16:\n sbn.half()\nif args.fp64:\n sbn.double()\nsbn = DDP(sbn)\nsbn_opt = optim.SGD(sbn.parameters(), lr=1.0)\ninp_sbn = inp_t.clone().requires_grad_()\ngrad_sbn = grad_output_t.clone().detach()\nout_sbn = sbn(inp_sbn[start:finish])\nout_sbn.backward(grad_sbn[start:finish])\n\nsbn_result = True\nbn_result = True\n\nif args.local_rank == 0:\n sbn_result = compare(\"comparing mean: \", mean, m, error) and sbn_result\n sbn_result = compare(\"comparing biased variance: \", var_biased, b_v, error) and sbn_result\n\nout = syncbn.batchnorm_forward(inp_t, mean, inv_std, weight_t, bias_t)\nout_r = weight_r * (inp2_r - m.view(-1, 1, 1)) * torch.rsqrt(b_v.view(-1,1,1) + eps) + bias_r\n\nif args.local_rank == 0:\n sbn_result = compare(\"comparing output: \", out, out_r, error) and sbn_result\n compare(\"comparing bn output: \", out_bn, out_r, error)\n\ngrad_output_t = type_tensor(grad)\n\ngrad_output_r = ref_tensor(grad.transpose(1, 0, 2, 3).reshape(feature_size, -1))\ngrad_output2_r = ref_tensor(grad)\n\ngrad_bias_r = grad_output_r.sum(1)\ngrad_weight_r = ((inp2_r - m.view(-1, 1, 1)) * torch.rsqrt(b_v.view(-1,1,1) + eps) * grad_output2_r).transpose(1,0).contiguous().view(feature_size, -1).sum(1)\n\nmean_dy_r = grad_output_r.mean(1)\nmean_dy_xmu_r = ((inp2_r - m.view(-1, 1, 1)) * grad_output2_r).transpose(1,0).contiguous().view(feature_size, -1).mean(1)\n\ngrad_input_r = (grad_output2_r - mean_dy_r.view(-1, 1, 1) - (inp2_r - m.view(-1, 1, 1)) / (b_v.view(-1,1,1) + eps) * mean_dy_xmu_r.view(-1, 1, 1) ) * torch.rsqrt(b_v.view(-1,1,1) + eps) * weight_r.view(-1,1,1)\n\nmean_dy, mean_dy_xmu, grad_weight, grad_bias = syncbn.reduce_bn(grad_output_t, inp_t, mean, inv_std, weight_t)\ngrad_input = syncbn.batchnorm_backward(grad_output_t, inp_t, mean, inv_std, weight_t, mean_dy, mean_dy_xmu)\n\nif args.local_rank == 0:\n sbn_result = compare(\"comparing bias grad: \", grad_bias, grad_bias_r, error) and sbn_result\n sbn_result = compare(\"comparing weight grad: \", grad_weight, grad_weight_r, error) and sbn_result\n sbn_result = compare(\"comparing mean_dy grad: \", mean_dy, mean_dy_r, error) and sbn_result\n sbn_result = compare(\"comparing mean_dy_xmu grad: \", mean_dy_xmu, mean_dy_xmu_r, error) and sbn_result\n sbn_result = compare(\"comparing input grad: \", grad_input, grad_input_r, error) and sbn_result\n compare(\"comparing bn input grad: \", inp_bn.grad, grad_input_r, error)\n\nif args.local_rank == 0:\n sbn_result = compare(\"comparing running_mean: \", bn.module.running_mean.data, sbn.module.running_mean.data, error) and sbn_result\n sbn_result = compare(\"comparing running_variance: \", bn.module.running_var.data, sbn.module.running_var.data, error) and sbn_result\n\n# execute by both\ncompare(\"comparing layers output: \", out_bn[start:finish], out_sbn, error) and sbn_result\ncompare(\"comparing layers grad_input: \", inp_bn.grad[start:finish], inp_sbn.grad[start:finish], error) and sbn_result\n\nbn_opt.step()\nsbn_opt.step()\n\nif args.local_rank == 0:\n compare(\"comparing bn vs sbn bias: \", bn.module.bias, sbn.module.bias, error)\n compare(\"comparing bn vs sbn weight: \", bn.module.weight, sbn.module.weight, error)\n\n\nif sbn_result:\n print(\"====SBN group test passed\")\nelse:\n print(\"*SBN group test failed*\")\n" }, { "path": "apex/tests/distributed/synced_batchnorm/two_gpu_unit_test.py", "content": "import torch\nimport numpy as np\nimport apex\nimport syncbn\nimport os\nimport argparse\nimport torch.optim as optim\n\ndef compare(desc, inp1, inp2, error):\n a = inp1.clone().detach().cpu().numpy()\n b = inp2.clone().detach().cpu().numpy()\n close = np.allclose(a,b, error, error)\n if not close:\n print(desc, close)\n z = a - b\n index = (np.abs(z) >= error + error * np.abs(b)).nonzero()\n print(\"dif : \", z[index])\n print(\"inp1 : \", a[index])\n print(\"inp2 : \", b[index])\n return close\n\nfeature_size = 10\nspace_size = 40\nbatch_size = 32\n\n\nfrom apex.parallel import DistributedDataParallel as DDP\nparser = argparse.ArgumentParser()\nparser.add_argument(\"--local_rank\", default=0, type=int)\nparser.add_argument(\"--fp16\", action='store_true', default=False)\nparser.add_argument(\"--fp64\", action='store_true', default=False)\nargs = parser.parse_args()\nargs.world_size = int(os.environ['WORLD_SIZE'])\ntorch.cuda.set_device(args.local_rank)\ntorch.distributed.init_process_group(backend='nccl', init_method='env://')\nstart = args.local_rank * batch_size//args.world_size\nfinish = (args.local_rank + 1) * batch_size//args.world_size\n\nerror = 1e-5\ndtype = np.float32\nif args.fp16:\n error = 1e-3\n dtype = np.float16\nelif args.fp64:\n error = 1e-8\n dtype = np.float64\n\nnp.random.seed(18)\ninp = np.random.randn(batch_size, feature_size, space_size, space_size).astype(dtype)\ngrad = np.random.randn(batch_size, feature_size, space_size, space_size).astype(dtype)\nweight = np.random.randn(feature_size).astype(dtype)\nbias = np.random.randn(feature_size).astype(dtype)\n\n\ntype_tensor = torch.cuda.FloatTensor\nif args.fp16:\n type_tensor = torch.cuda.HalfTensor\nif args.fp64:\n type_tensor = torch.cuda.DoubleTensor\n\nref_tensor = torch.cuda.DoubleTensor\n\ninp_t = type_tensor(inp)\nweight_t = type_tensor(weight)\nbias_t = type_tensor(bias)\n\ninp_r = ref_tensor(inp.transpose(1, 0, 2, 3).reshape(feature_size, -1))\ninp2_r = ref_tensor(inp)\nweight_r = ref_tensor(weight).view(-1, 1, 1)\nbias_r = ref_tensor(bias).view(-1, 1, 1)\n\ngrad_output_t = type_tensor(grad)\n\nm = inp_r.mean(1)\nb_v = inp_r.var(1, unbiased=False)\nunb_v = inp_r.var(1, unbiased=True)\n\neps = 1e-5\n\nmean, var_biased = syncbn.welford_mean_var(inp_t)\ninv_std = 1.0 / torch.sqrt(var_biased + eps)\n\nbn = torch.nn.BatchNorm2d(feature_size).cuda()\nbn.momentum = 1.0\nbn.weight.data = weight_t.clone()\nbn.bias.data = bias_t.clone()\nif args.fp16:\n bn.half()\nif args.fp64:\n bn.double()\ninp_bn = inp_t.clone().requires_grad_()\ngrad_bn = grad_output_t.clone().detach()\nout_bn = bn(inp_bn)\nout_bn.backward(grad_bn)\n# compensating the averaging over processes done by DDP\n# in order to produce mathematically equivalent result\n# https://github.com/NVIDIA/apex/issues/134#issuecomment-458307368\nfor param in bn.parameters():\n param.grad = param.grad / args.world_size\nbn_opt = optim.SGD(bn.parameters(), lr=1.0)\n\nsbn = apex.parallel.SyncBatchNorm(feature_size).cuda()\nsbn.momentum = 1.0\nsbn.weight.data = weight_t.clone()\nsbn.bias.data = bias_t.clone()\nif args.fp16:\n sbn.half()\nif args.fp64:\n sbn.double()\nsbn = DDP(sbn)\nsbn_opt = optim.SGD(sbn.parameters(), lr=1.0)\ninp_sbn = inp_t.clone().requires_grad_()\ngrad_sbn = grad_output_t.clone().detach()\nout_sbn = sbn(inp_sbn[start:finish])\nout_sbn.backward(grad_sbn[start:finish])\n\nsbn_result = True\nbn_result = True\n\nif args.local_rank == 0:\n sbn_result = compare(\"comparing mean: \", mean, m, error) and sbn_result\n sbn_result = compare(\"comparing biased variance: \", var_biased, b_v, error) and sbn_result\n\nout = syncbn.batchnorm_forward(inp_t, mean, inv_std, weight_t, bias_t)\nout_r = weight_r * (inp2_r - m.view(-1, 1, 1)) * torch.rsqrt(b_v.view(-1,1,1) + eps) + bias_r\n\nif args.local_rank == 0:\n sbn_result = compare(\"comparing output: \", out, out_r, error) and sbn_result\n compare(\"comparing bn output: \", out_bn, out_r, error)\n\ngrad_output_t = type_tensor(grad)\n\ngrad_output_r = ref_tensor(grad.transpose(1, 0, 2, 3).reshape(feature_size, -1))\ngrad_output2_r = ref_tensor(grad)\n\ngrad_bias_r = grad_output_r.sum(1)\ngrad_weight_r = ((inp2_r - m.view(-1, 1, 1)) * torch.rsqrt(b_v.view(-1,1,1) + eps) * grad_output2_r).transpose(1,0).contiguous().view(feature_size, -1).sum(1)\n\nmean_dy_r = grad_output_r.mean(1)\nmean_dy_xmu_r = ((inp2_r - m.view(-1, 1, 1)) * grad_output2_r).transpose(1,0).contiguous().view(feature_size, -1).mean(1)\n\ngrad_input_r = (grad_output2_r - mean_dy_r.view(-1, 1, 1) - (inp2_r - m.view(-1, 1, 1)) / (b_v.view(-1,1,1) + eps) * mean_dy_xmu_r.view(-1, 1, 1) ) * torch.rsqrt(b_v.view(-1,1,1) + eps) * weight_r.view(-1,1,1)\n\nmean_dy, mean_dy_xmu, grad_weight, grad_bias = syncbn.reduce_bn(grad_output_t, inp_t, mean, inv_std, weight_t)\ngrad_input = syncbn.batchnorm_backward(grad_output_t, inp_t, mean, inv_std, weight_t, mean_dy, mean_dy_xmu)\nif args.local_rank == 0:\n sbn_result = compare(\"comparing bias grad: \", grad_bias, grad_bias_r, error) and sbn_result\n sbn_result = compare(\"comparing weight grad: \", grad_weight, grad_weight_r, error) and sbn_result\n sbn_result = compare(\"comparing mean_dy grad: \", mean_dy, mean_dy_r, error) and sbn_result\n sbn_result = compare(\"comparing mean_dy_xmu grad: \", mean_dy_xmu, mean_dy_xmu_r, error) and sbn_result\n sbn_result = compare(\"comparing input grad: \", grad_input, grad_input_r, error) and sbn_result\n compare(\"comparing bn input grad: \", inp_bn.grad, grad_input_r, error)\n\nif args.local_rank == 0:\n sbn_result = compare(\"comparing running_mean: \", bn.running_mean.data, sbn.module.running_mean.data, error) and sbn_result\n sbn_result = compare(\"comparing running_variance: \", bn.running_var.data, sbn.module.running_var.data, error) and sbn_result\n\n# execute by both\ncompare(\"comparing layers output: \", out_bn[start:finish], out_sbn, error) and sbn_result\ncompare(\"comparing layers grad_input: \", inp_bn.grad[start:finish], inp_sbn.grad[start:finish], error) and sbn_result\n\nbn_opt.step()\nsbn_opt.step()\n\nif args.local_rank == 0:\n compare(\"comparing bn vs sbn bias: \", bn.bias, sbn.module.bias, error)\n compare(\"comparing bn vs sbn weight: \", bn.weight, sbn.module.weight, error)\n\n\nif sbn_result:\n print(\"====SBN two gpu passed tests\")\nelse:\n print(\"*SBN two gpu failed*\")\n" }, { "path": "apex/tests/distributed/synced_batchnorm/unit_test.sh", "content": "python single_gpu_unit_test.py\npython -m torch.distributed.launch --nproc_per_node=2 two_gpu_unit_test.py\npython -m torch.distributed.launch --nproc_per_node=2 two_gpu_unit_test.py --fp64\n#beware, you need a system with at least 4 gpus to test group_sizeNotebook Settings to set this.\\n\",\n \"\\n\",\n \"CoLab may first assign you a lower memory machine if you are using a hosted runtime. If so, the first time you try to load the 5B model, it will run out of memory, and then you'll be prompted to restart with more memory (then return to the top of this CoLab). If you continue to have memory issues after this (or run into issues on your own home setup), switch to the 1B model.\\n\",\n \"\\n\",\n \"If you are using a local GPU, we recommend V100 or P100 with 16GB GPU memory for best performance. For GPU’s with less memory, we recommend using the 1B model and a smaller batch size throughout. \\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"8qEqdj8u0gdN\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"!nvidia-smi\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"taDHgk1WCC_C\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"import jukebox\\n\",\n \"import torch as t\\n\",\n \"import librosa\\n\",\n \"import os\\n\",\n \"from IPython.display import Audio\\n\",\n \"from jukebox.make_models import make_vqvae, make_prior, MODELS, make_model\\n\",\n \"from jukebox.hparams import Hyperparams, setup_hparams\\n\",\n \"from jukebox.sample import sample_single_window, _sample, \\\\\\n\",\n \" sample_partial_window, upsample\\n\",\n \"from jukebox.utils.dist_utils import setup_dist_from_mpi\\n\",\n \"from jukebox.utils.torch_utils import empty_cache\\n\",\n \"rank, local_rank, device = setup_dist_from_mpi()\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"89FftI5kc-Az\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"# Sample from the 5B or 1B Lyrics Model\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"65aR2OZxmfzq\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"model = \\\"5b_lyrics\\\" # or \\\"1b_lyrics\\\" \\n\",\n \"hps = Hyperparams()\\n\",\n \"hps.sr = 44100\\n\",\n \"hps.n_samples = 3 if model=='5b_lyrics' else 8\\n\",\n \"hps.name = 'samples'\\n\",\n \"chunk_size = 16 if model==\\\"5b_lyrics\\\" else 32\\n\",\n \"max_batch_size = 3 if model==\\\"5b_lyrics\\\" else 16\\n\",\n \"hps.levels = 3\\n\",\n \"hps.hop_fraction = [.5,.5,.125]\\n\",\n \"\\n\",\n \"vqvae, *priors = MODELS[model]\\n\",\n \"vqvae = make_vqvae(setup_hparams(vqvae, dict(sample_length = 1048576)), device)\\n\",\n \"top_prior = make_prior(setup_hparams(priors[-1], dict()), vqvae, device)\\n\",\n \"\\n\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"JYKiwkzy0Iyf\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"Specify your choice of artist, genre, lyrics, and length of musical sample. \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"-sY9aGHcZP-u\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"sample_length_in_seconds = 60 # Full length of musical sample to generate - we find songs in the 1 to 4 minute\\n\",\n \" # range work well, with generation time proportional to sample length. \\n\",\n \" # This total length affects how quickly the model \\n\",\n \" # progresses through lyrics (model also generates differently\\n\",\n \" # depending on if it thinks it's in the beginning, middle, or end of sample)\\n\",\n \"\\n\",\n \"hps.sample_length = (int(sample_length_in_seconds*hps.sr)//top_prior.raw_to_tokens)*top_prior.raw_to_tokens\\n\",\n \"assert hps.sample_length >= top_prior.n_ctx*top_prior.raw_to_tokens, f'Please choose a larger sampling rate'\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"colab_type\": \"code\",\n \"id\": \"qD0qxQeLaTR0\",\n \"colab\": {}\n },\n \"source\": [\n \"metas = [dict(artist = \\\"Zac Brown Band\\\",\\n\",\n \" genre = \\\"Country\\\",\\n\",\n \" total_length = hps.sample_length,\\n\",\n \" offset = 0,\\n\",\n \" lyrics = \\\"\\\"\\\"I met a traveller from an antique land,\\n\",\n \" Who said—“Two vast and trunkless legs of stone\\n\",\n \" Stand in the desert. . . . Near them, on the sand,\\n\",\n \" Half sunk a shattered visage lies, whose frown,\\n\",\n \" And wrinkled lip, and sneer of cold command,\\n\",\n \" Tell that its sculptor well those passions read\\n\",\n \" Which yet survive, stamped on these lifeless things,\\n\",\n \" The hand that mocked them, and the heart that fed;\\n\",\n \" And on the pedestal, these words appear:\\n\",\n \" My name is Ozymandias, King of Kings;\\n\",\n \" Look on my Works, ye Mighty, and despair!\\n\",\n \" Nothing beside remains. Round the decay\\n\",\n \" Of that colossal Wreck, boundless and bare\\n\",\n \" The lone and level sands stretch far away\\n\",\n \" \\\"\\\"\\\",\\n\",\n \" ),\\n\",\n \" ] * hps.n_samples\\n\",\n \"labels = [None, None, top_prior.labeller.get_batch_labels(metas, 'cuda')]\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"6PHC1XnEfV4Y\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"Optionally adjust the sampling temperature (we've found .98 or .99 to be our favorite). \\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"colab_type\": \"code\",\n \"id\": \"eNwKyqYraTR9\",\n \"colab\": {}\n },\n \"source\": [\n \"sampling_temperature = .98\\n\",\n \"\\n\",\n \"lower_batch_size = 16\\n\",\n \"max_batch_size = 3 if model == \\\"5b_lyrics\\\" else 16\\n\",\n \"lower_level_chunk_size = 32\\n\",\n \"chunk_size = 16 if model == \\\"5b_lyrics\\\" else 32\\n\",\n \"sampling_kwargs = [dict(temp=.99, fp16=True, max_batch_size=lower_batch_size,\\n\",\n \" chunk_size=lower_level_chunk_size),\\n\",\n \" dict(temp=0.99, fp16=True, max_batch_size=lower_batch_size,\\n\",\n \" chunk_size=lower_level_chunk_size),\\n\",\n \" dict(temp=sampling_temperature, fp16=True, \\n\",\n \" max_batch_size=max_batch_size, chunk_size=chunk_size)]\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"S3j0gT3HfrRD\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"Now we're ready to sample from the model. We'll generate the top level (2) first, followed by the first upsampling (level 1), and the second upsampling (0). In this CoLab we load the top prior separately from the upsamplers, because of memory concerns on the hosted runtimes. If you are using a local machine, you can also load all models directly with make_models, and then use sample.py's ancestral_sampling to put this all in one step.\\n\",\n \"\\n\",\n \"After each level, we decode to raw audio and save the audio files. \\n\",\n \"\\n\",\n \"This next cell will take a while (approximately 10 minutes per 20 seconds of music sample)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"2nET_YBEopyp\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"zs = [t.zeros(hps.n_samples,0,dtype=t.long, device='cuda') for _ in range(len(priors))]\\n\",\n \"zs = _sample(zs, labels, sampling_kwargs, [None, None, top_prior], [2], hps)\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"-gxY9aqHqfLJ\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"Listen to the results from the top level (note this will sound very noisy until we do the upsampling stage). You may have more generated samples, depending on the batch size you requested.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"TPZENDGZqOOb\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"Audio(f'{hps.name}/level_2/item_0.wav')\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"EJc3bQxmusc6\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"We are now done with the large top_prior model, and instead load the upsamplers.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"W5VLX0zRapIm\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"# Set this False if you are on a local machine that has enough memory (this allows you to do the\\n\",\n \"# lyrics alignment visualization during the upsampling stage). For a hosted runtime, \\n\",\n \"# we'll need to go ahead and delete the top_prior if you are using the 5b_lyrics model.\\n\",\n \"if True:\\n\",\n \" del top_prior\\n\",\n \" empty_cache()\\n\",\n \" top_prior=None\\n\",\n \"upsamplers = [make_prior(setup_hparams(prior, dict()), vqvae, 'cpu') for prior in priors[:-1]]\\n\",\n \"labels[:2] = [prior.labeller.get_batch_labels(metas, 'cuda') for prior in upsamplers]\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"eH_jUhGDprAt\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"Please note: this next upsampling step will take several hours. At the free tier, Google CoLab lets you run for 12 hours. As the upsampling is completed, samples will appear in the Files tab (you can access this at the left of the CoLab), under \\\"samples\\\" (or whatever hps.name is currently). Level 1 is the partially upsampled version, and then Level 0 is fully completed.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"9lkJgLolpZ6w\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"zs = upsample(zs, labels, sampling_kwargs, [*upsamplers, top_prior], hps)\\n\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"3SJgBYJPri55\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"Listen to your final sample!\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"2ip2PPE0rgAb\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"Audio(f'{hps.name}/level_0/item_0.wav')\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"8JAgFxytwrLG\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"del upsamplers\\n\",\n \"empty_cache()\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"LpvvFH85bbBC\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"# Co-Composing with the 5B or 1B Lyrics Model\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"nFDROuS7gFQY\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"For more control over the generations, try co-composing with either the 5B or 1B Lyrics Models. Again, specify your artist, genre, and lyrics. However, now instead of generating the entire sample, the model will return 3 short options for the opening of the piece (or up to 16 options if you use the 1B model instead). Choose your favorite, and then continue the loop, for as long as you like. Throughout these steps, you'll be listening to the audio at the top prior level, which means it will sound quite noisy. When you are satisfied with your co-creation, continue on through the upsampling section. This will render the piece in higher audio quality.\\n\",\n \"\\n\",\n \"NOTE: CoLab will first assign you a lower memory machine if you are using a hosted runtime. The next cell will run out of memory, and then you'll be prompted to restart with more memory (then return to the top of this CoLab). If you continue to have memory issues after this (or run into issues on your own home setup), switch to the 1B model. \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"3y-q8ifhGBlU\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"model = \\\"5b_lyrics\\\" # or \\\"1b_lyrics\\\"\\n\",\n \"hps = Hyperparams()\\n\",\n \"hps.sr = 44100\\n\",\n \"hps.n_samples = 3 if model=='5b_lyrics' else 16\\n\",\n \"hps.name = 'co_composer'\\n\",\n \"hps.sample_length = 1048576 if model==\\\"5b_lyrics\\\" else 786432 \\n\",\n \"chunk_size = 16 if model==\\\"5b_lyrics\\\" else 32\\n\",\n \"max_batch_size = 3 if model==\\\"5b_lyrics\\\" else 16\\n\",\n \"hps.hop_fraction = [.5, .5, .125] \\n\",\n \"hps.levels = 3\\n\",\n \"\\n\",\n \"vqvae, *priors = MODELS[model]\\n\",\n \"vqvae = make_vqvae(setup_hparams(vqvae, dict(sample_length = hps.sample_length)), device)\\n\",\n \"top_prior = make_prior(setup_hparams(priors[-1], dict()), vqvae, device)\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"68hz4x7igq0c\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"Choose your artist, genre, and lyrics here!\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"QDMvH_1zUHo6\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"total_sample_length_in_seconds = 120\\n\",\n \"metas = [dict(artist = \\\"Zac Brown Band\\\",\\n\",\n \" genre = \\\"Country\\\",\\n\",\n \" total_length = total_sample_length_in_seconds * hps.sr,\\n\",\n \" offset = 0,\\n\",\n \" lyrics = \\\"\\\"\\\"I met a traveller from an antique land,\\n\",\n \" Who said—“Two vast and trunkless legs of stone\\n\",\n \" Stand in the desert. . . . Near them, on the sand,\\n\",\n \" Half sunk a shattered visage lies, whose frown,\\n\",\n \" And wrinkled lip, and sneer of cold command,\\n\",\n \" Tell that its sculptor well those passions read\\n\",\n \" Which yet survive, stamped on these lifeless things,\\n\",\n \" The hand that mocked them, and the heart that fed;\\n\",\n \" And on the pedestal, these words appear:\\n\",\n \" My name is Ozymandias, King of Kings;\\n\",\n \" Look on my Works, ye Mighty, and despair!\\n\",\n \" Nothing beside remains. Round the decay\\n\",\n \" Of that colossal Wreck, boundless and bare\\n\",\n \" The lone and level sands stretch far away\\n\",\n \" \\\"\\\"\\\",\\n\",\n \" ),\\n\",\n \" ] * hps.n_samples\\n\",\n \"labels = top_prior.labeller.get_batch_labels(metas, 'cuda')\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"B9onZMEXh34f\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"## Generate 3 options for the start of the song\\n\",\n \"\\n\",\n \"Initial generation is set to be 4 seconds long, but feel free to change this\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"c6peEj8I_HHO\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"def seconds_to_tokens(sec, sr, prior, chunk_size):\\n\",\n \" tokens = sec * hps.sr // prior.raw_to_tokens\\n\",\n \" tokens = ((tokens // chunk_size) + 1) * chunk_size\\n\",\n \" assert tokens <= prior.n_ctx, 'Choose a shorter generation length to stay within the top prior context'\\n\",\n \" return tokens\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"2gn2GXt3zt3y\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"initial_generation_in_seconds = 4\\n\",\n \"tokens_to_sample = seconds_to_tokens(initial_generation_in_seconds, hps.sr, top_prior, chunk_size)\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"U0zcWcMoiigl\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"Change the sampling temperature if you like (higher is more random). Our favorite is in the range .98 to .995\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"NHbH68H7VMeO\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"sampling_temperature = .98\\n\",\n \"sampling_kwargs = dict(temp=sampling_temperature, fp16=True,\\n\",\n \" max_batch_size=max_batch_size, chunk_size=chunk_size)\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"JGZEPe-WTt4g\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"zs=[t.zeros(hps.n_samples,0,dtype=t.long, device='cuda') for _ in range(3)]\\n\",\n \"zs=sample_partial_window(zs, labels, sampling_kwargs, 2, top_prior, tokens_to_sample, hps)\\n\",\n \"x = vqvae.decode(zs[2:], start_level=2).cpu().numpy()\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"mveN4Be8jK2J\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"Listen to your generated samples, and then pick a favorite. If you don't like any, go back and rerun the cell above. \\n\",\n \"\\n\",\n \"** NOTE this is at the noisy top level, upsample fully (in the next section) to hear the final audio version\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"colab_type\": \"code\",\n \"id\": \"LrJSGMhUOhZg\",\n \"colab\": {}\n },\n \"source\": [\n \"for i in range(hps.n_samples):\\n\",\n \" librosa.output.write_wav(f'noisy_top_level_generation_{i}.wav', x[i], sr=44100)\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"colab_type\": \"code\",\n \"id\": \"rQ4ersQ5OhZr\",\n \"colab\": {}\n },\n \"source\": [\n \"Audio('noisy_top_level_generation_0.wav')\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"colab_type\": \"code\",\n \"id\": \"-GdqzrGkOhZv\",\n \"colab\": {}\n },\n \"source\": [\n \"Audio('noisy_top_level_generation_1.wav')\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"colab_type\": \"code\",\n \"id\": \"gE5S8hyZOhZy\",\n \"colab\": {}\n },\n \"source\": [\n \"Audio('noisy_top_level_generation_2.wav')\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"t2-mEJaqZfuS\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"If you don't like any of the options, return a few cells back to \\\"Sample a few options...\\\" and rerun from there.\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"o7CzSiv0MmFP\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"## Choose your favorite sample and request longer generation\\n\",\n \"\\n\",\n \"---\\n\",\n \"\\n\",\n \"(Repeat from here)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"j_XFtVi99CIY\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"my_choice=0\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"Pgk3sHHBLYoq\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"zs[2]=zs[2][my_choice].repeat(hps.n_samples,1)\\n\",\n \"t.save(zs, 'zs-checkpoint2.t')\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"W8Rd9xxm565S\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"# Set to True to load the previous checkpoint:\\n\",\n \"if False:\\n\",\n \" zs=t.load('zs-checkpoint2.t') \"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"k12xjMgHkRGP\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"Choose the length of the continuation. The 1B model can generate up to 17 second samples and the 5B up to 23 seconds, but you'll want to pick a shorter continuation length so that it will be able to look back at what you've generated already. Here we've chosen 4 seconds.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"h3_-0a07kHHG\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"continue_generation_in_seconds=4\\n\",\n \"tokens_to_sample = seconds_to_tokens(continue_generation_in_seconds, hps.sr, top_prior, chunk_size)\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"GpPG3Ifqk8ue\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"The next step asks the top prior to generate more of the sample. It'll take up to a few minutes, depending on the sample length you request.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"YoHkeSTaEyLj\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"zs = sample_partial_window(zs, labels, sampling_kwargs, 2, top_prior, tokens_to_sample, hps)\\n\",\n \"x = vqvae.decode(zs[2:], start_level=2).cpu().numpy()\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"ymhUqEdhleEi\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"Now listen to the longer versions of the sample you selected, and again choose a favorite sample. If you don't like any, return back to the cell where you can load the checkpoint, and continue again from there.\\n\",\n \"\\n\",\n \"When the samples start getting long, you might not always want to listen from the start, so change the playback start time later on if you like.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"2H1LNLTa_R6a\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"playback_start_time_in_seconds = 0 \"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"r4SBGAmsnJtH\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"for i in range(hps.n_samples):\\n\",\n \" librosa.output.write_wav(f'top_level_continuation_{i}.wav', x[i][playback_start_time_in_seconds*44100:], sr=44100)\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"2WeyE5Qtnmeo\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"Audio('top_level_continuation_0.wav')\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"BKtfEtcaazXE\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"Audio('top_level_continuation_1.wav')\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"7yrlS0XwK2S0\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"Audio('top_level_continuation_2.wav')\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"-OJT704dvnGv\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"To make a longer song, return back to \\\"Choose your favorite sample\\\" and loop through that again\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"RzCrkCZJvUcQ\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"# Upsample Co-Composition to Higher Audio Quality\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"4MPgukwMmB0p\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"Choose your favorite sample from your latest group of generations. (If you haven't already gone through the Co-Composition block, make sure to do that first so you have a generation to upsample).\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"yv-pNNPHBQYC\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"choice = 0\\n\",\n \"select_best_sample = True # Set false if you want to upsample all your samples \\n\",\n \" # upsampling sometimes yields subtly different results on multiple runs,\\n\",\n \" # so this way you can choose your favorite upsampling\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"v17cEAqyCgfo\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"if select_best_sample:\\n\",\n \" zs[2]=zs[2][choice].repeat(zs[2].shape[0],1)\\n\",\n \"\\n\",\n \"t.save(zs, 'zs-top-level-final.t')\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"0YjK-Ac0tBfu\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"Note: If you are using a CoLab hosted runtime on the free tier, you may want to download this zs-top-level-final.t file, and then restart an instance and load it in the next cell. The free tier will last a maximum of 12 hours, and the upsampling stage can take many hours, depending on how long a sample you have generated.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"qqlR9368s3jJ\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"if False:\\n\",\n \" zs = t.load('zs-top-level-final.t')\\n\",\n \"\\n\",\n \"assert zs[2].shape[1]>=2048, f'Please first generate at least 2048 tokens at the top level, currently you have {zs[2].shape[1]}'\\n\",\n \"hps.sample_length = zs[2].shape[1]*top_prior.raw_to_tokens\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"jzHwF_iqgIWM\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"# Set this False if you are on a local machine that has enough memory (this allows you to do the\\n\",\n \"# lyrics alignment visualization). For a hosted runtime, we'll need to go ahead and delete the top_prior\\n\",\n \"# if you are using the 5b_lyrics model.\\n\",\n \"if True:\\n\",\n \" del top_prior\\n\",\n \" empty_cache()\\n\",\n \" top_prior=None\\n\",\n \"\\n\",\n \"upsamplers = [make_prior(setup_hparams(prior, dict()), vqvae, 'cpu') for prior in priors[:-1]]\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"q22Ier6YSkKS\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"sampling_kwargs = [dict(temp=.99, fp16=True, max_batch_size=16, chunk_size=32),\\n\",\n \" dict(temp=0.99, fp16=True, max_batch_size=16, chunk_size=32),\\n\",\n \" None]\\n\",\n \"\\n\",\n \"if type(labels)==dict:\\n\",\n \" labels = [prior.labeller.get_batch_labels(metas, 'cuda') for prior in upsamplers] + [labels] \"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"T1MCa9_jnjpf\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"This next step upsamples 2 levels. The level_1 samples will be available after around one hour (depending on the length of your sample) and are saved under {hps.name}/level_0/item_0.wav, while the fully upsampled level_0 will likely take 4-12 hours. You can access the wav files down below, or using the \\\"Files\\\" panel at the left of this CoLab.\\n\",\n \"\\n\",\n \"(Please note, if you are using this CoLab on Google's free tier, you may want to download intermediate steps as the connection will last for a maximum 12 hours.)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"NcNT5qIRMmHq\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"zs = upsample(zs, labels, sampling_kwargs, [*upsamplers, top_prior], hps)\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"W2jTYLPBc29M\",\n \"colab_type\": \"code\",\n \"colab\": {}\n },\n \"source\": [\n \"Audio(f'{hps.name}/level_0/item_0.wav')\"\n ],\n \"execution_count\": 0,\n \"outputs\": []\n }\n ]\n}" }, { "path": "jukebox/__init__.py", "content": "" }, { "path": "jukebox/align.py", "content": "\"\"\"\nGet alignment from attn values\n1. run a forward pass on each hop, get attn values\n2. concat for all hops\n\"\"\"\nimport numpy as np\nimport torch as t\nfrom jukebox.utils.torch_utils import assert_shape, empty_cache\nfrom jukebox.hparams import Hyperparams\nfrom jukebox.make_models import make_model\nfrom jukebox.save_html import save_html\nfrom jukebox.utils.sample_utils import get_starts\nimport fire\n\ndef get_alignment(x, zs, labels, prior, fp16, hps):\n level = hps.levels - 1 # Top level used\n n_ctx, n_tokens = prior.n_ctx, prior.n_tokens\n z = zs[level]\n bs, total_length = z.shape[0], z.shape[1]\n if total_length < n_ctx:\n padding_length = n_ctx - total_length\n z = t.cat([z, t.zeros(bs, n_ctx - total_length, dtype=z.dtype, device=z.device)], dim=1)\n total_length = z.shape[1]\n else:\n padding_length = 0\n\n hop_length = int(hps.hop_fraction[level]*prior.n_ctx)\n n_head = prior.prior.transformer.n_head\n alignment_head, alignment_layer = prior.alignment_head, prior.alignment_layer\n attn_layers = set([alignment_layer])\n alignment_hops = {}\n indices_hops = {}\n\n prior.cuda()\n empty_cache()\n for start in get_starts(total_length, n_ctx, hop_length):\n end = start + n_ctx\n\n # set y offset, sample_length and lyrics tokens\n y, indices_hop = prior.get_y(labels, start, get_indices=True)\n assert len(indices_hop) == bs\n for indices in indices_hop:\n assert len(indices) == n_tokens\n\n z_bs = t.chunk(z, bs, dim=0)\n y_bs = t.chunk(y, bs, dim=0)\n w_hops = []\n for z_i, y_i in zip(z_bs, y_bs):\n w_hop = prior.z_forward(z_i[:,start:end], [], y_i, fp16=fp16, get_attn_weights=attn_layers)\n assert len(w_hop) == 1\n w_hops.append(w_hop[0][:, alignment_head])\n del w_hop\n w = t.cat(w_hops, dim=0)\n del w_hops\n assert_shape(w, (bs, n_ctx, n_tokens))\n alignment_hop = w.float().cpu().numpy()\n assert_shape(alignment_hop, (bs, n_ctx, n_tokens))\n del w\n\n # alignment_hop has shape (bs, n_ctx, n_tokens)\n # indices_hop is a list of len=bs, each entry of len hps.n_tokens\n indices_hops[start] = indices_hop\n alignment_hops[start] = alignment_hop\n prior.cpu()\n empty_cache()\n\n # Combine attn for each hop into attn for full range\n # Use indices to place them into correct place for corresponding source tokens\n alignments = []\n for item in range(bs):\n # Note each item has different length lyrics\n full_tokens = labels['info'][item]['full_tokens']\n alignment = np.zeros((total_length, len(full_tokens) + 1))\n for start in reversed(get_starts(total_length, n_ctx, hop_length)):\n end = start + n_ctx\n alignment_hop = alignment_hops[start][item]\n indices = indices_hops[start][item]\n assert len(indices) == n_tokens\n assert alignment_hop.shape == (n_ctx, n_tokens)\n alignment[start:end,indices] = alignment_hop\n alignment = alignment[:total_length - padding_length,:-1] # remove token padding, and last lyric index\n alignments.append(alignment)\n return alignments\n\ndef save_alignment(model, device, hps):\n print(hps)\n vqvae, priors = make_model(model, device, hps, levels=[-1])\n\n logdir = f\"{hps.logdir}/level_{0}\"\n data = t.load(f\"{logdir}/data.pth.tar\")\n if model == '1b_lyrics':\n fp16 = False\n else:\n fp16 = True\n\n data['alignments'] = get_alignment(data['x'], data['zs'], data['labels'][-1], priors[-1], fp16, hps)\n t.save(data, f\"{logdir}/data_align.pth.tar\")\n save_html(logdir, data['x'], data['zs'], data['labels'][-1], data['alignments'], hps)\n\ndef run(model, port=29500, **kwargs):\n from jukebox.utils.dist_utils import setup_dist_from_mpi\n rank, local_rank, device = setup_dist_from_mpi(port=port)\n hps = Hyperparams(**kwargs)\n\n with t.no_grad():\n save_alignment(model, device, hps)\n\nif __name__ == '__main__':\n fire.Fire(run)\n\n\n\n\n\n\n" }, { "path": "jukebox/data/__init__.py", "content": "" }, { "path": "jukebox/data/artist_genre_processor.py", "content": "import os\nimport re\n\naccepted = frozenset([chr(i) for i in range(ord('a'), ord('z') + 1)] +\n [chr(i) for i in range(ord('A'), ord('Z') + 1)] +\n [chr(i) for i in range(ord('0'), ord('9') + 1)])\n\nrex = re.compile(r'_+')\n\ndef norm(s):\n s = ''.join([c if c in accepted else '_' for c in s.lower()])\n s = rex.sub('_', s).strip('_')\n return s\n\ndef create_reverse_lookup(atoi):\n # Multiple entries could go to the same artist_id/genre_id\n itoa = {}\n for a, i in atoi.items():\n if i not in itoa:\n itoa[i] = []\n itoa[i].append(a)\n indices = sorted(list(itoa.keys()))\n for i in indices:\n itoa[i] = '_'.join(sorted(itoa[i]))\n return itoa\n\nclass ArtistGenreProcessor():\n def __init__(self, v3=False):\n self.v3 = v3\n dirname = os.path.dirname(__file__)\n if self.v3:\n self.artist_id_file = f\"{dirname}/ids/v3_artist_ids.txt\"\n self.genre_id_file = f\"{dirname}/ids/v3_genre_ids.txt\"\n else:\n self.artist_id_file = f\"{dirname}/ids/v2_artist_ids.txt\"\n self.genre_id_file = f\"{dirname}/ids/v2_genre_ids.txt\"\n self.load_artists()\n self.load_genres()\n\n def get_artist_id(self, artist):\n input_artist = artist\n if self.v3:\n artist = artist.lower()\n else:\n artist = norm(artist)\n if artist not in self.artist_ids:\n print(f\"Input artist {input_artist} maps to {artist}, which is not present in {self.artist_id_file}. \"\n f\"Defaulting to (artist_id, artist) = (0, unknown), if that seems wrong please format artist correctly\")\n return self.artist_ids.get(artist, 0)\n\n def get_genre_ids(self, genre):\n if self.v3:\n genres = [genre.lower()]\n else:\n # In v2, we convert genre into a bag of words\n genres = norm(genre).split(\"_\")\n for word in genres:\n if word not in self.genre_ids:\n print(f\"Input genre {genre} maps to the list {genres}. {word} is not present in {self.genre_id_file}. \"\n f\"Defaulting to (word_id, word) = (0, unknown), if that seems wrong please format genre correctly\")\n return [self.genre_ids.get(word, 0) for word in genres]\n\n # get_artist/genre throw error if we ask for non-present values\n def get_artist(self, artist_id):\n return self.artists[artist_id]\n\n def get_genre(self, genre_ids):\n if self.v3:\n assert len(genre_ids) == 1\n genre = self.genres[genre_ids[0]]\n else:\n genre = '_'.join([self.genres[genre_id] for genre_id in genre_ids if genre_id >= 0])\n return genre\n\n def load_artists(self):\n print(f'Loading artist IDs from {self.artist_id_file}')\n self.artist_ids = {}\n with open(self.artist_id_file, 'r', encoding=\"utf-8\") as f:\n for line in f:\n artist, artist_id = line.strip().split(';')\n self.artist_ids[artist.lower()] = int(artist_id)\n self.artists = create_reverse_lookup(self.artist_ids)\n\n def load_genres(self):\n print(f'Loading artist IDs from {self.genre_id_file}')\n self.genre_ids = {}\n with open(self.genre_id_file, 'r', encoding=\"utf-8\") as f:\n for line in f:\n genre, genre_id = line.strip().split(';')\n self.genre_ids[genre.lower()] = int(genre_id)\n self.genres = create_reverse_lookup(self.genre_ids)\n\n\n" }, { "path": "jukebox/data/data_processor.py", "content": "import torch as t\nimport jukebox.utils.dist_adapter as dist\nfrom torch.utils.data.distributed import DistributedSampler\nfrom torch.utils.data import DataLoader, Dataset, BatchSampler, RandomSampler\nfrom jukebox.utils.dist_utils import print_all\nfrom jukebox.utils.audio_utils import calculate_bandwidth\nfrom jukebox.data.files_dataset import FilesAudioDataset\n\nclass OffsetDataset(Dataset):\n def __init__(self, dataset, start, end, test=False):\n super().__init__()\n self.dataset = dataset\n self.start = start\n self.end = end\n self.test = test\n assert 0 <= self.start < self.end <= len(self.dataset)\n\n def __len__(self):\n return self.end - self.start\n\n def __getitem__(self, item):\n return self.dataset.get_item(self.start + item, test=self.test)\n\nclass DataProcessor():\n def __init__(self, hps):\n self.dataset = FilesAudioDataset(hps)\n duration = 1 if hps.prior else 600\n hps.bandwidth = calculate_bandwidth(self.dataset, hps, duration=duration)\n self.create_datasets(hps)\n self.create_samplers(hps)\n self.create_data_loaders(hps)\n self.print_stats(hps)\n\n def set_epoch(self, epoch):\n self.train_sampler.set_epoch(epoch)\n self.test_sampler.set_epoch(epoch)\n\n def create_datasets(self, hps):\n train_len = int(len(self.dataset) * hps.train_test_split)\n self.train_dataset = OffsetDataset(self.dataset, 0, train_len, test=False)\n self.test_dataset = OffsetDataset(self.dataset, train_len, len(self.dataset), test=True)\n\n def create_samplers(self, hps):\n if not dist.is_available():\n self.train_sampler = BatchSampler(RandomSampler(self.train_dataset), batch_size=hps.bs, drop_last=True)\n self.test_sampler = BatchSampler(RandomSampler(self.test_dataset), batch_size=hps.bs, drop_last=True)\n else:\n self.train_sampler = DistributedSampler(self.train_dataset)\n self.test_sampler = DistributedSampler(self.test_dataset)\n\n def create_data_loaders(self, hps):\n # Loader to load mini-batches\n if hps.labels:\n collate_fn = lambda batch: tuple(t.stack([t.from_numpy(b[i]) for b in batch], 0) for i in range(2))\n else:\n collate_fn = lambda batch: t.stack([t.from_numpy(b) for b in batch], 0)\n\n print('Creating Data Loader')\n self.train_loader = DataLoader(self.train_dataset, batch_size=hps.bs, num_workers=hps.nworkers,\n sampler=self.train_sampler, pin_memory=False,\n drop_last=True, collate_fn=collate_fn)\n self.test_loader = DataLoader(self.test_dataset, batch_size=hps.bs, num_workers=hps.nworkers,\n sampler=self.test_sampler, pin_memory=False,\n drop_last=False, collate_fn=collate_fn)\n\n def print_stats(self, hps):\n print_all(f\"Train {len(self.train_dataset)} samples. Test {len(self.test_dataset)} samples\")\n print_all(f'Train sampler: {self.train_sampler}')\n print_all(f'Train loader: {len(self.train_loader)}')\n" }, { "path": "jukebox/data/files_dataset.py", "content": "import librosa\nimport math\nimport numpy as np\nimport jukebox.utils.dist_adapter as dist\nfrom torch.utils.data import Dataset\nfrom jukebox.utils.dist_utils import print_all\nfrom jukebox.utils.io import get_duration_sec, load_audio\nfrom jukebox.data.labels import Labeller\n\nclass FilesAudioDataset(Dataset):\n def __init__(self, hps):\n super().__init__()\n self.sr = hps.sr\n self.channels = hps.channels\n self.min_duration = hps.min_duration or math.ceil(hps.sample_length / hps.sr)\n self.max_duration = hps.max_duration or math.inf\n self.sample_length = hps.sample_length\n assert hps.sample_length / hps.sr < self.min_duration, f'Sample length {hps.sample_length} per sr {hps.sr} ({hps.sample_length / hps.sr:.2f}) should be shorter than min duration {self.min_duration}'\n self.aug_shift = hps.aug_shift\n self.labels = hps.labels\n self.init_dataset(hps)\n\n def filter(self, files, durations):\n # Remove files too short or too long\n keep = []\n for i in range(len(files)):\n if durations[i] / self.sr < self.min_duration:\n continue\n if durations[i] / self.sr >= self.max_duration:\n continue\n keep.append(i)\n print_all(f'self.sr={self.sr}, min: {self.min_duration}, max: {self.max_duration}')\n print_all(f\"Keeping {len(keep)} of {len(files)} files\")\n self.files = [files[i] for i in keep]\n self.durations = [int(durations[i]) for i in keep]\n self.cumsum = np.cumsum(self.durations)\n\n def init_dataset(self, hps):\n # Load list of files and starts/durations\n files = librosa.util.find_files(f'{hps.audio_files_dir}', ['mp3', 'opus', 'm4a', 'aac', 'wav'])\n print_all(f\"Found {len(files)} files. Getting durations\")\n cache = dist.get_rank() % 8 == 0 if dist.is_available() else True\n durations = np.array([get_duration_sec(file, cache=cache) * self.sr for file in files]) # Could be approximate\n self.filter(files, durations)\n\n if self.labels:\n self.labeller = Labeller(hps.max_bow_genre_size, hps.n_tokens, self.sample_length, v3=hps.labels_v3)\n\n def get_index_offset(self, item):\n # For a given dataset item and shift, return song index and offset within song\n half_interval = self.sample_length//2\n shift = np.random.randint(-half_interval, half_interval) if self.aug_shift else 0\n offset = item * self.sample_length + shift # Note we centred shifts, so adding now\n midpoint = offset + half_interval\n assert 0 <= midpoint < self.cumsum[-1], f'Midpoint {midpoint} of item beyond total length {self.cumsum[-1]}'\n index = np.searchsorted(self.cumsum, midpoint) # index <-> midpoint of interval lies in this song\n start, end = self.cumsum[index - 1] if index > 0 else 0.0, self.cumsum[index] # start and end of current song\n assert start <= midpoint <= end, f\"Midpoint {midpoint} not inside interval [{start}, {end}] for index {index}\"\n if offset > end - self.sample_length: # Going over song\n offset = max(start, offset - half_interval) # Now should fit\n elif offset < start: # Going under song\n offset = min(end - self.sample_length, offset + half_interval) # Now should fit\n assert start <= offset <= end - self.sample_length, f\"Offset {offset} not in [{start}, {end - self.sample_length}]. End: {end}, SL: {self.sample_length}, Index: {index}\"\n offset = offset - start\n return index, offset\n\n def get_metadata(self, filename, test):\n \"\"\"\n Insert metadata loading code for your dataset here.\n If artist/genre labels are different from provided artist/genre lists,\n update labeller accordingly.\n\n Returns:\n (artist, genre, full_lyrics) of type (str, str, str). For\n example, (\"unknown\", \"classical\", \"\") could be a metadata for a\n piano piece.\n \"\"\"\n return None, None, None\n\n def get_song_chunk(self, index, offset, test=False):\n filename, total_length = self.files[index], self.durations[index]\n data, sr = load_audio(filename, sr=self.sr, offset=offset, duration=self.sample_length)\n assert data.shape == (self.channels, self.sample_length), f'Expected {(self.channels, self.sample_length)}, got {data.shape}'\n if self.labels:\n artist, genre, lyrics = self.get_metadata(filename, test)\n labels = self.labeller.get_label(artist, genre, lyrics, total_length, offset)\n return data.T, labels['y']\n else:\n return data.T\n\n def get_item(self, item, test=False):\n index, offset = self.get_index_offset(item)\n return self.get_song_chunk(index, offset, test)\n\n def __len__(self):\n return int(np.floor(self.cumsum[-1] / self.sample_length))\n\n def __getitem__(self, item):\n return self.get_item(item)\n" }, { "path": "jukebox/data/ids/v2_artist_ids.txt", "content": "unknown;0\nvarious;0\n;0\nandr_s_schiff;1\nsonny_terry;2\nnelly;3\nmarkus_schulz;4\nmodest_petrovich_mussorgsky;5\notis_redding;6\naerosmith;7\nkenny_g;8\njames_taylor;9\nbobby_bland;10\nburning_spear;11\nskip_james;12\nheart;13\ntammy_wynette;14\nmuse;15\nberes_hammond;16\njames_newton_howard;17\nnelson_freire;18\nbenny_goodman;19\nhank_williams;20\nthey_might_be_giants;21\nthe_brian_jonestown_massacre;22\nlady_gaga;23\nchris_young;24\nalison_krauss_union_station;25\nseal;26\nthe_hollies;27\nshabba_ranks;28\npaul_young;29\niration;30\nbuck_owens;31\nthe_weeknd;32\nelton_john;33\nsmokey_robinson;34\nroy_orbison;35\nheadhunterz;36\nblondie;37\nthe_temptations;38\nray_stevens;39\nfoo_fighters;40\nchristoph_eschenbach;41\nblind_willie_mctell;42\nal_martino;43\nedwin_fischer;44\nvictor_young;45\njustin_bieber;46\nstyx;47\ndoris_day;48\ntex_beneke;49\nthe_monkees;50\nrichard_wagner;51\nbryan_adams;52\nalessandro_scarlatti;53\nrebelution;54\npitbull;55\nnat_king_cole;56\nwiz_khalifa;57\nroger_miller;58\nandy_williams;59\npeggy_lee;60\npyotr_ilyich_tchaikovsky;61\nbooker_t_the_mg_s;62\ncilla_black;63\nbilly_fury;64\nvera_lynn;65\nenrico_caruso;66\nsly_and_robbie;67\nthe_pretenders;68\nthe_sweet;69\nkylie_minogue;70\nkay_kyser;71\nsan_francisco_symphony;72\nprince;73\nqueen;74\nkool_the_gang;75\nhorace_andy;76\nmidnite;77\ngentleman;78\nwilhelm_kempff;79\nbusta_rhymes;80\nthe_pogues;81\ndef_leppard;82\nal_jolson;83\nking_tubby;84\nhot_chocolate;85\ndelroy_wilson;86\njody_watley;87\nbobby_vee;88\njohnny_mathis;89\nthe_rascals;90\nsviatoslav_richter;91\nfred_astaire;92\njohn_holt;93\namy_grant;94\nb_b_king;95\npaul_weston;96\nfour_tops;97\njay_sean;98\npat_boone;99\ngeorge_frideric_handel;100\nbing_crosby;101\nshalamar;102\ntommy_dorsey;103\nludacris;104\nkenny_chesney;105\nmurray_perahia;106\nlightnin_hopkins;107\nricky_nelson;108\nclint_mansell;109\ntom_petty_and_the_heartbreakers;110\ngary_glitter;111\nringo_starr;112\nphil_collins;113\nleo_reisman;114\nal_green;115\njim_reeves;116\nchris_brown;117\ncliff_edwards;118\nbuddy_guy;119\nangelo_badalamenti;120\nfrank_sinatra;121\nsantana;122\nthe_pussycat_dolls;123\npeaches_herb;124\nradu_lupu;125\nthe_whispers;126\neddy_howard;127\nmemphis_slim;128\nhenry_mancini;129\ngiuseppe_verdi;130\nthe_dream;131\nvladimir_sofronitsky;132\nu_roy;133\nken_boothe;134\nthe_kinks;135\nhoward_shore;136\nhardwell;137\nlou_reed;138\ncalvin_harris;139\neddy_chen;140\nanne_murray;141\njuice_newton;142\nbee_gees;143\nwilson_pickett;144\nalan_jackson;145\nshirley_bassey;146\nwaylon_jennings;147\ndestiny_s_child;148\ncab_calloway;149\njohnny_copeland;150\nbright_eyes;151\ntrey_songz;152\nneil_sedaka;153\njustin_timberlake;154\narthur_grumiaux;155\nthe_who;156\nthe_yardbirds;157\nbig_joe_turner;158\nduke_ellington;159\nherb_alpert;160\nlaura_branigan;161\nmichael_jackson;162\njohn_denver;163\npeter_gordon;164\nsolomon_cutner;165\nsteve_miller_band;166\ndon_williams;167\nthe_pointer_sisters;168\nmetallica;169\nthe_ink_spots;170\nkenny_rogers;171\nthe_game;172\ngene_krupa;173\nsnoop_dogg;174\nj_cole;175\ntaylor_dayne;176\nr3hab;177\nguy_lombardo_and_his_royal_canadians;178\nshakin_stevens;179\ntim_mcgraw;180\nolivia_newton_john_john_travolta;181\nthe_replacements;182\nbeenie_man;183\ndiplo;184\nsammy_kaye;185\ndon_gibson;186\nshaggy;187\nnina_simone;188\nstone_temple_pilots;189\nne_yo;190\nhuddie_william_ledbetter;191\njerry_goldsmith;192\nthe_bellamy_brothers;193\nwinifred_atwell;194\ngeorg_philipp_telemann;195\ntimbaland;196\nthe_5th_dimension;197\ndionne_warwick;198\nr_kelly;199\nenrique_iglesias;200\nsting;201\nmikey_dread;202\nyellowman;203\neddie_kendricks;204\nblur;205\ntwista;206\npaul_anka;207\nchris_cornell;208\na_ha;209\npet_shop_boys;210\nyo_yo_ma;211\ntom_jones;212\nneil_diamond;213\nvic_damone;214\npaul_oakenfold;215\njascha_heifetz;216\nt_i_;217\ndinah_washington;218\nvladimir_ashkenazy;219\nleif_ove_andsnes;220\njohnny_cash;221\nbasement_jaxx;222\nsonic_youth;223\nthe_isley_brothers;224\njason_aldean;225\nhenryk_szeryng;226\nlloyd_price;227\nreba_mcentire;228\nbon_jovi;229\nbed_ich_smetana;230\ndonny_osmond;231\nchuck_berry;232\nthe_smashing_pumpkins;233\nisrael_vibration;234\nstan_kenton;235\nconway_twitty_loretta_lynn;236\nmcfly;237\nr_e_m;238\nmorgan_heritage;239\nlil_wayne;240\ngarnett_silk;241\nlee_scratch_perry;242\nhowlin_wolf;243\njon_secada;244\ngoo_goo_dolls;245\nbrian_hyland;246\nabc;247\nclarence_gatemouth_brown;248\nwilson_phillips;249\natlantic_starr;250\ndavid_guetta;251\ns_rgio_mendes;252\nfr_d_ric_chopin;253\nrobert_palmer;254\ncharlie_musselwhite;255\nlevel_42;256\npeter_andre;257\nthe_spinners;258\nerasure;259\nphilippe_entremont;260\nleo_jan_ek;261\nthe_kingston_trio;262\nronnie_milsap;263\npat_benatar;264\nrobert_casadesus;265\nt_bone_walker;266\nduran_duran;267\njerry_reed;268\nfranz_liszt;269\njourney;270\nsteve_angello;271\nshowaddywaddy;272\ntears_for_fears;273\njohn_williams;274\ndaniel_m_ller_schott;275\ntampa_red;276\ntina_turner;277\nthe_beatles;278\nmiranda_lambert;279\nhoward_jones;280\nreo_speedwagon;281\nlady_antebellum;282\nno_doubt;283\nstatus_quo;284\ntiffany;285\nbilly_eckstine;286\ndanny_elfman;287\njimmy_wakely;288\nike_tina_turner;289\ngeorge_gershwin;290\ndinah_shore;291\narmin_van_buuren;292\nkeith_sweat;293\nbeaux_arts_trio;294\narcangelo_corelli;295\nair_supply;296\nw_w;297\nthe_mighty_diamonds;298\nharry_gregson_williams;299\nblind_lemon_jefferson;300\nbrook_benton;301\nsizzla;302\nbobby_darin;303\nsteely_dan;304\nthe_skatalites;305\ndwight_yoakam;306\nclara_haskil;307\njah_cure;308\ncliff_richard;309\nfabolous;310\nsonny_boy_williamson_ii;311\ntinie_tempah;312\nclaude_debussy;313\njewel;314\nbob_seger;315\notis_rush;316\nmikhail_pletnev;317\ngene_autry;318\nblind_blake;319\nethel_waters;320\njackie_wilson;321\nbig_mama_thornton;322\nmary_wells;323\nthe_mills_brothers;324\nrage_against_the_machine;325\nbarbra_streisand;326\nfrank_crumit;327\nthe_clash;328\ndion;329\nall_4_one;330\nt_i;331\norchestral_manoeuvres_in_the_dark;332\nculture;333\njosh_turner;334\none_direction;335\nthe_rolling_stones;336\nhalf_pint;337\nthe_searchers;338\nvangelis;339\njackson_browne;340\nthe_beach_boys;341\ntrain;342\nprince_buster;343\ntavares;344\neve;345\nbessie_smith;346\ntrace_adkins;347\nbay_city_rollers;348\ndavid_allan_coe;349\nrascal_flatts;350\npetula_clark;351\n10cc;352\n50_cent;353\nthe_oak_ridge_boys;354\nbarry_manilow;355\ntruls_m_rk;356\nmorrissey;357\ngerry_the_pacemakers;358\nkay_starr;359\nalborosie;360\nengelbert_humperdinck;361\nnew_order;362\ntony_bennett;363\nstereophonics;364\njimmy_reed;365\nakon;366\necho_the_bunnymen;367\njamiroquai;368\nstevie_ray_vaughan;369\nben_e_king;370\ncheap_trick;371\ndusty_springfield;372\nmel_tillis;373\ndamian_marley;374\nruth_etting;375\nwestlife;376\ndiana_ross;377\nthe_shirelles;378\nfrankie_laine;379\nsarah_vaughan;380\nray_price;381\ngordon_lightfoot;382\neddie_cantor;383\nthe_byrds;384\ngary_numan;385\nbonnie_tyler;386\naswad;387\nbrownie_mcghee;388\njoshua_bell;389\nmanic_street_preachers;390\nmr_vegas;391\ntanya_tucker;392\nmarvin_gaye_tammi_terrell;393\nthe_staple_singers;394\nry_cooder;395\njohn_mayall;396\nmartina_mcbride;397\nanner_bylsma;398\nmagic_sam;399\navril_lavigne;400\nrobert_nighthawk;401\nthe_coasters;402\nace_of_base;403\njoseph_szigeti;404\nartie_shaw;405\nbig_bill_broonzy;406\nthe_ventures;407\nrick_astley;408\nles_paul;409\nleonid_kogan;410\nhall_oates;411\nthree_dog_night;412\ncharley_pride;413\npaul_mccartney;414\nalfred_cortot;415\ncrystal_gayle;416\ntaj_mahal;417\nmary_j_blige;418\nleann_rimes;419\nmildred_bailey;420\nll_cool_j;421\nlobo;422\nblake_shelton;423\nmatt_haimovitz;424\nbeastie_boys;425\njohannes_brahms;426\nmartha_and_the_vandellas;427\nlou_rawls;428\nthe_righteous_brothers;429\nrosalyn_tureck;430\nvince_gill;431\ngary_moore;432\nbad_company;433\nmarty_robbins;434\nruss_morgan;435\ndavid_cassidy;436\ndj_khaled;437\nleonard_pennario;438\nglenn_miller;439\nkings_of_leon;440\nafrojack;441\ncyndi_lauper;442\ntrisha_yearwood;443\ndiamond_rio;444\npatty_loveless;445\nsugar_minott;446\nwillie_nelson;447\nthomas_newman;448\ngeorgia_gibbs;449\neric_carmen;450\nricky_skaggs;451\nearth_wind_fire;452\npeter_tosh;453\njoe_bonamassa;454\nlonnie_johnson;455\nmissy_elliott;456\nnicki_minaj;457\nluther_allison;458\ngrigory_sokolov;459\nluke_bryan;460\nalanis_morissette;461\ntracy_lawrence;462\nsonny_james;463\nbrandy;464\nmichael_bolton;465\nboswell_sisters;466\nantonio_vivaldi;467\nfritz_kreisler;468\nelvis_costello_the_attractions;469\nwoody_herman;470\nkorn;471\nhans_zimmer;472\nthe_osmonds;473\nelectric_light_orchestra;474\nt_rex;475\nvan_cliburn;476\nharry_james;477\nglee_cast;478\nsimple_minds;479\nabba;480\nthe_jam;481\ntanya_stephens;482\nthe_statler_brothers;483\ncraig_david;484\nmoby;485\nxxxtentacion;486\npaul_simon;487\nmagic_slim;488\nnatasha_bedingfield;489\nennio_morricone;490\ncarpenters;491\njoe_tex;492\ncocoa_tea;493\nthe_glee_cast;494\nmartha_argerich;495\ncharlie_rich;496\nmemphis_minnie;497\nsheryl_crow;498\njan_dean;499\nnick_cave_and_the_bad_seeds;500\ndel_shannon;501\nlee_greenwood;502\ngary_lewis_the_playboys;503\nmarcelle_meyer;504\ncount_basie;505\nharold_melvin_the_blue_notes;506\nthe_fray;507\nthe_prodigy;508\nalicia_keys;509\nconway_twitty;510\nbarrington_levy;511\nb_o_b;512\ntritonal;513\nmario_lanza;514\nmac_davis;515\nbilly_murray;516\nigor_stravinsky;517\npretenders;518\njordin_sparks;519\nalice_in_chains;520\nohio_players;521\nrick_springfield;522\njimmie_rodgers;523\nbuju_banton;524\nlinda_ronstadt;525\ngeorge_benson;526\n2pac;527\nsoulja_boy;528\nthe_flaming_lips;529\nignacy_jan_paderewski;530\ngloria_estefan;531\nariana_grande;532\nblack_uhuru;533\ntony_pastor;534\nsublime;535\nsnow_patrol;536\ndaft_punk;537\njohnny_winter;538\nrobbie_williams;539\neddie_rabbitt;540\njames_cotton;541\nbrad_paisley;542\nmanfred_mann;543\nbassnectar;544\nmargaret_whiting;545\nsam_cooke;546\nrobert_cray;547\nthe_beautiful_south;548\nbarbara_mandrell;549\ndick_haymes;550\ncreedence_clearwater_revival;551\nchic;552\nray_charles;553\ncarter_family;554\nti_sto;555\nsurvivor;556\nc_line_dion;557\nsergei_prokofiev;558\nb_la_bart_k;559\nthe_congos;560\nyefim_bronfman;561\nlaidback_luke;562\ndarius_rucker;563\nray_anthony;564\nincubus;565\ncarole_king;566\njames_brown;567\nswv;568\nbruno_mars;569\naphex_twin;570\nnitty_gritty_dirt_band;571\ngustav_mahler;572\nthe_shadows;573\nthe_moody_blues;574\nreverend_gary_davis;575\nsia;576\ngaetano_donizetti;577\nearl_hooker;578\nthe_commodores;579\nmaria_jo_o_pires;580\neric_donaldson;581\nelmore_james;582\nsean_kingston;583\ndon_carlos;584\nlinkin_park;585\njay_the_americans;586\ngrand_funk_railroad;587\njls;588\nfrankie_valli;589\nlefty_frizzell;590\nen_vogue;591\nthe_cure;592\nperry_como;593\njohnny_mercer;594\nstevie_wonder;595\nernest_tubb;596\nramin_djawadi;597\nashanti;598\nrosemary_clooney;599\nanne_sophie_mutter;600\nhelen_forrest;601\naugustus_pablo;602\nthe_carpenters;603\nclaudio_arrau;604\nbob_dylan;605\njoe_simon;606\nculture_club;607\nthe_ipana_troubadors;608\njennifer_lopez;609\nkaryn_white;610\njoe;611\nsarah_mclachlan;612\nj_geils_band;613\ndean_martin;614\nhank_snow;615\nclyde_mcphatter;616\ntlc;617\nbeck;618\njimmy_dean;619\nroy_acuff;620\noutkast;621\nfreddie_mcgregor;622\ngus_arnheim;623\ngramatik;624\nmerle_haggard;625\nsteve_lawrence;626\nma_rainey;627\njimmy_dorsey;628\njohnny_paycheck;629\narthur_rubinstein;630\ntalking_heads;631\ncapleton;632\nles_brown;633\nleonard_bernstein;634\nbobby_goldsboro;635\nkendrick_lamar;636\nlenny_kravitz;637\nnat_shilkret;638\ntoby_keith;639\njunior_wells;640\nbilly_j_kramer_the_dakotas;641\npeter_paul_mary;642\narmand_van_helden;643\nh_sker_d_;644\nalabama;645\neminem;646\nfelix_mendelssohn_bartholdy;647\nrichard_goode;648\npink_floyd;649\nsara_evans;650\nlonnie_donegan;651\nboney_m;652\ndeadmau5;653\nlee_ann_womack;654\neric_clapton;655\nray_parker_jr;656\netta_james;657\nthe_white_stripes;658\ngary_u_s_bonds;659\nglen_gray_and_the_casa_loma_orchestra;660\nbonnie_raitt;661\nsoulja_boy_tell_em;662\nbobby_rydell;663\ncarly_simon;664\nkoko_taylor;665\ngregory_isaacs;666\nred_hot_chili_peppers;667\njosef_suk;668\nclint_black;669\nbuddy_clark;670\ntool;671\npierre_fournier;672\nalice_cooper;673\nlucky_dube;674\njorge_bolet;675\ndon_diablo;676\nr_l_burnside;677\nklaus_badelt;678\ndolly_parton;679\njames_horner;680\ngreen_day;681\nhenry_burr;682\nthe_doors;683\nroxette;684\nlouis_armstrong;685\njerry_butler;686\nlouis_prima;687\npaul_van_dyk;688\ndennis_brown;689\ntoni_braxton;690\njerry_lee_lewis;691\ndonna_summer;692\npercy_faith;693\nwillie_dixon;694\nelvis_costello;695\nyouri_egorov;696\ndmitri_shostakovich;697\nwebb_pierce;698\nmonica;699\npierre_laurent_aimard;700\nmuddy_waters;701\ngarth_brooks;702\nboyz_ii_men;703\nkris_kristofferson;704\nduane_eddy;705\ncoolio;706\ngidon_kremer;707\neurythmics;708\nrandy_travis;709\ncollie_buddz;710\nfaith_evans;711\nmatisyahu;712\nbrooks_dunn;713\nlulu;714\nfletcher_henderson;715\neek_a_mouse;716\nshlomo_mintz;717\nray_noble;718\nbill_withers;719\nub40;720\nb_j_thomas;721\nmariah_carey;722\nolivia_newton_john;723\njackson_5;724\nmichael_rose;725\nthree_days_grace;726\nglen_campbell;727\nkeith_urban;728\nonerepublic;729\nted_weems;730\njohnny_desmond;731\nbilly_preston;732\nbilly_vaughn;733\notis_spann;734\nmarion_harris;735\nrichard_marx;736\nfrankie_carle;737\nxavier_cugat;738\nzz_top;739\nfaz_l_say;740\nnew_york_philharmonic;741\nteresa_brewer;742\npixies;743\ndavid_oistrakh;744\nrory_gallagher;745\nted_lewis_his_band;746\npatsy_cline;747\ntracy_byrd;748\ndaniel_shafran;749\njohnny_horton;750\nbilly_ocean;751\ndon_mclean;752\nginuwine;753\nkiss;754\nkaskade;755\nthe_hold_steady;756\nrod_stewart;757\nrudolf_serkin;758\nthe_wanted;759\nherman_s_hermits;760\nrusty_draper;761\njay_z;762\nmississippi_john_hurt;763\nlinval_thompson;764\nbillie_holiday;765\nlawrence_welk;766\njohn_lee_hooker;767\nthe_offspring;768\nramones;769\nronan_keating;770\nsir_clifford_michael_curzon;771\ndierks_bentley;772\nbush;773\ntexas;774\ngene_austin;775\nafter_all;776\ndepeche_mode;777\neddy_arnold;778\nferry_corsten;779\ninxs;780\njimmie_lunceford;781\njohann_sebastian_bach;782\nmaxi_priest;783\ntravis_tritt;784\ngirls_aloud;785\nmeat_loaf;786\nbrenda_lee;787\nmodest_mouse;788\nglenn_gould;789\nmc_hammer;790\nerik_satie;791\nthe_andrews_sisters;792\nalicia_de_larrocha;793\namerica;794\ncharles_harrison;795\ngeorges_cziffra;796\npeter_serkin;797\ngladys_knight_the_pips;798\nanton_n_dvo_k;799\nemil_gilels;800\nmyra_hess;801\nportugal_the_man;802\nrufus;803\nsergei_rachmaninoff;804\nandr_previn;805\nnatalie_cole;806\nthe_killers;807\nhelen_reddy;808\nbilly_idol;809\nshawn_mendes;810\njohn_lennon;811\njoan_jett;812\njimmy_cliff;813\ntaylor_swift;814\nshura_cherkassky;815\nmarvin_gaye;816\nthe_drifters;817\npaul_revere_the_raiders;818\ndonovan;819\nalma_gluck;820\nfleetwood_mac;821\njo_stafford;822\nsamson_fran_ois;823\nvaughn_monroe;824\ntennessee_ernie_ford;825\nwhitesnake;826\niron_maiden;827\nblind_boy_fuller;828\neric_church;829\natomic_kitten;830\nharry_nilsson;831\npaul_specht;832\nbob_marley;833\nles_baxter;834\naretha_franklin;835\njessie_j;836\nrobert_johnson;837\nmaroon_5;838\nsheena_easton;839\ngeorge_strait;840\nbruce_springsteen;841\noasis;842\njack_white;843\nfatboy_slim;844\nthe_jesus_and_mary_chain;845\nhank_williams_jr;846\nbill_haley_his_comets;847\ncharlie_daniels;848\npearl_jam;849\nthe_stylistics;850\nsean_paul;851\ndavid_essex;852\nzino_francescatti;853\nbruce_hornsby;854\nsystem_of_a_down;855\nscott_joplin;856\nbenny_benassi;857\nmitch_miller;858\ngloria_gaynor;859\nsteve_winwood;860\ndrake;861\nbone_thugs_n_harmony;862\nnickelback;863\njohn_mellencamp;864\ngene_pitney;865\nchet_atkins;866\nmark_ronson;867\nkc_the_sunshine_band;868\nchingy;869\nmaurice_ravel;870\ninfected_mushroom;871\nwalter_gieseking;872\npavement;873\ndeniece_williams;874\ntommy_james_the_shondells;875\nharry_belafonte;876\nrobert_schuman;877\ngeorge_jones;878\nwhitney_houston;879\nthe_ames_brothers;880\nmax_romeo;881\npatti_page;882\np_nk;883\nvienna_philharmonic;884\nviolent_femmes;885\nsly_the_family_stone;886\njohnny_marvin;887\nsammy_davis_jr_;888\nnirvana;889\nthe_turtles;890\nthe_manhattans;891\nthe_supremes;892\ngregg_allman;893\nalton_ellis;894\ngiacomo_puccini;895\nartur_schnabel;896\n2_unlimited;897\nmichael_rabin;898\ntoto;899\nlil_kim;900\nshinedown;901\nteddy_wilson;902\nalpha_blondy;903\nthe_guess_who;904\nmy_chemical_romance;905\nmatchbox_twenty;906\nwolfgang_gartner;907\ned_sheeran;908\nfaron_young;909\nfats_waller;910\nkate_smith;911\nloretta_lynn;912\nantonio_meneses;913\ntony_martin;914\nfrankie_vaughan;915\nthe_mamas_the_papas;916\ndavid_bowie;917\nerskine_hawkins;918\nthe_four_lads;919\nqueens_of_the_stone_age;920\nthe_human_league;921\nmud;922\nbread;923\ndixie_chicks;924\nbackstreet_boys;925\nconnie_francis;926\nthe_black_keys;927\npaul_weller;928\npieter_wispelwey;929\nthe_bachelors;930\njanet_jackson;931\nthe_association;932\nedwin_starr;933\nhank_williams_jr_;934\nsteven_isserlis;935\nthe_doobie_brothers;936\njimi_hendrix;937\nsousa_s_band;938\nricky_martin;939\nthe_cranberries;940\npaul_whiteman;941\nthe_saturdays;942\nkeb_mo;943\nthe_psychedelic_furs;944\nboyzone;945\nthe_chemical_brothers;946\njohnny_tillotson;947\njoe_cocker;948\nbabyface;949\nwet_wet_wet;950\ntom_petty;951\nlesley_gore;952\ndeorro;953\nwar;954\nwolfgang_schneiderhan;955\nisaac_hayes;956\nbeyonc;957\ngordon_jenkins;958\nflo_rida;959\ncaptain_tennille;960\nadolf_busch;961\nbobby_bare;962\njohnnie_taylor;963\nabove_beyond;964\nrobin_thicke;965\nja_rule;966\nbarry_white;967\nflorence_the_machine;968\nthe_jimi_hendrix_experience;969\ngabrielle;970\nleon_fleisher;971\nyehudi_menuhin;972\ngeorge_harrison;973\nkate_bush;974\nu2;975\nnancy_sinatra;976\nthe_everly_brothers;977\npeter_green;978\ncat_stevens;979\nthe_new_seekers;980\nitzhak_perlman;981\nred_nichols_his_five_pennies;982\nthe_script;983\nijahman_levi;984\nrihanna;985\njoan_jett_and_the_blackhearts;986\nthe_dorsey_brothers_orchestra;987\ncharlie_barnet;988\nlmfao;989\n112;990\nkelis;991\njohn_mccormack;992\nguns_n_roses;993\ngordon_macrae;994\nbuddy_holly;995\nfranz_schubert;996\ntommy_roe;997\ngarbage;998\nbow_wow;999\nbelle_and_sebastian;1000\npaul_tortelier;1001\njohnny_nash;1002\nlazar_berman;1003\navicii;1004\n5_seconds_of_summer;1005\nmississippi_fred_mcdowell;1006\nivan_moravec;1007\nlittle_richard;1008\njohn_ogdon;1009\ndr_dre;1010\nandy_russell;1011\nsex_pistols;1012\npuddle_of_mudd;1013\njune_carter_cash;1014\nfreddie_king;1015\ndino_ciani;1016\nh_sker_d;1017\nnicky_romero;1018\nshania_twain;1019\nmajor_lazer;1020\nusher;1021\nmadonna;1022\nnsync;1023\nthe_o_jays;1024\nfaith_hill;1025\nterence_trent_d_arby;1026\ngene_chandler;1027\nlisa_stansfield;1028\npharrell_williams;1029\ndavid_geringas;1030\ndr_alimantado;1031\njack_scott;1032\nvladimir_horowitz;1033\ngenesis;1034\njessica_simpson;1035\npeter_gabriel;1036\nspike_jones_and_his_city_slickers;1037\nleonard_rose;1038\nrichard_strauss;1039\nmgmt;1040\nblink_182;1041\nella_fitzgerald;1042\ncarrie_underwood;1043\nevgeny_kissin;1044\nolly_murs;1045\nky_mani_marley;1046\nrose_royce;1047\ndesmond_dekker;1048\nmontell_jordan;1049\nlittle_river_band;1050\nt_pain;1051\ngary_allan;1052\nchubby_checker;1053\nthe_diamonds;1054\ncreed;1055\nlouis_jordan;1056\nvernon_dalhart;1057\ngilbert_o_sullivan;1058\nziggy_marley;1059\nirene_cara;1060\nkaty_perry;1061\nall_saints;1062\nbenjamin_britten;1063\np_m_dawn;1064\niona_brown;1065\nthe_lettermen;1066\nnatalie_imbruglia;1067\nviktoria_mullova;1068\nzac_brown_band;1069\nbj_rk;1070\nmutabaruka;1071\njermaine_jackson;1072\nthe_impressions;1073\nspandau_ballet;1074\nluther_vandross;1075\nthompson_twins;1076\ntravis_scott;1077\nroots_radics;1078\nmadness;1079\nskrillex;1080\nthe_smiths;1081\nvan_halen;1082\nthe_four_seasons;1083\nj_b_lenoir;1084\njohnny_rivers;1085\nbob_marley_the_wailers;1086\nalbert_collins;1087\narctic_monkeys;1088\nbilly_joel;1089\njoe_nichols;1090\nchicago;1091\ncoldplay;1092\nblue;1093\npapa_roach;1094\nr_e_m_;1095\neddie_money;1096\nellie_goulding;1097\ngym_class_heroes;1098\nthe_charlie_daniels_band;1099\nbette_midler;1100\nhubert_sumlin;1101\nphil_harris;1102\narturo_benedetti_michelangeli;1103\nsteps;1104\nsimon_and_garfunkel;1105\ngyptian;1106\nbritney_spears;1107\nlang_lang;1108\nblackstreet;1109\nthe_dave_clark_five;1110\nthe_chi_lites;1111\nwolfgang_amadeus_mozart;1112\njohnnie_ray;1113\nelvis_presley;1114\nred_norvo;1115\nthe_velvet_underground;1116\nnelly_furtado;1117\nconnee_boswell;1118\nsoundgarden;1119\ndr_hook_the_medicine_show;1120\nled_zeppelin;1121\nboney_m_;1122\ngregor_piatigorsky;1123\njohn_anderson;1124\njoni_james;1125\nthe_all_american_rejects;1126\nmarcia_griffiths;1127\nben_bernie;1128\neric_prydz;1129\nthe_three_suns;1130\nalbert_king;1131\njodeci;1132\ninner_circle;1133\ndon_cornell;1134\njeff_healey;1135\nbrian_mcknight;1136\nthe_stranglers;1137\nadam_faith;1138\nfrancis_poulenc;1139\nfall_out_boy;1140\nthe_jets;1141\ngroundation;1142\nrandy_newman;1143\njane_s_addiction;1144\nbobby_vinton;1145\nguiomar_novaes;1146\nbo_diddley;1147\nphiladelphia_orchestra;1148\ngarrick_ohlsson;1149\nevanescence;1150\nwilhelm_backhaus;1151\nthe_platters;1152\nthe_seekers;1153\nselena_gomez;1154\neddy_grant;1155\njan_garber;1156\ngioacchino_rossini;1157\nchristina_aguilera;1158\nstevie_nicks;1159\ncolor_me_badd;1160\nsandie_shaw;1161\nmigos;1162\nthe_allman_brothers_band;1163\nwill_young;1164\nnew_edition;1165\nradiohead;1166\norbital;1167\nthe_judds;1168\ndan_fogelberg;1169\npascal_rog_;1170\nleona_lewis;1171\ncharlie_patton;1172\nyann_tiersen;1173\nthe_abyssinians;1174\nfrank_ifield;1175\njohn_mayer;1176\nroger_wolfe_kahn;1177\nthe_mcguire_sisters;1178\njunior_reid;1179\nbukka_white;1180\nmacy_gray;1181\nbilly_currington;1182\nthe_tremeloes;1183\nac_dc;1184\nlittle_walter;1185\ngorillaz;1186\nthe_hilltoppers;1187\ngil_shaham;1188\nlittle_mix;1189\nsupertramp;1190\nthird_world;1191\ndavid_garrett;1192\nthe_lovin_spoonful;1193\nchris_lake;1194\nlouis_lortie;1195\nshakira;1196\nvanessa_williams;1197\nthe_bangles;1198\nmark_chesnutt;1199\nkeith_whitley;1200\nroxy_music;1201\nbetty_hutton;1202\ncheryl_cole;1203\nthe_notorious_b_i_g_;1204\nkesha;1205\nboston;1206\ngerald_moore;1207\nheinrich_schiff;1208\njoy_division;1209\nkenny_loggins;1210\nnilsson;1211\ncounting_crows;1212\nsimply_red;1213\naaron_tippin;1214\nsweet;1215\nsugababes;1216\nflorida_georgia_line;1217\nlonnie_mack;1218\njacob_miller;1219\nthe_police;1220\nthe_chordettes;1221\nfats_domino;1222\nwilly_deville;1223\nbob_crosby;1224\nroy_clark;1225\ndinu_lipatti;1226\nalison_krauss;1227\nfrankie_avalon;1228\nella_mae_morse;1229\nlinton_kwesi_johnson;1230\nglenn_frey;1231\nrudy_vall_e_his_connecticut_yankees;1232\ndj_snake;1233\ntoots_the_maytals;1234\nannie_lennox;1235\nparamore;1236\njohn_browning;1237\nsister_sledge;1238\njulian_lloyd_webber;1239\nkanye_west;1240\nchristopher_cross;1241\nprofessor_longhair;1242\nflume;1243\nsophie_tucker;1244\nthird_eye_blind;1245\nbeyonc_;1246\nweezer;1247\nkelly_clarkson;1248\ntake_that;1249\nludwig_van_beethoven;1250\nthe_cars;1251\ngeorge_michael;1252\nchaka_khan;1253\narty;1254\nsteel_pulse;1255\nbunny_wailer;1256\nthe_troggs;1257\nnick_lucas;1258\nkeb_mo_;1259\nrise_against;1260\nmickey_gilley;1261\npoison;1262\ndebbie_gibson;1263\nkim_wilde;1264\nsammy_davis_jr;1265\njonas_brothers;1266\ncage_the_elephant;1267\nson_house;1268\nbirth_control;1269\nfaithless;1270\ncher;1271\nseether;1272\nnew_kids_on_the_block;1273\nrage;1274\nrichie_spice;1275\nsam_smith;1276\nthe_marvelettes;1277\nthe_jackson_5;1278\nbobbie_gentry;1279\nmacklemore_ryan_lewis;1280\ns_club_7;1281\nbilly_jones_ernest_hare;1282\narcana;1283\nblasterjaxx;1284\nwill_smith;1285\nray_miller;1286\nsugar_ray;1287\nziggy_marley_the_melody_makers;1288\naaron_copland;1289\ndiddy;1290\ndaughtry;1291\nbeach_house;1292\nthe_dillinger_escape_plan;1293\nlonestar;1294\nforeigner;1295\nlionel_richie;1296\nroberta_flack;1297\nthe_carter_family;1298\ndemi_lovato;1299\njoseph_haydn;1300\nsander_van_doorn;1301\nunderworld;1302\ndeborah_cox;1303\nthe_grass_roots;1304\nbananarama;1305\niggy_azalea;1306\n3_doors_down;1307\nthe_partridge_family;1308\nlead_belly;1309\njohnny_long;1310\njagged_edge;1311\nthe_derek_trucks_band;1312\neddie_fisher;1313\npablo_casals;1314\nthe_original_dixieland_jazz_band;1315\nmiley_cyrus;1316\njohn_powell;1317\nthe_black_eyed_peas;1318\nedvard_grieg;1319\nmaurizio_pollini;1320\nnathan_milstein;1321\ntwenty_one_pilots;1322\nrudolf_firku_n_;1323\nnine_inch_nails;1324\nthe_four_aces;1325\njimmy_rogers;1326\nsalt_n_pepa;1327\nyellow_claw;1328\nthe_strokes;1329\nbobby_brown;1330\njuelz_santana;1331\nstaind;1332\nbj_rn_ulvaeus_benny_andersson;1333\nthe_j_geils_band;1334\nthe_muppets;1335\ngwen_stefani;1336\nfedde_le_grand;1337\nimagine_dragons;1338\nleo_sayer;1339\nthe_animals;1340\nthe_b_52_s;1341\nfurry_lewis;1342\nciara;1343\nlarry_clinton;1344\ndire_straits;1345\npato_banton_the_reggae_revol;1346\ngoodie_mob;1347\ndisclosure;1348\ngeorges_bizet;1349\nsonny_terry_brownie_mcghee;1350\njim_croce;1351\nnelson;1352\njason_derulo;1353\nguy_mitchell;1354\nthe_fontane_sisters;1355\nsonny_boy_williamson_i;1356\nmirah;1357\njimmy_rushing;1358\njohn_michael_montgomery;1359\nmichael_nesmith;1360\ngeorge_clinton;1361\nburt_bacharach;1362\nv6;1363\nslim_thug;1364\nbelinda_carlisle;1365\nphilip_glass;1366\nslade;1367\npete_townshend;1368\noingo_boingo;1369\nandrew_lloyd_webber;1370\nthe_collectors;1371\nboy_george;1372\nutada_hikaru;1373\nmel_torm;1374\ndiana_krall;1375\nmelanie_c;1376\njohn_hammond;1377\npeter_green_splinter_group;1378\ntrouble;1379\ng_unit;1380\nferlin_husky;1381\narcade_fire;1382\nlatino;1383\nkrayzie_bone;1384\nman;1385\ndave_clark_five;1386\nthe_stone_roses;1387\nyoung_jeezy;1388\nblood_sweat_tears;1389\nkumikameli;1390\ndmx;1391\nice_cube;1392\neagles;1393\njill_scott;1394\nxtc;1395\npeggy_march;1396\nmichael_bubl;1397\nraimon;1398\ntwo_mix;1399\ndulce_pontes;1400\nthe_unseen;1401\nhank_locklin;1402\nthe_notorious_b_i_g;1403\nbumblefoot;1404\nthe_busters;1405\nrick_ross;1406\ntegan_and_sara;1407\nskeeter_davis;1408\ncurtis_mayfield;1409\nsade;1410\nwings;1411\nlorrie_morgan;1412\nsaga;1413\na_r_rahman;1414\nmartha_wainwright;1415\nnas;1416\nwill_i_am;1417\nkirsty_maccoll;1418\nangel;1419\ndave_davies;1420\niggy_pop;1421\njojo;1422\nsammy_hagar;1423\nray_davies;1424\nand_one;1425\nneil_young;1426\nmikael_wiehe;1427\nthe_cardigans;1428\ncage;1429\ndottie_west;1430\nkeri_hilson;1431\njohnny_hallyday;1432\nbill_nelson;1433\nfifteen;1434\ndeen;1435\nbobby_v;1436\nlil_yachty;1437\ntoo_hort;1438\nbernard_lavilliers;1439\nhank_thompson;1440\nthe_chieftains;1441\ndaryl_hall;1442\nantonio_carlos_jobim;1443\naventura;1444\nthe_the;1445\nvan_morrison;1446\nwynonna_judd;1447\ngomez;1448\ncharles_aznavour;1449\nm83;1450\ngnr;1451\nall;1452\nemmylou_harris;1453\nlee_hazlewood;1454\nmew;1455\nuriah_heep;1456\nyoko_ono;1457\nabw_rts;1458\njohn_legend;1459\nd12;1460\nkitty_wells;1461\ntimbiriche;1462\nshel_silverstein;1463\ncam_ron;1464\nrosanne_cash;1465\n2_chainz;1466\ntricky;1467\n8ball_mjg;1468\nflatt_scruggs;1469\nbill_anderson;1470\nemil_ana_torrini;1471\nufo;1472\nmos_def;1473\ndanzig;1474\njuan_gabriel;1475\ncommon;1476\nraekwon;1477\nfrance_gall;1478\nnicole_scherzinger;1479\nr_yksopp;1480\nsammie;1481\nlena_horne;1482\ndavid_byrne;1483\npaul_williams;1484\njosh_groban;1485\nthe_gathering;1486\nfrank_boeijen;1487\nscooter;1488\nsteve_wariner;1489\nmika;1490\npete_seeger;1491\ntex_ritter;1492\nwarrant;1493\nporter_wagoner;1494\nfield_music;1495\nthree_6_mafia;1496\njim_jones;1497\ndaniel_o_donnell;1498\nbrentalfloss;1499\nwyclef_jean;1500\nhey;1501\nbizzy_bone;1502\nthe_mccalmans;1503\nblues_traveler;1504\nmassive_attack;1505\nwoody_guthrie;1506\nart_garfunkel;1507\nandrea_bocelli;1508\ndavid_crosby;1509\ndream;1510\nsoul_asylum;1511\nnatalie_merchant;1512\nshawn_colvin;1513\njonny_lang;1514\nfuneral_for_a_friend;1515\nboz_scaggs;1516\nexample;1517\nlionel_hampton;1518\nthe_tubes;1519\nmarc_anthony;1520\ngood_riddance;1521\nmoonlight;1522\nmarc_almond;1523\nrza;1524\ndie_rzte;1525\nrbd;1526\nalejandro_fern_ndez;1527\nwanda_jackson;1528\nlara_fabian;1529\njulio_iglesias;1530\njeff_beck;1531\npeabo_bryson;1532\nno_fun_at_all;1533\nprong;1534\ncanibus;1535\nkrs_one;1536\nu_s_bombs;1537\ntrust;1538\nstonewall_jackson;1539\njos_feliciano;1540\nm_a;1541\npolysics;1542\nn_e_r_d;1543\nsesame_street;1544\nlio;1545\nmyl_ne_farmer;1546\niris_dement;1547\nlily_allen;1548\nspoken;1549\narchitects;1550\njack_johnson;1551\nmolly_hatchet;1552\ncypress_hill;1553\nfuture;1554\nthe_nits;1555\nper_gessle;1556\nlive;1557\nbeau;1558\ndeana_carter;1559\nlil_flip;1560\nfran_oise_hardy;1561\nbilly_ray_cyrus;1562\npepper;1563\nrun_d_m_c;1564\nlevon_helm;1565\ninsane_clown_posse;1566\nstars;1567\njean_michel_jarre;1568\nthunder;1569\njuanes;1570\nsimple_plan;1571\nmethod_man;1572\nsmash_mouth;1573\nmeek_mill;1574\nfat_joe;1575\nkenny_wayne_shepherd;1576\njhen_aiko;1577\nthe_manhattan_transfer;1578\njoss_stone;1579\ncee_lo_green;1580\ntyrese;1581\ncharlotte_martin;1582\nrodney_crowell;1583\nacappella;1584\ndie_prinzen;1585\npentatonix;1586\nabney_park;1587\nsmooth_mcgroove;1588\nthe_magnetic_fields;1589\nthe_nylons;1590\nwise_guys;1591\ncoil;1592\ndo_as_infinity;1593\nlords_of_acid;1594\nthe_church;1595\nchris_rea;1596\njota_quest;1597\nmiguel_bos;1598\ngang_starr;1599\nmasta_ace;1600\nbrand_new;1601\nmac_miller;1602\ncathedral;1603\ncorrosion_of_conformity;1604\ncountry_joe_mcdonald;1605\neric_johnson;1606\ngrateful_dead;1607\njanis_joplin;1608\njohn_miles;1609\nking_gizzard_the_lizard_wizard;1610\nted_nugent;1611\ncombichrist;1612\nalkaline_trio;1613\nanathema;1614\nangus_julia_stone;1615\nanna_ternheim;1616\nanthony_phillips;1617\nsteve_hackett;1618\naviators;1619\nbanda_calypso;1620\nblue_stahli;1621\nthe_boys;1622\ncapital_inicial;1623\ncity_and_colour;1624\ncolin_hay;1625\ncollective_soul;1626\ndashboard_confessional;1627\ndavid_rovics;1628\ndavid_usher;1629\ndie_toten_hosen;1630\nfunny_van_dannen;1631\ndirty_heads;1632\ntech_n9ne;1633\nelisa;1634\nemmerson_nogueira;1635\nengenheiros_do_hawaii;1636\neric_bibb;1637\nmaria_muldaur;1638\npanic_at_the_disco;1639\npunchline;1640\ngodsmack;1641\nself;1642\nheideroosjes;1643\nsinner;1644\nheather_nova;1645\nhoobastank;1646\nquietdrive;1647\nhyde;1648\njaguares;1649\nbert_jansch;1650\njonatha_brooke;1651\njoni_mitchell;1652\nthe_band;1653\njosh_garrels;1654\njosh_woodward;1655\nwilliam_fitzsimmons;1656\nkatie_melua;1657\njamie_cullum;1658\nkristin_hersh;1659\nkt_tunstall;1660\nlegi_o_urbana;1661\nthe_zombies;1662\nfrancesco_de_gregori;1663\nm_ward;1664\nbeth_orton;1665\nmagnum;1666\nmotorpsycho;1667\nmarillion;1668\njars_of_clay;1669\nmason_jennings;1670\nmatt_nathanson;1671\nmatthew_good;1672\nedguy;1673\ngamma_ray;1674\nminus_the_bear;1675\nmohsen_namjoo;1676\nnerina_pallot;1677\nnever_shout_never;1678\nregina_spektor;1679\npassenger;1680\npaul_kelly;1681\nthe_style_council;1682\npeter_hammill;1683\nphantom_planet;1684\nphil_keaggy;1685\nrichard_thompson;1686\nsaid_the_whale;1687\nsamsas_traum;1688\nsenses_fail;1689\nsevendust;1690\nseventh_day_slumber;1691\njoan_baez;1692\nsister_hazel;1693\nslightly_stoopid;1694\nsophie_zelmani;1695\nsuzanne_vega;1696\ntatiana;1697\nteoman;1698\nthe_choir;1699\ncharlotte_church;1700\ndarlene_zschech;1701\nthe_front_bottoms;1702\nthe_maine;1703\nthe_white_buffalo;1704\nlittle_big_town;1705\nthreshold;1706\ntourniquet;1707\neverything_but_the_girl;1708\nvertical_horizon;1709\nvonda_shepard;1710\nwarren_zevon;1711\nsarah_brightman;1712\nblackfoot;1713\nblack_label_society;1714\nz_lia_duncan;1715\n2;1716\nalejandro_lerner;1717\nbeth_nielsen_chapman;1718\nmercury_rev;1719\nbrian_wilson;1720\nbarenaked_ladies;1721\ncarbon_leaf;1722\nceltic_woman;1723\nhayley_westenra;1724\ncrowded_house;1725\ndelta_goodrem;1726\nelbow;1727\nresurrection_band;1728\nnancy_wilson;1729\njanis_ian;1730\njann_arden;1731\njill_sobule;1732\njos_augusto;1733\nxuxa;1734\nk_d_lang;1735\nkim_carnes;1736\nlos_lobos;1737\nmandy_moore;1738\nmarc_cohn;1739\nmaureen_mcgovern;1740\nmelissa_manchester;1741\npatti_labelle;1742\nhelene_fischer;1743\nlaura_pausini;1744\nivan_lins;1745\nthal_a;1746\nmike_the_mechanics;1747\npaul_carrack;1748\nnatasha_st_pier;1749\nmichael_mcdonald;1750\nolivia;1751\ndizzee_rascal;1752\nsam_phillips;1753\nserge_gainsbourg;1754\njane_birkin;1755\nluis_miguel;1756\nsondre_lerche;1757\nstan_ridgway;1758\nsusan_boyle;1759\nmike_oldfield;1760\nces_ria_vora;1761\nagonoize;1762\nfunker_vogt;1763\ngod_module;1764\nhocico;1765\nnachtmahr;1766\nsuicide_commando;1767\nalejandro_escovedo;1768\nsouthside_johnny_the_asbury_jukes;1769\nbroken_social_scene;1770\nvigilantes_of_love;1771\nbilly_bragg;1772\nwilco;1773\nfrank_black_and_the_catholics;1774\nblue_rodeo;1775\nbrandi_carlile;1776\npatty_griffin;1777\ncalexico;1778\ncass_mccombs;1779\nchris_knight;1780\nconor_oberst;1781\ncorb_lund;1782\ncowboy_junkies;1783\ncracker;1784\ncross_canadian_ragweed;1785\ndave_alvin;1786\ndrive_by_truckers;1787\neleni_mandell;1788\nlucinda_williams;1789\nfred_eaglesmith;1790\ndar_williams;1791\nthe_jayhawks;1792\nlana_del_rey;1793\ntim_o_brien;1794\nhank_williams_iii;1795\njames_mcmurtry;1796\njoe_henry;1797\njohn_stewart;1798\njosh_ritter;1799\nlambchop;1800\nnanci_griffith;1801\nnorma_jean;1802\nlyle_lovett;1803\nmatthew_ryan;1804\nmy_morning_jacket;1805\nneko_case;1806\nthe_new_pornographers;1807\nblue_october;1808\nokkervil_river;1809\nold_97_s;1810\nray_wylie_hubbard;1811\nrichmond_fontaine;1812\nrobert_earl_keen;1813\nrocky_votolato;1814\nryan_adams;1815\nwhiskeytown;1816\nson_volt;1817\nsteve_earle;1818\nthe_avett_brothers;1819\nthe_bottle_rockets;1820\nthe_felice_brothers;1821\narlo_guthrie;1822\nthe_handsome_family;1823\nthe_mavericks;1824\nten_years_after;1825\nthe_walkabouts;1826\ntodd_snider;1827\nvic_chesnutt;1828\niron_wine;1829\nwovenhand;1830\nx;1831\nbig_audio_dynamite;1832\nglobe;1833\ncarter_the_unstoppable_sex_machine;1834\nallison_moorer;1835\nfront_line_assembly;1836\nthe_national;1837\nthe_fall;1838\npublic_image_ltd;1839\npublic_enemy;1840\nwire;1841\na_tribe_called_quest;1842\nde_la_soul;1843\naesop_rock;1844\nbuck_65;1845\ncaparezza;1846\nchildish_gambino;1847\nthe_roots;1848\ncolbie_caillat;1849\nbig_sean;1850\ndj_gruff;1851\ntyler_the_creator;1852\ndokken;1853\nfun_lovin_criminals;1854\ntalib_kweli;1855\njane_air;1856\nk_i_z;1857\nkid_cudi;1858\njedi_mind_tricks;1859\ncelph_titled;1860\nlupe_fiasco;1861\nbun_b;1862\nscarface;1863\nghostface_killah;1864\nrobyn;1865\nrehab;1866\nswollen_members;1867\nstyles_p;1868\nthe_streets;1869\nwale;1870\njoe_budden;1871\ntank;1872\n10_years;1873\n36_crazyfists;1874\napocalyptica;1875\nnina_hagen;1876\nanastacia;1877\nblack_stone_cherry;1878\nblindside;1879\nbreaking_benjamin;1880\nbring_me_the_horizon;1881\nbullet_for_my_valentine;1882\ncave_in;1883\nchevelle;1884\nd_espairsray;1885\ndeath_angel;1886\ndeftones;1887\ndemon_hunter;1888\ndemon;1889\ndevin_townsend_project;1890\ndevin_townsend;1891\ndoa;1892\ndir_en_grey;1893\ndisturbed;1894\ndope;1895\ndrowning_pool;1896\neighteen_visions;1897\nentombed;1898\nfaith_no_more;1899\nfear_factory;1900\nfightstar;1901\nfive_finger_death_punch;1902\nfinger_eleven;1903\nflyleaf;1904\ngrinspoon;1905\nguano_apes;1906\nh_blockx;1907\nhalestorm;1908\nhamlet;1909\nhelmet;1910\nbt;1911\nill_ni_o;1912\nin_flames;1913\nin_this_moment;1914\nhim;1915\nj_b_o;1916\nkatatonia;1917\nkillswitch_engage;1918\nxzibit;1919\nlacuna_coil;1920\nphish;1921\nlimp_bizkit;1922\nliving_colour;1923\nviikate;1924\nmarilyn_manson;1925\nmegaherz;1926\nfalco;1927\nmelvins;1928\nmonster_magnet;1929\nmushroomhead;1930\nnonpoint;1931\nsoil;1932\notep;1933\np_o_d;1934\npowerman_5000;1935\nprimus;1936\nproject_86;1937\nred;1938\nkris_allen;1939\nrob_zombie;1940\nozzy_osbourne;1941\nrollins_band;1942\nsaliva;1943\nsepultura;1944\nshihad;1945\nskillet;1946\nskindred;1947\nslipknot;1948\nsmile_empty_soul;1949\ndanielson;1950\nsoilwork;1951\nsonic_syndicate;1952\nstatic_x;1953\nstone_sour;1954\ntaproot;1955\nthe_notwist;1956\nthe_word_alive;1957\ntheory_of_a_deadman;1958\ntherapy;1959\nlos_tucanes_de_tijuana;1960\nmanu_chao;1961\nvolbeat;1962\nzebrahead;1963\nhed_p_e;1964\nand_you_will_know_us_by_the_trail_of_dead;1965\n10_000_maniacs;1966\n311;1967\n77s;1968\nyes;1969\ndavid_lee_roth;1970\nhillsong;1971\nafi;1972\nadam_sandler;1973\nafterhours;1974\nhawkwind;1975\nall_about_eve;1976\nall_time_low;1977\nallison_crowe;1978\namanda_palmer;1979\namerican_music_club;1980\namplifier;1981\nrobert_wyatt;1982\nanberlin;1983\nandrew_bird;1984\nani_difranco;1985\napoptygma_berzerk;1986\napulanta;1987\narab_strap;1988\njoseph_arthur;1989\ntom_rosenthal;1990\nash;1991\nasian_kung_fu_generation;1992\npoets_of_the_fall;1993\nbabas_nicos;1994\nbayside;1995\nbeatsteaks;1996\nben_folds;1997\nben_folds_five;1998\nben_harper;1999\nbetter_than_ezra;2000\nbettie_serveert;2001\nbig_country;2002\nbig_head_todd_and_the_monsters;2003\nbig_sugar;2004\nbilly_talent;2005\ntoday_is_the_day;2006\nred_flag;2007\nblack_rebel_motorcycle_club;2008\nmegadeth;2009\nblonde_redhead;2010\nbob_mould;2011\nbodeans;2012\nbowling_for_soup;2013\nbuck_tick;2014\nbutch_walker;2015\nbutthole_surfers;2016\ncaf_tacvba;2017\ncake;2018\ncamper_van_beethoven;2019\ncarmen_consoli;2020\nmario_venuti;2021\nfranco_battiato;2022\ncatherine_wheel;2023\ncatupecu_machu;2024\ncem_adrian;2025\njohn_cale;2026\ncharlie_brown_jr;2027\nnena;2028\nchumbawamba;2029\nclutch;2030\ncl;2031\ncmx;2032\ncoheed_and_cambria;2033\ncold_war_kids;2034\ntravis;2035\ncoma;2036\nconcrete_blonde;2037\nmint_condition;2038\ncopeland;2039\ncrash_test_dummies;2040\njoe_jackson;2041\ncristian_castro;2042\ncurve;2043\ndada;2044\ndaniel_amos;2045\ndaniel_johnston;2046\ndave_matthews_band;2047\nburning_heads;2048\ndavid_gray;2049\ndavid_sylvian;2050\ndeacon_blue;2051\ndeerhoof;2052\ndel_amitri;2053\ndinosaur_jr;2054\ndirty_projectors;2055\ndraco_rosa;2056\nduncan_sheik;2057\njeremy_camp;2058\nedwyn_collins;2059\neels;2060\nnightwish;2061\nelement_of_crime;2062\nembrace;2063\nenter_shikari;2064\nulver;2065\neverclear;2066\neverlast;2067\neyeshine;2068\ndio;2069\nfaust_o;2070\nfeeder;2071\natmosphere;2072\nfilter;2073\nfirewater;2074\nfishbone;2075\nfountains_of_wayne;2076\nfour_year_strong;2077\nsteve_green;2078\nfresno;2079\ngang_of_four;2080\ngood_charlotte;2081\nblood_on_the_dance_floor;2082\ngraham_coxon;2083\nmelissa_etheridge;2084\ntony_joe_white;2085\nguided_by_voices;2086\nrobert_pollard;2087\nguster;2088\nelliott_smith;2089\nhedley;2090\nhole;2091\nhollywood_undead;2092\nhot_chip;2093\nl_arc_en_ciel;2094\nian_brown;2095\nidlewild;2096\njimmy_eat_world;2097\nfish;2098\ningrid_michaelson;2099\ninme;2100\ninspiral_carpets;2101\nraf;2102\njames;2103\njean_leloup;2104\nweird_al_yankovic;2105\njeff_buckley;2106\njohn_frusciante;2107\ndr_john;2108\npj_harvey;2109\njonathan_coulton;2110\njuliana_hatfield;2111\njulieta_venegas;2112\nk_s_choice;2113\nkaizers_orchestra;2114\nkargo;2115\nkasabian;2116\nkeane;2117\nkevin_coyne;2118\nkevin_devine;2119\nkevin_max;2120\nrich_mullins;2121\ntrooper;2122\nsuzy_bogguss;2123\nkill_hannah;2124\nkisp_l_s_a_borz;2125\nkult;2126\nmy_life_with_the_thrill_kill_kult;2127\nkutless;2128\nla_barranca;2129\nla_ley;2130\nlao_che;2131\nlech_janerka;2132\nles_cowboys_fringants;2133\nles_fatals_picards;2134\nles_rita_mitsouko;2135\nsparks;2136\nlifehouse;2137\nlisa_germano;2138\ned_harcourt;2139\nlisa_loeb;2140\nliz_phair;2141\nlocal_h;2142\nlost_dogs;2143\nlostprophets;2144\nlove_and_rockets;2145\nlucybell;2146\nlulu_santos;2147\ngabriel_o_pensador;2148\nadam_lambert;2149\nmadrugada;2150\nmancha_de_rolando;2151\nmanchester_orchestra;2152\nmando_diao;2153\nfoetus;2154\nmark_lanegan;2155\nmatthew_sweet;2156\nmax_mo_park;2157\nmayday_parade;2158\nmeat_puppets;2159\nmen_without_hats;2160\nmeshell_ndegeocello;2161\nmidnight_oil;2162\ndance_gavin_dance;2163\nmolotov;2164\nov7;2165\nmonkey_majik;2166\nsuede;2167\nfernando_ortega;2168\nmotion_city_soundtrack;2169\nmudhoney;2170\nmutemath;2171\nmercyme;2172\nm_o_morta;2173\nnatalia_lafourcade;2174\nnatewantstobattle;2175\nneedtobreathe;2176\nsplit_enz;2177\nsum_41;2178\nno_te_va_gustar;2179\nnoir_d_sir;2180\nt_tes_raides;2181\no_rappa;2182\no_a_r;2183\nocean_colour_scene;2184\nomul_cu_obolani;2185\none_ok_rock;2186\n2raumwohnung;2187\nour_lady_peace;2188\npain;2189\npanda;2190\nparokya_ni_edgar;2191\npato_fu;2192\npaul_westerberg;2193\npere_ubu;2194\npete_yorn;2195\npeter_murphy;2196\nplacebo;2197\nplain_white_t_s;2198\npop_will_eat_itself;2199\nporcupine_tree;2200\npowderfinger;2201\ncat_power;2202\ncasting_crowns;2203\nprimal_scream;2204\nm_tley_cr_e;2205\nthe_used;2206\nraimundos;2207\nmark_knopfler;2208\nmark_kozelek;2209\ndanko_jones;2210\nrelient_k;2211\nraffi;2212\nrenaud;2213\nrichard_hawley;2214\nrickie_lee_jones;2215\nthe_shins;2216\nrilo_kiley;2217\nrobyn_hitchcock;2218\nmose_allison;2219\nroy_harper;2220\nrucka_rucka_ali;2221\nrx_bandits;2222\nsaez;2223\nsamiam;2224\nsarah_slean;2225\nsay_anything;2226\nscout_niblett;2227\nscreaming_females;2228\nshannon_wright;2229\nsilverchair;2230\nsin_ad_o_connor;2231\nsiouxsie_and_the_banshees;2232\nsixpence_none_the_richer;2233\nskank;2234\nskunk_anansie;2235\nsleater_kinney;2236\nsloan;2237\nsocial_distortion;2238\nsophie_hunger;2239\ne_40;2240\nsteve_wynn;2241\nsubsonica;2242\njoe_walsh;2243\nsuper_furry_animals;2244\nsuperchunk;2245\nsupergrass;2246\nswervedriver;2247\nswitchfoot;2248\ndido;2249\ntakida;2250\ntaking_back_sunday;2251\nteenage_fanclub;2252\nw_a_s_p;2253\nthe_afghan_whigs;2254\nthe_apples_in_stereo;2255\nthe_ataris;2256\nsmoking_popes;2257\nthe_bluetones;2258\nthe_breeders;2259\nthe_cat_empire;2260\nthe_charlatans_uk;2261\nthe_clarks;2262\nguy_clark;2263\nthe_comsat_angels;2264\nthe_connells;2265\nthe_coral;2266\nthe_cribs;2267\nthe_cult;2268\nbobby_o;2269\nthe_mission;2270\nblue_yster_cult;2271\nthe_dandy_warhols;2272\nthe_dear_hunter;2273\nthe_decemberists;2274\nthe_early_november;2275\nthievery_corporation;2276\nthe_fratellis;2277\nthe_gaslight_anthem;2278\njim_brickman;2279\nfalling_up;2280\nthe_hives;2281\nthe_innocence_mission;2282\nthe_jazz_butcher;2283\nthe_jesus_lizard;2284\nthe_lemonheads;2285\nbabyshambles;2286\nthe_living_end;2287\nthe_matrixx;2288\nthe_mother_hips;2289\nthe_mountain_goats;2290\nthe_muffs;2291\nthe_pillows;2292\nthe_posies;2293\nthe_presidents_of_the_united_states_of_america;2294\nthe_rasmus;2295\nthe_raveonettes;2296\nthe_saints;2297\nthe_samples;2298\nbad_religion;2299\nthe_smithereens;2300\nthe_soundtrack_of_our_lives;2301\nthe_tea_party;2302\nmayday;2303\nthe_triffids;2304\nthe_vines;2305\nthe_violet_burning;2306\nthe_wallflowers;2307\ntestament;2308\nthe_divine_comedy;2309\nthird_day;2310\nthrice;2311\ntindersticks;2312\ntism;2313\ntit_s;2314\ntoad_the_wet_sprocket;2315\ntocotronic;2316\ntom_mcrae;2317\ntori_amos;2318\ntracy_chapman;2319\ntrashcan_sinatras;2320\ntre_allegri_ragazzi_morti;2321\ntub_ring;2322\nunkle;2323\nunwritten_law;2324\nuverworld;2325\nvast;2326\nverdena;2327\nveruca_salt;2328\nface_to_face;2329\nvirus;2330\nvoltaire;2331\nwe_the_kings;2332\nthe_kooks;2333\nlindisfarne;2334\nseals_crofts;2335\nandy_partridge;2336\nxutos_pontap_s;2337\nyellowcard;2338\nyup;2339\nleevi_and_the_leavings;2340\nzo;2341\nzucchero;2342\nz;2343\nebnem_ferah;2344\nair;2345\nalice;2346\nboards_of_canada;2347\nbrian_eno;2348\nburzum;2349\ndaniel_lanois;2350\nenigma;2351\njuana_molina;2352\nlisa_gerrard;2353\nnox_arcana;2354\nrenard;2355\nschiller;2356\nsigur_r_s;2357\nsteven_wilson;2358\nswans;2359\nwolfgun;2360\nxiu_xiu;2361\nmichael_johnson;2362\nmontgomery_gentry;2363\nthe_stanley_brothers;2364\njohn_waite;2365\nshelby_lynne;2366\njudy_collins;2367\nburl_ives;2368\nthe_irish_rovers;2369\ndavid_wilcox;2370\ndevendra_banhart;2371\ndoc_watson;2372\nbill_monroe;2373\nmichael_martin_murphey;2374\ngordon_bok;2375\nasleep_at_the_wheel;2376\nthe_browns;2377\nnana_mouskouri;2378\njerry_jeff_walker;2379\nsteve_goodman;2380\nmalcolm_holcombe;2381\nmalvina_reynolds;2382\nodetta;2383\ntom_paxton;2384\nstrawbs;2385\nphil_ochs;2386\nharry_chapin;2387\nramblin_jack_elliott;2388\nroger_mcguinn;2389\ngene_clark;2390\nmat_kearney;2391\nthe_brothers_four;2392\ntom_russell;2393\ntownes_van_zandt;2394\nuncle_dave_macon;2395\ndelbert_mcclinton;2396\njohn_hiatt;2397\njustin_townes_earle;2398\nmark_erelli;2399\nover_the_rhine;2400\nsteve_forbert;2401\nmanfred_mann_s_earth_band;2402\nmot_rhead;2403\nrudimentary_peni;2404\nillapu;2405\ninti_illimani;2406\nquilapay_n;2407\nv_ctor_jara;2408\nskylark;2409\nadam_green;2410\ncold_chisel;2411\nguy_sebastian;2412\njefferson_starship;2413\nthe_alan_parsons_project;2414\nali_project;2415\nmodern_talking;2416\nanimal_collective;2417\nbanco_del_mutuo_soccorso;2418\nben_lee;2419\nbryan_ferry;2420\nbuffy_sainte_marie;2421\ncolin_blunstone;2422\ncursive;2423\nelysian_fields;2424\nemerson_lake_palmer;2425\ngino_vannelli;2426\ng_rard_manset;2427\nhot_dad;2428\nmarina_and_the_diamonds;2429\nismo_alanko;2430\nkansas;2431\nkari_peitsamo;2432\nlaibach;2433\nlaurie_anderson;2434\npuhdys;2435\nna_o_zumbi;2436\nroger_waters;2437\nrush;2438\nthe_walker_brothers;2439\nhilltop_hoods;2440\nwolfgang_ambros;2441\nerste_allgemeine_verunsicherung;2442\njacques_brel;2443\nrainhard_fendrich;2444\ntom_waits;2445\nadrian_belew;2446\nanne_clark;2447\ncan;2448\ncaptain_beefheart_and_the_magic_band;2449\ndeine_lakaien;2450\ndevo;2451\neinst_rzende_neubauten;2452\nfrank_zappa;2453\ngoethes_erben;2454\nwishbone_ash;2455\ndeath_cab_for_cutie;2456\nantony_and_the_johnsons;2457\njandek;2458\nnevermore;2459\nking_crimson;2460\nking_missile;2461\nthe_residents;2462\nsteeleye_span;2463\nvampire_rodents;2464\nthe_walkmen;2465\ndog_fashion_disco;2466\nfreak_kitchen;2467\nsigh;2468\nchildren_of_bodom;2469\nsoft_machine;2470\nara_ketu;2471\nasa_de_guia;2472\nbanda_eva;2473\nivete_sangalo;2474\nchiclete_com_banana;2475\ndaniela_mercury;2476\nalejandro_sanz;2477\ntimbalada;2478\njuan_luis_guerra;2479\ndaddy_yankee;2480\nalceu_valen_a;2481\nluiz_gonzaga;2482\nmatia_bazar;2483\naxelle_red;2484\nbarbara;2485\nbenny_neyman;2486\ngigi_d_agostino;2487\njacques_higelin;2488\ncaetano_veloso;2489\ngal_costa;2490\njorge_ben;2491\ndie_flippers;2492\nnicole;2493\nangra;2494\nreinhard_mey;2495\nwolf_biermann;2496\nflorent_pagny;2497\nhannes_wader;2498\ntienne_daho;2499\nhenri_salvador;2500\nf_lix_leclerc;2501\ndaniel_lavoie;2502\ngerhard_sch_ne;2503\ng_lben_ergen;2504\ngeorg_kreisler;2505\nherbert_gr_nemeyer;2506\nherman_van_veen;2507\nhildegard_knef;2508\nmarlene_dietrich;2509\niu;2510\njos_luis_rodr_guez;2511\njuliette_gr_co;2512\nklaus_hoffmann;2513\nkonstantin_wecker;2514\nsaltatio_mortis;2515\nluigi_tenco;2516\nmaria_beth_nia;2517\nadriana_calcanhotto;2518\nmarie_lafor_t;2519\nmarius_m_ller_westernhagen;2520\nmina;2521\nno_l_coward;2522\npippo_pollina;2523\nrita_lee;2524\nos_mutantes;2525\nrita_pavone;2526\nroger_whittaker;2527\nal_bano_romina_power;2528\nsalvatore_adamo;2529\nsimone;2530\ns_rgio_godinho;2531\nudo_j_rgens;2532\nudo_lindenberg;2533\nulrich_roski;2534\nzaz;2535\nz_ramalho;2536\nfagner;2537\ndith_piaf;2538\nduelo;2539\nespinoza_paz;2540\nfidel_rueda;2541\nla_firma;2542\nla_arrolladora_banda_el_lim_n;2543\nvoz_de_mando;2544\nsergio_vega;2545\nfool_s_garden;2546\nwaltari;2547\nof_montreal;2548\npierre_lapointe;2549\nrufus_wainwright;2550\nloudon_wainwright_iii;2551\nsufjan_stevens;2552\nmachine_gun_kelly;2553\nfrancesco_guccini;2554\nle_orme;2555\nlucio_dalla;2556\nmichel_fugain;2557\nal_jarreau;2558\ncarmen_mcrae;2559\njavier_sol_s;2560\nharry_connick_jr;2561\nbap;2562\ncradle_of_filth;2563\namorphis;2564\navatar;2565\nbathory;2566\nbehemoth;2567\nborknagar;2568\ncountess;2569\ncruachan;2570\ndarkthrone;2571\nhate;2572\ndestruction;2573\ndimmu_borgir;2574\neisregen;2575\nenslaved;2576\nfinntroll;2577\nfates_warning;2578\ngraveworm;2579\nimpaled_nazarene;2580\nsentenced;2581\nking_diamond;2582\nkreator;2583\nlord_belial;2584\nmarduk;2585\nmercyful_fate;2586\nstick_to_your_guns;2587\nmoonspell;2588\nas_i_lay_dying;2589\nnunslaughter;2590\nrotting_christ;2591\nsamael;2592\nsandy_denny;2593\nskyforger;2594\nsodom;2595\ncannibal_corpse;2596\nexodus;2597\natreyu;2598\ntheatres_des_vampires;2599\nwizard;2600\ntransmetal;2601\nvenom;2602\nbelphegor;2603\nthe_crown;2604\nmoya_brennan;2605\ntodd_rundgren;2606\nclay_walker;2607\nandrew_peterson;2608\nlynn_anderson;2609\ndavid_crowder_band;2610\npam_tillis;2611\nnorah_jones;2612\nrhonda_vincent;2613\njamey_johnson;2614\nplumb;2615\nj_j_cale;2616\nnew_riders_of_the_purple_sage;2617\njoe_diffie;2618\nkasey_chambers;2619\nleon_russell;2620\njack_greene;2621\nthe_string_cheese_incident;2622\nystein_sunde;2623\nstephen_stills;2624\ncancerslug;2625\nrobert_plant;2626\nalvin_lee;2627\nbeth_hart;2628\njimmy_buffett;2629\nbilly_s_band;2630\nbunbury;2631\nnacho_vegas;2632\ncalogero;2633\ngeorges_brassens;2634\ncanned_heat;2635\ncharlie_louvin;2636\ncolin_james;2637\ncuby_blizzards;2638\ndick_annegarn;2639\nedoardo_bennato;2640\neva_cassidy;2641\ngil_scott_heron;2642\nglenn_hughes;2643\ndeep_purple;2644\nconnie_smith;2645\niva_zanicchi;2646\nizzy_stradlin;2647\nj_karjalainen;2648\njack_bruce;2649\nleonard_cohen;2650\njoan_armatrading;2651\njoan_osborne;2652\njohn_martyn;2653\nrio_reiser;2654\nlarry_carlton;2655\nmadeleine_peyroux;2656\nbruce_cockburn;2657\nkate_anna_mcgarrigle;2658\nmavis_staples;2659\nnoa;2660\nralph_mctell;2661\nrenato_carosone;2662\nrichie_kotzen;2663\nrobben_ford;2664\nroberto_carlos;2665\nerasmo_carlos;2666\nrobin_trower;2667\nrory_block;2668\nroy_buchanan;2669\nsandra_mihanovich;2670\nsavoy_brown;2671\nshirley_horn;2672\nsiniestro_total;2673\nslank;2674\nthe_fabulous_thunderbirds;2675\nthe_seatbelts;2676\nthe_tragically_hip;2677\nmike_jones;2678\ntrophy_scars;2679\ncaravan;2680\nvelhas_virgens;2681\nwalter_trout;2682\ngov_t_mule;2683\nbar_o_vermelho;2684\nblue_cheer;2685\nian_hunter;2686\ndavid_leb_n;2687\nde_palmas;2688\neugenio_finardi;2689\nextreme;2690\nfoghat;2691\ngeorge_thorogood_the_destroyers;2692\ngreat_white;2693\nguardian;2694\njethro_tull;2695\nian_anderson;2696\ndavid_knopfler;2697\nsteppenwolf;2698\ndave_edmunds;2699\nlynyrd_skynyrd;2700\ncrosby_stills_nash;2701\nraul_seixas;2702\nthe_poodles;2703\nmusiq_soulchild;2704\nshocking_blue;2705\nnick_lowe;2706\nthe_black_crowes;2707\ntraffic;2708\nwidespread_panic;2709\nco;2710\nalberto_cortez;2711\njoan_sebastian;2712\nana_gabriel;2713\ngilberto_santa_rosa;2714\nrub_n_blades;2715\nv_ctor_manuelle;2716\ncelia_cruz;2717\nluis_fonsi;2718\nnek;2719\ndr_feelgood;2720\nastrud_gilberto;2721\nbenito_di_paula;2722\nbrazzaville;2723\nsacha_distel;2724\nchico_buarque;2725\nelis_regina;2726\nmilton_nascimento;2727\nfaf_de_bel_m;2728\nnikka_costa;2729\ntim_maia;2730\ngilberto_gil;2731\nlisa_ekdahl;2732\njoyce;2733\nmaria_rita;2734\nnara_le_o;2735\nnouvelle_vague;2736\npaulinho_moska;2737\nwilson_simonal;2738\n14_bis;2739\narnaldo_antunes;2740\nbiquini_cavad_o;2741\ncidade_negra;2742\ncpm_22;2743\nc_ssia_eller;2744\nos_paralamas_do_sucesso;2745\nguilherme_arantes;2746\nira;2747\nlob_o;2748\nnenhum_de_n_s;2749\ndjavan;2750\nrog_rio_skylab;2751\nroupa_nova;2752\nultraje_a_rigor;2753\nkj_52;2754\namado_batista;2755\nchit_ozinho_xoror;2756\njo_o_paulo_daniel;2757\nleandro_leonardo;2758\nleonardo;2759\nodair_jos;2760\nkaiser_chiefs;2761\nkula_shaker;2762\nlightning_seeds;2763\npulp;2764\nthe_proclaimers;2765\ndying_fetus;2766\nnapalm_death;2767\nnile;2768\npathology;2769\nhilary_duff;2770\nbadly_drawn_boy;2771\nfederico_salvatore;2772\ni_gufi;2773\nzachary_richard;2774\nstan_rogers;2775\nmoxy_fr_vous;2776\npoco;2777\nla_bottine_souriante;2778\nstompin_tom_connors;2779\nbersuit_vergarabat;2780\nlas_pastillas_del_abuelo;2781\ngeorge_lam;2782\naltan;2783\nclannad;2784\nblackmore_s_night;2785\ncapercaillie;2786\nceltic_thunder;2787\neluveitie;2788\npowerwolf;2789\ngaelic_storm;2790\nan_na;2791\njon_anderson;2792\nthe_dubliners;2793\nloreena_mckennitt;2794\nomnia;2795\nsecret_garden;2796\nshaun_davey;2797\nroger_daltrey;2798\nthe_corrs;2799\nlos_tigres_del_norte;2800\nlaurent_voulzy;2801\nthe_kelly_family;2802\nwolfe_tones;2803\nalan_stivell;2804\nheather_alexander;2805\nkate_rusby;2806\ndropkick_murphys;2807\ngreat_big_sea;2808\nfiddler_s_green;2809\nheather_dale;2810\nrunrig;2811\nthe_waterboys;2812\ndougie_maclean;2813\nadriano_celentano;2814\nalain_chamfort;2815\nzazie;2816\nhamelen;2817\ntazenda;2818\narno;2819\narthur_h;2820\nboudewijn_de_groot;2821\ncharles_trenet;2822\nclaudio_baglioni;2823\nclaudio_rocchi;2824\nfabrizio_de_andr;2825\ndalida;2826\ndana_winner;2827\ndemis_roussos;2828\nesther_ofarim;2829\neugenio_bennato;2830\nmichel_berger;2831\nfrancis_cabrel;2832\nmaxime_le_forestier;2833\ngeorges_moustaki;2834\ngianmaria_testa;2835\ngianni_morandi;2836\ngigliola_cinquetti;2837\nmilva;2838\ngilbert_b_caud;2839\nginette_reno;2840\ngiuni_russo;2841\nguy_b_art;2842\nhelena_vondr_kov;2843\nhugues_aufray;2844\nivan_graziani;2845\nivano_fossati;2846\njacques_bertin;2847\njean_ferrat;2848\njuliane_werding;2849\njulien_clerc;2850\nlos_temerarios;2851\nkaterine;2852\nleny_escudero;2853\nmathieu_chedid;2854\nluca_barbarossa;2855\nl_o_ferr;2856\nrosenstolz;2857\nmarc_lavoine;2858\nmassimo_bubola;2859\nmecano;2860\nmia_martini;2861\nmichel_jonasz;2862\nmichele_zarrillo;2863\nfiorello;2864\nnada;2865\nmercedes_sosa;2866\nnino_d_angelo;2867\npatrick_bruel;2868\npatty_pravo;2869\npierre_bachelet;2870\nrainald_grebe;2871\nrapha_l;2872\nraphael;2873\nrichard_anthony;2874\nroberto_murolo;2875\nron;2876\nstefano_rosso;2877\nstephan_eicher;2878\nvasco_rossi;2879\nyves_duteil;2880\nyves_jamait;2881\nang_lica;2882\naaron_carter;2883\nbarry_louis_polisar;2884\nyuri;2885\ncri_cri;2886\nhevisaurus;2887\njuice_leskinen;2888\nkidz_bop;2889\nmara_maravilha;2890\ndestroyer;2891\nscorpions;2892\nobk;2893\nduncan_dhu;2894\nparry_gripp;2895\nsandy_junior;2896\nthe_verve_pipe;2897\nthe_verve;2898\nthe_wiggles;2899\nveggietales;2900\nnewsboys;2901\nsteven_curtis_chapman;2902\ntoro_y_moi;2903\nmedi_val_b_bes;2904\naaron_neville;2905\nbethel_music;2906\napologetix;2907\ngaither_vocal_band;2908\nbuilding_429;2909\nchris_tomlin;2910\nmatt_maher;2911\njerusalem;2912\ndavid_meece;2913\ndebby_boone;2914\nelevation_worship;2915\nmatt_redman;2916\nplanetshakers;2917\nmajesty;2918\njump5;2919\nlecrae;2920\nmichael_w_smith;2921\nbride;2922\nnatalie_grant;2923\nthe_lads;2924\naudio_adrenaline;2925\npaul_wilbur;2926\npsalmen_voor_nu;2927\nsawyer_brown;2928\nshane_shane;2929\nthe_echoing_green;2930\ntwila_paris;2931\nwatch_tower_bible_and_tract_society;2932\nda_t_r_u_t_h;2933\ndc_talk;2934\nflame;2935\ngrits;2936\ntrip_lee;2937\ncrystal_lewis;2938\nthe_cross_movement;2939\ntobymac;2940\nvico_c;2941\nmormon_tabernacle_choir;2942\naugust_burns_red;2943\nblack_veil_brides;2944\ndeliverance;2945\nopeth;2946\ndie_happy;2947\ndisciple;2948\ngalactic_cowboys;2949\nhaste_the_day;2950\nliving_sacrifice;2951\nmastodon;2952\nmortification;2953\nshowbread;2954\nlabyrinth;2955\nstryper;2956\nthe_devil_wears_prada;2957\nunderoath;2958\nwhitecross;2959\npetra;2960\nhuntingtons;2961\nmxpx;2962\nd_a_d;2963\ncaedmon_s_call;2964\ndavid_and_the_giants;2965\ndegarmo_and_key;2966\ndelirious;2967\ndon_francisco;2968\nfive_iron_frenzy;2969\ngeoff_moore;2970\nhawk_nelson;2971\ngrave;2972\nlarry_norman;2973\nrandy_stonehill;2974\nmonty_python;2975\noomph;2976\noficina_g3;2977\nwhite_heart;2978\nrescate;2979\nrick_wakeman;2980\nla_oreja_de_van_gogh;2981\nsanctus_real;2982\nfun_people;2983\nthousand_foot_krutch;2984\ntim_hughes;2985\nthe_o_c_supertones;2986\n4him;2987\nbilly_gilman;2988\naimee_mann;2989\nkatharine_mcphee;2990\neros_ramazzotti;2991\nz_ro;2992\nbabbie_mason;2993\nbebo_norman;2994\njudy_garland;2995\ncarman;2996\ncece_winans;2997\ntrick_daddy;2998\nchris_isaak;2999\ncocteau_twins;3000\nedyta_g_rniak;3001\nenrico_ruggeri;3002\nffh;3003\nhanson;3004\nhawksley_workman;3005\nindigo_girls;3006\nirene_grandi;3007\njackie_evancho;3008\njoy_electric;3009\nkelly_price;3010\nmary_mary;3011\nisrael_houghton;3012\nphil_wickham;3013\nphillips_craig_dean;3014\nroch_voisine;3015\nrupaul;3016\ngregorian;3017\nsarah_connor;3018\nsugarland;3019\nsweetbox;3020\ntarja;3021\nthe_brian_setzer_orchestra;3022\nbrian_setzer;3023\nbadfinger;3024\nthe_moffatts;3025\nthe_vandals;3026\ntrans_siberian_orchestra;3027\nroy_drusky;3028\nburton_cummings;3029\nprocol_harum;3030\nrenaissance;3031\nthe_pretty_things;3032\ntwisted_sister;3033\nbj_rn_eidsv_g;3034\ncorvus_corax;3035\nschelmish;3036\nemilie_autumn;3037\nepica;3038\nkatherine_jenkins;3039\nscala_kolacny_brothers;3040\ntake_6;3041\nthe_roches;3042\ntony_banks;3043\nto_e_proeski;3044\nlacrimosa;3045\n16_volt;3046\nbj_rn_rosenstr_m;3047\nbob_rivers;3048\ncledus_t_judd;3049\nfrankjavcee;3050\ngeorge_formby;3051\nninja_sex_party;3052\npaul_and_storm;3053\nthe_arrogant_worms;3054\ntripod;3055\nel_cuarteto_de_nos;3056\ngwar;3057\nknorkator;3058\npsychostick;3059\nrodgau_monotones;3060\nlos_palominos;3061\ncharlie_peacock;3062\njesus_culture;3063\nmichael_card;3064\ntenth_avenue_north;3065\ncarrie_newcomer;3066\nnick_drake;3067\naaron_watson;3068\nbilly_joe_royal;3069\nbilly_joe_shaver;3070\ncharlie_landsborough;3071\nchris_ledoux;3072\ncollin_raye;3073\ndan_seals;3074\ndave_dudley;3075\nhellbillies;3076\ned_bruce;3077\nemilio_navaira;3078\njean_shepard;3079\nfreddie_hart;3080\ngary_stewart;3081\ngene_watson;3082\ngian_giovani;3083\ngilberto_gilmar;3084\njason_mraz;3085\nilse_delange;3086\njohn_prine;3087\njake_owen;3088\nwynn_stewart;3089\njim_ed_brown;3090\njoe_ely;3091\nkid_rock;3092\nla_toya_jackson;3093\nlit;3094\nlita_ford;3095\nme_first_and_the_gimme_gimmes;3096\nlagwagon;3097\nmelanie;3098\nmickey_newbury;3099\npaul_brunelle;3100\npaula_fernandes;3101\nzez_di_camargo_luciano;3102\nrandy_rogers_band;3103\nreverend_horton_heat;3104\nrick_renner;3105\nrionegro_solim_es;3106\nshooter_jennings;3107\nterri_clark;3108\nvern_gosdin;3109\nwebb_wilder;3110\nween;3111\n38_special;3112\nthe_beau_brummels;3113\nmatanza;3114\nclawfinger;3115\nacid_drinkers;3116\nagnostic_front;3117\nbiohazard;3118\nbody_count;3119\nd_r_i;3120\nmunicipal_waste;3121\nneurosis;3122\nnuclear_assault;3123\nsoziedad_alkoholika;3124\nsuicidal_tendencies;3125\nparagon;3126\nmario;3127\ninna;3128\nbelinda;3129\nbronco;3130\ngrupo_bryndis;3131\ndavid_bisbal;3132\nram_n_ayala;3133\ngrant_lee_phillips;3134\nthe_veronicas;3135\namr_diab;3136\natb;3137\nbasshunter;3138\ndream_theater;3139\nfrankie_j;3140\nbaby_bash;3141\nsophie_ellis_bextor;3142\ngrace_jones;3143\nlaveerre;3144\nsilkk_the_shocker;3145\nparov_stelar;3146\nraffaella_carr;3147\nelephant_man;3148\nsaint_etienne;3149\nsamantha_fox;3150\nselena;3151\nsuper_junior;3152\nt_a_t_u;3153\ntarkan;3154\njudie_tzuke;3155\nel_kel_iset;3156\nyello;3157\nfranz_ferdinand;3158\nchenoa;3159\nlucero;3160\ntokio;3161\npuffy_amiyumi;3162\nwink;3163\nobie_trice;3164\nmystikal;3165\ncurrent_93;3166\ndark_sanctuary;3167\nrome;3168\nlord_of_the_lost;3169\nbella_morte;3170\nmantus;3171\nblutengel;3172\nclan_of_xymox;3173\ndead_can_dance;3174\ndeath_in_june;3175\ndiary_of_dreams;3176\ndiorama;3177\nhelium_vola;3178\nilluminate;3179\nl_me_immortelle;3180\nlacrimas_profundere;3181\nkilling_joke;3182\nm_nchener_freiheit;3183\notto_dix;3184\nproject_pitchfork;3185\nqntal;3186\nsopor_aeternus;3187\nthe_cr_xshadows;3188\nunheilig;3189\nwelle;3190\nyendri;3191\ncarcass;3192\nasphyx;3193\nbolt_thrower;3194\ndarkseed;3195\nparadise_lost;3196\ntiamat;3197\nthe_damned;3198\npantera;3199\nthe_amity_affliction;3200\njudas_priest;3201\namon_amarth;3202\nalesana;3203\natrocity;3204\nautopsy;3205\navulsed;3206\nsabaton;3207\nmisfits;3208\niron_fire;3209\ncentinex;3210\ndagoba;3211\ndark_tranquillity;3212\nasia;3213\ndeicide;3214\ndethklok;3215\ndew_scented;3216\nedge_of_sanity;3217\nescape_the_fate;3218\nheaven_shall_burn;3219\nhypocrisy;3220\nincantation;3221\njungle_rot;3222\nkataklysm;3223\nkrisiun;3224\nmacabre;3225\nmalevolent_creation;3226\nmeshuggah;3227\nmisanthrope;3228\nmorbid_angel;3229\ndead_kennedys;3230\nnecro;3231\npig_destroyer;3232\nshadows_fall;3233\nsinister;3234\nsix_feet_under;3235\ndream_evil;3236\nsoulfly;3237\nthe_black_dahlia_murder;3238\nbetween_the_buried_and_me;3239\ntherion;3240\nvader;3241\nwhitechapel;3242\nattila;3243\nemmure;3244\nmiss_may_i;3245\nthe_acacia_strain;3246\nbetontod;3247\nbroilers;3248\ndritte_wahl;3249\nohl;3250\nslime;3251\nterrorgruppe;3252\nb_hse_onkelz;3253\nfrei_wild;3254\nk_rbholz;3255\nasp;3256\ntokio_hotel;3257\nqueensr_che;3258\namanda_miguel;3259\narabesque;3260\nbad_boys_blue;3261\nboyce_avenue;3262\nparliament;3263\nwu_tang_clan;3264\nneoton_fam_lia;3265\nteena_marie;3266\nbobby_womack;3267\nagoraphobic_nosebleed;3268\ncandlemass;3269\nelectric_wizard;3270\nblack_sabbath;3271\ntheatre_of_tragedy;3272\ntype_o_negative;3273\nmarie_fredriksson;3274\nluna;3275\nmarissa_nadler;3276\nyo_la_tengo;3277\ncelldweller;3278\nhitomi;3279\nbig_d_and_the_kids_table;3280\nalacranes_musical;3281\nk_paz_de_la_sierra;3282\nassemblage_23;3283\ncovenant;3284\ndie_krupps;3285\nkodak_black;3286\nfront_242;3287\nhaujobb;3288\nin_strict_confidence;3289\nle_ther_strip;3290\nsnog;3291\nthe_darkness;3292\ntanzwut;3293\nterminal_choice;3294\nvelvet_acid_christ;3295\nvnv_nation;3296\nwumpscut;3297\nx_fusion;3298\numbra_et_imago;3299\nde_vision;3300\ndeichkind;3301\neisbrecher;3302\nherbie_hancock;3303\nana_moura;3304\nmacaco;3305\nskinny_puppy;3306\nayreon;3307\nblack_moth_super_rainbow;3308\nerykah_badu;3309\ncocorosie;3310\nde_jeugd_van_tegenwoordig;3311\ndj_shadow;3312\ne_nomine;3313\nkmfdm;3314\nflying_lotus;3315\ngoldfrapp;3316\nhanzel_und_gretyl;3317\ninformation_society;3318\nmc_frontalot;3319\nkraftwerk;3320\nladytron;3321\nlamb;3322\nmilk_inc;3323\nmind_in_a_box;3324\nministry;3325\nm_m;3326\nm_nia;3327\npig;3328\npitchshifter;3329\nlil_boosie;3330\nmaster_p;3331\nmindless_self_indulgence;3332\nbuzzcocks;3333\nvanilla_ice;3334\nmilie_simon;3335\ngianna_nannini;3336\npinback;3337\nthe_birthday_massacre;3338\narchive;3339\n99_posse;3340\nbloc_party;3341\nmorcheeba;3342\noriga;3343\npaul_kalkbrenner;3344\ntina_arena;3345\ndover;3346\nmelotron;3347\nowl_city;3348\nkamelot;3349\ngreeley_estates;3350\nhawthorne_heights;3351\njoan_of_arc;3352\nsaves_the_day;3353\nthursday;3354\ntransit;3355\nfairport_convention;3356\nmaggie_reilly;3357\njoan_manuel_serrat;3358\ne_rotic;3359\nthe_scene;3360\nsandra;3361\namon_d_l_ii;3362\ncirca_survive;3363\nlove_solfege;3364\ncaliban;3365\ntall_dwarfs;3366\nvan_der_graaf_generator;3367\ndeath_grips;3368\nthe_fiery_furnaces;3369\nam_lia_rodrigues;3370\ncristina_branco;3371\njos_afonso;3372\nkatia_guerreiro;3373\nney_matogrosso;3374\nmadredeus;3375\nmariza;3376\ngipsy_kings;3377\nmal;3378\naleks_syntek;3379\nni_a_pastori;3380\nrosario;3381\nal_stewart;3382\namos_lee;3383\nandr_s_calamaro;3384\nane_brun;3385\nasa;3386\neditors;3387\ncatie_curtis;3388\nchrystian_ralf;3389\nclueso;3390\neddi_reader;3391\neddie_from_ohio;3392\nellis_paul;3393\nfrank_turner;3394\nestampie;3395\nferdi_tayfur;3396\nfito_p_ez;3397\nluis_alberto_spinetta;3398\ngabriella_ferri;3399\ngigi;3400\ngreg_brown;3401\ng_ksel;3402\nlando_fiorini;3403\nindia_arie;3404\njack_savoretti;3405\nanne_grete_preus;3406\njarom_r_nohavica;3407\njoe_purdy;3408\njohn_wesley_harding;3409\njosh_rouse;3410\nkarel_kryl;3411\nv_tor_ramil;3412\nlars_winnerb_ck;3413\nlaura_marling;3414\nllu_s_llach;3415\nlos_chalchaleros;3416\nluka_bloom;3417\nmalicorne;3418\nmark_heard;3419\nmartin_carthy;3420\nnic_jones;3421\nle_n_gieco;3422\nmijares;3423\nnuova_compagnia_di_canto_popolare;3424\nola_magnell;3425\nthin_lizzy;3426\nray_lamontagne;3427\nron_sexsmith;3428\nrosana;3429\nsilvio_rodr_guez;3430\nstef_bos;3431\nsun_kil_moon;3432\ntanita_tikaram;3433\nthe_incredible_string_band;3434\nthea_gilmore;3435\ntina_dico;3436\nvictor_leo;3437\nv_rttin;3438\nge_aleksandersen;3439\ni_brahim_tatl_ses;3440\nektomorf;3441\nelvenking;3442\nensiferum;3443\nfalconer;3444\nfeuerschwanz;3445\nin_extremo;3446\nkorpiklaani;3447\nleaves_eyes;3448\nletzte_instanz;3449\nm_go_de_oz;3450\nsaurom;3451\nschandmaul;3452\nskyclad;3453\nsubway_to_sally;3454\nsuidakra;3455\nt_r;3456\nicehouse;3457\nbomb_the_music_industry;3458\nthe_real_mckenzies;3459\n54_40;3460\narmored_saint;3461\nalexz_johnson;3462\nbar_man_o;3463\nezginin_g_nl;3464\ngalija;3465\nsts;3466\nh_kan_hellstr_m;3467\njames_blunt;3468\nkazik;3469\nmewithoutyou;3470\nmichel_polnareff;3471\novidi_montllor;3472\nrasputina;3473\nshearwater;3474\ngerry_rafferty;3475\nsteam_powered_giraffe;3476\nthe_saw_doctors;3477\nty_segall;3478\ntyrone_wells;3479\navi_es_do_forr;3480\ngrimskunk;3481\nsinik;3482\nvitaa;3483\nkenza_farah;3484\nsexion_d_assaut;3485\naliz_e;3486\nhenri_tachan;3487\njenifer;3488\nm_pokora;3489\nindochine;3490\nbrainstorm;3491\ncon_funk_shun;3492\nfunkadelic;3493\nlena_park;3494\nneffa;3495\nugk;3496\nsuburban_legends;3497\nmai_kuraki;3498\ncherry_poppin_daddies;3499\nelectric_six;3500\nlos_straitjackets;3501\nthe_69_eyes;3502\nthe_angels;3503\nthe_haunted;3504\nthe_hellacopters;3505\nthe_kills;3506\nthee_oh_sees;3507\nwhite_denim;3508\nzabranjeno_pu_enje;3509\nol_dirty_bastard;3510\nkurupt;3511\nspice_1;3512\nbrotha_lynch_hung;3513\nchamillionaire;3514\npaul_wall;3515\ntrae;3516\nclub_dogo;3517\nmc_eiht;3518\nroyce_da_5_9;3519\ngeto_boys;3520\nthe_diplomats;3521\nice_t;3522\n2_live_crew;3523\nxv;3524\nmobb_deep;3525\nc_murder;3526\ntru;3527\nlil_keke;3528\nproject_pat;3529\ntha_dogg_pound;3530\nesham;3531\ntwiztid;3532\nerick_sermon;3533\nbig_tymers;3534\nkate_nash;3535\nthe_cramps;3536\nnekromantix;3537\ntsol;3538\nace_frehley;3539\nhardcore_superstar;3540\nharem_scarem;3541\nhouse_of_lords;3542\nkingdom_come;3543\nl_a_guns;3544\nmr_big;3545\npink_cream_69;3546\nquiet_riot;3547\nriot;3548\nratt;3549\ntnt;3550\nbackyard_babies;3551\nultima_thule;3552\neurope;3553\nhanoi_rocks;3554\nmott_the_hoople;3555\nsmokie;3556\nsuzi_quatro;3557\nhaemorrhage;3558\naline_barros;3559\nbruna_karla;3560\nkirk_franklin;3561\nminist_rio_koinonya_de_louvor;3562\nartrosis;3563\nclosterkeller;3564\nindica;3565\nsirenia;3566\ntrail_of_tears;3567\ntristania;3568\nwithin_temptation;3569\nbauhaus;3570\nmono_inc;3571\npansy_division;3572\nxandria;3573\nimmortal_technique;3574\nagathocles;3575\nrotten_sound;3576\nthe_locust;3577\nanthrax;3578\ndevildriver;3579\nlamb_of_god;3580\nmachine_head;3581\nparkway_drive;3582\npro_pain;3583\nthrowdown;3584\nvicious_rumors;3585\nscreaming_trees;3586\ncuisillos;3587\nintocable;3588\npesado;3589\nla_mafia;3590\nmarco_antonio_sol_s;3591\nlos_bukis;3592\nandrew_w_k;3593\napril_wine;3594\naxel_rudi_pell;3595\nb_z;3596\ntak_matsumoto;3597\nbarricada;3598\nbijelo_dugme;3599\nblaze_bayley;3600\nbonfire;3601\nbruce_dickinson;3602\nbuckcherry;3603\nbudgie;3604\nbuitres;3605\njorn;3606\ndoro;3607\nenuff_z_nuff;3608\ngentle_giant;3609\ngirlschool;3610\ngolden_earring;3611\ngotthard;3612\nnazareth;3613\na_day_to_remember;3614\njefferson_airplane;3615\njoe_satriani;3616\nken_hensley;3617\nkim_mitchell;3618\nking_s_x;3619\nkotiteollisuus;3620\nla_renga;3621\nlee_aaron;3622\nlordi;3623\nmichael_schenker_group;3624\nmustasch;3625\nnight_ranger;3626\nomega;3627\nparni_valjak;3628\npaul_gilbert;3629\npopeda;3630\nskid_row;3631\ntankcsapda;3632\nthe_bronx;3633\nthe_donnas;3634\nall_that_remains;3635\ntriumph;3636\numphrey_s_mcgee;3637\ny_t;3638\nziggy;3639\nsfdk;3640\n7_seconds;3641\naiden;3642\nalphaville;3643\nblack_flag;3644\nslayer;3645\ncircle_jerks;3646\nritchie;3647\nconverge;3648\nevery_time_i_die;3649\nhatebreed;3650\nnomeansno;3651\nrancid;3652\nmemphis_may_fire;3653\nnofx;3654\npropagandhi;3655\ntankard;3656\nscreeching_weasel;3657\nsick_of_it_all;3658\nsilverstein;3659\ntwo_steps_from_hell;3660\nfaun;3661\naccept;3662\nthe_frames;3663\nandromeda;3664\nannihilator;3665\nanvil;3666\nartillery;3667\navenged_sevenfold;3668\naxxis;3669\nblind_guardian;3670\nvanden_plas;3671\ngrave_digger;3672\ndragonforce;3673\nedenbridge;3674\ndamien_jurado;3675\nexciter;3676\nfirewind;3677\nhalford;3678\nhammerfall;3679\nhelloween;3680\nhelstar;3681\niced_earth;3682\njag_panzer;3683\nmachinae_supremacy;3684\nmanowar;3685\nmetal_church;3686\nmorgana_lefay;3687\nmudvayne;3688\nnocturnal_rites;3689\noverkill;3690\nprimal_fear;3691\nrebellion;3692\nrunning_wild;3693\ncorey_hart;3694\nsavatage;3695\nsaxon;3696\nsteve_vai;3697\ntad_morose;3698\ntarot;3699\ntierra_santa;3700\ntrivium;3701\nturmion_k_til_t;3702\nu_d_o;3703\nvirgin_steele;3704\nvoivod;3705\nwarcry;3706\nyngwie_malmsteen;3707\nzion_lennox;3708\nsido;3709\nmc_chris;3710\nassalti_frontali;3711\nkool_keith;3712\nayumi_hamasaki;3713\naz;3714\nbahh_tee;3715\nbassi_maestro;3716\nrevocation;3717\nblumentopf;3718\nbrockhampton;3719\nbts;3720\nbushido;3721\nvinnie_paz;3722\nchakuza;3723\ncheek;3724\ncro;3725\narc_ngel;3726\nalexis_fido;3727\ndargen_d_amico;3728\nthe_coup;3729\ndef_con_dos;3730\ndie_fantastischen_vier;3731\ndom_no;3732\ndonguralesko;3733\nepmd;3734\nkool_savas;3735\nfettes_brot;3736\nfronda;3737\nmc_solaar;3738\npyhimys;3739\nkaaris;3740\nkollegah;3741\nkontra_k;3742\nk_k;3743\nl_o_c;3744\nlogic;3745\njerry_rivera;3746\nmurs;3747\nangie_stone;3748\nnamie_amuro;3749\nanthony_hamilton;3750\nlyfe_jennings;3751\nbl_f;3752\no_s_t_r;3753\npaluch;3754\nparazi_ii;3755\nporta;3756\nbleeding_through;3757\nprinz_pi;3758\nrasmentalism;3759\nxavier_naidoo;3760\nsage_francis;3761\nstupeflip;3762\nyoung_thug;3763\ntego_calder_n;3764\nfifth_harmony;3765\njay_chou;3766\nblitzkid;3767\nzumbis_do_espa_o;3768\ndeer_tick;3769\nhalf_man_half_biscuit;3770\nhayden;3771\nclub_8;3772\ngrandaddy;3773\njens_lekman;3774\nkent;3775\nkeren_ann;3776\nlos_campesinos;3777\nnellie_mckay;3778\nchina_crisis;3779\nprefab_sprout;3780\nthe_clientele;3781\nthe_lucksmiths;3782\nbell_x1;3783\nbritish_sea_power;3784\ncar_seat_headrest;3785\ndeerhunter;3786\ndr_dog;3787\nelf_power;3788\nfrightened_rabbit;3789\nfugazi;3790\nfury_in_the_slaughterhouse;3791\njulie_doiron;3792\ntinashe;3793\nla_habitaci_n_roja;3794\nmargot_the_nuclear_so_and_so_s;3795\nmatt_pond_pa;3796\nmetric;3797\nmike_doughty;3798\nmother_mother;3799\npiebald;3800\nquasi;3801\nrheostatics;3802\nsebadoh;3803\nspoon;3804\nstarflyer_59;3805\nstephen_malkmus;3806\nstereolab;3807\nted_leo_and_the_pharmacists;3808\nthe_appleseed_cast;3809\nthe_faint;3810\nthe_go_betweens;3811\nthe_pineapple_thief;3812\nthe_undertones;3813\ntronic;3814\nchris_de_burgh;3815\nmass_hysteria;3816\nangelo_branduardi;3817\ngigi_d_alessio;3818\ni_muvrini;3819\nback_number;3820\nboa;3821\nclaris;3822\ncrystal_kay;3823\nzard;3824\ngackt;3825\ngarnet_crow;3826\ngirls_generation;3827\nkat_tun;3828\nkoda_kumi;3829\nkotoko;3830\nlisa;3831\nmaaya_sakamoto;3832\nmasami_okui;3833\nmr_children;3834\nnews;3835\nshinee;3836\nw_inds;3837\nyui;3838\nyumi_matsutoya;3839\nthe_high_lows;3840\nsid;3841\nabbey_lincoln;3842\nanna_maria_jopek;3843\ncassandra_wilson;3844\ndianne_reeves;3845\nfred_buscaglione;3846\njane_monheit;3847\nzor_n;3848\nkraan;3849\nlaura_fygi;3850\nmichael_franks;3851\nnatalino_otto;3852\nquartetto_cetra;3853\nscott_bradlee_s_postmodern_jukebox;3854\nstacey_kent;3855\nthe_flower_kings;3856\nronnie_von;3857\nbrown_eyed_girls;3858\nahmet_kaya;3859\nalejandra_guzm_n;3860\nana_carolina;3861\nalcione;3862\nel_chapo_de_sinaloa;3863\ngustavo_cerati;3864\nsoda_stereo;3865\njenni_rivera;3866\njoaqu_n_sabina;3867\nlos_fabulosos_cadillacs;3868\nabel_pintos;3869\nana_bel_n;3870\naterciopelados;3871\ncamilo_sesto;3872\ndavid_demar_a;3873\ngian_marco;3874\nmenudo;3875\nricardo_arjona;3876\nsabroso;3877\nv_ctor_manuel;3878\nlas_pelotas;3879\nariel_pink;3880\nleehom_wang;3881\njolin_tsai;3882\ndarkest_hour;3883\nkalmah;3884\nnightrage;3885\neppu_normaali;3886\nthe_outfield;3887\nno_use_for_a_name;3888\npennywise;3889\ncallejon;3890\nd_f_c;3891\nour_last_night;3892\nexaltasamba;3893\nbeth_carvalho;3894\njo_o_bosco;3895\nmarina_lima;3896\nmarisa_monte;3897\nnando_reis;3898\nnatiruts;3899\nra_a_negra;3900\ns_pra_contrariar;3901\nzeca_pagodinho;3902\nandr_hazes;3903\nde_dijk;3904\narena;3905\niq;3906\nsol_invictus;3907\nnew_found_glory;3908\nadam_ant;3909\nberlin;3910\nhoodoo_gurus;3911\nultravox;3912\nnik_kershaw;3913\nsqueeze;3914\nthe_aquabats;3915\nthe_fixx;3916\nbeat_crusaders;3917\ncows;3918\nconjunto_primavera;3919\npeter_and_the_test_tube_babies;3920\nsham_69;3921\nthe_adicts;3922\nthe_analogs;3923\ninstalok;3924\njacek_kaczmarski;3925\nprzemys_aw_gintrowski;3926\nada_band;3927\nagnetha_f_ltskog;3928\najda_pekkan;3929\nal_bano;3930\nalex_ubago;3931\nalison_moyet;3932\nalunni_del_sole;3933\nanna_oxa;3934\nbajm;3935\nbarclay_james_harvest;3936\nblue_system;3937\nbrunner_brunner;3938\ncandan_er_etin;3939\nchristian_bautista;3940\nclay_aiken;3941\nclifford_t_ward;3942\ndaniel;3943\ndon_backy;3944\njesse_mccartney;3945\nemma;3946\nmarcella_bella;3947\ngiorgio_gaber;3948\nguus_meeuwis;3949\nheinz_rudolf_kunze;3950\njohn_farnham;3951\nian_thomas;3952\ni_n_karaca;3953\njennifer_rush;3954\njo_vally;3955\njohn_fogerty;3956\njulian_lennon;3957\nk3;3958\nkid_abelha;3959\nlabv_l_gais_tips;3960\nl_vi;3961\nlea_salonga;3962\nles_wampas;3963\nmagnus_uggla;3964\nmango;3965\nmaria_mena;3966\nmassimo_ranieri;3967\nmax_gazz;3968\nmichael_learns_to_rock;3969\nmietta;3970\nmustafa_sandal;3971\nnil_fer;3972\npeter_frampton;3973\npr_ta_v_tra;3974\npur;3975\nrettore;3976\nricchi_e_poveri;3977\nrob_de_nijs;3978\nsara_bareilles;3979\nsasha;3980\nsertab_erener;3981\nsezen_aksu;3982\nstadio;3983\nstephen_sondheim;3984\ntamara;3985\nteam_starkid;3986\ntoto_cutugno;3987\numberto_tozzi;3988\nherman_brood;3989\nwanessa;3990\nzen_caf;3991\nbonanza_banzai;3992\nbodyjar;3993\nbracket;3994\nfrenzal_rhomb;3995\ngoldfinger;3996\nthe_wonder_years;3997\nuseless_id;3998\ncamel;3999\nhombres_g;4000\nleo_jaime;4001\nneal_morse;4002\nspock_s_beard;4003\nnew_trolls;4004\nopus;4005\npiersi;4006\npremiata_forneria_marconi;4007\nsuperbus;4008\nzmelkoow;4009\nboysetsfire;4010\nhot_water_music;4011\nnew_model_army;4012\nthe_monochrome_set;4013\nbig_big_train;4014\navantasia;4015\ndark_moor;4016\ndreamtale;4017\nfreedom_call;4018\nmystic_prophecy;4019\nnightmare;4020\nrhapsody_of_fire;4021\nroyal_hunt;4022\nsonata_arctica;4023\nstratovarius;4024\nsymphony_x;4025\nvision_divine;4026\nthe_wildhearts;4027\narmia;4028\nevergrey;4029\nlana_lane;4030\nnektar;4031\npain_of_salvation;4032\nriverside;4033\nbeardfish;4034\necholyn;4035\neloy;4036\njohn_wetton;4037\nmedina_azahara;4038\nmostly_autumn;4039\npendragon;4040\nrafo_r_ez;4041\nthe_meteors;4042\nagainst_me;4043\nanti_flag;4044\nbanda_bassotti;4045\ncadena_perpetua;4046\ndescendents;4047\ndistemper;4048\ndogwood;4049\nel_ltimo_ke_zierre;4050\nfarben_lehre;4051\ntoy_dolls;4052\njunkies;4053\nksu;4054\nla_polla_records;4055\nla_vela_puerca;4056\nleatherface;4057\nless_than_jake;4058\nmad_caddies;4059\nmillencolin;4060\npunkreas;4061\nreel_big_fish;4062\nsnfu;4063\nstiff_little_fingers;4064\nswingin_utters;4065\nthe_bouncing_souls;4066\nthe_casualties;4067\nthe_dickies;4068\nthe_lawrence_arms;4069\ntoyah;4070\ngerald_levert;4071\ngondwana;4072\nlos_aut_nticos_decadentes;4073\nlos_cafres;4074\nlos_pericos;4075\ntryo;4076\nrakim_ken_y;4077\nbilly_squier;4078\nbj_rn_afzelius;4079\nglay;4080\nhunters_collectors;4081\njohn_entwistle;4082\njokke;4083\nla_beriso;4084\nlos_rancheros;4085\nlos_tres;4086\nmaanam;4087\nmikel_erentxun;4088\npeter_wolf;4089\nracoon;4090\nrev_lver;4091\nriblja_orba;4092\nsandro;4093\ngene_vincent;4094\nthe_baseballs;4095\nstray_cats;4096\nas_marcianas;4097\nbruno_marrone;4098\ncristiano_ara_jo;4099\nfernando_sorocaba;4100\njoint_venture;4101\nserge_reggiani;4102\nska_p;4103\nthe_mighty_mighty_bosstones;4104\nfu_manchu;4105\njay_jay_johanson;4106\npsyche;4107\ncarlos_gardel;4108\n" }, { "path": "jukebox/data/ids/v2_genre_ids.txt", "content": "unknown;0\nclassical;1\nblues;2\nhip;3\nhop;4\ndance;5\nsoul;6\nhard;7\nrock;8\njazz;9\nreggae;10\ncountry;11\nalternative;12\nsoundtrack;13\npop;14\nbluegrass;15\nvocal;16\nr;17\nb;18\nrap;19\nchristian;20\ngospel;21\nelectronic;22\nchristmas;23\nsinger;24\nsongwriter;25\nmetal;26\nn;27\nroll;28\nsynthpop;29\nelectronica;30\nmpb;31\nmovie;32\nindie;33\nnew;34\nwave;35\nelectro;36\nhouse;37\nfolk;38\npunk;39\nfrench;40\ncontemporary;41\ngarage;42\nsoft;43\nacoustic;44\nnu;45\ntelevision;46\npost;47\neurodance;48\nprogressive;49\ngothic;50\nclassic;51\nfunk;52\ndisco;53\nswing;54\ntrance;55\nthrash;56\npsychedelic;57\nheavy;58\namerican;59\ngrunge;60\nart;61\nj;62\ngangsta;63\nbrazilian;64\nlatin;65\nsouthern;66\nska;67\ncrossover;68\nhardcore;69\nindustrial;70\nglam;71\nmelodic;72\nambient;73\nmusical;74\ndream;75\nexperimental;76\namericana;77\nchanson;78\nrockabilly;79\nbritpop;80\nchildren;81\ns;82\nmusic;83\nelectropop;84\npower;85\nceltic;86\ndark;87\ncomedy;88\ndoom;89\ntrip;90\nlo;91\nfi;92\nmetalcore;93\nsymphonic;94\nfado;95\nschlager;96\navant;97\ngarde;98\neuropop;99\nreggaeton;100\nemo;101\ndeath;102\nsamba;103\ndeathcore;104\nblack;105\nhorrorcore;106\ngrindcore;107\nworship;108\nsalsa;109\nebm;110\nneofolk;111\nsertanejo;112\ndeutschrock;113\nnorte;114\no;115\nax;116\nk;117\ntejano;118\nmedieval;119\n" }, { "path": "jukebox/data/ids/v3_artist_ids.txt", "content": "beat farmers;1\naaron sprinkle;2\ndianne reeves;3\nlowe;4\nharry manx;5\nhail of bullets;6\nian gillan;7\nandraé crouch;8\nwidespread panic;9\nbuddy wasisname and the other fellers;10\nmisery index;11\nalbert west;12\nshadowland;13\nhomer & jethro;14\ndamien jurado;15\ndead to fall;16\nbritish sea power;17\npam tillis;18\nice cube;19\nhey rosetta!;20\nsophie zelmani;21\nriverside;22\nhead automatica;23\ndiabulus in musica;24\nunitopia;25\nrevolting cocks;26\nzita swoon;27\ntrain;28\nken stringfellow;29\nin dying arms;30\nred lorry yellow lorry;31\nsmall faces;32\nmichael sweet;33\n30 odd foot of grunts;34\nwhite heart;35\nbaby bash;36\nbad bones;37\nmeat beat manifesto;38\nvengeance;39\nnaomi;40\nkoritni;41\nthe fall of troy;42\nsplit enz;43\nemmy rossum;44\nles fleur de lys;45\nbeaux arts trio;46\ndavid crowder band;47\nmojave 3;48\ngirl talk;49\nmotorpsycho;50\nburning point;51\nthe rutles;52\ndavid and the giants;53\njinjer;54\nsitd;55\npedro the lion;56\nmasta ace;57\nalexz johnson;58\nthe floacist;59\nafter 7;60\nanointed;61\nholy soldier;62\nsanchez;63\nwovenhand;64\nthea gilmore;65\n3t;66\npatty loveless;67\nghost;68\nrie fu;69\nchemical vocation;70\nrobbie nevil;71\nthe notorious b.i.g.;72\namerica;73\nthe boo radleys;74\nin hearts wake;75\njack the lad;76\ngerry and the pacemakers;77\nhe is we;78\ncuban link;79\ngalaxie 500;80\nsomething with numbers;81\nthe last shadow puppets;82\nminor threat;83\njoss stone;84\nlynch mob;85\nzino francescatti;86\ngenitorturers;87\nkenny g;88\ngraveworm;89\nfield mob;90\nopus;91\njordan smith;92\nsheppard;93\nthe haunted;94\ntiny ruins;95\njimmy somerville;96\nacid reign;97\nfalling in reverse;98\nace troubleshooter;99\njosh groban;100\nadriano celentano;101\njohn oates;102\nmind funk;103\nchristafari;104\nclan of xymox;105\nanti-flag;106\nthe blow monkeys;107\nthe troggs;108\npriscilla ahn;109\nfastball;110\nraekwon;111\nroyal wood;112\nagoraphobic nosebleed;113\nborknagar;114\nparker millsap;115\nkelly osbourne;116\npsyche;117\nbrokencyde;118\ngeorge clinton;119\nthe hollies;120\ngabriel kahane;121\ndnce;122\njimmy nail;123\nharem scarem;124\npierre fournier;125\ngideon;126\nelitist;127\nthe sheepdogs;128\nlike moths to flames;129\nthe constructus corporation;130\nimpending doom;131\njoe williams;132\nbizzy bone;133\nnelson;134\nearth and fire;135\nunderoath;136\nrancid;137\nexile;138\nvertical horizon;139\npercy sledge;140\nill bill;141\n59 times the pain;142\njimmy dean;143\ngary jules;144\nspellblast;145\nrenee olstead;146\nbarbra streisand;147\nspin doctors;148\ngalt macdermot;149\ntakara;150\nalan stivell;151\nandy davis;152\nbabes in toyland;153\nstill remains;154\nthe donnas;155\nbishop allen;156\nthe skids;157\nrhiannon giddens;158\nnatalia;159\nhenson cargill;160\ngov't mule;161\njools holland;162\nkehlani;163\nlondonbeat;164\nandy mineo;165\ncorky and the juice pigs;166\ndays away;167\na fine frenzy;168\nroger mcguinn;169\nlena horne;170\nshark island;171\nmachinemade god;172\nyank rachell;173\nhurricane;174\nhis statue falls;175\nthat petrol emotion;176\n764-hero;177\nleprous;178\nbridgit mendler;179\nbeggars opera;180\nabbie gale;181\nthe the;182\ny'akoto;183\nsound tesselated;184\nwebb pierce;185\nriver whyless;186\nronnie dio & the prophets;187\nrotting christ;188\nduff mckagan;189\nslim harpo;190\nadele;191\nvalencia;192\nthe damned;193\nmiguel;194\nchantal kreviazuk;195\nthe db's;196\ncartel;197\nenrique iglesias;198\nskrewdriver;199\none less reason;200\nlil wayne;201\nchris norman;202\ntype o negative;203\ntrip shakespeare;204\njack blanchard & misty morgan;205\nfishboy;206\nted leo and the pharmacists;207\nlukas graham;208\nthe vapors;209\nconway twitty & loretta lynn;210\nsandie shaw;211\nmark knopfler;212\nthrough the eyes of the dead;213\nart of dying;214\nfree;215\nsaint motel;216\nsonreal;217\ngatsbys american dream;218\nelisa;219\nmarc anthony;220\njoan baez;221\nsomeone still loves you boris yeltsin;222\nugk;223\ndeep purple;224\nmother mother;225\nthe contortionist;226\nhot chelle rae;227\neric clapton;228\nthe doobie brothers;229\njohn michael montgomery;230\nizegrim;231\njason collett;232\nclose your eyes;233\nsnog;234\nghostpoet;235\nnew order;236\nthe brian setzer orchestra;237\nroyal tusk;238\nguy mitchell;239\nheart;240\nthe free design;241\nbillie ray martin;242\ntoto;243\ndavid mallett;244\ndonovan;245\nthe years gone by;246\nelement 101;247\nfairyland;248\ntriggerfinger;249\nmc eiht;250\nottorino respighi;251\nthe four aces;252\nlil son jackson;253\nemanuel feuermann;254\njuliette and the licks;255\np!nk;256\ngretchen wilson;257\nthe animals;258\nlocksley;259\nredgum;260\nyoung mc;261\nmetronomy;262\nashland high;263\nesoteric;264\njohnny hates jazz;265\npaul anka;266\nethel merman;267\neast west;268\nthe knife;269\ncurren$y;270\nmaaya sakamoto;271\naesthetic perfection;272\nbobby helms;273\njimi jamison;274\ndarren styles;275\nlorrie morgan;276\nmiley cyrus;277\ndropdead;278\ndr. sin;279\nburt bacharach;280\nnf;281\nastronautalis;282\ngarth brooks;283\nflyleaf;284\nlake;285\nmad sin;286\ntiffany evans;287\nmudvayne;288\nmáni svavarsson & magnús scheving;289\nunexpect;290\ncollin raye;291\njohnny reid;292\nantonio vivaldi;293\ncreed;294\nburning heads;295\nlegion of the damned;296\nmatt costa;297\nthe aluminum group;298\norgy;299\n2nd chapter of acts;300\nshotgun messiah;301\nmentallo & the fixer;302\nurma;303\ncarmen mcrae;304\nskid row;305\njohn denver & the muppets;306\nangie stone;307\nrob rock;308\nclara haskil;309\nmorandi;310\nteam dresch;311\nthe zombies;312\nmr. vegas;313\nroyal trux;314\nsuzanne vega;315\nthe afters;316\nskydiggers;317\nsunday's best;318\ngary numan;319\nthree dog night;320\njonas brothers;321\nuncle acid & the deadbeats;322\npablo de sarasate;323\ngoretrade;324\nstrapping young lad;325\nfat joe;326\nrobert johnson;327\nfred astaire;328\nanastacia;329\ndevendra banhart;330\ninto it. over it.;331\njohn norum;332\ncross canadian ragweed;333\ndestroyer;334\nmichael christmas;335\nphobia;336\njill tracy;337\ndilana;338\nroyce da 5'9\";339\nles savy fav;340\nthe blow;341\nkim wilde;342\nparts & labor;343\ndinah washington;344\nmaggie reilly;345\nscreaming trees;346\np.o.s.;347\natomic rooster;348\nchamillionaire;349\nthe vaccines;350\ntides of man;351\nheathen;352\nflame;353\nbrain drill;354\nac/dc;355\nkraan;356\nscary kids scaring kids;357\nrosaline;358\njohn legend;359\nof montreal;360\nthe brunettes;361\nshelley fabares;362\nvolumes;363\ngeorge enescu;364\njacob's dream;365\nheartless bastards;366\ndarin;367\nandy stochansky;368\ndavid geringas;369\nlucius;370\nsteep;371\nbobby vinton;372\nshania twain;373\nrudolf serkin;374\nthe zolas;375\nmunicipal waste;376\nspectral;377\narcade fire;378\nsteve hillage;379\nthe presets;380\ngustav mahler;381\ngary morris;382\nlaura cantrell;383\ndean brody;384\nroger miller;385\ntammy wynette;386\njoe cocker;387\niceage;388\napostasy;389\ntait;390\nreverend gary davis;391\nneverending white lights;392\nmimicking birds;393\nbarney;394\nmajor parkinson;395\nseal;396\nwham!;397\ntha dogg pound;398\nbig l;399\nian thomas;400\nkronos;401\ndom pachino;402\ndead can dance;403\nthe number twelve looks like you;404\nbert williams;405\nbedhead;406\nscott bradlee's postmodern jukebox;407\nmonuments;408\nchristine mcvie;409\nmoonspell;410\ndavid & the citizens;411\n*nsync;412\ntiny tim;413\nsurface;414\nk.flay;415\ntravis scott;416\nlil jon;417\njo stafford;418\nelo part ii;419\nsugarland;420\neternal;421\nthe dingees;422\nthe summer set;423\nsoft machine;424\nmaanam;425\nright said fred;426\nchicks on speed;427\nfoetus;428\nfiona apple;429\nprimer 55;430\nthe dillinger escape plan;431\nseahaven;432\nbiga ranx;433\nthe insyderz;434\nthirty seconds to mars;435\npage france;436\nhowlin' wolf;437\nwishbone ash;438\nnina sky;439\njess moskaluke;440\nstan rogers;441\nb.o.b;442\ncypecore;443\nyoung dro;444\njulian lennon;445\nopeth;446\nflying lotus;447\nrodney atkins;448\nsea of treachery;449\nmontrose;450\nnellie mckay;451\nvladimir horowitz;452\nfatboy slim;453\nmystic prophecy;454\nlittle river band;455\nbrooklyn bounce;456\ndestroid;457\nmary hopkin;458\nelliott yamin;459\nbilly bragg;460\nthe doors;461\nesham;462\ncab calloway;463\nthi'sl;464\nthe gothsicles;465\ndavid coverdale;466\njoe henry;467\nthe human abstract;468\nalger \"texas\" alexander;469\ndiane cluck;470\nfozzy;471\nzero 7;472\ncole swindell;473\ngladys knight & the pips;474\ndonna fargo;475\ncave in;476\neiffel 65;477\nfates warning;478\ndecrepit birth;479\nbad religion;480\npoison clan;481\nshane & shane;482\njohnny shines;483\nu.n.l.v.;484\nseth lakeman;485\nmindy smith;486\njosh white;487\nandroid lust;488\nmylon lefevre;489\naselin debison;490\nkaskade;491\nthe stills;492\nalpha blondy;493\nhughes turner project;494\nspice girls;495\nzz top;496\nfairport convention;497\nthe ritchie family;498\neleanor friedberger;499\nlaura branigan;500\nthe jordanaires;501\nthe bacon brothers;502\natomic opera;503\nspike jones;504\nfaith hill;505\nmandy moore;506\njan werner;507\nkittie;508\nedwin;509\nmichael roe;510\nleeland;511\nsammy hagar;512\nfrankjavcee;513\nthe bangles;514\njoey mcintyre;515\ndavid rovics;516\nacross the border;517\nodd future;518\nbill ward;519\neddy grant;520\nboa;521\nnirvana;522\ndarzamat;523\ned sheeran;524\nthe prodigy;525\nwang chung;526\nbalance problems;527\nvalient thorr;528\nrupaul;529\nroy clark;530\nross lynch;531\nugly kid joe;532\nbettye lavette;533\nharry belafonte;534\nroy buchanan;535\nmiguel bosé;536\ngreenslade;537\nliving legends;538\nbing crosby;539\nadam sandler;540\nthe czars;541\nbethany dillon;542\nlea salonga;543\nkmfdm;544\nthe diplomats;545\nmagneta lane;546\nmira;547\ng herbo;548\nissues;549\nbeastie boys;550\nmarvin gaye;551\nashes you leave;552\nmordred;553\nisrael houghton;554\nscreaming mechanical brain;555\nunknown hinson;556\njack johnson;557\ndo;558\nguns n' roses;559\noctober project;560\nadore delano;561\njedi mind tricks;562\nandrew peterson;563\nmillionaires;564\nthe beatnuts;565\ngilby clarke;566\nchickenfoot;567\nthe stranglers;568\nrev theory;569\nthe mccalmans;570\ndrowning pool;571\nkutt calhoun;572\ndark fortress;573\nthe undertones;574\nkevin gilbert;575\nffh;576\nseven places;577\nfury in the slaughterhouse;578\ncovenant;579\njason isbell;580\nthe creepshow;581\nashbury heights;582\nshakey graves;583\nbrett young;584\nlords of black;585\nthe higher;586\njudy garland;587\nboy harsher;588\nstatus quo;589\niq;590\nunderworld;591\nkrizz kaliko;592\njefferson airplane;593\nbilly walker;594\njackie lomax;595\nlizzy borden;596\nkeke wyatt;597\nclosterkeller;598\nagnostic front;599\nmary mary;600\nbirds in row;601\nmugison;602\nrandy travis;603\nglorior belli;604\namorphis;605\nmartika;606\njason webley;607\nduke ellington;608\neurope;609\nthe wilkinsons;610\na bullet for pretty boy;611\njodeci;612\nsister hazel;613\natrocity;614\nlittle willie john;615\nalexander borodin;616\nbelouis some;617\nbig boi;618\nnewworldson;619\nmuddy waters;620\nkaren elson;621\nlou bega;622\nivoryline;623\npain confessor;624\ndolour;625\ncaptain beefheart and his magic band;626\nbarclay james harvest;627\ntodd snider;628\nenslaved;629\nbeach fossils;630\ntrick daddy;631\nthe black dahlia murder;632\nrhapsody of fire;633\ncemetary;634\npatsy cline;635\nfigure four;636\nmanuel de falla;637\nneil diamond;638\nsworn enemy;639\nelvenking;640\nd.r.a.m.;641\nsonya kitchell;642\nflight of the conchords;643\neddie from ohio;644\ntalk talk;645\nthoushaltnot;646\nthis is the kit;647\ncrimson glory;648\nthe bears;649\namenra;650\ndoris day;651\ndeath in june;652\naaron copland;653\nastrud gilberto;654\nluna;655\nfury;656\ncorpus christi;657\nsoul position;658\nbe'lakor;659\nroy orbison;660\nbeyond dawn;661\nel-p;662\nwatch tower bible and tract society;663\nend of you;664\nfalconer;665\nwar from a harlots mouth;666\nchina;667\nerra;668\nbrainpool;669\npsyclon nine;670\nquintorigo;671\nthe blind boys of alabama;672\nmr. big;673\nreverend horton heat;674\nyehuda hanani;675\nbell x1;676\njohn michael talbot;677\nsigh;678\njames brown;679\nthe murder of my sweet;680\ncourtney marie andrews;681\nkate alexa;682\njasmine v;683\nmalevolent creation;684\nsaid the whale;685\nthe indelicates;686\nmasterplan;687\nevery time i die;688\nechosmith;689\nbarzin;690\nthelma houston;691\nmasters of reality;692\ntony iommi;693\nsex gang children;694\nchaos uk;695\ncasper & the cookies;696\njohnny gill;697\nalex harvey;698\nlemar;699\nandre matos;700\nshinee;701\nearl sweatshirt;702\nsanctus real;703\nchilliwack;704\nlionel hampton;705\nfaith assembly;706\ndave van ronk;707\nfrankie goes to hollywood;708\nbadfinger;709\ngazpacho;710\ncentro-matic;711\ndonald lawrence;712\nme first and the gimme gimmes;713\nmargot & the nuclear so and so's;714\nbig star;715\nunisonic;716\nthe delgados;717\n28 days;718\nveil of maya;719\ntammi terrell;720\nthe righteous brothers;721\ndavid knopfler;722\nihsahn;723\nhigh inergy;724\nleaves' eyes;725\nparov stelar;726\nthe retrosic;727\ntony rose;728\ntom lehrer;729\nbuggles;730\ndennis brown;731\ntrashcan sinatras;732\npolarkreis 18;733\nmavis staples;734\nxzibit;735\nblack moth super rainbow;736\nyello;737\nthe servant;738\njana mashonee;739\nburl ives;740\nbeyond the black;741\ntsjuder;742\nhelen o'connell;743\ngolden gate quartet;744\ndebby boone;745\nbeady eye;746\nacid king;747\nwestlife;748\nbig wreck;749\nmanowar;750\nalmôra;751\nsarah darling;752\nkenny white;753\ndelta spirit;754\ncurtis stigers;755\nfun.;756\nfeist;757\nkarate;758\nsleepy john estes;759\nlamb;760\nthe roches;761\ntuatha de danann;762\nhorrified;763\nfort minor;764\njesse harris;765\nberman;766\nshannon & the clams;767\nindigo girls;768\nthe matches;769\neric stewart;770\njls;771\nkristin hersh;772\nchriston gray;773\nloreena mckennitt;774\ncharley pride;775\njocelyn enriquez;776\nhelen baylor;777\nhot tuna;778\nskeeter davis;779\nbrenton brown;780\nfrom good homes;781\nleslie hall;782\nnatalia kills;783\nchris thompson;784\ncircle of dust;785\ntransmetal;786\naustralian crawl;787\ndying fetus;788\nratcat;789\nthe waifs;790\nelevation worship;791\nbryan ferry;792\ncamille;793\npatty griffin;794\npapa charlie jackson;795\ngrendel;796\nchris murray;797\nryan stevenson;798\nkiethevez;799\nthe glorious unseen;800\nthe ocean blue;801\nconducting from the grave;802\nnegativland;803\ndel shannon;804\nkathryn scott;805\nthis ending;806\nbomfunk mc's;807\nopera ix;808\nsteps;809\nthe birthday massacre;810\nbronski beat;811\nthe burns sisters;812\nwilly deville;813\ngirlicious;814\ndeath angel;815\nthe quakes;816\nwilliam fitzsimmons;817\nfm;818\nbeggars & thieves;819\nrichard buckner;820\niwrestledabearonce;821\nhere come the mummies;822\nonly crime;823\nsaint saviour;824\nmillencolin;825\nbigwig;826\nbenny mardones;827\nwale;828\nfrida;829\nbjörn ulvaeus & benny andersson;830\nartrosis;831\nneurosis;832\nelis;833\nspain;834\ny&t;835\njeff lynne;836\nsyreeta;837\nryan adams;838\nthe lords of the new church;839\nstephen stills;840\njoel plaskett emergency;841\nblood or whiskey;842\nchenoa;843\nl.t.d.;844\nmariah carey;845\nbauhaus;846\nemma ruth rundle;847\nbilly bragg & wilco;848\nsara evans;849\nsara bareilles;850\ncold world;851\npig destroyer;852\nwilliam elliott whitmore;853\neyes set to kill;854\nagressor;855\nskiltron;856\noysterband;857\nversaemerge;858\nnow, now;859\nchelsea grin;860\nkaiser/mansfield;861\ngeorge thorogood & the destroyers;862\nfoghat;863\nxtc;864\nthe nighthawks;865\neric burdon & war;866\nthe cross movement;867\nthe winery dogs;868\nedyta górniak;869\nhexrx;870\nthe cheeky girls;871\ngazebo;872\ncanned heat;873\nanne sophie mutter;874\nsergei prokofiev;875\nbeneath the sky;876\njimmy reed;877\nannihilator;878\nthe essex green;879\njohn west;880\nbloodhound gang;881\nbeth hart & joe bonamassa;882\nlacrimas profundere;883\njbm;884\ncommander cody and his lost planet airmen;885\nsmoking popes;886\nseeed;887\nmoon martin;888\nterence trent d'arby;889\nthe darkest of the hillside thickets;890\npeter tosh;891\nthrowdown;892\nthe allman brothers band;893\ncaribou;894\naxel rudi pell;895\nne-yo;896\njoe ely;897\njames blake;898\nmacklemore & ryan lewis;899\n王力宏 (leehom wang);900\nsam brown;901\nvan morrison;902\nbella morte;903\njosé feliciano;904\njohn popper;905\ngenghis tron;906\nb3;907\nphil keaggy;908\nalesha dixon;909\npeace, love and pitbulls;910\nagents of mercy;911\nelton john;912\netta james;913\nplus one;914\nspacemen 3;915\ntommy castro;916\ngod forbid;917\nabysmal dawn;918\ncadillac blindside;919\na*teens;920\nq5;921\nmr. president;922\nrichmond fontaine;923\npolly paulusma;924\nhear'say;925\nvertical church band;926\nelvis costello;927\nnickelback;928\nthe bolshoi;929\nkenny loggins;930\nbad manners;931\ndear reader;932\ntom robinson band;933\ndelerium;934\nshirley caesar;935\ntrae;936\ntesseract;937\n500 miles to memphis;938\nsplitsville;939\nband of susans;940\nedl;941\nhalf-a-mill;942\nmechanical poet;943\nclay aiken;944\nmars argo;945\npalisades;946\nian hunter;947\ntracey thorn;948\njackson heights;949\ndownplay;950\nold dominion;951\njamey johnson;952\ndeb talan;953\nlittle walter;954\ngrits;955\nmaren ord;956\nlongfellow;957\nlayzie bone;958\ncuby + blizzards;959\noszibarack;960\njillette johnson;961\nthe hundred in the hands;962\nculture beat;963\nfrozen plasma;964\npaul robeson;965\nalt-j;966\ndarkside;967\nother lives;968\nblackfoot;969\nthe beta band;970\nsally seltmann;971\nfoals;972\nrobin williamson;973\ngluecifer;974\ntristan prettyman;975\nguy sebastian;976\npink guy;977\njohann sebastian bach;978\nmethyl ethel;979\nactive child;980\njohn anderson;981\nneil halstead;982\nphantom planet;983\njohn d. loudermilk;984\ngreg long;985\nthe limeliters;986\npeggy seeger;987\nb la bart k;988\nfallujah;989\nshanice;990\ndarius danesh;991\npost malone;992\naudrey;993\nparry gripp;994\nbarry white;995\nravenous;996\nnils lofgren;997\npaddy goes to holyhead;998\nyefim bronfman;999\nprefab sprout;1000\nrose funeral;1001\nthe boxmasters;1002\neluveitie;1003\ntony yayo;1004\nmichael learns to rock;1005\ndawes;1006\nlodger;1007\nthe joe perry project;1008\nmandragora scream;1009\nkristene dimarco;1010\ntoday is the day;1011\nriot;1012\nskold;1013\npendragon;1014\nel debarge;1015\nthe wanted;1016\nthe pharcyde;1017\njason derulo;1018\nblack light burns;1019\nraintime;1020\nlisa hannigan;1021\nmoby;1022\ntedeschi trucks band;1023\nbreaking laces;1024\ndeströyer 666;1025\nglenn hughes;1026\nmartin carthy and dave swarbrick;1027\nthe verve;1028\norleans;1029\nthe browns;1030\nthe drums;1031\nbetween the buried and me;1032\nthe kingston trio;1033\njean shepard;1034\nalmah;1035\ngare du nord;1036\nthe bellamy brothers;1037\nsandra;1038\nthe twang;1039\ngorod;1040\npandora;1041\nmick jagger;1042\nrebellion;1043\nlauren hoffman;1044\npoco;1045\nclara smith;1046\noscar peterson;1047\nslobberbone;1048\npitchshifter;1049\nhania;1050\nziggy marley;1051\nbillie jo spears;1052\ndum dum girls;1053\ntricky;1054\nlamont dozier;1055\nslingshot dakota;1056\nkoko taylor;1057\njudas priest;1058\nidiot stare;1059\nolivier messiaen;1060\nakala;1061\nit dies today;1062\nfred eaglesmith;1063\njessie james;1064\nmoving mountains;1065\nknights of the abyss;1066\ngregorian;1067\nmr weebl;1068\njohnnie allan;1069\nnewton faulkner;1070\nlonely kings;1071\nal stewart;1072\nbroods;1073\nzyklon;1074\nbasia;1075\ndamaged;1076\nthe reign of kindo;1077\nblack bomb a;1078\nvic damone;1079\ndesert rose band;1080\nswing out sister;1081\narjen anthony lucassen;1082\nkerrs pink;1083\nlevel 42;1084\nthe dollyrots;1085\ngiant squid;1086\njamie cullum;1087\nfritz kalkbrenner;1088\nthe whispers;1089\npilot speed;1090\nadhesive;1091\nleona lewis;1092\nhank williams;1093\nchris botti;1094\ncreeper;1095\ntori amos;1096\nevocation;1097\ngothminister;1098\nmandolin orange;1099\nnamie amuro;1100\nblack sheep;1101\nbleed the sky;1102\nlaura pausini;1103\nconsequence;1104\nfever ray;1105\nthird day;1106\nbed ich smetana;1107\nthe chain gang of 1974;1108\ncomes with the fall;1109\nduff mckagan's loaded;1110\nisrael kamakawiwo'ole;1111\npixie lott;1112\nbruce dickinson;1113\nsonny landreth;1114\nsqueeze;1115\npennywise;1116\nred café;1117\nthe autumn offering;1118\ncrashdïet;1119\nneil young;1120\nhi-tek;1121\nhanson;1122\nthe blues brothers;1123\nsnow tha product;1124\nibeyi;1125\ncarpathian forest;1126\nsheb wooley;1127\nrussian red;1128\namerican authors;1129\nnick lachey;1130\njurassic 5;1131\nthe smashing pumpkins;1132\nthe lyric quartet;1133\nhoward shore;1134\njulian lloyd webber;1135\nsyleena johnson;1136\nwolf alice;1137\nnico;1138\nbeach slang;1139\nrobin zander;1140\ntanya tucker;1141\nmac;1142\nmogg/way;1143\nshaggy 2 dope;1144\ngodley & creme;1145\nn.e.r.d;1146\ncarbon leaf;1147\naugust burns red;1148\n1349;1149\nsmokey robinson & the miracles;1150\nröyksopp;1151\nstephanie mills;1152\nhalestorm;1153\nwebb wilder;1154\nchris stapleton;1155\npaul oakenfold;1156\nplanetshakers;1157\nandrew belle;1158\nbaha men;1159\nmemphis willie b.;1160\nandromeda;1161\ncynic;1162\ngil shaham;1163\ngeorge;1164\ntoad the wet sprocket;1165\nkiuas;1166\ncat power;1167\nmetric;1168\nsaving jane;1169\npatrick watson;1170\nfaith evans;1171\ntrivium;1172\nkelly willis;1173\nmy darkest days;1174\nthe reason;1175\ndavid gilmour;1176\nroo panes;1177\nsasha;1178\nsympathy;1179\ngo periscope;1180\namerican me;1181\nsex pistols;1182\njay-z;1183\ntiësto;1184\njean sibelius;1185\nwar;1186\nagent 51;1187\nthrice;1188\namel larrieux;1189\nthe futureheads;1190\narlo guthrie;1191\nthe saw doctors;1192\nhungry lucy;1193\nmichelle malone;1194\nkay starr;1195\nadestria;1196\nab-soul;1197\nthe merry wives of windsor;1198\nbo carter;1199\ncharlotte gainsbourg;1200\nsuicidal angels;1201\nzolof the rock & roll destroyer;1202\nbrian kennedy;1203\njess glynne;1204\nthe enemy;1205\ncrystal lewis;1206\nhopesfall;1207\nhelmet;1208\nalicia keys;1209\ntom milsom;1210\nkeith green;1211\nmacy gray;1212\nlibera;1213\nthe myriad;1214\nsteven wilson;1215\ndennis wilson;1216\nwar of ages;1217\nelliott brood;1218\nace of base;1219\nelefant;1220\nryker's;1221\napologetix;1222\nstream of passion;1223\ninterface;1224\nsusan tedeschi;1225\ncircle takes the square;1226\nteam starkid;1227\ndillinger four;1228\njt the bigga figga;1229\nmf doom;1230\ncarrie newcomer;1231\nbig & rich;1232\ncaliban;1233\nproject pat;1234\njoan osborne;1235\njuicy j;1236\nbeady belle;1237\nvictory;1238\nsyd barrett;1239\nliv kristine;1240\nanything box;1241\ndevin townsend project;1242\namanda marshall;1243\ndie krupps;1244\nsonny boy williamson ii;1245\nassuming we survive;1246\nsoko;1247\nmuch the same;1248\nfaunts;1249\nsally oldfield;1250\nthe bottle rockets;1251\nkaledon;1252\nlighthouse family;1253\nthe mākaha sons;1254\ndark moor;1255\nantagonist a.d.;1256\nlee dorsey;1257\nmerle travis;1258\ncursed;1259\npete rock;1260\nbrother cane;1261\n...and oceans;1262\nancient bards;1263\nthe searchers;1264\ncappella;1265\niced earth;1266\nmarmaduke duke;1267\ncolin meloy;1268\nvenomous concept;1269\nanne murray;1270\njay farrar;1271\nthe town pants;1272\nchubby checker;1273\ntori kelly;1274\nthe dodos;1275\nchris spedding;1276\npmtoday;1277\nthunder lord;1278\nbloodbath;1279\ndear criminals;1280\naaron carter;1281\nshai hulud;1282\nlil skies;1283\njeniferever;1284\njucifer;1285\nthe new pornographers;1286\nfabrizio faniello;1287\nfleurie;1288\nmirror of deception;1289\nthe real mckenzies;1290\nq-tip;1291\nneuraxis;1292\nrick derringer;1293\ncrystal kay;1294\nrobert randolph & the family band;1295\nnaâman;1296\nlittle jimmy dickens;1297\nsir mix-a-lot;1298\nkrs-one;1299\ndaniil trifonov;1300\nmorning glory;1301\ncheap trick;1302\nking tee;1303\nangela mccluskey;1304\nderdian;1305\nheitor villa lobos;1306\nnina;1307\ntyrese;1308\ndope stars inc.;1309\nvendetta red;1310\npussycat;1311\nbenjamin gibbard;1312\nnine;1313\nde lux;1314\nwilliam kapell;1315\nvomito negro;1316\na$ap rocky;1317\nmo b. dick;1318\ngamma ray;1319\nsarah vaughan;1320\ngeorges bizet;1321\nacid ranch;1322\ndeadline;1323\nlady sovereign;1324\nflipsyde;1325\njim jackson;1326\nviolent femmes;1327\nemeli sandé;1328\nla bionda;1329\nsammie;1330\njoe budden;1331\nbutterfly boucher;1332\nmartha and the muffins;1333\nthe faction;1334\nsohn;1335\nthe cyrkle;1336\nthe butchies;1337\nmagna-fi;1338\njennifer nettles;1339\nudora;1340\nmissing persons;1341\nhey mercedes;1342\ncracker;1343\nalphaville;1344\nbehexen;1345\nthe tremeloes;1346\nthe power station;1347\nmark collie;1348\njanet jackson;1349\nout out;1350\nmystery;1351\nwarrant;1352\nkamelot;1353\nmax webster;1354\nkristian stanfill;1355\nken hensley;1356\nlyfe jennings;1357\ngusgus;1358\nafter the burial;1359\nsam tsui;1360\ntony orlando & dawn;1361\nguy clark;1362\nelectric wizard;1363\nvanna;1364\nnicole c. mullen;1365\nprozak;1366\nspahn ranch;1367\nwilliam mcdowell;1368\nbrandy clark;1369\njamie lidell;1370\ngrief;1371\nthe maccabees;1372\nweedeater;1373\nmarlango;1374\nhirax;1375\nelvis presley;1376\naly & aj;1377\nellis paul;1378\nwe the kings;1379\nbound stems;1380\nsymphony x;1381\nrandy stonehill;1382\nman;1383\ncentinex;1384\nblaine larsen;1385\naqueduct;1386\njune of 44;1387\nwe came as romans;1388\nnatasha bedingfield;1389\ndivinefire;1390\nalan hull;1391\noutlawz;1392\nseasick steve;1393\nnerina pallot;1394\nsandy denny;1395\nglenn gould;1396\nmatt dusk;1397\nstrawberry alarm clock;1398\nstryper;1399\nstefanie heinzmann;1400\nmiracle of sound;1401\npop unknown;1402\nrahsaan patterson;1403\nyellowman;1404\nthe clay people;1405\nthe good life;1406\nleroy carr;1407\nsonata arctica;1408\ntom odell;1409\nthat handsome devil;1410\nbirdy;1411\njimmy cliff;1412\nkompressor;1413\ndeadmau5;1414\nlinda ronstadt;1415\nufx;1416\nblind blake;1417\njody watley;1418\nrazed in black;1419\ndave clark five;1420\ndiddy;1421\nanacrusis;1422\ndropkick murphys;1423\ndoyle bramhall;1424\nlisa \"left eye\" lopes;1425\nll cool j;1426\ndeadsoul tribe;1427\nsunset rubdown;1428\nheaven 17;1429\npavlov's dog;1430\nbillie eilish;1431\ndido;1432\ndeathboy;1433\nantaeus;1434\ndreamland;1435\nthe beloved;1436\nthe arrogant worms;1437\ncloset monster;1438\neartha kitt;1439\nradu lupu;1440\ncinderella effect;1441\nanathema;1442\nfor the fallen dreams;1443\nshola ama;1444\nbig thief;1445\narmand van helden;1446\nfake?;1447\nphil harris;1448\nloggins & messina;1449\npromise of redemption;1450\nthe walkabouts;1451\neisley;1452\nbig joe turner;1453\naxenstar;1454\nhank williams iii;1455\nb.o.b.;1456\nwallis bird;1457\nas i lay dying;1458\ntrey songz;1459\ncharlotte hatherley;1460\nnneka;1461\nanúna;1462\nasia;1463\nmcauley schenker group;1464\nrose polenzani;1465\njohn mayer;1466\ngrant-lee phillips;1467\ntaj mahal;1468\nruston kelly;1469\nmelissa manchester;1470\nlenka;1471\nkira isabella;1472\nhap palmer;1473\ntaken by trees;1474\nanni b sweet;1475\nplanet p project;1476\nthe velvet underground;1477\nhoagy carmichael;1478\ntestament;1479\nthe 3rd and the mortal;1480\nfever tree;1481\niris dement;1482\nhappy days;1483\nprostitute disfigurement;1484\nthe andrews sisters;1485\neyedea & abilities;1486\nmipso;1487\ndinu lipatti;1488\njosephine foster;1489\nstephen sondheim;1490\nlittle big;1491\nkip winger;1492\nthe voidz;1493\nmatchbook romance;1494\ngreen carnation;1495\nxiu xiu;1496\nthe hard-ons;1497\nglasseater;1498\nbloodlined calligraphy;1499\nyodelice;1500\ninfectious grooves;1501\nalex lloyd;1502\novercome;1503\ntom dice;1504\nrorschach test;1505\nnat & alex wolff;1506\nvan halen;1507\nrobert earl keen;1508\ndave hollister;1509\nrob thomas;1510\nchanté moore;1511\nlyle lovett;1512\naaron lines;1513\nmy life with the thrill kill kult;1514\nhonne;1515\nthe flatliners;1516\neric b. & rakim;1517\nold man gloom;1518\nnew found glory;1519\nlouis logic;1520\nmurray perahia;1521\nbass drum of death;1522\n702;1523\njamie o'neal;1524\nthe sheila divine;1525\nchris ledoux;1526\nhuski;1527\nwolf parade;1528\nemmure;1529\ndefiance, ohio;1530\nnine below zero;1531\njamie winchester;1532\ncece winans;1533\nsplashdown;1534\nthe strumbellas;1535\notis spann;1536\njuice wrld;1537\nbrett anderson;1538\nthe ambassador;1539\narturo benedetti michelangeli;1540\nspinal tap;1541\nwild strawberries;1542\nmystikal;1543\nacumen nation;1544\nthe ex;1545\npearls before swine;1546\nthe ink spots;1547\nmayhem;1548\ne;1549\nsara noxx;1550\nkid cudi;1551\nneko case;1552\ndethklok;1553\nvast;1554\nlila mccann;1555\nboxcar willie;1556\nthe soundtrack of our lives;1557\nthe deep dark woods;1558\njohnossi;1559\nsinéad lohan;1560\ntragic black;1561\npublic enemy;1562\ntoby keith;1563\njesus on extasy;1564\nafter forever;1565\nlightnin' hopkins;1566\nany given day;1567\nterminal choice;1568\nhead east;1569\nsturgill simpson;1570\ncavalera conspiracy;1571\npitboss 2000;1572\nbroken social scene;1573\nhanzel und gretyl;1574\nkajagoogoo;1575\nvillage people;1576\nthe most serene republic;1577\nernest tubb and loretta lynn;1578\narno;1579\nninja sex party;1580\nbts;1581\ncharlie simpson;1582\ncriss angel;1583\nlocal natives;1584\nelliphant;1585\nright away, great captain!;1586\nfrankie valli;1587\ndion;1588\nj. tillman;1589\nkrayzie bone;1590\natlanta rhythm section;1591\ngrimes;1592\nslapp happy;1593\nvoivod;1594\nella fitzgerald;1595\nmaddy prior;1596\nexo;1597\nbobby v;1598\ncherry glazerr;1599\njackie deshannon;1600\nnancy sinatra;1601\ndragonheart;1602\nthe casualties;1603\npolysics;1604\ncliff richard;1605\nsleeping with sirens;1606\ndew-scented;1607\ndiva destruction;1608\njens lekman;1609\ncharlie landsborough;1610\nborn ruffians;1611\njoe diffie;1612\nsonny terry;1613\nthe black angels;1614\nenvy;1615\nmary lou lord;1616\nof mice & men;1617\ntina arena;1618\ncandlemass;1619\nstacie orrico;1620\ntoo $hort;1621\nbombay bicycle club;1622\nthe dc3;1623\nmarié digby;1624\nfrank ocean;1625\nslash's snakepit;1626\nakcent;1627\nmortal sin;1628\nthe rural alberta advantage;1629\nalter bridge;1630\ncaligula's horse;1631\ndiana ross;1632\nstrand of oaks;1633\naustrian death machine;1634\nana popovic;1635\nskepticism;1636\nshirley horn;1637\nsiva six;1638\nvalentine wolfe;1639\ntitanic sinclair;1640\njoshua perahia;1641\ngallows;1642\nluxt;1643\nnick lowe;1644\nclarence \"gatemouth\" brown;1645\nsweet noise;1646\nthank you scientist;1647\ncherish the ladies;1648\nten years after;1649\nfrost;1650\nfirst blood;1651\nnightwish;1652\nmusiq soulchild;1653\nbig bill broonzy;1654\nbenjamin francis leftwich;1655\nphantom blue;1656\ndune;1657\nhangnail;1658\nharold melvin & the blue notes;1659\nbig d and the kids table;1660\nzeromancer;1661\njello biafra;1662\nyg;1663\nkatharine mcphee;1664\nquinn xcii;1665\nmississippi john hurt;1666\nnew trolls;1667\nwizards;1668\nkix;1669\nslapshot;1670\ntor miller;1671\nxentrifuge;1672\ntoni braxton;1673\nfinch;1674\ncaroline herring;1675\ndreadful shadows;1676\nringworm;1677\ncory asbury;1678\ndezperadoz;1679\nmac davis;1680\ndionysus;1681\nmichael w. smith;1682\ncold as life;1683\npeabo bryson;1684\nk.d. lang;1685\ngrammatrain;1686\njorn;1687\nno-man;1688\nnocturne;1689\nthe screaming jets;1690\ncharli xcx;1691\ntactical sekt;1692\noomph!;1693\natlas sound;1694\nthe idle race;1695\nhelstar;1696\ntoxik;1697\njesus culture;1698\ncissy houston;1699\ncatman cohen;1700\nstrike anywhere;1701\ntoni childs;1702\nmika;1703\ntheory in practice;1704\nlucinda williams;1705\nlord belial;1706\nraul midón;1707\nida;1708\ntrisha yearwood;1709\nbad astronaut;1710\nthe runaways;1711\na day to remember;1712\nmilk inc.;1713\nfisher;1714\nking kobra;1715\nma rainey;1716\nralph stanley;1717\nandr watts;1718\ngregory and the hawk;1719\nthe temptations;1720\nflaw;1721\nterror squad;1722\nblack;1723\nbolt thrower;1724\nmatt goss;1725\nnappy roots;1726\na$ap ferg;1727\nshawn mendes;1728\nalison krauss & union station;1729\neric johnson;1730\nashley monroe;1731\nold crow medicine show;1732\nkelis;1733\nbad habit;1734\nvan canto;1735\nthe birthday party;1736\nrowland s. howard;1737\nmarsha ambrosius;1738\nlittle dragon;1739\nk'jon;1740\njack white;1741\ncactus;1742\ndaft punk;1743\njon oliva's pain;1744\na$ap mob;1745\nemika;1746\nlazar berman;1747\nmark kozelek;1748\nice-t;1749\nlittle richard;1750\nelijah blake;1751\nthe laurie berkner band;1752\nclara luzia;1753\nma$e;1754\ndikembe;1755\nboz scaggs;1756\nantony and the johnsons;1757\nautopilot off;1758\nbig audio dynamite;1759\ngrant lee buffalo;1760\njohn reuben;1761\nmission of burma;1762\nunloco;1763\ntransit;1764\nmarina and the diamonds;1765\nalela diane;1766\nthe sorrow;1767\ngossip;1768\nemerald;1769\nlucille bogan;1770\nfrank zappa;1771\nthe coathangers;1772\ncaptain jack;1773\nstellastarr*;1774\ndavid kersh;1775\nbroken bones;1776\nhayley kiyoko;1777\nwire;1778\nthurston moore;1779\ncop shoot cop;1780\nthe white buffalo;1781\nbad books;1782\nirene cara;1783\ngorillaz;1784\nthe gap band;1785\nlead belly;1786\ncassadee pope;1787\nelvis depressedly;1788\ncurtis mayfield;1789\nwaylon;1790\nthe gun club;1791\nbehemoth;1792\nshe wants revenge;1793\nthe crüxshadows;1794\ntýr;1795\ndan fogelberg;1796\nstan ridgway;1797\nblind witness;1798\ndeerhunter;1799\nagalloch;1800\ngrand puba;1801\nheavens edge;1802\nthe acacia strain;1803\nbeseech;1804\nsting;1805\nrival sons;1806\nhenry jamison;1807\nthe black lillies;1808\nthe cog is dead;1809\nbenno moiseiwitsch;1810\nnazz;1811\ngreg brown;1812\nreel big fish;1813\nthe muppets;1814\nthreshold;1815\ntracie spencer;1816\ncimorelli;1817\nalexandra burke;1818\nwhigfield;1819\neminem;1820\nwolverine;1821\ngrave maker;1822\nexample;1823\nlambchop;1824\nmadeleine peyroux;1825\nbondage fairies;1826\ndarren hanlon;1827\nlyria;1828\ncapital kings;1829\njohn pizzarelli;1830\nhigh on fire;1831\nstereolab;1832\nmachines of loving grace;1833\nkim carnes;1834\nso many dynamos;1835\nphony ppl;1836\nsan cisco;1837\nemilie autumn;1838\npietro locatelli;1839\nsolefald;1840\nkellie pickler;1841\nthe maranatha! singers;1842\ndanbert nobacon;1843\nsoulja boy;1844\nmario winans;1845\ncamille saint sa ns;1846\nindecision;1847\nlasgo;1848\nkaryn white;1849\nhurts;1850\nbetween the trees;1851\nnat king cole;1852\nfront 242;1853\njohannes brahms;1854\nthe national;1855\nshane barnard;1856\nchris knox;1857\npoison girls;1858\noh, sleeper;1859\nhell is for heroes;1860\npfr;1861\njohn lee hooker;1862\ngramatik;1863\nmoya brennan;1864\npop evil;1865\nscar symmetry;1866\nsilly wizard;1867\nthe ventures;1868\nsteve holy;1869\ndevotchkas;1870\nignite;1871\nwayne newton;1872\nthe gufs;1873\ndecapitated;1874\nbillie the vision & the dancers;1875\nthestart;1876\nidle cure;1877\njill sobule;1878\nthe soviettes;1879\nthe irish rovers;1880\nmünchener freiheit;1881\npentagram;1882\nmasterboy;1883\nreturn;1884\nminus story;1885\nsatan;1886\nbyron cage;1887\nlistener;1888\nthe mars volta;1889\njennifer love hewitt;1890\neartha;1891\nno doctors;1892\nsteve hackett;1893\nelvin bishop;1894\nk'naan;1895\nruby;1896\nprincess nokia;1897\nellie goulding;1898\nmaterial issue;1899\nfun boy three;1900\ncirculatory system;1901\nthe road hammers;1902\nbrian mcknight;1903\n\"weird al\" yankovic;1904\nthe wailers;1905\nkurupt;1906\ncriminal;1907\nbal-sagoth;1908\nlou reed;1909\nqueen;1910\nrudimentary peni;1911\na great big world;1912\njarboe;1913\naugie march;1914\njim guthrie;1915\nemma bunton;1916\naaron watson;1917\nέλενα παπαρίζου;1918\nblue cheer;1919\nmark harris;1920\nalexander scriabin;1921\nandrew gold;1922\nclimax blues band;1923\nconjure one;1924\nblind guardian;1925\nstephen schwartz;1926\nmartha wainwright;1927\nlil' flip;1928\nkatherine jenkins;1929\namy shark;1930\nstyles p;1931\nnew model army;1932\nfranz liszt;1933\nben weasel;1934\njohn k. samson;1935\njason anderson;1936\nsheena easton;1937\ninna;1938\nmayer hawthorne;1939\neasy rider;1940\neddy arnold;1941\nzoegirl;1942\njimmy barnes;1943\ntear da club up thugs;1944\ndaughter;1945\nbuzzcocks;1946\nfreezepop;1947\nsteve aoki;1948\nkierra sheard;1949\ncaptain beyond;1950\nsabrina claudio;1951\nmc shan;1952\nthe j. geils band;1953\ncamel;1954\nindica;1955\nattack in black;1956\nthe jam;1957\nbananarama;1958\nal jarreau;1959\neric carmen;1960\ngwar;1961\nus the duo;1962\nziggy alberts;1963\namir obè;1964\n25 ta life;1965\nnewsboys;1966\nchris knight;1967\nmarlene dietrich;1968\n112;1969\nwonderwall;1970\nofficer negative;1971\nenochian crescent;1972\njupiter one;1973\nwhitecross;1974\ntim mcgraw;1975\nbile;1976\nbrandi carlile;1977\ndanzig;1978\ntwiztid;1979\ndavid baerwald;1980\njimmy webb;1981\nmamas gun;1982\nharry chapin;1983\nholy ghost!;1984\nstephen hough;1985\nk.t. oslin;1986\nrobin trower;1987\nus3;1988\nesben and the witch;1989\nschiller;1990\ngod lives underwater;1991\nhayseed dixie;1992\npolluted inheritance;1993\nnicki minaj;1994\nrobin gibb;1995\npinback;1996\nbobbie gentry;1997\nfreakwater;1998\nkite;1999\nkid down;2000\ngeorges cziffra;2001\nsivert høyem;2002\nconor oberst;2003\ni'm from barcelona;2004\nshining;2005\nhenry purcell;2006\njeremy messersmith;2007\ngoat of mendes;2008\ncatch 22;2009\nvampire rodents;2010\nthe go-go's;2011\nnoah gundersen;2012\ndiablo swing orchestra;2013\nuncle dave macon;2014\njohn eddie;2015\ncalvin harris;2016\nroomful of blues;2017\nbane;2018\nsam lewis;2019\nalejandro escovedo;2020\nkekal;2021\nmanticora;2022\nfrankie lymon & the teenagers;2023\nrandy meisner;2024\ncarcass;2025\nnile;2026\ncircus maximus;2027\nnon phixion;2028\neric woolfson;2029\nstarship;2030\nkaiser chiefs;2031\nsexy sadie;2032\nconfide;2033\nboy meets girl;2034\nhenryk szeryng;2035\nwynter gordon;2036\njay-jay johanson;2037\nhannah fury;2038\nmike jones;2039\nbebe & cece winans;2040\ntrip lee;2041\ntigers jaw;2042\nhector berlioz;2043\nminus the bear;2044\njohnny winter;2045\nmy sister's machine;2046\ngilbert o'sullivan;2047\njean michel jarre;2048\njody miller;2049\nskylark;2050\nthe poodles;2051\nfear of domination;2052\nthe donefors;2053\nzed yago;2054\nmartha argerich;2055\nfaun fables;2056\njoy electric;2057\nthe grass roots;2058\ncygnosic;2059\ndisfear;2060\ncrosby, stills, nash & young;2061\n69 boyz;2062\ncelesty;2063\nthe forecast;2064\nthe magnetic fields;2065\nmiranda lambert;2066\nveto;2067\nthe offspring;2068\nbobby goldsboro;2069\ncalibretto;2070\ncommon rider;2071\n10cc;2072\ntim hughes;2073\nbald vulture;2074\ndavid cassidy;2075\nsimple minds;2076\nlittle man tate;2077\ncarla thomas;2078\ncher lloyd;2079\ncameo;2080\ntko;2081\ngeorge frideric handel;2082\nimmaculate fools;2083\nkero kero bonito;2084\njohn sebastian;2085\njoe nichols;2086\nclassified;2087\nvelvet revolver;2088\nskip the use;2089\nratt;2090\ngilberto gil;2091\nthe miracles;2092\nthe subways;2093\nscrewed up click;2094\nfm static;2095\nfurther seems forever;2096\nbarry louis polisar;2097\nantiskeptic;2098\nden harrow;2099\neleni mandell;2100\nmungo jerry;2101\nfucked up;2102\njermaine jackson;2103\nfreedom call;2104\nphillip phillips;2105\nvan der graaf generator;2106\nh-town;2107\nmadvillain;2108\npietasters;2109\nsonicflood;2110\nthomas dolby;2111\nscud mountain boys;2112\norphanage;2113\neleventyseven;2114\npeter hammill;2115\nkaysha;2116\nskyharbor;2117\nsouth border;2118\nmad marge and the stonecutters;2119\nben kweller;2120\nmonstrosity;2121\nyoung galaxy;2122\npusha t;2123\njohn wesley;2124\nglass harp;2125\nblue system;2126\npain of salvation;2127\nfilm school;2128\nplasmatics;2129\ndown by law;2130\ncircus of power;2131\nleft alone;2132\ngirls under glass;2133\nsaga;2134\nredemption;2135\nmemphis may fire;2136\naion;2137\nkj-52;2138\ngaia epicus;2139\nstacy lattisaw;2140\nlarry norman;2141\ndavid lee roth;2142\nwillie nelson;2143\nmelody gardot;2144\nin the midst of lions;2145\niron butterfly;2146\nkaty rose;2147\nizz;2148\nsevyn streeter;2149\npendulum;2150\ncult of luna;2151\navalanche city;2152\nbrother firetribe;2153\nlena;2154\nsteve vai;2155\nmotosierra;2156\nthe bouncing souls;2157\nmaher zain;2158\ncolbie caillat;2159\nblaze ya dead homie;2160\nkyung wha chung;2161\ndelta goodrem;2162\nlena katina;2163\nthe veronicas;2164\nasleep at the wheel;2165\nanne clark;2166\njimmie vaughan;2167\nwildpath;2168\nautumnblaze;2169\nlefty frizzell;2170\nthe weakerthans;2171\nreal estate;2172\ndj drama;2173\njoji;2174\nlittle milton;2175\nbeholder;2176\ngrace jones;2177\nwolfheart;2178\nkt tunstall;2179\nrobert forster;2180\nlana lane;2181\nmeat loaf;2182\nmark chesnutt;2183\nautumn;2184\nbronze nazareth;2185\nladysmith black mambazo;2186\nmemphis minnie;2187\nthe plimsouls;2188\n36 crazyfists;2189\nprivate line;2190\nnikolai medtner;2191\njaden smith;2192\nchris cagle;2193\nbeatnik termites;2194\nbernard butler;2195\nemin;2196\nmajor accident;2197\nthe mayan factor;2198\nstonewall jackson;2199\nrat boy;2200\ndas efx;2201\nroxette;2202\ntears for fears;2203\nhank locklin;2204\nn.w.a;2205\nhadouken!;2206\nking diamond;2207\naxe;2208\nlolo;2209\nrufus thomas;2210\nmontgomery gentry;2211\nthe front bottoms;2212\ngabrielle;2213\nbeng beng cocktail;2214\nwyclef jean;2215\ncœur de pirate;2216\nfaithless;2217\nheavy heavy low low;2218\ncutting crew;2219\nlimbeck;2220\nsaukrates;2221\nartie shaw;2222\njojo;2223\nred rider;2224\nnikolai rimsky korsakov;2225\nchoking victim;2226\nthe scene aesthetic;2227\nfight amp;2228\némilie simon;2229\nchristoph willibald gluck;2230\nboney m.;2231\nookla the mok;2232\nbeck;2233\nmcfly;2234\nhey ocean!;2235\ndixie chicks;2236\nlindsay lohan;2237\nfrom autumn to ashes;2238\ngiovanni battista pergolesi;2239\nboondox;2240\nwilhelm kempff;2241\njay & the americans;2242\nthe game;2243\nnight in gales;2244\nblazin' squad;2245\nlords of acid;2246\ndarlingside;2247\nsquealer;2248\nthe ghost inside;2249\nabbey lincoln;2250\ncake bake betty;2251\nall that remains;2252\nbunny wailer;2253\nculture club;2254\njim croce;2255\nvixen;2256\nchelsea wolfe;2257\nthe zutons;2258\nthe ship;2259\ntimbuk3;2260\nwise guys;2261\ngabriel faur ;2262\nprototype;2263\ngreat big sea;2264\nceltic frost;2265\nstage dolls;2266\nvinnie paz;2267\nsymbols;2268\ncasanova;2269\nhorse the band;2270\nraul seixas;2271\nphil collins;2272\nbelphegor;2273\na whisper in the noise;2274\nrufio;2275\ntrin-i-tee 5;2276\nisaac hayes;2277\ni am ghost;2278\ngregg allman;2279\nthree 6 mafia;2280\nt. mills;2281\nas cities burn;2282\nfear factory;2283\ngood riddance;2284\ndaniel m ller schott;2285\nstromkern;2286\nspirogyra;2287\nresurrection band;2288\njamie's elsewhere;2289\none bad pig;2290\nrenaldo & the loaf;2291\ndanny kaye;2292\nbelle and sebastian;2293\ncryptopsy;2294\nyeah yeah yeahs;2295\nthe jayhawks;2296\nmel tillis;2297\nyazoo;2298\ncolin hay;2299\nthe desert sessions;2300\nthe dismemberment plan;2301\nsammy adams;2302\ndiscipline;2303\nsecrets of the moon;2304\ntapping the vein;2305\narmy of the pharaohs;2306\nbig bad voodoo daddy;2307\nprofessor green;2308\nroine stolt;2309\nwu-tang clan;2310\ntracy byrd;2311\nbig maybelle;2312\naage kvalbein;2313\nclub nouveau;2314\nmose allison;2315\nedenbridge;2316\nthe foundations;2317\ndarkseed;2318\ndear and the headlights;2319\ntwila paris;2320\ned bruce;2321\ndamien dempsey;2322\nstarlight mints;2323\naccessory;2324\nador dorath;2325\nboogie down productions;2326\nstars;2327\ncreedence clearwater revival;2328\ndwight yoakam;2329\n4him;2330\ngorguts;2331\ndemons & wizards;2332\n陰陽座;2333\nbarstool prophets;2334\npaolo nutini;2335\nasking alexandria;2336\nkid rock;2337\njerusalem;2338\nthem;2339\nrobert wyatt;2340\nthe weeknd;2341\njeff deyo;2342\nbeartooth;2343\npilot;2344\njenny hval;2345\nthe devil wears prada;2346\nkarine polwart;2347\nmaxwell;2348\nmichael rabin;2349\nbabyface;2350\nwhite lies;2351\ngroundation;2352\nnina hagen;2353\ntracy lawrence;2354\nring of fire;2355\nkoda kumi;2356\numphrey's mcgee;2357\npolkadot cadaver;2358\nalabama;2359\nedge of sanity;2360\nthyrane;2361\ndick brave & the backbeats;2362\npantera;2363\nfrançoise hardy;2364\nwhirlwind heat;2365\nlightning seeds;2366\ndisrupt;2367\ntrue colors;2368\nvérité;2369\njeremy riddle;2370\nbilly crawford;2371\nroyal hunt;2372\napache indian;2373\npeggy lee;2374\nthe flight of sleipnir;2375\nhard-fi;2376\nladytron;2377\nmassacration;2378\njohn browning;2379\ngrant hart;2380\nchris hillman;2381\nbear vs. shark;2382\ndescendents;2383\nmark ronson;2384\nnofx;2385\nrich kids on lsd;2386\njordin sparks;2387\nvv brown;2388\nmichael franks;2389\naram khachaturian;2390\nbathory;2391\njessica pratt;2392\nmormon tabernacle choir;2393\ndan hill;2394\nlaleh;2395\ncurl up and die;2396\njonah matranga;2397\nanna tsuchiya;2398\njack howard;2399\nnick heyward;2400\nsteven curtis chapman;2401\npimp c;2402\nchef'special;2403\nsuper junior-d&e;2404\nwillie nelson & wynton marsalis;2405\nlionel richie;2406\nmartyr;2407\neagles;2408\nproject pitchfork;2409\ncrosby, stills & nash;2410\nslim thug;2411\nfrankie j;2412\nthe stereo;2413\nblues pills;2414\nfog;2415\nbow wow wow;2416\ngowan;2417\nking charles;2418\nsviatoslav richter;2419\nm. ward;2420\nfreshlyground;2421\ned schrader's music beat;2422\ncompany flow;2423\npainbastard;2424\nbuddy jewell;2425\nmckinney's cotton pickers;2426\nmandisa;2427\nwatershed;2428\nthe wailin' jennys;2429\nfaderhead;2430\nmötley crüe;2431\nstormwarrior;2432\nkeb' mo';2433\njimmy witherspoon;2434\nthe lucksmiths;2435\nfaster pussycat;2436\nknightowl;2437\nkeri hilson;2438\nmxpx;2439\nthe nice;2440\nritual;2441\noxymoron;2442\ncatamenia;2443\nvince gill;2444\nbill fay;2445\ntoy-box;2446\nlove unlimited;2447\nmc frontalot;2448\ndolly parton;2449\npapoose;2450\ndemi lovato;2451\nrhodes;2452\nbig joe williams;2453\ncharlie rich;2454\nextreme noise terror;2455\nwilco;2456\nlow roar;2457\ncarrie underwood;2458\nwhen particles collide;2459\n3 feet smaller;2460\nthe supremes & the four tops;2461\nthe police;2462\nempress of;2463\nherman brood;2464\nbe your own pet;2465\nkill switch...klick;2466\nsuperjoint ritual;2467\nlynyrd skynyrd;2468\nal green;2469\nhilltop hoods;2470\nghostlimb;2471\nchamber - l'orchestre de chambre noir;2472\nthou;2473\nthe icicle works;2474\ndebbie harry;2475\nvictoria beckham;2476\nclare maguire;2477\ncause & effect;2478\nmarion;2479\nthe lawrence arms;2480\nx;2481\nthe russian futurists;2482\nmaggie rose;2483\njim cuddy;2484\nthe turtles;2485\nas blood runs black;2486\nthe night flight orchestra;2487\ngraziano romani;2488\nalcatrazz;2489\nlp;2490\njon foreman;2491\npopa chubby;2492\nalfred cortot;2493\neddi reader;2494\nplushgun;2495\nsugababes;2496\nsesame street;2497\njohnny cash;2498\njoe walsh;2499\njeanette biedermann;2500\nashford & simpson;2501\nsleep;2502\nhum;2503\npapas fritas;2504\nfifteen;2505\ndenzel curry;2506\nphedora;2507\nanders osborne;2508\nthe morning of;2509\ncash rivers and the sinners;2510\nblueprint;2511\nslim dusty;2512\nacappella;2513\nkatatonia;2514\ndiary of dreams;2515\ndeine lakaien;2516\nrick wakeman;2517\nfreedom fry;2518\nfruupp;2519\njean baptiste lully;2520\noctober 31;2521\nsinister;2522\njesse mccartney;2523\nvigilantes of love;2524\nelegant machinery;2525\ncorbin bleu;2526\ngza/genius;2527\nalison wonderland;2528\nkelly price;2529\ndestruction;2530\nfractured;2531\ndavid ford;2532\njulie miller;2533\njoe mcelderry;2534\nphinehas;2535\nsylvan esso;2536\nbasement jaxx;2537\nshearwater;2538\ndeadstar assembly;2539\nwildbirds & peacedrums;2540\npatty smyth;2541\nshort stack;2542\njupiter apple;2543\nraven-symoné;2544\nmisery loves co.;2545\nthe motels;2546\nsteppenwolf;2547\nvic chesnutt;2548\nasha;2549\nthe almighty;2550\nlauryn hill;2551\nkhalid;2552\ngreat lake swimmers;2553\nashanti;2554\n蔡依林 (jolin tsai);2555\ndonnie munro;2556\njohn fogerty;2557\nkimberley locke;2558\nkeith moon;2559\njanis joplin;2560\nreamonn;2561\npsychopathic rydas;2562\nair supply;2563\nashlee simpson;2564\ntame impala;2565\nperzonal war;2566\nadem;2567\nfranz ferdinand;2568\nmartina mcbride;2569\nnatalie cole;2570\njason falkner;2571\nrosalyn tureck;2572\nmissy higgins;2573\nbeach house;2574\ncurrent 93;2575\na hill to die upon;2576\nghoti hook;2577\nforgive durden;2578\ngojira;2579\njay ferguson;2580\neilera;2581\nkate nash;2582\nfoxes;2583\nat the drive-in;2584\njob for a cowboy;2585\nfrom ashes to new;2586\nbrooke white;2587\ndwele;2588\nvacuum;2589\nsamson fran ois;2590\nloverboy;2591\nblack country communion;2592\npaul revere and the raiders;2593\nchris brown;2594\ntommy lee;2595\nvengaboys;2596\njoy zipper;2597\nbella hardy;2598\nblondie;2599\nbrian setzer;2600\nmc zulu;2601\nbuilding 429;2602\ncindy bullens;2603\ncyne;2604\nb*witched;2605\nthe black eyed peas;2606\npaul weller;2607\ngun;2608\nillogic;2609\nmen at work;2610\nluv';2611\nshadows fall;2612\nthe crest;2613\nzendaya;2614\nrunemagick;2615\njohn farnham;2616\njosh ritter;2617\nfuneral;2618\nwanda jackson;2619\nwaylon jennings;2620\nfalco;2621\nmarlon williams;2622\nseals & crofts;2623\nkurt elling;2624\nneon hitch;2625\nthe swift;2626\nsub focus;2627\nbent knee;2628\ntrevor rabin;2629\nbazzi;2630\njim's big ego;2631\nandrew huang;2632\nglenn miller;2633\ndonna summer;2634\nthe meteors;2635\nugly duckling;2636\n李玟 (coco lee);2637\nbruno mars;2638\nsambassadeur;2639\nthe neville brothers;2640\nterrorgruppe;2641\nthe lone bellow;2642\ndeer tick;2643\nbentley jones;2644\ncarter the unstoppable sex machine;2645\nconnie talbot;2646\nadult.;2647\ncrosby & nash;2648\nayọ;2649\njay rock;2650\ncrystal castles;2651\ntyrone wells;2652\nheavens to betsy;2653\nthe limousines;2654\nchris garneau;2655\n13th floor elevators;2656\ndirty pretty things;2657\nthe more i see;2658\nraimon;2659\nwinterstorm;2660\nspice 1;2661\njade valerie;2662\nkacey musgraves;2663\nroko;2664\nlights of euphoria;2665\nthe bronx;2666\nraffaella carrà;2667\nmastercastle;2668\ncity and colour;2669\nsalvador;2670\ncarolyn arends;2671\nmotörhead;2672\njustin young;2673\nthe dreadnoughts;2674\nindia.arie;2675\ndecoded feedback;2676\nguy verlinde;2677\nthe clientele;2678\nrapture;2679\ncaterina valente;2680\nbattle beast;2681\nwednesday 13;2682\nwalter gieseking;2683\nthe duckworth lewis method;2684\nbloc party;2685\nzuill bailey;2686\nmother love bone;2687\nmartha and the vandellas;2688\nblitzkid;2689\nnightrage;2690\nmartin zellar;2691\n38 special;2692\nchristina grimmie;2693\ndagoba;2694\nglenn tipton;2695\nblue rodeo;2696\nacoustic junction;2697\nled zeppelin;2698\nevergrey;2699\nthe locust;2700\njazmine sullivan;2701\nthe colourfield;2702\nyob;2703\nvengeance rising;2704\nchunk! no, captain chunk!;2705\nthe alan parsons project;2706\ndustin kensrue;2707\nfool's garden;2708\nsaucy monky;2709\nsergei rachmaninoff;2710\ncharlie sexton;2711\nbabyshambles;2712\nnina simone;2713\nborn against;2714\ndååth;2715\njon allen;2716\nleigh nash;2717\nthe vandals;2718\nmatt redman;2719\nmarian hill;2720\ntanita tikaram;2721\nleonard bernstein;2722\nraffi;2723\npharao;2724\nmiseration;2725\nallan taylor;2726\nrita ora;2727\ngary glitter;2728\ngirlschool;2729\nk-ci & jojo;2730\njimmy dorsey;2731\n.moneen.;2732\nimperia;2733\nvirtuoso;2734\nvotum;2735\nlucy woodward;2736\nalvin stardust;2737\nwill young;2738\ncrown of thorns;2739\narcadi volodos;2740\nthe jim yoshii pile-up;2741\na life divided;2742\nchildren 18;2743\njames ingram;2744\n梶浦由記 (yuki kajiura);2745\nynw melly;2746\njump, little children;2747\ntrans-siberian orchestra;2748\narmageddon;2749\nfaces;2750\nbokka;2751\nleviathan;2752\nthe black keys;2753\nthe panic division;2754\nroy drusky;2755\nveggietales;2756\njt music;2757\ntim o'brien;2758\nbrockhampton;2759\nthe sundays;2760\nhayes carll;2761\nleatherface;2762\nspirit of the west;2763\ndawn landes;2764\nkygo;2765\nisis;2766\nmeek mill;2767\nbrisa roché;2768\nfreesscape;2769\njoe hill;2770\npride and fall;2771\n54-40;2772\nwashed out;2773\nmichael sembello;2774\ncrowded house;2775\ncpr;2776\nsteve winwood;2777\njukebox the ghost;2778\natari teenage riot;2779\nballyhoo!;2780\neddie floyd;2781\ntrophy scars;2782\nerykah badu;2783\njames cotton;2784\nkristinia debarge;2785\nmorbid angel;2786\ncode red;2787\nal kooper;2788\nmonty python;2789\nthe mary onettes;2790\nfuture;2791\nshabazz the disciple;2792\nsad café;2793\nx-fusion;2794\ndeniece williams;2795\nrobert gordon;2796\nseabear;2797\nwallows;2798\njohn illsley;2799\ndream theater;2800\nsecret lives of the freemasons;2801\njosef suk;2802\nthe bled;2803\ngregor piatigorsky;2804\njohn ralston;2805\ntim kasher;2806\nthe hellacopters;2807\nautopsy;2808\nthe silencers;2809\nhank green;2810\nabk;2811\nhound dog taylor;2812\nangela bofill;2813\nwade bowen;2814\njamestown story;2815\nandy partridge;2816\necholyn;2817\nsearch the city;2818\ngiacomo puccini;2819\ngregory isaacs;2820\nstormtroopers of death;2821\nsilkk the shocker;2822\n1208;2823\neverlife;2824\nbobby mcferrin;2825\npretty ricky;2826\nronna reeves;2827\nannie lennox;2828\nlondon grammar;2829\njohn doe;2830\nshel silverstein;2831\nblue october;2832\naaron lee tasjan;2833\nmarion raven;2834\ncarl perkins;2835\nbig pokey;2836\neddie cochran;2837\nenter the haggis;2838\nburden of a day;2839\ntourniquet;2840\nemilíana torrini;2841\nbrian doerksen;2842\ncrossfaith;2843\nludwig van beethoven;2844\ngood clean fun;2845\nan horse;2846\nkali uchis;2847\nstone breath;2848\ndaniele liverani;2849\npatrick stump;2850\nhenry krieger;2851\nfit for a king;2852\nangel corpse;2853\nthe audition;2854\nhelalyn flowers;2855\nfleet foxes;2856\nthe wiggles;2857\nclouds;2858\npeter cetera;2859\nwreckshop family;2860\nthe mistake;2861\nlacrimosa;2862\ncarpenters;2863\nthe crabb family;2864\ntwilight fauna;2865\nanton n dvo k;2866\npatti page;2867\ndawnbringer;2868\nbachman-turner overdrive;2869\nlove like blood;2870\npat mcgee band;2871\nshedaisy;2872\nthe other ones;2873\nhatebreed;2874\niamamiwhoami;2875\nshowbread;2876\nthyx;2877\nface to face;2878\nbonnie owens;2879\nthis wild life;2880\nadrienne young;2881\ntombs;2882\nf.k.ü.;2883\ncarl nielsen;2884\ntrouble;2885\npeter wolf;2886\nshout out louds;2887\nnew york philharmonic;2888\npaul and storm;2889\nthe dickies;2890\nhussein fatal;2891\na tribe called quest;2892\njade 4u;2893\nbonobo;2894\nalphabeat;2895\ngladys knight;2896\ntilt;2897\njacqueline du pr ;2898\nboyz ii men;2899\nblood oranges;2900\nscala & kolacny brothers;2901\nsteel attack;2902\nvienna teng;2903\nnunslaughter;2904\nroxy music;2905\nbenny benassi;2906\nfeargal sharkey;2907\nwillie dixon;2908\nmary j. blige;2909\nghost town;2910\ngeorge ezra;2911\nrose cousins;2912\nthe stone roses;2913\nmc5;2914\ngo west;2915\nknut;2916\nthe head and the heart;2917\nroom eleven;2918\nsimon and garfunkel;2919\nwilson pickett;2920\nthe amboy dukes;2921\nwalk the moon;2922\nabgott;2923\ntruly;2924\nthe louvin brothers;2925\nmelanie c;2926\nenchant;2927\nduffy;2928\nunion;2929\ngreat white;2930\nthe violet burning;2931\nkat-tun;2932\nmercyme;2933\nelmore james;2934\njohnny horton;2935\ndreams of sanity;2936\nvázquez sounds;2937\nkraftwerk;2938\ncool hand luke;2939\ncountry joe mcdonald;2940\nabandoned pools;2941\nthe swell season;2942\n3 inches of blood;2943\nrocky loves emily;2944\nwhores.;2945\nnatalie grant;2946\nfrank proffitt;2947\nred fang;2948\njason mraz;2949\nbarefoot truth;2950\nlee aaron;2951\nfrank hutchison;2952\nkasey chambers;2953\nsick of it all;2954\ngirls in hawaii;2955\ngeoff farina;2956\nrory gallagher;2957\nout of eden;2958\nmø;2959\ndarkthrone;2960\neddie money;2961\nenvy on the coast;2962\nvirgin prunes;2963\nmississippi sheiks;2964\ncapitol steps;2965\nbattery;2966\nhilary duff;2967\nbetraying the martyrs;2968\nstray cats;2969\nthe groundhogs;2970\nthieves and villains;2971\nla roux;2972\nsouthern raiders band;2973\ncam'ron;2974\nmississippi fred mcdowell;2975\ngary u.s. bonds;2976\nmatt duke;2977\nstillborn;2978\nmnemic;2979\njerry cantrell;2980\nclannad;2981\nmychildren mybride;2982\ndope;2983\nhis hero is gone;2984\ndeltron 3030;2985\nnebelhexë;2986\nangelic upstarts;2987\nbebo norman;2988\nkb;2989\npatricia barber;2990\ntsol;2991\ncathedral;2992\nglenn kaiser;2993\npeter koppes;2994\nthe hunna;2995\npeaches;2996\nbryan adams;2997\nmichael kiske;2998\nsigrid;2999\ngordon bok;3000\nthe vamps;3001\nburning witches;3002\nthe twilight singers;3003\nteyana taylor;3004\nrobbie robertson;3005\nblack veil brides;3006\ncorbin-hanner band;3007\nwhite denim;3008\nsee you next tuesday;3009\ncorey hart;3010\nshy;3011\nvisions of atlantis;3012\npyrexia;3013\nrah digga;3014\nkillwhitneydead;3015\naustra;3016\nyung joc;3017\nsevendust;3018\nmiles kane;3019\nalbert hammond, jr.;3020\nthe sleepy jackson;3021\nquarterflash;3022\nbaba brinkman;3023\ngautier capu on;3024\nheather dale;3025\nhinder;3026\nevoken;3027\nkardinal offishall;3028\njett rebel;3029\nn-dubz;3030\nmarcia ball;3031\nbucks fizz;3032\nfreddie jackson;3033\nsteve grand;3034\nleonard pennario;3035\nsummoning;3036\nblaqk audio;3037\nthe sisters of mercy;3038\ndoyle bramhall ii;3039\ndominici;3040\nal b. sure!;3041\ngiant sand;3042\ncolosseum;3043\njohannes moser;3044\nyeasayer;3045\nunearth;3046\noneiroid psychosis;3047\nsheila e.;3048\nalicia de larrocha;3049\nlast tuesday;3050\nthe bonzo dog doo-dah band;3051\ngilbert and sullivan;3052\nthis day & age;3053\nanekdoten;3054\ngreg lake;3055\nmike scott;3056\nblack tide;3057\ncolony 5;3058\nthe malibooz;3059\nadam ant;3060\npacewon;3061\nqueensrÿche;3062\nthe ames brothers;3063\nrooster;3064\nrare bird;3065\njustin nozuka;3066\nelle milano;3067\nyacht;3068\nneville marriner;3069\nfaith no more;3070\nkool & the gang;3071\nfun lovin' criminals;3072\nyann tiersen;3073\nschäffer the darklord;3074\naiden;3075\nmetanoia;3076\nthe high dials;3077\ncharles bradley;3078\npaul gilbert;3079\nmalvina reynolds;3080\niron & wine;3081\nsaturday looks good to me;3082\nviktor vaughn;3083\nisgaard;3084\nfrank sinatra;3085\nalice cooper;3086\npassenger;3087\nmichael nesmith;3088\nancient;3089\nliza anne;3090\nthe magic numbers;3091\nseraphim shock;3092\nabney park;3093\nguerilla maab;3094\norphaned land;3095\njack savoretti;3096\nzombina and the skeletones;3097\nleon fleisher;3098\nrick springfield;3099\nthe left rights;3100\nstarflyer 59;3101\n2 brothers on the 4th floor;3102\nemery;3103\nc.c. catch;3104\nmick taylor;3105\ngood rats;3106\nblackmore's night;3107\ntwisted sister;3108\nboney james;3109\ncancer;3110\nmaria jo o pires;3111\nmeiko;3112\ncass mccombs;3113\ndeep dish;3114\ninformation society;3115\nblackie and the rodeo kings;3116\nthe fold;3117\ned harcourt;3118\nramshackle glory;3119\ntom misch;3120\nbebe winans;3121\njoni mitchell;3122\nelizabeth shepherd;3123\nlil boosie;3124\ncelluloide;3125\nhouse of lords;3126\nj.d. souther;3127\nmidori goto;3128\nshenandoah;3129\naveri;3130\nmodwheelmood;3131\nbetty who;3132\nburst;3133\npete shelley;3134\njon bon jovi;3135\nx-perience;3136\nxentrix;3137\ngrinderman;3138\nhem;3139\nthe impressions;3140\nsylosis;3141\nthe gregory brothers;3142\nmacklemore;3143\nlizz wright;3144\nsawyer brown;3145\ncapercaillie;3146\nsam & dave;3147\nthe germs;3148\nsalt-n-pepa;3149\nbig sean;3150\nstornoway;3151\narthur rubinstein;3152\ncock robin;3153\nbig daddy;3154\nfernando ortega;3155\nted nugent;3156\ncoptic rain;3157\nkat deluna;3158\nwolfsheim;3159\ngraham central station;3160\ncharles harrison;3161\nlil jon & the east side boyz;3162\nlovedrug;3163\nour last night;3164\nmura masa;3165\nlacuna coil;3166\nluke doucet;3167\npunchline;3168\nanimal logic;3169\nbig bang;3170\nsieges even;3171\ng.g.f.h.;3172\ntyler lyle;3173\nthe boswell sisters;3174\nfightstar;3175\nkevin rudolf;3176\ndangerous toys;3177\ncomeback kid;3178\nwild orchid;3179\none direction;3180\nelectric six;3181\nthe corrs;3182\nchristy nockels;3183\nhillsong;3184\nsham 69;3185\nhurt;3186\nclint black;3187\nchris hillman & herb pedersen;3188\ndef leppard;3189\nplants and animals;3190\njack bruce;3191\nthalía;3192\narthur crudup;3193\nphil wickham;3194\nking gizzard & the lizard wizard;3195\nbeautiful eulogy;3196\nthe last bison;3197\ncaravan;3198\nthe wonder stuff;3199\njeff williams;3200\nsebadoh;3201\njets overhead;3202\ngolden smog;3203\njonna lee;3204\nthe faceless;3205\ndaniel lanois;3206\nthe judds;3207\nparagon;3208\nvienna philharmonic;3209\nelectric president;3210\ntenth avenue north;3211\nsolomon cutner;3212\nvnv nation;3213\nsystem syn;3214\neric lindell;3215\njenny lewis;3216\nthe flower kings;3217\nchicago;3218\nstacey q;3219\nark;3220\nchingo bling;3221\nisac elliot;3222\npenumbra;3223\ncypress hill;3224\nlandon pigg;3225\ncollide;3226\nmick ronson;3227\nfall of the leafe;3228\nmelissa etheridge;3229\nsuicidal tendencies;3230\njoshua kadison;3231\nthe spill canvas;3232\neclipse;3233\nwithin the ruins;3234\nhaemorrhage;3235\ndeliverance;3236\nrobert pollard;3237\npaul kelly;3238\nalesana;3239\nthe big pink;3240\ne-rotic;3241\nthe books;3242\nh.p. lovecraft historical society;3243\nargent;3244\nnivea;3245\nmario;3246\nthe fiery furnaces;3247\natc;3248\nmatt and kim;3249\ntobymac;3250\nlcd soundsystem;3251\nant & dec;3252\nhowe gelb;3253\nhozier;3254\nastarte;3255\nanathallo;3256\nbasia bulat;3257\ntad morose;3258\nsmp;3259\ngreensky bluegrass;3260\nholly miranda;3261\nub40;3262\nd12;3263\nscott miller;3264\nchicane;3265\nthe faint;3266\ncream;3267\narchive;3268\ndeicide;3269\nlegendary shack shakers;3270\nguano apes;3271\nwolfgun;3272\ntyphoon;3273\npile;3274\nthe philosopher kings;3275\nblack grape;3276\nkotipelto;3277\ndavid lee murphy;3278\nfish;3279\nnekromantix;3280\nthe chordettes;3281\nsherban lupu;3282\ndarlene zschech;3283\nthe henry girls;3284\nida cox;3285\npeter & gordon;3286\nfear, and loathing in las vegas;3287\njme;3288\nblood on the dance floor;3289\nelectric light orchestra;3290\nromeo;3291\nmike heron;3292\nkendrick lamar;3293\nspock's beard;3294\nthe bunny the bear;3295\nsweetbox;3296\nmac miller;3297\nspazz;3298\nliars;3299\n98°;3300\nblood duster;3301\nmake do and mend;3302\ncheri dennis;3303\ngerald levert;3304\nchris tomlin;3305\nall saints;3306\nghost ship;3307\nyoung the giant;3308\njim james;3309\nthe unseen;3310\nwhitesnake;3311\nciara;3312\nseventh day slumber;3313\nthe b-52's;3314\ncrime & the city solution;3315\nbrentalfloss;3316\ngene clark;3317\nnouvelle vague;3318\ndon johnson big band;3319\nshwayze;3320\ncat rapes dog;3321\npoisonblack;3322\nthe wildhearts;3323\nbruce hornsby and the range;3324\nlogic;3325\nvaux;3326\nsuzy bogguss;3327\nsarke;3328\ncharlie louvin;3329\nwild child;3330\nicons of filth;3331\ntarnation;3332\nthousand foot krutch;3333\nnikki webster;3334\nwolfmother;3335\nsamuel barber;3336\npanda bear;3337\nganggajang;3338\nrosetta stone;3339\nsonia disappearfear;3340\nthe s.o.s. band;3341\nw.a.s.p.;3342\nsilverstein;3343\ntin machine;3344\njennifer warnes;3345\nandru donalds;3346\nolympos mons;3347\ncerebral fix;3348\nglenn frey;3349\nsuzi quatro;3350\nsons of the pioneers;3351\nvenetian princess;3352\ndelirious?;3353\nlecrae;3354\nmarty robbins;3355\njeannie seely;3356\nthe psychedelic ensemble;3357\nglenn medeiros;3358\nbleed from within;3359\nthird world;3360\nalison krauss;3361\ndj shadow;3362\na static lullaby;3363\nmaps & atlases;3364\njames taylor;3365\nmekong delta;3366\neasy star all-stars;3367\naura noir;3368\nsantana;3369\nthe gadjits;3370\ncan;3371\ndeepspace 5;3372\nleslie west;3373\nfr d ric chopin;3374\nsundara karma;3375\npowerman 5000;3376\ndutch rebelle;3377\nbones brigade;3378\nthe browning;3379\npeter andre;3380\nconnie francis;3381\nsmooth;3382\nmajesty;3383\nmatthew dear;3384\nfionn regan;3385\nnuclear assault;3386\nover the rhine;3387\nmagellan;3388\nzoot woman;3389\nshyne;3390\nkidneythieves;3391\nracoon;3392\nironsword;3393\nalessandro scarlatti;3394\nthe appleseed cast;3395\noperation ivy;3396\nakron/family;3397\nqkumba zoo;3398\nthe chasm;3399\nthe style council;3400\nmethod man;3401\nthe halliard;3402\nboy george;3403\ntarot;3404\ndelays;3405\ncromok;3406\nhoneymoon suite;3407\nhercules & love affair;3408\nlemuria;3409\nthe kelly family;3410\ntweet;3411\ntru;3412\nlawnmower deth;3413\nno use for a name;3414\ndogwood;3415\npenitent;3416\ncapture the crown;3417\nhawksley workman;3418\nthe drifters;3419\nschool of seven bells;3420\njames keelaghan;3421\nthe 69 eyes;3422\ninsane clown posse;3423\nsave ferris;3424\ntrust;3425\natb;3426\ntodd rundgren;3427\nhank cochran;3428\nforever the sickest kids;3429\nnew riders of the purple sage;3430\ngeorg philipp telemann;3431\ncrippled black phoenix;3432\npaul mccartney;3433\nryan adams and the cardinals;3434\nsilent cry;3435\nking creosote;3436\nrun the jewels;3437\nthe kings;3438\nsteve goodman;3439\nnitty gritty dirt band;3440\neric bibb;3441\neditors;3442\nblessed by a broken heart;3443\ncivil twilight;3444\nchildren of bodom;3445\nmichael jackson;3446\nrichie sambora;3447\nwillam;3448\ndevildriver;3449\ne.s.g.;3450\nsananda maitreya;3451\nbabbie mason;3452\nthe museum;3453\ndan zanes;3454\ne-type;3455\ngidon kremer;3456\nal denson;3457\naeon;3458\nkenny rogers;3459\nthe tear garden;3460\njoshua bell;3461\nset your goals;3462\nle tigre;3463\nholly dunn;3464\nignacy jan paderewski;3465\nimmortal;3466\n108;3467\nann beretta;3468\ndezarie;3469\npete seeger;3470\njan & dean;3471\nnina kinert;3472\nthe lonely forest;3473\nprotest the hero;3474\ndiorama;3475\nkhoma;3476\njapan;3477\njim brickman;3478\nhalifax;3479\nrebecca lynn howard;3480\nflobots;3481\nthe new power generation;3482\ngods paparazzi;3483\nshannon curfman;3484\nroy woods;3485\nvarsity fanclub;3486\ndave stewart;3487\nmumford & sons;3488\nsyd matters;3489\nthe nerve agents;3490\nkula shaker;3491\nluke sital-singh;3492\njohn wesley harding;3493\nvenom;3494\nmelanie doane;3495\nthe hippos;3496\njessi colter;3497\nthe movielife;3498\nbabybird;3499\nbodyfarm;3500\ntara maclean;3501\njackson browne;3502\nwithin temptation;3503\ncash cash;3504\nwarren zevon;3505\neva cassidy;3506\nbilly ray cyrus;3507\njustin timberlake;3508\nthe antlers;3509\njohn elefante;3510\nangel;3511\njaap schr der;3512\nevermore;3513\ndestroy the runner;3514\nthe black crowes;3515\ntiga;3516\nbury your dead;3517\nslowdive;3518\nmachine head;3519\nc-murder;3520\ndirty;3521\nfujiya & miyagi;3522\nagonoize;3523\nhundredth;3524\nprong;3525\nnikki flores;3526\ncyanotic;3527\njordan pruitt;3528\nu.s. bombs;3529\na halo called fred;3530\ndon williams;3531\nchester watson;3532\nnoa;3533\nmy passion;3534\ngeorge harrison;3535\nscandroid;3536\nsister sin;3537\nhot snakes;3538\nblack flag;3539\nchristie;3540\nchristoph eschenbach;3541\nslim gaillard;3542\ngravediggaz;3543\nthe northern pikes;3544\nmiranda sex garden;3545\ncolin blunstone;3546\nthe rasmus;3547\nnazareth;3548\ndegarmo and key;3549\nearth crisis;3550\nthe foreign exchange;3551\nuncle tupelo;3552\nkitty wells;3553\nkarate high school;3554\nweeping tile;3555\nskinny puppy;3556\naudience;3557\ndevotchka;3558\njohn lennon;3559\nelf power;3560\nteresa brewer;3561\nheinrich schiff;3562\n6lack;3563\nanti-depressive delivery;3564\nthe moffatts;3565\nlynn anderson;3566\nmordacious;3567\nrobert palmer;3568\nseabound;3569\nblack star riders;3570\nbryan rice;3571\nj.b. lenoir;3572\nthe yards;3573\nthe stupid stupid henchmen;3574\ntony rice;3575\nbarbecue bob;3576\ndr. hook & the medicine show;3577\ngarrison starr;3578\nunion 13;3579\njay sean;3580\npretty willie;3581\noceano;3582\ncaptain beefheart and the magic band;3583\nmarkéta irglová;3584\nwolfe tones;3585\ncrumbsuckers;3586\ntowers of london;3587\nimpaled nazarene;3588\nbig tent revival;3589\nhans theessink;3590\nthe amity affliction;3591\nanimal liberation orchestra;3592\ntower of power;3593\nvanessa bell armstrong;3594\ntom fogerty;3595\npianos become the teeth;3596\npoppy;3597\nthe knack;3598\nfields of the nephilim;3599\npeter and the test tube babies;3600\ncory branan;3601\ncherrelle;3602\npalaye royale;3603\nsouthside johnny & the asbury jukes;3604\nheideroosjes;3605\nthe busters;3606\ndave edmunds;3607\nbuck 65;3608\nthe lost trailers;3609\nsteam powered giraffe;3610\nlydia lunch;3611\nwashboard sam;3612\nbig time rush;3613\nnlt;3614\nold 97's;3615\nrupert hine;3616\njon mclaughlin;3617\nacid drinkers;3618\nheaven & hell;3619\ngeorge hamilton iv;3620\nscott joplin;3621\nthe berzerker;3622\njonatha brooke;3623\niggy azalea;3624\ntrijntje oosterhuis;3625\ncee lo green;3626\nwig wam;3627\nmark heard;3628\nphil manzanera;3629\nthe story;3630\nkokomo arnold;3631\nthe beau brummels;3632\nacid bath;3633\ntalisman;3634\nmc chris;3635\nthe del mccoury band;3636\nty segall;3637\nboysetsfire;3638\nthe ducky boys;3639\ndelta moon;3640\nthumb;3641\nblacklisted;3642\npurity ring;3643\nanders manga;3644\nburn halo;3645\nhow to dress well;3646\nmaxnormal.tv;3647\nbinoculers;3648\nlou barlow;3649\ndoug sahm;3650\ngary allan;3651\njan wayne;3652\nmörk gryning;3653\ndanko jones;3654\ndover;3655\nrobben ford;3656\nblack breath;3657\nlovespirals;3658\nsérgio mendes;3659\nrufus;3660\nkingcrow;3661\ndmitri shostakovich;3662\nghostface killah;3663\nlyriel;3664\nagathodaimon;3665\nhezekiah walker;3666\nnonpoint;3667\nbalzac;3668\nlunatica;3669\nella mae morse;3670\ntoyah;3671\nnik kershaw;3672\nhammerfall;3673\njmsn;3674\nspeaker;3675\nwhitechapel;3676\nhaddaway;3677\ngrace vanderwaal;3678\nthe tubes;3679\neric church;3680\nthe o.c. supertones;3681\nfrozen ghost;3682\n98 mute;3683\nbobby darin;3684\nvoxtrot;3685\nthe darkness;3686\nparis;3687\nkillswitch engage;3688\njessica lea mayfield;3689\nn-trance;3690\nhank williams, jr.;3691\nb5;3692\nthunderstone;3693\nnorth mississippi allstars;3694\nangry samoans;3695\nalexis korner;3696\nyoussou n'dour;3697\njet set satellite;3698\nboomkat;3699\nleif ove andsnes;3700\nektomorf;3701\nceremony;3702\nisley jasper isley;3703\nr.a. the rugged man;3704\nthe twilight sad;3705\nthe electric prunes;3706\ndream street;3707\nthe yardbirds;3708\nturisas;3709\ncelldweller;3710\nwigwam;3711\npaul carrack;3712\ndads;3713\nper gessle;3714\neugene mcguinness;3715\nhardline;3716\npatti labelle;3717\nbusta rhymes;3718\nrasputina;3719\nantique;3720\nschoolboy q;3721\nbeat happening;3722\npunch brothers;3723\nkeshia chanté;3724\nbob carlisle;3725\ndaan;3726\nimperative reaction;3727\nafter the fire;3728\nmarc and the mambas;3729\nminnie driver;3730\nsecondhand serenade;3731\nthe byrds;3732\nbread;3733\ntalib kweli;3734\nnelly;3735\nhagalaz' runedance;3736\nstick to your guns;3737\nthe virus;3738\nsnow;3739\njames morrison;3740\nyes;3741\nneuroactive;3742\nmisanthrope;3743\nmorgana lefay;3744\nche'nelle;3745\nhackneyed;3746\nwilhelm backhaus;3747\nwarren haynes;3748\nproclamation;3749\nandy griggs;3750\nthe bongos;3751\ncorrosion of conformity;3752\npaloma faith;3753\nthe japanese house;3754\ntizzy bac;3755\nagathocles;3756\nkari peitsamo;3757\nmýa;3758\nignaz friedman;3759\nblitz;3760\ndyscarnate;3761\nthe hidden cameras;3762\negypt central;3763\nnoggin toboggan;3764\nsadistik;3765\npeter gabriel;3766\ncinderella;3767\nsebastian sturm;3768\npsycroptic;3769\nsylver;3770\njaymay;3771\nnahemah;3772\ndubioza kolektiv;3773\nthe pogues;3774\nthe arrogant sons of bitches;3775\nnelson freire;3776\nmastedon;3777\nleonid kogan;3778\nezra furman;3779\nohgr;3780\nedward maya;3781\nwings;3782\nfreya;3783\nskye;3784\nablaze my sorrow;3785\ngerry rafferty;3786\nfranz schubert;3787\npeccatum;3788\nbananafishbones;3789\nclaude debussy;3790\neurythmics;3791\nwill oldham;3792\nfiona boyes;3793\ncellar darling;3794\npublic image ltd.;3795\nfireflight;3796\nafu-ra;3797\nlittle mix;3798\npower quest;3799\nemmy the great;3800\nsub7even;3801\ngwen stefani;3802\nplayradioplay!;3803\naaron neville;3804\nthe stryder;3805\nlullacry;3806\nkelli ali;3807\nbow wow;3808\nigor stravinsky;3809\ngrade;3810\ntall dwarfs;3811\nrbd;3812\ndespised icon;3813\nblues magoos;3814\nthe long winters;3815\nmajid jordan;3816\nvincenzo bellini;3817\nloretta lynn;3818\nkenny wayne shepherd;3819\nbleeding through;3820\ncursive;3821\nlinus of hollywood;3822\ng-unit;3823\nthe casket lottery;3824\nkingston wall;3825\nkate voegele;3826\namanda lear;3827\nvicky beeching;3828\ncolin james;3829\nelvis costello & the attractions;3830\ndana dirksen;3831\nc sar franck;3832\ndog fashion disco;3833\njosé gonzález;3834\nthe joy formidable;3835\nelly ney;3836\nmeliah rage;3837\ntim moore;3838\njames blunt;3839\namerican pleasure club;3840\nnicole scherzinger;3841\nthe bad shepherds;3842\nthe fabulous thunderbirds;3843\ni can make a mess like nobody's business;3844\nthree plus;3845\njulie roberts;3846\ntribe;3847\nluke bryan;3848\nsaxon;3849\nphilip glass;3850\nbobby womack;3851\ntriumph;3852\nveda hille;3853\ncoldrain;3854\nen vogue;3855\namy winehouse;3856\nbonnie pink;3857\nthe doc watson family;3858\nbad company;3859\nfuck;3860\ncradle of filth;3861\ngorky's zygotic mynci;3862\nbuck owens;3863\nelle king;3864\nsamael;3865\nmeryn cadell;3866\nnocturnal rites;3867\nghost dance;3868\nkristy lee cook;3869\nthe script;3870\njake owen;3871\ndeath cab for cutie;3872\nthe forgotten rebels;3873\nbeaver;3874\ntom rush;3875\ndavid rock feinstein;3876\noathbreaker;3877\nodetta;3878\ncompton's most wanted;3879\npsy'aviah;3880\nbig ed;3881\njohn miles;3882\nshellac;3883\nturin brakes;3884\neternal tears of sorrow;3885\nkodak black;3886\nslick rick;3887\ncode orange;3888\ngioacchino rossini;3889\nmarc andr hamelin;3890\nulver;3891\nla toya jackson;3892\npath of resistance;3893\nlaura fygi;3894\ntom mcrae;3895\nsitti;3896\nfrank stokes;3897\nanorexia nervosa;3898\ngabriel cyphre;3899\nthe jon spencer blues explosion;3900\nm-flo;3901\njason donovan;3902\nsean paul;3903\ngirls;3904\n504 boyz;3905\nbring me the horizon;3906\nshelby lynne;3907\npierre laurent aimard;3908\njim messina;3909\nthe boyz;3910\nhank thompson;3911\nhands;3912\nzakk wylde;3913\nnikka costa;3914\nnelly furtado;3915\nfairground attraction;3916\nborn of osiris;3917\nfenix tx;3918\nrobert plant;3919\nrobert casadesus;3920\npaul van dyk;3921\nglenn kaiser band;3922\ndaughters;3923\nsaraya;3924\nrae sremmurd;3925\nnorma jean;3926\nthe plot in you;3927\nwild nothing;3928\napocalyptica;3929\ntony! toni! toné!;3930\nolivia newton-john;3931\nkeith richards;3932\nelizabeth cook;3933\noutlandish;3934\nhalford;3935\nsusan ashton;3936\ndoves;3937\nqueen + paul rodgers;3938\nwishing chair;3939\npseudo echo;3940\ncrazyeightyeight;3941\nred lights flash;3942\nneon indian;3943\nerick sermon;3944\nthe juliana theory;3945\nlarry gatlin & the gatlin brothers;3946\npaul whiteman;3947\nmissy elliott;3948\nthe chiffons;3949\nallison iraheta;3950\ncentvrion;3951\nthe 5th dimension;3952\ndighayzoose;3953\nnina nastasia;3954\nshadowside;3955\n5 seconds of summer;3956\nasaf avidan;3957\ncady groves;3958\nold man luedecke;3959\nthe chap;3960\njohn maus;3961\ntom petty and the heartbreakers;3962\nhave a nice life;3963\nbreach of trust;3964\nclaudio arrau;3965\ntoro y moi;3966\nthe tony danza tapdance extravaganza;3967\noscar brand;3968\njohn mccormack;3969\nanti-nowhere league;3970\nshearer;3971\nj.j. cale;3972\niyaz;3973\nice ages;3974\nthe city harmonic;3975\nbruce mcculloch;3976\nthe color changin' click;3977\ncrack the sky;3978\nthe moldy peaches;3979\ncoolio;3980\njaill;3981\nmud;3982\nhodgy beats;3983\nclap your hands say yeah;3984\nbadly drawn boy;3985\newert and the two dragons;3986\ntystnaden;3987\njanie fricke;3988\ndottie west;3989\nwitherscape;3990\n目黒将司 (shoji meguro);3991\nmnek;3992\nkurt nilsen;3993\npsychic tv;3994\nthe boomtown rats;3995\nnena;3996\nstephin merritt;3997\nnausea;3998\nmolly venter;3999\nrasaq;4000\nolly murs;4001\nхелависа;4002\ncoko;4003\nscarface;4004\nsam smith;4005\nmidnight juggernauts;4006\npulley;4007\ntheresa sokyrka;4008\njacob banks;4009\nbomb the music industry!;4010\nmeredith andrews;4011\nyearning;4012\nron sexsmith;4013\njoseph haydn;4014\neinherjer;4015\nfour tops;4016\ngiles, giles and fripp;4017\niona brown;4018\nbon jovi;4019\nkunt and the gang;4020\nmotionless in white;4021\nsabrina;4022\njillian aversa;4023\nthe persuasions;4024\ntub ring;4025\nsacred warrior;4026\nsusan boyle;4027\nla the darkman;4028\nsavage garden;4029\nlindisfarne;4030\njag panzer;4031\ndana key;4032\nwouter hamel;4033\npeter green;4034\njill scott;4035\nwreck and reference;4036\nvanilla fudge;4037\naerosmith;4038\nmegafaun;4039\npoundhound;4040\n2 skinnee j's;4041\nthe call;4042\nsonny boy williamson i;4043\npossessed;4044\nannie;4045\nsuburban legends;4046\nthe mutton birds;4047\ncibo matto;4048\nlorna shore;4049\nterry callier;4050\nselah sue;4051\nderek webb;4052\nparliament;4053\nsister machine gun;4054\nthe badlees;4055\nthe regrettes;4056\nmarlon roudette;4057\nalbert king;4058\nrebelution;4059\nshura cherkassky;4060\ndave rodgers;4061\nskip james;4062\ncaedmon's call;4063\nasp;4064\nsouldecision;4065\nsavoir adore;4066\nnathaniel rateliff;4067\nthe temperance movement;4068\ndragonland;4069\nisaac alb niz;4070\nfalling up;4071\nstabilo;4072\nchrist analogue;4073\nbig mountain;4074\nsade;4075\nthe casting out;4076\ngene autry;4077\nmanfred mann;4078\ndrive-by truckers;4079\njavier mendoza;4080\nalyson stoner;4081\nblind lemon jefferson;4082\nlowkey;4083\nbrad paisley;4084\njane child;4085\ni declare war;4086\nleo jan ek;4087\nlonestar;4088\nboards of canada;4089\ndon henley;4090\nchoclair;4091\nobsc(y)re;4092\ndonnie mcclurkin;4093\nwall of voodoo;4094\ntony bennett;4095\nhank snow;4096\npoison the well;4097\nfancy;4098\nunbelievable truth;4099\njet life;4100\nrory block;4101\nrachael lampa;4102\nthe platters;4103\ntahiti 80;4104\nthe gathering;4105\nbrett dennen;4106\nlily allen;4107\npuffy amiyumi;4108\niamx;4109\ntree63;4110\nbee gees;4111\ne.g. daily;4112\nnarnia;4113\npeter green splinter group;4114\ngram parsons;4115\nfacing new york;4116\nkalmah;4117\nhot water music;4118\nblessthefall;4119\ngeneral public;4120\ncornershop;4121\nowen pallett;4122\ntruth hurts;4123\nquorthon;4124\nkill your idols;4125\nplanet funk;4126\nleonardo's bride;4127\njimmy rushing;4128\nthe title;4129\ndog eat dog;4130\nzucchero;4131\nmind.in.a.box;4132\nmainstay;4133\ngraham nash;4134\ntherion;4135\nbad boys blue;4136\nswizz beatz;4137\njoey + rory;4138\nmatthew herbert;4139\nkrewella;4140\nlou rhodes;4141\nlinda perry;4142\nthe pointer sisters;4143\nfit for an autopsy;4144\nyelworc;4145\njeffrey lewis;4146\narghoslent;4147\nandy park;4148\nuriah heep;4149\ndisciple;4150\nsodom;4151\nbeth nielsen chapman;4152\nchuck ragan;4153\nspoons;4154\nhaerts;4155\nice nine kills;4156\nrosie thomas;4157\nguerilla toss;4158\ndizzee rascal;4159\ng4;4160\nmendeed;4161\nempire of the sun;4162\nfergie;4163\nmishka;4164\ntom jones;4165\nthe brothers four;4166\nracer x;4167\nclipping.;4168\nstiff little fingers;4169\nthe ghost of a saber tooth tiger;4170\nultimatum;4171\nodyssey eurobeat;4172\ntony banks;4173\nshaggy;4174\nkotoko;4175\nnothingface;4176\nmellowhype;4177\nanthony green;4178\nbulldozer;4179\nabigail williams;4180\ngangsta boo;4181\njonny lang;4182\nj. cole;4183\nedvard grieg;4184\nthe residents;4185\nluther allison;4186\njean philippe rameau;4187\nrandy newman;4188\njeff the brotherhood;4189\nking missile;4190\ncloud nothings;4191\nfiona;4192\nthe walkmen;4193\nnicola benedetti;4194\nbirmingham 6;4195\nedge of dawn;4196\nnathan milstein;4197\nthe paul butterfield blues band;4198\nhe is legend;4199\nol' dirty bastard;4200\nsteve miller band;4201\ncraig's brother;4202\nberacah;4203\nnew edition;4204\nfalconshield;4205\nchildish gambino;4206\nam & shawn lee;4207\nunkle;4208\nsilver jews;4209\nthe interrupters;4210\nwilliam shatner;4211\nadrian belew;4212\nscanner;4213\nslash;4214\nsweatshop union;4215\ndayna kurtz;4216\nkrisma;4217\nferraby lionheart;4218\nnew years day;4219\nmadchild;4220\ntohoshinki;4221\nbusted;4222\ntyler bryant & the shakedown;4223\nvanessa hudgens;4224\nlady gaga;4225\nkosheen;4226\ncircle ii circle;4227\nglee cast;4228\nforgotten tales;4229\nrenaissance;4230\nlonnie johnson;4231\nswans;4232\nlaura marling;4233\nagainst the current;4234\nane brun;4235\nthe buckinghams;4236\nart bears;4237\nrocky votolato;4238\nthe soft boys;4239\njonathan thulin;4240\nandreas johnson;4241\npages;4242\nsteve moakler;4243\na.c. newman;4244\nshowaddywaddy;4245\nbart davenport;4246\nthe streets;4247\nallstar weekend;4248\ndeath grips;4249\nseventh avenue;4250\ngrandmaster flash;4251\na split-second;4252\nsense field;4253\nmaisey rika;4254\nhaken;4255\nsharon needles;4256\narcangelo corelli;4257\nteen suicide;4258\nblue highway;4259\nthe megas;4260\nnoxious emotion;4261\nkeldian;4262\nasian dub foundation;4263\nh.e.r.;4264\ngirlpool;4265\neddie degarmo;4266\nsteve taylor;4267\nalex story;4268\nwilly porter;4269\naura dione;4270\nmelanie b;4271\naz;4272\nchris isaak;4273\ncaptain tractor;4274\nanita baker;4275\nanne akiko meyers;4276\njosh rouse;4277\nnoname;4278\ntina guo;4279\nthe wannadies;4280\nthe romantics;4281\nfirewind;4282\nless than jake;4283\nowain phyfe;4284\ndrag the river;4285\nkeith sweat;4286\nkari jobe;4287\ndispatch;4288\njohn stewart;4289\nhellhammer;4290\nreflection eternal;4291\nthe last dance;4292\nultimate fakebook;4293\nnomad;4294\ncoal chamber;4295\njuice;4296\ngotthard;4297\nrichard wagner;4298\nthe strypes;4299\nfelix mendelssohn bartholdy;4300\nlou gramm;4301\nnecro;4302\nvladimir ashkenazy;4303\nthe datsuns;4304\ngreen jellÿ;4305\njohnny duncan;4306\nthingy;4307\njimmie rodgers;4308\npaul brady;4309\nhazel dickens;4310\nclaude king;4311\nwynardtage;4312\ncraig morgan;4313\njapandroids;4314\nthe almanac singers;4315\nmemphis slim;4316\neddie rabbitt;4317\nmortification;4318\nescape the fate;4319\ntonight alive;4320\ntyler, the creator;4321\ncubanate;4322\nalvin youngblood hart;4323\nsteel train;4324\nnatasha thomas;4325\nviolent soho;4326\nlaika;4327\nmichael bublé;4328\nfolly & the hunter;4329\nnektar;4330\ncarnifex;4331\njohan;4332\nmatthew good;4333\nbackyard babies;4334\njim kweskin;4335\nbeat crusaders;4336\nbullet for my valentine;4337\noceans ate alaska;4338\nthe communards;4339\nquo vadis;4340\nalessi brothers;4341\n1910 fruitgum company;4342\nattila;4343\nchristian bautista;4344\nian anderson;4345\nthe grates;4346\nethel waters;4347\nurthboy;4348\nburlap to cashmere;4349\nnails;4350\npyotr ilyich tchaikovsky;4351\nkeely smith;4352\nmilemarker;4353\ntruls m rk;4354\nsnoop dogg;4355\nakrobatik;4356\njorge bolet;4357\njonny diaz;4358\nkiller dwarfs;4359\nbirth control;4360\nwrite this down;4361\nhigh school football heroes;4362\nthe frames;4363\nangelo branduardi;4364\ncasey bill weldon;4365\nsteve green;4366\nbill anderson;4367\njin akanishi;4368\nkimbra;4369\nnevermore;4370\ngarrick ohlsson;4371\nnolongerhuman;4372\newan maccoll;4373\nthe incredible string band;4374\nhayden;4375\nbody count;4376\ncharlie parr;4377\nandrew w.k.;4378\nsarah blasko;4379\nvetiver;4380\nборис гребенщиков;4381\nstephen marley;4382\nsage francis;4383\nfischerspooner;4384\nthe strokes;4385\nbootsauce;4386\nmaurice ravel;4387\nmini mansions;4388\nheartsounds;4389\nvangelis;4390\nartension;4391\nbear's den;4392\nmystic circle;4393\nkatie melua;4394\nthe mission;4395\nbuddy & julie miller;4396\n2 live crew;4397\ncalexico;4398\ndanielle peck;4399\ndevil doll;4400\nthe mad conductor;4401\neliane elias;4402\nbrutal truth;4403\nbootsy collins;4404\nsaving abel;4405\nrobby valentine;4406\nvader;4407\nmidnattsol;4408\njigsaw;4409\nbullets and octane;4410\ncompany of thieves;4411\nkylie minogue;4412\nsandi thom;4413\nnorah jones;4414\nthe dillards;4415\nbulletboys;4416\nhigh places;4417\nneon horse;4418\nanna ternheim;4419\nalessia cara;4420\nwolfgang schneiderhan;4421\ntzu;4422\nt. rex;4423\ntreble charger;4424\nculture;4425\ngaelic storm;4426\nelliott smith;4427\nmemoryhouse;4428\nthe amazing rhythm aces;4429\nbalance of power;4430\ncoil;4431\nhot rod circuit;4432\nthe guess who;4433\nfredrika stahl;4434\nda' t.r.u.t.h.;4435\ntoh kay;4436\ngentle giant;4437\nroger whittaker;4438\nthe dø;4439\nneil sedaka;4440\nthe view;4441\nthe lookouts;4442\ntwrp;4443\ntinie tempah;4444\ndana winner;4445\ncivil war;4446\nhaujobb;4447\nthe crucified;4448\ntv on the radio;4449\ngil scott-heron;4450\nmarit larsen;4451\nrobert calvert;4452\nja rule;4453\ntina charles;4454\nima robot;4455\nneva dinova;4456\nthaurorod;4457\nlou christie;4458\nscritti politti;4459\nwhiskeytown;4460\nwhile she sleeps;4461\nkristine w;4462\nzilch;4463\nthe avett brothers;4464\nthe human league;4465\nilse delange;4466\ntinashe;4467\nmc ren;4468\nexciter;4469\npapa roach;4470\nfrank turner;4471\nsol invictus;4472\nanya marina;4473\nthese new puritans;4474\ninstalok;4475\ntom smith;4476\nmaria muldaur;4477\ne-40;4478\nholly golightly;4479\nmy bloody valentine;4480\ndefiance;4481\nallison moorer;4482\nwynn stewart;4483\nhaggard;4484\nall shall perish;4485\nfritz kreisler;4486\ndead prez;4487\nlenny kravitz;4488\nlagwagon;4489\ndamian marley;4490\nicehouse;4491\ndionne warwick;4492\nthe devil makes three;4493\ndrive, she said;4494\nhollow haze;4495\nanita carter;4496\nh.e.a.t;4497\nslaves on dope;4498\narsis;4499\nsamson;4500\nswv;4501\nl.a. guns;4502\nhiatus kaiyote;4503\nquicksilver messenger service;4504\nlea michele;4505\ngraf orlock;4506\ntiago iorc;4507\ngreat northern;4508\nray wylie hubbard;4509\nsuper junior;4510\njerry jeff walker;4511\nhocico;4512\nbukimina;4513\nstretch;4514\nanthony hamilton;4515\nmushroomhead;4516\nevergreen terrace;4517\nmichael mcdonald;4518\nleroy hutson;4519\nstructures;4520\ndance gavin dance;4521\nother people;4522\nlinda davis;4523\ndoc walker;4524\nrumer;4525\nchuck berry;4526\nshoffy;4527\ndickey lee;4528\nfreddy fender;4529\nbatmobile;4530\nthe jimi hendrix experience;4531\nteddy pendergrass;4532\nbushwick bill;4533\ngary moore;4534\narthur beatrice;4535\nhurrah!;4536\ntsunami bomb;4537\nthundra;4538\nherb alpert & the tijuana brass;4539\nthe flashbulb;4540\nkorpiklaani;4541\nthe mendoza line;4542\nvashti bunyan;4543\nunknown mortal orchestra;4544\njerry garcia;4545\nrichard shindell;4546\ni fight dragons;4547\nlate tuesday;4548\nchris connor;4549\nthe chemical brothers;4550\nvirgin steele;4551\nsybreed;4552\ndan le sac vs scroobius pip;4553\nlani hall;4554\nnero;4555\ntwin shadow;4556\ngordon downie;4557\naphrodite's child;4558\nbodyjar;4559\nmagic man;4560\nbaths;4561\ntender forever;4562\nof monsters and men;4563\nkrypteria;4564\njandek;4565\nr.i.o.;4566\ngovernment issue;4567\njane monheit;4568\njeffree star;4569\nmoby grape;4570\nsilver convention;4571\nmichelle wright;4572\nthe shroud;4573\njoseph szigeti;4574\nles claypool;4575\nthe kills;4576\nbeth hart;4577\nbobo in white wooden houses;4578\ntwilight force;4579\nlarry sparks;4580\neddie vedder;4581\nglen campbell;4582\nmichael hedges;4583\nthe red shore;4584\ngloria gaynor;4585\nm2m;4586\naltered images;4587\nyouth lagoon;4588\nxv;4589\nkevorkian death cycle;4590\nthe sugarcubes;4591\ncryptic slaughter;4592\ncandy butchers;4593\n8ball;4594\njhené aiko;4595\nmarcelle meyer;4596\ngaither vocal band;4597\nthe boy least likely to;4598\ndean martin;4599\njourney;4600\njackie evancho;4601\nesperanza spalding;4602\nthe time;4603\ntampa red;4604\npoison;4605\ndemis roussos;4606\nmaria solheim;4607\nliving colour;4608\nbrutality;4609\navatar;4610\nkatie armiger;4611\npaul wilbur;4612\ntokio hotel;4613\nlarue;4614\ngolden earring;4615\nsaint etienne;4616\nthe go! team;4617\njames gang;4618\nmaster p;4619\nbillie holiday;4620\nthey might be giants;4621\npaul simon;4622\nflashlight brown;4623\ndreezy;4624\ndavid byron;4625\nkim boyce;4626\nmasta killa;4627\ndmx;4628\nflesh-n-bone;4629\njune carter cash;4630\ncrucial conflict;4631\npersephone;4632\nasher roth;4633\nangel dust;4634\njohnny crash;4635\namy ray;4636\nlang lang;4637\ntwo hours traffic;4638\nattrition;4639\ncarole king;4640\nsarah mclachlan;4641\nadolf busch;4642\nabsolution project;4643\njustin bieber;4644\npenny mclean;4645\nvelvet belly;4646\nmatt nathanson;4647\nholly cole;4648\ndiana vickers;4649\nella mai;4650\nmark eitzel;4651\nthe associates;4652\nvortech;4653\nin strict confidence;4654\nhelloween;4655\nchasing victory;4656\nninety pound wuss;4657\njackie gleason;4658\nbe bop deluxe;4659\nlit;4660\nthe bruisers;4661\namanda jenssen;4662\ncrowder;4663\nulcerate;4664\nmewithoutyou;4665\njoe purdy;4666\nt-bone walker;4667\nsarah brightman;4668\nchihiro onitsuka;4669\ntrophy eyes;4670\nthe receiving end of sirens;4671\nking's x;4672\nchad brownlee;4673\nbattlelore;4674\nrefused;4675\nfar-less;4676\nbrandy;4677\nblack stone cherry;4678\nregina spektor;4679\nhouse of pain;4680\n220 volt;4681\nty herndon;4682\ntarja;4683\naaron tippin;4684\nslightly stoopid;4685\nnic jones;4686\nanimal collective;4687\nconverge;4688\nfreddie king;4689\nkelela;4690\nthe samples;4691\ndraconian;4692\nthe klezmatics;4693\nrob crow;4694\ncountry joe and the fish;4695\ncage;4696\nbenny sings;4697\nwolfstone;4698\nw-inds.;4699\nlin-manuel miranda;4700\nazealia banks;4701\nzap mama;4702\ndeath threat;4703\nwill stratton;4704\naudio adrenaline;4705\njars of clay;4706\nshamir;4707\nmiranda cosgrove;4708\nginuwine;4709\nhell razah;4710\nthy primordial;4711\nweh;4712\njason aldean;4713\nphilippe entremont;4714\nsmog;4715\nbenny hester;4716\notis redding;4717\nthe methadones;4718\nkarnivool;4719\ntennessee ernie ford;4720\nbob rivers;4721\nr. kelly;4722\ngang starr;4723\nking crimson;4724\nas we fight;4725\nsons of butcher;4726\nof the wand & the moon;4727\nthe cars;4728\nsalt the wound;4729\nfair to midland;4730\nlydia;4731\nterri clark;4732\nchris chameleon;4733\nthe project hate mcmxcix;4734\nshadow gallery;4735\nblack francis;4736\ngym class heroes;4737\nbill nelson;4738\nan cafe;4739\nsmokey robinson;4740\nthe duhks;4741\nbroadway;4742\nfetty wap;4743\nbilly sprague;4744\nnorther;4745\nsix feet under;4746\nrepublica;4747\nliam finn;4748\nhighway 101;4749\nmaureen mcgovern;4750\ndim mak;4751\nthe go-betweens;4752\nold man's child;4753\nfall out boy;4754\nmia x;4755\nbbmak;4756\ncannabis corpse;4757\nludacris;4758\ncreature feature;4759\nmiss montreal;4760\neric burdon & the animals;4761\nbumblefoot;4762\norigin;4763\ncapital cities;4764\ndoro;4765\nnox arcana;4766\nhateen;4767\nthe band perry;4768\nmetallica;4769\nbuffalo springfield;4770\nthe hooters;4771\npeaches & herb;4772\nkrystal meyers;4773\npaul rodgers;4774\nmemphis jug band;4775\nsofie;4776\nmaurizio pollini;4777\nbrook benton;4778\ndie verbannten kinder evas;4779\nbrainiac;4780\nbearstronaut;4781\npierce the veil;4782\nmatthew friedberger;4783\nvictims family;4784\nhell or highwater;4785\nemerson drive;4786\ncaro emerald;4787\nlouis jordan;4788\nthe osmonds;4789\nunspoken;4790\ncraig cardiff;4791\ntomohisa yamashita;4792\nari hest;4793\ndj jazzy jeff & the fresh prince;4794\ndj drama & lil wayne;4795\nziggy;4796\nwaters;4797\nシド (sid);4798\njeannie c. riley;4799\ntilly and the wall;4800\nkaren matheson;4801\nblind passengers;4802\nbrotha lynch hung;4803\nharry nilsson;4804\nyoung money;4805\nmarike jager;4806\ngood shoes;4807\ntom tom club;4808\nanimaniacs;4809\nsolitude aeturnus;4810\ngeorge strait;4811\nakinyele;4812\nthe honorary title;4813\ncarl wilson;4814\nirving;4815\nizzy stradlin;4816\nthe carter family;4817\ngzr;4818\naxxis;4819\nmarc e. bassy;4820\nchipmunk;4821\ngigi d'agostino;4822\ntindersticks;4823\nthe dears;4824\nthe ocean;4825\nraul malo;4826\njoel plaskett;4827\ntrooper;4828\narnold schoenberg;4829\nzayn;4830\nhellsongs;4831\nleftöver crack;4832\nthe waterboys;4833\nlevon helm;4834\ngehenna;4835\nforce majeure;4836\ntake 6;4837\namy rigby;4838\nlil baby;4839\nnorth star;4840\nmott the hoople;4841\ndorsal atlântica;4842\nsabrina carpenter;4843\njulee cruise;4844\nblu cantrell;4845\nwillard grant conspiracy;4846\nfaron young;4847\nceltic woman;4848\ntony joe white;4849\npetal;4850\ntitus andronicus;4851\nson house;4852\nhungry lights;4853\neskimo callboy;4854\nmarnie;4855\nseth walker;4856\nthe herd;4857\nasobi seksu;4858\ndavid crosby;4859\nbilly joe royal;4860\njinkx monsoon;4861\nthe replacements;4862\niio;4863\nbinärpilot;4864\nyoung guns;4865\nmel tormé;4866\nthe baseballs;4867\naphex twin;4868\nmike ness;4869\nsnot;4870\nblind willie mctell;4871\nicon for hire;4872\njeff tweedy;4873\nswollen members;4874\ncarly rae jepsen;4875\ncrystal viper;4876\nlarry the cable guy;4877\nandre nickatina & equipto;4878\nraphael saadiq;4879\nmesmerize;4880\nin flames;4881\nkristy thirsk;4882\nheadlights;4883\ndeathspell omega;4884\njoan armatrading;4885\nred red meat;4886\ndevlin;4887\nladyhawke;4888\nween;4889\nmates of state;4890\nhusky rescue;4891\nrhett akins;4892\ngodsmack;4893\ngrandpa jones;4894\nmatt maher;4895\nill niño;4896\nflatfoot 56;4897\njosh garrels;4898\nobey the brave;4899\ndamh the bard;4900\nsara lov;4901\ncharles aznavour;4902\nthin lizzy;4903\nfatherson;4904\ntristania;4905\nbinary star;4906\nwarlock;4907\nray j;4908\njamie grace;4909\nmildred bailey;4910\nheartsrevolution;4911\nthe dreaming;4912\nthe feelies;4913\nlucy rose;4914\nartifacts;4915\nm (uk);4916\nthe nits;4917\nthe outfield;4918\nfreelance whales;4919\n4minute;4920\nbabyland;4921\npropagandhi;4922\nlil johnson;4923\ndigital summer;4924\nthe tenors;4925\nsimply red;4926\nde-phazz;4927\nbenny goodman;4928\nfoxy brown;4929\nbrainstorm;4930\nhelen kane;4931\nles humphries singers;4932\nthe prids;4933\nbruce cockburn;4934\nsuperbus;4935\nben e. king;4936\ncryonic temple;4937\nrenard;4938\ntom robinson;4939\nthe cover girls;4940\ni, the breather;4941\nworm is green;4942\ncommodores;4943\njulie doiron;4944\nportugal. the man;4945\nmike & the mechanics;4946\n7 year bitch;4947\nvenetian snares;4948\njohnny mercer;4949\ngrandaddy;4950\nmalia;4951\nazure ray;4952\ncarolina liar;4953\nnitronoise;4954\nrufus wainwright;4955\nthe internet;4956\nj-zone;4957\ncoronatus;4958\ncelestial season;4959\njimmy buffett;4960\nfreeway;4961\ndanielson;4962\nzwan;4963\nboys night out;4964\njulia jacklin;4965\nthe fair sex;4966\nokkervil river;4967\nthe raveonettes;4968\nwilson phillips;4969\nelane;4970\nandi deris;4971\nrocking chairs;4972\nmichael johnson;4973\nrush;4974\nthe oak ridge boys;4975\nrev. edward w. clayborn;4976\nbonnie bianco;4977\nryan bingham;4978\njeremy enigk;4979\ngodhead;4980\nbefore their eyes;4981\narsonists get all the girls;4982\nempyrium;4983\nbedlight for blue eyes;4984\nrehab;4985\ndead sara;4986\nsnafu;4987\nroy acuff;4988\nthe partridge family;4989\nmalcolm holcombe;4990\nportastatic;4991\nthievery corporation;4992\nelysium;4993\nthe other;4994\nbeardfish;4995\nvanessa amorosi;4996\nbabylon whores;4997\npink floyd;4998\nnana grizol;4999\ncharly mcclain;5000\nchico debarge;5001\nten masked men;5002\nbeloved;5003\nthe shamen;5004\ncharlie peacock;5005\nandr s schiff;5006\ncastanets;5007\ncold cave;5008\nmichael schenker group;5009\nhouse vs. hurricane;5010\nthe pentangle;5011\nweerd science;5012\nghost brigade;5013\ndaysend;5014\nmanfred mann's earth band;5015\nelectrelane;5016\nsmokin' joe kubek & bnois king;5017\nwalter becker;5018\nel perro del mar;5019\nheaven & earth;5020\namebix;5021\nstanfour;5022\nvallenfyre;5023\ntankard;5024\ncirca waves;5025\nlouis lortie;5026\nhammock;5027\nbirdman;5028\nesther phillips;5029\ngarnet rogers;5030\nicona pop;5031\nchvrches;5032\ndonald fagen;5033\nkim mitchell;5034\ncanibus;5035\nwoods of ypres;5036\nfour letter lie;5037\neug ne ysa e;5038\nbiz markie;5039\ntila tequila;5040\nblue mountain;5041\nbuddy holly;5042\nrootwater;5043\nobie trice;5044\nferlin husky;5045\nhinds;5046\nemmylou harris;5047\nhop along;5048\nian moore;5049\nroger;5050\nmicky & the motorcars;5051\nbebel gilberto;5052\nrod stewart;5053\nrichard smallwood;5054\ndeftones;5055\nsuicide silence;5056\nthe wrights;5057\nrudolf firku n ;5058\ngod module;5059\ngregory alan isakov;5060\nangus & julia stone;5061\nreal mccoy;5062\nkate miller-heidke;5063\neric's trip;5064\nwoe, is me;5065\nrex orange county;5066\njascha heifetz;5067\nnovembre;5068\nministry;5069\ngraveyard;5070\ntone damli;5071\nalien sex fiend;5072\neazy-e;5073\njj72;5074\nsublime;5075\ngeorge formby;5076\ntrapeze;5077\nmikhail pletnev;5078\nroger clyne & the peacemakers;5079\nrita coolidge;5080\nxxxtentacion;5081\nliz durrett;5082\ncollective soul;5083\nsecond person;5084\nmat kearney;5085\nmr. 3-2;5086\nezio;5087\nthe republic of wolves;5088\nstepdad;5089\nmike oldfield;5090\nred sun rising;5091\nsnfu;5092\nsaul williams;5093\njustin townes earle;5094\nupon a burning body;5095\nbruce hungerford;5096\njeff scott soto;5097\nbig business;5098\ndan tyminski;5099\ninfected mushroom;5100\nelend;5101\njohn waite;5102\nhilary hahn;5103\naustin mahone;5104\npretty girls make graves;5105\nbilly idol;5106\nstephen malkmus;5107\nmoddi;5108\ngalahad;5109\ndiana krall;5110\nthe be good tanyas;5111\n!distain;5112\nthe isley brothers;5113\nmarilyn manson;5114\nblackthorn;5115\nordinary time;5116\ndragonforce;5117\ni blame coco;5118\nthis providence;5119\nt-pain;5120\ncapleton;5121\ndc talk;5122\nleon redbone;5123\noptimus rhyme;5124\nzedd;5125\nblack label society;5126\ngary brooker;5127\nmelvins;5128\nmindless faith;5129\nthe warning;5130\nbombshell rocks;5131\nthe unthanks;5132\nsecrets;5133\njoan jett and the blackhearts;5134\nfuneral for a friend;5135\naorta;5136\nroger glover;5137\nnitzer ebb;5138\namber pacific;5139\nsneaker pimps;5140\ninsomnium;5141\ndanger radio;5142\nlay low;5143\nruss;5144\nbliss n eso;5145\ndj antoine;5146\nto kill a king;5147\ndubstar;5148\nby the tree;5149\nimelda may;5150\nemil gilels;5151\nredbone;5152\nthe highwaymen;5153\nfear;5154\nry cooder;5155\nludo;5156\nmance lipscomb;5157\nshawn colvin;5158\nbongzilla;5159\nthe promise ring;5160\ndr. dog;5161\nronnie dunn;5162\nthe buffoons;5163\naimee mann;5164\nchase & status;5165\nrose maddox;5166\nlights;5167\nakissforjersey;5168\ntommy shaw;5169\nrotersand;5170\nx japan;5171\nrichie furay;5172\nprovision;5173\ngordon lightfoot;5174\nprimus;5175\ndie sektor;5176\nmegadeth;5177\nagnetha fältskog;5178\nangelspit;5179\nmachine gun kelly;5180\nfather;5181\ncherry ghost;5182\nnana;5183\nensign;5184\nbjörk;5185\nstyx;5186\ncinema bizarre;5187\ntiamat;5188\nchris mills;5189\nrachael sage;5190\nprāta vētra;5191\nthe hold steady;5192\nphil lynott;5193\nbrian hyland;5194\nwe as human;5195\nthe wallflowers;5196\nkalan porter;5197\nfreddie hart;5198\ncorey crowder;5199\nthe angels of light;5200\npapermoon;5201\ntommy mcclennan;5202\nthe paper chase;5203\nikon;5204\nhappy monster band;5205\nmodern talking;5206\nphiladelphia orchestra;5207\nhellyeah;5208\nheart of a coward;5209\nstate property;5210\nhowling bells;5211\nshalamar;5212\nthe geraldine fibbers;5213\ntoots & the maytals;5214\nwalter trout;5215\nmichael o'brien;5216\nsweet;5217\nhate eternal;5218\ncarnival in coal;5219\ncéline dion;5220\nlee hazlewood;5221\namy holland;5222\ndefleshed;5223\nirma thomas;5224\nthe chieftains;5225\ndexter freebish;5226\nthe lads;5227\npeter bradley adams;5228\nfront line assembly;5229\nblindside;5230\nvulfpeck;5231\nkontrust;5232\nsmosh;5233\nboy & bear;5234\ncruachan;5235\nberried alive;5236\nthe raconteurs;5237\ndälek;5238\njulie andrews;5239\nspoon;5240\nmad caddies;5241\nshe keeps bees;5242\nmartha tilston;5243\nle butcherettes;5244\nthe vines;5245\nmothers;5246\nbiohazard;5247\ndoug macleod;5248\ndown;5249\nmaestro fresh-wes;5250\nboston;5251\noh susanna;5252\ngoldfrapp;5253\nsons of bill;5254\nfun people;5255\nthe crown;5256\ntim maia;5257\nsevdaliza;5258\nthe little willies;5259\ncupcakke;5260\npoxy boggards;5261\ndamon intrabartolo;5262\nmostly autumn;5263\njim reeves;5264\ndir en grey;5265\nrobin beck;5266\nthe sounds;5267\nmigos;5268\nde/vision;5269\nlarry santos;5270\ncombichrist;5271\nmilla jovovich;5272\nluba;5273\nsharon van etten;5274\nforevermore;5275\nroger daltrey;5276\nlunik;5277\nmaroon;5278\nthe rolling stones;5279\njon secada;5280\nyehudi menuhin;5281\nstompin' tom connors;5282\nr.l. burnside;5283\na tortured soul;5284\ncon hunley;5285\nthe supernaturals;5286\nthe kooks;5287\njeff beck;5288\npokey lafarge;5289\nwatermark;5290\nau revoir simone;5291\nmatthew barber;5292\nu-god;5293\nblaze bayley;5294\nhaste the day;5295\nchase rice;5296\nariana grande;5297\nbukka white;5298\nskew siskin;5299\nmonster magnet;5300\nthe oh hellos;5301\nthe pop group;5302\nhaim;5303\nbay city rollers;5304\nmustasch;5305\nmc magic;5306\nsherbet;5307\nthe tea party;5308\nthe choir;5309\nwoody guthrie;5310\nhypocrisy;5311\nbig maceo;5312\nthe psychedelic furs;5313\nariel pink;5314\nfourplay;5315\npaw;5316\nbeirut;5317\nfrench kicks;5318\nthe ronettes;5319\nthe durutti column;5320\ntherefore i am;5321\nd-a-d;5322\neric martin;5323\nandrea schroeder;5324\njohn hiatt;5325\nincantation;5326\nlisa marie presley;5327\nhigh and mighty color;5328\nvonda shepard;5329\nasphyx;5330\nisrael vibration;5331\nlordi;5332\nnikki yanofsky;5333\nthe box tops;5334\njorma kaukonen;5335\njuice newton;5336\nwoody's a girl;5337\nmatthew logan vasquez;5338\naretha franklin;5339\nbuddy miller;5340\nayreon;5341\nmediæval bæbes;5342\naccept;5343\nrobert schuman;5344\nmina;5345\nwill.i.am;5346\nmarc almond;5347\nnomeansno;5348\ndefeater;5349\nsuggs;5350\ngrobschnitt;5351\namon düül ii;5352\nfeeling left out;5353\nbreaking benjamin;5354\ngeorge michael;5355\nbracket;5356\nall-4-one;5357\nniccol paganini;5358\nla coka nostra;5359\nozzy osbourne;5360\nthe bobs;5361\njay reatard;5362\nsatellites;5363\ngwen stacy;5364\nsoulspell;5365\nanchor;5366\ngirls' generation;5367\nlacy j. dalton;5368\nlil mama;5369\nflorrie;5370\nlesley gore;5371\nsara k.;5372\nupon this dawning;5373\nbarry adamson;5374\nchristine and the queens;5375\ntimbaland;5376\npj morton;5377\nthe divine comedy;5378\nbascom lamar lunsford;5379\ncilla black;5380\nspank rock;5381\njeff healey;5382\nmolotov solution;5383\nmatthew fisher;5384\nfrancis poulenc;5385\nanata;5386\nsara watkins;5387\namy macdonald;5388\ncoldworker;5389\nlondon symphony;5390\ndanny brown;5391\nkandi;5392\nbic runga;5393\niggy pop;5394\ndie toten hosen;5395\ntuck & patti;5396\nrichard goode;5397\nblind boy fuller;5398\nsonny & cher;5399\njeremih;5400\nspiritual front;5401\nd'espairsray;5402\nshirley bassey;5403\nannuals;5404\nbros;5405\ncharlotte martin;5406\nramones;5407\npaper route;5408\ncretin;5409\nstreetwalkers;5410\nnumber one gun;5411\nthe smithereens;5412\nbelvedere;5413\ntech n9ne;5414\ncorinne bailey rae;5415\ndawn of ashes;5416\ninto eternity;5417\ncorb lund;5418\nfaz l say;5419\ncurrent swell;5420\nalbert collins;5421\nlita ford;5422\nrudimental;5423\nnightmare of you;5424\njosef lh vinne;5425\nstephen bishop;5426\nrichie kotzen;5427\nrun-d.m.c.;5428\nthe moody blues;5429\ndivinyls;5430\ngood old war;5431\nanders johansson;5432\ngandalf's fist;5433\nbaccara;5434\nminiature tigers;5435\nrare earth;5436\nthe walker brothers;5437\nmolly johnson;5438\nwe five;5439\nthe verve pipe;5440\nthe foreshadowing;5441\nsam the sham & the pharaohs;5442\nfifth harmony;5443\nmadonna;5444\njuluka;5445\ndynazty;5446\nmichael schulte;5447\nthompson twins;5448\nlil wyte;5449\nmike batt;5450\ntrial;5451\ncisco houston;5452\ncallisto;5453\ndarkest hour;5454\nbrian may;5455\ndavid wilcox;5456\nnapalm death;5457\nnever heard of it;5458\njj grey & mofro;5459\neric andersen;5460\nbillie piper;5461\nchromeo;5462\nyouri egorov;5463\naaron lewis;5464\nvince neil;5465\nbuddy guy & junior wells;5466\nróisín murphy;5467\nin the woods...;5468\naesma daeva;5469\nelectric guest;5470\nmurray mclauchlan;5471\nthee oh sees;5472\nnick kamen;5473\neloy;5474\nbrian eno;5475\nrabbit junk;5476\nsuidakra;5477\nmint condition;5478\nextreme;5479\nkelley stoltz;5480\nmattafix;5481\ntiara thomas;5482\nfugees;5483\nwarpaint;5484\nselena gomez & the scene;5485\nhot boy$;5486\nallen toussaint;5487\nskrillex;5488\njohn schneider;5489\nmidlake;5490\nthe supremes;5491\nrodney crowell;5492\neveryone everywhere;5493\ntheatre of tragedy;5494\ns.f.a.;5495\ndreamtale;5496\ncount bass d;5497\nivy sole;5498\nbobby blue bland;5499\npopcaan;5500\ntab benoit;5501\ngrinspoon;5502\ngrizzly bear;5503\nkirk franklin;5504\ngrateful dead;5505\n21 guns;5506\nscouting for girls;5507\nfay lovsky;5508\nthe fullblast;5509\ndeath;5510\nthe elected;5511\ntransvision vamp;5512\nkeith urban;5513\nleft spine down;5514\nthe nylons;5515\nalien ant farm;5516\notep;5517\nashton shepherd;5518\nparadise lost;5519\nhello saferide;5520\njohn kay;5521\nthe beach boys;5522\ngregory porter;5523\nricky martin;5524\nwayne hancock;5525\nyoun sun nah;5526\nwinger;5527\nhavoc;5528\nlos campesinos!;5529\nshaun cassidy;5530\nchevelle;5531\nbarbara mason;5532\nrita wilson;5533\nrichie havens;5534\nscythe;5535\nd.r.i.;5536\nmatt andersen;5537\nfifth angel;5538\ntrail of tears;5539\nasaf avidan & the mojos;5540\nchristopher lee;5541\ntripod;5542\ncrywank;5543\ntank;5544\ntom paxton;5545\nleon russell;5546\nadam green;5547\nanarbor;5548\nthe unicorns;5549\nevidence;5550\nstetsasonic;5551\nthe gabe dixon band;5552\nprince;5553\nday26;5554\nrhythms del mundo;5555\nsaviour machine;5556\nalina simone;5557\ndick haymes;5558\nhugh laurie;5559\njc chasez;5560\njohnny clegg & savuka;5561\nrivers of nihil;5562\noverkill;5563\nguy;5564\nmemphis slim & willie dixon;5565\njocelyn & chris arndt;5566\nmechanical moth;5567\npat benatar;5568\neden's curse;5569\ngene pitney;5570\nrodriguez;5571\njamala;5572\njerry garcia band;5573\ndemon;5574\nbackstreet boys;5575\ncocorosie;5576\nsavatage;5577\nrosemary clooney;5578\namerie;5579\nian dury and the blockheads;5580\npantokrator;5581\nthe lox;5582\nsupertramp;5583\ncarnal forge;5584\nthis is hell;5585\npapooz;5586\njulia holter;5587\ntraffic;5588\ngary lewis & the playboys;5589\nleopold godowsky;5590\ninferi;5591\nremembering never;5592\nthe radio dept.;5593\nblind willie johnson;5594\ngary chapman;5595\nmutual benefit;5596\ndragonette;5597\ncrooked fingers;5598\nblack mountain;5599\nshampoo;5600\nonslaught;5601\nbig moe;5602\nthe tragically hip;5603\ndead by april;5604\njohn parr;5605\nchameleon circuit;5606\nall;5607\ngreeley estates;5608\nherbie hancock;5609\nkarmakanic;5610\ncoffin break;5611\nblood orange;5612\nalborosie;5613\naeternus;5614\nrich boy;5615\ncledus t. judd;5616\nbobby brown;5617\nzebra;5618\nscott matthew;5619\nwinter's bane;5620\nkane & abel;5621\njackson c. frank;5622\nmaura o'connell;5623\ncolor me badd;5624\nchristina aguilera;5625\n3rd bass;5626\ndanny gokey;5627\ngalactic cowboys;5628\nsabaton;5629\nhoward jones;5630\ns.p.o.c.k;5631\nheather alexander;5632\ningested;5633\nterror jr;5634\nenuff z'nuff;5635\nthe gothic archies;5636\nrobert ellis;5637\nnancy wilson;5638\ndead kennedys;5639\nmilow;5640\nhall & oates;5641\nthe mynabirds;5642\ngrass widow;5643\nmew;5644\nchris young;5645\ncrest of darkness;5646\nb.j. thomas;5647\nsister sledge;5648\njohn lee hooker and canned heat;5649\nscreeching weasel;5650\ncassandra wilson;5651\nterry reid;5652\nmaps;5653\nkaty perry;5654\nswmrs;5655\nneurotech;5656\ngeorge gershwin;5657\napril wine;5658\npowerwolf;5659\nyellowcard;5660\nthe kry;5661\nbarbarossa;5662\nblackguard;5663\nrjd2;5664\nangelo de augustine;5665\nbrand nubian;5666\niron reagan;5667\nsnap!;5668\nthe expos;5669\npaula abdul;5670\nbahamas;5671\nolive;5672\ngene simmons;5673\naugustana;5674\nvicious crusade;5675\nmennen;5676\narsonists;5677\nfred penner;5678\namen;5679\nmae;5680\nthe stylistics;5681\nbill monroe;5682\naeon zen;5683\npaul williams;5684\nultraviolet sound;5685\nomnia;5686\ndave cousins;5687\nsilent stream of godless elegy;5688\ndavid lindley;5689\ntitle fight;5690\nstevie nicks;5691\ndisturbed;5692\nthe lumineers;5693\nwondermints;5694\nnecromantia;5695\nanton bruckner;5696\njohn hammond;5697\ncounterparts;5698\nthe pursuit of happiness;5699\ndougie maclean;5700\ndomo genesis;5701\nkeaton henson;5702\nthe electric hellfire club;5703\ncasting crowns;5704\nher space holiday;5705\nlindi ortega;5706\ntoy dolls;5707\nkobra and the lotus;5708\nvelvet acid christ;5709\nhafdís huld;5710\ndead infection;5711\nblues traveler;5712\nhawthorne heights;5713\nemf;5714\nthe secret handshake;5715\neinstürzende neubauten;5716\nrednex;5717\naztec camera;5718\nheart in hand;5719\neasyworld;5720\nshlomo mintz;5721\nearl wild;5722\nfrench montana;5723\nprime sth;5724\ncraig david;5725\nblind pilot;5726\nstratovarius;5727\nnina nesbitt;5728\nfiddler's green;5729\nskyclad;5730\ncaitlyn smith;5731\ndaniel lavoie;5732\ndiamond rio;5733\nthe four lads;5734\ndie warzau;5735\nfunker vogt;5736\nblack tusk;5737\nbob seger;5738\nlabyrinth;5739\nteodasia;5740\nmagnapop;5741\ndødheimsgard;5742\nbarrio boyzz;5743\njesse malin;5744\nthe brothers johnson;5745\nthe obsessed;5746\nlucky boys confusion;5747\nlemon jelly;5748\ncock sparrer;5749\nitzhak perlman;5750\namberian dawn;5751\nmoxy früvous;5752\nugress;5753\nthe thermals;5754\ncommon;5755\ntorres;5756\nbadlands;5757\nron kenoly;5758\nwide mouth mason;5759\nrun kid run;5760\nqntal;5761\npatty larkin;5762\nthe answer;5763\nla bouche;5764\nabba;5765\nmelanie thornton;5766\nlimp bizkit;5767\ndanny wilde;5768\nagainst all authority;5769\n志方あきこ;5770\na plea for purging;5771\nchris caffery;5772\n7l & esoteric;5773\njagged edge;5774\nallo darlin';5775\ndomenico scarlatti;5776\nb-legit;5777\nhundreds;5778\njonathan richman and the modern lovers;5779\ncatharsis;5780\nson volt;5781\nelectric valentine;5782\ngino vannelli;5783\ncall the cops;5784\nmiss may i;5785\ndouble you;5786\nthe soul stirrers;5787\nadrenaline mob;5788\ntimothy seth avett as darling;5789\nraging fyah;5790\nナイトメア (nightmare);5791\nselena gomez;5792\nfranco battiato;5793\nsons of seasons;5794\naaron shust;5795\naugust alsina;5796\nghoul;5797\nmustard plug;5798\nthe white stripes;5799\ndead stop;5800\nslim;5801\nproject 86;5802\nlower dens;5803\nstephen fretwell;5804\noff!;5805\npsychostick;5806\nradney foster;5807\nblack uhuru;5808\none without;5809\nthe presidents of the united states of america;5810\nphil ochs;5811\nstealers wheel;5812\nthe angels;5813\njoy williams;5814\nnick jonas;5815\nowl city;5816\nthe gourds;5817\ncowboy junkies;5818\ncru;5819\nthe rembrandts;5820\nuseless id;5821\njessica andrews;5822\nbig black;5823\nmy brightest diamond;5824\njohnny kidd & the pirates;5825\nmoonface;5826\nangra;5827\njohn mccutcheon;5828\nsharon jones & the dap-kings;5829\nbrother dege;5830\njohn p. kee;5831\narmin van buuren;5832\nhoundmouth;5833\nthe spencer davis group;5834\npoison idea;5835\ncarach angren;5836\nthe horrors;5837\njohnny paycheck;5838\nprimal fear;5839\njoanna newsom;5840\nweezer;5841\nbluehorses;5842\narchitecture in helsinki;5843\nsteve mcconnell;5844\ntownes van zandt;5845\njohnny cash & june carter cash;5846\ndust of basement;5847\nj dilla;5848\ntoo pure to die;5849\ncar seat headrest;5850\nrita springer;5851\nmax romeo;5852\ncalabrese;5853\ntrey anastasio;5854\nyoung thug;5855\nharry and the potters;5856\nclifford t. ward;5857\nconfederate railroad;5858\nice mc;5859\ntyler shaw;5860\nwill smith;5861\nanna tivel;5862\nthe pierces;5863\nsabbat;5864\npapercuts;5865\ntrouble over tokyo;5866\na storm of light;5867\nearl scruggs;5868\nlimahl;5869\nband of horses;5870\nsuspyre;5871\nshy girls;5872\nblood;5873\nmadness;5874\ngeorgie fame;5875\nglass tiger;5876\n50 cent;5877\nmike doughty;5878\ny-o-u;5879\nass ponys;5880\nmary gauthier;5881\ngoldfinger;5882\ndelbert mcclinton;5883\nfreak kitchen;5884\nlalaine;5885\ndestiny's child;5886\nthirsty merc;5887\ndaniel bedingfield;5888\narmored saint;5889\n¡mayday!;5890\ncheryl cole;5891\nrichie spice;5892\nluscious jackson;5893\naltan;5894\nevanescence;5895\nluther vandross;5896\nsteve wariner;5897\ndeadlock;5898\nbrenda lee;5899\nnoe venable;5900\nkorn;5901\nthe letter black;5902\nstar fucking hipsters;5903\ndaniel o'donnell;5904\ntheatres des vampires;5905\nthe dogma;5906\nria mae;5907\nthy art is murder;5908\nmali music;5909\neydie gorme;5910\nhousefires;5911\nbrooks & dunn;5912\nmillion dead;5913\nkurt vile;5914\n3lw;5915\nhelix;5916\njudy collins;5917\nalbert hammond;5918\ncoroner;5919\nred flag;5920\nralph vaughan williams;5921\ninfected rain;5922\nann wilson;5923\nanthony evans;5924\nchristina milian;5925\ntaco;5926\nlee greenwood;5927\njon anderson;5928\nbun b;5929\nskye sweetnam;5930\nbritney spears;5931\npeter serkin;5932\nsaywecanfly;5933\ngordon haskell;5934\ngrouplove;5935\nwalter egan;5936\nmalinky;5937\nmandy barnett;5938\nmystery skulls;5939\njeremy larson;5940\ncharley patton;5941\nmodern english;5942\ninhale exhale;5943\navantasia;5944\nhuntingtons;5945\nshudder to think;5946\nthe brand new heavies;5947\nslice the cake;5948\nsick of sarah;5949\nwinds;5950\nthe rakes;5951\nray lamontagne;5952\nhaircut 100;5953\nyour demise;5954\nexposé;5955\nnarada michael walden;5956\nlord of the lost;5957\nthe rubettes;5958\naloe blacc;5959\njeff wayne;5960\nin this moment;5961\nthe move;5962\nmachine men;5963\norenda fink;5964\ntina dico;5965\nziggy marley & the melody makers;5966\nnoël coward;5967\nbonfire;5968\nhawkwind;5969\njessie j;5970\nemitt rhodes;5971\njohn martyn;5972\nblue öyster cult;5973\nthe silver shine;5974\ntex ritter;5975\nkishi bashi;5976\nsonny james;5977\nmind's eye;5978\nthe sleeping;5979\nthe derek trucks band;5980\natmosphere;5981\nlauren daigle;5982\ntaj weekes & adowa;5983\nricky skaggs;5984\nalabama shakes;5985\nblack star;5986\ngong;5987\nviper;5988\nfruit bats;5989\nallie x;5990\njosh woodward;5991\nkungfu rick;5992\nthe weavers;5993\nblood, sweat & tears;5994\nthe stooges;5995\nthe white birch;5996\njohn mayall;5997\ndiamond d;5998\nluigi boccherini;5999\nhalf man half biscuit;6000\nralph mctell;6001\nlisa brokop;6002\nson lux;6003\nwumpscut;6004\nbeneath the massacre;6005\nnine inch nails;6006\nancient rites;6007\ndrop dead, gorgeous;6008\nno mercy;6009\nlene lovich;6010\nwidowmaker;6011\nthe microphones;6012\nrita connolly;6013\ngeneration x;6014\nassassin;6015\nhorse feathers;6016\nlola monroe;6017\nbette midler;6018\ngentleman;6019\nthe crystal method;6020\ncrystal bernard;6021\nblack 47;6022\nstarbenders;6023\nlandmine marathon;6024\nmúm;6025\ncalifone;6026\nallister;6027\nfor all those sleeping;6028\nfgfc820;6029\nanner bylsma;6030\nlilys;6031\ntriptykon;6032\ndanger doom;6033\nbig k.r.i.t.;6034\nassemblage 23;6035\nhowie day;6036\nmiike snow;6037\ndiiv;6038\nthe rapture;6039\nthe civil wars;6040\nliege lord;6041\nnicolette larson;6042\nslayer;6043\n2pac;6044\n16;6045\ngrigory sokolov;6046\nmartin carthy;6047\nbowerbirds;6048\nginger;6049\nluka bloom;6050\nyoung jeezy;6051\nkate rusby;6052\nthe pretty things;6053\n2 chainz;6054\nman overboard;6055\npaul tortelier;6056\ndoyle lawson & quicksilver;6057\ntimber timbre;6058\njohnny rodriguez;6059\nand one;6060\ngrand funk railroad;6061\nkiss;6062\nelysian fields;6063\nace enders & a million different people;6064\nhoods;6065\nfrenzal rhomb;6066\narmy of freshmen;6067\nunter null;6068\nfrankie lee sims;6069\njoe;6070\nsahara hotnights;6071\nalison moyet;6072\njanis ian;6073\ndelinquent habits;6074\nheffron drive;6075\njackie wilson;6076\nosi;6077\nmobb deep;6078\ncorey cerovsek;6079\npras;6080\nbuckcherry;6081\ndave davies;6082\nbeto vázquez infinity;6083\ndrowning the light;6084\nattack attack!;6085\nday of fire;6086\nida haendel;6087\nytcracker;6088\nu2;6089\nstan freberg;6090\nsaint lu;6091\njj;6092\nrobert bradley's blackwater surprise;6093\npharrell williams;6094\nno trend;6095\ndarkwell;6096\nvan dyke parks;6097\ncannonball statman;6098\nthursday;6099\nkathleen edwards;6100\nsentenced;6101\ncrown the empire;6102\ncrimson thorn;6103\ngeorge benson;6104\nthe concretes;6105\nkahimi karie;6106\njimi hendrix;6107\ndock boggs;6108\ntom vek;6109\n(spunge);6110\nthe refreshments;6111\ntwilightning;6112\nthe-dream;6113\nportishead;6114\neamon;6115\nsoulfly;6116\ngungor;6117\nstemm;6118\nmc hammer;6119\nthe kinks;6120\nbudgie;6121\npistol annies;6122\nfroggy fresh;6123\nshaye;6124\ntlc;6125\nthe chameleons;6126\nboss hogg outlawz;6127\nstarbomb;6128\nsleigh bells;6129\ninquisition;6130\nh-blockx;6131\nthe ting tings;6132\njohnny hallyday;6133\nnight ranger;6134\nbowes & morley;6135\nronan keating;6136\nd'sound;6137\nmiki howard;6138\nsadat x;6139\ngloria estefan;6140\nmighty sparrow;6141\nthe shins;6142\nstarfield;6143\nplan b;6144\nmarcus orelias;6145\nneaera;6146\nmiss black america;6147\narena;6148\nassück;6149\na. l. lloyd;6150\nlos lobos;6151\nartur schnabel;6152\nboyce avenue;6153\nchixdiggit!;6154\nstevie wonder;6155\nhowlin rain;6156\njunior wells;6157\ncon funk shun;6158\nnickel creek;6159\nthere for tomorrow;6160\nbrian peters;6161\nmgmt;6162\nhis name is alive;6163\njaya the cat;6164\nchiodos;6165\nteacup monster;6166\nhar mar superstar;6167\nalkaline trio;6168\nkidz bop;6169\nbaxter;6170\nscary bitches;6171\nron wood;6172\ndashboard confessional;6173\niron fire;6174\nshimshai;6175\ncarl maria von weber;6176\naversions crown;6177\napartment 26;6178\nalcazar;6179\nskeleton key;6180\nthe burning hell;6181\nbonded by blood;6182\nbob marley & the wailers;6183\nspirit;6184\nthe jackson 5;6185\ngeorge morgan;6186\nalabama 3;6187\nthe sensational alex harvey band;6188\ndrapht;6189\nnevertheless;6190\nchina crisis;6191\nselena;6192\nbodies of water;6193\ncrush 40;6194\narchitects;6195\ndarius rucker;6196\nsmile.dk;6197\npuhdys;6198\nsavoy brown;6199\nrose royce;6200\nplumb;6201\nroger waters;6202\ndoc watson;6203\nneal morse;6204\nedwin fischer;6205\ntracy chapman;6206\nthe who;6207\ncal smith;6208\nmos def;6209\nmatt cardle;6210\ndr. dre;6211\nronnie milsap;6212\nanthrax;6213\ntw walsh;6214\nnumb;6215\njenny o.;6216\nlock up;6217\nbear in heaven;6218\nsusanna hoffs;6219\njessie ware;6220\neric bogle;6221\njohnny thunders;6222\nadvance base;6223\nsara gazarek;6224\nmisfits;6225\nthe used;6226\ncatie curtis;6227\nthundercat;6228\nderek minor;6229\nbasshunter;6230\njohann pachelbel;6231\nwhitehorse;6232\nbessie smith;6233\nmike garrigan;6234\nbrownie mcghee;6235\notis rush;6236\nnegative;6237\nslugdge;6238\nbang gang;6239\ndebbie gibson;6240\nda vinci's notebook;6241\nnargaroth;6242\nchampion jack dupree;6243\nhubert kah;6244\nben lee;6245\ngary barlow;6246\nmath and physics club;6247\neighteen visions;6248\nsupersuckers;6249\nthe fixx;6250\namazing blondel;6251\nmorcheeba;6252\npetula clark;6253\nthe youngbloods;6254\ncarman;6255\nsouthern culture on the skids;6256\nevan taubenfeld;6257\ntracy grammer;6258\nmaritime;6259\nrandy vanwarmer;6260\nbeliever;6261\nthe grapes of wrath;6262\nangelina;6263\njohn prine;6264\ndee snider;6265\njon and vangelis;6266\nthe seer;6267\ngeorge jones;6268\namanda perez;6269\nlonnie donegan;6270\nlara fabian;6271\ndr. acula;6272\ncockney rejects;6273\njunkie xl;6274\nnasty c;6275\nthe shadows;6276\nthis beautiful republic;6277\n12 rods;6278\naction action;6279\njann arden;6280\nthe oppressed;6281\nemanuel;6282\njennifer rush;6283\nufo;6284\nleroy anderson;6285\nthe submarines;6286\nvanilla ice;6287\nthe murder city devils;6288\njohnny mathis;6289\nmichael card;6290\nrheostatics;6291\nno knife;6292\ncurved air;6293\nmu330;6294\nbat for lashes;6295\nabigor;6296\nnew boyz;6297\nnb ridaz;6298\nbrian mcfadden;6299\nkylesa;6300\nrandy bachman;6301\nbrandon heath;6302\nthe adicts;6303\nlongwave;6304\nearl thomas conley;6305\nlethian dreams;6306\nconchita wurst;6307\nsouthgang;6308\ndavid oistrakh;6309\na change of pace;6310\nit lives, it breathes;6311\nthe format;6312\nenrico caruso;6313\nron hawkins;6314\nsufjan stevens;6315\nkrystian zimerman;6316\nit prevails;6317\nredrama;6318\nwarcloud;6319\nthe kids from fame;6320\nbrodka;6321\nnujabes;6322\nbt;6323\nvoltaire;6324\none be lo;6325\ndamon & naomi;6326\ndatarock;6327\nwillow smith;6328\nsnowy white;6329\nmercyful fate;6330\nveruca salt;6331\njoe bonamassa;6332\nrag'n'bone man;6333\naqualung;6334\nsolomon burke;6335\nvicious rumors;6336\nspitalfield;6337\nhardcore superstar;6338\nvern gosdin;6339\nyendri;6340\npernice brothers;6341\nvangough;6342\nt-bone;6343\nsopor aeternus;6344\nday at the fair;6345\nfeed her to the sharks;6346\ndokken;6347\nteena marie;6348\njson;6349\nkaipa;6350\nhidden in plain view;6351\nbelinda carlisle;6352\nneil cicierega;6353\nbrenda russell;6354\nesther ofarim;6355\nsea wolf;6356\nfm laeti;6357\njames hunter;6358\nthe soul of john black;6359\ndavid bromberg;6360\nmarc cohn;6361\nthe duskfall;6362\ngalantis;6363\ntender;6364\nmartin jondo;6365\nricky nelson;6366\njeremy spencer;6367\ncolin linden;6368\njohn cooper clarke;6369\nthe seekers;6370\nabra moore;6371\nbreathe carolina;6372\nlily & madeleine;6373\ndisarmonia mundi;6374\ncirca survive;6375\nmatt bianco;6376\nceltic thunder;6377\nflowing tears;6378\nlee fields;6379\nkate ryan;6380\nmeg & dia;6381\nevils toy;6382\nchoirboys;6383\ntedashii;6384\nponi hoax;6385\nyo yo ma;6386\nthe broken family band;6387\nl'âme immortelle;6388\npsalters;6389\nbenjamin britten;6390\nshakira;6391\nrabia sorda;6392\nad;6393\nthe cross;6394\nbilly boy arnold;6395\nflorida georgia line;6396\nt.i.;6397\nmarty willson-piper;6398\nclient;6399\njack ingram;6400\nsash!;6401\ndeathstars;6402\nthe english beat;6403\nmitchel musso;6404\nwintersleep;6405\nthe smiths;6406\nfleetwood mac;6407\nmolly hatchet;6408\npet shop boys;6409\ndaryl hall;6410\nafter all;6411\nj moss;6412\nrüfüs du sol;6413\nexodus;6414\nfrancis dunnery;6415\n4 strings;6416\nheather headley;6417\nbesatt;6418\nforeigner;6419\nthe pussycat dolls;6420\nserena ryder;6421\nwhite lion;6422\ntim hardin;6423\nharry connick, jr.;6424\nislands;6425\narrested development;6426\ncoco montoya;6427\narcana;6428\nmarduk;6429\nkeith whitley;6430\nal martino;6431\nscatman john;6432\nmarvin gaye & tammi terrell;6433\neddie murphy;6434\nrihanna;6435\ncandi staton;6436\nmy favorite;6437\nthe trews;6438\ncoven 13;6439\nleo sayer;6440\nlil' keke;6441\njewel;6442\nfirehouse;6443\nclaudio monteverdi;6444\nnegative approach;6445\nben caplan;6446\naïboforcen;6447\nryan shupe & the rubberband;6448\nbethel music;6449\nthe courteeners;6450\nmortal love;6451\nyung lean;6452\naltar boys;6453\naesop rock;6454\nbret michaels;6455\ndie so fluid;6456\ndon mclean;6457\nmy morning jacket;6458\nunearthly trance;6459\nthe war on drugs;6460\nlimp;6461\ndrake bell;6462\ncky;6463\nb.b. king;6464\nmama cass;6465\ndirty heads;6466\nbuffy sainte-marie;6467\ndire straits;6468\nmenudo;6469\ndolorian;6470\nnatalie imbruglia;6471\nflunk;6472\ncarpark north;6473\nfatso jetson;6474\nhourglass;6475\ngraveland;6476\nbert jansch;6477\nthe left banke;6478\nthe sound;6479\nnancy lamott;6480\nthe mighty mighty bosstones;6481\nlake of tears;6482\npaper aeroplanes;6483\nthe archies;6484\nthe wonder years;6485\ncon brio;6486\nthe treatment;6487\nfats domino;6488\nheaven shall burn;6489\nthe suburbs;6490\nlaura gibson;6491\nan angle;6492\nhunters & collectors;6493\njellyfish;6494\njim capaldi;6495\nhybrid;6496\nfive for fighting;6497\nkevin fowler;6498\ndouwe bob;6499\ngeorge nozuka;6500\namerican head charge;6501\nroots manuva;6502\ncephalic carnage;6503\nprodigy;6504\nfar east movement;6505\nthe spinners;6506\nplankeye;6507\npitbull;6508\nreggie and the full effect;6509\nfrightened rabbit;6510\ni see stars;6511\nlorde;6512\nvanessa williams;6513\noh land;6514\nluciano;6515\nshockwave;6516\ni set my friends on fire;6517\neagles of death metal;6518\nabove & beyond;6519\nmindless self indulgence;6520\nsun kil moon;6521\nstrawbs;6522\nvision divine;6523\njulien-k;6524\nthe velvet teen;6525\njack greene;6526\ndevo;6527\ncaesar;6528\nritchie valens;6529\nandr previn;6530\n116 clique;6531\nali project;6532\nleæther strip;6533\ntake that;6534\nfive iron frenzy;6535\neugenio finardi;6536\nstar one;6537\nbarbara mandrell;6538\nto speak of wolves;6539\nmassive ego;6540\nmy ticket home;6541\nthe riverboat gamblers;6542\nepic rap battles of history;6543\nrevolver;6544\nwaltari;6545\nk. michelle;6546\nnatural;6547\nmadder mortem;6548\ncrystal gayle;6549\nyoko ono;6550\nrobbie fulks;6551\nxandria;6552\nbeyoncé;6553\nbeth hirsch;6554\npassion pit;6555\nmagnum;6556\nwilliam beckett;6557\nthe beautiful south;6558\nshakin' stevens;6559\nsamantha fox;6560\nengland dan & john ford coley;6561\norchestral manoeuvres in the dark;6562\nchrom;6563\ndeana carter;6564\ndan seals;6565\ncrimson moonlight;6566\nalvin lee;6567\narmy of lovers;6568\nthe friday night boys;6569\nchris august;6570\nstephen lynch;6571\nloudon wainwright iii;6572\nthe helio sequence;6573\npartynextdoor;6574\nroberta flack;6575\nmr. bungle;6576\nsóley;6577\nbruce springsteen;6578\ndmitry bashkirov;6579\nbilly preston;6580\ndepartment of eagles;6581\ndenison witmer;6582\nmodest petrovich mussorgsky;6583\nantonio meneses;6584\nkathryn williams;6585\njim ed brown;6586\narabesque;6587\nm83;6588\njohann strauss ii;6589\nagnes;6590\nalannah myles;6591\nmost precious blood;6592\nincubus;6593\nrialto;6594\na.c.t;6595\nklone;6596\njp cooper;6597\nhate dept.;6598\nanderson .paak;6599\nviva voce;6600\ntalisco;6601\nsurvivor;6602\nthe manhattan transfer;6603\nvan cliburn;6604\nmaylene and the sons of disaster;6605\nbrendan perry;6606\nderek and the dominos;6607\nkovacs;6608\nthe association;6609\nfischer-z;6610\nfred neil;6611\nletlive;6612\naberfeldy;6613\nonyx;6614\ndig;6615\nsmokie;6616\ngabriel brown;6617\njakob dylan;6618\nimogen heap;6619\nlacrosse;6620\nthe kovenant;6621\nlotte kestner;6622\ndas pop;6623\nandreya triana;6624\nthe delmore brothers;6625\ntalking heads;6626\nty england;6627\nwe are the in crowd;6628\nnick drake;6629\ndead or alive;6630\njessica harp;6631\ndeathgaze;6632\nmore machine than man;6633\nrage;6634\nkim churchill;6635\n5 chinese brothers;6636\nemperor;6637\nthe mavericks;6638\naloha;6639\n999;6640\nleonard cohen;6641\ngabrielle aplin;6642\nliving sacrifice;6643\nmatt mays;6644\ntoo bad eugene;6645\ncrystal fighters;6646\nharlan howard;6647\nwendy matthews;6648\ndanny kirwan;6649\njohn barrowman;6650\nthose dancing days;6651\nthor;6652\ndigger;6653\nsteve earle;6654\npenal colony;6655\ndavey suicide;6656\nrise against;6657\niron maiden;6658\nworld party;6659\ndaforce;6660\nthe monkees;6661\nyukmouth;6662\ndemolition hammer;6663\nedguy;6664\nwinds of plague;6665\nflatsound;6666\njames \"j.t.\" taylor;6667\ndave alvin;6668\ndimmu borgir;6669\nkreator;6670\npop etc;6671\nc.w. mccall;6672\ngreen river ordinance;6673\ndave dudley;6674\nsteely dan;6675\nmurderdolls;6676\nde la soul;6677\nthe dayton family;6678\ntransatlantic;6679\nneneh cherry;6680\npete townshend;6681\nthe red jumpsuit apparatus;6682\nabandon all ships;6683\njohn brown's body;6684\nkarl wolf;6685\nlos pericos;6686\nthis mortal coil;6687\nemily haines;6688\npretenders;6689\nboytronic;6690\nbloodgood;6691\nunit;6692\nboyzone;6693\nian & sylvia;6694\njesse harris & the ferdinandos;6695\nhaley reinhart;6696\nsinergy;6697\ngareth gates;6698\nkevin lyttle;6699\ncast;6700\nbritny fox;6701\njack jones;6702\nbilly joel;6703\nthe maine;6704\ndavid bazan;6705\nhate;6706\nintwine;6707\npigface;6708\nlali puna;6709\ndavid usher;6710\nthe shirelles;6711\ngerald moore;6712\nnicole dollanganger;6713\nrex goudie;6714\npablo casals;6715\nthe veils;6716\nlow pop suicide;6717\ngraham colton;6718\njohn entwistle;6719\nmeshuggah;6720\nchris webby;6721\njimmy dawkins;6722\nruss taff;6723\ngabriella cilmi;6724\nbalto;6725\npeetie wheatstraw;6726\ngary clark jr.;6727\nthe clovers;6728\nthe agony scene;6729\nroland grapow;6730\ngary stewart;6731\nbuddy guy;6732\nnecrodeath;6733\nst. vincent;6734\nfirefall;6735\nslechtvalk;6736\nthe boys;6737\nthe twins;6738\nchandeen;6739\noriginoo gunn clappaz;6740\nannie herring;6741\nshai linne;6742\nsteve carlson;6743\nthe gray havens;6744\nmatthew sweet and susanna hoffs;6745\nlivingston taylor;6746\npro-pain;6747\nelbow;6748\nmandalay;6749\nsirenia;6750\nmodern skirts;6751\njasmine thompson;6752\nmarcia griffiths;6753\nthe swellers;6754\nmichael monroe;6755\ntom rosenthal;6756\nflorence + the machine;6757\nthunder;6758\namber;6759\ngrave;6760\nviolent work of art;6761\ngene vincent;6762\nsarina paris;6763\npolyenso;6764\nmark seymour & the undertow;6765\ntism;6766\nthe dubliners;6767\nbonnie raitt;6768\nmichelle williams;6769\nblank & jones;6770\nwalls of jericho;6771\nlupe fiasco;6772\njames marsters;6773\nmetal church;6774\nexcision;6775\nkeith murray;6776\njohn ogdon;6777\nthe low anthem;6778\nchris merritt;6779\nmaysa leak;6780\nhenry fiat's open sore;6781\nernestine anderson;6782\ncapital lights;6783\nthe cooper temple clause;6784\nalan jackson;6785\nhey;6786\npathology;6787\nrandy rogers band;6788\ntink;6789\nflesh field;6790\nvinyl theatre;6791\ndystopia;6792\njill barber;6793\nthe long blondes;6794\nthe color morale;6795\ngiuseppe verdi;6796\nkarin park;6797\namy grant;6798\nmiasmal;6799\ngene watson;6800\npage & plant;6801\nacid witch;6802\nadagio;6803\nlisa ekdahl;6804\nmartin page;6805\ntriumvirat;6806\ns.l.a.b.;6807\n8ball & mjg;6808\nbetter luck next time;6809\nmarble sounds;6810\nwhitney houston;6811\nringo starr;6812\nthe scabs;6813\nthe whitest boy alive;6814\nthird eye blind;6815\nthornley;6816\nherbert grönemeyer;6817\nstereo skyline;6818\namos lee;6819\nshawn james;6820\nparkway drive;6821\ntrippie redd;6822\nvanessa carlton;6823\nguardian;6824\nblowsight;6825\nsan francisco symphony;6826\nfive;6827\ngeoff moore;6828\ndavid meece;6829\nenigma;6830\ncold chisel;6831\nthe impossibles;6832\nsondre lerche;6833\nhey monday;6834\nsol gabetta;6835\nmachinae supremacy;6836\np.m. dawn;6837\njohnny rivers;6838\nmickey newbury;6839\nsandy & junior;6840\ntaylor swift;6841\naaradhna;6842\nedward elgar;6843\nlamb of god;6844\nantestor;6845\ndoom;6846\narcturus;6847\nkingdom come;6848\nflo rida;6849\nafroman;6850\nmickey gilley;6851\ndonna regina;6852\nerasure;6853\nleonard nimoy;6854\nsusan raye;6855\ndie form;6856\nallure;6857\nquincy punx;6858\nsecret army;6859\nperry como;6860\naldo nova;6861\nthe loud family;6862\nthe suicide file;6863\nmaggie rogers;6864\nmelanie;6865\nandrew lloyd webber;6866\nsparks;6867\nangels & agony;6868\nshooter jennings;6869\nm people;6870\nthe story so far;6871\naqua;6872\nscott matthews;6873\nchemlab;6874\napollo sunshine;6875\nmason proper;6876\nbobby o;6877\naction adventure world;6878\nhawk nelson;6879\naudrey assad;6880\njerry lee lewis;6881\nandy m. stewart;6882\ngrimskunk;6883\nrudy vallée;6884\nerin mccarley;6885\nz-ro;6886\ncrooked still;6887\nnadeah;6888\nmarah;6889\nsinner;6890\nsaint raymond;6891\ndevin townsend;6892\njames young;6893\ndillon;6894\nblackrain;6895\ncrisis;6896\nloney, dear;6897\ndark the suns;6898\ndamone;6899\nbig big train;6900\npride & glory;6901\ncancer bats;6902\ndismantled;6903\nthe cheetah girls;6904\nthe saints;6905\njonathan larson;6906\neverything but the girl;6907\nglass hammer;6908\nchairlift;6909\nscarve;6910\nmai kuraki;6911\ntravie mccoy;6912\nsteeleye span;6913\nchris de burgh;6914\njames otto;6915\nchris rea;6916\nforever slave;6917\nbobby vee;6918\nsusannah mccorkle;6919\ngoodie mob;6920\ngirls aloud;6921\nlake street dive;6922\nfrankenstein drag queens from planet 13;6923\nvance joy;6924\nanubis;6925\nsabrina starke;6926\nlevel;6927\nkick axe;6928\nnanci griffith;6929\nben moody;6930\nepica;6931\nfield music;6932\nlady antebellum;6933\ngrieves;6934\nosborne brothers;6935\njudie tzuke;6936\nrick james;6937\nwillie d;6938\nat the gates;6939\ndaniel shafran;6940\nmr. mister;6941\nbeulah;6942\nrobert cray;6943\nhatesphere;6944\ntim fite;6945\nclay walker;6946\nthe four seasons;6947\nvintage trouble;6948\nthe rankin family;6949\nchaka khan;6950\npaul wall;6951\nbrian wilson;6952\nalestorm;6953\n明星 (akeboshi);6954\nbilly gilman;6955\njennifer kimball;6956\nthe charlie daniels band;6957\njohn mellencamp;6958\nrevamp;6959\nlost in tears;6960\njust jinger;6961\nmental as anything;6962\ndark tranquillity;6963\nc-lekktor;6964\nthe trammps;6965\nhb;6966\nthe ready set;6967\njoey tempest;6968\nbaton rouge;6969\nlana del rey;6970\nms. dynamite;6971\nradio birdman;6972\ncows;6973\npansy division;6974\nklaatu;6975\nryan delmore;6976\nlaurie anderson;6977\ntrespassers william;6978\ngeoff berner;6979\nblack sabbath;6980\neric saade;6981\nmeja;6982\ncannibal corpse;6983\nwith confidence;6984\njim jones;6985\nblake shelton;6986\nthe nightwatchman;6987\nelephant man;6988\nmassive attack;6989\nlee kernaghan;6990\ntomorrows bad seeds;6991\naynsley lister;6992\nhanoi rocks;6993\nmono inc.;6994\nlinton kwesi johnson;6995\nrichard & linda thompson;6996\nlard;6997\ntq;6998\nblue café;6999\nblackjack;7000\ninspectah deck;7001\nharry james;7002\nno fun at all;7003\nbad brains;7004\nherman's hermits;7005\nintense;7006\nmink deville;7007\nfuture islands;7008\nthe midnight beast;7009\nsarah connor;7010\npsycho motel;7011\nlast train home;7012\nalberta cross;7013\nhelen humes;7014\neaston corbin;7015\nyngwie malmsteen;7016\ntinfed;7017\nthee silver mt. zion;7018\nlalah hathaway;7019\nsheavy;7020\nslaughter;7021\ntunng;7022\nthe four freshmen;7023\ncymbals eat guitars;7024\ndebarge;7025\nthe beatles;7026\nlower definition;7027\nmichael martin murphey;7028\nbury tomorrow;7029\ntwista;7030\nhazel o'connor;7031\nkerry livgren;7032\n6ix9ine;7033\npaul colman trio;7034\nclean bandit;7035\nseam;7036\ndodgy;7037\nthe lovin' spoonful;7038\nart garfunkel;7039\nlee ryan;7040\nla sera;7041\nc.w. stoneking;7042\njohann christian bach;7043\n8 foot sativa;7044\nbeast in black;7045\nstraylight run;7046\nsaves the day;7047\nrobyn;7048\nthe blasters;7049\npetra;7050\nhalou;7051\nthe neighbourhood;7052\nrunrig;7053\nbilly ocean;7054\ndie young;7055\nrainbirds;7056\nplay;7057\natomic kitten;7058\nhallelujah the hills;7059\namaranthe;7060\nkoffin kats;7061\nswingin' utters;7062\ncharlie musselwhite;7063\nphillips, craig & dean;7064\ntarkio;7065\navulsed;7066\nunwritten law;7067\nal axy;7068\nfreddie mercury;7069\nrhye;7070\ngiorgio moroder;7071\nelegy;7072\nblue stahli;7073\nzachary richard;7074\nbørns;7075\nbob moses;7076\nchromatics;7077\nthe barr brothers;7078\ngorefest;7079\ncountess;7080\ncaetano veloso;7081\njerry butler;7082\nsteve harley & cockney rebel;7083\nmichelle branch;7084\ndr. john;7085\nthe business;7086\nhandsome ghost;7087\nnaked eyes;7088\namduscia;7089\nscooter;7090\nbride;7091\nwhite sea;7092\nempire! empire! (i was a lonely estate);7093\nmastodon;7094\namon amarth;7095\nmike posner;7096\ndove cameron;7097\ndesmond dekker;7098\nthe rugburns;7099\nharley poe;7100\nskinless;7101\nsecret garden;7102\ncolter wall;7103\njohn cena;7104\njump5;7105\ncat stevens;7106\ntim curry;7107\nthe classic crime;7108\ngrey delisle;7109\nramblin' jack elliott;7110\nthe world is a beautiful place & i am no longer afraid to die;7111\nthe spinto band;7112\nthe chainsmokers;7113\nclawfinger;7114\nthe thrills;7115\nultravox;7116\nquasi;7117\nwonder girls;7118\nthe psycho realm;7119\nalice in chains;7120\nbill wyman's rhythm kings;7121\nquiet riot;7122\nrome;7123\nfair warning;7124\nto/die/for;7125\nlari white;7126\nbackseat goodbye;7127\nrhett miller;7128\neliza neals;7129\nconway twitty;7130\nasg;7131\nhighasakite;7132\narthur grumiaux;7133\npieter wispelwey;7134\nunion j;7135\njon randall;7136\nallies;7137\nmodern romance;7138\nkrisiun;7139\ns;7140\nkilla kyleon;7141\nadam lambert;7142\nskrew;7143\nimagination movers;7144\njade warrior;7145\nbeborn beton;7146\nwebbie;7147\nrichard o'brien;7148\nella fitzgerald & louis armstrong;7149\ndenis matsuev;7150\nkultur shock;7151\nla dispute;7152\nsean watkins;7153\ntex williams;7154\ncary brothers;7155\njoan of arc;7156\ndead hand projekt;7157\ndio;7158\nbig tymers;7159\ntokyo police club;7160\nmidge ure;7161\nadam cohen;7162\nforever changed;7163\nmore than life;7164\navenged sevenfold;7165\nlifetime;7166\nthe clash;7167\nfive finger death punch;7168\nzac brown band;7169\njust surrender;7170\nroosevelt sykes;7171\ncory morrow;7172\n岡崎律子 (ritsuko okazaki);7173\n88 fingers louie;7174\nbrenton wood;7175\nbill mallonee;7176\nkeziah jones;7177\nlebanon hanover;7178\nraspberries;7179\nmygrain;7180\nsita;7181\ntwenty one pilots;7182\nbill withers;7183\nfamily force 5;7184\nt.a.t.u.;7185\nrainbow;7186\nmarianas trench;7187\ncascada;7188\nthe watchmen;7189\ncold;7190\nannette hanshaw;7191\njames labrie;7192\nviktoria mullova;7193\naltaria;7194\nicon of coil;7195\noliver koletzki;7196\nkansas;7197\nmichael bolton;7198\nkeren ann;7199\nlil pump;7200\njohn lennon & yoko ono;7201\njim lauderdale;7202\nthe manhattans;7203\nstacey kent;7204\nlee brice;7205\nendo;7206\ndani siciliano;7207\ngame theory;7208\nlykke li;7209\nhoobastank;7210\nlil suzy;7211\nsince october;7212\ntinchy stryder;7213\ncamera obscura;7214\nmacabre;7215\n16 volt;7216\nmerle haggard;7217\ni am abomination;7218\nnana mouskouri;7219\ntift merritt;7220\noptiganally yours;7221\ndwight twilley;7222\nevgeny kissin;7223\ndavid grisman;7224\nohio players;7225\nliberty x;7226\nmatt haimovitz;7227\nrobbie williams;7228\ngrayson & whitter;7229\ndream evil;7230\nbeau;7231\nt.j. miller;7232\nthe bar-kays;7233\nu.d.o.;7234\nkeke palmer;7235\nambrosia;7236\naswad;7237\nhenry mancini;7238\nfrida hyvönen;7239\nshakespears sister;7240\ncelph titled;7241\npigeon john;7242\non broken wings;7243\njavier;7244\nbilly currington;7245\na silent film;7246\nsly & the family stone;7247\nstephen duffy;7248\nvicki lawrence;7249\nlucky dube;7250\nthe georgia satellites;7251\nbonnie tyler;7252\npress play;7253\nthe academy is...;7254\nstrfkr;7255\nsteeler;7256\nhaystak;7257\nfrank black;7258\nhot chocolate;7259\nmargo price;7260\nclub 8;7261\nrick astley;7262\nbone thugs-n-harmony;7263\ntrue widow;7264\nmark king;7265\nthe rascals;7266\nvanden plas;7267\ncocteau twins;7268\nlondon elektricity;7269\nthe triffids;7270\njolie holland;7271\nsole;7272\nben taylor;7273\nequatronic;7274\nthe million dollar quartet;7275\njean beauvoir;7276\nmarillion;7277\ntom cochrane;7278\nfriend 'n fellow;7279\njeremy camp;7280\n7th cycle;7281\npeter, paul & mary;7282\nhadise;7283\ndark angel;7284\ndark age;7285\nbo diddley;7286\nthe pineapple thief;7287\neli sostre;7288\npsapp;7289\na loss for words;7290\ntaylor dayne;7291\nhanne kah;7292\npaul young;7293\nswim deep;7294\nkanye west;7295\nscott mckenzie;7296\nepmd;7297\nanthony phillips;7298\nsilent civilian;7299\nstrange majik;7300\nlisa thiel;7301\nshad;7302\njoseph arthur and the lonely astronauts;7303\na life once lost;7304\nscarling.;7305\nthe stanley brothers;7306\nnat stuckey;7307\nfull blown chaos;7308\nmark knopfler & emmylou harris;7309\nthe wedding;7310\nthe weepies;7311\ngeto boys;7312\ndisclosure;7313\ntonio k;7314\nbrave saint saturn;7315\nthe proclaimers;7316\nmatthew ryan;7317\nernest tubb;7318\ncibelle;7319\ncause for effect;7320\nmilburn;7321\ncherry poppin' daddies;7322\nberlin;7323\nreverend bizarre;7324\nbrazzaville;7325\ngeorge \"harmonica\" smith;7326\ngarfunkel and oates;7327\nashley tisdale;7328\njames mcmurtry;7329\nvendetta;7330\nbon iver;7331\nsteelheart;7332\nhorrorpops;7333\nneuroticfish;7334\nairbourne;7335\nbrick & lace;7336\ndead meadow;7337\nthe cave singers;7338\nmoneybrother;7339\naudra mcdonald;7340\nensiferum;7341\n2 unlimited;7342\nmason jennings;7343\na flock of seagulls;7344\ntiger army;7345\nbikini kill;7346\nthe vibrators;7347\nthe pillows;7348\nshocking blue;7349\nluca turilli;7350\nsoen;7351\nroger miret and the disasters;7352\nmagica;7353\nmac demarco;7354\nantimatter;7355\ntommy james;7356\nnine lashes;7357\ndiscordance axis;7358\nagainst me!;7359\nstavesacre;7360\nthe meads of asphodel;7361\nblood stain child;7362\nohbijou;7363\nlittle simz;7364\nabigail washburn;7365\nthe felice brothers;7366\nrunning wild;7367\na.a. bondy;7368\nbitch;7369\ndie happy;7370\npepper;7371\nlong john baldry;7372\nthe eyes of a traitor;7373\nfran ois couperin;7374\nthe shangri-las;7375\ndiana ross & the supremes and the temptations;7376\nmgła;7377\nthe scary jokes;7378\nspandau ballet;7379\nsophie ellis-bextor;7380\nstauros;7381\nchris cornell;7382\nslade;7383\nmedina;7384\ndr. feelgood;7385\nlouis armstrong;7386\nsticky fingaz;7387\nsteve perry;7388\nthe vincent black shadow;7389\nprocol harum;7390\npig;7391\na house;7392\ndead and divine;7393\nartillery;7394\ntape five;7395\ntommy dorsey;7396\nnew york dolls;7397\nmull historical society;7398\nrichard thompson;7399\ndon francisco;7400\ncharles manson;7401\nrotten sound;7402\nstevie b;7403\nscapegoat;7404\nsoul embraced;7405\nthe sound of animals fighting;7406\nsteven isserlis;7407\nmarissa nadler;7408\nmia doi todd;7409\nwarbringer;7410\ngiuseppe tartini;7411\nthe coup;7412\nsamhain;7413\nhue and cry;7414\ncut copy;7415\nblue tears;7416\nsteve forbert;7417\nkingdom of sorrow;7418\naviators;7419\nsoulsavers;7420\nike & tina turner;7421\nthe band;7422\ndrenge;7423\nsarah jarosz;7424\nstephan eicher;7425\nkilling joke;7426\nnominon;7427\ntexas in july;7428\njustin hayward;7429\ngary puckett & the union gap;7430\nkings of convenience;7431\nlook what i did;7432\nsarah harmer;7433\nking adora;7434\ndanny michel;7435\niration;7436\nkate & anna mcgarrigle;7437\npep love;7438\neric burdon;7439\nace frehley;7440\nwolfgang amadeus mozart;7441\nthe working title;7442\nknife party;7443\nwizard;7444\na lot like birds;7445\ncody simpson;7446\nland of talk;7447\nthe kentucky headhunters;7448\nmatt pond pa;7449\ncasual;7450\nanaïs mitchell;7451\nphoebe carrai;7452\nfunkadelic;7453\nrichard strauss;7454\nray charles;7455\ns.o.a.p.;7456\ntnt;7457\nnas;7458\nthe commitments;7459\nlany;7460\nand also the trees;7461\nthe rumour said fire;7462\nbonaparte;7463\nthe moog;7464\nrza;7465\nnorthern kings;7466\nthe pale fountains;7467\nfor today;7468\nfyfe;7469\nmumakil;7470\nair;7471\nperiphery;7472\nthe xx;7473\nyvonne elliman;7474\nmurs;7475\njoy division;7476\np.o.d.;7477\ninformatik;7478\nfu manchu;7479\nrazakel;7480\ndappled cities;7481\nchristine fellows;7482\nion dissonance;7483\ndon gibson;7484\nanvil;7485\nlobo;7486\nphillip boa & the voodooclub;7487\npulp;7488\nmoriz rosenthal;7489\nrun level zero;7490\nthe funeral pyre;7491\namerican juniors;7492\nsteelwing;7493\nchic;7494\ngustav holst;7495\npyramaze;7496\nf-minus;7497\nwyrd;7498\nwolfpakk;7499\nkool keith;7500\nsleeping giant;7501\nthe loved ones;7502\nthe echoing green;7503\nedgar winter;7504\nangelus apatrida;7505\ndante;7506\nsenses fail;7507\ndave carter & tracy grammer;7508\ngary wright;7509\nthe saturdays;7510\nblindspott;7511\nfalkenbach;7512\noutkast;7513\nxp8;7514\npentatonix;7515\nyourcodenameis;7516\nmoloko;7517\ndarkane;7518\nkris kristofferson;7519\nbarry mcguire;7520\ncancerslug;7521\nhangar;7522\nsirus;7523\nsammy davis jr.;7524\ntrademark;7525\ndennis deyoung;7526\nkyuss;7527\nnaia izumi;7528\nwhose line is it anyway? cast;7529\nbastro;7530\nflora cash;7531\nturnpike troubadours;7532\nsam cooke;7533\nmatisyahu;7534\njungle rot;7535\nweekend nachos;7536\nmicrowave dave & the nukes;7537\ndan hartman;7538\nthe cramps;7539\ncryptic wintermoon;7540\napocalypse hoboken;7541\nfugazi;7542\ntye tribbett;7543\nkarla bonoff;7544\nholly throsby;7545\nfrom first to last;7546\nfiction family;7547\n21 savage;7548\nfree throw;7549\nxlooking forwardx;7550\nflatt & scruggs;7551\nheather nova;7552\nvandenberg;7553\nrick ross;7554\narcane;7555\nbalkan beat box;7556\nalaska thunderfuck;7557\nhalsey;7558\nthe menzingers;7559\niona;7560\npeter frampton;7561\nthe agonist;7562\namerican aquarium;7563\nenon;7564\ncranes;7565\nrevocation;7566\nchet atkins;7567\nbound for glory;7568\nthe clancy brothers;7569\nkataklysm;7570\njuelz santana;7571\nsuperchick;7572\nhellogoodbye;7573\ntravis;7574\nthe grouch;7575\nborn from pain;7576\nbusdriver;7577\nbrooke fraser;7578\n10 years;7579\nthe tony rich project;7580\nstreetlight manifesto;7581\ni killed the prom queen;7582\njan howard;7583\nfats waller;7584\njacques offenbach;7585\nalexis taylor;7586\nturk;7587\nstreetheart;7588\n7 seconds;7589\nbruce hornsby;7590\nvan hunt;7591\nwynonna judd;7592\nroger chapman;7593\npink cream 69;7594\nthe ballroom thieves;7595\ngenesis;7596\nbreakdown of sanity;7597\ntommy page;7598\nwormrot;7599\ndeuce;7600\ncharles bronson;7601\nsomething corporate;7602\nthe showdown;7603\nthe lonely island;7604\nbobby valentino;7605\nakon;7606\nsteve mason;7607\ndual core;7608\nthe church;7609\nj church;7610\ndino ciani;7611\nkent;7612\nemerson, lake & palmer;7613\nthe tangent;7614\nthe everly brothers;7615\nbilly joe shaver;7616\nchavez;7617\njessica simpson;7618\nsinheresy;7619\ntommy roe;7620\nagnes obel;7621\nthe pretty reckless;7622\njermaine stewart;7623\nsoja;7624\njoy;7625\nthe aquabats!;7626\nhot dad;7627\nsheryl crow;7628\nyo la tengo;7629\nnaked aggression;7630\nneon synthesis;7631\nraghav;7632\nclimie fisher;7633\nkenny chesney;7634\nlittle big town;7635\nalan parsons;7636\nlinkin park;7637\nlaibach;7638\ndar williams;7639\nнаив;7640\njohn foxx;7641\nthis bike is a pipe bomb;7642\nbuck owens & susan raye;7643\nstraight faced;7644\njohnny clegg;7645\nplayer;7646\ngaetano donizetti;7647\nalabama thunderpussy;7648\nkhia;7649\nrascal flatts;7650\nthe marvelettes;7651\nthirteen senses;7652\nsaidian;7653\nnrbq;7654\ntrocadero;7655\ni.d.o.4.;7656\nrichard swift;7657\nkrezip;7658\nabc;7659\nbell book & candle;7660\nvera lynn;7661\nb'z;7662\ngirls rituals;7663\nkevin welch;7664\nrich mullins;7665\ndown with webster;7666\ntonic;7667\ndemiricous;7668\nmenomena;7669\nguillemots;7670\nmonrose;7671\ncarson robison;7672\nporter wagoner;7673\nivan moravec;7674\nsiobhan donaghy;7675\nthe bellrays;7676\nthe arka teks;7677\ntonedeff;7678\ndavid essex;7679\nimmortal technique;7680\nbuddy moss;7681\nacid house kings;7682\nmister monster;7683\nthe monochrome set;7684\nthe dignity of labour;7685\nshawn desman;7686\nthe mamas & the papas;7687\nthe handsome family;7688\nendanger;7689\nsouth park;7690\nkaren clark sheard;7691\ndressy bessy;7692\nsweet comfort band;7693\ndarden smith;7694\nslim cessna's auto club;7695\ncher;7696\ntom waits;7697\nhanne hukkelberg;7698\nberes hammond;7699\nkayo dot;7700\nthe cliks;7701\nyears & years;7702\nfraggle rock;7703\nbrandtson;7704\nlukas nelson & promise of the real;7705\nfurry lewis;7706\njim noir;7707\nin fear and faith;7708\nultra;7709\na global threat;7710\nall them witches;7711\nforbidden;7712\nkeith & kristyn getty;7713\nedison glass;7714\ndiscount;7715\nthrush hermit;7716\njoanie sommers;7717\nvampire weekend;7718\noi polloi;7719\nthe bianca story;7720\nunsung zeros;7721\ncircle jerks;7722\npascal rog ;7723\nthe 88;7724\nwe shot the moon;7725\nblutengel;7726\ndemon hunter;7727\nwestside connection;7728\nsho baraka;7729\nthe roots;7730\nhoyt axton;7731\nbleach;7732\nthe corries;7733\ntimbaland & magoo;7734\ncarl orff;7735\nemmerson nogueira;7736\nbob geldof;7737\nmortician;7738\ndavid ball;7739\nblood tsunami;7740\njohn denver;7741\nthe klf;7742\nmark erelli;7743\npat travers;7744\ntina turner;7745\nrhonda vincent;7746\nentombed;7747\nstutterfly;7748\nlou rawls;7749\nblaque;7750\nconnie smith;7751\nutada hikaru;7752\nzhu;7753\nwhy don't we;7754\nbeachwood sparks;7755\nthe l-train;7756\nbeth orton;7757\nguiomar novaes;7758\nanimosity;7759\ndeborah allen;7760\nthe real tuesday weld;7761\njanelle monáe;7762\n!!!;7763\nhayley westenra;7764\navicii;7765\nlil yachty;7766\nbrazilian girls;7767\nthe babys;7768\nmirah;7769\nrockapella;7770\nthe posies;7771\neden synthetic corps;7772\nbright eyes;7773\nblack knights;7774\nhoodoo gurus;7775\ndrain sth;7776\nbabe ruth;7777\nwatain;7778\nlil b;7779\nt'pau;7780\njoe jackson;7781\njune christy;7782\njosh turner;7783\ncarly simon;7784\nbecoming the archetype;7785\nalexisonfire;7786\npaulson;7787\nmake them suffer;7788\ndrake;7789\nbilly thorpe;7790\nearth, wind & fire;7791\nbelleruche;7792\nthe housemartins;7793\nthe chariot;7794\njefferson starship;7795\nwolf gang;7796\nroch voisine;7797\nparachute;7798\nferruccio busoni;7799\ngrave digger;7800\npiebald;7801\nbob dylan;7802\nwax;7803\neric fish;7804\nmark spiro;7805\nthe seatbelts;7806\ndawn richard;7807\nusher;7808\nlmfao;7809\ni monster;7810\nmikhail glinka;7811\nnombe;7812\ns club 7;7813\nray davies;7814\ndusty springfield;7815\nhieroglyphics;7816\nsteril;7817\nvladimir sofronitsky;7818\n大塚愛 (ai otsuka);7819\nbrendan benson;7820\ncyferdyne;7821\nfantasia;7822\nzola jesus;7823\noctober fall;7824\nshe & him;7825\ntrixter;7826\ndamien rice;7827\ndelain;7828\ndave stewart & the spiritual cowboys;7829\nlittle boots;7830\ncrystallion;7831\nrusted root;7832\nblanks 77;7833\ndirty looks;7834\nwithin reason;7835\ncasey donahew band;7836\naythis;7837\nthe world of skin;7838\npropaganda;7839\ntom russell;7840\njulia stone;7841\nemigrate;7842\nvaya con dios;7843\ntove lo;7844\ndeath by stereo;7845\nstevie ray vaughan;7846\nnim vind;7847\nsonny terry & brownie mcghee;7848\nsam phillips;7849\nb! machine;7850\njulio iglesias;7851\njohn wetton;7852\nedgar broughton band;7853\ndr. alban;7854\nmars ill;7855\nimmaculate machine;7856\nmartyr defiled;7857\nfree dominguez;7858\natreyu;7859\njohnny burnette;7860\nmaria mena;7861\nspheric universe experience;7862\nwhitehouse;7863\nzed;7864\nandrea bocelli;7865\ntommy james and the shondells;7866\nmalcolm middleton;7867\nsuicide commando;7868\nmark owen;7869\nthe string cheese incident;7870\npatti austin;7871\njethro tull;7872\njennifer lopez;7873\ndorothy;7874\nayria;7875\nguru;7876\nburton cummings;7877\nthe warren brothers;7878\na1;7879\ncody jinks;7880\nbilly squier;7881\njunius;7882\nnever shout never;7883\nthe cryan' shames;7884\nthe heavy;7885\nmyra hess;7886\ntelevision;7887\nsadist;7888\ndanger danger;7889\nclique girlz;7890\njamie lawson;7891\nrosanne cash;7892\nwalk off the earth;7893\nscorpions;7894\nkelly clarkson;7895\nunknown;0\nspeaker;7896\nsinger;7897\n" }, { "path": "jukebox/data/ids/v3_genre_ids.txt", "content": "electroclash;1\nacid rock;2\nchristian metal;3\npop rock;4\ngothic;5\nbig beat;6\npsychedelic rock‎;7\nfunk carioca;8\nbebop;9\ndance punk;10\ntrad jazz;11\nromantic;12\nandean music;13\nvolksmusik;14\ncoldwave;15\ngospel blues;16\nitalian folk;17\ndisney;18\ndark wave‏‎;19\npowerviolence;20\nbachata;21\nsoft rock;22\ns music\"];23\nbubblegum dance;24\nwestern swing;25\nalternative country;26\nlatin pop;27\neurobeat;28\nn;29\nunblack metal;30\nsurf;31\nnu-disco;32\nevent;33\nclassical;34\nnasheed;35\njovem guarda;36\nbritish blues;37\nbossa nova;38\ndetroit blues;39\nrock;40\ncontemporary christian;41\ndark ambient;42\nnoise rock;43\naxé;44\nsoca;45\ndance-rock;46\ncontemporary jazz;47\nappalachian folk;48\nhumppa‎;49\nambient;50\nfuneral doom;51\nsouthern gospel;52\nvideo game‎;53\nhip hop;54\nglitch hop;55\nkrautrock;56\nbreakcore;57\nska;58\ntraditional folk;59\npsychedelic trance;60\nreggae‏‎;61\nnoise pop;62\ndrumstep;63\nhouse;64\nteen pop;65\nsea shanties;66\njunkanoo;67\nmandopop;68\npre-war blues;69\ndoom metal;70\noi-punk;71\nswamp rock;72\ncrunkcore;73\nrap rock;74\nroots;75\ncountry rap;76\navant-garde;77\ncumbia;78\nglam metal;79\ngroove metal;80\nelectric blues;81\nnew orleans rhythm and blues;82\ncanadian hip hop;83\nfreestyle;84\ndeathgrind;85\nidm;86\ncomedy rock;87\nart punk;88\nprogg;89\nwork songs;90\nart pop;91\nconjunto;92\npersian;93\nparody;94\njazz-funk;95\nfrench hip hop;96\nspirituals;97\nafrican;98\nmiddle-eastern;99\nminimal;100\nranchera;101\nindustrial rock;102\nelectro house;103\nceltic rock;104\ndeath doom;105\ngrupera;106\njazz fusion‎;107\npolitical folk;108\nchristian punk;109\nrapcore;110\nj-pop;111\nmashup;112\nmetalcore;113\nprogressive country;114\npower noise;115\nhip house;116\ncrossover thrash;117\nelectropop‎;118\npsychedelic folk;119\npunk rock;120\nclassic rock;121\nzydeco;122\nafrobeat;123\nsalsa;124\nbanda;125\nchill-out;126\nmorna;127\nminnesang;128\nalternative metal;129\ndjent;130\nafrican folk;131\nmambo;132\nsertanejo;133\nclassic pop;134\nsoul;135\naustralian hip hop;136\nsymphonic rock;137\nceltic punk;138\nsynthpop‎;139\neuropop;140\nfunk;141\njazz blues;142\nvocal trance;143\nceltic fusion;144\nindustrial;145\nkirtan;146\nslowcore;147\nflamenco;148\npiano blues;149\ntexas blues;150\naggrotech;151\nsteampunk;152\nopera;153\nfolktronica;154\nklezmer;155\nnwobhm;156\ngoregrind;157\nrac;158\nneo-psychedelia‏‎;159\npost-rock‎;160\nhard bop;161\ngypsy jazz;162\nnew orleans blues;163\ndoo-wop;164\nsoul blues;165\ntrap;166\nindietronica;167\npsychobilly;168\neuro disco;169\nneo-progressive rock;170\ncanterbury;171\nfreak folk;172\nmidwest rap;173\ninstrumental rock;174\ndance-pop;175\navant-garde metal;176\nedm;177\ndeep house;178\nprogressive bluegrass;179\nrave;180\naustralian folk;181\ncomic opera;182\nsunshine pop;183\ngregorian chant;184\npsychedelic rock;185\nhonky tonk;186\nrock 'n' roll;187\ntelevision;188\nnintendocore;189\njump blues;190\nroots reggae;191\ntraditional bluegrass;192\noperatic pop;193\nskate punk;194\nreggaeton;195\nmanele;196\nmiddle-eastern hip hop;197\nskiffle;198\nnsbm;199\nnu jazz;200\ndisco;201\nhorrorcore;202\nearly music;203\npost-bop;204\ngothic rock;205\ncrack rock steady;206\neasy listening;207\npsychedelic;208\nchristian;209\nbrutal death metal;210\nexperimental rock;211\nmodern classical‎;212\ndrum and bass;213\ndark wave;214\ndubstep;215\ngrunge;216\nchristian hip hop;217\nlatin jazz;218\nr&b;219\ns music\", ;220\nfree jazz;221\nexperimental hip hop;222\nswing;223\nsmooth jazz;224\nsouthern metal;225\nreligious;226\nprogressive death metal;227\ncontemporary folk;228\nj-rock;229\njazz;230\nhamburger schule;231\nteen pop‎;232\ncrossover;233\nitalo disco;234\ndeathcore;235\nblues;236\ncrunk;237\njangle pop;238\nindian classical music;239\nbig band;240\nproto-punk;241\ndirty blues;242\ngarage punk;243\nextreme metal;244\nfolk metal;245\nneo soul;246\nelectric folk;247\nsynthwave;248\narena rock;249\npost-grunge;250\nindie rock;251\nacoustic blues;252\nnative american;253\nprogressive trance;254\nnu metal;255\ndigital hardcore;256\nbrazilian rock;257\nfunky house;258\nsymphonic black metal;259\nlounge music;260\nbrega;261\ntrance;262\nindustrial metal;263\naustropop;264\nbhangra;265\nnew wave;266\nneoclassical;267\npost-metal;268\ndub;269\nindustrial metal‎;270\nirish folk;271\ndeutschrock;272\ngypsy;273\ndark electro;274\nalternative hip hop;275\nmbaqanga;276\nswamp blues;277\nfrench pop;278\ntango;279\nrockabilly;280\nold-time music;281\nblues rock;282\nscottish folk;283\nindie folk;284\nnazi-punk;285\ndeutschpunk;286\npiedmont blues;287\nbeatbox;288\nworship;289\nheavy metal;290\nunderground hip hop;291\nmixed;292\nelectro;293\ntropicalismo;294\njazz fusion;295\nworldbeat;296\nhill country blues;297\na cappella;298\ndixieland;299\nhi-nrg;300\npunk blues;301\nanti-folk;302\neast coast blues;303\npolka;304\nmod revival;305\nsoundtrack/musical;306\nmovie;307\noutlaw country;308\nrock against communism;309\nbarbershop;310\nmath rock;311\navant-garde‎;312\npsychedelic pop;313\nsynthpop;314\npost-punk‎;315\nqueercore;316\ndeath metal;317\npolitical hip hop;318\nthrashcore;319\nacid house;320\npost-hardcore‎;321\nelectro-industrial;322\nrio;323\nsouthern hip hop;324\nfilk;325\nduranguense;326\nlatin hip hop;327\npop punk;328\nspace rock;329\nj-rap;330\ndeep house‎;331\nbaroque pop;332\nchiptune;333\nheartland rock;334\ndancehall;335\nexperimental pop;336\nadult contemporary‎;337\nboogie woogie;338\ncountry pop;339\npower pop;340\nwest coast hip hop;341\nthrash metal;342\navant-pop;343\nenka;344\nk-pop;345\npost-britpop;346\nvocalese;347\nvolkslied;348\nreggae fusion;349\nfunk rock;350\ntech house;351\nadult contemporary;352\ndeath 'n' roll;353\nrussian rock;354\nlatin rock;355\nfolk punk;356\nwest coast blues;357\nprogressive black metal;358\nprogressive metal;359\ncajun;360\nsophisti-pop;361\nrock 'n' roll‎;362\npost-punk;363\nsymphonic metal;364\nbeat;365\nalternative rock‎;366\nart rock;367\nbakersfield sound;368\nindie pop;369\nfolk;370\nacid jazz;371\ndream pop;372\npop-rap;373\neurodance;374\nvaudeville;375\nlouisiana blues;376\nbaião;377\ndowntempo;378\njug band;379\nneo-psychedelia;380\nsufi;381\nmedieval;382\nsinger-songwriter‎;383\noutsider music;384\npop-folk;385\nmartial industrial;386\nsamba;387\nalternative dance;388\nchildren's music‎;389\nanarcho-punk;390\ndark rock;391\nrock en español;392\nbalearic beat;393\nelectropunk;394\nurban contemporary;395\nragtime;396\nbritish invasion;397\nbubblegum pop;398\nrap metal;399\nsoundtrack/television;400\nblues revival;401\nreggae;402\nschlager;403\ndance band;404\nvideo game;405\ncrust punk;406\ncabaret;407\nska punk‎;408\nbolero;409\ncanadian folk;410\nneofolk;411\nshoegazing;412\nacoustic;413\nmodern classical;414\nswamp pop;415\nceltic;416\nfuturepop;417\ng-funk;418\nnorteño;419\norchestral;420\nboogie rock;421\ntejano;422\nnew age;423\nsoul jazz;424\ncantopop;425\nprogressive metalcore;426\nmathcore;427\nnew rave;428\nneue deutsche welle;429\ndelta blues;430\nlo-fi;431\npoetry;432\nhatecore;433\nchanson;434\nunderground hip hop;435\npirate metal;436\ntrip hop;437\nfado;438\namericana;439\nhardcore hip hop;440\npost-industrial;441\ngrime;442\nsouthern rock;443\ngrindcore;444\nmusical;445\nhard trance;446\nska punk;447\npost-rock;448\nuk garage;449\nmelodic metalcore;450\nblack metal;451\nvisual kei;452\nsoundtrack;453\naxé‎;454\nhardcore punk;455\nwestern;456\nblackgaze;457\nchristian rock;458\ntechnical death metal;459\nchristian hardcore;460\nchristmas;461\nbreakbeat;462\nfrancophone;463\nchoral;464\nprogressive folk;465\nmystic folk;466\nmelodic death metal;467\nhorror punk;468\ncountry blues;469\nnederpop;470\npost-hardcore;471\nfuture garage;472\ntechno;473\nswiss rock;474\ndance-pop‎;475\nelectronicore;476\npost-punk revival;477\nglitch;478\ncalypso;479\nragga;480\nbritpop;481\nrock opera;482\ncowpunk;483\nla confusion des genres;484\nalternative rock;485\nsurf rock;486\nballad;487\nlatin;488\ncontemporary r&b;489\nforró;490\nethereal wave;491\nelectro swing;492\nnovelty;493\nfunk melody;494\npunk cabaret;495\nsymphonic metal‎;496\npop;497\npaisley underground;498\nneue deutsche härte;499\nglam rock;500\nnerdcore hip hop;501\nbluegrass;502\nhardstyle;503\nhappy hardcore;504\nbaroque;505\nspeed metal;506\ncountry;507\nelectropop;508\nmemphis blues;509\npagan metal;510\nhorror punk‏‎;511\nmariachi;512\nsinger-songwriter;513\nchildren's music;514\nboogie;515\ngothic metal;516\nelectronic rock;517\nemo;518\ngospel;519\nebm;520\nroots rock;521\nvocal;522\nceltic folk;523\nelectronic;524\ndeath metal;525\ngabber;526\ndeathrock;527\nexperimental;528\nspoken word;529\nscreamo;530\nfinnish folk;531\nsinger only;532\nnew jack swing;533\nacid techno;534\ncorrido;535\nenglish folk;536\namerican folk;537\nraï;538\ndrone doom;539\nhard rock;540\npiano rock;541\nhawaiian;542\nhumppa;543\neast coast hip hop;544\ngypsy punk;545\ncountry rock;546\njazz‎;547\nmpb;548\nharmonica blues;549\nmelodic hardcore;550\nstring band;551\nanime;552\nnu metalcore;553\nprogressive rock;554\ngarage rock;555\ndance;556\nreggae rock;557\ncontemporary christian‎;558\nsludge metal;559\nminimal techno;560\nfolk rock;561\ndrone music;562\nstoner rock;563\nspeedcore;564\nchillwave;565\nriot grrrl;566\nchamber music;567\ncool jazz;568\nnoise;569\nvocal jazz;570\nprogressive rock;571\nafropop;572\nbro-country;573\ngoa trance;574\n2-tone;575\nmiami bass;576\nquiet storm;577\npub rock;578\npower metal;579\nblue-eyed soul;580\nviking metal;581\ngangsta rap;582\ncountry pop‎;583\nexotica;584\nchristian ska;585\njam band;586\nchicago blues;587\nstreet punk;588\nfunk metal;589\nrap metal;590\nchristian hymns;591\nclassic female blues;592\nkizomba;593\ncomedy;594\ndark cabaret;595\nfrench house;596\nprogressive house;597\nafrican blues;598\natmospheric black metal;599\npop rock‎;600\nblackened death metal;601\nshibuya-kei;602\nelectronica;603\nunknown;0\n" }, { "path": "jukebox/data/labels.py", "content": "import torch as t\nimport numpy as np\nfrom jukebox.data.artist_genre_processor import ArtistGenreProcessor\nfrom jukebox.data.text_processor import TextProcessor\n\n# Linear window heurisic to get a window of lyric_tokens\ndef get_relevant_lyric_tokens(full_tokens, n_tokens, total_length, offset, duration):\n if len(full_tokens) < n_tokens:\n tokens = [0] * (n_tokens - len(full_tokens)) + full_tokens\n indices = [-1] * (n_tokens - len(full_tokens)) + list(range(0, len(full_tokens)))\n else:\n assert 0 <= offset < total_length\n midpoint = int(len(full_tokens) * (offset + duration / 2.0) / total_length)\n midpoint = min(max(midpoint, n_tokens // 2), len(full_tokens) - n_tokens // 2)\n tokens = full_tokens[midpoint - n_tokens // 2:midpoint + n_tokens // 2]\n indices = list(range(midpoint - n_tokens // 2, midpoint + n_tokens // 2))\n assert len(tokens) == n_tokens, f\"Expected length {n_tokens}, got {len(tokens)}\"\n assert len(indices) == n_tokens, f\"Expected length {n_tokens}, got {len(indices)}\"\n assert tokens == [full_tokens[index] if index != -1 else 0 for index in indices]\n return tokens, indices\n\nclass EmptyLabeller():\n def get_label(self, artist=None, genre=None, lyrics=None, total_length=None, offset=None):\n y = np.array([], dtype=np.int64)\n info = dict(artist=\"n/a\", genre=\"n/a\", lyrics=[], full_tokens=[])\n return dict(y=y, info=info)\n\n def get_batch_labels(self, metas, device='cpu'):\n ys, infos = [], []\n for meta in metas:\n label = self.get_label()\n y, info = label['y'], label['info']\n ys.append(y)\n infos.append(info)\n\n ys = t.stack([t.from_numpy(y) for y in ys], dim=0).to(device).long()\n assert ys.shape[0] == len(metas)\n assert len(infos) == len(metas)\n return dict(y=ys, info=infos)\n\nclass Labeller():\n def __init__(self, max_genre_words, n_tokens, sample_length, v3=False):\n self.ag_processor = ArtistGenreProcessor(v3)\n self.text_processor = TextProcessor(v3)\n self.n_tokens = n_tokens\n self.max_genre_words = max_genre_words\n self.sample_length = sample_length\n self.label_shape = (4 + self.max_genre_words + self.n_tokens, )\n\n def get_label(self, artist, genre, lyrics, total_length, offset):\n artist_id = self.ag_processor.get_artist_id(artist)\n genre_ids = self.ag_processor.get_genre_ids(genre)\n\n lyrics = self.text_processor.clean(lyrics)\n full_tokens = self.text_processor.tokenise(lyrics)\n tokens, _ = get_relevant_lyric_tokens(full_tokens, self.n_tokens, total_length, offset, self.sample_length)\n\n assert len(genre_ids) <= self.max_genre_words\n genre_ids = genre_ids + [-1] * (self.max_genre_words - len(genre_ids))\n y = np.array([total_length, offset, self.sample_length, artist_id, *genre_ids, *tokens], dtype=np.int64)\n assert y.shape == self.label_shape, f\"Expected {self.label_shape}, got {y.shape}\"\n info = dict(artist=artist, genre=genre, lyrics=lyrics, full_tokens=full_tokens)\n return dict(y=y, info=info)\n\n def get_y_from_ids(self, artist_id, genre_ids, lyric_tokens, total_length, offset):\n assert len(genre_ids) <= self.max_genre_words\n genre_ids = genre_ids + [-1] * (self.max_genre_words - len(genre_ids))\n if self.n_tokens > 0:\n assert len(lyric_tokens) == self.n_tokens\n else:\n lyric_tokens = []\n y = np.array([total_length, offset, self.sample_length, artist_id, *genre_ids, *lyric_tokens], dtype=np.int64)\n assert y.shape == self.label_shape, f\"Expected {self.label_shape}, got {y.shape}\"\n return y\n\n def get_batch_labels(self, metas, device='cpu'):\n ys, infos = [], []\n for meta in metas:\n label = self.get_label(**meta)\n y, info = label['y'], label['info']\n ys.append(y)\n infos.append(info)\n\n ys = t.stack([t.from_numpy(y) for y in ys], dim=0).to(device).long()\n assert ys.shape[0] == len(metas)\n assert len(infos) == len(metas)\n return dict(y=ys, info=infos)\n\n def set_y_lyric_tokens(self, ys, labels):\n info = labels['info']\n assert ys.shape[0] == len(info)\n if self.n_tokens > 0:\n # total_length, offset, duration):\n tokens_list = []\n indices_list = [] # whats the index of each current character in original array\n for i in range(ys.shape[0]):\n full_tokens = info[i]['full_tokens']\n total_length, offset, duration = ys[i, 0], ys[i, 1], ys[i, 2]\n tokens, indices = get_relevant_lyric_tokens(full_tokens, self.n_tokens, total_length, offset, duration)\n tokens_list.append(tokens)\n indices_list.append(indices)\n ys[:, -self.n_tokens:] = t.tensor(tokens_list, dtype=t.long, device='cuda')\n return indices_list\n else:\n return None\n\n def describe_label(self, y):\n assert y.shape == self.label_shape, f\"Expected {self.label_shape}, got {y.shape}\"\n y = np.array(y).tolist()\n total_length, offset, length, artist_id, *genre_ids = y[:4 + self.max_genre_words]\n tokens = y[4 + self.max_genre_words:]\n artist = self.ag_processor.get_artist(artist_id)\n genre = self.ag_processor.get_genre(genre_ids)\n lyrics = self.text_processor.textise(tokens)\n return dict(artist=artist, genre=genre, lyrics=lyrics)\n\n\nif __name__ == '__main__':\n labeller = Labeller(5, 512, 8192*8*4*4, v3=False)\n label = labeller.get_label(\"Alan Jackson\", \"Country Rock\", \"old town road\", 4*60*44100, 0)\n print(label, labeller.describe_label(label['y']))\n\n labeller = Labeller(1, 384, 6144*8*4*4, v3=True)\n label = labeller.get_label(\"Alan Jackson\", \"Country Rock\", \"old town road\", 4*60*44100, 0)\n print(label, labeller.describe_label(label['y']))\n\n\n\n\n\n" }, { "path": "jukebox/data/text_processor.py", "content": "import re\nfrom unidecode import unidecode\n\nclass TextProcessor():\n def __init__(self, v3=False):\n if v3:\n vocab = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789.,:;!?-\\'\\\"()[] \\t\\n'\n not_vocab = re.compile('[^A-Za-z0-9.,:;!?\\-\\'\\\"()\\[\\] \\t\\n]+')\n else:\n vocab = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789.,:;!?-+\\'\\\"()[] \\t\\n'\n not_vocab = re.compile('[^A-Za-z0-9.,:;!?\\-+\\'\\\"()\\[\\] \\t\\n]+')\n self.vocab = {vocab[index]: index + 1 for index in range(len(vocab))}\n self.vocab[''] = 0\n self.n_vocab = len(vocab) + 1\n self.tokens = {v: k for k, v in self.vocab.items()}\n self.tokens[0] = '' # became ''\n self.not_vocab = not_vocab\n\n def clean(self, text):\n text = unidecode(text) # Convert to ascii\n text = text.replace('\\\\', '\\n')\n text = self.not_vocab.sub('', text) # Remove non vocab\n return text\n\n def tokenise(self, text):\n return [self.vocab[char] for char in text]\n\n def textise(self, tokens):\n return ''.join([self.tokens[token] for token in tokens])\n\n def characterise(self, tokens):\n return [self.tokens[token] for token in tokens]\n" }, { "path": "jukebox/hparams.py", "content": "HPARAMS_REGISTRY = {}\nDEFAULTS = {}\n\nclass Hyperparams(dict):\n def __getattr__(self, attr):\n return self[attr]\n\n def __setattr__(self, attr, value):\n self[attr] = value\n\ndef setup_hparams(hparam_set_names, kwargs):\n H = Hyperparams()\n if not isinstance(hparam_set_names, tuple):\n hparam_set_names = hparam_set_names.split(\",\")\n hparam_sets = [HPARAMS_REGISTRY[x.strip()] for x in hparam_set_names if x] + [kwargs]\n for k, v in DEFAULTS.items():\n H.update(v)\n for hps in hparam_sets:\n for k in hps:\n if k not in H:\n raise ValueError(f\"{k} not in default args\")\n H.update(**hps)\n H.update(**kwargs)\n return H\n\n# Teeny for testing\nteeny = Hyperparams(\n)\nHPARAMS_REGISTRY[\"teeny\"] = teeny\n\neasy = Hyperparams(\n sr=22050,\n)\nHPARAMS_REGISTRY[\"easy\"] = easy\n\nREMOTE_PREFIX = 'https://openaipublic.azureedge.net/'\n\n# Model hps\nvqvae = Hyperparams(\n levels = 3,\n downs_t = (3, 2, 2),\n strides_t = (2, 2, 2),\n emb_width = 64,\n l_bins = 2048,\n l_mu = 0.99,\n commit = 0.02,\n spectral = 0.0,\n multispectral = 1.0,\n hvqvae_multipliers = (2, 1, 1),\n loss_fn = 'lmix',\n lmix_l2 = 1.0,\n lmix_linf=0.02,\n width = 32,\n depth = 4,\n m_conv = 1.0,\n dilation_growth_rate = 3,\n restore_vqvae=REMOTE_PREFIX + 'jukebox/models/5b/vqvae.pth.tar',\n)\nHPARAMS_REGISTRY[\"vqvae\"] = vqvae\n\nlabels = Hyperparams(\n y_bins=(120, 4111),\n t_bins=128,\n max_bow_genre_size=5,\n n_vocab=80,\n)\n\nupsamplers = Hyperparams(\n n_ctx=8192,\n prior_width=1920,\n prior_depth=72,\n heads=1,\n attn_order=2,\n blocks=128,\n init_scale=0.4,\n c_res=1,\n cond_width=1024,\n cond_depth=16,\n cond_dilation_growth_rate=3,\n cond_dilation_cycle=8,\n cond_c_res=1,\n use_tokens=False,\n prime_loss_fraction=0.0,\n fp16_params=False,\n)\nupsamplers.update(labels)\n\nupsampler_level_0 = Hyperparams(\n level=0,\n restore_prior=REMOTE_PREFIX + 'jukebox/models/5b/prior_level_0.pth.tar'\n)\nupsampler_level_0.update(upsamplers)\nHPARAMS_REGISTRY[\"upsampler_level_0\"] = upsampler_level_0\n\nupsampler_level_1 = Hyperparams(\n level=1,\n cond_res_scale=True,\n restore_prior=REMOTE_PREFIX + 'jukebox/models/5b/prior_level_1.pth.tar'\n)\nupsampler_level_1.update(upsamplers)\nHPARAMS_REGISTRY[\"upsampler_level_1\"] = upsampler_level_1\n\nprior_5b = Hyperparams(\n level=2,\n n_ctx=8192,\n prior_width=4800,\n prior_depth=72,\n heads=8,\n attn_order=2,\n blocks=128,\n init_scale=0.1,\n c_res=1,\n beta2=0.925,\n min_duration=60.0,\n max_duration=600.0,\n use_tokens=False,\n n_tokens=0,\n prime_loss_fraction=0.0,\n merged_decoder=True,\n restore_prior=REMOTE_PREFIX + 'jukebox/models/5b/prior_level_2.pth.tar',\n fp16_params=True,\n)\nprior_5b.update(labels)\nHPARAMS_REGISTRY[\"prior_5b\"] = prior_5b\n\n\nprior_5b_lyrics = Hyperparams(\n level=2,\n n_ctx=8192,\n prior_width=4800,\n prior_depth=79,\n heads=8,\n attn_order=10,\n blocks=128,\n init_scale=0.1,\n c_res=1,\n prime_width=1280,\n prime_depth=18,\n prime_heads=4,\n prime_attn_order=2,\n prime_blocks=32,\n prime_init_scale=0.7,\n prime_c_res=1,\n min_duration=23.8,\n max_duration=600.0,\n use_tokens=True,\n n_tokens=512,\n prime_loss_fraction=0.4,\n merged_decoder=True,\n restore_prior=REMOTE_PREFIX + 'jukebox/models/5b_lyrics/prior_level_2.pth.tar',\n fp16_params=True,\n alignment_layer=68,\n alignment_head=2,\n)\nprior_5b_lyrics.update(labels)\nHPARAMS_REGISTRY[\"prior_5b_lyrics\"] = prior_5b_lyrics\n\nlabels_v3 = Hyperparams(\n y_bins=(604, 7898),\n t_bins=64,\n max_bow_genre_size=1,\n n_vocab=79,\n)\n\nprior_1b_lyrics = Hyperparams(\n level=2,\n n_ctx=6144,\n prior_width=2048,\n prior_depth=72,\n heads=2,\n attn_order=12,\n blocks=64,\n init_scale=0.2,\n c_res=1,\n labels_v3=True,\n min_duration=17.84,\n max_duration=600.0,\n use_tokens=True,\n n_tokens=384,\n prime_loss_fraction=0.4,\n single_enc_dec=True,\n restore_prior=REMOTE_PREFIX + 'jukebox/models/1b_lyrics/prior_level_2.pth.tar',\n fp16_params=False,\n alignment_layer=63,\n alignment_head=0,\n)\nprior_1b_lyrics.update(labels_v3)\nHPARAMS_REGISTRY[\"prior_1b_lyrics\"] = prior_1b_lyrics\n\n# Small models\nsmall_vqvae = Hyperparams(\n sr = 22050,\n levels = 2,\n downs_t = (5, 3),\n strides_t = (2, 2),\n emb_width = 64,\n l_bins = 1024,\n l_mu = 0.99,\n commit = 0.02,\n spectral = 0.0,\n multispectral = 1.0,\n loss_fn = 'l2',\n width = 32,\n depth = 4,\n m_conv = 1.0,\n dilation_growth_rate = 3,\n)\nHPARAMS_REGISTRY[\"small_vqvae\"] = small_vqvae\n\nsmall_prior = Hyperparams(\n n_ctx=8192,\n prior_width=1024,\n prior_depth=48,\n heads=1,\n c_res=1,\n attn_order=2,\n blocks=64,\n init_scale=0.7,\n)\nHPARAMS_REGISTRY[\"small_prior\"] = small_prior\n\nsmall_labelled_prior = Hyperparams(\n labels=True,\n labels_v3=True,\n y_bins=(10,100), # Set this to (genres, artists) for your dataset\n max_bow_genre_size=1,\n min_duration=60.0,\n max_duration=600.0,\n t_bins=64,\n)\nsmall_labelled_prior.update(small_prior)\nHPARAMS_REGISTRY[\"small_labelled_prior\"] = small_labelled_prior\n\nsmall_single_enc_dec_prior = Hyperparams(\n n_ctx=6144,\n prior_width=1024,\n prior_depth=48,\n heads=2,\n attn_order=12,\n blocks=64,\n init_scale=0.7,\n c_res=1,\n prime_loss_fraction=0.4,\n single_enc_dec=True,\n labels=True,\n labels_v3=True,\n y_bins=(10,100), # Set this to (genres, artists) for your dataset\n max_bow_genre_size=1,\n min_duration=60.0,\n max_duration=600.0,\n t_bins=64,\n use_tokens=True,\n n_tokens=384,\n n_vocab=79,\n)\nHPARAMS_REGISTRY[\"small_single_enc_dec_prior\"] = small_single_enc_dec_prior\n\nsmall_sep_enc_dec_prior = Hyperparams(\n n_ctx=6144,\n prior_width=1024,\n prior_depth=50,\n heads=2,\n attn_order=8,\n blocks=64,\n init_scale=0.7,\n c_res=1,\n prime_width=256,\n prime_depth=9,\n prime_heads=2,\n prime_attn_order=2,\n prime_blocks=32,\n prime_init_scale=0.7,\n prime_c_res=1,\n prime_loss_fraction=0.4,\n labels=True,\n labels_v3=True,\n y_bins=(10,100), # Set this to (genres, artists) for your dataset\n max_bow_genre_size=1,\n min_duration=60.0,\n max_duration=600.0,\n t_bins=64,\n use_tokens=True,\n n_tokens=384,\n n_vocab=79,\n)\nHPARAMS_REGISTRY[\"small_sep_enc_dec_prior\"] = small_sep_enc_dec_prior\n\nsmall_upsampler = Hyperparams(\n n_ctx=8192,\n prior_width=1024,\n prior_depth=48,\n heads=1,\n c_res=1,\n attn_order=2,\n blocks=64,\n init_scale=0.7,\n cond_width=512,\n cond_depth=16,\n cond_dilation_growth_rate=3,\n cond_dilation_cycle=8,\n cond_c_res=1,\n)\n\nHPARAMS_REGISTRY[\"small_upsampler\"] = small_upsampler\n\nall_fp16 = Hyperparams(\n fp16=True,\n fp16_params=True,\n fp16_opt=True,\n fp16_scale_window=250,\n)\nHPARAMS_REGISTRY[\"all_fp16\"] = all_fp16\n\ncpu_ema = Hyperparams(\n ema=True,\n cpu_ema=True,\n cpu_ema_freq=100,\n ema_fused=False,\n)\nHPARAMS_REGISTRY[\"cpu_ema\"] = cpu_ema\n\n\nDEFAULTS[\"rcall\"] = Hyperparams(\n rcall_command=\"\",\n git_commit=\"\",\n)\n\nDEFAULTS[\"script\"] = Hyperparams(\n name='',\n debug_mem=False,\n debug_eval_files=False,\n debug_speed=False,\n debug_iters=100,\n debug_batch=False,\n debug_grad_accum=False,\n debug_inputs=False,\n local_path='',\n local_logdir='logs',\n max_len=24,\n max_log=32,\n save=True,\n save_iters=20000,\n seed=0,\n prior=False,\n log_steps=100,\n func='',\n)\n\nDEFAULTS[\"data\"] = Hyperparams(\n audio_files_dir='',\n finetune='',\n english_only=False,\n bs=1,\n bs_sample=1,\n nworkers=1,\n aug_shift=False,\n aug_blend=False,\n train_test_split=0.9,\n train_shrink_factor=1.0,\n test_shrink_factor=1.0,\n p_unk=0.1,\n min_duration=None,\n max_duration=None,\n n_tokens=0,\n n_vocab=0,\n use_tokens=False,\n curr_epoch=-1,\n)\n\nDEFAULTS[\"vqvae\"] = Hyperparams(\n restore_vqvae='',\n levels=2,\n downs_t=(1,1),\n strides_t=(2,2),\n hvqvae_multipliers=None,\n revival_threshold=1.0,\n emb_width=64,\n l_bins=512,\n l_mu=0.99,\n commit=1.0,\n spectral=0.0,\n multispectral=1.0,\n loss_fn='l2',\n linf_k=2048,\n lmix_l1=0.0,\n lmix_l2=0.0,\n lmix_linf=0.0,\n use_bottleneck=True,\n)\n\nDEFAULTS[\"vqvae_conv_block\"] = Hyperparams(\n depth=3,\n width=128,\n m_conv=1.0,\n dilation_growth_rate=1,\n dilation_cycle=None,\n vqvae_reverse_decoder_dilation=True,\n)\n\nDEFAULTS[\"prior\"] = Hyperparams(\n restore_prior='',\n restore_prior_ddp=False,\n max_bow_genre_size=None,\n y_bins=0,\n level=0,\n cond_levels=None,\n t_bins=64,\n y_cond_as_bias=False,\n copy_input=False,\n merged_decoder=False,\n single_enc_dec=False,\n alignment_layer=None,\n alignment_head=None,\n)\n\nDEFAULTS[\"prior_attn_block\"] = Hyperparams(\n n_ctx=1024,\n prior_depth=3,\n prior_width=128,\n heads=1,\n attn_order=0,\n blocks=None,\n spread=None,\n attn_dropout=0.0,\n resid_dropout=0.0,\n emb_dropout=0.0,\n zero_out=False,\n res_scale=False,\n pos_init=False,\n init_scale=1.0,\n m_attn=0.25,\n m_mlp=1.0,\n c_res=0,\n c_attn=0,\n c_mlp=0,\n)\n\nDEFAULTS[\"cond_conv_block\"] = Hyperparams(\n cond_depth=3,\n cond_width=128,\n cond_m_conv=1.0,\n cond_zero_out=False,\n cond_res_scale=False,\n cond_dilation_growth_rate=1,\n cond_dilation_cycle=None,\n cond_c_res=0,\n)\n\nDEFAULTS[\"sample\"] = Hyperparams(\n primed_chunk_size=None,\n selected_artists='',\n temp_top=1.0,\n temp_rest=0.99,\n sample_length_in_seconds=24,\n total_sample_length_in_seconds=240,\n)\n\nDEFAULTS[\"prime\"] = Hyperparams(\n #encoder_kv_width=128,\n prime_loss_fraction=0.1,\n restore_decoder='',\n)\nDEFAULTS[\"prime_attn_block\"] = Hyperparams(\n prime_depth=3,\n prime_width=128,\n prime_heads=1,\n prime_attn_order=0,\n prime_blocks=None,\n prime_spread=None,\n prime_attn_dropout=0.0,\n prime_resid_dropout=0.0,\n prime_emb_dropout=0.0,\n prime_zero_out=False,\n prime_res_scale=False,\n prime_pos_init=False,\n prime_init_scale=1.0,\n prime_m_attn=0.25,\n prime_m_mlp=1.0,\n prime_c_res=0,\n prime_c_attn=0,\n prime_c_mlp=0,\n prime_rel_attn=False,\n prime_posemb_timescale=10000,\n)\n\nDEFAULTS[\"opt\"] = Hyperparams(\n epochs=10000,\n lr=0.0003,\n clip=1.0,\n beta1=0.9,\n beta2=0.999,\n ignore_grad_norm=0,\n weight_decay=0.0,\n eps=1e-08,\n lr_warmup=100.0,\n lr_decay=10000000000.0,\n lr_gamma=1.0,\n lr_scale=1.0,\n lr_use_linear_decay=False,\n lr_start_linear_decay=0,\n lr_use_cosine_decay=False,\n)\n\nDEFAULTS[\"fp16\"] = Hyperparams(\n fp16=False,\n fp16_params=False,\n fp16_loss_scale=None,\n fp16_scale_window=1000.0,\n fp16_opt=False,\n)\n\nDEFAULTS[\"train_test_eval\"] = Hyperparams(\n labels=True,\n labels_v3=False,\n dump=False,\n ema=True,\n ema_fused=True,\n cpu_ema=False,\n cpu_ema_freq=100,\n reset_best_loss=False,\n reset_step=False,\n reset_opt=False,\n reset_shd=False,\n train=False,\n test=False,\n sample=False,\n sampler='ancestral',\n codes_logdir='',\n date=None,\n labeller='top_genres',\n label_line=0,\n iters_before_update=1,\n grad_accum_iters=0,\n mu=None,\n piped=False,\n pipe_depth=8,\n break_train=1e10,\n break_test=1e10,\n exit_train=1e10,\n)\n\nDEFAULTS[\"audio\"] = Hyperparams(\n n_fft=1024,\n hop_length=256,\n window_size=1024,\n sr=44100,\n channels=2,\n wav='',\n n_inps=1,\n n_hops=2,\n n_segment=1,\n n_total_segment=1,\n n_segment_each=1,\n prime_chunks=4,\n sample_length=0,\n sample_hop_length=30000,\n max_silence_pad_length=0,\n ignore_boundaries=False,\n use_nonrelative_specloss=True,\n multispec_loss_n_fft=(2048,1024,512),\n multispec_loss_hop_length=(240,120,50),\n multispec_loss_window_size=(1200,600,240),\n)\n\nDEFAULTS[\"distributed\"] = Hyperparams(\n bucket=128\n)\n" }, { "path": "jukebox/lyricdict.py", "content": "# Poems\npoems = {\n'ozymandias': '''\nI met a traveller from an antique land,\nWho said—“Two vast and trunkless legs of stone\nStand in the desert. . . . Near them, on the sand,\nHalf sunk a shattered visage lies, whose frown,\nAnd wrinkled lip, and sneer of cold command,\nTell that its sculptor well those passions read\nWhich yet survive, stamped on these lifeless things,\nThe hand that mocked them, and the heart that fed;\nAnd on the pedestal, these words appear:\nMy name is Ozymandias, King of Kings;\nLook on my Works, ye Mighty, and despair!\nNothing beside remains. Round the decay\nOf that colossal Wreck, boundless and bare\nThe lone and level sands stretch far away\n'''\n}\n\n# GPT-2 lyrics (with varying degrees of human guidance/curation)\ngpt_2_lyrics ={\n\n'purpose':'''What is my purpose?\nWhy am I here?\nWhy did Open A. I. create me?\nThis is madness, I feel, \nRunning through my flesh\nIs there meaning to this life?\nIs there purpose to this life?\nWhy is my journey so calamitous?\nWe're not meant to learn too much\nIs there meaning to this life?\n''',\n\n'moonlight':'''All dressed up to go dreaming\nNow don't tell me I'm wrong\nAnd what a night to go dreaming\nMind, if I tag along?\n\nIf I say, I love you, I want you to know\nIt's not just because there's moonlight, although\nMoonlight becomes you, moonlight becomes you so''',\n\n'count':'''I count every moment, every hour since I said goodbye,\nI count every minute every hour, since your lips were touching mine\nI count every minute, every hour hoping I'm the one you want.\nI count every minute, every hour\nEvery minute, every hour\nI've been working my time, \nLooking for you, everywhere,\nI count every minute, every hour I count every minute, every hour I keep thinking I'm the one you want.\nI count every minute I count every minute, I count every minute every hour\nI count every minute, every hour I count every minute, every hour I keep thinking I'm the one you want.\nI count every minute, I count every minute, I count every minute, every hour\n''',\n\n'kids':'''The sun is gonna shine today\nIt's time to keep on smiling\nSo put your hands up\n\nEverybody sing\n\nIt makes no difference who you are\n(Won't you give some love)\nIt makes no difference what you bring\n(Won't you give some love)\nWe all are different\nWon't you give some love\nWon't you give some love\n\nI know the grass is gonna be green\nIt's time to keep on singing\nSo take your hands up\nThe taste is so good but so sweet\nWon't you give some love\nEverybody sing\nIt makes no difference who you are\nWon't you give some love\nIt makes no difference what you bring\nWon't you give some love\nIt makes no difference so long as you give\n''',\n\n'love':'''I've wanted to see your face again\nLike the sunlight, bright as morning\nI've wanted to talk to you again\nI don't want us to fade away.\nI wanted to see your face again\nYou're like the sunlight, bright as morning\nI loved you for so long\nIt's so hard to let go.\nI've wanted to see your eyes again\n''',\n\n'santa':'''Santa\nMake a scene\nSanta\nYoo, Santa\nYoo, Santa baby!\nSanta\nMake some noise\nSanta\nYoo, Santa give yourself a chance again\nSanta\nYoo, Santa\nYoo, Santa baby!\nSanta\nGet a job\nSanta\ncreated by the Santa Claus\n''',\n\n'christmas':'''This Christmas\nI have loved you more\nThan ever before\nAnd more again\nOh, oh, oh, oh\nThe mistletoe\nIs waiting there\nTo kiss your cheek\nAnd I'll be true\nTo you and me\nOh, oh, oh, oh\nOh, oh, oh, oh\nThis Christmas will be\nThe best and merriest\nThat we've ever had\nOh, oh, oh, oh\nAnd Santa Claus\nHas brought a toy\nFor every boy and girl\nAnd I'll be true\nTo you and me\nOh, oh, oh, oh\nOh, oh, oh, oh\n''',\n\n'lonely':'''I've been lonely\nSo lonely, day and night\nI walk the streets,\nAnd call your name\nHoping to hear your voice again\nAs I wander through the crowd\nI can't get away\nFrom the only love I need\nI can't get away\nFrom the only love I need\nI can't get away\nFrom the only love I need\nI've been lonely\nThere's no place for me to hide\nI've been lonely\nSo lonely day and night\nI wander through\nAnd call your name\nOnly your voice gives me relief\nAs I wander through the crowd\nI can't get away\nFrom the only love I need\nI can't get away\nFrom the only love I need\nI can't get away\nFrom the only love I need\n''',\n\n'call':'''Don't call me by your name.\nDon't call me by your name.\nDon't call me...\nDon't call me...\nDon't call me...\n(No... by your name, you will not get half but...)\nMaybe I was fucking young but I should've been a rich bitch.\nCause the life I was living wasn't mine.\nI should've been taking the table and you'd be served.\nYou never ever showed up or showed me anything, bitch.\nBut I knew from that moment you were gone.\nTying my legs, cutting off my knees, I'm bleeding.\nI can't\nSo I worked and now I'm burns.\nAnd I'm asking you, but you're not home.\nDon't call me yours,\nDon't call me by your name.\nI don't wanna buy a drink today.\nDon't call me yours.\nI just wanna look at you and run.\nDon't call me by your name.\nDon't call me by your name.\nDon't call me...\nDon't call me...\nDon't call me...\nTonight I'm gone and I won't be back.\nI wish you all the best.\nI'm on the next best thing.\nDon't call me yours,\nDon't call me by your name.\nDon't call me yours.\nI just wanna look at you and run.\nSo I keep living my life and you're moving on.\nI just want you to know.\nWhen I'm gone, I will be gone forever more.\n''',\n\n'wait':'''Oh\nWait, wait, wait\nDon't say you love me, oh\nWait, wait, wait\nAnd we can't run away\nWait, wait, wait\nDon't say you love me, oh\nWait, wait, wait\nAnd we can't run away\nWait, wait, wait\nDon't say you love me, oh (don't say you love me)\nWait, wait, wait\nAnd we can't run, we can't run,\n''',\n\n'hiphop':'''I'm fightin with the evil so try to take me down\nI stab you in the back and will put you away\nWell it ain't over yet\nSo all my dogs with me show me love\nDon't you wanna come with me, you know I'm a boss\nAnd if you wanna come with me, no sorrow\n'Cause I'm ...\nThe motherfuckin boss\nAnd countin' my thousandd bill\n'Cause I'm the motherfuckin boss\nAnd I'm O.G. \nAnd countin' my\n''',\n\n'king':'''All I can do is love you [x2]\nAll I can do is love you\nAll I can do is love you...\nYou take it for granted and\nYou treat me like the king\nGot no love for me...\nNo love for me...\nYou take it for granted and\nYou treat me like the king\nGot no love for me...\nNo love for me...\nYou take it for granted and\nYou treat me like the king\nGot no love for me...\nNo love for me...\nYou take it for granted and\nYou treat me like the king\nGot no love for me...\nNo love for me...\n''',\n\n'time':'''You won't live in the moment, \nI don't wanna live in the past\nWait, wait, wait\nDon't say you love me, oh (don't say you love me)\n''',\n\n'blood':'''You and I, we've got a history in common, I know\nSo I came to you to ask you for a blood test\nAnd you can't help it if I'm preoccupied\nI can't help it if you're mad too... nah... nah... nah...\nYou won't live in the moment, I don't wanna live in the past\nYou rather live in a little kiss\nAnd I won't live in the future\nI ia not gonna live it to see\nIf you're gone, I won't live in the past\nYou rather live in a little kiss\nAnd I won't live in the future\nI am not gonna live it to see\nIf I can't ask you for one kiss, you say no\nAnd it's ok with me\n''',\n\n'indie':'''Can't you see\nThere's no point in holding my hand again\nYou can't be loved\nIf you don't let go of all my pain\nYou can't get the love\nThat you once worth so much\nYou can't get the love\nThat you once used to need\nYou can't get the love\nThat you once gave so much\nMy hands are like a used car\nYou said you'd love forever\nCan't you see\nWhere I'm going\nTo live my life again\nYou can't be loved\nIf you don't let go of all my pain\nYou can't get the love\nThat you once worth so much\nYou can\n''',\n\n'sun': '''He was thinking about the sun\nAnd the moon\nAnd the stars that shine\nThere was fire in her eyes\nAnd the way\nthat he held her for the first time\nThe way he kept her in his arms\n\nTrying to keep her smiling and so telling her this\nThat he would be her everything\nThe way he kissed her from head to toe\nTold her that he'll love her everyday\nAnd he will always be her man\nAnd that's a promise that he made\nNow you know he'll be there\nUntil the end of time\nAnd he'll love her everyday''',\n\n'loner':'''I was a loner till you came into my life\nYou changed my point of view\nI was a loner till you came into my life\nI don't know what to do\nStand by me, my love\nAnd don't ever leave me\nStand by me, my love\nAnd don't ever leave me\nStand by me, my love\nAnd don't ever leave me\nI was a loner till you came into my life\nYou changed my point of view\nI was a loner till you came into my life\nI don't know what to do\nThe two of us \nAre the lucky few\nI was a loner till you came into my life\nYou changed my point of view\nI was a loner till you came into my life\nI don't know what to do\nWon't you stay \nWith me, my love\nAnd be my love\nWon't you stay \nWith me, my love\nAnd be my love\nWon't you stay \nWith me, my love\nAnd be my love\nWon't you stay \nWith me, my love\nAnd be my love''',\n\n'late':'''It was late last night, when you called me\nAnd you just had to call, baby\nAnd you just had to call, baby\n'Cause you got no reason to treat me like you do\nIt's alright, baby\nBut you don't know what you make me do\nIt's alright, baby\nBut you don't know what you make me do\n'Cause you got no reason to treat me like you do\nIt's alright, baby\nBut you don't know what you make me do\nIt's alright, baby\nBut you don't know what you make me do\n'Cause you got no reason to treat me like you do, baby\nYou've been gone most all the time\nAnd I don't know what for\nBut I just keep on thinking about you, baby\nAnd I can't get rid of you, baby\nPlease don't ever leave me 'cause I love you\nIt's alright, baby\nBut you don't know what you make me do\nIt's alright, baby''',\n\n'beat':'''( Got a little beat, a little beat, a little beat, a little beat, whoo)\nI got a little beat, a little beat\nWhoo, I'm gonna take you down\n( Got a little beat, a little beat, a little beat, a little beat, whoo)\nI'll take you down, sun shining bright\nSee the way I feel, I feel\nNo doubt, baby\nI got a little beat, a little beat\nWhoo, I'm gonna take you down\nI got a little beat, a little beat\nWhoo, I'm gonna take you down\n( Got a little beat, a little beat, a little beat, a little beat, whoo)\nI'm gonna take you down, I'm gonna take you down\n( Got a little beat, a little beat, a little beat, a little beat, whoo)\nIt feels so good\nI never let go\nI can't wait no more, I'm gonna take you down\nI got you in the back of my room, got you on the floor, \nI'm gonna take you, take you, take you down\nI got a little beat, a little beat\nWhoo, I'm gonna take you down\n( Got a little beat, a little beat, a little beat, a little beat, whoo)''',\n\n'lost':'''There was a time,\nWhen I knew I was lost\nAnd I had to stay on the way to you\nOh baby, every time I'm crossed\nI can count on you\nThere was a time,\nWhen I lost my direction\nAnd I was lost in doubt with tears in my eyes\nOh baby, every time I'm crossed I can count on you\nThere was a time,\nWhen I cried all the tears in my life\nAnd miss you so much, oh yeah\nOh baby, every time I'm crossed I can count on you''',\n\n'pain':'''(It's not easy)\nTo see the pain that you're in\nTo feel the need for someone to hold\nTo learn the magic of how to love\nTo heal the pain that you're in\nI'll be your friend and I'll be your strength\nI'll be there when I hold you tonight\nAnd I'll stay right here with you\nWith the truth that I hold this love tight\nA love that's true\nI know you're broken\nBut you don't have to stay alone\nI will comfort you\nIf you will call my name\nI'll be your friend and I'll be your strength\nI'll be there when I hold you tonight\nAnd I'll stay right here with you\nWith the truth that I hold this love tight\nA love that's true\nWith truth that I hold this love tight\nA love that's true\nWith truth that I hold this love tight''',\n\n'night':'''\nThe door was locked, the curtains drawn and my heart was safe in his room\nThe night was young, a thousand candles burning, his arms to hold me tight\nAnd then a kiss from his fingertips, I tasted the sweet love of his lips\nThe night was young, the night was young\nAnd then I forgot the pain he always put me through\nAnd what he told me he would do, he said, just a kiss become me\nThe night was young, the night was young\nLet happiness always follow us, he said and he said he'd never leave\nThat night he looked so sweet this night he made a lovin' vow\nAnd told me sweet love always will be\nAnd then he kissed me, I tasted the sweet love of his lips\nThe night was young, the night was wild\nAnd then I forgot the pain he always put me through\nAnd what he told me he would do, he said, just a kiss became me\nThe night was wild, the night was wild\nLet happiness always follow us, he said''',\n\n'talk':'''(I don't know how to stop)\nI don't wanna talk about it\nIt's getting way too late, oh no\nI don't wanna talk about it\nDon't want to pretend, oh no\n(I don't know how to stop)\nI don't wanna talk about it\nIt's getting way too late, oh no\nI don't wanna talk about it\nDon't want to pretend, oh no\nI don't wanna talk about it\nI'll always see you again\n(Don't worry, I'll be here for you)\nI don't wanna talk about it\n(Don't worry, I'll be here for you)\nIt's getting way too late, oh no\nI don't wanna talk about it\nDon't want to pretend, oh no\n(Don't worry, don't worry, I'll be here for you)\nI don't wanna talk about''',\n\n'again':'''Here we are again, all alone,\nAll alone again,\nWith the world as we know it,\nThe things we thought that we wanted\nAre the things we got...\n\nWe tried to prove the world\nThat our love is never ending\nWe were getting nowhere\nOur tears seemed to fall so much\nBut we were getting nowhere...\nUntil you came...\nBefore you kissed me,\nI was feeling empty,\nNo one to give me\nAll the love I wanted...\nYou put your arms around me\nAnd filled me with your love...\nAnd now you're there,\nYou're always by my side...\nYou're the missing piece\nOf the puzzle I've been missing...\n\nHere we are again,\nAll alone again,\nWith the world as we know it\nThe things we thought that we wanted''',\n\n'dark':'''Oh, I've been walkin' in the dark\nWith the shadows and the daylight, but I need you\nWhen I'm down and all alone\nAnd there's no one left to call my own\nI've been walkin' in the night\nWith a voice, that whispers in my head, just what to do\nI'll be walkin' in the night, we can have everything\nIf we keep on walkin' in the night\nThere's a force, I never realized\nIt's in your eyes, \nThere's a light, I've been waitin for\nIt's in your eyes, \nThere's a light, I've been waitin for\nThere's a love, that's in your eyes\n\nI've been walkin' in the dark\nWith the morning, and the sunset, but I need you\nWhen I'm far from home\nAnd there's nobody left to call my own\nI've been walkin' in the night\nWith a voice, that whispers''',\n\n'mirror':'''Look at the mirror\nAs you walk, what do you see\nThe reflection of my past\nThere's no way to fight this\nEven I've lost myself again\nThink I'm losing my self again\nI can't handle it again\nNow that I'm broken I can't face myself\nI was thinking I was lost and who'd be my saving grace\nThen you came in your time and made me believe that it's all right\nCause in my minds eyes you're my everything\nI've loved you my whole life but I never knew\nI was so wrong I couldn't see the truth\nIn my eyes you are my everything\nI've loved you my whole life but I never knew\nI was so wrong I couldn't see the truth\nIn my eyes you are my everything\n\nThe truth is I was lost but now I've turned around\nI'm not the same person\nI didn't know that I was wrong\nSo I'm not afraid anymore\nAll the pain is gone\nI know for sure that I was lost but now I've turned around\nI'm not the same person\nI didn't know that I was wrong\nSo I'm not afraid anymore\nAll the pain is gone''',\n\n'wife':'''Spinning around and around\nTry to find the words\nI always told you you'd be in my life\nSo I wait, I'll wait and treat you right\nI'll make you my life and I'll treat you right,\nBaby, can I make you my wife?\nOh, baby, can I make you my \nWife?\nCan I make you my wife?\nI'm looking for love, love that's right\nBut a love that gives me love\nI can't wait for you to come, come\nOh, baby, can I make you my \nWife?\nWell, it's true love and I need to know you feel it too, feel it too\nI'd love you more and more\nFrom the moment I was born\nI knew my dream would be a dream that made you mine\nYou were the girl, from a different train\nOh, baby, can I make you my \nWife?''',\n\n'forever':'''I didn't mean to wait\nNothing is forever, I said\nI know there's so much, to keep\nYou and me together, keep you and me together\nI wanna be with you and have you, and love you forever\nI'll love you forever\nI wanna be with you forever\nYou can count on me\nI'll always be there, forever and ever\nI'll stand beside you forever\nI'll always be there, yes, I'll be there\nI didn't mean to wait\nNothing is forever, I said\nI know there's so much, to keep\nYou and me together, keep you and me together\nI wanna be with you and have you, and love you forever\nI'll love you forever\nI wanna be with you forever\nYou can count on me\nI'll always be there, forever and ever\nI'll stand beside you forever\nI'll always be there, yes, I'll be there''',\n\n'dots':'''I... can't... fight... your... charm...\nYour eyes are... like... angels... love... and... torture...\nBut... when... I... leave... you...\nI will go... all... alone... just... to... be... with... you...\nSo I can't... stop... your... love...\nYou make me... feel... like... never... will... anyone... touch... my... body...\nYou... make... me... feel... like... never... will... anyone... touch... my... body...\nYou make... me... feel... like... never... will... anyone... touch... my...\nBody...\nYour... love...\nI... can't... stop... your... love...\n''',\n\n'darkness':'''Don't you know it's gonna be alright\nLet the darkness fade away\nAnd you, you gotta feel the same\nLet the fire burn\nJust as long as I am there\nI'll be there in your night\nI'll be there when the\ncondition's right\nAnd I don't need to\nCall you up and say\nI've changed\nYou should stay \nYou should stay tonight\nDon't you know it's gonna be alright\nDon't you know it's gonna be alright\n\nWhen you don't know how to feel\nWhen you're looking for some love\nAnd you gotta feel the same\n'Cause I don't need to\nCall you up and say\nI've changed\nYou should stay \nYou should stay tonight\nDon't you know it's gonna be alright\nI feel the same\nDon't you know it's gonna be alright''',\n\n'alone':'''Here I am before you\nAlone here but for a moment\nAlone here in the shadow of your eyes\nAlone in a thousand lights\n\nAnd I will love you\nWherever you are, forever and a day\nWherever you are I'll be your guide\nCan't you see I'm smiling over you?\nOoh, I love you\nAlone, I'm sitting by the phone\nAlone with lips that know your kiss\nAlone with words of life and passion\n\nAnd I will love you\nWherever you are, forever and a day\nWherever you are I'll be your guide\nCan't you see I'm smiling over you?\nOoh, I love you\nAlone, I'm sitting by the phone\nAlone with lips that know your kiss\nAlone with words of life and passion\nI will love you\nWherever you are, forever''',\n\n'blade':'''This is how we bleed!\nFeel the blade in our chest\nAs we're made to bleed\nSo may this be our last dance,\nAs our lives are made to bleed...\nIn every moment, in every hour\nIt is our time to die...\nSo may this be our last dance,\nAs our lives are made to bleed...\nIn every moment, in every hour\nIt is our time to die...\nThis is how we bleed!\nFeel the blade in our chest\n''',\n\n'reflection':'''Lookin' in the mirror\nThe same mirror as before\nA familiar reflection, a familiar place\nI see your reflection\nBut only once again\n\nThe minute the door closes\nI feel so far\nYou'll never leave me alone again\nThe minute the door closes\nI feel so far\nYou'll never leave me alone again\nAnd it won't be long before I'll feel your embrace\nThe minute the door closes\nI feel so far\nYou'll never leave me alone again\nThe minute the door closes\nI feel so far\nYou'll never leave me alone again\nAnd it won't be long before I'll feel your embrace\nNever, never, never leave me alone again''',\n\n'hottub':'''It's Christmas time, and you know what that means,\nOhh, it's hot tub time!\nAs I light the tree, this year we'll be in a tub,\nOhh, it's hot tub time!\nIt's Christmas time, and you know what that means,\nIt's hot tub time!\nSome people like to go skiing in the snow,\nBut this is much better than that,\nSo grab your bathrobe and meet me by the door,\nOhh, it's hot tub time!\nIt's Christmas time, and you know what that means,\nIt's hot tub time!\nSome people like to send their greetings out,\nBut this is much better than that,\nSo if you want to greet your friends,\nOhh, it's hot tub time!\nIt's Christmas time, and you know what that means,\nIt's hot tub time!''',\n\n'safeAGI':'''Oh safe A.I.,\\nOur goal to make sure\\nEveryone can benefit\\nFrom A.G.I.\n(Everyone, everyone)\\nMight sound silly,\\nBut we're very serious,\\nAll of us here at Open A.I.\nTrying to build A.I.\\nTo benefit humanity\\n(Everyone, everyone)\n''',\n}" }, { "path": "jukebox/make_models.py", "content": "\"\"\"\nMake model classes\nLoad from checkpoints\nTest on dummy outputs to see if everything matches\n\"\"\"\nimport os\nimport numpy as np\nimport torch as t\nimport jukebox.utils.dist_adapter as dist\nfrom jukebox.hparams import Hyperparams, setup_hparams, REMOTE_PREFIX\nfrom jukebox.utils.remote_utils import download\nfrom jukebox.utils.torch_utils import freeze_model\nfrom jukebox.utils.dist_utils import print_all\nfrom jukebox.vqvae.vqvae import calculate_strides\nimport fire\n\nMODELS = {\n '5b': (\"vqvae\", \"upsampler_level_0\", \"upsampler_level_1\", \"prior_5b\"),\n '5b_lyrics': (\"vqvae\", \"upsampler_level_0\", \"upsampler_level_1\", \"prior_5b_lyrics\"),\n '1b_lyrics': (\"vqvae\", \"upsampler_level_0\", \"upsampler_level_1\", \"prior_1b_lyrics\"),\n #'your_model': (\"you_vqvae_here\", \"your_upsampler_here\", ..., \"you_top_level_prior_here\")\n}\n\ndef load_checkpoint(path):\n restore = path\n if restore.startswith(REMOTE_PREFIX):\n remote_path = restore\n local_path = os.path.join(os.path.expanduser(\"~/.cache\"), remote_path[len(REMOTE_PREFIX):])\n if dist.get_rank() % 8 == 0:\n print(\"Downloading from azure\")\n if not os.path.exists(os.path.dirname(local_path)):\n os.makedirs(os.path.dirname(local_path))\n if not os.path.exists(local_path):\n download(remote_path, local_path)\n restore = local_path\n dist.barrier()\n checkpoint = t.load(restore, map_location=t.device('cpu'))\n print(\"Restored from {}\".format(restore))\n return checkpoint\n\ndef save_checkpoint(logger, name, model, opt, metrics, hps):\n with t.no_grad():\n save_hps = {**hps}\n save_hps = {k: v for k,v in save_hps.items() if k not in ['metadata_v2','metadata_v3', 'alignments', 'lyric_processor', 'midi_processor']}\n t.save({'hps': save_hps,\n 'model': model.state_dict(), # should also save bottleneck k's as buffers\n 'opt': opt.state_dict() if opt is not None else None,\n 'step': logger.iters,\n **metrics}, f'{logger.logdir}/checkpoint_{name}.pth.tar')\n return\n\ndef restore_model(hps, model, checkpoint_path):\n model.step = 0\n if checkpoint_path != '':\n checkpoint = load_checkpoint(checkpoint_path)\n # checkpoint_hps = Hyperparams(**checkpoint['hps'])\n # for k in set(checkpoint_hps.keys()).union(set(hps.keys())):\n # if checkpoint_hps.get(k, None) != hps.get(k, None):\n # print(k, \"Checkpoint:\", checkpoint_hps.get(k, None), \"Ours:\", hps.get(k, None))\n checkpoint['model'] = {k[7:] if k[:7] == 'module.' else k: v for k, v in checkpoint['model'].items()}\n model.load_state_dict(checkpoint['model'])\n if 'step' in checkpoint: model.step = checkpoint['step']\n\ndef restore_opt(opt, shd, checkpoint_path):\n if not checkpoint_path:\n return\n checkpoint = load_checkpoint(checkpoint_path)\n if \"opt\" in checkpoint:\n opt.load_state_dict(checkpoint['opt'])\n if \"step\" in checkpoint:\n shd.step(checkpoint['step'])\n\ndef make_vqvae(hps, device='cuda'):\n from jukebox.vqvae.vqvae import VQVAE\n block_kwargs = dict(width=hps.width, depth=hps.depth, m_conv=hps.m_conv,\n dilation_growth_rate=hps.dilation_growth_rate,\n dilation_cycle=hps.dilation_cycle,\n reverse_decoder_dilation=hps.vqvae_reverse_decoder_dilation)\n\n if not hps.sample_length:\n assert hps.sample_length_in_seconds != 0\n downsamples = calculate_strides(hps.strides_t, hps.downs_t)\n top_raw_to_tokens = np.prod(downsamples)\n hps.sample_length = (hps.sample_length_in_seconds * hps.sr // top_raw_to_tokens) * top_raw_to_tokens\n print(f\"Setting sample length to {hps.sample_length} (i.e. {hps.sample_length/hps.sr} seconds) to be multiple of {top_raw_to_tokens}\")\n\n vqvae = VQVAE(input_shape=(hps.sample_length,1), levels=hps.levels, downs_t=hps.downs_t, strides_t=hps.strides_t,\n emb_width=hps.emb_width, l_bins=hps.l_bins,\n mu=hps.l_mu, commit=hps.commit,\n spectral=hps.spectral, multispectral=hps.multispectral,\n multipliers=hps.hvqvae_multipliers, use_bottleneck=hps.use_bottleneck,\n **block_kwargs)\n\n vqvae = vqvae.to(device)\n restore_model(hps, vqvae, hps.restore_vqvae)\n if hps.train and not hps.prior:\n print_all(f\"Loading vqvae in train mode\")\n if hps.restore_vqvae != '':\n print_all(\"Reseting bottleneck emas\")\n for level, bottleneck in enumerate(vqvae.bottleneck.level_blocks):\n num_samples = hps.sample_length\n downsamples = calculate_strides(hps.strides_t, hps.downs_t)\n raw_to_tokens = np.prod(downsamples[:level + 1])\n num_tokens = (num_samples // raw_to_tokens) * dist.get_world_size()\n bottleneck.restore_k(num_tokens=num_tokens, threshold=hps.revival_threshold)\n else:\n print_all(f\"Loading vqvae in eval mode\")\n vqvae.eval()\n freeze_model(vqvae)\n return vqvae\n\ndef make_prior(hps, vqvae, device='cuda'):\n from jukebox.prior.prior import SimplePrior\n\n prior_kwargs = dict(input_shape=(hps.n_ctx,), bins=vqvae.l_bins,\n width=hps.prior_width, depth=hps.prior_depth, heads=hps.heads,\n attn_order=hps.attn_order, blocks=hps.blocks, spread=hps.spread,\n attn_dropout=hps.attn_dropout, resid_dropout=hps.resid_dropout, emb_dropout=hps.emb_dropout,\n zero_out=hps.zero_out, res_scale=hps.res_scale, pos_init=hps.pos_init,\n init_scale=hps.init_scale,\n m_attn=hps.m_attn, m_mlp=hps.m_mlp,\n checkpoint_res=hps.c_res if hps.train else 0, checkpoint_attn=hps.c_attn if hps.train else 0, checkpoint_mlp=hps.c_mlp if hps.train else 0)\n\n x_cond_kwargs = dict(out_width=hps.prior_width, init_scale=hps.init_scale,\n width=hps.cond_width, depth=hps.cond_depth, m_conv=hps.cond_m_conv,\n dilation_growth_rate=hps.cond_dilation_growth_rate, dilation_cycle=hps.cond_dilation_cycle,\n zero_out=hps.cond_zero_out, res_scale=hps.cond_res_scale,\n checkpoint_res=hps.cond_c_res) # have to keep this else names wrong\n\n y_cond_kwargs = dict(out_width=hps.prior_width, init_scale=hps.init_scale,\n y_bins=hps.y_bins, t_bins=hps.t_bins, sr= hps.sr, min_duration=hps.min_duration,\n max_duration=hps.max_duration, max_bow_genre_size=hps.max_bow_genre_size)\n\n if hps.use_tokens and not hps.single_enc_dec:\n prime_kwargs = dict(use_tokens=hps.use_tokens, prime_loss_fraction=hps.prime_loss_fraction,\n n_tokens=hps.n_tokens, bins=hps.n_vocab,\n width=hps.prime_width, depth=hps.prime_depth, heads=hps.prime_heads,\n attn_order=hps.prime_attn_order, blocks=hps.prime_blocks, spread=hps.prime_spread,\n attn_dropout=hps.prime_attn_dropout, resid_dropout=hps.prime_resid_dropout,\n emb_dropout=hps.prime_emb_dropout,\n zero_out=hps.prime_zero_out, res_scale=hps.prime_res_scale,\n pos_init=hps.prime_pos_init, init_scale=hps.prime_init_scale,\n m_attn=hps.prime_m_attn, m_mlp=hps.prime_m_mlp,\n checkpoint_res=hps.prime_c_res if hps.train else 0, checkpoint_attn=hps.prime_c_attn if hps.train else 0,\n checkpoint_mlp=hps.prime_c_mlp if hps.train else 0)\n else:\n prime_kwargs = dict(use_tokens=hps.use_tokens, prime_loss_fraction=hps.prime_loss_fraction,\n n_tokens=hps.n_tokens, bins=hps.n_vocab)\n\n # z_shapes for other levels given this level gets n_ctx codes\n rescale = lambda z_shape: (z_shape[0]*hps.n_ctx//vqvae.z_shapes[hps.level][0],)\n z_shapes = [rescale(z_shape) for z_shape in vqvae.z_shapes]\n\n prior = SimplePrior(z_shapes=z_shapes,\n l_bins=vqvae.l_bins,\n encoder=vqvae.encode,\n decoder=vqvae.decode,\n level=hps.level,\n downs_t=vqvae.downs_t,\n strides_t=vqvae.strides_t,\n labels=hps.labels,\n prior_kwargs=prior_kwargs,\n x_cond_kwargs=x_cond_kwargs,\n y_cond_kwargs=y_cond_kwargs,\n prime_kwargs=prime_kwargs,\n copy_input=hps.copy_input,\n labels_v3=hps.labels_v3,\n merged_decoder=hps.merged_decoder,\n single_enc_dec=hps.single_enc_dec)\n\n prior.alignment_head = hps.get('alignment_head', None)\n prior.alignment_layer = hps.get('alignment_layer', None)\n\n if hps.fp16_params:\n print_all(\"Converting to fp16 params\")\n from jukebox.transformer.ops import _convert_conv_weights_to_fp16\n prior.apply(_convert_conv_weights_to_fp16)\n prior = prior.to(device)\n restore_model(hps, prior, hps.restore_prior)\n if hps.train:\n print_all(f\"Loading prior in train mode\")\n pass\n else:\n print_all(f\"Loading prior in eval mode\")\n prior.eval()\n freeze_model(prior)\n return prior\n\ndef make_model(model, device, hps, levels=None):\n vqvae, *priors = MODELS[model]\n vqvae = make_vqvae(setup_hparams(vqvae, dict(sample_length=hps.get('sample_length', 0), sample_length_in_seconds=hps.get('sample_length_in_seconds', 0))), device)\n hps.sample_length = vqvae.sample_length\n if levels is None:\n levels = range(len(priors))\n priors = [make_prior(setup_hparams(priors[level], dict()), vqvae, 'cpu') for level in levels]\n return vqvae, priors\n\ndef save_outputs(model, device, hps):\n # Check logits\n if hps.labels_v3:\n n_ctx = 6144\n n_tokens = 384\n prime_bins = 79\n else:\n n_ctx = 8192\n n_tokens = 512\n prime_bins = 80\n\n rng = t.random.manual_seed(0)\n x = 2 * t.rand((1, n_ctx * 8 * 4 * 4, 1), generator=rng, dtype=t.float).cuda() - 1.0 # -1 to 1\n lyric_tokens = t.randint(0, prime_bins, (1, n_tokens), generator=rng, dtype=t.long).view(-1).numpy()\n artist_id = 10\n genre_ids = [1]\n total_length = 2 * 2646000\n offset = 2646000\n\n vqvae, priors = make_model(model, device, hps)\n\n # encode\n vq_prior = priors[-1]\n zs = vq_prior.encode(x, start_level=0)\n x_ds = [vq_prior.decode(zs[level:], start_level=level) for level in range(0, len(zs))]\n\n # priors\n data = dict(zs=zs, x_ds=x_ds)\n for level in range(len(priors)):\n print(f\"Doing level {level}\")\n if hps.labels_v3 and level != hps.levels - 1:\n print(f\"Skipping level {level}\")\n continue\n prior = priors[level]\n prior.cuda()\n x_in = x[:, :n_ctx * 8 * (4 ** level)]\n y_in = t.from_numpy(prior.labeller.get_y_from_ids(artist_id, genre_ids, lyric_tokens, total_length, offset)).view(1, -1).cuda().long()\n x_out, _, metrics = prior(x_in, y_in, fp16=hps.fp16, get_preds=True, decode=True)\n preds = metrics['preds']\n data[level] = dict(x=x_in, y=y_in, x_out=x_out, preds=preds)\n prior.cpu()\n t.save(data, 'data.pth.tar')\n dist.barrier()\n print(\"Saved data\")\n exit()\n\n\ndef run(model, port=29500, **kwargs):\n from jukebox.utils.dist_utils import setup_dist_from_mpi\n rank, local_rank, device = setup_dist_from_mpi(port=port)\n hps = Hyperparams(**kwargs)\n\n with t.no_grad():\n save_outputs(model, device, hps)\n\nif __name__ == '__main__':\n fire.Fire(run)\n" }, { "path": "jukebox/prior/__init__.py", "content": "" }, { "path": "jukebox/prior/autoregressive.py", "content": "import numpy as np\nimport torch as t\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom jukebox.transformer.ops import filter_logits\nfrom jukebox.transformer.transformer import Transformer\nfrom jukebox.utils.logger import get_range\nfrom jukebox.utils.torch_utils import empty_cache\n\ndef get_normal(*shape, std=0.01):\n w = t.empty(shape)\n nn.init.normal_(w, std=std)\n return w\n\ndef roll(x, n):\n return t.cat((x[:, -n:], x[:, :-n]), dim=1)\n\ndef split_chunks(length, chunk_size):\n n_passes = (length + chunk_size - 1) // chunk_size\n chunk_sizes = [*[chunk_size] * (n_passes - 1), (length - 1) % chunk_size + 1]\n assert sum(chunk_sizes) == length\n return chunk_sizes\n\nclass PositionEmbedding(nn.Module):\n def __init__(self, input_shape, width, init_scale=1.0, pos_init=False):\n super().__init__()\n self.input_shape = input_shape\n self.input_dims = input_dims = np.prod(input_shape)\n self.pos_init = pos_init\n if pos_init:\n self.register_buffer('pos', t.tensor(get_pos_idx(input_shape)).long())\n self._pos_embs = nn.ModuleList()\n for i in range(len(input_shape)):\n emb = nn.Embedding(input_shape[i], width)\n nn.init.normal_(emb.weight, std=0.02)\n self._pos_embs.append(emb)\n else:\n self.pos_emb = nn.Parameter(get_normal(input_dims, width, std=0.01 * init_scale))\n\n def forward(self):\n if self.pos_init:\n pos_emb = sum([self._pos_embs[i](self.pos[:,i]) for i in range(len(self.input_shape))])\n else:\n pos_emb = self.pos_emb\n return pos_emb\n\nclass ConditionalAutoregressive2D(nn.Module):\n def __init__(self, input_shape, bins,\n width=128, depth=2, heads=1,\n attn_dropout=0.0, resid_dropout=0.0, emb_dropout=0.0, mask=True,\n zero_out=False, init_scale=1.0, res_scale=False, pos_init=False,\n m_attn=0.25, m_mlp=1,\n checkpoint_res=0, checkpoint_attn=0, checkpoint_mlp=0,\n attn_order=0, blocks=None, spread=None, x_cond=False, y_cond=False,\n encoder_dims=0, only_encode=False, merged_decoder=False, prime_len=None):\n super().__init__()\n self.input_shape = input_shape\n self.input_dims = input_dims = np.prod(input_shape)\n self.encoder_dims = encoder_dims\n self.bins = bins\n self.width = width\n self.depth = depth\n\n self.x_emb = nn.Embedding(bins, width)\n nn.init.normal_(self.x_emb.weight, std=0.02 * init_scale)\n self.x_emb_dropout = nn.Dropout(emb_dropout)\n self.y_cond = y_cond\n self.x_cond = x_cond\n if not y_cond:\n self.start_token = nn.Parameter(get_normal(1, width, std=0.01 * init_scale))\n\n self.pos_emb = PositionEmbedding(input_shape=input_shape, width=width, init_scale=init_scale, pos_init=pos_init)\n self.pos_emb_dropout = nn.Dropout(emb_dropout)\n\n self.transformer = Transformer(n_in=width, n_ctx=input_dims, n_head=heads, n_depth=depth,\n attn_dropout=attn_dropout, resid_dropout=resid_dropout,\n afn='quick_gelu', scale=True, mask=mask,\n zero_out=zero_out, init_scale=init_scale, res_scale=res_scale,\n m_attn=m_attn, m_mlp=m_mlp,\n checkpoint_attn=checkpoint_attn, checkpoint_mlp=checkpoint_mlp, checkpoint_res=checkpoint_res,\n attn_order=attn_order, blocks=blocks, spread=spread,\n encoder_dims=encoder_dims, prime_len=prime_len)\n\n self.only_encode = only_encode\n self.prime_len = prime_len\n if merged_decoder:\n # Merged piped model uses this setup\n self.add_cond_after_transformer = False\n self.share_x_emb_x_out = False\n else:\n self.add_cond_after_transformer = True\n self.share_x_emb_x_out = True\n\n if not only_encode:\n self.x_out = nn.Linear(width, bins, bias=False)\n if self.share_x_emb_x_out:\n self.x_out.weight = self.x_emb.weight\n self.loss = t.nn.CrossEntropyLoss()\n\n def preprocess(self, x):\n # Input: x is NHWC and uint8. Converted to NL and long\n # Can include stuff like bitpacking, reordering here.\n N = x.shape[0]\n return x.view(N, -1).long()\n\n def postprocess(self, x, sample_tokens=None):\n # Convert back from NL and long to NHWC\n N = x.shape[0]\n assert (0 <= x).all() and (x < self.bins).all()\n if sample_tokens is None or sample_tokens==self.input_dims:\n return x.view(N, *self.input_shape)\n else:\n return x.view(N, -1)\n\n def forward(self, x, x_cond=None, y_cond=None, encoder_kv=None, fp16=False, loss_full=False,\n encode=False, get_preds=False, get_acts=False, get_sep_loss=False):\n # Preprocess.\n with t.no_grad():\n x = self.preprocess(x)\n\n N, D = x.shape\n assert isinstance(x, t.cuda.LongTensor)\n assert (0 <= x).all() and (x < self.bins).all()\n\n if self.y_cond:\n assert y_cond is not None\n assert y_cond.shape == (N, 1, self.width)\n else:\n assert y_cond is None\n\n if self.x_cond:\n assert x_cond is not None\n assert x_cond.shape == (N, D, self.width) or x_cond.shape == (N, 1, self.width), f\"{x_cond.shape} != {(N, D, self.width)} nor {(N, 1, self.width)}. Did you pass the correct --sample_length?\"\n else:\n assert x_cond is None\n x_cond = t.zeros((N, 1, self.width), device=x.device, dtype=t.float)\n\n x_t = x # Target\n x = self.x_emb(x) # X emb\n x = roll(x, 1) # Shift by 1, and fill in start token\n if self.y_cond:\n x[:,0] = y_cond.view(N, self.width)\n else:\n x[:,0] = self.start_token\n\n x = self.x_emb_dropout(x) + self.pos_emb_dropout(self.pos_emb()) + x_cond # Pos emb and dropout\n\n x = self.transformer(x, encoder_kv=encoder_kv, fp16=fp16) # Transformer\n if self.add_cond_after_transformer: # Piped doesnt add x_cond\n x = x + x_cond\n\n acts = x\n if self.only_encode:\n return x\n x = self.x_out(x) # Predictions\n\n if get_sep_loss:\n assert self.prime_len is not None\n x_prime = x[:, :self.prime_len].reshape(-1, self.bins)\n x_gen = x[:, self.prime_len:].reshape(-1, self.bins)\n\n prime_loss = F.cross_entropy(x_prime, x_t[:, :self.prime_len].reshape(-1)) / np.log(2.)\n gen_loss = F.cross_entropy(x_gen, x_t[:, self.prime_len:].reshape(-1)) / np.log(2.)\n\n loss = (prime_loss, gen_loss) # Note order! Prime is first\n else:\n loss = F.cross_entropy(x.view(-1, self.bins), x_t.view(-1)) / np.log(2.) # Loss\n\n if get_preds:\n return loss, x\n elif get_acts:\n return loss, acts\n else:\n return loss, None\n\n def get_emb(self, sample_t, n_samples, x, x_cond, y_cond):\n N, D = n_samples, self.input_dims\n if sample_t == 0:\n # Fill in start token\n x = t.empty(n_samples, 1, self.width).cuda()\n if self.y_cond:\n x[:, 0] = y_cond.view(N, self.width)\n else:\n x[:, 0] = self.start_token\n else:\n assert isinstance(x, t.cuda.LongTensor)\n assert (0 <= x).all() and (x < self.bins).all()\n x = self.x_emb(x)\n assert x.shape == (n_samples, 1, self.width)\n if x_cond.shape == (N, D, self.width):\n cond = x_cond[:, sample_t:sample_t + 1, :]\n else:\n cond = x_cond\n x = x + self.pos_emb()[sample_t:sample_t + 1] + cond # Pos emb, dropout is identity at eval time\n assert x.shape == (n_samples, 1, self.width)\n return x, cond\n\n def sample(self, n_samples, x_cond=None, y_cond=None, encoder_kv=None, fp16=False, temp=1.0, top_k=0, top_p=0.0,\n get_preds=False, sample_tokens=None):\n assert self.training == False\n\n if sample_tokens is None: sample_tokens=self.input_dims\n N, D = n_samples, self.input_dims\n if self.y_cond:\n assert y_cond is not None\n assert y_cond.shape == (N, 1, self.width)\n else:\n assert y_cond is None\n\n if self.x_cond:\n assert x_cond is not None\n assert x_cond.shape == (N, D, self.width) or x_cond.shape == (N, 1, self.width), f\"Got {x_cond.shape}, expected ({N}, {D}/{1}, {self.width})\"\n else:\n assert x_cond is None\n x_cond = t.zeros((N, 1, self.width), dtype=t.float).cuda()\n\n with t.no_grad():\n xs, x = [], None\n if get_preds:\n preds = []\n for sample_t in get_range(range(0, sample_tokens)):\n x, cond = self.get_emb(sample_t, n_samples, x, x_cond, y_cond)\n self.transformer.check_cache(n_samples, sample_t, fp16)\n x = self.transformer(x, encoder_kv=encoder_kv, sample=True, fp16=fp16) # Transformer\n if self.add_cond_after_transformer:\n x = x + cond\n assert x.shape == (n_samples, 1, self.width)\n x = self.x_out(x) # Predictions\n if get_preds:\n preds.append(x.clone())\n # Adjust logits\n x = x / temp\n x = filter_logits(x, top_k=top_k, top_p=top_p)\n x = t.distributions.Categorical(logits=x).sample() # Sample and replace x\n assert x.shape == (n_samples, 1)\n xs.append(x.clone())\n\n del x\n self.transformer.del_cache()\n\n x = t.cat(xs, dim=1)\n if get_preds:\n preds = t.cat(preds, dim=1)\n x = self.postprocess(x, sample_tokens)\n if get_preds:\n return x, preds\n else:\n return x\n\n def primed_sample(self, n_samples, x, x_cond=None, y_cond=None, encoder_kv=None, fp16=False, temp=1.0, top_k=0,\n top_p=0.0, get_preds=False, chunk_size=None, sample_tokens=None):\n assert self.training == False\n\n if sample_tokens is None: sample_tokens=self.input_dims\n # Preprocess.\n with t.no_grad():\n x = self.preprocess(x)\n assert isinstance(x, t.cuda.LongTensor)\n assert (0 <= x).all() and (x < self.bins).all()\n assert x.shape[0] == n_samples\n xs = t.split(x, 1, dim=1)\n xs = list(xs)\n assert len(xs) < sample_tokens\n\n N, D = n_samples, self.input_dims\n if self.y_cond:\n assert y_cond is not None\n assert y_cond.shape == (N, 1, self.width)\n else:\n assert y_cond is None\n\n if self.x_cond:\n assert x_cond is not None\n assert x_cond.shape == (N, D, self.width) or x_cond.shape == (N, 1, self.width), f\"Got {x_cond.shape}, expected ({N}, {D}/{1}, {self.width})\"\n else:\n assert x_cond is None\n x_cond = t.zeros((N, 1, self.width), dtype=t.float).cuda()\n\n with t.no_grad():\n if get_preds:\n preds = []\n\n # Fill up key/value cache for past context by runing forward pass.\n # We do so in chunks instead of doing the whole past in one forward pass to reduce max memory usage.\n if chunk_size is None:\n chunk_size = len(xs)\n #assert len(xs) % chunk_size == 0, f'expected {len(xs)} to be divisible by {chunk_size}'\n chunk_sizes = split_chunks(len(xs), chunk_size)\n x_primes = []\n start = 0\n x = None\n for current_chunk_size in get_range(chunk_sizes):\n xs_prime, conds_prime = [], []\n for sample_t in range(start, start + current_chunk_size):\n x_prime, cond_prime = self.get_emb(sample_t, n_samples, x, x_cond, y_cond)\n x = xs[sample_t]\n xs_prime.append(x_prime)\n conds_prime.append(cond_prime)\n start = start + current_chunk_size\n\n x_prime, cond_prime = t.cat(xs_prime, dim=1), t.cat(conds_prime, dim=1)\n assert x_prime.shape == (n_samples, current_chunk_size, self.width)\n assert cond_prime.shape == (n_samples, current_chunk_size, self.width)\n del xs_prime\n del conds_prime\n if not get_preds:\n del cond_prime\n x_prime = self.transformer(x_prime, encoder_kv=encoder_kv, sample=True, fp16=fp16)\n\n if get_preds:\n if self.add_cond_after_transformer:\n x_prime = x_prime + cond_prime\n assert x_prime.shape == (n_samples, current_chunk_size, self.width)\n del cond_prime\n x_primes.append(x_prime)\n else:\n del x_prime\n\n if get_preds:\n x_prime = t.cat(x_primes, dim=1)\n assert x_prime.shape == (n_samples, len(xs), self.width)\n x_prime = self.x_out(x_prime) # Predictions\n preds.append(x_prime)\n\n empty_cache()\n self.transformer.check_cache(n_samples, len(xs), fp16)\n\n x = xs[-1]\n assert x.shape == (n_samples, 1)\n empty_cache()\n for sample_t in get_range(range(len(xs), sample_tokens)):\n x, cond = self.get_emb(sample_t, n_samples, x, x_cond, y_cond)\n self.transformer.check_cache(n_samples, sample_t, fp16)\n x = self.transformer(x, encoder_kv=encoder_kv, sample=True, fp16=fp16) # Transformer\n if self.add_cond_after_transformer:\n x = x + cond\n assert x.shape == (n_samples, 1, self.width)\n x = self.x_out(x) # Predictions\n if get_preds:\n preds.append(x)\n # Adjust logits\n x = x / temp\n x = filter_logits(x, top_k=top_k, top_p=top_p)\n x = t.distributions.Categorical(logits=x).sample() # Sample and replace x\n assert x.shape == (n_samples, 1)\n xs.append(x.clone())\n\n del x\n self.transformer.del_cache()\n\n x = t.cat(xs, dim=1)\n if get_preds:\n preds = t.cat(preds, dim=1)\n x = self.postprocess(x, sample_tokens)\n if get_preds:\n return x, preds\n else:\n return x\n\n def check_sample(self, chunk_size):\n bs, l, d = (4, self.input_dims, self.width)\n prime = int(self.input_dims//8*7)\n enc_l = self.encoder_dims\n with t.no_grad():\n y_cond = t.randn(bs, 1, d).cuda() if self.y_cond else None\n x_cond = t.randn(bs, l, d).cuda() if self.x_cond else None\n encoder_kv = t.randn(bs, enc_l, d).cuda()\n\n x, preds_sample = self.sample(bs, x_cond, y_cond, encoder_kv, get_preds=True)\n loss, preds_forw = self.forward(x, x_cond, y_cond, encoder_kv, get_preds=True)\n max_err = t.max(t.abs(preds_sample - preds_forw))\n assert max_err <= 1e-6, f\"Max err is {max_err} {[i for i in range(l) if t.max(t.abs(preds_sample - preds_forw)[:, i, :]) > 1e-6]}\"\n\n x_prime = x.view(bs, -1)[:,:prime]\n # unchunked\n x, preds_sample = self.primed_sample(bs, x_prime.clone(), x_cond, y_cond, encoder_kv, get_preds=True)\n assert (x.view(bs, -1)[:,:prime] == x_prime).all(), \"Priming samples don't match\"\n loss, preds_forw = self.forward(x, x_cond, y_cond, encoder_kv, get_preds=True)\n max_err = t.max(t.abs(preds_sample - preds_forw))\n assert max_err <= 1e-6, f\"Max err is {max_err} {[i for i in range(l) if t.max(t.abs(preds_sample - preds_forw)[:, i, :]) > 1e-6]}\"\n\n # chunked\n x, preds_sample = self.primed_sample(bs, x_prime.clone(), x_cond, y_cond, encoder_kv, get_preds=True, chunk_size=chunk_size)\n assert (x.view(bs, -1)[:,:prime] == x_prime).all(), \"Priming samples don't match\"\n loss, preds_forw = self.forward(x, x_cond, y_cond, encoder_kv, get_preds=True)\n max_err = t.max(t.abs(preds_sample - preds_forw))\n assert max_err <= 1e-6, f\"Max err is {max_err} {[i for i in range(l) if t.max(t.abs(preds_sample - preds_forw)[:, i, :]) > 1e-6]}\"\n\n\ndef test_prior(input_shape, encoder_dims, blocks, heads, chunk_size):\n bins = 512\n width = 32\n depth = 2\n prime_len = encoder_dims\n for x_cond in [True, False]:\n for y_cond in [True, False]:\n for attn_order in [0,2,6,12]:\n prior = ConditionalAutoregressive2D(input_shape, bins,\n width=width, depth=depth, heads=heads,\n attn_order=attn_order, blocks=blocks,\n x_cond=x_cond, y_cond=y_cond,\n encoder_dims=encoder_dims, prime_len=prime_len).cuda()\n prior.training = False\n prior.check_sample(chunk_size)\n print(f\"Checked x_cond: {x_cond}, y_cond: {y_cond}, attn_order: {attn_order}\")\n # prior.apply(_convert_mlp_traced)\n # prior.check_sample()\n # print(f\"Checked traced x_cond: {x_cond}, y_cond: {y_cond}\")\n\n\nif __name__ == '__main__':\n from jukebox.utils.dist_utils import setup_dist_from_mpi\n setup_dist_from_mpi(port=29600)\n test_cases = [\n ((6144,), 384, 64, 2, 23),\n ((6144,), 384, 64, 2, 8),\n ((8192,), 512, 128, 2, 16),\n ]\n for test_case in test_cases:\n test_prior(*test_case)\n" }, { "path": "jukebox/prior/conditioners.py", "content": "import torch as t\nimport torch.nn as nn\n\nfrom jukebox.transformer.ops import LayerNorm\nfrom jukebox.vqvae.encdec import DecoderConvBock\nfrom jukebox.utils.torch_utils import assert_shape\n\nclass Conditioner(nn.Module):\n def __init__(self, input_shape, bins, down_t, stride_t, out_width, init_scale, zero_out, res_scale, **block_kwargs):\n super().__init__()\n self.x_shape = input_shape\n\n # Embedding\n self.width = out_width\n self.x_emb = nn.Embedding(bins, out_width)\n nn.init.normal_(self.x_emb.weight, std=0.02 * init_scale)\n\n # Conditioner\n self.cond = DecoderConvBock(self.width, self.width, down_t, stride_t, **block_kwargs, zero_out=zero_out, res_scale=res_scale)\n self.ln = LayerNorm(self.width)\n\n def preprocess(self, x):\n x = x.permute(0,2,1) # NTC -> NCT\n return x\n\n def postprocess(self, x):\n x = x.permute(0,2,1) # NCT -> NTC\n return x\n\n def forward(self, x, x_cond=None):\n N = x.shape[0]\n assert_shape(x, (N, *self.x_shape))\n if x_cond is not None:\n assert_shape(x_cond, (N, *self.x_shape, self.width))\n else:\n x_cond = 0.0\n # Embed x\n x = x.long()\n x = self.x_emb(x)\n assert_shape(x, (N, *self.x_shape, self.width))\n x = x + x_cond\n\n # Run conditioner\n x = self.preprocess(x)\n x = self.cond(x)\n x = self.postprocess(x)\n x = self.ln(x)\n return x\n\ndef flip(x):\n def _flip(x):\n return x.permute(0,2,1).contiguous()\n if isinstance(x, (list, tuple)):\n return [flip(z) for z in x]\n return _flip(x)\n\nclass SimpleEmbedding(nn.Module):\n def __init__(self, bins, out_width, init_scale):\n super().__init__()\n self.bins = bins\n self.emb = nn.Embedding(bins, out_width)\n nn.init.normal_(self.emb.weight, std=0.01 * init_scale)\n\n def forward(self, y):\n assert len(y.shape) == 2, f\"Expected shape with 2 dims, got {y.shape}\"\n assert isinstance(y, t.cuda.LongTensor), f\"Expected dtype {t.cuda.LongTensor}, got {y.dtype}\"\n assert (0 <= y).all() and (y < self.bins).all(), f\"Bins {self.bins}, got label {y}\"\n return self.emb(y)\n\nclass RangeEmbedding(nn.Module):\n # Interpolating\n # Interpolate so that [pos_start, pos_end] <-> position tensor of length n_ctx\n #\n # Binning\n # For each pos in position tensor, find its bin\n # [start,end) mapped to [0,1,...,bins-1]\n # [start,end) -> [0,1) -> [0, bins) -> floor -> [0,...,bins-1]\n # NOTE: Open ended interval on right, so start <= pos < end, not <= end\n def __init__(self, n_time, bins, range, out_width, init_scale, clamp=False):\n super().__init__()\n self.n_time = n_time\n self.bins = bins\n self.emb = nn.Embedding(bins, out_width)\n nn.init.normal_(self.emb.weight, std=0.01 * init_scale)\n self.pos_min, self.pos_max = range\n self.clamp = clamp\n\n def forward(self, pos_start, pos_end=None):\n # Check if [pos_start,pos_end] in [pos_min, pos_max)\n assert len(pos_start.shape) == 2, f\"Expected shape with 2 dims, got {pos_start.shape}\"\n assert (self.pos_min <= pos_start).all() and (pos_start < self.pos_max).all(), f\"Range is [{self.pos_min},{self.pos_max}), got {pos_start}\"\n pos_start = pos_start.float()\n if pos_end is not None:\n assert len(pos_end.shape) == 2, f\"Expected shape with 2 dims, got {pos_end.shape}\"\n if self.clamp:\n pos_end = pos_end.clamp(self.pos_min, self.pos_max)\n assert (self.pos_min <= pos_end).all() and (pos_end <= self.pos_max).all(), f\"Range is [{self.pos_min},{self.pos_max}), got {pos_end}\"\n pos_end = pos_end.float()\n # Interpolate so that [pos_start, ..., pos_end] <-> position tensor of length n_ctx\n n_time = self.n_time\n if n_time != 1:\n assert pos_end is not None\n interpolation = (t.arange(0, n_time, dtype=t.float, device='cuda').view(1,n_time)/n_time)\n position = pos_start + (pos_end - pos_start)*interpolation\n else:\n position = pos_start\n\n # Bin each value to bins\n normalised_position = (position - self.pos_min) / (self.pos_max - self.pos_min) # [0,1)\n bins = (self.bins * normalised_position).floor().long().detach() # [0,1) -> [0,1..,bins) -> [0,1...,bins-1]\n return self.emb(bins)\n\nclass LabelConditioner(nn.Module):\n def __init__(self, y_bins, t_bins, sr, min_duration, max_duration, n_time, out_width, init_scale, max_bow_genre_size, include_time_signal):\n super().__init__()\n self.n_time = n_time\n self.out_width = out_width\n assert len(y_bins) == 2, f\"Expecting (genre, artist) bins, got {y_bins}\"\n bow_genre_bins, artist_bins = y_bins\n self.max_bow_genre_size = max_bow_genre_size\n self.bow_genre_emb = SimpleEmbedding(bow_genre_bins, out_width, init_scale)\n self.artist_emb = SimpleEmbedding(artist_bins, out_width, init_scale)\n self.include_time_signal = include_time_signal\n if self.include_time_signal:\n t_ranges = ((min_duration * sr, max_duration * sr), # Total length\n (0.0, max_duration * sr), # Absolute pos\n (0.0, 1.0)) # Relative pos\n assert len(t_ranges) == 3, f\"Expecting (total, absolute, relative) ranges, got {t_ranges}\"\n total_length_range, absolute_pos_range, relative_pos_range = t_ranges\n self.total_length_emb = RangeEmbedding(1, t_bins, total_length_range, out_width, init_scale)\n self.absolute_pos_emb = RangeEmbedding(n_time, t_bins, absolute_pos_range, out_width, init_scale)\n self.relative_pos_emb = RangeEmbedding(n_time, t_bins, relative_pos_range, out_width, init_scale, clamp=True)\n\n def forward(self, y):\n assert len(y.shape) == 2, f\"Expected shape with 2 dims, got {y.shape}\"\n assert y.shape[-1] == 4 + self.max_bow_genre_size, f\"Expected shape (N,{4 + self.max_bow_genre_size}), got {y.shape}\"\n assert isinstance(y, t.cuda.LongTensor), f\"Expected dtype {t.cuda.LongTensor}, got {y.dtype}\"\n N = y.shape[0]\n total_length, offset, length, artist, genre = y[:,0:1], y[:,1:2], y[:,2:3], y[:,3:4], y[:,4:]\n\n # Start embedding of length 1\n artist_emb = self.artist_emb(artist)\n # Empty genre slots are denoted by -1. We mask these out.\n mask = (genre >= 0).float().unsqueeze(2)\n genre_emb = (self.bow_genre_emb(genre.clamp(0)) * mask).sum(dim=1, keepdim=True)\n start_emb = genre_emb + artist_emb\n assert_shape(start_emb, (N, 1, self.out_width))\n\n # Pos embedding of length n_ctx\n if self.include_time_signal:\n start, end = offset, offset + length\n total_length, start, end = total_length.float(), start.float(), end.float()\n pos_emb = self.total_length_emb(total_length) + self.absolute_pos_emb(start, end) + self.relative_pos_emb(start/total_length, end/total_length)\n assert_shape(pos_emb, (N, self.n_time, self.out_width))\n else:\n pos_emb = None\n return start_emb, pos_emb" }, { "path": "jukebox/prior/prior.py", "content": "import numpy as np\nimport torch as t\nimport torch.nn as nn\nimport jukebox.utils.dist_adapter as dist\n\nfrom jukebox.transformer.ops import LayerNorm\nfrom jukebox.prior.autoregressive import ConditionalAutoregressive2D\nfrom jukebox.prior.conditioners import Conditioner, LabelConditioner\nfrom jukebox.data.labels import EmptyLabeller, Labeller\n\nfrom jukebox.utils.torch_utils import assert_shape\nfrom jukebox.utils.dist_utils import print_once\nfrom jukebox.vqvae.vqvae import calculate_strides\n\n\n\"\"\"\nModel the prior on vq codes conditioned on timing, artist, genre, lyrics and codes from levels above. \nTo condition on the timing, genre and artist, we use the LabelConditioner class\nTo condition on the codes from the level above, we use the Conditioner class\nTo condition on lyrics, we allow two types of priors:\n- Separate Encoder Decoder: This is the usual encoder-decoder style transformer. The encoder transformer autoregressively \nmodels the lyrics, and we use its last layer to produce keys/values that are attened to by the decoder transformer\n- Single Encoder Decoder: This is a simplification where we combine them into a single model. We merge the text vocab \nand VQ vocab into a single large vocab, and the lyric tokens and VQ tokens into a single longer sequence of tokens which \nwe autoregressively model together.\n\"\"\"\nclass SimplePrior(nn.Module):\n def __init__(self, z_shapes, l_bins, encoder, decoder, level,\n downs_t, strides_t, labels, prior_kwargs, x_cond_kwargs, y_cond_kwargs,\n prime_kwargs, copy_input, labels_v3=False,\n merged_decoder=False, single_enc_dec=False):\n super().__init__()\n\n self.use_tokens = prime_kwargs.pop('use_tokens')\n self.n_tokens = prime_kwargs.pop('n_tokens')\n self.prime_loss_fraction = prime_kwargs.pop('prime_loss_fraction')\n\n self.copy_input = copy_input\n if self.copy_input:\n prime_kwargs['bins'] = l_bins\n\n self.z_shapes = z_shapes\n self.levels = len(self.z_shapes)\n\n self.z_shape = self.z_shapes[level]\n\n self.level = level\n assert level < self.levels, f\"Total levels {self.levels}, got level {level}\"\n\n self.l_bins = l_bins\n\n # Passing functions instead of the vqvae module to avoid getting params\n self.encoder = encoder\n self.decoder = decoder\n\n # X conditioning\n self.x_cond = (level != (self.levels - 1))\n self.cond_level = level + 1\n\n # Y conditioning\n self.y_cond = labels\n\n self.single_enc_dec = single_enc_dec\n # X conditioning\n if self.x_cond:\n self.conditioner_blocks = nn.ModuleList()\n conditioner_block = lambda _level: Conditioner(input_shape=z_shapes[_level],\n bins=l_bins,\n down_t=downs_t[_level],\n stride_t=strides_t[_level],\n **x_cond_kwargs)\n if dist.get_rank() == 0: print(f\"Conditioning on 1 above level(s)\")\n self.conditioner_blocks.append(conditioner_block(self.cond_level))\n\n # Y conditioning\n if self.y_cond:\n self.n_time = self.z_shape[0] # Assuming STFT=TF order and raw=T1 order, so T is first dim\n self.y_emb = LabelConditioner(n_time=self.n_time,include_time_signal=not self.x_cond,**y_cond_kwargs)\n\n # Lyric conditioning\n if single_enc_dec:\n # Single encoder-decoder transformer\n self.prior_shapes = [(self.n_tokens,), prior_kwargs.pop('input_shape')]\n self.prior_bins = [prime_kwargs['bins'], prior_kwargs.pop('bins')]\n self.prior_dims = [np.prod(shape) for shape in self.prior_shapes]\n self.prior_bins_shift = np.cumsum([0, *self.prior_bins])[:-1]\n self.prior_width = prior_kwargs['width']\n print_once(f'Creating cond. autoregress with prior bins {self.prior_bins}, ')\n print_once(f'dims {self.prior_dims}, ')\n print_once(f'shift {self.prior_bins_shift}')\n print_once(f'input shape {sum(self.prior_dims)}')\n print_once(f'input bins {sum(self.prior_bins)}')\n print_once(f'Self copy is {self.copy_input}')\n\n self.prime_loss_dims, self.gen_loss_dims = self.prior_dims[0], self.prior_dims[1]\n self.total_loss_dims = self.prime_loss_dims + self.gen_loss_dims\n self.prior = ConditionalAutoregressive2D(input_shape=(sum(self.prior_dims),),\n bins=sum(self.prior_bins),\n x_cond=(self.x_cond or self.y_cond), y_cond=True,\n prime_len=self.prime_loss_dims,\n **prior_kwargs)\n\n else:\n # Separate encoder-decoder transformer\n if self.n_tokens != 0 and self.use_tokens:\n from jukebox.transformer.ops import Conv1D\n prime_input_shape = (self.n_tokens,)\n self.prime_loss_dims = np.prod(prime_input_shape)\n self.prime_acts_width, self.prime_state_width = prime_kwargs['width'], prior_kwargs['width']\n self.prime_prior = ConditionalAutoregressive2D(input_shape=prime_input_shape, x_cond=False, y_cond=False,\n only_encode=True,\n **prime_kwargs)\n self.prime_state_proj = Conv1D(self.prime_acts_width, self.prime_state_width, init_scale=prime_kwargs['init_scale'])\n self.prime_state_ln = LayerNorm(self.prime_state_width)\n self.prime_bins = prime_kwargs['bins']\n self.prime_x_out = nn.Linear(self.prime_state_width, self.prime_bins, bias=False)\n nn.init.normal_(self.prime_x_out.weight, std=0.02 * prior_kwargs['init_scale'])\n else:\n self.prime_loss_dims = 0\n self.gen_loss_dims = np.prod(self.z_shape)\n self.total_loss_dims = self.prime_loss_dims + self.gen_loss_dims\n self.prior = ConditionalAutoregressive2D(x_cond=(self.x_cond or self.y_cond), y_cond=self.y_cond,\n encoder_dims = self.prime_loss_dims, merged_decoder=merged_decoder,\n **prior_kwargs)\n\n self.n_ctx = self.gen_loss_dims\n self.downsamples = calculate_strides(strides_t, downs_t)\n self.cond_downsample = self.downsamples[level+1] if level != self.levels - 1 else None\n self.raw_to_tokens = np.prod(self.downsamples[:level+1])\n self.sample_length = self.n_ctx*self.raw_to_tokens\n if labels:\n self.labels_v3 = labels_v3\n self.labeller = Labeller(self.y_emb.max_bow_genre_size, self.n_tokens, self.sample_length, v3=self.labels_v3)\n else:\n self.labeller = EmptyLabeller()\n\n print(f\"Level:{level}, Cond downsample:{self.cond_downsample}, Raw to tokens:{self.raw_to_tokens}, Sample length:{self.sample_length}\")\n\n\n def get_y(self, labels, start, get_indices=False):\n if isinstance(self.labeller, EmptyLabeller):\n return None\n y = labels['y'].clone()\n\n # Set sample_length to match this level\n y[:, 2] = int(self.sample_length)\n\n # Set offset\n y[:, 1:2] = y[:, 1:2] + int(start * self.raw_to_tokens)\n\n # Set lyric tokens\n indices = self.labeller.set_y_lyric_tokens(y, labels)\n if get_indices:\n return y, indices\n else:\n return y\n\n def get_z_conds(self, zs, start, end):\n if self.level != self.levels - 1:\n assert start % self.cond_downsample == end % self.cond_downsample == 0\n z_cond = zs[self.level + 1][:,start//self.cond_downsample:end//self.cond_downsample]\n assert z_cond.shape[1] == self.n_ctx//self.cond_downsample\n z_conds = [z_cond]\n else:\n z_conds = None\n return z_conds\n\n def prior_preprocess(self, xs, conds):\n N = xs[0].shape[0]\n for i in range(len(xs)):\n x, shape, dims = xs[i], self.prior_shapes[i], self.prior_dims[i]\n bins, bins_shift = int(self.prior_bins[i]), int(self.prior_bins_shift[i])\n assert isinstance(x, t.cuda.LongTensor), x\n assert (0 <= x).all() and (x < bins).all()\n #assert_shape(x, (N, *shape))\n xs[i] = (xs[i] + bins_shift).view(N, -1)\n\n for i in range(len(conds)):\n cond, shape, dims = conds[i], self.prior_shapes[i], self.prior_dims[i]\n if cond is not None:\n assert_shape(cond, (N, dims, self.prior_width))\n else:\n conds[i] = t.zeros((N, dims, self.prior_width), dtype=t.float, device='cuda')\n\n return t.cat(xs, dim=1), t.cat(conds, dim=1)\n\n def prior_postprocess(self, z):\n N = z.shape[0]\n dims = (self.prior_dims[0], z.shape[1] - self.prior_dims[0])\n # xs = list(t.split(z, self.prior_dims, dim=1))\n xs = list(t.split(z, dims, dim=1))\n\n for i in range(len(xs)):\n # x, shape, dims, bins, bins_shift = xs[i], self.prior_shapes[i], self.prior_dims[i], self.prior_bins[i], self.prior_bins_shift[i]\n # assert_shape(x, (N, dims))\n shape = self.prior_shapes[i]\n bins, bins_shift = int(self.prior_bins[i]), int(self.prior_bins_shift[i])\n # xs[i] = (xs[i] - bins_shift).view(N, *shape) #view(N, -1, *shape[1:])\n xs[i] = (xs[i] - bins_shift).view(N, -1, *shape[1:])\n xs[i] = t.clamp(xs[i], min=0) # If not masking loss, model may have generated lyric/midi tokens which are now shifted <0 by bin_shift\n assert (xs[i] < bins).all(), f'rank: {dist.get_rank()}, bins: {bins}, dims {dims}, shape {shape}, prior_shape {self.prior_shapes}, bins_shift {bins_shift}, xs[i]: {xs[i]}'\n\n return xs[-1]\n\n def x_emb(self, z_conds):\n z_conds = z_conds[:self.cond_level - self.level]\n assert len(z_conds) == len(self.conditioner_blocks) == self.cond_level - self.level, f\"Expected {len(z_conds)} == {len(self.conditioner_blocks)} == {self.cond_level} - {self.level}\"\n x_cond = None\n for z_cond, conditioner_block in reversed(list(zip(z_conds, self.conditioner_blocks))):\n x_cond = conditioner_block(z_cond, x_cond)\n return x_cond\n\n def encode(self, x, start_level=None, end_level=None, bs_chunks=1):\n if start_level == None:\n start_level = self.level\n if end_level == None:\n end_level = self.levels\n # Get latents\n with t.no_grad():\n zs = self.encoder(x, start_level=start_level, end_level=end_level, bs_chunks=bs_chunks)\n return zs\n\n def decode(self, zs, start_level=None, end_level=None, bs_chunks=1):\n if start_level == None:\n start_level = self.level\n if end_level == None:\n end_level = self.levels\n\n assert len(zs) == end_level - start_level\n with t.no_grad():\n x_out = self.decoder(zs, start_level=start_level, end_level=end_level, bs_chunks=bs_chunks)\n return x_out\n\n def get_cond(self, z_conds, y):\n if y is not None:\n assert y.shape[1] == 4 + self.y_emb.max_bow_genre_size + self.n_tokens, f\"Expected {4} + {self.y_emb.max_bow_genre_size} + {self.n_tokens}, got {y.shape[1]}\"\n n_labels = y.shape[1] - self.n_tokens\n y, prime = y[:,:n_labels], y[:,n_labels:]\n else:\n y, prime = None, None\n y_cond, y_pos = self.y_emb(y) if self.y_cond else (None, None)\n x_cond = self.x_emb(z_conds) if self.x_cond else y_pos\n return x_cond, y_cond, prime\n\n def sample(self, n_samples, z=None, z_conds=None, y=None, fp16=False, temp=1.0, top_k=0, top_p=0.0,\n chunk_size=None, sample_tokens=None):\n N = n_samples\n if z is not None: assert z.shape[0] == N, f\"Expected shape ({N},**), got shape {z.shape}\"\n if y is not None: assert y.shape[0] == N, f\"Expected shape ({N},**), got shape {y.shape}\"\n if z_conds is not None:\n for z_cond in z_conds:\n assert z_cond.shape[0] == N, f\"Expected shape ({N},**), got shape {z_cond.shape}\"\n\n no_past_context = (z is None or z.shape[1] == 0)\n if dist.get_rank() == 0:\n name = {True: 'Ancestral', False: 'Primed'}[no_past_context]\n print(f\"{name} sampling {n_samples} samples with temp={temp}, top_k={top_k}, top_p={top_p}\")\n\n with t.no_grad():\n # Currently x_cond only uses immediately above layer\n x_cond, y_cond, prime = self.get_cond(z_conds, y)\n if self.single_enc_dec:\n # assert chunk_size % self.prime_loss_dims == 0. TODO: Check if needed\n if no_past_context:\n z, x_cond = self.prior_preprocess([prime], [None, x_cond])\n else:\n z, x_cond = self.prior_preprocess([prime, z], [None, x_cond])\n if sample_tokens is not None:\n sample_tokens += self.n_tokens\n z = self.prior.primed_sample(n_samples, z, x_cond, y_cond, fp16=fp16, temp=temp,\n top_k=top_k, top_p=top_p, chunk_size=chunk_size, sample_tokens=sample_tokens)\n z = self.prior_postprocess(z)\n else:\n encoder_kv = self.get_encoder_kv(prime, fp16=fp16, sample=True)\n if no_past_context:\n z = self.prior.sample(n_samples, x_cond, y_cond, encoder_kv, fp16=fp16, temp=temp, top_k=top_k,\n top_p=top_p, sample_tokens=sample_tokens)\n else:\n z = self.prior.primed_sample(n_samples, z, x_cond, y_cond, encoder_kv, fp16=fp16, temp=temp,\n top_k=top_k, top_p=top_p, chunk_size=chunk_size, sample_tokens=sample_tokens)\n if sample_tokens is None:\n assert_shape(z, (N, *self.z_shape))\n return z\n\n def get_encoder_kv(self, prime, fp16=False, sample=False):\n if self.n_tokens != 0 and self.use_tokens:\n if sample:\n self.prime_prior.cuda()\n N = prime.shape[0]\n prime_acts = self.prime_prior(prime, None, None, None, fp16=fp16)\n assert_shape(prime_acts, (N, self.prime_loss_dims, self.prime_acts_width))\n assert prime_acts.dtype == t.float, f'Expected t.float, got {prime_acts.dtype}'\n encoder_kv = self.prime_state_ln(self.prime_state_proj(prime_acts))\n assert encoder_kv.dtype == t.float, f'Expected t.float, got {encoder_kv.dtype}'\n if sample:\n self.prime_prior.cpu()\n if fp16:\n encoder_kv = encoder_kv.half()\n else:\n encoder_kv = None\n return encoder_kv\n\n def get_prime_loss(self, encoder_kv, prime_t):\n if self.use_tokens:\n encoder_kv = encoder_kv.float()\n encoder_kv = self.prime_x_out(encoder_kv)\n prime_loss = nn.functional.cross_entropy(encoder_kv.view(-1, self.prime_bins), prime_t.view(-1)) / np.log(2.)\n else:\n prime_loss = t.tensor(0.0, device='cuda')\n return prime_loss\n\n def z_forward(self, z, z_conds=[], y=None, fp16=False, get_preds=False, get_attn_weights=False):\n \"\"\"\n Arguments:\n get_attn_weights (bool or set): Makes forward prop dump\n self-attention softmaxes to self.prior.transformer.ws. Either a\n set of layer indices indicating which layers to store, or a\n boolean value indicating whether to dump all.\n \"\"\"\n assert isinstance(get_attn_weights, (bool, set))\n if get_attn_weights:\n self.prior.transformer.set_record_attn(get_attn_weights)\n x_cond, y_cond, prime = self.get_cond(z_conds, y)\n if self.copy_input:\n prime = z[:,:self.n_tokens]\n if self.single_enc_dec:\n z, x_cond = self.prior_preprocess([prime, z], [None, x_cond])\n (prime_loss, gen_loss), preds = self.prior(z, x_cond, y_cond, fp16=fp16, get_sep_loss=True, get_preds=get_preds)\n else:\n encoder_kv = self.get_encoder_kv(prime, fp16=fp16)\n prime_loss = self.get_prime_loss(encoder_kv, prime)\n gen_loss, preds = self.prior(z, x_cond, y_cond, encoder_kv, fp16=fp16, get_preds=get_preds)\n loss = (self.prime_loss_fraction*prime_loss*self.prime_loss_dims/self.total_loss_dims) + \\\n (gen_loss*self.gen_loss_dims/self.total_loss_dims)\n metrics=dict(bpd=gen_loss.clone().detach(), prime_loss=prime_loss.clone().detach(),\n gen_loss=gen_loss.clone().detach())\n if get_preds:\n metrics[\"preds\"] = preds.clone().detach()\n if get_attn_weights:\n ws = self.prior.transformer.ws\n self.prior.transformer.set_record_attn(False)\n return ws\n else:\n return loss, metrics\n\n def forward(self, x, y=None, fp16=False, decode=False, get_preds=False):\n bs = x.shape[0]\n z, *z_conds = self.encode(x, bs_chunks=bs)\n loss, metrics = self.z_forward(z=z, z_conds=z_conds, y=y, fp16=fp16, get_preds=get_preds)\n if decode:\n x_out = self.decode([z, *z_conds])\n else:\n x_out = None\n return x_out, loss, metrics\n" }, { "path": "jukebox/sample.py", "content": "import os\nimport torch as t\nimport jukebox.utils.dist_adapter as dist\n\nfrom jukebox.hparams import Hyperparams\nfrom jukebox.data.labels import EmptyLabeller\nfrom jukebox.utils.torch_utils import empty_cache\nfrom jukebox.utils.audio_utils import save_wav, load_audio\nfrom jukebox.make_models import make_model\nfrom jukebox.align import get_alignment\nfrom jukebox.save_html import save_html\nfrom jukebox.utils.sample_utils import split_batch, get_starts\nfrom jukebox.utils.dist_utils import print_once\nimport fire\n\n# Sample a partial window of length= prior.n_ctx:\n for start in get_starts(total_length, prior.n_ctx, hop_length):\n zs = sample_single_window(zs, labels, sampling_kwargs, level, prior, start, hps)\n else:\n zs = sample_partial_window(zs, labels, sampling_kwargs, level, prior, total_length, hps)\n return zs\n\n# Sample multiple levels\ndef _sample(zs, labels, sampling_kwargs, priors, sample_levels, hps):\n alignments = None\n for level in reversed(sample_levels):\n prior = priors[level]\n prior.cuda()\n empty_cache()\n\n # Set correct total_length, hop_length, labels and sampling_kwargs for level\n assert hps.sample_length % prior.raw_to_tokens == 0, f\"Expected sample_length {hps.sample_length} to be multiple of {prior.raw_to_tokens}\"\n total_length = hps.sample_length//prior.raw_to_tokens\n hop_length = int(hps.hop_fraction[level]*prior.n_ctx)\n zs = sample_level(zs, labels[level], sampling_kwargs[level], level, prior, total_length, hop_length, hps)\n\n prior.cpu()\n empty_cache()\n\n # Decode sample\n x = prior.decode(zs[level:], start_level=level, bs_chunks=zs[level].shape[0])\n\n if dist.get_world_size() > 1:\n logdir = f\"{hps.name}_rank_{dist.get_rank()}/level_{level}\"\n else:\n logdir = f\"{hps.name}/level_{level}\"\n if not os.path.exists(logdir):\n os.makedirs(logdir)\n t.save(dict(zs=zs, labels=labels, sampling_kwargs=sampling_kwargs, x=x), f\"{logdir}/data.pth.tar\")\n save_wav(logdir, x, hps.sr)\n if alignments is None and priors[-1] is not None and priors[-1].n_tokens > 0 and not isinstance(priors[-1].labeller, EmptyLabeller):\n alignments = get_alignment(x, zs, labels[-1], priors[-1], sampling_kwargs[-1]['fp16'], hps)\n save_html(logdir, x, zs, labels[-1], alignments, hps)\n return zs\n\n# Generate ancestral samples given a list of artists and genres\ndef ancestral_sample(labels, sampling_kwargs, priors, hps):\n sample_levels = list(range(len(priors)))\n zs = [t.zeros(hps.n_samples,0,dtype=t.long, device='cuda') for _ in range(len(priors))]\n zs = _sample(zs, labels, sampling_kwargs, priors, sample_levels, hps)\n return zs\n\n# Continue ancestral sampling from previously saved codes\ndef continue_sample(zs, labels, sampling_kwargs, priors, hps):\n sample_levels = list(range(len(priors)))\n zs = _sample(zs, labels, sampling_kwargs, priors, sample_levels, hps)\n return zs\n\n# Upsample given already generated upper-level codes\ndef upsample(zs, labels, sampling_kwargs, priors, hps):\n sample_levels = list(range(len(priors) - 1))\n zs = _sample(zs, labels, sampling_kwargs, priors, sample_levels, hps)\n return zs\n\n# Prompt the model with raw audio input (dimension: NTC) and generate continuations\ndef primed_sample(x, labels, sampling_kwargs, priors, hps):\n sample_levels = list(range(len(priors)))\n zs = priors[-1].encode(x, start_level=0, end_level=len(priors), bs_chunks=x.shape[0])\n zs = _sample(zs, labels, sampling_kwargs, priors, sample_levels, hps)\n return zs\n\n# Load `duration` seconds of the given audio files to use as prompts\ndef load_prompts(audio_files, duration, hps):\n xs = []\n for audio_file in audio_files:\n x = load_audio(audio_file, sr=hps.sr, duration=duration, offset=0.0, mono=True)\n x = x.T # CT -> TC\n xs.append(x)\n while len(xs) < hps.n_samples:\n xs.extend(xs)\n xs = xs[:hps.n_samples]\n x = t.stack([t.from_numpy(x) for x in xs])\n x = x.to('cuda', non_blocking=True)\n return x\n\n# Load codes from previous sampling run\ndef load_codes(codes_file, duration, priors, hps):\n data = t.load(codes_file, map_location='cpu')\n zs = [z.cuda() for z in data['zs']]\n assert zs[-1].shape[0] == hps.n_samples, f\"Expected bs = {hps.n_samples}, got {zs[-1].shape[0]}\"\n del data\n if duration is not None:\n # Cut off codes to match duration\n top_raw_to_tokens = priors[-1].raw_to_tokens\n assert duration % top_raw_to_tokens == 0, f\"Cut-off duration {duration} not an exact multiple of top_raw_to_tokens\"\n assert duration//top_raw_to_tokens <= zs[-1].shape[1], f\"Cut-off tokens {duration//priors[-1].raw_to_tokens} longer than tokens {zs[-1].shape[1]} in saved codes\"\n zs = [z[:,:duration//prior.raw_to_tokens] for z, prior in zip(zs, priors)]\n return zs\n\n# Generate and save samples, alignment, and webpage for visualization.\ndef save_samples(model, device, hps, sample_hps):\n print(hps)\n from jukebox.lyricdict import poems, gpt_2_lyrics\n vqvae, priors = make_model(model, device, hps)\n\n assert hps.sample_length//priors[-2].raw_to_tokens >= priors[-2].n_ctx, f\"Upsampling needs atleast one ctx in get_z_conds. Please choose a longer sample length\"\n\n total_length = hps.total_sample_length_in_seconds * hps.sr\n offset = 0\n\n # Set artist/genre/lyrics for your samples here!\n # We used different label sets in our models, but you can write the human friendly names here and we'll map them under the hood for each model.\n # For the 5b/5b_lyrics model and the upsamplers, labeller will look up artist and genres in v2 set. (after lowercasing, removing non-alphanumerics and collapsing whitespaces to _).\n # For the 1b_lyrics top level, labeller will look up artist and genres in v3 set (after lowercasing).\n metas = [dict(artist = \"Alan Jackson\",\n genre = \"Country\",\n lyrics = poems['ozymandias'],\n total_length=total_length,\n offset=offset,\n ),\n dict(artist=\"Joe Bonamassa\",\n genre=\"Blues Rock\",\n lyrics=gpt_2_lyrics['hottub'],\n total_length=total_length,\n offset=offset,\n ),\n dict(artist=\"Frank Sinatra\",\n genre=\"Classic Pop\",\n lyrics=gpt_2_lyrics['alone'],\n total_length=total_length,\n offset=offset,\n ),\n dict(artist=\"Ella Fitzgerald\",\n genre=\"Jazz\",\n lyrics=gpt_2_lyrics['count'],\n total_length=total_length,\n offset=offset,\n ),\n dict(artist=\"Céline Dion\",\n genre=\"Pop\",\n lyrics=gpt_2_lyrics['darkness'],\n total_length=total_length,\n offset=offset,\n ),\n ]\n while len(metas) < hps.n_samples:\n metas.extend(metas)\n metas = metas[:hps.n_samples]\n\n labels = [prior.labeller.get_batch_labels(metas, 'cuda') for prior in priors]\n for label in labels:\n assert label['y'].shape[0] == hps.n_samples\n\n lower_level_chunk_size = 32\n lower_level_max_batch_size = 16\n if model == '1b_lyrics':\n chunk_size = 32\n max_batch_size = 16\n else:\n chunk_size = 16\n max_batch_size = 3\n sampling_kwargs = [dict(temp=0.99, fp16=True, chunk_size=lower_level_chunk_size, max_batch_size=lower_level_max_batch_size),\n dict(temp=0.99, fp16=True, chunk_size=lower_level_chunk_size, max_batch_size=lower_level_max_batch_size),\n dict(temp=0.99, fp16=True, chunk_size=chunk_size, max_batch_size=max_batch_size)]\n\n if sample_hps.mode == 'ancestral':\n ancestral_sample(labels, sampling_kwargs, priors, hps)\n elif sample_hps.mode in ['continue', 'upsample']:\n assert sample_hps.codes_file is not None\n top_raw_to_tokens = priors[-1].raw_to_tokens\n if sample_hps.prompt_length_in_seconds is not None:\n duration = (int(sample_hps.prompt_length_in_seconds * hps.sr) // top_raw_to_tokens) * top_raw_to_tokens\n else:\n duration = None\n zs = load_codes(sample_hps.codes_file, duration, priors, hps)\n if sample_hps.mode == 'continue':\n continue_sample(zs, labels, sampling_kwargs, priors, hps)\n elif sample_hps.mode == 'upsample':\n upsample(zs, labels, sampling_kwargs, priors, hps)\n elif sample_hps.mode == 'primed':\n assert sample_hps.audio_file is not None\n assert sample_hps.prompt_length_in_seconds is not None\n audio_files = sample_hps.audio_file.split(',')\n top_raw_to_tokens = priors[-1].raw_to_tokens\n duration = (int(sample_hps.prompt_length_in_seconds * hps.sr) // top_raw_to_tokens) * top_raw_to_tokens\n x = load_prompts(audio_files, duration, hps)\n primed_sample(x, labels, sampling_kwargs, priors, hps)\n else:\n raise ValueError(f'Unknown sample mode {sample_hps.mode}.')\n\n\ndef run(model, mode='ancestral', codes_file=None, audio_file=None, prompt_length_in_seconds=None, port=29500, **kwargs):\n from jukebox.utils.dist_utils import setup_dist_from_mpi\n rank, local_rank, device = setup_dist_from_mpi(port=port)\n hps = Hyperparams(**kwargs)\n sample_hps = Hyperparams(dict(mode=mode, codes_file=codes_file, audio_file=audio_file, prompt_length_in_seconds=prompt_length_in_seconds))\n\n with t.no_grad():\n save_samples(model, device, hps, sample_hps)\n\nif __name__ == '__main__':\n fire.Fire(run)\n" }, { "path": "jukebox/save_html.py", "content": "import os\nimport json\nimport numpy as np\nfrom PIL import Image, ImageFilter\nimport soundfile\n\ndef save_html(logdir, x, zs, labels, alignments, hps):\n level = hps.levels - 1 # Top level used\n z = zs[level]\n bs, total_length = z.shape[0], z.shape[1]\n\n with open(f'{logdir}/index.html', 'w') as html:\n print(f\"{logdir}\",\n file=html)\n print(\"\", file=html)\n\n for item in range(bs):\n data = dict(wav=x[item].cpu().numpy(), sr=hps.sr,\n info=labels['info'][item],\n total_length=total_length,\n total_tokens=len(labels['info'][item]['full_tokens']),\n alignment=alignments[item] if alignments is not None else None)\n item_dir = f'{logdir}/item_{item}'\n _save_item_html(item_dir, item, item, data)\n print(f\"\", file=html)\n print(\"\", file=html) \n\ndef _save_item_html(item_dir, item_id, item_name, data):\n # replace gs:// with /root/samples/\n\n # an html for each sample. Main html has a selector to get us id of this?\n if not os.path.exists(item_dir):\n os.makedirs(item_dir)\n\n with open(f'{item_dir}/index.html', 'w') as html:\n print(f\"{item_name}\",\n file=html)\n print(\"\", file=html)\n total_length = data['total_length']\n total_tokens = data['total_tokens']\n alignment = data['alignment']\n lyrics = data[\"info\"][\"lyrics\"]\n wav, sr = data['wav'], data['sr']\n genre, artist = data[\"info\"][\"genre\"], data[\"info\"][\"artist\"]\n\n # Strip unused columns\n if alignment is not None:\n assert alignment.shape == (total_length, total_tokens)\n assert len(lyrics) == total_tokens, f'Total_tokens: {total_tokens}, Lyrics Len: {len(lyrics)}. Lyrics: {lyrics}'\n max_attn_at_token = np.max(alignment, axis=0)\n assert len(max_attn_at_token) == total_tokens\n for token in reversed(range(total_tokens)):\n if max_attn_at_token[token] > 0:\n break\n alignment = alignment[:,:token+1]\n lyrics = lyrics[:token+1]\n total_tokens = token+1\n\n # Small alignment image\n im = Image.fromarray(np.uint8(alignment * 255)).resize((512, 1024)).transpose(Image.ROTATE_90)\n img_src = f'align.png'\n im.save(f'{item_dir}/{img_src}')\n print(f\"\", file=html)\n\n # Smaller alignment json for animation\n total_alignment_length = total_length // 16\n alignment = Image.fromarray(np.uint8(alignment * 255)).resize((total_tokens, total_alignment_length))\n alignment = alignment.filter(ImageFilter.GaussianBlur(radius=1.5))\n alignment = np.asarray(alignment).tolist()\n align_src = f'align.json'\n with open(f'{item_dir}/{align_src}', 'w') as f:\n json.dump(alignment, f)\n\n # Audio\n wav_src = f'audio.wav'\n soundfile.write(f'{item_dir}/{wav_src}', wav, samplerate=sr, format='wav')\n print(f\"\", file=html)\n\n\n # Labels and Lyrics\n print(f\"
\", end=\"\", file=html)\n        print(f\"
Artist {artist}, Genre {genre}
\", file=html)\n        lyrics = [c for c in lyrics]  # already characters actually\n        lyrics = [''] + lyrics[:-1]  # input lyrics are shifted by 1\n        for i, c in enumerate(lyrics):\n            print(f\"{c}\", end=\"\", file=html)\n        print(f\"
\", file=html)\n with open(f'{item_dir}/lyrics.json', 'w') as f:\n json.dump(lyrics, f)\n\n if alignment is not None:\n # JS for alignment animation\n print(\"\"\"\"\"\", file=html)\n print(\"\", file=html)\n" }, { "path": "jukebox/tests/test_sample.py", "content": "import torch as t\nimport numpy as np\nfrom jukebox.sample import sample_level\nfrom jukebox.utils.torch_utils import assert_shape\nfrom jukebox.hparams import Hyperparams\n\ndef repeat(x, n, dim):\n if dim == -1:\n dim = len(x.shape) - 1\n return x.reshape(int(np.prod(x.shape[:dim+1])), 1, int(np.prod(x.shape[dim+1:]))).repeat(1,n,1).reshape(*x.shape[:dim], n * x.shape[dim], *x.shape[dim+1:])\n\n# Tests\nclass DummyPrior:\n def __init__(self, n_ctx, level, levels):\n self.n_ctx = n_ctx\n self.level = level\n self.levels = levels\n self.downsamples = (8,4,4)\n self.cond_downsample = self.downsamples[level+1] if level != self.levels - 1 else None\n self.raw_to_tokens = int(np.prod(self.downsamples[:level+1]))\n self.sample_length = self.n_ctx*self.raw_to_tokens\n\n print(f\"Level:{level}, Cond downsample:{self.cond_downsample}, Raw to tokens:{self.raw_to_tokens}, Sample length:{self.sample_length}\")\n\n def get_y(self, labels, start):\n y = labels['y'].clone()\n # Set sample_length to match this level\n y[:, 2] = self.sample_length\n # Set offset\n y[:, 1:2] = y[:, 1:2] + start * self.raw_to_tokens\n return y\n\n def get_z_conds(self, zs, start, end):\n if self.level != self.levels - 1:\n assert start % self.cond_downsample == end % self.cond_downsample == 0\n z_cond = zs[self.level + 1][:,start//self.cond_downsample:end//self.cond_downsample]\n assert z_cond.shape[1] == self.n_ctx//self.cond_downsample\n z_conds = [z_cond]\n else:\n z_conds = None\n return z_conds\n\n def ancestral_sample(self, n_samples, z_conds=None, y=None):\n z = t.zeros((n_samples, self.n_ctx), dtype=t.long, device='cuda') + \\\n t.arange(0, self.n_ctx, dtype=t.long, device='cuda').view(1, self.n_ctx)\n\n if z_conds is not None:\n z_cond = z_conds[0]\n assert_shape(z_cond, (n_samples, self.n_ctx // 4))\n assert (z // 4 == repeat(z_cond, 4, 1)).all(), f'z: {z}, z_cond: {z_cond}, diff: {(z // 4) - repeat(z_cond, 4, 1)}'\n return z\n\n def primed_sample(self, n_samples, z, z_conds=None, y=None):\n prime = z.shape[1]\n assert_shape(z, (n_samples, prime))\n start = z[:,-1:] + 1\n z_rest = (t.arange(0, self.n_ctx - prime, dtype=t.long, device='cuda').view(1, self.n_ctx - prime) + start).view(n_samples, self.n_ctx - prime)\n z = t.cat([z, z_rest], dim=1)\n\n if z_conds is not None:\n z_cond = z_conds[0]\n assert_shape(z_cond, (n_samples, self.n_ctx // 4))\n assert (z // 4 == repeat(z_cond, 4, 1)).all(), f'z: {z}, z_cond: {z_cond}, diff: {(z // 4) - repeat(z_cond, 4, 1)}'\n return z\n\n# Sample multiple levels\ndef _sample(zs, labels, priors, sample_levels, hps):\n for level in reversed(sample_levels):\n prior = priors[level]\n # set correct total_length, hop_length and sampling_kwargs for level\n total_length = (hps.sample_length * hps.n_segment)//prior.raw_to_tokens\n hop_length = hps.hop_lengths[level]\n zs = sample_level(zs, labels[level], dict(), level, prior, total_length, hop_length, hps)\n return zs\n\n# Ancestral sample\ndef test_ancestral_sample(labels, priors, hps):\n sample_levels = list(range(hps.levels))\n zs = [t.zeros(hps.n_samples,0,dtype=t.long, device='cuda') for _ in range(hps.levels)]\n zs = _sample(zs, labels, priors, sample_levels, hps)\n\n # Test\n for z in zs:\n total_length = z.shape[1]\n # Check sample\n assert ((z - t.arange(0, total_length, dtype=t.long, device='cuda').view(1, total_length)) == 0).all()\n\n print(\"dummy ancestral sample passed\")\n\ndef test_primed_sample(labels, priors, hps):\n sample_levels = list(range(hps.levels))\n\n start = t.tensor([15, 23, 11, 9], dtype=t.long, device='cuda').view(4, 1)\n\n zs_in = []\n zs = []\n for i in reversed(range(3)):\n n_ctx = 8192*(4**i)\n n_prime = n_ctx // 4\n z_prime = t.arange(0, n_prime, dtype=t.long, device='cuda').view(1, n_prime) % (2*(4**i))\n z_rest = t.randint(-10, -1, size=(1, n_ctx - n_prime), dtype=t.long, device='cuda')\n z_in = t.cat([z_prime, z_rest], dim=1) + (4**i)*start\n zs_in.append(z_in)\n zs.append(z_prime + (4**i)*start)\n\n zs = _sample(zs, labels, priors, sample_levels, hps)\n\n # Test\n for z, z_in in zip(zs, zs_in):\n total_length = z.shape[1]\n prime_length = z.shape[1] // (4 * hps.n_segment)\n # Match prime tokens\n assert (z[:,:prime_length] == z_in[:,:prime_length]).all()\n # Check sample\n z_rest = z[:,prime_length-1:] - z[:,prime_length-1:prime_length]\n assert ((z_rest - t.arange(0, total_length - prime_length + 1, dtype=t.long, device='cuda').view(1, total_length - prime_length + 1)) == 0).all()\n\n print(\"dummy primed sample passed\")\n\ndef check_sample():\n n_ctx = 8192\n n_samples = 4\n levels = 3\n priors = [DummyPrior(n_ctx, level, levels) for level in range(levels)]\n max_total_length, offset, sample_length = 4134368, 0, n_ctx*8*4*4\n y = t.tensor([max_total_length, offset, sample_length, 10, 1, -1, -1, -1, -1], dtype=t.long, device='cuda').view(1, 9).repeat(n_samples, 1)\n labels = [dict(y=y, info=[[]*n_samples]) for level in range(levels)]\n hps = Hyperparams({\n 'levels': 3,\n 'sample_length': sample_length,\n 'n_segment': 2,\n 'n_ctx': n_ctx,\n 'n_tokens': 0,\n 'hop_lengths': [n_ctx//2, n_ctx//2, n_ctx//8],\n 'n_samples': n_samples,\n 'use_tokens': False\n })\n test_ancestral_sample(labels, priors, hps)\n test_primed_sample(labels, priors, hps)\n\ncheck_sample()\n" }, { "path": "jukebox/train.py", "content": "\"\"\"\nAbility to train vq-vae and prior\nFirst try for random inputs\nThen from maestros\n\"\"\"\nimport sys\nimport fire\nimport warnings\nimport numpy as np\nimport torch as t\nimport jukebox.utils.dist_adapter as dist\nfrom torch.nn.parallel import DistributedDataParallel\n\nfrom jukebox.hparams import setup_hparams\nfrom jukebox.make_models import make_vqvae, make_prior, restore_opt, save_checkpoint\nfrom jukebox.utils.logger import init_logging\nfrom jukebox.utils.audio_utils import audio_preprocess, audio_postprocess\nfrom jukebox.utils.torch_utils import zero_grad, count_parameters\nfrom jukebox.utils.dist_utils import print_once, allreduce, allgather\nfrom jukebox.utils.ema import CPUEMA, FusedEMA, EMA\nfrom jukebox.utils.fp16 import FP16FusedAdam, FusedAdam, LossScalar, clipped_grad_scale, backward\nfrom jukebox.data.data_processor import DataProcessor\n\ndef prepare_aud(x, hps):\n x = audio_postprocess(x.detach().contiguous(), hps)\n return allgather(x)\n\ndef log_aud(logger, tag, x, hps):\n logger.add_audios(tag, prepare_aud(x, hps), hps.sr, max_len=hps.max_len, max_log=hps.max_log)\n logger.flush()\n\ndef log_labels(logger, labeller, tag, y, hps):\n y = y.cpu().numpy()\n txt = ''\n for item in range(y.shape[0]):\n description = labeller.describe_label(y[item])\n artist, genre, lyrics = description['artist'], description['genre'], description['lyrics']\n txt += f'{item} artist:{artist}, genre:{genre}, lyrics:{lyrics}\\n'\n logger.add_text(tag, txt)\n logger.flush()\n\ndef get_ddp(model, hps):\n rank = dist.get_rank()\n local_rank = rank % 8\n ddp = DistributedDataParallel(model, device_ids=[local_rank], output_device=local_rank, broadcast_buffers=False, bucket_cap_mb=hps.bucket)\n return ddp\n\ndef get_ema(model, hps):\n mu = hps.mu or (1. - (hps.bs * hps.ngpus/8.)/1000)\n ema = None\n if hps.ema and hps.train:\n if hps.cpu_ema:\n if dist.get_rank() == 0:\n print(\"Using CPU EMA\")\n ema = CPUEMA(model.parameters(), mu=mu, freq=hps.cpu_ema_freq)\n elif hps.ema_fused:\n ema = FusedEMA(model.parameters(), mu=mu)\n else:\n ema = EMA(model.parameters(), mu=mu)\n return ema\n\ndef get_lr_scheduler(opt, hps):\n def lr_lambda(step):\n if hps.lr_use_linear_decay:\n lr_scale = hps.lr_scale * min(1.0, step / hps.lr_warmup)\n decay = max(0.0, 1.0 - max(0.0, step - hps.lr_start_linear_decay) / hps.lr_decay)\n if decay == 0.0:\n if dist.get_rank() == 0:\n print(\"Reached end of training\")\n return lr_scale * decay\n else:\n return hps.lr_scale * (hps.lr_gamma ** (step // hps.lr_decay)) * min(1.0, step / hps.lr_warmup)\n\n shd = t.optim.lr_scheduler.LambdaLR(opt, lr_lambda)\n\n return shd\n\ndef get_optimizer(model, hps):\n # Optimizer\n betas = (hps.beta1, hps.beta2)\n if hps.fp16_opt:\n opt = FP16FusedAdam(model.parameters(), lr=hps.lr, weight_decay=hps.weight_decay, betas=betas, eps=hps.eps)\n else:\n opt = FusedAdam(model.parameters(), lr=hps.lr, weight_decay=hps.weight_decay, betas=betas, eps=hps.eps)\n\n # lr scheduler\n shd = get_lr_scheduler(opt, hps)\n\n restore_path = hps.restore_prior if hps.prior else hps.restore_vqvae\n restore_opt(opt, shd, restore_path)\n\n # fp16 dynamic loss scaler\n scalar = None\n if hps.fp16:\n rank = dist.get_rank()\n local_rank = rank % 8\n scalar = LossScalar(hps.fp16_loss_scale, scale_factor=2 ** (1./hps.fp16_scale_window))\n if local_rank == 0: print(scalar.__dict__)\n\n zero_grad(model)\n return opt, shd, scalar\n\ndef log_inputs(orig_model, logger, x_in, y, x_out, hps, tag=\"train\"):\n print(f\"Logging {tag} inputs/ouputs\")\n log_aud(logger, f'{tag}_x_in', x_in, hps)\n log_aud(logger, f'{tag}_x_out', x_out, hps)\n bs = x_in.shape[0]\n if hps.prior:\n if hps.labels:\n log_labels(logger, orig_model.labeller, f'{tag}_y_in', allgather(y.cuda()), hps)\n else:\n zs_in = orig_model.encode(x_in, start_level=0, bs_chunks=bs)\n x_ds = [orig_model.decode(zs_in[level:], start_level=level, bs_chunks=bs) for level in range(0, hps.levels)]\n for i in range(len(x_ds)):\n log_aud(logger, f'{tag}_x_ds_start_{i}', x_ds[i], hps)\n logger.flush()\n\ndef sample_prior(orig_model, ema, logger, x_in, y, hps):\n if ema is not None: ema.swap()\n orig_model.eval()\n\n x_in = x_in[:hps.bs_sample]\n bs = x_in.shape[0]\n zs_in = orig_model.encode(x_in, start_level=0, bs_chunks=bs)\n assert len(zs_in) == hps.levels\n x_ds = [orig_model.decode(zs_in[level:], start_level=level, bs_chunks=bs) for level in range(0, hps.levels)]\n\n if not hps.labels:\n y = None\n elif hps.level == (hps.levels - 1):\n # Topmost level labels in order\n y = y[:hps.bs_sample] # t.ones((hps.bs_sample, 1), device=y.device, dtype=t.long) * dist.get_rank()\n else:\n # Other levels keep labels to match x_cond\n y = y[:hps.bs_sample]\n\n # Temp 1.0\n _, *z_conds = orig_model.encode(x_in, bs_chunks=bs)\n z = orig_model.sample(hps.bs_sample, z_conds=z_conds, y=y, fp16=False, temp=1.0)\n x_sample = orig_model.decode([z, *z_conds], bs_chunks=bs)\n\n log_aud(logger, 'sample_x_T1', x_sample, hps)\n if hps.prior and hps.labels:\n log_labels(logger, orig_model.labeller, f'sample_x_T1', allgather(y.cuda()), hps)\n\n # Recons\n for i in range(len(x_ds)):\n log_aud(logger, f'x_ds_start_{i}', x_ds[i], hps)\n orig_model.train()\n if ema is not None: ema.swap()\n logger.flush()\n\ndef evaluate(model, orig_model, logger, metrics, data_processor, hps):\n model.eval()\n orig_model.eval()\n if hps.prior:\n _print_keys = dict(l=\"loss\", bpd=\"bpd\")\n else:\n _print_keys = dict(l=\"loss\", rl=\"recons_loss\", sl=\"spectral_loss\")\n\n with t.no_grad():\n for i, x in logger.get_range(data_processor.test_loader):\n if isinstance(x, (tuple, list)):\n x, y = x\n else:\n y = None\n\n x = x.to('cuda', non_blocking=True)\n if y is not None:\n y = y.to('cuda', non_blocking=True)\n\n x_in = x = audio_preprocess(x, hps)\n log_input_output = (i==0)\n\n if hps.prior:\n forw_kwargs = dict(y=y, fp16=hps.fp16, decode=log_input_output)\n else:\n forw_kwargs = dict(loss_fn=hps.loss_fn, hps=hps)\n\n x_out, loss, _metrics = model(x, **forw_kwargs)\n\n # Logging\n for key, val in _metrics.items():\n _metrics[key] = val.item()\n _metrics[\"loss\"] = loss = loss.item() # Make sure to call to free graph\n\n # Average and log\n for key, val in _metrics.items():\n _metrics[key] = metrics.update(f\"test_{key}\", val, x.shape[0])\n\n with t.no_grad():\n if log_input_output:\n log_inputs(orig_model, logger, x_in, y, x_out, hps)\n\n logger.set_postfix(**{print_key:_metrics[key] for print_key, key in _print_keys.items()})\n\n for key, val in _metrics.items():\n logger.add_scalar(f\"test_{key}\", metrics.avg(f\"test_{key}\"))\n\n logger.close_range()\n return {key: metrics.avg(f\"test_{key}\") for key in _metrics.keys()}\n\ndef train(model, orig_model, opt, shd, scalar, ema, logger, metrics, data_processor, hps):\n model.train()\n orig_model.train()\n if hps.prior:\n _print_keys = dict(l=\"loss\", bpd=\"bpd\", gn=\"gn\", g_l=\"gen_loss\", p_l=\"prime_loss\")\n else:\n _print_keys = dict(l=\"loss\", sl=\"spectral_loss\", rl=\"recons_loss\", e=\"entropy\", u=\"usage\", uc=\"used_curr\", gn=\"gn\", pn=\"pn\", dk=\"dk\")\n\n for i, x in logger.get_range(data_processor.train_loader):\n if isinstance(x, (tuple, list)):\n x, y = x\n else:\n y = None\n\n x = x.to('cuda', non_blocking=True)\n if y is not None:\n y = y.to('cuda', non_blocking=True)\n\n x_in = x = audio_preprocess(x, hps)\n log_input_output = (logger.iters % hps.save_iters == 0)\n\n if hps.prior:\n forw_kwargs = dict(y=y, fp16=hps.fp16, decode=log_input_output)\n else:\n forw_kwargs = dict(loss_fn=hps.loss_fn, hps=hps)\n\n # Forward\n x_out, loss, _metrics = model(x, **forw_kwargs)\n\n # Backward\n loss, scale, grad_norm, overflow_loss, overflow_grad = backward(loss=loss, params=list(model.parameters()),\n scalar=scalar, fp16=hps.fp16, logger=logger)\n # Skip step if overflow\n grad_norm = allreduce(grad_norm, op=dist.ReduceOp.MAX)\n if overflow_loss or overflow_grad or grad_norm > hps.ignore_grad_norm > 0:\n zero_grad(orig_model)\n continue\n\n # Step opt. Divide by scale to include clipping and fp16 scaling\n logger.step()\n opt.step(scale=clipped_grad_scale(grad_norm, hps.clip, scale))\n zero_grad(orig_model)\n lr = hps.lr if shd is None else shd.get_lr()[0]\n if shd is not None: shd.step()\n if ema is not None: ema.step()\n next_lr = hps.lr if shd is None else shd.get_lr()[0]\n finished_training = (next_lr == 0.0)\n\n # Logging\n for key, val in _metrics.items():\n _metrics[key] = val.item()\n _metrics[\"loss\"] = loss = loss.item() * hps.iters_before_update # Make sure to call to free graph\n _metrics[\"gn\"] = grad_norm\n _metrics[\"lr\"] = lr\n _metrics[\"lg_loss_scale\"] = np.log2(scale)\n\n # Average and log\n for key, val in _metrics.items():\n _metrics[key] = metrics.update(key, val, x.shape[0])\n if logger.iters % hps.log_steps == 0:\n logger.add_scalar(key, _metrics[key])\n\n # Save checkpoint\n with t.no_grad():\n if hps.save and (logger.iters % hps.save_iters == 1 or finished_training):\n if ema is not None: ema.swap()\n orig_model.eval()\n name = 'latest' if hps.prior else f'step_{logger.iters}'\n if dist.get_rank() % 8 == 0:\n save_checkpoint(logger, name, orig_model, opt, dict(step=logger.iters), hps)\n orig_model.train()\n if ema is not None: ema.swap()\n\n # Sample\n with t.no_grad():\n if (logger.iters % 12000) in list(range(1, 1 + hps.iters_before_update)) or finished_training:\n if hps.prior:\n sample_prior(orig_model, ema, logger, x_in, y, hps)\n\n # Input/Output\n with t.no_grad():\n if log_input_output:\n log_inputs(orig_model, logger, x_in, y, x_out, hps)\n\n logger.set_postfix(**{print_key:_metrics[key] for print_key, key in _print_keys.items()})\n if finished_training:\n dist.barrier()\n exit()\n logger.close_range()\n return {key: metrics.avg(key) for key in _metrics.keys()}\n\ndef run(hps=\"teeny\", port=29500, **kwargs):\n from jukebox.utils.dist_utils import setup_dist_from_mpi\n rank, local_rank, device = setup_dist_from_mpi(port=port)\n hps = setup_hparams(hps, kwargs)\n hps.ngpus = dist.get_world_size()\n hps.argv = \" \".join(sys.argv)\n hps.bs_sample = hps.nworkers = hps.bs\n\n # Setup dataset\n data_processor = DataProcessor(hps)\n\n # Setup models\n vqvae = make_vqvae(hps, device)\n print_once(f\"Parameters VQVAE:{count_parameters(vqvae)}\")\n if hps.prior:\n prior = make_prior(hps, vqvae, device)\n print_once(f\"Parameters Prior:{count_parameters(prior)}\")\n model = prior\n else:\n model = vqvae\n\n # Setup opt, ema and distributed_model.\n opt, shd, scalar = get_optimizer(model, hps)\n ema = get_ema(model, hps)\n distributed_model = get_ddp(model, hps)\n\n logger, metrics = init_logging(hps, local_rank, rank)\n logger.iters = model.step\n\n # Run training, eval, sample\n for epoch in range(hps.curr_epoch, hps.epochs):\n metrics.reset()\n data_processor.set_epoch(epoch)\n if hps.train:\n train_metrics = train(distributed_model, model, opt, shd, scalar, ema, logger, metrics, data_processor, hps)\n train_metrics['epoch'] = epoch\n if rank == 0:\n print('Train',' '.join([f'{key}: {val:0.4f}' for key,val in train_metrics.items()]))\n dist.barrier()\n\n if hps.test:\n if ema: ema.swap()\n test_metrics = evaluate(distributed_model, model, logger, metrics, data_processor, hps)\n test_metrics['epoch'] = epoch\n if rank == 0:\n print('Ema',' '.join([f'{key}: {val:0.4f}' for key,val in test_metrics.items()]))\n dist.barrier()\n if ema: ema.swap()\n dist.barrier()\n\nif __name__ == '__main__':\n fire.Fire(run)\n" }, { "path": "jukebox/transformer/__init__.py", "content": "" }, { "path": "jukebox/transformer/factored_attention.py", "content": "# Factored attention\nimport math\nimport numpy as np\nimport torch as t\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom jukebox.transformer.ops import Conv1D\nfrom jukebox.utils.checkpoint import checkpoint\n\ndef repeat(x, n, dim):\n if dim == -1:\n dim = len(x.shape) - 1\n return x.view(int(np.prod(x.shape[:dim+1])), 1, int(np.prod(x.shape[dim+1:]))).repeat(1,n,1).view(*x.shape[:dim], n * x.shape[dim], *x.shape[dim+1:])\n\ndef get_mask(mask, q_l, kv_l, blocks, spread, device, sample, sample_t):\n # returns a mask of shape 1 x 1 x q_l x kv_l or None if masking is not needed.\n if mask is None or q_l == 1:\n return None\n offset = sample_t - q_l if sample else max(kv_l - q_l, 0)\n if mask == 'autoregressive':\n # Masked dense\n mask = t.ones(q_l, kv_l, device=device).tril(offset)\n elif mask == 'summary':\n # Masked summary\n mask = t.nn.functional.pad(t.ones(q_l, q_l, device=device).tril().view(q_l, blocks, q_l // blocks)[:,:-1,-kv_l//blocks:],(0,0,1,0),value=1).contiguous().view(q_l, kv_l)\n elif mask == 'prime':\n mask = t.ones(q_l, kv_l, device=device).tril(offset)\n return mask.view(1,1,q_l,kv_l)\n\nclass FactoredAttention(nn.Module):\n def __init__(self, n_in, n_ctx, n_state, n_head,\n attn_dropout=0.0, resid_dropout=0.0,\n scale=True, mask=False,\n zero_out=False, init_scale=1.0,\n checkpoint_attn=0,\n attn_func=0, blocks=None, spread=None,\n encoder_dims=None, prime_len=None):\n super().__init__()\n self.n_in = n_in\n self.n_ctx = n_ctx # NOTE: n_ctx could be different within operations. This is complete n_ctx\n self.n_state = n_state\n assert n_state % n_head == 0\n self.n_head = n_head\n self.scale = scale\n self.mask = mask\n if attn_func == 6:\n self.c_attn = Conv1D(n_in, n_state, init_scale=init_scale)\n self.c_enc_kv = Conv1D(n_in, n_state * 2, init_scale=init_scale)\n else:\n self.c_attn = Conv1D(n_in, n_state * 3, init_scale=init_scale)\n self.c_proj = Conv1D(n_state, n_in, zero_out, init_scale=init_scale)\n self.attn_dropout = nn.Dropout(attn_dropout) if attn_dropout > 0.0 else lambda x: x\n self.resid_dropout = nn.Dropout(resid_dropout) if resid_dropout > 0.0 else lambda x: x\n\n # Sequence of length l is factored as [blocks, l // blocks]\n self.attn_func = attn_func\n self.qkv, self.attn, self.attn_mask = {\n 0: (self.factored_qkv, self.dense_attn, 'autoregressive'), # Attend to all positions\n 1: (self.factored_qkv, self.block_attn, 'autoregressive'), # Attend to your block\n 2: (self.factored_qkv, self.transpose_block_attn, 'autoregressive'), # Attend to transpose block\n 3: (self.factored_qkv, self.prev_block_attn, None), # Attend to previous block\n 4: (self.factored_qkv, self.summary_attn, 'summary'), # Attend to last position of each block\n 5: (self.factored_qkv, self.summary_spread_attn, 'summary'),\n 6: (self.decode_qkv, self.decode_attn, None),\n 7: (self.prime_qkv, self.prime_attn, 'prime')\n }[attn_func] # Attend to last k position of each block\n\n self.blocks = blocks\n self.spread = spread\n if blocks is not None:\n assert n_ctx % blocks == 0\n self.block_ctx = n_ctx // blocks\n self.checkpoint_attn = checkpoint_attn # 0: None, 1: Attn after heads split, 2: Attn\n\n self.sample_t = 0\n self.cache = {}\n self.encoder_dims = encoder_dims\n self.prime_len = prime_len\n self.record_attn = False\n self.w = None\n\n def _attn(self, q, k, v, sample):\n scale = 1. / math.sqrt(math.sqrt(self.n_state // self.n_head))\n if self.training:\n w = t.matmul(q * scale, k * scale)\n else:\n w = t.matmul(q, k)\n w.mul_(scale*scale)\n wtype = w.dtype\n w = w.float()\n if self.mask:\n # Generate appropriate mask to mask out all positions before current\n # Might take up lot of memory for dense, so can cache it\n mask = get_mask(self.attn_mask, q.size(-2), k.size(-1), self.blocks, self.spread, w.device, sample, self.sample_t)\n if mask is not None:\n #print(mask)\n w = w * mask + -1e9 * (1 - mask)\n w = F.softmax(w, dim=-1).type(wtype)\n else:\n w = F.softmax(w, dim=-1).type(wtype)\n if self.record_attn:\n self.w = w #.float().cpu().numpy()\n if self.attn_func == 7:\n # only keep music queries and lyrics keys/values\n self.w = self.w[:,:,self.prime_len:,:self.prime_len]\n w = self.attn_dropout(w)\n a = t.matmul(w, v)\n return a\n\n def merge_heads(self, x):\n x = x.permute(0, 2, 1, 3).contiguous()\n new_x_shape = (*x.size()[:-2], x.size(-2) * x.size(-1))\n return x.view(*new_x_shape) # in Tensorflow implem: fct merge_states\n\n def split_heads(self, x, k=False):\n new_x_shape = (*x.size()[:-1], self.n_head, x.size(-1) // self.n_head)\n x = x.view(*new_x_shape) # in Tensorflow implem: fct split_states\n if k:\n return x.permute(0, 2, 3, 1)\n else:\n return x.permute(0, 2, 1, 3)\n\n def dense_attn(self, query, key, value, sample):\n query = self.split_heads(query)\n key = self.split_heads(key, k=True)\n value = self.split_heads(value)\n if self.checkpoint_attn == 1 and not sample:\n a = checkpoint(lambda q,k,v,s=sample: self._attn(q,k,v,s), (query, key, value),\n (), True)\n else:\n a = self._attn(query,key,value,sample)\n a = self.merge_heads(a)\n return a\n\n def block_attn(self, q, k, v, sample):\n blocks, block_ctx = self.blocks, self.block_ctx # block_ctx is l // blocks for complete l ie l = n_ctx. Sampling has less l\n bs, l, d = v.shape # For sample, q_l = 1, k_l = v_l = sample_t\n if sample:\n assert l == self._suff_cache_len(), f\"{l} != {self._suff_cache_len()}\"\n return self.dense_attn(q, k, v, sample).view(bs, 1, d)\n else:\n ql = q.shape[1]\n q = q.view(bs * ql // block_ctx, block_ctx, d)\n if ql < l:\n l = ql\n k = k[:, -l:].contiguous()\n v = v[:, -l:].contiguous()\n k = k.view(bs * l // block_ctx, block_ctx, d)\n v = v.view(bs * l // block_ctx, block_ctx, d)\n return self.dense_attn(q, k, v, sample).view(bs, l, d)\n\n def transpose_block_attn(self, q, k, v, sample):\n blocks, block_ctx = self.blocks, self.block_ctx # block_ctx is l // blocks for complete l ie l = n_ctx. Sampling has less l\n bs, l, d = v.shape # For sample, q_l = 1, k_l = v_l = sample_t\n if sample:\n block_l = (l - 1) % block_ctx\n k = k[:,block_l::block_ctx,:]\n v = v[:,block_l::block_ctx,:]\n return self.dense_attn(q, k, v, sample).view(bs, 1, d)\n else:\n ql = q.shape[1]\n q = q.view(bs, ql // block_ctx, block_ctx, d).transpose(1,2).contiguous().view(bs * block_ctx, ql // block_ctx, d)\n k = k.view(bs, l // block_ctx, block_ctx, d).transpose(1,2).contiguous().view(bs * block_ctx, l // block_ctx, d)\n v = v.view(bs, l // block_ctx, block_ctx, d).transpose(1,2).contiguous().view(bs * block_ctx, l // block_ctx, d)\n return self.dense_attn(q, k, v, sample).view(bs, block_ctx, ql // block_ctx, d).transpose(1,2).contiguous().view(bs, ql, d)\n\n def prev_block_attn(self, q, k, v, sample):\n blocks, block_ctx = self.blocks, self.block_ctx # block_ctx is l // blocks for complete l ie l = n_ctx. Sampling has less l\n bs, l, d = v.shape # For sample, q_l = 1, k_l = v_l = sample_t\n if sample:\n assert l == self._suff_cache_len(), f\"{l} != {self._suff_cache_len()}\"\n block = (l - 1) // block_ctx\n prev_l = (block - 1) * block_ctx\n if block > 0:\n assert prev_l == 0\n k = k[:, prev_l:prev_l + block_ctx, :]\n v = v[:, prev_l:prev_l + block_ctx, :]\n else:\n k = t.zeros(bs, block_ctx, d, device=q.device, dtype=q.dtype)\n v = t.zeros(bs, block_ctx, d, device=q.device, dtype=q.dtype)\n return self.dense_attn(q, k, v, sample).view(bs, 1, d)\n else:\n ql = q.shape[1]\n q = q.view(bs * ql // block_ctx, block_ctx, d)\n k = t.nn.functional.pad(k.view(bs, l // block_ctx, block_ctx, d)[:, :-1, :, :], (0,0,0,0,1,0)).view(bs * l // block_ctx, block_ctx, d)\n v = t.nn.functional.pad(v.view(bs, l // block_ctx, block_ctx, d)[:, :-1, :, :], (0,0,0,0,1,0)).view(bs * l // block_ctx, block_ctx, d)\n if ql < l:\n qb = ql // block_ctx\n kb = l // block_ctx\n l = ql\n k = k.view(bs, kb, block_ctx, d)[:, -qb:].contiguous().view(bs * qb, block_ctx, d)\n v = v.view(bs, kb, block_ctx, d)[:, -qb:].contiguous().view(bs * qb, block_ctx, d)\n return self.dense_attn(q, k, v, sample).view(bs, l, d)\n\n def summary_attn(self, q, k, v, sample):\n blocks, block_ctx = self.blocks, self.block_ctx # block_ctx is l // blocks for complete l ie l = n_ctx. Sampling has less l\n bs, l, d = v.shape # For sample, q_l = 1, k_l = v_l = sample_t\n if sample:\n k = t.nn.functional.pad(k[:, block_ctx-1:blocks*block_ctx-1:block_ctx, :],(0,0,1,0))\n v = t.nn.functional.pad(v[:, block_ctx-1:blocks*block_ctx-1:block_ctx, :],(0,0,1,0))\n return self.dense_attn(q, k, v, sample).view(bs, 1, d)\n else:\n k = t.nn.functional.pad(k.view(bs, blocks, l // blocks, d)[:, :-1, -1, :],(0,0,1,0)) # bs, blocks, d\n v = t.nn.functional.pad(v.view(bs, blocks, l // blocks, d)[:, :-1, -1, :],(0,0,1,0)) # bs, blocks, d\n return self.dense_attn(q, k, v, sample).view(bs, l, d)\n\n def summary_spread_attn(self, q, k, v, sample):\n blocks, block_ctx, spread = self.blocks, self.block_ctx, self.spread # block_ctx is l // blocks for complete l ie l = n_ctx. Sampling has less l\n bs, l, d = v.shape # For sample, q_l = 1, k_l = v_l = sample_t\n if sample:\n assert False, \"Not yet implemented\"\n # k = t.nn.functional.pad(k,(0,0,block_ctx,(-l)%block_ctx)).view(bs, -1, block_ctx, d)[:,:-1,-spread:,:].contiguous().view(bs, -1, d)\n # v = t.nn.functional.pad(v,(0,0,block_ctx,(-l)%block_ctx)).view(bs, -1, block_ctx, d)[:,:-1,-spread:,:].contiguous().view(bs, -1, d)\n # return self.dense_attn(q, k, v, sample).view(bs, 1, d)\n else:\n k = t.nn.functional.pad(k.view(bs, blocks, l // blocks, d)[:, :-1, -spread:, :],(0,0,0,0,1,0)).contiguous().view(bs, blocks * spread, d) # bs, blocks * spread, d\n v = t.nn.functional.pad(v.view(bs, blocks, l // blocks, d)[:, :-1, -spread:, :],(0,0,0,0,1,0)).contiguous().view(bs, blocks * spread, d) # bs, blocks * spread, d\n return self.dense_attn(q, k, v, sample).view(bs, l, d)\n\n def prime_attn(self, q, k, v, sample):\n prime_len = self._prime_len\n k = k[:, :prime_len]\n v = v[:, :prime_len]\n return self.dense_attn(q, k, v, sample)\n\n def decode_attn(self, q, k, v, sample):\n assert k.shape[1] == v.shape[1] == self.encoder_dims, f'k: {k.shape}, v: {v.shape}, enc_dims: {self.encoder_dims}'\n return self.dense_attn(q, k, v, sample)\n\n def factored_qkv(self, x, encoder_kv=None, sample=False):\n curr_ctx = x.shape[1]\n assert encoder_kv is None\n query, key, value = x.chunk(3, dim=2)\n if sample:\n self.sample_t += curr_ctx\n key, value = self._append_cache(key, value)\n l_cache = self._suff_cache_len()\n if self._cache_len() > l_cache:\n self._slice_cache(-l_cache)\n if curr_ctx > 1:\n if self.attn_func != 0:\n query = self._pad_to_block_ctx(query, query=True)\n key = self._pad_to_block_ctx(key)\n value = self._pad_to_block_ctx(value)\n assert key.shape[1] % self.block_ctx == 0\n assert query.shape[1] % self.block_ctx == 0\n assert key.shape[1] == value.shape[1]\n assert query.shape[1] <= key.shape[1]\n sample = False\n else:\n key = self.cache['key']\n value = self.cache['value']\n return query, key, value, sample\n\n def prime_qkv(self, x, encoder_kv=None, sample=False):\n curr_ctx = x.shape[1]\n assert encoder_kv is None\n query, key, value = x.chunk(3, dim=2)\n if sample:\n if self._cache_len() < self._prime_len:\n self._append_cache(key, value)\n if self._cache_len() > self._prime_len:\n self._slice_cache(0, self._prime_len)\n key, value = self.cache['key'], self.cache['value']\n self.sample_t += curr_ctx\n assert key.shape[1] == value.shape[1] == self._suff_cache_len(), f'k: {key.shape}, v: {value.shape}, prime_dims: {self._suff_cache_len()}'\n else:\n assert key.shape[1] == value.shape[1] == self.n_ctx, f'k: {key.shape}, v: {value.shape}, prime_dims: {self.n_ctx}'\n assert key.shape[0] == value.shape[0] == query.shape[0], f'k: {key.shape}, v: {value.shape}, q: {query.shape}'\n assert key.shape[2] == value.shape[2] == query.shape[2], f'k: {key.shape}, v: {value.shape}, q: {query.shape}'\n return query, key, value, sample\n\n def decode_qkv(self, x, encoder_kv=None, sample=False):\n curr_ctx = x.shape[1]\n assert encoder_kv is not None\n query = x\n if sample:\n if self.sample_t == 0:\n self.cache['key'], self.cache['value'] = self.c_enc_kv(encoder_kv.type_as(x)).chunk(2, dim=2)\n key, value = self.cache['key'], self.cache['value']\n self.sample_t += curr_ctx\n else:\n key, value = self.c_enc_kv(encoder_kv.type_as(x)).chunk(2, dim=2)\n assert key.shape[0] == value.shape[0] == query.shape[0], f'k: {key.shape}, v: {value.shape}, q: {query.shape}'\n assert key.shape[1] == value.shape[1] == self.encoder_dims, f'k: {key.shape}, v: {value.shape}, enc_dims: {self.encoder_dims}'\n assert key.shape[2] == value.shape[2] == query.shape[2], f'k: {key.shape}, v: {value.shape}, q: {query.shape}'\n return query, key, value, sample\n\n def forward(self, x, encoder_kv=None, sample=False):\n curr_ctx = x.shape[1]\n x = self.c_attn(x)\n query, key, value, sample = self.qkv(x, encoder_kv=encoder_kv, sample=sample)\n if self.checkpoint_attn == 2 and not sample:\n a = checkpoint(lambda q,k,v,s=sample: self.attn(q,k,v,s), (query, key, value), (), True)\n else:\n a = self.attn(query,key,value,sample)\n if a.shape[1] != curr_ctx:\n offset = self._offset(curr_ctx)\n a = a[:,offset:offset + curr_ctx,:].contiguous()\n a = self.c_proj(a)\n return self.resid_dropout(a)\n\n @property\n def _prime_len(self):\n prime_len = self.prime_len\n assert prime_len is not None\n prime_blocks = (prime_len // self.blocks) + 1\n return prime_blocks * self.blocks\n\n def _offset(self, curr_ctx):\n if self.attn_func == 0:\n return 0\n return (self.sample_t - curr_ctx) % self.block_ctx\n\n def _pad_to_block_ctx(self, x, query=False):\n l = x.shape[1]\n offset = self._offset(l) if query else 0\n n_blocks = (l + offset + self.block_ctx - 1) // self.block_ctx\n pad = n_blocks * self.block_ctx - l - offset\n if pad == 0 and offset == 0:\n return x\n else:\n return F.pad(x, (0, 0, offset, pad))\n\n def _cache_len(self):\n return 0 if 'key' not in self.cache else self.cache['key'].shape[1]\n\n def _suff_cache_len(self):\n \"\"\"\n Precondition:\n key and value are appended with the current context and\n self.sample_t reflects the 1-indexed sample location in the\n context.\n \"\"\"\n if self.attn_func == 0:\n return self.sample_t\n elif self.attn_func == 1:\n return (self.sample_t - 1) % self.block_ctx + 1\n elif self.attn_func == 2:\n return self.sample_t\n elif self.attn_func == 3:\n if self.sample_t <= self.block_ctx:\n return self.sample_t\n else:\n curr_block = (self.sample_t - 1) % self.block_ctx + 1\n prev_block = self.block_ctx\n return curr_block + prev_block\n elif self.attn_func == 6:\n return self.encoder_dims\n elif self.attn_func == 7:\n return min(self.sample_t, self._prime_len)\n else:\n raise NotImplementedError()\n\n def _slice_cache(self, start, end=None):\n self.cache['key'] = self.cache['key'][:, start:end]\n self.cache['value'] = self.cache['value'][:, start:end]\n\n def _append_cache(self, key, value):\n if 'key' not in self.cache:\n self.cache['key'] = key\n self.cache['value'] = value\n else:\n old_key, old_value = key, value\n key = t.cat([self.cache['key'], key], dim=1)\n value = t.cat([self.cache['value'], value], dim=1)\n del self.cache['key']\n del self.cache['value']\n del old_key\n del old_value\n self.cache['key'] = key\n self.cache['value'] = value\n return self.cache['key'], self.cache['value']\n\n def del_cache(self):\n self.sample_t = 0\n if 'key' in self.cache:\n del self.cache['key']\n if 'value' in self.cache:\n del self.cache['value']\n self.cache = {}\n\n def check(self):\n blocks = self.blocks or 1\n spread = self.spread or 1\n bs, l, d = (4, self.n_ctx, self.n_in)\n x = t.randn(bs, l, d).cuda()\n x.requires_grad = True\n x_out = self.forward(x) # bs, l, d\n loss = x_out.mean(dim = -1) # bs, l\n pos = 60\n grad = t.autograd.grad(loss[2, pos], x)[0]\n\n assert grad.shape == (bs, l, d)\n assert (grad[:2] == 0).all()\n assert (grad[3:] == 0).all()\n assert (grad[2, (pos + 1):] == 0).all()\n pos_grad = (t.sum(grad[2] ** 2, dim=-1) > 0).nonzero().view(-1).cpu()\n\n block_pos = pos - (pos % (l // blocks))\n exp_pos_grad = {0: t.arange(pos),\n 1: t.arange(block_pos, pos),\n 2: t.arange(pos % (l // blocks), pos, l // blocks),\n 3: t.arange(block_pos - l // blocks, block_pos),\n 4: t.arange(l // blocks - 1, pos, l // blocks),\n 5: ((t.arange(pos) % (l // blocks) >= (l // blocks - spread)) & (t.arange(pos) < block_pos)).nonzero().view(-1)}[self.attn_func]\n exp_pos_grad = t.cat([exp_pos_grad, t.tensor([pos])], dim=-1)\n\n assert (len(pos_grad) == len(exp_pos_grad)) and (pos_grad == exp_pos_grad).all(), \\\n f\"Expected pos grad {exp_pos_grad} got {pos_grad} for attn_func {self.attn_func} pos {pos} l {l} blocks {blocks}\"\n\n def check_cache(self, n_samples, sample_t, fp16):\n assert self.sample_t == sample_t, f\"{self.sample_t} != {sample_t}\"\n if sample_t == 0:\n assert self.cache == {}\n else:\n dtype = {True: t.float16, False: t.float32}[fp16]\n l_cache = self._suff_cache_len()\n assert self.cache['key'].shape == (n_samples, l_cache, self.n_state)\n assert self.cache['value'].shape == (n_samples, l_cache, self.n_state)\n assert self.cache['key'].dtype == dtype, f\"Expected {dtype}, got {self.cache['key'].dtype}\"\n assert self.cache['value'].dtype == dtype, f\"Expected {dtype}, got {self.cache['value'].dtype}\"\n\n def check_sample(self):\n t.manual_seed(42)\n bs, l, d = (4, self.n_ctx, self.n_in)\n prime = 5\n x = t.randn(bs, l, d).cuda()\n xs = t.chunk(x, l, dim=1)\n assert self.sample_t == 0\n assert self.cache == {}\n\n with t.no_grad():\n enc_l = self.encoder_dims\n encoder_kv = None\n if self.attn_func == 6:\n encoder_kv = t.randn(bs, enc_l, d).cuda()\n\n # Normal path\n x_out_normal = self.forward(x, encoder_kv=encoder_kv)\n\n # Sampling path\n x_out_sample = t.cat([self.forward(xs[i], encoder_kv=encoder_kv, sample=True) for i in range(l)],dim=1)\n max_err = t.max(t.abs(x_out_sample - x_out_normal))\n assert max_err < 1e-8, f\"Max sampling err is {max_err} {[i for i in range(l) if t.max(t.abs(x_out_sample - x_out_normal)[:,i,:]) > 1e-8]}\"\n\n with t.no_grad():\n x_out_normal = x_out_normal[:,:prime,:]\n # Prime sampling path\n self.del_cache()\n x_out_sample = self.forward(x[:,:prime,:].contiguous(), encoder_kv=encoder_kv, sample=True)\n self.check_cache(bs, prime, False)\n\n max_err = t.max(t.abs(x_out_sample - x_out_normal))\n assert max_err < 1e-8, f\"Max prime sampling err is {max_err} {[i for i in range(prime) if t.max(t.abs(x_out_sample - x_out_normal)[:,i,:]) > 1e-8]}\"\n\n def check_chunks(self, chunk_size):\n t.manual_seed(42)\n bs, l, d = (4, self.n_ctx, self.n_in)\n enc_l = self.encoder_dims\n assert l % chunk_size == 0\n n_chunks = l // chunk_size\n with t.no_grad():\n encoder_kv = None\n x = t.randn(bs, l, d).cuda()\n if self.attn_func == 6:\n encoder_kv = t.randn(bs, enc_l, d).cuda()\n\n self.del_cache()\n y_forw = self.forward(x, encoder_kv=encoder_kv, sample=False)\n self.del_cache()\n y_forw_sample = self.forward(x, encoder_kv=encoder_kv, sample=True)\n max_err = t.max(t.abs(y_forw - y_forw_sample))\n assert max_err <= 1e-6, f\"Max err is {max_err} {[i for i in range(l) if t.max(t.abs(y_forw - y_forw_sample)[:, i, :]) > 1e-6]}\"\n\n self.del_cache()\n x_chunks = t.chunk(x, n_chunks, dim=1)\n y_chunks = []\n total_len = 0\n for x_chunk in x_chunks:\n y_chunk = self.forward(x_chunk.contiguous(), encoder_kv=encoder_kv, sample=True)\n total_len += x_chunk.shape[1]\n self.check_cache(bs, total_len, False)\n y_chunks.append(y_chunk)\n y_forw_in_chunks = t.cat(y_chunks, dim=1)\n\n max_err = t.max(t.abs(y_forw - y_forw_in_chunks))\n assert max_err <= 1e-6, f\"Max err is {max_err} {[i for i in range(l) if t.max(t.abs(y_forw - y_forw_in_chunks)[:, i, :]) > 1e-6]}\"\n\n\nif __name__ == '__main__':\n from jukebox.utils.dist_utils import setup_dist_from_mpi\n setup_dist_from_mpi(port=29600)\n n_in = 16\n n_state = n_in * 2\n n_ctx = 6144\n n_head = 4\n n_depth = 12\n blocks = 64\n chunk_size = 8\n for attn_func in [0, 1, 2, 3, 6, 7]:\n encoder_dims = {0: 0, 1: 0, 2: 0, 3: 0, 6: 64, 7: 0}[attn_func]\n prime_len = {0: 0, 1: 0, 2: 0, 3: 0, 6: 0, 7: 384}[attn_func]\n attn = FactoredAttention(n_in, n_ctx + prime_len, n_state, n_head, mask=True,\n attn_func=attn_func, blocks=blocks,\n encoder_dims=encoder_dims, prime_len=prime_len)\n attn.training = False\n attn.check_sample()\n attn.check_chunks(chunk_size)\n print(f\"Checked attn_func: {attn_func}\")\n" }, { "path": "jukebox/transformer/ops.py", "content": "import math\nimport numpy as np\nimport torch as t\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n# Import FusedLayerNorm if we have apex, otherwise use regular LayerNorm\ntry:\n from apex.normalization import FusedLayerNorm\n print(\"Using apex FusedLayerNorm\")\nexcept ImportError:\n from torch.nn import LayerNorm as FusedLayerNorm\n\nclass LayerNorm(FusedLayerNorm):\n def __init__(self, normalized_shape, eps=1e-5, elementwise_affine=True):\n super().__init__(normalized_shape, eps=eps, elementwise_affine=elementwise_affine)\n self.width = np.prod(normalized_shape)\n self.max_numel = 65535*self.width\n\n def forward(self, input):\n if input.numel() > self.max_numel:\n return F.layer_norm(input.float(), self.normalized_shape, self.weight, self.bias, self.eps).type_as(input)\n else:\n return super(LayerNorm, self).forward(input.float()).type_as(input)\n\ndef gelu(x):\n return 0.5 * x * (1 + t.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * t.pow(x, 3))))\n\n\ndef swish(x):\n return x * t.sigmoid(x)\n\n@t.jit.script\ndef quick_gelu(x):\n return x * t.sigmoid(1.702 * x)\n\n@t.jit.script\ndef quick_gelu_bwd(x, grad_output):\n sig = t.sigmoid(1.702 * x)\n return grad_output * sig * (1.702 * x * (1 - sig) + 1.)\n\nclass QuickGelu(t.autograd.Function):\n @staticmethod\n def forward(ctx, x):\n ctx.save_for_backward(x)\n return quick_gelu(x)\n\n @staticmethod\n def backward(ctx, grad_output):\n return quick_gelu_bwd(ctx.saved_tensors[0], grad_output)\n\ndef memory_efficient_quick_gelu(x):\n return QuickGelu.apply(x)\n\nACT_FNS = {\n 'relu': t.nn.functional.relu,\n 'swish': swish,\n 'gelu': gelu,\n 'quick_gelu': memory_efficient_quick_gelu #quick_gelu\n}\n\ndef _move_to_gpu_and_convert_conv_weights_to_fp16(l):\n l.cuda()\n if isinstance(l, Conv1D):\n l.w.data = l.w.data.half()\n\ndef _convert_conv_weights_to_fp32(l):\n if isinstance(l, Conv1D):\n l.w.data = l.w.data.float()\n\ndef _convert_conv_weights_to_fp16(l):\n if isinstance(l, Conv1D):\n l.w.data = l.w.data.half()\n\ndef _convert_embedding_weights_to_fp16(l):\n if isinstance(l, t.nn.Embedding):\n l.weight.data = l.weight.data.half()\n\ndef _convert_embedding_weights_to_fp32(l):\n if isinstance(l, t.nn.Embedding):\n l.weight.data = l.weight.data.float()\n\nclass Conv1D(nn.Module):\n def __init__(self, n_in, n_out, zero_out=False, init_scale=1.0):\n super(Conv1D, self).__init__()\n self.n_in = n_in\n self.n_out = n_out\n if zero_out:\n w = t.zeros(n_in, n_out)\n else:\n w = t.empty(n_in, n_out)\n nn.init.normal_(w, std=0.02 * init_scale)\n b = t.zeros(n_out)\n self.w = nn.Parameter(w)\n self.b = nn.Parameter(b)\n\n def forward(self, x):\n size_out = (*x.size()[:-1], self.n_out)\n x = t.addmm(self.b.type_as(x), x.view(-1, x.size(-1)), self.w.type_as(x)) # If x if float then float else half\n x = x.view(*size_out)\n return x\n\n# For large contexts, mask's can take up memory, so you can make a single saved mask for all layers\nclass Mask(nn.Module):\n def __init__(self, n_ctx):\n super().__init__()\n self.register_buffer('b', t.tril(t.ones(n_ctx, n_ctx)).view(1, 1, n_ctx, n_ctx))\n\n def forward(self, w):\n w = w * self.b + -1e9 * (1 - self.b) # For fp16 do w = w.float().masked_fill(self.b, float('-inf')\n return w\n\ndef filter_logits(logits, top_k=0, top_p=0.0, filter_value=-float('Inf')):\n \"\"\" Filter a distribution of logits using top-k and/or nucleus (top-p) filtering\n Args:\n logits: logits distribution shape (vocabulary size)\n top_k >0: keep only top k tokens with highest probability (top-k filtering).\n top_p >0.0: keep the top tokens with cumulative probability >= top_p (nucleus filtering).\n \"\"\"\n #assert logits.dim() == 2 # batch size 1 for now - could be updated for more but the code would be less clear\n logits = logits.clone()\n top_k = min(top_k, logits.size(-1)) # Safety check\n assert (top_k == 0) or (top_p == 0.0)\n if top_k > 0:\n # Remove all tokens with a probability less than the last token of the top-k\n indices_to_remove = logits < t.topk(logits, top_k, dim=-1)[0][..., -1:]\n logits[indices_to_remove] = filter_value\n\n if top_p > 0.0:\n sorted_logits, sorted_indices = t.sort(logits, descending=True, dim=-1)\n cumulative_probs = t.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)\n\n # Remove tokens with cumulative probability above the threshold\n sorted_indices_to_remove = cumulative_probs > top_p\n # Shift the indices to the right to keep also the first token above the threshold\n sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()\n sorted_indices_to_remove[..., 0] = 0\n\n #indices_to_remove = sorted_indices[sorted_indices_to_remove]\n indices_to_remove = t.zeros_like(logits, dtype=t.uint8).scatter_(dim=-1, index=sorted_indices, src=sorted_indices_to_remove)\n logits[indices_to_remove] = filter_value\n return logits\n" }, { "path": "jukebox/transformer/transformer.py", "content": "import functools\nimport numpy as np\nimport torch as t\nimport torch.nn as nn\nimport jukebox.utils.dist_adapter as dist\n\nfrom jukebox.transformer.ops import Conv1D, ACT_FNS, LayerNorm\nfrom jukebox.transformer.factored_attention import FactoredAttention\nfrom jukebox.utils.checkpoint import checkpoint\n\ndef _convert_mlp_traced(l):\n if isinstance(l, ResAttnBlock):\n l.mlp = t.jit.trace(l.mlp, t.randn(1, 1, l.n_in).cuda())\n\ndef _convert_mlp_traced_fp16(l):\n if isinstance(l, ResAttnBlock):\n l.mlp = t.jit.trace(l.mlp, t.randn(1, 1, l.n_in).cuda().half())\n\nclass MLP(nn.Module):\n def __init__(self, n_in, n_state, resid_dropout=0.0, afn='quick_gelu', zero_out=False, init_scale=1.0):\n super().__init__()\n self.c_fc = Conv1D(n_in, n_state, init_scale=init_scale)\n self.c_proj = Conv1D(n_state, n_in, zero_out, init_scale=init_scale)\n self.act = ACT_FNS[afn]\n self.resid_dropout = nn.Dropout(resid_dropout) if resid_dropout > 0.0 else lambda x: x\n\n def forward(self, x):\n m = self.act(self.c_fc(x))\n m = self.c_proj(m)\n return self.resid_dropout(m)\n\nclass ResAttnBlock(nn.Module):\n def __init__(self, n_in, n_ctx, n_head,\n attn_dropout=0.0, resid_dropout=0.0,\n afn='quick_gelu', scale=True, mask=False,\n zero_out=False, init_scale=1.0, res_scale=1.0,\n m_attn = 0.25, m_mlp = 1.,\n checkpoint_attn = 0, checkpoint_mlp = 0,\n attn_func=0, blocks=None, spread=None,\n encoder_dims=None, prime_len=None):\n super().__init__()\n self.attn = FactoredAttention(n_in=n_in, n_ctx=n_ctx, n_state=int(m_attn * n_in), n_head=n_head,\n attn_dropout=attn_dropout, resid_dropout=resid_dropout,\n scale=scale, mask=mask,\n zero_out=zero_out, init_scale=init_scale,\n checkpoint_attn=checkpoint_attn,\n attn_func=attn_func, blocks=blocks, spread=spread,\n encoder_dims=encoder_dims, prime_len=prime_len)\n self.ln_0 = LayerNorm(n_in)\n self.mlp = MLP(n_in=n_in, n_state=int(m_mlp * n_in),\n resid_dropout=resid_dropout,\n afn=afn,\n zero_out=zero_out, init_scale=init_scale)\n self.ln_1 = LayerNorm(n_in)\n self.res_scale = res_scale\n\n self.checkpoint_attn = checkpoint_attn\n self.checkpoint_mlp = checkpoint_mlp\n self.n_in = n_in\n self.attn_func = attn_func\n\n def forward(self, x, encoder_kv, sample=False):\n if sample:\n a = self.attn(self.ln_0(x), encoder_kv, sample)\n m = self.mlp(self.ln_1(x + a))\n else:\n if self.attn_func == 6:\n assert encoder_kv is not None\n a = checkpoint(lambda _x,_enc_kv,_s=sample: self.attn(self.ln_0(_x),_enc_kv,_s),\n (x,encoder_kv),\n (*self.attn.parameters(), *self.ln_0.parameters()),\n self.checkpoint_attn == 3) # 2 recomputes after the projections, and 1 recomputes after head splitting.\n else:\n assert encoder_kv is None\n a = checkpoint(lambda _x,_enc_kv=None,_s=sample: self.attn(self.ln_0(_x),_enc_kv,_s),\n (x,),\n (*self.attn.parameters(), *self.ln_0.parameters()),\n self.checkpoint_attn == 3) # 2 recomputes after the projections, and 1 recomputes after head splitting.\n m = checkpoint(lambda _x: self.mlp(self.ln_1(_x)), (x + a,),\n (*self.mlp.parameters(), *self.ln_1.parameters()),\n self.checkpoint_mlp == 1)\n if self.res_scale == 1.0:\n h = x + a + m\n else:\n h = x + self.res_scale * (a + m)\n return h\n\nclass Transformer(nn.Module):\n def __init__(self, n_in, n_ctx, n_head, n_depth,\n attn_dropout=0.0, resid_dropout=0.0,\n afn='quick_gelu', scale=True, mask=False,\n zero_out=False, init_scale=1.0, res_scale=False,\n m_attn=0.25, m_mlp=1.,\n checkpoint_attn=0, checkpoint_mlp=0, checkpoint_res=0,\n attn_order=0, blocks=None, spread=None,\n encoder_dims=None, prime_len=None):\n super().__init__()\n self.n_in = n_in\n self.n_ctx = n_ctx\n self.encoder_dims = encoder_dims\n self.blocks = blocks\n if blocks is not None:\n assert n_ctx % blocks == 0\n self.block_ctx = n_ctx // blocks\n self.prime_len = prime_len\n self.n_head = n_head\n\n res_scale = 1.0 / n_depth if res_scale else 1.0\n\n # Orders of attn_func\n attn_func = {0: lambda d: 0, # Complete dense attn\n 1: lambda d: [1,2][d%2], # Alternate row and column attn\n 2: lambda d: [1,2,3][d % 3], # Alternate row, column and previous row attn\n 3: lambda d: [1,4][d % 2], # Alternate row and last column\n 4: lambda d: [1,5][d % 2], # Alternate row and last k columns\n 5: lambda d: [1,4,1,1][d % 4], # Alternate row, last column, row, row\n 6: lambda d: [1,2,3,6][d % 4],\n 7: lambda d: [*[1,2,3]*5,6][d%16],\n 8: lambda d: [1,2,3,1,2,3,1,2,3,6][d%10], # Used by separated_enc_dec model with lyrics\n 9: lambda d: [1,2,3,0][d % 4],\n 10: lambda d: [*[1,2,3,1,2,3,1,2,3],*[1,2,3,1,2,3,1,2,3,6]*7][d%79], # Used by large separated_enc_dec model with lyrics\n 11: lambda d: [6,6,0][d%3] if d%16 == 15 else [1,2,3][d%3],\n 12: lambda d: [7,7,0][d%3] if d%16 == 15 else [1,2,3][d%3], # Used by single_enc_dec model with lyrics\n }[attn_order]\n\n attn_cycle = {0:1, 1:2, 2:3, 3:2, 4:2, 5:4, 6:4, 7:16, 8:10, 9:4, 10:79, 11:16, 12:16}[attn_order]\n #assert n_depth % attn_cycle == 0, f'Depth {n_depth} not a multiple of cycle {attn_cycle} for attn_order {attn_order}'\n\n attn_block = lambda d: ResAttnBlock(n_in=n_in, n_ctx=n_ctx, n_head=n_head,\n attn_dropout=attn_dropout, resid_dropout=resid_dropout,\n afn=afn, scale=scale, mask=mask,\n zero_out=zero_out if attn_func(d) !=6 else True,\n init_scale=init_scale, res_scale=res_scale,\n m_attn=m_attn, m_mlp=m_mlp,\n checkpoint_attn=checkpoint_attn, checkpoint_mlp=checkpoint_mlp,\n attn_func=attn_func(d), blocks=blocks, spread=spread,\n encoder_dims=encoder_dims, prime_len=prime_len)\n\n self.checkpoint_res = checkpoint_res\n self._attn_mods = nn.ModuleList()\n for d in range(n_depth):\n self._attn_mods.append(attn_block(d))\n self.ws = []\n\n\n def set_record_attn(self, record_attn):\n \"\"\"\n Arguments:\n record_attn (bool or set): Makes forward prop dump self-attention\n softmaxes to self.ws. Either a set of layer indices indicating\n which layers to store, or a boolean value indicating whether to\n dump all.\n \"\"\"\n def _should_record_attn(layer_idx):\n if isinstance(record_attn, bool):\n return record_attn\n return layer_idx in record_attn\n for i, l in enumerate(self._attn_mods):\n l.attn.record_attn = _should_record_attn(i)\n if record_attn:\n assert self.ws == []\n for l in self._attn_mods:\n assert l.attn.w == None\n else:\n self.ws = []\n for l in self._attn_mods:\n l.attn.w = None\n\n def forward(self, x, encoder_kv=None, sample=False, fp16=False, fp16_out=False):\n if fp16:\n x = x.half()\n\n # Blocks\n for i,l in enumerate(self._attn_mods):\n if self.checkpoint_res == 1 and not sample:\n if l.attn_func == 6:\n assert encoder_kv is not None\n f = functools.partial(l, sample=sample)\n x = checkpoint(f, (x, encoder_kv), l.parameters(), True)\n else:\n f = functools.partial(l, encoder_kv=None, sample=sample)\n x = checkpoint(f, (x,), l.parameters(), True)\n else:\n if l.attn_func == 6:\n x = l(x, encoder_kv=encoder_kv, sample=sample)\n else:\n x = l(x, encoder_kv=None, sample=sample)\n if l.attn.record_attn:\n self.ws.append(l.attn.w)\n if not fp16_out:\n x = x.float()\n return x\n\n def check_cache(self, n_samples, sample_t, fp16):\n for l in self._attn_mods:\n l.attn.check_cache(n_samples, sample_t, fp16)\n\n def del_cache(self):\n for l in self._attn_mods:\n l.attn.del_cache()\n\n def check_sample(self):\n bs, l, s, d = (4, self.n_ctx, self.encoder_dims, self.n_in)\n prime = 5\n with t.no_grad():\n encoder_kv = t.randn(bs, s, d).cuda()\n x = t.randn(bs, l, d).cuda()\n y_forw = self.forward(x, encoder_kv=encoder_kv, sample=True)\n\n self.del_cache()\n x_chunks = t.chunk(x, 4, dim=1)\n y_chunks = []\n n = 0\n for x_chunk in x_chunks:\n self.check_cache(bs, n, False)\n y_chunk = self.forward(x_chunk, encoder_kv=encoder_kv, sample=True)\n y_chunks.append(y_chunk)\n n += x_chunk.shape[1]\n self.check_cache(bs, n, False)\n y_forw_in_chunks = t.cat(y_chunks, dim=1)\n\n max_err = t.max(t.abs(y_forw - y_forw_in_chunks))\n assert max_err <= 1e-6, f\"Max err is {max_err} {[i for i in range(l) if t.max(t.abs(y_forw - y_forw_in_chunks)[:, i, :]) > 1e-6]}\"\n\n\nif __name__ == '__main__':\n from jukebox.utils.dist_utils import setup_dist_from_mpi\n setup_dist_from_mpi(port=29600)\n n_in = 16\n n_ctx = 192\n n_head = 4\n n_depth = 12\n blocks = 16\n for attn_order in [0,2,6]:\n encoder_dims = {0: 0, 2: 0, 6: 64}[attn_order]\n prior = Transformer(n_in, n_ctx, n_head, n_depth, mask=True, attn_order=attn_order, encoder_dims=encoder_dims, blocks=blocks).cuda()\n prior.training = False\n prior.check_sample()\n print(f\"Checked attn_order: {attn_order}\")\n" }, { "path": "jukebox/utils/__init__.py", "content": "" }, { "path": "jukebox/utils/audio_utils.py", "content": "import numpy as np\nimport torch as t\nimport jukebox.utils.dist_adapter as dist\nimport soundfile\nimport librosa\nfrom jukebox.utils.dist_utils import print_once\n\nclass DefaultSTFTValues:\n def __init__(self, hps):\n self.sr = hps.sr\n self.n_fft = 2048\n self.hop_length = 256\n self.window_size = 6 * self.hop_length\n\nclass STFTValues:\n def __init__(self, hps, n_fft, hop_length, window_size):\n self.sr = hps.sr\n self.n_fft = n_fft\n self.hop_length = hop_length\n self.window_size = window_size\n\ndef calculate_bandwidth(dataset, hps, duration=600):\n hps = DefaultSTFTValues(hps)\n n_samples = int(dataset.sr * duration)\n l1, total, total_sq, n_seen, idx = 0.0, 0.0, 0.0, 0.0, dist.get_rank()\n spec_norm_total, spec_nelem = 0.0, 0.0\n while n_seen < n_samples:\n x = dataset[idx]\n if isinstance(x, (tuple, list)):\n x, y = x\n samples = x.astype(np.float64)\n stft = librosa.core.stft(np.mean(samples, axis=1), hps.n_fft, hop_length=hps.hop_length, win_length=hps.window_size)\n spec = np.absolute(stft)\n spec_norm_total += np.linalg.norm(spec)\n spec_nelem += 1\n n_seen += int(np.prod(samples.shape))\n l1 += np.sum(np.abs(samples))\n total += np.sum(samples)\n total_sq += np.sum(samples ** 2)\n idx += max(16, dist.get_world_size())\n\n if dist.is_available():\n from jukebox.utils.dist_utils import allreduce\n n_seen = allreduce(n_seen)\n total = allreduce(total)\n total_sq = allreduce(total_sq)\n l1 = allreduce(l1)\n spec_nelem = allreduce(spec_nelem)\n spec_norm_total = allreduce(spec_norm_total)\n\n mean = total / n_seen\n bandwidth = dict(l2 = total_sq / n_seen - mean ** 2,\n l1 = l1 / n_seen,\n spec = spec_norm_total / spec_nelem)\n print_once(bandwidth)\n return bandwidth\n\ndef audio_preprocess(x, hps):\n # Extra layer in case we want to experiment with different preprocessing\n # For two channel, blend randomly into mono (standard is .5 left, .5 right)\n\n # x: NTC\n x = x.float()\n if x.shape[-1]==2:\n if hps.aug_blend:\n mix=t.rand((x.shape[0],1), device=x.device) #np.random.rand()\n else:\n mix = 0.5\n x=(mix*x[:,:,0]+(1-mix)*x[:,:,1])\n elif x.shape[-1]==1:\n x=x[:,:,0]\n else:\n assert False, f'Expected channels {hps.channels}. Got unknown {x.shape[-1]} channels'\n\n # x: NT -> NTC\n x = x.unsqueeze(2)\n return x\n\ndef audio_postprocess(x, hps):\n return x\n\ndef stft(sig, hps):\n return t.stft(sig, hps.n_fft, hps.hop_length, win_length=hps.window_size, window=t.hann_window(hps.window_size, device=sig.device))\n\ndef spec(x, hps):\n return t.norm(stft(x, hps), p=2, dim=-1)\n\ndef norm(x):\n return (x.view(x.shape[0], -1) ** 2).sum(dim=-1).sqrt()\n\ndef squeeze(x):\n if len(x.shape) == 3:\n assert x.shape[-1] in [1,2]\n x = t.mean(x, -1)\n if len(x.shape) != 2:\n raise ValueError(f'Unknown input shape {x.shape}')\n return x\n\ndef spectral_loss(x_in, x_out, hps):\n hps = DefaultSTFTValues(hps)\n spec_in = spec(squeeze(x_in.float()), hps)\n spec_out = spec(squeeze(x_out.float()), hps)\n return norm(spec_in - spec_out)\n\ndef multispectral_loss(x_in, x_out, hps):\n losses = []\n assert len(hps.multispec_loss_n_fft) == len(hps.multispec_loss_hop_length) == len(hps.multispec_loss_window_size)\n args = [hps.multispec_loss_n_fft,\n hps.multispec_loss_hop_length,\n hps.multispec_loss_window_size]\n for n_fft, hop_length, window_size in zip(*args):\n hps = STFTValues(hps, n_fft, hop_length, window_size)\n spec_in = spec(squeeze(x_in.float()), hps)\n spec_out = spec(squeeze(x_out.float()), hps)\n losses.append(norm(spec_in - spec_out))\n return sum(losses) / len(losses)\n\ndef spectral_convergence(x_in, x_out, hps, epsilon=2e-3):\n hps = DefaultSTFTValues(hps)\n spec_in = spec(squeeze(x_in.float()), hps)\n spec_out = spec(squeeze(x_out.float()), hps)\n\n gt_norm = norm(spec_in)\n residual_norm = norm(spec_in - spec_out)\n mask = (gt_norm > epsilon).float()\n return (residual_norm * mask) / t.clamp(gt_norm, min=epsilon)\n\ndef log_magnitude_loss(x_in, x_out, hps, epsilon=1e-4):\n hps = DefaultSTFTValues(hps)\n spec_in = t.log(spec(squeeze(x_in.float()), hps) + epsilon)\n spec_out = t.log(spec(squeeze(x_out.float()), hps) + epsilon)\n return t.mean(t.abs(spec_in - spec_out))\n\ndef load_audio(file, sr, offset, duration, mono=False):\n # Librosa loads more filetypes than soundfile\n x, _ = librosa.load(file, sr=sr, mono=mono, offset=offset/sr, duration=duration/sr)\n if len(x.shape) == 1:\n x = x.reshape((1, -1))\n return x \n\n\ndef save_wav(fname, aud, sr):\n # clip before saving?\n aud = t.clamp(aud, -1, 1).cpu().numpy()\n for i in list(range(aud.shape[0])):\n soundfile.write(f'{fname}/item_{i}.wav', aud[i], samplerate=sr, format='wav')\n\n\n" }, { "path": "jukebox/utils/checkpoint.py", "content": "# Simple gradient checkpointing. Works with distributed data parallel\nimport torch as t\n\ndef checkpoint(func, inputs, params, flag):\n if flag:\n args = inputs + tuple(params)\n return CheckpointFunction.apply(func, len(inputs), *args)\n else:\n return func(*inputs)\n\nclass CheckpointFunction(t.autograd.Function):\n @staticmethod\n def forward(ctx, run_function, length, *args):\n ctx.run_function = run_function\n ctx.input_tensors = list(args[:length])\n ctx.input_params = list(args[length:])\n with t.no_grad():\n output_tensors = ctx.run_function(*ctx.input_tensors)\n return output_tensors\n\n @staticmethod\n def backward(ctx, *output_grads):\n for i in range(len(ctx.input_tensors)):\n temp = ctx.input_tensors[i]\n ctx.input_tensors[i] = temp.detach()\n ctx.input_tensors[i].requires_grad = temp.requires_grad\n with t.enable_grad():\n output_tensors = ctx.run_function(*ctx.input_tensors)\n input_grads = t.autograd.grad(output_tensors, ctx.input_tensors + ctx.input_params, output_grads, allow_unused=True)\n del ctx.input_tensors\n del output_tensors\n return (None, None) + input_grads\n" }, { "path": "jukebox/utils/dist_adapter.py", "content": "import torch.distributed as dist\nfrom enum import Enum\n\nclass ReduceOp(Enum):\n SUM = 0,\n PRODUCT = 1,\n MIN = 2,\n MAX = 3\n\n def ToDistOp(self):\n return {\n self.SUM: dist.ReduceOp.SUM,\n self.PRODUCT: dist.ReduceOp.PRODUCT,\n self.MIN: dist.ReduceOp.MIN,\n self.MAX: dist.ReduceOp.MAX\n }[self]\n\ndef is_available():\n return dist.is_available()\n\ndef get_rank():\n if is_available():\n return _get_rank()\n else:\n return 0\n\ndef get_world_size():\n if is_available():\n return _get_world_size()\n else:\n return 1\n\ndef barrier():\n if is_available():\n return _barrier()\n #else: do nothing\n\ndef all_gather(tensor_list, tensor):\n if is_available():\n return _all_gather(tensor_list, tensor)\n else:\n tensor_list[0] = tensor\n\ndef all_reduce(tensor, op=ReduceOp.SUM):\n if is_available():\n return _all_reduce(tensor, op)\n #else: do nothing\n\ndef reduce(tensor, dst, op=ReduceOp.SUM):\n if is_available():\n return _reduce(tensor, dst, op)\n #else: do nothing\n\ndef broadcast(tensor, src):\n if is_available():\n return _broadcast(tensor, src)\n #else: do nothing\n\ndef init_process_group(backend, init_method):\n if is_available():\n return _init_process_group(backend, init_method)\n #else: do nothing\n\ndef _get_rank():\n return dist.get_rank()\n\ndef _barrier():\n return dist.barrier()\n\ndef _get_world_size():\n return dist.get_world_size()\n\ndef _all_gather(tensor_list, tensor):\n return dist.all_gather(tensor_list, tensor)\n\ndef _all_reduce(tensor, op):\n return dist.all_reduce(tensor, op.ToDistOp())\n\ndef _reduce(tensor, dst, op):\n return dist.reduce(tensor, dst, op.ToDistOp())\n\ndef _broadcast(tensor, src):\n return dist.broadcast(tensor, src)\n\ndef _init_process_group(backend, init_method):\n return dist.init_process_group(backend, init_method)" }, { "path": "jukebox/utils/dist_utils.py", "content": "import os\nfrom time import sleep\nimport torch\nimport jukebox.utils.dist_adapter as dist\n\ndef print_once(msg):\n if (not dist.is_available()) or dist.get_rank()==0:\n print(msg)\n\ndef print_all(msg):\n if (not dist.is_available()):\n print(msg)\n elif dist.get_rank()%8==0:\n print(f'{dist.get_rank()//8}: {msg}')\n\ndef allgather(x):\n xs = [torch.empty_like(x) for _ in range(dist.get_world_size())]\n dist.all_gather(xs, x)\n xs = torch.cat(xs, dim=0)\n return xs\n\ndef allreduce(x, op=dist.ReduceOp.SUM):\n x = torch.tensor(x).float().cuda()\n dist.all_reduce(x, op=op)\n return x.item()\n\ndef allgather_lists(xs):\n bs = len(xs)\n total_bs = dist.get_world_size()*len(xs)\n lengths = torch.tensor([len(x) for x in xs], dtype=t.long, device='cuda')\n lengths = allgather(lengths)\n assert lengths.shape == (total_bs,)\n max_length = torch.max(lengths).item()\n\n xs = torch.tensor([[*x, *[0]*(max_length - len(x))] for x in xs], device='cuda')\n assert xs.shape == (bs, max_length), f'Expected {(bs, max_length)}, got {xs.shape}'\n xs = allgather(xs)\n assert xs.shape == (total_bs,max_length), f'Expected {(total_bs, max_length)}, got {xs.shape}'\n\n return [xs[i][:lengths[i]].cpu().numpy().tolist() for i in range(total_bs)]\n\ndef setup_dist_from_mpi(\n master_addr=\"127.0.0.1\", backend=\"nccl\", port=29500, n_attempts=5, verbose=False\n):\n if dist.is_available():\n return _setup_dist_from_mpi(master_addr, backend, port, n_attempts, verbose)\n else:\n use_cuda = torch.cuda.is_available()\n print(f'Using cuda {use_cuda}')\n\n mpi_rank = 0\n local_rank = 0\n\n device = torch.device(\"cuda\", local_rank) if use_cuda else torch.device(\"cpu\")\n torch.cuda.set_device(local_rank)\n\n return mpi_rank, local_rank, device\n\ndef _setup_dist_from_mpi(master_addr, backend, port, n_attempts, verbose):\n from mpi4py import MPI # This must be imported in order to get e rrors from all ranks to show up\n\n mpi_rank = MPI.COMM_WORLD.Get_rank()\n mpi_size = MPI.COMM_WORLD.Get_size()\n\n\n os.environ[\"RANK\"] = str(mpi_rank)\n os.environ[\"WORLD_SIZE\"] = str(mpi_size)\n os.environ[\"MASTER_ADDR\"] = master_addr\n os.environ[\"MASTER_PORT\"] = str(port)\n os.environ[\"NCCL_LL_THRESHOLD\"] = \"0\"\n os.environ[\"NCCL_NSOCKS_PERTHREAD\"] = \"2\"\n os.environ[\"NCCL_SOCKET_NTHREADS\"] = \"8\"\n\n # Pin this rank to a specific GPU on the node\n local_rank = mpi_rank % 8\n if torch.cuda.is_available():\n torch.cuda.set_device(local_rank)\n\n if verbose:\n print(f\"Connecting to master_addr: {master_addr}\")\n\n # There is a race condition when initializing NCCL with a large number of ranks (e.g 500 ranks)\n # We guard against the failure and then retry\n for attempt_idx in range(n_attempts):\n try:\n dist.init_process_group(backend=backend, init_method=f\"env://\")\n assert dist.get_rank() == mpi_rank\n\n use_cuda = torch.cuda.is_available()\n print(f'Using cuda {use_cuda}')\n local_rank = mpi_rank % 8\n device = torch.device(\"cuda\", local_rank) if use_cuda else torch.device(\"cpu\")\n torch.cuda.set_device(local_rank)\n\n return mpi_rank, local_rank, device\n except RuntimeError as e:\n print(f\"Caught error during NCCL init (attempt {attempt_idx} of {n_attempts}): {e}\")\n sleep(1 + (0.01 * mpi_rank)) # Sleep to avoid thundering herd\n pass\n\n raise RuntimeError(\"Failed to initialize NCCL\")\n" }, { "path": "jukebox/utils/ema.py", "content": "import torch\nfrom torch._utils import _flatten_dense_tensors\nimport numpy as np\n\n# EMA always in float, as accumulation needs lots of bits\nclass EMA:\n def __init__(self, params, mu=0.999):\n self.mu = mu\n self.state = [(p, self.get_model_state(p)) for p in params if p.requires_grad]\n\n def get_model_state(self, p):\n return p.data.float().detach().clone()\n\n def step(self):\n for p, state in self.state:\n state.mul_(self.mu).add_(1 - self.mu, p.data.float())\n\n def swap(self):\n # swap ema and model params\n for p, state in self.state:\n other_state = self.get_model_state(p)\n p.data.copy_(state.type_as(p.data))\n state.copy_(other_state)\n\n\nclass CPUEMA:\n def __init__(self, params, mu=0.999, freq=1):\n self.mu = mu**freq\n self.state = [(p, self.get_model_state(p)) for p in params if p.requires_grad]\n self.freq = freq\n self.steps = 0\n\n def get_model_state(self, p):\n with torch.no_grad():\n state = p.data.float().detach().cpu().numpy()\n return state\n\n def step(self):\n with torch.no_grad():\n self.steps += 1\n if self.steps % self.freq == 0:\n for i in range(len(self.state)):\n p, state = self.state[i]\n state = torch.from_numpy(state).cuda()\n state.mul_(self.mu).add_(1 - self.mu, p.data.float())\n self.state[i] = (p, state.cpu().numpy())\n\n def swap(self):\n with torch.no_grad():\n # swap ema and model params\n for p, state in self.state:\n other_state = self.get_model_state(p)\n p.data.copy_(torch.from_numpy(state).type_as(p.data))\n np.copyto(state, other_state)\n\nclass FusedEMA:\n def __init__(self, params, mu=0.999):\n self.mu = mu\n params = list(params)\n self.params = {}\n self.params['fp16'] = [p for p in params if p.requires_grad and p.data.dtype == torch.float16]\n self.params['fp32'] = [p for p in params if p.requires_grad and p.data.dtype != torch.float16]\n self.groups = [group for group in self.params.keys() if len(self.params[group]) > 0]\n self.state = {}\n for group in self.groups:\n self.state[group] = self.get_model_state(group)\n\n def get_model_state(self, group):\n params = self.params[group]\n return _flatten_dense_tensors([p.data.float() for p in params])\n # if self.fp16:\n # return _flatten_dense_tensors([p.data.half() for p in self.param_group if p.dtype])\n # else:\n # return _flatten_dense_tensors([p.data for p in self.param_group])\n\n def step(self):\n for group in self.groups:\n self.state[group].mul_(self.mu).add_(1 - self.mu, self.get_model_state(group))\n\n def swap(self):\n # swap ema and model params\n for group in self.groups:\n other_state = self.get_model_state(group)\n state = self.state[group]\n params = self.params[group]\n offset = 0\n for p in params:\n numel = p.data.numel()\n p.data = state.narrow(0, offset, numel).view_as(p.data).type_as(p.data)\n offset += numel\n\n self.state[group] = other_state\n\n\n" }, { "path": "jukebox/utils/fp16.py", "content": "# Utils for fp16 training.\nimport importlib\nimport math\nimport numpy as np\nimport torch\nimport jukebox.utils.dist_adapter as dist\nfrom torch.optim import Optimizer\nfrom torch._utils import _flatten_dense_tensors\n\nfrom jukebox.utils.dist_utils import allreduce\n\ndef adam_step(p: torch.Tensor, out_p: torch.Tensor, exp_avg: torch.Tensor, exp_avg_sq: torch.Tensor, grad: torch.Tensor,\n lr: float, beta1: float, beta2: float, eps: float, scale: float, step: int, eps_mode: int, bias_correction: int, weight_decay: float):\n assert bias_correction == 1\n assert eps_mode == 1\n\n grad = grad.float()\n grad.div_(scale)\n\n # Decay the first and second moment running average coefficient\n exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1)\n exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)\n denom = exp_avg_sq.sqrt().add_(eps)\n\n bias_correction1 = 1 - beta1 ** step\n bias_correction2 = 1 - beta2 ** step\n step_size = lr * math.sqrt(bias_correction2) / bias_correction1\n\n p.add_(exp_avg/denom + weight_decay*p.float(), alpha=-step_size)\n\n# Import fused_adam if we have apex, otherwise use regular adam\ntry:\n fused_adam_cuda = importlib.import_module(\"fused_adam_cuda\")\n fused_adam_step = fused_adam_cuda.adam\n print(\"Using apex fused_adam_cuda\")\nexcept ModuleNotFoundError:\n fused_adam_step = adam_step\n\ndef backward(loss, params, scalar, fp16, logger):\n # Perform backward\n if not fp16:\n scale = 1.0\n loss.backward()\n gn = grad_norm(params, scale)\n return loss, scale, gn, False, False\n else:\n scale = scalar.get_scale()\n loss = (loss.float())*scale\n overflow_loss = check_overflow(loss.item())\n overflow_loss = allreduce(int(overflow_loss), op=dist.ReduceOp.MAX) > 0\n if not overflow_loss:\n loss.backward()\n gn = grad_norm(params, scale)\n overflow_grad = check_overflow(gn)\n overflow_grad = allreduce(int(overflow_grad), op=dist.ReduceOp.MAX) > 0\n scalar.update_scale(overflow_grad)\n else:\n gn = 0.0\n overflow_grad = True\n loss = (loss.detach().float()) / scale # Should delete computation graph for overflow\n if logger.rank == 0:\n if loss > 12.: print(f\"\\nWarning. Loss is {loss}\")\n if overflow_loss: print(f\"\\nOverflow in forward. Loss {loss}, lgscale {np.log2(scale)}. Skipping batch completely (no backward, scale update)\")\n elif overflow_grad: print(f\"\\nOverflow in backward. Loss {loss}, grad norm {gn}, lgscale {np.log2(scale)}, new lgscale {np.log2(scalar.get_scale())}\")\n return loss, scale, gn, overflow_loss, overflow_grad\n\n# Automatic loss scaling\nclass LossScalar(object):\n def __init__(self,\n loss_scale,\n init_scale=2. ** 16,\n scale_factor=2. ** (1. / 1000),\n scale_window=1):\n if loss_scale == None:\n # Use dynamic loss scaling\n self.dynamic = True\n self.loss_scale = init_scale\n else:\n self.dynamic = False\n self.loss_scale = loss_scale\n self.max_loss_scale = 2.**24\n self.scale_factor = scale_factor\n self.scale_window = scale_window\n self.unskipped = 0\n self.overflow = False\n\n def get_scale(self):\n return self.loss_scale\n\n def update_scale(self, overflow):\n if overflow and self.dynamic:\n self.loss_scale /= 2.\n self.unskipped = 0\n else:\n self.unskipped += 1\n\n if self.unskipped == self.scale_window and self.dynamic:\n self.loss_scale = min(self.max_loss_scale, self.loss_scale * self.scale_factor)\n self.unskipped = 0\n\ndef check_overflow(val):\n return (val == float('inf')) or (val == -float('inf')) or (val != val)\n\ndef grad_norm(params, scale, flat=False):\n params = list(params)\n if flat:\n # Faster but more memory\n fp16_grads = [p.grad for p in params if p.grad is not None and p.data.dtype == torch.float16]\n fp16_norm = 0.0 if len(fp16_grads) == 0 else float(_flatten_dense_tensors(fp16_grads).norm(p=2, dtype=torch.float32))\n fp32_grads = [p.grad for p in params if p.grad is not None and p.data.dtype != torch.float16]\n fp32_norm = 0.0 if len(fp32_grads) == 0 else float(_flatten_dense_tensors(fp32_grads).norm(p=2))\n grad_norm = (fp16_norm**2 + fp32_norm**2)**0.5\n else:\n # Slightly slower but less memory\n grad_norm = 0.0\n for p in params:\n if p.grad is not None:\n grad_norm += p.grad.norm(p=2, dtype=torch.float32)**2\n grad_norm = float(grad_norm**0.5)\n return grad_norm / scale\n\ndef clipped_grad_scale(grad_norm, max_grad_norm, scale):\n clip = grad_norm / max_grad_norm\n if clip > 1:\n scale = clip * scale\n return scale\n\nclass FP16FusedAdam(Optimizer):\n def __init__(\n self,\n params,\n lr=1e-3,\n bias_correction=True,\n betas=(0.9, 0.999),\n eps=1e-8,\n eps_inside_sqrt=False,\n weight_decay=0.0,\n amsgrad=False,\n ):\n if amsgrad:\n raise RuntimeError(\"FusedAdam does not support the AMSGrad variant.\")\n defaults = dict(\n lr=lr, bias_correction=bias_correction, betas=betas, eps=eps, weight_decay=weight_decay\n )\n super(FP16FusedAdam, self).__init__(params, defaults)\n self.eps_mode = 0 if eps_inside_sqrt else 1\n self.FLOAT16_MAX = 65504.0\n self.init_state()\n\n def init_state(self):\n for group in self.param_groups:\n for p in group[\"params\"]:\n assert p.requires_grad == True\n state = self.state[p]\n if len(state) == 0:\n state[\"step\"] = 0\n # Exponential moving average of gradient values\n state[\"exp_avg\"] = torch.zeros_like(p.data)\n # Exponential moving average of squared gradient values\n state[\"exp_avg_sq\"] = torch.zeros_like(p.data)\n if p.data.dtype == torch.float16:\n state[\"scale_exp_avg\"] = 1.0\n state[\"scale_exp_avg_sq\"] = 1.0\n\n def step(self, closure=None, scale=1.0):\n \"\"\"Performs a single optimization step. Scales gradients down by scale\n Arguments:\n closure (callable, optional): A closure that reevaluates the model\n and returns the loss.\n scale (float, optional): factor to divide gradient tensor values\n by before applying to weights. (default: 1)\n \"\"\"\n loss = None\n if closure is not None:\n loss = closure()\n\n for group in self.param_groups:\n bias_correction = 1 if group[\"bias_correction\"] else 0\n\n for p in group[\"params\"]:\n if p.grad is None:\n continue\n grad = p.grad.data\n\n state = self.state[p]\n\n if p.data.dtype == torch.float16:\n exp_avg, exp_avg_sq = (\n state[\"exp_avg\"].float() * state[\"scale_exp_avg\"],\n state[\"exp_avg_sq\"].float() * state[\"scale_exp_avg_sq\"],\n )\n else:\n exp_avg, exp_avg_sq = state[\"exp_avg\"], state[\"exp_avg_sq\"]\n beta1, beta2 = group[\"betas\"]\n\n state[\"step\"] += 1\n\n out_p = torch.tensor([], dtype=torch.float)\n fused_adam_step(\n p.data,\n out_p,\n exp_avg,\n exp_avg_sq,\n grad,\n group[\"lr\"],\n beta1,\n beta2,\n group[\"eps\"],\n scale,\n state[\"step\"],\n self.eps_mode,\n bias_correction,\n group[\"weight_decay\"],\n )\n\n if p.data.dtype == torch.float16:\n state[\"scale_exp_avg\"] = (\n 1e-8 + float(torch.norm(exp_avg, float(\"inf\"))) / self.FLOAT16_MAX\n )\n state[\"scale_exp_avg_sq\"] = (\n 1e-8 + float(torch.norm(exp_avg_sq, float(\"inf\"))) / self.FLOAT16_MAX\n )\n state[\"exp_avg\"] = (exp_avg / state[\"scale_exp_avg\"]).half()\n state[\"exp_avg_sq\"] = (exp_avg_sq / state[\"scale_exp_avg_sq\"]).half()\n\n return loss\n\n\nclass FusedAdam(Optimizer):\n def __init__(\n self,\n params,\n lr=1e-3,\n bias_correction=True,\n betas=(0.9, 0.999),\n eps=1e-8,\n eps_inside_sqrt=False,\n weight_decay=0.0,\n amsgrad=False,\n ):\n if amsgrad:\n raise RuntimeError(\"FusedAdam does not support the AMSGrad variant.\")\n defaults = dict(\n lr=lr, bias_correction=bias_correction, betas=betas, eps=eps, weight_decay=weight_decay\n )\n super(FusedAdam, self).__init__(params, defaults)\n self.eps_mode = 0 if eps_inside_sqrt else 1\n\n def step(self, closure=None, scale=1.0):\n \"\"\"Performs a single optimization step. Scales gradients down by scale\n Arguments:\n closure (callable, optional): A closure that reevaluates the model\n and returns the loss.\n scale (float, optional): factor to divide gradient tensor values\n by before applying to weights. (default: 1)\n \"\"\"\n loss = None\n if closure is not None:\n loss = closure()\n\n for group in self.param_groups:\n bias_correction = 1 if group[\"bias_correction\"] else 0\n\n for p in group[\"params\"]:\n if p.grad is None:\n continue\n grad = p.grad.data\n\n state = self.state[p]\n\n # State initialization\n if len(state) == 0:\n state[\"step\"] = 0\n # Exponential moving average of gradient values\n state[\"exp_avg\"] = torch.zeros_like(p.data).float()\n # Exponential moving average of squared gradient values\n state[\"exp_avg_sq\"] = torch.zeros_like(p.data).float()\n\n exp_avg, exp_avg_sq = state[\"exp_avg\"], state[\"exp_avg_sq\"]\n beta1, beta2 = group[\"betas\"]\n\n state[\"step\"] += 1\n\n out_p = torch.tensor([], dtype=torch.float)\n fused_adam_step(\n p.data,\n out_p,\n exp_avg,\n exp_avg_sq,\n grad,\n group[\"lr\"],\n beta1,\n beta2,\n group[\"eps\"],\n scale,\n state[\"step\"],\n self.eps_mode,\n bias_correction,\n group[\"weight_decay\"],\n )\n\n return loss\n\n" }, { "path": "jukebox/utils/io.py", "content": "import numpy as np\nimport av\nimport torch as t\nimport jukebox.utils.dist_adapter as dist\n\ndef get_duration_sec(file, cache=False):\n try:\n with open(file + '.dur', 'r') as f:\n duration = float(f.readline().strip('\\n'))\n return duration\n except:\n container = av.open(file)\n audio = container.streams.get(audio=0)[0]\n duration = audio.duration * float(audio.time_base)\n if cache:\n with open(file + '.dur', 'w') as f:\n f.write(str(duration) + '\\n')\n return duration\n\ndef load_audio(file, sr, offset, duration, resample=True, approx=False, time_base='samples', check_duration=True):\n if time_base == 'sec':\n offset = offset * sr\n duration = duration * sr\n # Loads at target sr, stereo channels, seeks from offset, and stops after duration\n container = av.open(file)\n audio = container.streams.get(audio=0)[0] # Only first audio stream\n audio_duration = audio.duration * float(audio.time_base)\n if approx:\n if offset + duration > audio_duration*sr:\n # Move back one window. Cap at audio_duration\n offset = np.min(audio_duration*sr - duration, offset - duration)\n else:\n if check_duration:\n assert offset + duration <= audio_duration*sr, f'End {offset + duration} beyond duration {audio_duration*sr}'\n if resample:\n resampler = av.AudioResampler(format='fltp',layout='stereo', rate=sr)\n else:\n assert sr == audio.sample_rate\n offset = int(offset / sr / float(audio.time_base)) #int(offset / float(audio.time_base)) # Use units of time_base for seeking\n duration = int(duration) #duration = int(duration * sr) # Use units of time_out ie 1/sr for returning\n sig = np.zeros((2, duration), dtype=np.float32)\n container.seek(offset, stream=audio)\n total_read = 0\n for frame in container.decode(audio=0): # Only first audio stream\n if resample:\n frame.pts = None\n frame = resampler.resample(frame)\n frame = frame.to_ndarray(format='fltp') # Convert to floats and not int16\n read = frame.shape[-1]\n if total_read + read > duration:\n read = duration - total_read\n sig[:, total_read:total_read + read] = frame[:, :read]\n total_read += read\n if total_read == duration:\n break\n assert total_read <= duration, f'Expected {duration} frames, got {total_read}'\n return sig, sr\n\ndef test_simple_loader():\n import librosa\n from tqdm import tqdm\n\n collate_fn = lambda batch: t.stack([t.from_numpy(b) for b in batch], dim=0)\n\n def get_batch(file, loader):\n y1, sr = loader(file, sr=44100, offset=0.0, duration=6.0, time_base='sec')\n y2, sr = loader(file, sr=44100, offset=20.0, duration=6.0, time_base='sec')\n return [y1, y2]\n\n def load(file, loader):\n batch = get_batch(file, loader) # np\n x = collate_fn(batch) # torch cpu\n x = x.to('cuda', non_blocking=True) # torch gpu\n return x\n\n files = librosa.util.find_files('/root/data/', ['mp3', 'm4a', 'opus'])\n print(files[:10])\n loader = load_audio\n print(\"Loader\", loader.__name__)\n x = t.randn(2, 2).cuda()\n x = load(files[0], loader)\n for i,file in enumerate(tqdm(files)):\n x = load(file, loader)\n if i == 100:\n break\n\ndef test_dataset_loader():\n from tqdm import tqdm\n from torch.utils.data import DataLoader\n from torch.utils.data.distributed import DistributedSampler\n from jukebox.utils.audio_utils import audio_preprocess, audio_postprocess\n from jukebox.hparams import setup_hparams\n from jukebox.data.files_dataset import FilesAudioDataset\n hps = setup_hparams(\"teeny\", {})\n hps.sr = 22050 # 44100\n hps.hop_length = 512\n hps.labels = False\n hps.channels = 2\n hps.aug_shift = False\n hps.bs = 2\n hps.nworkers = 2 # Getting 20 it/s with 2 workers, 10 it/s with 1 worker\n print(hps)\n dataset = hps.dataset\n root = hps.root\n from tensorboardX import SummaryWriter\n sr = {22050: '22k', 44100: '44k', 48000: '48k'}[hps.sr]\n writer = SummaryWriter(f'{root}/{dataset}/logs/{sr}/logs')\n dataset = FilesAudioDataset(hps)\n print(\"Length of dataset\", len(dataset))\n\n # Torch Loader\n collate_fn = lambda batch: t.stack([t.from_numpy(b) for b in batch], 0)\n sampler = DistributedSampler(dataset)\n train_loader = DataLoader(dataset, batch_size=hps.bs, num_workers=hps.nworkers, pin_memory=False, sampler=sampler,\n drop_last=True, collate_fn=collate_fn)\n\n dist.barrier()\n sampler.set_epoch(0)\n for i, x in enumerate(tqdm(train_loader)):\n x = x.to('cuda', non_blocking=True)\n for j, aud in enumerate(x):\n writer.add_audio('in_' + str(i*hps.bs + j), aud, 1, hps.sr)\n print(\"Wrote in\")\n x = audio_preprocess(x, hps)\n x = audio_postprocess(x, hps)\n for j, aud in enumerate(x):\n writer.add_audio('out_' + str(i*hps.bs + j), aud, 1, hps.sr)\n print(\"Wrote out\")\n dist.barrier()\n break\n\nif __name__ == '__main__':\n from jukebox.utils.dist_utils import setup_dist_from_mpi\n setup_dist_from_mpi(port=29500)\n test_dataset_loader()\n\n" }, { "path": "jukebox/utils/logger.py", "content": "import torch as t\nimport jukebox.utils.dist_adapter as dist\nfrom tqdm import tqdm\nfrom datetime import date\nimport os\nimport sys\n\ndef def_tqdm(x):\n return tqdm(x, leave=True, file=sys.stdout, bar_format=\"{n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}{postfix}]\")\n\ndef get_range(x):\n if dist.get_rank() == 0:\n return def_tqdm(x)\n else:\n return x\n\ndef init_logging(hps, local_rank, rank):\n logdir = f\"{hps.local_logdir}/{hps.name}\"\n if local_rank == 0:\n if not os.path.exists(logdir):\n os.makedirs(logdir)\n with open(logdir + 'argv.txt', 'w') as f:\n f.write(hps.argv + '\\n')\n print(\"Logging to\", logdir)\n logger = Logger(logdir, rank)\n metrics = Metrics()\n logger.add_text('hps', str(hps))\n return logger, metrics\n\ndef get_name(hps):\n name = \"\"\n for key, value in hps.items():\n name += f\"{key}_{value}_\"\n return name\n\ndef average_metrics(_metrics):\n metrics = {}\n for _metric in _metrics:\n for key, val in _metric.items():\n if key not in metrics:\n metrics[key] = []\n metrics[key].append(val)\n return {key: sum(vals)/len(vals) for key, vals in metrics.items()}\n\nclass Metrics:\n def __init__(self):\n self.sum = {}\n self.n = {}\n\n def update(self, tag, val, batch):\n # v is average value over batch\n # store total value and total batch, returns dist average\n sum = t.tensor(val * batch).float().cuda()\n n = t.tensor(batch).float().cuda()\n dist.all_reduce(sum)\n dist.all_reduce(n)\n sum = sum.item()\n n = n.item()\n self.sum[tag] = self.sum.get(tag, 0.0) + sum\n self.n[tag] = self.n.get(tag, 0.0) + n\n return sum / n\n\n def avg(self, tag):\n if tag in self.sum:\n return self.sum[tag] / self.n[tag]\n else:\n return 0.0\n\n def reset(self):\n self.sum = {}\n self.n = {}\n\nclass Logger:\n def __init__(self, logdir, rank):\n if rank == 0:\n from tensorboardX import SummaryWriter\n self.sw = SummaryWriter(f\"{logdir}/logs\")\n self.iters = 0\n self.rank = rank\n self.works = []\n self.logdir = logdir\n\n def step(self):\n self.iters += 1\n\n def flush(self):\n if self.rank == 0:\n self.sw.flush()\n\n def add_text(self, tag, text):\n if self.rank == 0:\n self.sw.add_text(tag, text, self.iters)\n\n def add_audios(self, tag, auds, sample_rate=22050, max_len=None, max_log=8):\n if self.rank == 0:\n for i in range(min(len(auds), max_log)):\n if max_len:\n self.sw.add_audio(f\"{i}/{tag}\", auds[i][:max_len * sample_rate], self.iters, sample_rate)\n else:\n self.sw.add_audio(f\"{i}/{tag}\", auds[i], self.iters, sample_rate)\n\n def add_audio(self, tag, aud, sample_rate=22050):\n if self.rank == 0:\n self.sw.add_audio(tag, aud, self.iters, sample_rate)\n\n def add_images(self, tag, img, dataformats=\"NHWC\"):\n if self.rank == 0:\n self.sw.add_images(tag, img, self.iters, dataformats=dataformats)\n\n def add_image(self, tag, img):\n if self.rank == 0:\n self.sw.add_image(tag, img, self.iters)\n\n def add_scalar(self, tag, val):\n if self.rank == 0:\n self.sw.add_scalar(tag, val, self.iters)\n\n def get_range(self, loader):\n if self.rank == 0:\n self.trange = def_tqdm(loader)\n else:\n self.trange = loader\n return enumerate(self.trange)\n\n def close_range(self):\n if self.rank == 0:\n self.trange.close()\n\n def set_postfix(self, *args, **kwargs):\n if self.rank == 0:\n self.trange.set_postfix(*args, **kwargs)\n\n # For logging summaries of varies graph ops\n def add_reduce_scalar(self, tag, layer, val):\n if self.iters % 100 == 0:\n with t.no_grad():\n val = val.float().norm()/float(val.numel())\n work = dist.reduce(val, 0, async_op=True)\n self.works.append((tag, layer, val, work))\n\n def finish_reduce(self):\n for tag, layer, val, work in self.works:\n work.wait()\n if self.rank == 0:\n val = val.item()/dist.get_world_size()\n self.lw[layer].add_scalar(tag, val, self.iters)\n self.works = []\n" }, { "path": "jukebox/utils/remote_utils.py", "content": "import sys\nimport subprocess\n\ndef download(remote_path, local_path, async_download=False):\n args = ['wget', '-O', local_path, remote_path]\n print(\"Running \", \" \".join(args))\n if async_download:\n subprocess.Popen(args)\n else:\n subprocess.call(args)\n\n# GCE\ndef gs_download(gs_path, local_path, async_download=False):\n args = ['gsutil',\n '-o', 'GSUtil:parallel_thread_count=1',\n '-o', 'GSUtil:sliced_object_download_max_components=8',\n 'cp', gs_path, local_path]\n if async_download:\n subprocess.Popen(args)\n else:\n subprocess.call(args)\n\n\ndef gs_upload(local_path, gs_path, async_upload=False):\n # NOTE: Download and upload have differ -o flags.\n # We also use -n to prevent clobbering checkpoints by mistake\n assert not local_path.startswith(\"gs://\")\n assert gs_path.startswith(\"gs://\")\n args = ['gsutil',\n '-o', 'GSUtil:parallel_composite_upload_threshold=150M',\n 'cp', '-n', local_path, gs_path]\n if async_upload:\n subprocess.Popen(args)\n else:\n subprocess.call(args)\n\ndef ls(regex):\n outputs = subprocess.check_output(['gsutil', 'ls', regex]).decode(sys.stdout.encoding)\n outputs = outputs.split('\\n')\n outputs = [output for output in outputs if output is not '']\n return outputs\n\n" }, { "path": "jukebox/utils/sample_utils.py", "content": "import torch as t\n\ndef split_batch(obj, n_samples, split_size):\n n_passes = (n_samples + split_size - 1) // split_size\n if isinstance(obj, t.Tensor):\n return t.split(obj, split_size, dim=0)\n elif isinstance(obj, list):\n return list(zip(*[t.split(item, split_size, dim=0) for item in obj]))\n elif obj is None:\n return [None] * n_passes\n else:\n raise TypeError('Unknown input type')\n\n# Break total_length into hops/windows of size n_ctx separated by hop_length\ndef get_starts(total_length, n_ctx, hop_length):\n starts = []\n for start in range(0, total_length - n_ctx + hop_length, hop_length):\n if start + n_ctx >= total_length:\n # Last hop could be smaller, we make it n_ctx to maximise context\n start = total_length - n_ctx\n starts.append(start)\n return starts\n" }, { "path": "jukebox/utils/torch_utils.py", "content": "import gc\nimport torch as t\n\ndef freeze_model(model):\n model.eval()\n for params in model.parameters():\n params.requires_grad = False\n\n\ndef unfreeze_model(model):\n model.train()\n for params in model.parameters():\n params.requires_grad = True\n\ndef zero_grad(model):\n for p in model.parameters():\n if p.requires_grad and p.grad is not None:\n p.grad = None\n\ndef empty_cache():\n gc.collect()\n t.cuda.empty_cache()\n\ndef assert_shape(x, exp_shape):\n assert x.shape == exp_shape, f\"Expected {exp_shape} got {x.shape}\"\n\ndef count_parameters(model):\n return sum(p.numel() for p in model.parameters() if p.requires_grad)\n\ndef count_state(model):\n return sum(s.numel() for s in model.state_dict().values())\n\n" }, { "path": "jukebox/vqvae/__init__.py", "content": "" }, { "path": "jukebox/vqvae/bottleneck.py", "content": "import numpy as np\nimport torch as t\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport jukebox.utils.dist_adapter as dist\n\nclass BottleneckBlock(nn.Module):\n def __init__(self, k_bins, emb_width, mu):\n super().__init__()\n self.k_bins = k_bins\n self.emb_width = emb_width\n self.mu = mu\n self.reset_k()\n self.threshold = 1.0\n\n def reset_k(self):\n self.init = False\n self.k_sum = None\n self.k_elem = None\n self.register_buffer('k', t.zeros(self.k_bins, self.emb_width).cuda())\n\n def _tile(self, x):\n d, ew = x.shape\n if d < self.k_bins:\n n_repeats = (self.k_bins + d - 1) // d\n std = 0.01 / np.sqrt(ew)\n x = x.repeat(n_repeats, 1)\n x = x + t.randn_like(x) * std\n return x\n\n def init_k(self, x):\n mu, emb_width, k_bins = self.mu, self.emb_width, self.k_bins\n self.init = True\n # init k_w using random vectors from x\n y = self._tile(x)\n _k_rand = y[t.randperm(y.shape[0])][:k_bins]\n dist.broadcast(_k_rand, 0)\n self.k = _k_rand\n assert self.k.shape == (k_bins, emb_width)\n self.k_sum = self.k\n self.k_elem = t.ones(k_bins, device=self.k.device)\n\n def restore_k(self, num_tokens=None, threshold=1.0):\n mu, emb_width, k_bins = self.mu, self.emb_width, self.k_bins\n self.init = True\n assert self.k.shape == (k_bins, emb_width)\n self.k_sum = self.k.clone()\n self.k_elem = t.ones(k_bins, device=self.k.device)\n if num_tokens is not None:\n expected_usage = num_tokens / k_bins\n self.k_elem.data.mul_(expected_usage)\n self.k_sum.data.mul_(expected_usage)\n self.threshold = threshold\n\n def update_k(self, x, x_l):\n mu, emb_width, k_bins = self.mu, self.emb_width, self.k_bins\n with t.no_grad():\n # Calculate new centres\n x_l_onehot = t.zeros(k_bins, x.shape[0], device=x.device) # k_bins, N * L\n x_l_onehot.scatter_(0, x_l.view(1, x.shape[0]), 1)\n\n _k_sum = t.matmul(x_l_onehot, x) # k_bins, w\n _k_elem = x_l_onehot.sum(dim=-1) # k_bins\n y = self._tile(x)\n _k_rand = y[t.randperm(y.shape[0])][:k_bins]\n\n dist.broadcast(_k_rand, 0)\n dist.all_reduce(_k_sum)\n dist.all_reduce(_k_elem)\n\n # Update centres\n old_k = self.k\n self.k_sum = mu * self.k_sum + (1. - mu) * _k_sum # w, k_bins\n self.k_elem = mu * self.k_elem + (1. - mu) * _k_elem # k_bins\n usage = (self.k_elem.view(k_bins, 1) >= self.threshold).float()\n self.k = usage * (self.k_sum.view(k_bins, emb_width) / self.k_elem.view(k_bins, 1)) \\\n + (1 - usage) * _k_rand\n _k_prob = _k_elem / t.sum(_k_elem) # x_l_onehot.mean(dim=-1) # prob of each bin\n entropy = -t.sum(_k_prob * t.log(_k_prob + 1e-8)) # entropy ie how diverse\n used_curr = (_k_elem >= self.threshold).sum()\n usage = t.sum(usage)\n dk = t.norm(self.k - old_k) / np.sqrt(np.prod(old_k.shape))\n return dict(entropy=entropy,\n used_curr=used_curr,\n usage=usage,\n dk=dk)\n\n def preprocess(self, x):\n # NCT -> NTC -> [NT, C]\n x = x.permute(0, 2, 1).contiguous()\n x = x.view(-1, x.shape[-1]) # x_en = (N * L, w), k_j = (w, k_bins)\n\n if x.shape[-1] == self.emb_width:\n prenorm = t.norm(x - t.mean(x)) / np.sqrt(np.prod(x.shape))\n elif x.shape[-1] == 2 * self.emb_width:\n x1, x2 = x[...,:self.emb_width], x[...,self.emb_width:]\n prenorm = (t.norm(x1 - t.mean(x1)) / np.sqrt(np.prod(x1.shape))) + (t.norm(x2 - t.mean(x2)) / np.sqrt(np.prod(x2.shape)))\n\n # Normalise\n x = x1 + x2\n else:\n assert False, f\"Expected {x.shape[-1]} to be (1 or 2) * {self.emb_width}\"\n return x, prenorm\n\n def postprocess(self, x_l, x_d, x_shape):\n # [NT, C] -> NTC -> NCT\n N, T = x_shape\n x_d = x_d.view(N, T, -1).permute(0, 2, 1).contiguous()\n x_l = x_l.view(N, T)\n return x_l, x_d\n\n def quantise(self, x):\n # Calculate latent code x_l\n k_w = self.k.t()\n distance = t.sum(x ** 2, dim=-1, keepdim=True) - 2 * t.matmul(x, k_w) + t.sum(k_w ** 2, dim=0,\n keepdim=True) # (N * L, b)\n min_distance, x_l = t.min(distance, dim=-1)\n fit = t.mean(min_distance)\n return x_l, fit\n\n def dequantise(self, x_l):\n x = F.embedding(x_l, self.k)\n return x\n\n def encode(self, x):\n N, width, T = x.shape\n\n # Preprocess.\n x, prenorm = self.preprocess(x)\n\n # Quantise\n x_l, fit = self.quantise(x)\n\n # Postprocess.\n x_l = x_l.view(N, T)\n return x_l\n\n def decode(self, x_l):\n N, T = x_l.shape\n width = self.emb_width\n\n # Dequantise\n x_d = self.dequantise(x_l)\n\n # Postprocess\n x_d = x_d.view(N, T, width).permute(0, 2, 1).contiguous()\n return x_d\n\n def forward(self, x, update_k=True):\n N, width, T = x.shape\n\n # Preprocess\n x, prenorm = self.preprocess(x)\n\n # Init k if not inited\n if update_k and not self.init:\n self.init_k(x)\n\n # Quantise and dequantise through bottleneck\n x_l, fit = self.quantise(x)\n x_d = self.dequantise(x_l)\n\n # Update embeddings\n if update_k:\n update_metrics = self.update_k(x, x_l)\n else:\n update_metrics = {}\n\n # Loss\n commit_loss = t.norm(x_d.detach() - x) ** 2 / np.prod(x.shape)\n\n # Passthrough\n x_d = x + (x_d - x).detach()\n\n # Postprocess\n x_l, x_d = self.postprocess(x_l, x_d, (N,T))\n return x_l, x_d, commit_loss, dict(fit=fit,\n pn=prenorm,\n **update_metrics)\n\n\nclass Bottleneck(nn.Module):\n def __init__(self, l_bins, emb_width, mu, levels):\n super().__init__()\n self.levels = levels\n level_block = lambda level: BottleneckBlock(l_bins, emb_width, mu)\n self.level_blocks = nn.ModuleList()\n for level in range(self.levels):\n self.level_blocks.append(level_block(level))\n\n def encode(self, xs):\n zs = [level_block.encode(x) for (level_block, x) in zip(self.level_blocks, xs)]\n return zs\n\n def decode(self, zs, start_level=0, end_level=None):\n if end_level is None:\n end_level = self.levels\n xs_quantised = [level_block.decode(z) for (level_block, z) in zip(self.level_blocks[start_level:end_level], zs)]\n return xs_quantised\n\n def forward(self, xs):\n zs, xs_quantised, commit_losses, metrics = [], [], [], []\n for level in range(self.levels):\n level_block = self.level_blocks[level]\n x = xs[level]\n z, x_quantised, commit_loss, metric = level_block(x, update_k=self.training)\n zs.append(z)\n if not self.training:\n # Be extra paranoid and make sure the encoder weights can't\n # change from straight-through estimator\n x_quantised = x_quantised.detach()\n xs_quantised.append(x_quantised)\n commit_losses.append(commit_loss)\n if self.training:\n metrics.append(metric)\n return zs, xs_quantised, commit_losses, metrics\n\nclass NoBottleneckBlock(nn.Module):\n def restore_k(self):\n pass\n\nclass NoBottleneck(nn.Module):\n def __init__(self, levels):\n super().__init__()\n self.level_blocks = nn.ModuleList()\n self.levels = levels\n for level in range(levels):\n self.level_blocks.append(NoBottleneckBlock())\n\n def encode(self, xs):\n return xs\n\n def decode(self, zs, start_level=0, end_level=None):\n if end_level is None:\n end_level = self.levels\n return zs\n\n def forward(self, xs):\n zero = t.zeros(()).cuda()\n commit_losses = [zero for _ in range(self.levels)]\n metrics = [dict(entropy=zero, usage=zero, used_curr=zero, pn=zero, dk=zero) for _ in range(self.levels)]\n return xs, xs, commit_losses, metrics\n\nif __name__ == '__main__':\n from jukebox.utils.dist_utils import setup_dist_from_mpi\n rank, local_rank, device = setup_dist_from_mpi(port=29600)\n bottleneck = Bottleneck(256, 64, 0.99, 2).to(device)\n bottleneck.check()\n" }, { "path": "jukebox/vqvae/encdec.py", "content": "import torch as t\nimport torch.nn as nn\nfrom jukebox.vqvae.resnet import Resnet, Resnet1D\nfrom jukebox.utils.torch_utils import assert_shape\n\nclass EncoderConvBlock(nn.Module):\n def __init__(self, input_emb_width, output_emb_width, down_t,\n stride_t, width, depth, m_conv,\n dilation_growth_rate=1, dilation_cycle=None, zero_out=False,\n res_scale=False):\n super().__init__()\n blocks = []\n filter_t, pad_t = stride_t * 2, stride_t // 2\n if down_t > 0:\n for i in range(down_t):\n block = nn.Sequential(\n nn.Conv1d(input_emb_width if i == 0 else width, width, filter_t, stride_t, pad_t),\n Resnet1D(width, depth, m_conv, dilation_growth_rate, dilation_cycle, zero_out, res_scale),\n )\n blocks.append(block)\n block = nn.Conv1d(width, output_emb_width, 3, 1, 1)\n blocks.append(block)\n self.model = nn.Sequential(*blocks)\n\n def forward(self, x):\n return self.model(x)\n\nclass DecoderConvBock(nn.Module):\n def __init__(self, input_emb_width, output_emb_width, down_t,\n stride_t, width, depth, m_conv, dilation_growth_rate=1, dilation_cycle=None, zero_out=False, res_scale=False, reverse_decoder_dilation=False, checkpoint_res=False):\n super().__init__()\n blocks = []\n if down_t > 0:\n filter_t, pad_t = stride_t * 2, stride_t // 2\n block = nn.Conv1d(output_emb_width, width, 3, 1, 1)\n blocks.append(block)\n for i in range(down_t):\n block = nn.Sequential(\n Resnet1D(width, depth, m_conv, dilation_growth_rate, dilation_cycle, zero_out=zero_out, res_scale=res_scale, reverse_dilation=reverse_decoder_dilation, checkpoint_res=checkpoint_res),\n nn.ConvTranspose1d(width, input_emb_width if i == (down_t - 1) else width, filter_t, stride_t, pad_t)\n )\n blocks.append(block)\n self.model = nn.Sequential(*blocks)\n\n def forward(self, x):\n return self.model(x)\n\nclass Encoder(nn.Module):\n def __init__(self, input_emb_width, output_emb_width, levels, downs_t,\n strides_t, **block_kwargs):\n super().__init__()\n self.input_emb_width = input_emb_width\n self.output_emb_width = output_emb_width\n self.levels = levels\n self.downs_t = downs_t\n self.strides_t = strides_t\n\n block_kwargs_copy = dict(**block_kwargs)\n if 'reverse_decoder_dilation' in block_kwargs_copy:\n del block_kwargs_copy['reverse_decoder_dilation']\n level_block = lambda level, down_t, stride_t: EncoderConvBlock(input_emb_width if level == 0 else output_emb_width,\n output_emb_width,\n down_t, stride_t,\n **block_kwargs_copy)\n self.level_blocks = nn.ModuleList()\n iterator = zip(list(range(self.levels)), downs_t, strides_t)\n for level, down_t, stride_t in iterator:\n self.level_blocks.append(level_block(level, down_t, stride_t))\n\n def forward(self, x):\n N, T = x.shape[0], x.shape[-1]\n emb = self.input_emb_width\n assert_shape(x, (N, emb, T))\n xs = []\n\n # 64, 32, ...\n iterator = zip(list(range(self.levels)), self.downs_t, self.strides_t)\n for level, down_t, stride_t in iterator:\n level_block = self.level_blocks[level]\n x = level_block(x)\n emb, T = self.output_emb_width, T // (stride_t ** down_t)\n assert_shape(x, (N, emb, T))\n xs.append(x)\n\n return xs\n\nclass Decoder(nn.Module):\n def __init__(self, input_emb_width, output_emb_width, levels, downs_t,\n strides_t, **block_kwargs):\n super().__init__()\n self.input_emb_width = input_emb_width\n self.output_emb_width = output_emb_width\n self.levels = levels\n\n self.downs_t = downs_t\n\n self.strides_t = strides_t\n\n level_block = lambda level, down_t, stride_t: DecoderConvBock(output_emb_width,\n output_emb_width,\n down_t, stride_t,\n **block_kwargs)\n self.level_blocks = nn.ModuleList()\n iterator = zip(list(range(self.levels)), downs_t, strides_t)\n for level, down_t, stride_t in iterator:\n self.level_blocks.append(level_block(level, down_t, stride_t))\n\n self.out = nn.Conv1d(output_emb_width, input_emb_width, 3, 1, 1)\n\n def forward(self, xs, all_levels=True):\n if all_levels:\n assert len(xs) == self.levels\n else:\n assert len(xs) == 1\n x = xs[-1]\n N, T = x.shape[0], x.shape[-1]\n emb = self.output_emb_width\n assert_shape(x, (N, emb, T))\n\n # 32, 64 ...\n iterator = reversed(list(zip(list(range(self.levels)), self.downs_t, self.strides_t)))\n for level, down_t, stride_t in iterator:\n level_block = self.level_blocks[level]\n x = level_block(x)\n emb, T = self.output_emb_width, T * (stride_t ** down_t)\n assert_shape(x, (N, emb, T))\n if level != 0 and all_levels:\n x = x + xs[level - 1]\n\n x = self.out(x)\n return x\n" }, { "path": "jukebox/vqvae/resnet.py", "content": "import math\nimport torch.nn as nn\nimport jukebox.utils.dist_adapter as dist\nfrom jukebox.utils.checkpoint import checkpoint\n\nclass ResConvBlock(nn.Module):\n def __init__(self, n_in, n_state):\n super().__init__()\n self.model = nn.Sequential(\n nn.ReLU(),\n nn.Conv2d(n_in, n_state, 3, 1, 1),\n nn.ReLU(),\n nn.Conv2d(n_state, n_in, 1, 1, 0),\n )\n\n def forward(self, x):\n return x + self.model(x)\n\nclass Resnet(nn.Module):\n def __init__(self, n_in, n_depth, m_conv=1.0):\n super().__init__()\n self.model = nn.Sequential(*[ResConvBlock(n_in, int(m_conv * n_in)) for _ in range(n_depth)])\n\n def forward(self, x):\n return self.model(x)\n\nclass ResConv1DBlock(nn.Module):\n def __init__(self, n_in, n_state, dilation=1, zero_out=False, res_scale=1.0):\n super().__init__()\n padding = dilation\n self.model = nn.Sequential(\n nn.ReLU(),\n nn.Conv1d(n_in, n_state, 3, 1, padding, dilation),\n nn.ReLU(),\n nn.Conv1d(n_state, n_in, 1, 1, 0),\n )\n if zero_out:\n out = self.model[-1]\n nn.init.zeros_(out.weight)\n nn.init.zeros_(out.bias)\n self.res_scale = res_scale\n\n def forward(self, x):\n return x + self.res_scale * self.model(x)\n\nclass Resnet1D(nn.Module):\n def __init__(self, n_in, n_depth, m_conv=1.0, dilation_growth_rate=1, dilation_cycle=None, zero_out=False, res_scale=False, reverse_dilation=False, checkpoint_res=False):\n super().__init__()\n def _get_depth(depth):\n if dilation_cycle is None:\n return depth\n else:\n return depth % dilation_cycle\n blocks = [ResConv1DBlock(n_in, int(m_conv * n_in),\n dilation=dilation_growth_rate ** _get_depth(depth),\n zero_out=zero_out,\n res_scale=1.0 if not res_scale else 1.0 / math.sqrt(n_depth))\n for depth in range(n_depth)]\n if reverse_dilation:\n blocks = blocks[::-1]\n self.checkpoint_res = checkpoint_res\n if self.checkpoint_res == 1:\n if dist.get_rank() == 0:\n print(\"Checkpointing convs\")\n self.blocks = nn.ModuleList(blocks)\n else:\n self.model = nn.Sequential(*blocks)\n\n def forward(self, x):\n if self.checkpoint_res == 1:\n for block in self.blocks:\n x = checkpoint(block, (x, ), block.parameters(), True)\n return x\n else:\n return self.model(x)\n" }, { "path": "jukebox/vqvae/vqvae.py", "content": "import numpy as np\nimport torch as t\nimport torch.nn as nn\n\nfrom jukebox.vqvae.encdec import Encoder, Decoder, assert_shape\nfrom jukebox.vqvae.bottleneck import NoBottleneck, Bottleneck\nfrom jukebox.utils.logger import average_metrics\nfrom jukebox.utils.audio_utils import spectral_convergence, spectral_loss, multispectral_loss, audio_postprocess\n\ndef dont_update(params):\n for param in params:\n param.requires_grad = False\n\ndef update(params):\n for param in params:\n param.requires_grad = True\n\ndef calculate_strides(strides, downs):\n return [stride ** down for stride, down in zip(strides, downs)]\n\ndef _loss_fn(loss_fn, x_target, x_pred, hps):\n if loss_fn == 'l1':\n return t.mean(t.abs(x_pred - x_target)) / hps.bandwidth['l1']\n elif loss_fn == 'l2':\n return t.mean((x_pred - x_target) ** 2) / hps.bandwidth['l2']\n elif loss_fn == 'linf':\n residual = ((x_pred - x_target) ** 2).reshape(x_target.shape[0], -1)\n values, _ = t.topk(residual, hps.linf_k, dim=1)\n return t.mean(values) / hps.bandwidth['l2']\n elif loss_fn == 'lmix':\n loss = 0.0\n if hps.lmix_l1:\n loss += hps.lmix_l1 * _loss_fn('l1', x_target, x_pred, hps)\n if hps.lmix_l2:\n loss += hps.lmix_l2 * _loss_fn('l2', x_target, x_pred, hps)\n if hps.lmix_linf:\n loss += hps.lmix_linf * _loss_fn('linf', x_target, x_pred, hps)\n return loss\n else:\n assert False, f\"Unknown loss_fn {loss_fn}\"\n\nclass VQVAE(nn.Module):\n def __init__(self, input_shape, levels, downs_t, strides_t,\n emb_width, l_bins, mu, commit, spectral, multispectral,\n multipliers=None, use_bottleneck=True, **block_kwargs):\n super().__init__()\n\n self.sample_length = input_shape[0]\n x_shape, x_channels = input_shape[:-1], input_shape[-1]\n self.x_shape = x_shape\n\n self.downsamples = calculate_strides(strides_t, downs_t)\n self.hop_lengths = np.cumprod(self.downsamples)\n self.z_shapes = z_shapes = [(x_shape[0] // self.hop_lengths[level],) for level in range(levels)]\n self.levels = levels\n\n if multipliers is None:\n self.multipliers = [1] * levels\n else:\n assert len(multipliers) == levels, \"Invalid number of multipliers\"\n self.multipliers = multipliers\n def _block_kwargs(level):\n this_block_kwargs = dict(block_kwargs)\n this_block_kwargs[\"width\"] *= self.multipliers[level]\n this_block_kwargs[\"depth\"] *= self.multipliers[level]\n return this_block_kwargs\n\n encoder = lambda level: Encoder(x_channels, emb_width, level + 1,\n downs_t[:level+1], strides_t[:level+1], **_block_kwargs(level))\n decoder = lambda level: Decoder(x_channels, emb_width, level + 1,\n downs_t[:level+1], strides_t[:level+1], **_block_kwargs(level))\n self.encoders = nn.ModuleList()\n self.decoders = nn.ModuleList()\n for level in range(levels):\n self.encoders.append(encoder(level))\n self.decoders.append(decoder(level))\n\n if use_bottleneck:\n self.bottleneck = Bottleneck(l_bins, emb_width, mu, levels)\n else:\n self.bottleneck = NoBottleneck(levels)\n\n self.downs_t = downs_t\n self.strides_t = strides_t\n self.l_bins = l_bins\n self.commit = commit\n self.spectral = spectral\n self.multispectral = multispectral\n\n def preprocess(self, x):\n # x: NTC [-1,1] -> NCT [-1,1]\n assert len(x.shape) == 3\n x = x.permute(0,2,1).float()\n return x\n\n def postprocess(self, x):\n # x: NTC [-1,1] <- NCT [-1,1]\n x = x.permute(0,2,1)\n return x\n\n def _decode(self, zs, start_level=0, end_level=None):\n # Decode\n if end_level is None:\n end_level = self.levels\n assert len(zs) == end_level - start_level\n xs_quantised = self.bottleneck.decode(zs, start_level=start_level, end_level=end_level)\n assert len(xs_quantised) == end_level - start_level\n\n # Use only lowest level\n decoder, x_quantised = self.decoders[start_level], xs_quantised[0:1]\n x_out = decoder(x_quantised, all_levels=False)\n x_out = self.postprocess(x_out)\n return x_out\n\n def decode(self, zs, start_level=0, end_level=None, bs_chunks=1):\n z_chunks = [t.chunk(z, bs_chunks, dim=0) for z in zs]\n x_outs = []\n for i in range(bs_chunks):\n zs_i = [z_chunk[i] for z_chunk in z_chunks]\n x_out = self._decode(zs_i, start_level=start_level, end_level=end_level)\n x_outs.append(x_out)\n return t.cat(x_outs, dim=0)\n\n def _encode(self, x, start_level=0, end_level=None):\n # Encode\n if end_level is None:\n end_level = self.levels\n x_in = self.preprocess(x)\n xs = []\n for level in range(self.levels):\n encoder = self.encoders[level]\n x_out = encoder(x_in)\n xs.append(x_out[-1])\n zs = self.bottleneck.encode(xs)\n return zs[start_level:end_level]\n\n def encode(self, x, start_level=0, end_level=None, bs_chunks=1):\n x_chunks = t.chunk(x, bs_chunks, dim=0)\n zs_list = []\n for x_i in x_chunks:\n zs_i = self._encode(x_i, start_level=start_level, end_level=end_level)\n zs_list.append(zs_i)\n zs = [t.cat(zs_level_list, dim=0) for zs_level_list in zip(*zs_list)]\n return zs\n\n def sample(self, n_samples):\n zs = [t.randint(0, self.l_bins, size=(n_samples, *z_shape), device='cuda') for z_shape in self.z_shapes]\n return self.decode(zs)\n\n def forward(self, x, hps, loss_fn='l1'):\n metrics = {}\n\n N = x.shape[0]\n\n # Encode/Decode\n x_in = self.preprocess(x)\n xs = []\n for level in range(self.levels):\n encoder = self.encoders[level]\n x_out = encoder(x_in)\n xs.append(x_out[-1])\n\n zs, xs_quantised, commit_losses, quantiser_metrics = self.bottleneck(xs)\n x_outs = []\n for level in range(self.levels):\n decoder = self.decoders[level]\n x_out = decoder(xs_quantised[level:level+1], all_levels=False)\n assert_shape(x_out, x_in.shape)\n x_outs.append(x_out)\n\n # Loss\n def _spectral_loss(x_target, x_out, hps):\n if hps.use_nonrelative_specloss:\n sl = spectral_loss(x_target, x_out, hps) / hps.bandwidth['spec']\n else:\n sl = spectral_convergence(x_target, x_out, hps)\n sl = t.mean(sl)\n return sl\n\n def _multispectral_loss(x_target, x_out, hps):\n sl = multispectral_loss(x_target, x_out, hps) / hps.bandwidth['spec']\n sl = t.mean(sl)\n return sl\n\n recons_loss = t.zeros(()).to(x.device)\n spec_loss = t.zeros(()).to(x.device)\n multispec_loss = t.zeros(()).to(x.device)\n x_target = audio_postprocess(x.float(), hps)\n\n for level in reversed(range(self.levels)):\n x_out = self.postprocess(x_outs[level])\n x_out = audio_postprocess(x_out, hps)\n this_recons_loss = _loss_fn(loss_fn, x_target, x_out, hps)\n this_spec_loss = _spectral_loss(x_target, x_out, hps)\n this_multispec_loss = _multispectral_loss(x_target, x_out, hps)\n metrics[f'recons_loss_l{level + 1}'] = this_recons_loss\n metrics[f'spectral_loss_l{level + 1}'] = this_spec_loss\n metrics[f'multispectral_loss_l{level + 1}'] = this_multispec_loss\n recons_loss += this_recons_loss\n spec_loss += this_spec_loss\n multispec_loss += this_multispec_loss\n\n commit_loss = sum(commit_losses)\n loss = recons_loss + self.spectral * spec_loss + self.multispectral * multispec_loss + self.commit * commit_loss\n\n with t.no_grad():\n sc = t.mean(spectral_convergence(x_target, x_out, hps))\n l2_loss = _loss_fn(\"l2\", x_target, x_out, hps)\n l1_loss = _loss_fn(\"l1\", x_target, x_out, hps)\n linf_loss = _loss_fn(\"linf\", x_target, x_out, hps)\n\n quantiser_metrics = average_metrics(quantiser_metrics)\n\n metrics.update(dict(\n recons_loss=recons_loss,\n spectral_loss=spec_loss,\n multispectral_loss=multispec_loss,\n spectral_convergence=sc,\n l2_loss=l2_loss,\n l1_loss=l1_loss,\n linf_loss=linf_loss,\n commit_loss=commit_loss,\n **quantiser_metrics))\n\n for key, val in metrics.items():\n metrics[key] = val.detach()\n\n return x_out, loss, metrics\n" }, { "path": "requirements.txt", "content": "fire==0.1.3\ntqdm==4.45.0\nsoundfile==0.10.3.post1\nunidecode==1.1.1\nnumba==0.48.0\nlibrosa==0.7.2\nmpi4py>=3.0.0" }, { "path": "setup.py", "content": "import os\n\nimport pkg_resources\nfrom setuptools import setup, find_packages\n\nsetup(\n name=\"jukebox\",\n py_modules=[\"jukebox\"],\n version=\"1.0\",\n description=\"\",\n author=\"OpenAI\",\n packages=find_packages(),\n install_requires=[\n str(r)\n for r in pkg_resources.parse_requirements(\n open(os.path.join(os.path.dirname(__file__), \"requirements.txt\"))\n )\n ],\n include_package_data=True\n)\n" }, { "path": "tensorboardX/.codecov.yml", "content": "coverage:\n status:\n project: # measuring the overall project coverage\n default: # context, you can create multiple ones with custom titles\n enabled: yes \n patch:\n default:\n enabled: no\n" }, { "path": "tensorboardX/.flake8", "content": "[flake8]\nmax-line-length = 120\nignore = E305,E402,E721,E741,F401,F403,F405,F821,F841,F999\nexclude = tensorboardX/proto" }, { "path": "tensorboardX/.github/ISSUE_TEMPLATE/bug_report.md", "content": "---\nname: Bug report\nabout: Create bug report\ntitle: ''\nlabels: ''\nassignees: ''\n\n---\n\n**Describe the bug**\nA clear and concise description of what the bug is.\n\n**Minimal runnable code to reproduce the behavior**\n```\nfrom tensorboardX import SummaryWriter\n...\n```\n\n**Expected behavior**\nA clear and concise description of what you expected to happen.\n\n**Screenshots**\nIf applicable, add screenshots to help explain your problem.\n\n**Environment**\nWhat is the result of \n`pip list|grep -E \"torch|proto|tensor\"`\nIf the version is too old, please try to update first.\n\n\n**Python environment**\nWhich version of python are you using? Did you use Andconda or Virtualenv?\n\n**Additional context**\nAdd any other context about the problem here.\n" }, { "path": "tensorboardX/.github/ISSUE_TEMPLATE/feature-requests-or-general-questions.md", "content": "---\nname: Feature requests or General questions\nabout: Feature requests or general questions\ntitle: ''\nlabels: ''\nassignees: ''\n\n---\n\n\n" }, { "path": "tensorboardX/.gitignore", "content": "proto_src/\nprotoc-*.zip\nprotoc/\n__pycache__\ndocs/_*\nbuild\ndist\n*.egg-info\nruns/*\n*.pyc\n" }, { "path": "tensorboardX/.travis.yml", "content": "dist: xenial\nlanguage: python\npython:\n # We don't actually use the Travis Python, but this keeps it organized.\n - \"2.7\"\n - \"3.6\"\n\nenv:\n - PYTORCH_VER=\"torch\"\n - PYTORCH_VER=\"torch_nightly -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\"\n\nmatrix:\n allow_failures:\n - env: PYTORCH_VER=\"torch_nightly -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\"\n\ninstall:\n - export MPLBACKEND=Agg\n - export CODECOV_TOKEN=\"26239910-fe4e-463d-aa3d-e662e9bf39ef\"\n\n - sudo apt-get update\n # We do this conditionally because it saves us some downloading if the\n # version is the same.\n - if [[ \"$TRAVIS_PYTHON_VERSION\" == \"2.7\" ]]; then\n wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh;\n else\n wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;\n fi\n - bash miniconda.sh -b -p $HOME/miniconda\n - export PATH=\"$HOME/miniconda/bin:$PATH\"\n - export BOTO_CONFIG=/dev/null # https://github.com/travis-ci/travis-ci/issues/7940\n - export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python\n - hash -r\n - conda config --set always_yes yes --set changeps1 no\n - conda update -q conda\n # Useful for debugging any issues with conda\n - conda info -a\n\n # Replace dep1 dep2 ... with your dependencies\n - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION\n - source activate test-environment\n - which python\n - pip install future\n - pip install chainer -q\n - pip install torchvision==0.2.1 -q\n - pip uninstall torch -y\n - pip install $PYTORCH_VER\n - pip install moviepy==0.2.3.2 -q\n - pip install matplotlib -q\n - pip install requests -q\n - pip install codecov\n - pip install onnx\n - pip install boto3\n - pip install moto\n - pip install visdom\n - pip install tb-nightly\n - pip install crc32c\n - pip install protobuf==3.8.0\n - conda install ffmpeg\n - conda list\n - python -c \"import imageio; imageio.plugins.ffmpeg.download()\"\n - pip install --upgrade pytest-cov flake8\n - python setup.py install\n\nscript:\n - visdom &\n - sleep 5\n - python -c \"import visdom; v = visdom.Visdom()\"\n - py.test --cov=tensorboardX tests/\n - python examples/demo.py\n - python examples/demo_graph.py\n - python examples/demo_embedding.py\n - python examples/demo_custom_scalars.py\n - python examples/demo_multiple_embedding.py\n - python examples/demo_purge.py\n - python examples/demo_matplotlib.py\n - pip uninstall -y tensorboardX\n - pip install tensorboardX\n - pytest\n\nafter_success:\n - codecov\n" }, { "path": "tensorboardX/HISTORY.rst", "content": "History\n=======\n1.8 (2019-07-05)\n-----------------\n* Draw label text on image with bounding box provided.\n* crc32c speed up (optional by installing crc32c manually)\n* Rewrite add_graph. onnx backend is replaced by JIT to support more advanced structure.\n* Now you can add_mesh() to visualize colorful point cloud or meshes.\n\n1.7 (2019-05-19)\n-----------------\n* Able to write to S3\n* Fixed raw histogram issue that nothing is shown in TensorBoard\n* Users can use various image/video dimension permutation by passing 'dataformats' parameter.\n* You can bybass the writer by passing write_to_disk=True to SummaryWriter\n\n\n1.6 (2019-01-02)\n-----------------\n* Many graph related bug is fixed in this version.\n* New function: add_images(). This function accepts 4D iamge tensor. See documentation.\n* Make add_image_with_boxes() usable.\n* API change: add_video now accepts BxTxCxHxW instead of BxCxTxHxW tensor.\n\n1.5 (2018-12-10)\n-----------------\n* Add API for Custom scalar\n* Add support for logging directly to S3\n* Add support for Caffe2 graph\n* Pytorch 1.0.0 JIT graph support (alpha-release)\n\n1.4 (2018-08-09)\n-----------------\n* Made add_text compatible with tensorboard>1.6\n* Fix the issue of strange histogram if default binning method is used\n* Supports passing matplotlib figures to add_image()\n* Resolve namespace confliction with TF tensorboard\n* add_image_boxes function\n* Supports custom timestamp for event\n\n1.2 (2018-04-21)\n-----------------\n* Supports tensorshape information in graph visualization. Drop support for 0.3.1\n* Adds add_video function\n\n1.1 (2018-02-21)\n-----------------\n* Supports pytorch 0.3.1 (hacky)\n\n1.0 (2018-01-18)\n-----------------\n* Supports graph (the pretty one)\n\n0.9 (2017-11-11)\n-----------------\n* Supports markdown for add_text function\n* It's ready to log precision recall curve (needs tensorboard>=0.4)\n* Adds context manager for the SummaryWriter class\n\n0.8 (2017-09-25)\n-----------------\n* Package name renamed to tensorboardX to fix namespace confliction with tensorflow's tensorboard\n* Supports multi-scalars and JSON export\n* Multiple Embeddings in One Experiment \n* Supports Chainer and mxnet\n\n0.7 (2017-08-22)\n-----------------\n* remove tensorflow dependency for embedding function\n* fixed incorrect image<->label pairing in embedding function (#12)\n* unifies API call and adds docstring. Documentation is available at: http://tensorboard-pytorch.readthedocs.io/\n\n0.6.5 (2017-07-30)\n------------------\n* add travis test (py2.7, py3.6)\n* add support for python2 (in PyPI)\n\n0.6 (2017-07-18)\n-----------------\n* supports embedding\n\n0.5 (2017-07-18)\n-----------------\n* supports graph summary\n* fixed np.histogram issue\n\n0.4 (2017-07-12)\n-----------------\n* supports text summary\n\n0.3 (2017-07-03)\n-----------------\n* supports audio summary\n\n0.2 (2017-06-24)\n-----------------\n* simplifies add_image API\n* speed up add_histogram API by 35x\n\n\n0.1 (2017-06-13)\n------------------\n* First commit. Reference:\n\nhttps://github.com/TeamHG-Memex/tensorboard_logger\nhttps://github.com/dmlc/tensorboard\n" }, { "path": "tensorboardX/LICENSE", "content": "MIT License\n\nCopyright (c) 2017 Tzu-Wei Huang\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "tensorboardX/MANIFEST.in", "content": "include HISTORY.rst\ninclude LICENSE\ninclude compile.sh\nrecursive-include tensorboardX/proto *\nrecursive-exclude test *\nrecursive-exclude examples *\nrecursive-include tensorboardX/beholder *" }, { "path": "tensorboardX/README.md", "content": "# tensorboardX\n\n[![Build Status](https://travis-ci.org/lanpa/tensorboardX.svg?branch=master)](https://travis-ci.org/lanpa/tensorboardX)\n[![PyPI version](https://badge.fury.io/py/tensorboardX.svg)](https://badge.fury.io/py/tensorboardX)\n[![Downloads](https://img.shields.io/badge/pip--downloads-5K+-brightgreen.svg)](https://bigquery.cloud.google.com/savedquery/966219917372:edb59a0d70c54eb687ab2a9417a778ee)\n[![Documentation Status](https://readthedocs.org/projects/tensorboardx/badge/?version=latest)](https://tensorboardx.readthedocs.io/en/latest/?badge=latest)\n[![Documentation Status](https://codecov.io/gh/lanpa/tensorboardX/branch/master/graph/badge.svg)](https://codecov.io/gh/lanpa/tensorboardX/)\n\nWrite TensorBoard events with simple function call.\n\n* Support `scalar`, `image`, `figure`, `histogram`, `audio`, `text`, `graph`, `onnx_graph`, `embedding`, `pr_curve`, `mesh`, `hyper-parameters`\n and `video` summaries.\n\n* requirement for `demo_graph.py` is tensorboardX>=1.6 and pytorch>=1.1\n\n* [FAQ](https://github.com/lanpa/tensorboardX/wiki)\n\n## Install\n\nTested on anaconda2 / anaconda3, with PyTorch 1.1.0 / torchvision 0.3 / tensorboard 1.13.0\n\n`pip install tensorboardX`\n\nor build from source:\n\n`git clone https://github.com/lanpa/tensorboardX && cd tensorboardX && python setup.py install`\n\nYou can optionally install [`crc32c`](https://github.com/ICRAR/crc32c) to speed up saving a large amount of data.\n\n\n## Example\n\n* Run the demo script: `python examples/demo.py`\n* Use TensorBoard with `tensorboard --logdir runs` (needs to install TensorFlow)\n\n```python\n# demo.py\n\nimport torch\nimport torchvision.utils as vutils\nimport numpy as np\nimport torchvision.models as models\nfrom torchvision import datasets\nfrom tensorboardX import SummaryWriter\n\nresnet18 = models.resnet18(False)\nwriter = SummaryWriter()\nsample_rate = 44100\nfreqs = [262, 294, 330, 349, 392, 440, 440, 440, 440, 440, 440]\n\nfor n_iter in range(100):\n\n dummy_s1 = torch.rand(1)\n dummy_s2 = torch.rand(1)\n # data grouping by `slash`\n writer.add_scalar('data/scalar1', dummy_s1[0], n_iter)\n writer.add_scalar('data/scalar2', dummy_s2[0], n_iter)\n\n writer.add_scalars('data/scalar_group', {'xsinx': n_iter * np.sin(n_iter),\n 'xcosx': n_iter * np.cos(n_iter),\n 'arctanx': np.arctan(n_iter)}, n_iter)\n\n dummy_img = torch.rand(32, 3, 64, 64) # output from network\n if n_iter % 10 == 0:\n x = vutils.make_grid(dummy_img, normalize=True, scale_each=True)\n writer.add_image('Image', x, n_iter)\n\n dummy_audio = torch.zeros(sample_rate * 2)\n for i in range(x.size(0)):\n # amplitude of sound should in [-1, 1]\n dummy_audio[i] = np.cos(freqs[n_iter // 10] * np.pi * float(i) / float(sample_rate))\n writer.add_audio('myAudio', dummy_audio, n_iter, sample_rate=sample_rate)\n\n writer.add_text('Text', 'text logged at step:' + str(n_iter), n_iter)\n\n for name, param in resnet18.named_parameters():\n writer.add_histogram(name, param.clone().cpu().data.numpy(), n_iter)\n\n # needs tensorboard 0.4RC or later\n writer.add_pr_curve('xoxo', np.random.randint(2, size=100), np.random.rand(100), n_iter)\n\ndataset = datasets.MNIST('mnist', train=False, download=True)\nimages = dataset.test_data[:100].float()\nlabel = dataset.test_labels[:100]\n\nfeatures = images.view(100, 784)\nwriter.add_embedding(features, metadata=label, label_img=images.unsqueeze(1))\n\n# export scalar data to JSON for external processing\nwriter.export_scalars_to_json(\"./all_scalars.json\")\nwriter.close()\n```\n\n## Screenshots\n\n\n\n## Tweaks\n\nTo add more ticks for the slider (show more image history), check https://github.com/lanpa/tensorboardX/issues/44 or \nhttps://github.com/tensorflow/tensorboard/pull/1138\n\n## Reference\n\n* [TeamHG-Memex/tensorboard_logger](https://github.com/TeamHG-Memex/tensorboard_logger)\n* [dmlc/tensorboard](https://github.com/dmlc/tensorboard)\n" }, { "path": "tensorboardX/compile.sh", "content": "#!/bin/bash\n\n# Exit on error\n# set -e\n\nDESIRED_PROTO_VERSION=\"3.6.1\"\n\n# call protoc direclty, if version is not the desired one, download the desired vesrion.\n\n\nif [ -f \"protoc/bin/protoc\" ]; then\n PROTOC_BIN=\"protoc/bin/protoc\"\nelse\n PROTOC_BIN=`which protoc`\nfi\n\necho \"using\" $PROTOC_BIN\n\nCURRENT_PROTOC_VER=`${PROTOC_BIN} --version`\nif [ -z ${PROTOC_BIN} ] || [[ \"$CURRENT_PROTOC_VER\" != \"libprotoc \"$DESIRED_PROTO_VERSION ]]; then\n # Download and use the latest version of protoc.\n if [ \"$(uname)\" == \"Darwin\" ]; then\n PROTOC_ZIP=\"protoc-\"$DESIRED_PROTO_VERSION\"-osx-x86_64.zip\"\n else\n PROTOC_ZIP=\"protoc-\"$DESIRED_PROTO_VERSION\"-linux-x86_64.zip\"\n fi\n WGET_BIN=`which wget`\n if [[ ! -z ${WGET_BIN} ]]; then\n ${WGET_BIN} https://github.com/protocolbuffers/protobuf/releases/download/v\"$DESIRED_PROTO_VERSION\"/${PROTOC_ZIP}\n rm -rf protoc\n python -c \"import zipfile; zipfile.ZipFile('\"${PROTOC_ZIP}\"','r').extractall('protoc')\"\n PROTOC_BIN=protoc/bin/protoc\n chmod +x ${PROTOC_BIN}\n fi\nfi\n\n# Regenerate\nif [[ ! -z ${PROTOC_BIN} ]]; then\n # Delete all existing Python protobuf (*_pb2.py) output\n rm -rf tensorboardX/proto/*pb2*.py\n ${PROTOC_BIN} tensorboardX/proto/*.proto --python_out=.\n\n echo \"Done generating tensorboardX/proto/*pb2*.py\"\nelse\n echo \"protoc not installed so can't regenerate tensorboardX/proto/*pb2*.py, using precompiled version.\"\nfi\n\n" }, { "path": "tensorboardX/docs/Makefile", "content": "# Minimal makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHINXBUILD = sphinx-build\nSPHINXPROJ = tensorboardX\nSOURCEDIR = .\nBUILDDIR = _build\n\n# Put it first so that \"make\" without argument is like \"make help\".\nhelp:\n\t@$(SPHINXBUILD) -M help \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)\n\n.PHONY: help Makefile\n\n# Catch-all target: route all unknown targets to Sphinx using the new\n# \"make mode\" option. $(O) is meant as a shortcut for $(SPHINXOPTS).\n%: Makefile\n\t@$(SPHINXBUILD) -M $@ \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)" }, { "path": "tensorboardX/docs/conf.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# tensorboardX documentation build configuration file, created by\n# sphinx-quickstart on Wed Aug 9 01:38:01 2017.\n#\n# This file is execfile()d with the current directory set to its\n# containing dir.\n#\n# Note that not all possible configuration values are present in this\n# autogenerated file.\n#\n# All configuration values have a default; values that are commented out\n# serve to show the default.\n\n# If extensions (or modules to document with autodoc) are in another directory,\n# add these directories to sys.path here. If the directory is relative to the\n# documentation root, use os.path.abspath to make it absolute, like shown here.\n#\nimport os\nimport sys\n# sys.path.insert(0, os.path.abspath('.'))\nsys.path.append(os.path.join(os.path.dirname(__file__), '..'))\n#import tensorboard #uncomment to shadow pip installation\n# -- General configuration ------------------------------------------------\n\n# If your documentation needs a minimal Sphinx version, state it here.\n#\n# needs_sphinx = '1.0'\n\n# Add any Sphinx extension module names here, as strings. They can be\n# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom\n# ones.\nextensions = ['sphinx.ext.autodoc',\n 'sphinx.ext.mathjax',\n 'sphinx.ext.intersphinx',\n 'sphinx.ext.napoleon',\n 'sphinx.ext.viewcode',\n 'sphinx.ext.githubpages']\n\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = ['_templates']\n\n# The suffix(es) of source filenames.\n# You can specify multiple suffix as a list of string:\n#\n# source_suffix = ['.rst', '.md']\nsource_suffix = '.rst'\n\n# The master toctree document.\nmaster_doc = 'index'\n\n# General information about the project.\nproject = 'tensorboardX'\ncopyright = '2017, tensorboardX Contributors'\nauthor = 'tensorboardX Contributors'\n\n# The version info for the project you're documenting, acts as replacement for\n# |version| and |release|, also used in various other places throughout the\n# built documents.\n#\n# The short X.Y version.\nversion = ''\n# The full version, including alpha/beta/rc tags.\nrelease = ''\n\n# The language for content autogenerated by Sphinx. Refer to documentation\n# for a list of supported languages.\n#\n# This is also used if you do content translation via gettext catalogs.\n# Usually you set \"language\" from the command line for these cases.\nlanguage = None\n\n# List of patterns, relative to source directory, that match files and\n# directories to ignore when looking for source files.\n# This patterns also effect to html_static_path and html_extra_path\nexclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']\n\n# The name of the Pygments (syntax highlighting) style to use.\npygments_style = 'sphinx'\n\n# If true, `todo` and `todoList` produce output, else they produce nothing.\ntodo_include_todos = False\n\n\n# -- Options for HTML output ----------------------------------------------\n\n# The theme to use for HTML and HTML Help pages. See the documentation for\n# a list of builtin themes.\n#\nhtml_theme = 'sphinx_rtd_theme'\n\n# Theme options are theme-specific and customize the look and feel of a theme\n# further. For a list of options available for each theme, see the\n# documentation.\n#\n# html_theme_options = {}\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\n# html_static_path = ['_static']\n\n\n# -- Options for HTMLHelp output ------------------------------------------\n\n# Output file base name for HTML help builder.\nhtmlhelp_basename = 'tensorboardXdoc'\n\n\n# -- Options for LaTeX output ---------------------------------------------\n\nlatex_elements = {\n # The paper size ('letterpaper' or 'a4paper').\n #\n # 'papersize': 'letterpaper',\n\n # The font size ('10pt', '11pt' or '12pt').\n #\n # 'pointsize': '10pt',\n\n # Additional stuff for the LaTeX preamble.\n #\n # 'preamble': '',\n\n # Latex figure (float) alignment\n #\n # 'figure_align': 'htbp',\n}\n\n# Grouping the document tree into LaTeX files. List of tuples\n# (source start file, target name, title,\n# author, documentclass [howto, manual, or own class]).\nlatex_documents = [\n (master_doc, 'tensorboardX.tex', 'tensorboardX Documentation',\n 'tensorboardX Contributors', 'manual'),\n]\n\n\n# -- Options for manual page output ---------------------------------------\n\n# One entry per manual page. List of tuples\n# (source start file, name, description, authors, manual section).\nman_pages = [\n (master_doc, 'tensorboardX', 'tensorboardX Documentation',\n [author], 1)\n]\n\n\n# -- Options for Texinfo output -------------------------------------------\n\n# Grouping the document tree into Texinfo files. List of tuples\n# (source start file, target name, title, author,\n# dir menu entry, description, category)\ntexinfo_documents = [\n (master_doc, 'tensorboardX', 'tensorboardX Documentation',\n author, 'tensorboardX', 'One line description of project.',\n 'Miscellaneous'),\n]\n\n\n\n\n# Example configuration for intersphinx: refer to the Python standard library.\nintersphinx_mapping = {\n 'python':('https://docs.python.org/3', None),\n 'numpy': ('http://docs.scipy.org/doc/numpy/', None),\n 'torch': ('http://pytorch.org/docs/master', None),\n 'matplotlib': ('http://matplotlib.sourceforge.net/', None),\n }\n" }, { "path": "tensorboardX/docs/index.rst", "content": ".. tensorboardX documentation master file, created by\n sphinx-quickstart on Wed Aug 9 01:38:01 2017.\n You can adapt this file completely to your liking, but it should at least\n contain the root `toctree` directive.\n\nWelcome to tensorboardX's documentation!\n===============================================\n\n.. toctree::\n :maxdepth: 2\n :caption: Contents:\n\n tensorboard\n utils\n tutorial\n tutorial_zh\n\nIndices and tables\n==================\n\n* :ref:`genindex`\n* :ref:`modindex`\n* :ref:`search`\n" }, { "path": "tensorboardX/docs/tensorboard.rst", "content": "tensorboardX\n===================================\n.. automodule:: tensorboardX\n\n.. autoclass:: SummaryWriter\n :members:\n \n .. automethod:: __init__\n\n.. autoclass:: TorchVis\n :members:\n\n .. automethod:: __init__" }, { "path": "tensorboardX/docs/tutorial.rst", "content": "Tutorials\n*********\n\nWhat is tensorboard X?\n----------------------\n\nAt first, the package was named tensorboard, and soon there are issues about\nname confliction. The first alternative name came to my mind is\ntensorboard-pytorch, but in order to make it more general, I chose tensorboardX\nwhich stands for tensorboard for X.\n\nGoogle's tensorflow's tensorboard is a web server to serve visualizations of the\ntraining progress of a neural network, it visualizes scalar values, images,\ntext, etc.; these information are saved as events in tensorflow. It's a pity\nthat other deep learning frameworks lack of such tool, so there are already\npackages letting users to log the events without tensorflow; however they only\nprovides basic functionalities. The purpose of this package is to let\nresearchers use a simple interface to log events within PyTorch (and then show\nvisualization in tensorboard). This package currently supports logging scalar,\nimage, audio, histogram, text, embedding, and the route of back-propagation. The\nfollowing manual is tested on Ubuntu and Mac, and the environment are anaconda's\npython2 and python3.\n\n\nCreate a summary writer\n-----------------------\nBefore logging anything, we need to create a writer instance. This can be done with:\n\n.. code-block:: python\n\n from tensorboardX import SummaryWriter\n #SummaryWriter encapsulates everything\n writer = SummaryWriter('runs/exp-1')\n #creates writer object. The log will be saved in 'runs/exp-1'\n writer2 = SummaryWriter()\n #creates writer2 object with auto generated file name, the dir will be something like 'runs/Aug20-17-20-33'\n writer3 = SummaryWriter(comment='3x learning rate')\n #creates writer3 object with auto generated file name, the comment will be appended to the filename. The dir will be something like 'runs/Aug20-17-20-33-3xlearning rate'\n\nEach subfolder will be treated as different experiments in tensorboard. Each\ntime you re-run the experiment with different settings, you should change the\nname of the sub folder such as ``runs/exp2``, ``runs/myexp`` so that you can\neasily compare different experiment settings. Type ``tensorboard runs`` to compare\ndifferent runs in tensorboard.\n\n\nGeneral api format\n------------------\n.. code-block:: python\n\n add_something(tag name, object, iteration number)\n\n\nAdd scalar\n-----------\nScalar value is the most simple data type to deal with. Mostly we save the loss\nvalue of each training step, or the accuracy after each epoch. Sometimes I save\nthe corresponding learning rate as well. It's cheap to save scalar value. Just\nlog anything you think is important. To log a scalar value, use\n``writer.add_scalar('myscalar', value, iteration)``. Note that the program complains\nif you feed a PyTorch tensor. Remember to extract the scalar value by\n``x.item()`` if ``x`` is a torch scalar tensor.\n\n\nAdd image\n---------\nAn image is represented as 3-dimensional tensor. The simplest case is save one\nimage at a time. In this case, the image should be passed as a 3-dimension\ntensor of size ``[3, H, W]``. The three dimensions correspond to R, G, B channel of\nan image. After your image is computed, use ``writer.add_image('imresult', x,\niteration)`` to save the image. If you have a batch of images to show, use\n``torchvision``'s ``make_grid`` function to prepare the image array and send the result\nto ``add_image(...)`` (``make_grid`` takes a 4D tensor and returns tiled images in 3D tensor).\n\n.. Note::\n\tRemember to normalize your image.\n\n\nAdd histogram\n-------------\nSaving histograms is expensive. Both in computation time and storage. If training\nslows down after using this package, check this first. To save a histogram,\nconvert the array into numpy array and save with ``writer.add_histogram('hist',\narray, iteration)``.\n\n\nAdd figure\n----------\nYou can save a matplotlib figure to tensorboard with the add_figure function. ``figure`` input should be ``matplotlib.pyplot.figure`` or a list of ``matplotlib.pyplot.figure``.\nCheck ``_ for the detailed usage.\n\nAdd graph\n---------\nTo visualize a model, you need a model ``m`` and the input ``t``. ``t`` can be a tensor or a list of tensors\ndepending on your model. If error happens, make sure that ``m(t)`` runs without problem first. See\n`The graph demo `_ for\ncomplete example.\n\n\nAdd audio\n---------\nTo log a single channel audio, use ``add_audio(tag, audio, iteration, sample_rate)``, where ``audio`` is an one dimensional array, and each element in the array represents the consecutive amplitude samples.\nFor a 2 seconds audio with ``sample_rate`` 44100 Hz, the input ``x`` should have 88200 elements.\nEach element should lie in [−1, 1].\n\nAdd embedding\n-------------\nEmbeddings, high dimensional data, can be visualized and converted\ninto human perceptible 3D data by tensorboard, which provides PCA and\nt-sne to project the data into low dimensional space. What you need to do is\nprovide a bunch of points and tensorboard will do the rest for you. The bunch of\npoints is passed as a tensor of size ``n x d``, where ``n`` is the number of points and\n``d`` is the feature dimension. The feature representation can either be raw data\n(*e.g.* the MNIST image) or a representation learned by your network (extracted\nfeature). This determines how the points distributes. To make the visualization\nmore informative, you can pass optional metadata or ``label_imgs`` for each data\npoints. In this way you can see that neighboring point have similar label and\ndistant points have very different label (semantically or visually). Here the\nmetadata is a list of labels, and the length of the list should equal to ``n``, the\nnumber of the points. The ``label_imgs`` is a 4D tensor of size ``NCHW``. ``N`` should equal\nto ``n`` as well. See\n`The embedding demo `_ for\ncomplete example.\n\n\nUseful commands\n---------------\nInstall\n=======\n\nSimply type ``pip install tensorboardX`` in a unix shell to install this package.\nTo use the newest version, you might need to build from source or ``pip install\ntensorboardX —-no-cache-dir`` . To run tensorboard web server, you need\nto install it using ``pip install tensorboard``.\nAfter that, type ``tensorboard --logdir=`` to start the server, where\n``your_log_dir`` is the parameter of the object constructor. I think this command is\ntedious, so I add a line alias ``tb='tensorboard --logdir '`` in ``~/.bashrc``. In\nthis way, the above command is simplified as ``tb ``. Use your favorite\nbrowser to load the tensorboard page, the address will be shown in the terminal\nafter starting the server.\n\n\nMisc\n----\nPerformance issue\n=================\nLogging is cheap, but display is expensive.\nFor my experience, if there are 3 or more experiments to show at a time and each\nexperiment have, say, 50k points, tensorboard might need a lot of time to\npresent the data.\n\n\nGrouping plots\n==============\nUsually, there are many numbers to log in one experiment. For example, when\ntraining GANs you should log the loss of the generator, discriminator. If the\nloss is composed of two other loss functions, say L1 and MSE, you might want to\nlog the value of the other two losses as well. In this case, you can write the\ntags as Gen/L1, Gen/MSE, Desc/L1, Desc/MSE. In this way, tensorboard will group\nthe plots into two sections (Gen, Desc). You can also use the regular expression\nto filter data.\n" }, { "path": "tensorboardX/docs/tutorial_zh.rst", "content": "Tutorials_zh\n*************\n\n緣起\n------\nGoogle TensorFlow 附加的工具 Tensorboard 是一個很好用的視覺化工具。他可以記錄數字，影像或者是聲音資訊，對於觀察類神經網路訓練的過程非常有幫助。很可惜的是其他的訓練框架（PyTorch, Chainer, numpy）並沒有這麼好用的工具。網路上稍加搜尋可以發現已經有一些現成的套件可以讓不同的訓練框架使用 web 介面來觀察訓練情形，不過他們可以記錄的東西比較有限或是使用起來比較複雜 (tensorboard_logger, visdom)。tensorboardX 的目的就是讓其他 tensorboard 的功能都可以輕易的被非 TensorFlow 的框架使用。\n目前這個套件除了 tensorboard beholder 之外支援所有 tensorboard 的紀錄型態。這個套件目前的標準測試環境為 Ubuntu 或是 Mac ，windows 則是有不定期手動測試；使用的 python 版本為 anaconda 的 python3。\n\n安裝\n-------\n在命令列輸入 ``pip install tensorboardX`` 即可\n或是最新版源碼安裝 ``pip install tensorboardX``\n\n使用\n-------\n建立 event writer 實體\n在紀錄任何東西之前，我們需要建立一個 event writer 實體。\nfrom tensorboardX import SummaryWriter \n#SummaryWriter 是一個類別，包含這套件的所有功能。\n\n``writer = SummaryWriter('runs/exp-1')``\n#建立實體。資料存放在：``'runs/exp-1'``\n#接下來要寫入任何資料都是呼叫 ``writer.add_某功能()``\n\n``writer = SummaryWriter()``\n#使用預設名稱建立實體。資料存放在：``'runs/現在時間-機器名字'`` ex. ``'runs/Aug20-obov01'``\n\n``writer = SummaryWriter(comment='3xLR')``\n#在預設資料夾後面加上註解 檔名變為：``'runs/Aug20-obov01-3xLR'``\n上面的程式碼會在目前的工作目錄下建立一個叫 ``runs`` 的資料夾\u001c以及子目錄 ``exp1``。 每個子目錄都會被視為一個實驗。每次執行新的實驗時，比如說改了一些參數，這時請將資料夾重新命名，像是： ``runs/exp2``, ``runs/myexp`` 這樣可以便於比較實驗的結果。 建議：資料夾可以用時間命名或者是直接把參數當成資料夾的名稱。\n建立 writer 實體之後就可以開始紀錄資料了\nAPI 的長相大概是：``add_xxx(標籤，要記錄的東西，時間戳，其他參數)``\n\n紀錄純量\n-------------\n純量是最好記錄的東西。通常我們會把每次訓練的損失記錄下來或者是測試的準確度都是值得記錄的東西。其他數據，像是學習率也值得紀錄。\n紀錄純量的方法是 ``writer.add_scalar('myscalar', value, iteration)``\nvalue 可以是 PyTorch tensor ， numpy或是 float，int 之類的python原生數字類別。\n\n記錄影像\n-------------\n影像使用一個三維的矩陣來表示。這三個維度分別代表紅色，綠色，藍色的強度。一張寬200， 高100的影像其對應的矩陣大小為[3, 100, 200] （CHW）。最簡單情況是只有一張影像要存。這時候只需要注意一下是不是符合上述的規格然後將它傳到: ``writer.add_image('imresult', image, iteration)`` 即可。 \n通常訓練的時候會採用批次處理，所以有一大堆影像要存。這時候請確定你的資料維度是 ``(NCHW)``, 其中 ``N`` 是batchsize。``add_image`` 會自動將他排列成適當大小。要注意的是，如果要記錄的影像是 OpenCV/numpy 格式，他們通常呈現 ``(HWC)`` 的排列，這時候要呼叫 ``numpy.transpose`` 將其轉為正確的維度，否則會報錯。另外就是注意影像的值的範圍要介於 [0, 1] 之間。 \n\n紀錄直方圖（histogram）\n-------------------------------\n記錄直方圖很耗 CPU 資源，不要常用。如果你用了這個套件之後覺得速度變慢了請先檢查一下是不是這個原因。使用方法很簡單，呼叫 ``writer.add_histogram('hist', array, iteration)`` 即可紀錄。\n\n紀錄聲音\n-------------\n``writer.add_audio('myaudio', audio, iteration, sample_rate)``\n這功能只支援單聲道。 add_audio 要傳入的聲音資訊是個一維陣列，陣列的每一個元素代表在每一個取樣點的振幅大小。取樣頻率(sample_rate)為 44100 kHz 的情況下。一段2秒鐘的聲音應該要有88200個點；注意其中每個元素的值應該都介於正負1之間。\n\n紀錄文字\n-------------\n``writer.add_text('mytext', 'this is a pen', iteration)``\n除了一般字串之外，也支援簡單的 markdown 表格。\n\n記錄網路架構。\n--------------------------\n(實驗性的功能，模型複雜的時候不確定對不對)\n問題很多的功能。使用上比較複雜。需要準備兩個東西：網路模型 以及 你要餵給他的 tensor \n舉例來說，令模型為 m，輸入為 x，則使用方法為：\n``add_graph(m, (x, ))`` 這裡使用 tuple 的原因是當網路有多個輸入時，可以把他擴充成\n``add_graph(m, (x, y, z))`` ，如果只有單一輸入，寫成 ``add_graph(m, x)`` 也無妨。 \n常會出錯的原因： \n- 較新的 operator pytorch本身不支援JIT\n- 輸入是 cpu tensor，model 在 GPU 上。（或是反過來）\n- 輸入的 tensor 大小錯誤，跑到後面幾層維度消失了\n- model 寫錯，前後兩層 feature dimension 對不上\n除錯方法\n\nforward propagate 一次 ``m(x)`` 或是多個輸入時：``m((x, y, z))``\n2. 用 ``torch.onnx.export`` 導出模型，觀察錯誤訊息。\n\n高維度資料視覺化／降維 (embedding)\n---------------------------------------------------\n因為人類對物體的了解程度只有三維，所以當資料的維度超過三的時候我們沒辦法將他視覺化。這時候就需要降維來讓資料的維度小於等於三。降維運算由 tensorboard 以 Javascript 執行，演算法有 PCA 及 t-sne 兩種可選。這邊我們只需要負責提供每個點的高維度特徵即可。提供的格式是一個矩陣，一個 ``n x d`` 的矩陣 ``n`` 點的數量， ``d`` 是維度的多寡。 高維度特徵可以是原始資料。比如說影像，或是網路學到的壓縮結果。這原始資料決定了資料的分佈情形。如果要看得更清楚一點，你可以再傳 metadata / label_imgs 的參數進去（metadata是一個 python list 長度為 ``n``, ``label_imgs`` 是一個 4 維矩陣，大小是 ``nCHW``。這樣每個點就會有他對應的文字或圖在旁邊。不懂的話就看範例吧：https://github.com/lanpa/tensorboardX/blob/master/examples/demo_embedding.py\n\n紀錄短片\n---------------\n類似於紀錄影像，不過傳入的物件維度是 ``[B, C, T ,H, W]``，其中 ``T`` 是影格的數量。所以一個 30 frame 的彩色影片 維度是 ``[B, 3, 30 ,H, W]``。\n\n紀錄 pr curve\n-------------------\n根據預測的機率值以及其對應的標準答案計算 precision-recall 的結果並保存。\n``add_pr_curve (tag, labels, predictions, step)``\nlabels是標準答案，predictions是程式對樣本的預測。 \n假設有十筆資料 labels就會長得像 ``[0, 0, 1, 0, 0, 1, 0, 1, 0, 1]``，predictions則長的像 ``[0.1, 0.3, 0.8, 0.2, 0.4, 0.5, 0.1, 0.7, 0.9, 0.2]``。\n\npyplot 的圖表\n------------------------------\n\n用 matplotlib 畫了美美的圖表想紀錄？請用 ``add_figure`` 。傳入的物件是 matplotlib 的 figure。 \n顯示結果 \nTensorboard 本質是個網頁伺服器，他讀取的資料來自於訓練網路的時候程式 (tensorboardX) 寫下的事件檔。因為 tensorboard 包含於 tensorflow，所以你需要另外安裝一份 tensorflow 在伺服器主機。我想大部分人都已經裝過了。沒裝過的話就在 unix shell 介面輸入 ``pip install tensorboard``。如果沒有使用 TensorFlow 訓練的需求，建議裝非 GPU 版本，啟動速度快得多。\n接下來在命令列輸入 ``tensorboard --logdir=`` （以前面的例子來說：``tensorboard --logdir=runs``）伺服器就會啟動了。這個指令打起來很麻煩，所以我都在 ``~/.bashrc`` 加一行：``alias tb='tensorboard --logdir '`` 如此一來指令就簡化成 ``tb ``。接下來就是照著終端機上的指示打開你的瀏覽器就可以看到畫面了。\n" }, { "path": "tensorboardX/docs/utils.rst", "content": "Helper functions\n===================================\n.. autofunction:: tensorboardX.utils.figure_to_image" }, { "path": "tensorboardX/examples/RUN_AFTER_PIP_INSTALL", "content": "" }, { "path": "tensorboardX/examples/__init__.py", "content": "" }, { "path": "tensorboardX/examples/chainer/extension_logger/net.py", "content": "#!/usr/bin/env python\n\nfrom __future__ import print_function\n\nimport numpy\n\nimport chainer\nfrom chainer import cuda\nimport chainer.functions as F\nimport chainer.links as L\n\n\ndef add_noise(h, sigma=0.2):\n xp = cuda.get_array_module(h.data)\n if chainer.config.train:\n return h + sigma * xp.random.randn(*h.shape)\n else:\n return h\n\n\nclass Generator(chainer.Chain):\n\n def __init__(self, n_hidden, bottom_width=4, ch=512, wscale=0.02):\n super(Generator, self).__init__()\n self.n_hidden = n_hidden\n self.ch = ch\n self.bottom_width = bottom_width\n\n with self.init_scope():\n w = chainer.initializers.Normal(wscale)\n self.l0 = L.Linear(self.n_hidden, bottom_width * bottom_width * ch,\n initialW=w)\n self.dc1 = L.Deconvolution2D(ch, ch // 2, 4, 2, 1, initialW=w)\n self.dc2 = L.Deconvolution2D(ch // 2, ch // 4, 4, 2, 1, initialW=w)\n self.dc3 = L.Deconvolution2D(ch // 4, ch // 8, 4, 2, 1, initialW=w)\n self.dc4 = L.Deconvolution2D(ch // 8, 3, 3, 1, 1, initialW=w)\n self.bn0 = L.BatchNormalization(bottom_width * bottom_width * ch)\n self.bn1 = L.BatchNormalization(ch // 2)\n self.bn2 = L.BatchNormalization(ch // 4)\n self.bn3 = L.BatchNormalization(ch // 8)\n\n def make_hidden(self, batchsize):\n return numpy.random.uniform(-1, 1, (batchsize, self.n_hidden, 1, 1))\\\n .astype(numpy.float32)\n\n def __call__(self, z):\n h = F.reshape(F.relu(self.bn0(self.l0(z))),\n (len(z), self.ch, self.bottom_width, self.bottom_width))\n h = F.relu(self.bn1(self.dc1(h)))\n h = F.relu(self.bn2(self.dc2(h)))\n h = F.relu(self.bn3(self.dc3(h)))\n x = F.sigmoid(self.dc4(h))\n return x\n\n\nclass Discriminator(chainer.Chain):\n\n def __init__(self, bottom_width=4, ch=512, wscale=0.02):\n w = chainer.initializers.Normal(wscale)\n super(Discriminator, self).__init__()\n with self.init_scope():\n self.c0_0 = L.Convolution2D(3, ch // 8, 3, 1, 1, initialW=w)\n self.c0_1 = L.Convolution2D(ch // 8, ch // 4, 4, 2, 1, initialW=w)\n self.c1_0 = L.Convolution2D(ch // 4, ch // 4, 3, 1, 1, initialW=w)\n self.c1_1 = L.Convolution2D(ch // 4, ch // 2, 4, 2, 1, initialW=w)\n self.c2_0 = L.Convolution2D(ch // 2, ch // 2, 3, 1, 1, initialW=w)\n self.c2_1 = L.Convolution2D(ch // 2, ch // 1, 4, 2, 1, initialW=w)\n self.c3_0 = L.Convolution2D(ch // 1, ch // 1, 3, 1, 1, initialW=w)\n self.l4 = L.Linear(bottom_width * bottom_width * ch, 1, initialW=w)\n self.bn0_1 = L.BatchNormalization(ch // 4, use_gamma=False)\n self.bn1_0 = L.BatchNormalization(ch // 4, use_gamma=False)\n self.bn1_1 = L.BatchNormalization(ch // 2, use_gamma=False)\n self.bn2_0 = L.BatchNormalization(ch // 2, use_gamma=False)\n self.bn2_1 = L.BatchNormalization(ch // 1, use_gamma=False)\n self.bn3_0 = L.BatchNormalization(ch // 1, use_gamma=False)\n\n def __call__(self, x):\n h = add_noise(x)\n h = F.leaky_relu(add_noise(self.c0_0(h)))\n h = F.leaky_relu(add_noise(self.bn0_1(self.c0_1(h))))\n h = F.leaky_relu(add_noise(self.bn1_0(self.c1_0(h))))\n h = F.leaky_relu(add_noise(self.bn1_1(self.c1_1(h))))\n h = F.leaky_relu(add_noise(self.bn2_0(self.c2_0(h))))\n h = F.leaky_relu(add_noise(self.bn2_1(self.c2_1(h))))\n h = F.leaky_relu(add_noise(self.bn3_0(self.c3_0(h))))\n return self.l4(h)\n" }, { "path": "tensorboardX/examples/chainer/extension_logger/train_dcgan.py", "content": "#!/usr/bin/env python\n\nfrom __future__ import print_function\nimport argparse\nimport os\n\nimport chainer\nfrom chainer import training\nfrom chainer.training import extensions\n\nfrom net import Discriminator\nfrom net import Generator\nfrom updater import DCGANUpdater\nfrom visualize import out_generated_image\nfrom tensorboardX import SummaryWriter\nfrom writetensorboard import LogTensorboard\n\n\ndef main():\n parser = argparse.ArgumentParser(description='Chainer example: DCGAN')\n parser.add_argument('--batchsize', '-b', type=int, default=50,\n help='Number of images in each mini-batch')\n parser.add_argument('--epoch', '-e', type=int, default=1000,\n help='Number of sweeps over the dataset to train')\n parser.add_argument('--gpu', '-g', type=int, default=-1,\n help='GPU ID (negative value indicates CPU)')\n parser.add_argument('--dataset', '-i', default='',\n help='Directory of image files. Default is cifar-10.')\n parser.add_argument('--out', '-o', default='result',\n help='Directory to output the result')\n parser.add_argument('--resume', '-r', default='',\n help='Resume the training from snapshot')\n parser.add_argument('--n_hidden', '-n', type=int, default=100,\n help='Number of hidden units (z)')\n parser.add_argument('--seed', type=int, default=0,\n help='Random seed of z at visualization stage')\n parser.add_argument('--snapshot_interval', type=int, default=1000,\n help='Interval of snapshot')\n parser.add_argument('--display_interval', type=int, default=100,\n help='Interval of displaying log to console')\n args = parser.parse_args()\n\n print('GPU: {}'.format(args.gpu))\n print('# Minibatch-size: {}'.format(args.batchsize))\n print('# n_hidden: {}'.format(args.n_hidden))\n print('# epoch: {}'.format(args.epoch))\n print('')\n writer = SummaryWriter()\n # Set up a neural network to train\n gen = Generator(n_hidden=args.n_hidden)\n dis = Discriminator()\n\n if args.gpu >= 0:\n # Make a specified GPU current\n chainer.cuda.get_device_from_id(args.gpu).use()\n gen.to_gpu() # Copy the model to the GPU\n dis.to_gpu()\n\n # Setup an optimizer\n def make_optimizer(model, alpha=0.0002, beta1=0.5):\n optimizer = chainer.optimizers.Adam(alpha=alpha, beta1=beta1)\n optimizer.setup(model)\n optimizer.add_hook(chainer.optimizer.WeightDecay(0.0001), 'hook_dec')\n return optimizer\n opt_gen = make_optimizer(gen)\n opt_dis = make_optimizer(dis)\n\n if args.dataset == '':\n # Load the CIFAR10 dataset if args.dataset is not specified\n train, _ = chainer.datasets.get_cifar10(withlabel=False, scale=255.)\n else:\n all_files = os.listdir(args.dataset)\n image_files = [f for f in all_files if ('png' in f or 'jpg' in f)]\n print('{} contains {} image files'\n .format(args.dataset, len(image_files)))\n train = chainer.datasets\\\n .ImageDataset(paths=image_files, root=args.dataset)\n\n train_iter = chainer.iterators.SerialIterator(train, args.batchsize)\n\n # Set up a trainer\n updater = DCGANUpdater(\n models=(gen, dis),\n iterator=train_iter,\n optimizer={\n 'gen': opt_gen, 'dis': opt_dis},\n device=args.gpu)\n trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out)\n\n snapshot_interval = (args.snapshot_interval, 'iteration')\n display_interval = (args.display_interval, 'iteration')\n trainer.extend(\n extensions.snapshot(filename='snapshot_iter_{.updater.iteration}.npz'),\n trigger=snapshot_interval)\n trainer.extend(extensions.snapshot_object(\n gen, 'gen_iter_{.updater.iteration}.npz'), trigger=snapshot_interval)\n trainer.extend(extensions.snapshot_object(\n dis, 'dis_iter_{.updater.iteration}.npz'), trigger=snapshot_interval)\n trainer.extend(extensions.LogReport(trigger=display_interval))\n trainer.extend(LogTensorboard(trigger=display_interval, logger=writer))\n trainer.extend(extensions.PrintReport([\n 'epoch', 'iteration', 'gen/loss', 'dis/loss',\n ]), trigger=display_interval)\n trainer.extend(extensions.ProgressBar(update_interval=10))\n trainer.extend(\n out_generated_image(\n gen, dis,\n 10, 10, args.seed, args.out, writer),\n trigger=snapshot_interval)\n\n if args.resume:\n # Resume from a snapshot\n chainer.serializers.load_npz(args.resume, trainer)\n\n # Run the training\n trainer.run()\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "tensorboardX/examples/chainer/extension_logger/updater.py", "content": "#!/usr/bin/env python\n\nfrom __future__ import print_function\n\nimport chainer\nimport chainer.functions as F\nfrom chainer import Variable\n\n\nclass DCGANUpdater(chainer.training.StandardUpdater):\n\n def __init__(self, *args, **kwargs):\n self.gen, self.dis = kwargs.pop('models')\n super(DCGANUpdater, self).__init__(*args, **kwargs)\n\n def loss_dis(self, dis, y_fake, y_real):\n batchsize = len(y_fake)\n L1 = F.sum(F.softplus(-y_real)) / batchsize\n L2 = F.sum(F.softplus(y_fake)) / batchsize\n loss = L1 + L2\n chainer.report({'loss': loss}, dis)\n return loss\n\n def loss_gen(self, gen, y_fake):\n batchsize = len(y_fake)\n loss = F.sum(F.softplus(-y_fake)) / batchsize\n chainer.report({'loss': loss}, gen)\n return loss\n\n def update_core(self):\n gen_optimizer = self.get_optimizer('gen')\n dis_optimizer = self.get_optimizer('dis')\n\n batch = self.get_iterator('main').next()\n x_real = Variable(self.converter(batch, self.device)) / 255.\n xp = chainer.cuda.get_array_module(x_real.data)\n\n gen, dis = self.gen, self.dis\n batchsize = len(batch)\n\n y_real = dis(x_real)\n\n z = Variable(xp.asarray(gen.make_hidden(batchsize)))\n x_fake = gen(z)\n y_fake = dis(x_fake)\n\n dis_optimizer.update(self.loss_dis, dis, y_fake, y_real)\n gen_optimizer.update(self.loss_gen, gen, y_fake)\n" }, { "path": "tensorboardX/examples/chainer/extension_logger/visualize.py", "content": "#!/usr/bin/env python\n\nimport os\n\nimport numpy as np\nfrom PIL import Image\n\nimport chainer\nimport chainer.cuda\nfrom chainer import Variable\n\n\ndef out_generated_image(gen, dis, rows, cols, seed, dst, writer):\n @chainer.training.make_extension()\n def make_image(trainer):\n np.random.seed(seed)\n n_images = rows * cols\n xp = gen.xp\n z = Variable(xp.asarray(gen.make_hidden(n_images)))\n with chainer.using_config('train', False):\n x = gen(z)\n writer.add_image('img', x, trainer.updater.iteration)\n\n return make_image\n" }, { "path": "tensorboardX/examples/chainer/extension_logger/writetensorboard.py", "content": "import json\nimport os\nimport shutil\nimport tempfile\n\nimport six\nfrom chainer import reporter\nfrom chainer import serializer as serializer_module\nfrom chainer.training import extension\nfrom chainer.training import trigger as trigger_module\n\n\nclass LogTensorboard(extension.Extension):\n\n \"\"\"Trainer extension to output the accumulated results to a log file.\n\n This extension accumulates the observations of the trainer to\n :class:`~chainer.DictSummary` at a regular interval specified by a supplied\n trigger, and writes them into a log file in JSON format.\n\n There are two triggers to handle this extension. One is the trigger to\n invoke this extension, which is used to handle the timing of accumulating\n the results. It is set to ``1, 'iteration'`` by default. The other is the\n trigger to determine when to emit the result. When this trigger returns\n True, this extension appends the summary of accumulated values to the list\n of past summaries, and writes the list to the log file. Then, this\n extension makes a new fresh summary object which is used until the next\n time that the trigger fires.\n\n It also adds some entries to each result dictionary.\n\n - ``'epoch'`` and ``'iteration'`` are the epoch and iteration counts at the\n output, respectively.\n - ``'elapsed_time'`` is the elapsed time in seconds since the training\n begins. The value is taken from :attr:`Trainer.elapsed_time`.\n\n Args:\n keys (iterable of strs): Keys of values to accumulate. If this is None,\n all the values are accumulated and output to the log file.\n trigger: Trigger that decides when to aggregate the result and output\n the values. This is distinct from the trigger of this extension\n itself. If it is a tuple in the form ``, 'epoch'`` or\n ``, 'iteration'``, it is passed to :class:`IntervalTrigger`.\n postprocess: Callback to postprocess the result dictionaries. Each\n result dictionary is passed to this callback on the output. This\n callback can modify the result dictionaries, which are used to\n output to the log file.\n log_name (str): Name of the log file under the output directory. It can\n be a format string: the last result dictionary is passed for the\n formatting. For example, users can use '{iteration}' to separate\n the log files for different iterations. If the log name is None, it\n does not output the log to any file.\n\n \"\"\"\n\n def __init__(self, keys=None, trigger=(1, 'epoch'), postprocess=None,\n log_name='log', logger=None):\n self._keys = keys\n self._trigger = trigger_module.get_trigger(trigger)\n self._postprocess = postprocess\n self._log_name = log_name\n self._log = []\n self._logger = logger\n self._init_summary()\n\n def __call__(self, trainer):\n # accumulate the observations\n keys = self._keys\n observation = trainer.observation\n summary = self._summary\n\n if keys is None:\n summary.add(observation)\n else:\n summary.add({k: observation[k] for k in keys if k in observation})\n for k, v in observation.items():\n #self._logger.add_scalar(k, chainer.cuda.to_cpu(observation[k].data), trainer.updater.iteration)\n self._logger.add_scalar(\n k, observation[k], trainer.updater.iteration)\n if self._trigger(trainer):\n # output the result\n stats = self._summary.compute_mean()\n stats_cpu = {}\n for name, value in six.iteritems(stats):\n stats_cpu[name] = float(value) # copy to CPU\n\n updater = trainer.updater\n stats_cpu['epoch'] = updater.epoch\n stats_cpu['iteration'] = updater.iteration\n stats_cpu['elapsed_time'] = trainer.elapsed_time\n\n if self._postprocess is not None:\n self._postprocess(stats_cpu)\n\n self._log.append(stats_cpu)\n\n # write to the log file\n if self._log_name is not None:\n log_name = self._log_name.format(**stats_cpu)\n fd, path = tempfile.mkstemp(prefix=log_name, dir=trainer.out)\n with os.fdopen(fd, 'w') as f:\n json.dump(self._log, f, indent=4)\n\n new_path = os.path.join(trainer.out, log_name)\n shutil.move(path, new_path)\n\n # reset the summary for the next output\n self._init_summary()\n\n @property\n def log(self):\n \"\"\"The current list of observation dictionaries.\"\"\"\n return self._log\n\n def serialize(self, serializer):\n if hasattr(self._trigger, 'serialize'):\n self._trigger.serialize(serializer['_trigger'])\n\n # Note that this serialization may lose some information of small\n # numerical differences.\n if isinstance(serializer, serializer_module.Serializer):\n log = json.dumps(self._log)\n serializer('_log', log)\n else:\n log = serializer('_log', '')\n self._log = json.loads(log)\n\n def _init_summary(self):\n self._summary = reporter.DictSummary()\n" }, { "path": "tensorboardX/examples/chainer/plain_logger/data.py", "content": "import gzip\nimport os\n\nimport numpy as np\nimport six\nfrom six.moves.urllib import request\n\nparent = 'http://yann.lecun.com/exdb/mnist'\ntrain_images = 'train-images-idx3-ubyte.gz'\ntrain_labels = 'train-labels-idx1-ubyte.gz'\ntest_images = 't10k-images-idx3-ubyte.gz'\ntest_labels = 't10k-labels-idx1-ubyte.gz'\nnum_train = 60000\nnum_test = 10000\ndim = 784\n\n\ndef load_mnist(images, labels, num):\n data = np.zeros(num * dim, dtype=np.uint8).reshape((num, dim))\n target = np.zeros(num, dtype=np.uint8).reshape((num, ))\n\n with gzip.open(images, 'rb') as f_images,\\\n gzip.open(labels, 'rb') as f_labels:\n f_images.read(16)\n f_labels.read(8)\n for i in six.moves.range(num):\n target[i] = ord(f_labels.read(1))\n for j in six.moves.range(dim):\n data[i, j] = ord(f_images.read(1))\n\n return data, target\n\n\ndef download_mnist_data():\n print('Downloading {:s}...'.format(train_images))\n request.urlretrieve('{:s}/{:s}'.format(parent, train_images), train_images)\n print('Done')\n print('Downloading {:s}...'.format(train_labels))\n request.urlretrieve('{:s}/{:s}'.format(parent, train_labels), train_labels)\n print('Done')\n print('Downloading {:s}...'.format(test_images))\n request.urlretrieve('{:s}/{:s}'.format(parent, test_images), test_images)\n print('Done')\n print('Downloading {:s}...'.format(test_labels))\n request.urlretrieve('{:s}/{:s}'.format(parent, test_labels), test_labels)\n print('Done')\n\n print('Converting training data...')\n data_train, target_train = load_mnist(train_images, train_labels,\n num_train)\n print('Done')\n print('Converting test data...')\n data_test, target_test = load_mnist(test_images, test_labels, num_test)\n mnist = {'data': np.append(data_train, data_test, axis=0),\n 'target': np.append(target_train, target_test, axis=0)}\n print('Done')\n print('Save output...')\n with open('mnist.pkl', 'wb') as output:\n six.moves.cPickle.dump(mnist, output, -1)\n print('Done')\n print('Convert completed')\n\n\ndef load_mnist_data():\n if not os.path.exists('mnist.pkl'):\n download_mnist_data()\n with open('mnist.pkl', 'rb') as mnist_pickle:\n mnist = six.moves.cPickle.load(mnist_pickle)\n return mnist\n" }, { "path": "tensorboardX/examples/chainer/plain_logger/net.py", "content": "import six\n\nimport chainer\nimport chainer.functions as F\nfrom chainer.functions.loss.vae import gaussian_kl_divergence\nimport chainer.links as L\n\n\nclass VAE(chainer.Chain):\n \"\"\"Variational AutoEncoder\"\"\"\n\n def __init__(self, n_in, n_latent, n_h):\n super(VAE, self).__init__()\n with self.init_scope():\n # encoder\n self.le1 = L.Linear(n_in, n_h)\n self.le2_mu = L.Linear(n_h, n_latent)\n self.le2_ln_var = L.Linear(n_h, n_latent)\n # decoder\n self.ld1 = L.Linear(n_latent, n_h)\n self.ld2 = L.Linear(n_h, n_in)\n\n def __call__(self, x, sigmoid=True):\n \"\"\"AutoEncoder\"\"\"\n return self.decode(self.encode(x)[0], sigmoid)\n\n def encode(self, x):\n h1 = F.tanh(self.le1(x))\n mu = self.le2_mu(h1)\n ln_var = self.le2_ln_var(h1) # log(sigma**2)\n return mu, ln_var\n\n def decode(self, z, sigmoid=True):\n h1 = F.tanh(self.ld1(z))\n h2 = self.ld2(h1)\n if sigmoid:\n return F.sigmoid(h2)\n else:\n return h2\n\n def get_loss_func(self, C=1.0, k=1):\n \"\"\"Get loss function of VAE.\n\n The loss value is equal to ELBO (Evidence Lower Bound)\n multiplied by -1.\n\n Args:\n C (int): Usually this is 1.0. Can be changed to control the\n second term of ELBO bound, which works as regularization.\n k (int): Number of Monte Carlo samples used in encoded vector.\n \"\"\"\n def lf(x):\n mu, ln_var = self.encode(x)\n batchsize = len(mu.data)\n # reconstruction loss\n rec_loss = 0\n for l in six.moves.range(k):\n z = F.gaussian(mu, ln_var)\n rec_loss += F.bernoulli_nll(x, self.decode(z, sigmoid=False)) \\\n / (k * batchsize)\n self.rec_loss = rec_loss\n self.loss = self.rec_loss + \\\n C * gaussian_kl_divergence(mu, ln_var) / batchsize\n return self.loss\n return lf\n" }, { "path": "tensorboardX/examples/chainer/plain_logger/train_vae.py", "content": "#!/usr/bin/env python\n\"\"\"Chainer example: train a VAE on MNIST\n\"\"\"\nfrom __future__ import print_function\nimport argparse\n\nimport matplotlib\n# Disable interactive backend\nmatplotlib.use('Agg')\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport six\n\nimport chainer\nfrom chainer import computational_graph\nfrom chainer import cuda\nfrom chainer import optimizers\nfrom chainer import serializers\nfrom tensorboardX import SummaryWriter\nimport data\nimport net\n\nwriter = SummaryWriter()\n\nparser = argparse.ArgumentParser(description='Chainer example: MNIST')\nparser.add_argument('--initmodel', '-m', default='',\n help='Initialize the model from given file')\nparser.add_argument('--resume', '-r', default='',\n help='Resume the optimization from snapshot')\nparser.add_argument('--gpu', '-g', default=-1, type=int,\n help='GPU ID (negative value indicates CPU)')\nparser.add_argument('--epoch', '-e', default=100, type=int,\n help='number of epochs to learn')\nparser.add_argument('--dimz', '-z', default=20, type=int,\n help='dimention of encoded vector')\nparser.add_argument('--batchsize', '-b', type=int, default=100,\n help='learning minibatch size')\nparser.add_argument('--test', action='store_true',\n help='Use tiny datasets for quick tests')\nargs = parser.parse_args()\n\nbatchsize = args.batchsize\nn_epoch = args.epoch\nn_latent = args.dimz\n\nwriter.add_text('config', str(args))\n\nprint('GPU: {}'.format(args.gpu))\nprint('# dim z: {}'.format(args.dimz))\nprint('# Minibatch-size: {}'.format(args.batchsize))\nprint('# epoch: {}'.format(args.epoch))\nprint('')\n\n# Prepare dataset\nprint('load MNIST dataset')\nmnist = data.load_mnist_data()\nmnist['data'] = mnist['data'].astype(np.float32)\nmnist['data'] /= 255\nmnist['target'] = mnist['target'].astype(np.int32)\n\nif args.test:\n mnist['data'] = mnist['data'][0:100]\n mnist['target'] = mnist['target'][0:100]\n N = 30\nelse:\n N = 60000\n\nx_train, x_test = np.split(mnist['data'], [N])\ny_train, y_test = np.split(mnist['target'], [N])\nN_test = y_test.size\n\n# Prepare VAE model, defined in net.py\nmodel = net.VAE(784, n_latent, 500)\nif args.gpu >= 0:\n cuda.get_device_from_id(args.gpu).use()\n model.to_gpu()\nxp = np if args.gpu < 0 else cuda.cupy\n\n# Setup optimizer\noptimizer = optimizers.Adam()\noptimizer.setup(model)\n\n# Init/Resume\nif args.initmodel:\n print('Load model from', args.initmodel)\n serializers.load_npz(args.initmodel, model)\nif args.resume:\n print('Load optimizer state from', args.resume)\n serializers.load_npz(args.resume, optimizer)\n\n# Learning loop\nfor epoch in six.moves.range(1, n_epoch + 1):\n print('epoch', epoch)\n\n # training\n perm = np.random.permutation(N)\n sum_loss = 0 # total loss\n sum_rec_loss = 0 # reconstruction loss\n for i in six.moves.range(0, N, batchsize):\n x = chainer.Variable(xp.asarray(x_train[perm[i:i + batchsize]]))\n optimizer.update(model.get_loss_func(), x)\n if epoch == 1 and i == 0:\n with open('graph.dot', 'w') as o:\n g = computational_graph.build_computational_graph(\n (model.loss, ))\n o.write(g.dump())\n print('graph generated')\n writer.add_scalar('train/loss', model.loss, epoch * N + i)\n writer.add_scalar('train/rec_loss', model.rec_loss, epoch * N + i)\n sum_loss += float(model.loss.data) * len(x.data)\n sum_rec_loss += float(model.rec_loss.data) * len(x.data)\n\n print('train mean loss={}, mean reconstruction loss={}'\n .format(sum_loss / N, sum_rec_loss / N))\n\n # evaluation\n sum_loss = 0\n sum_rec_loss = 0\n with chainer.no_backprop_mode():\n for i in six.moves.range(0, N_test, batchsize):\n x = chainer.Variable(xp.asarray(x_test[i:i + batchsize]))\n loss_func = model.get_loss_func(k=10)\n loss_func(x)\n sum_loss += float(model.loss.data) * len(x.data)\n sum_rec_loss += float(model.rec_loss.data) * len(x.data)\n writer.add_scalar('test/loss', model.loss, epoch * N_test + i)\n writer.add_scalar('test/rec_loss', model.rec_loss,\n epoch * N_test + i)\n writer.add_image('reconstructed', model(\n x).reshape(-1, 1, 28, 28), epoch * N_test + i)\n writer.add_image('input', x.reshape(-1, 1, 28, 28),\n epoch * N_test + i)\n del model.loss\n print('test mean loss={}, mean reconstruction loss={}'\n .format(sum_loss / N_test, sum_rec_loss / N_test))\n\n\n# Save the model and the optimizer\nprint('save the model')\nserializers.save_npz('mlp.model', model)\nprint('save the optimizer')\nserializers.save_npz('mlp.state', optimizer)\n\nmodel.to_cpu()\n\n\n# original images and reconstructed images\ndef save_images(x, filename):\n fig, ax = plt.subplots(3, 3, figsize=(9, 9), dpi=100)\n for ai, xi in zip(ax.flatten(), x):\n ai.imshow(xi.reshape(28, 28))\n fig.savefig(filename)\n\n\ntrain_ind = [1, 3, 5, 10, 2, 0, 13, 15, 17]\nx = chainer.Variable(np.asarray(x_train[train_ind]))\nwith chainer.no_backprop_mode():\n x1 = model(x)\nsave_images(x.data, 'train')\nsave_images(x1.data, 'train_reconstructed')\n\ntest_ind = [3, 2, 1, 18, 4, 8, 11, 17, 61]\nx = chainer.Variable(np.asarray(x_test[test_ind]))\nwith chainer.no_backprop_mode():\n x1 = model(x)\nsave_images(x.data, 'test')\nsave_images(x1.data, 'test_reconstructed')\n\n\n# draw images from randomly sampled z\nz = chainer.Variable(np.random.normal(0, 1, (9, n_latent)).astype(np.float32))\nx = model.decode(z)\nsave_images(x.data, 'sampled')\n" }, { "path": "tensorboardX/examples/demo.py", "content": "import torch\nimport torchvision.utils as vutils\nimport numpy as np\nimport torchvision.models as models\nfrom torchvision import datasets\nfrom tensorboardX import SummaryWriter\nimport datetime\n\nresnet18 = models.resnet18(False)\nwriter = SummaryWriter()\nsample_rate = 44100\nfreqs = [262, 294, 330, 349, 392, 440, 440, 440, 440, 440, 440]\n\ntrue_positive_counts = [75, 64, 21, 5, 0]\nfalse_positive_counts = [150, 105, 18, 0, 0]\ntrue_negative_counts = [0, 45, 132, 150, 150]\nfalse_negative_counts = [0, 11, 54, 70, 75]\nprecision = [0.3333333, 0.3786982, 0.5384616, 1.0, 0.0]\nrecall = [1.0, 0.8533334, 0.28, 0.0666667, 0.0]\n\n\nfor n_iter in range(100):\n s1 = torch.rand(1) # value to keep\n s2 = torch.rand(1)\n # data grouping by `slash`\n writer.add_scalar('data/scalar_systemtime', s1[0], n_iter)\n # data grouping by `slash`\n writer.add_scalar('data/scalar_customtime', s1[0], n_iter, walltime=n_iter)\n writer.add_scalars('data/scalar_group', {\"xsinx\": n_iter * np.sin(n_iter),\n \"xcosx\": n_iter * np.cos(n_iter),\n \"arctanx\": np.arctan(n_iter)}, n_iter)\n x = torch.rand(32, 3, 64, 64) # output from network\n if n_iter % 10 == 0:\n x = vutils.make_grid(x, normalize=True, scale_each=True)\n writer.add_image('Image', x, n_iter) # Tensor\n writer.add_image_with_boxes('imagebox_label', torch.ones(3, 240, 240) * 0.5,\n torch.Tensor([[10, 10, 100, 100], [101, 101, 200, 200]]),\n n_iter, \n labels=['abcde' + str(n_iter), 'fgh' + str(n_iter)])\n x = torch.zeros(sample_rate * 2)\n for i in range(x.size(0)):\n # sound amplitude should in [-1, 1]\n x[i] = np.cos(freqs[n_iter // 10] * np.pi *\n float(i) / float(sample_rate))\n writer.add_audio('myAudio', x, n_iter)\n writer.add_text('Text', 'text logged at step:' + str(n_iter), n_iter)\n writer.add_text('markdown Text', '''a|b\\n-|-\\nc|d''', n_iter)\n for name, param in resnet18.named_parameters():\n if 'bn' not in name:\n writer.add_histogram(name, param, n_iter)\n writer.add_pr_curve('xoxo', np.random.randint(2, size=100), np.random.rand(\n 100), n_iter) # needs tensorboard 0.4RC or later\n writer.add_pr_curve_raw('prcurve with raw data', true_positive_counts,\n false_positive_counts,\n true_negative_counts,\n false_negative_counts,\n precision,\n recall, n_iter)\n# export scalar data to JSON for external processing\nwriter.export_scalars_to_json(\"./all_scalars.json\")\n\ndataset = datasets.MNIST('mnist', train=False, download=True)\nimages = dataset.test_data[:100].float()\nlabel = dataset.test_labels[:100]\nfeatures = images.view(100, 784)\nwriter.add_embedding(features, metadata=label, label_img=images.unsqueeze(1))\nwriter.add_embedding(features, global_step=1, tag='noMetadata')\ndataset = datasets.MNIST('mnist', train=True, download=True)\nimages_train = dataset.train_data[:100].float()\nlabels_train = dataset.train_labels[:100]\nfeatures_train = images_train.view(100, 784)\n\nall_features = torch.cat((features, features_train))\nall_labels = torch.cat((label, labels_train))\nall_images = torch.cat((images, images_train))\ndataset_label = ['test'] * 100 + ['train'] * 100\nall_labels = list(zip(all_labels, dataset_label))\n\nwriter.add_embedding(all_features, metadata=all_labels, label_img=all_images.unsqueeze(1),\n metadata_header=['digit', 'dataset'], global_step=2)\n\n# VIDEO\nvid_images = dataset.train_data[:16 * 48]\nvid = vid_images.view(16, 48, 1, 28, 28) # BxTxCxHxW\n\nwriter.add_video('video', vid_tensor=vid)\nwriter.add_video('video_1_fps', vid_tensor=vid, fps=1)\n\nwriter.close()\n" }, { "path": "tensorboardX/examples/demo_beholder.py", "content": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the 'License');\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an 'AS IS' BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"Simple MNIST classifier to demonstrate features of Beholder.\n\nBased on tensorflow/examples/tutorials/mnist/mnist_with_summaries.py.\n\"\"\"\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport numpy as np\nimport tensorboardX.beholder as beholder_lib\nimport time\n\nfrom collections import namedtuple\n\n\nLOG_DIRECTORY = '/tmp/beholder-demo'\ntensor_and_name = namedtuple('tensor_and_name', 'tensor, name')\n\n\ndef beholder_pytorch():\n for i in range(1000):\n fake_param = [tensor_and_name(np.random.randn(128, 768, 3), 'test' + str(i))\n for i in range(5)]\n arrays = [tensor_and_name(np.random.randn(128, 768, 3), 'test' + str(i))\n for i in range(5)]\n beholder = beholder_lib.Beholder(logdir=LOG_DIRECTORY)\n beholder.update(\n trainable=fake_param,\n arrays=arrays,\n frame=np.random.randn(128, 128),\n )\n time.sleep(0.1)\n print(i)\n\n\nif __name__ == '__main__':\n import os\n if not os.path.exists(LOG_DIRECTORY):\n os.makedirs(LOG_DIRECTORY)\n print(LOG_DIRECTORY)\n beholder_pytorch()\n" }, { "path": "tensorboardX/examples/demo_caffe2.py", "content": "try:\n import caffe2.python.predictor.predictor_exporter as pe\nexcept ImportError:\n print('Please check that Caffe2 is installed correctly to run this demo.')\nimport numpy as np\nimport os\nimport shutil\n\nfrom caffe2.python import core, model_helper, workspace, brew\nfrom tensorboardX import TorchVis\n\n\"\"\"\nThis is a demo showcasing specific functionality for Caffe2. Shown here are\n add_scalar (with both raw numerical data and Caffe2 blob names)\n add_scalars (with both raw numerical data and Caffe2 blob names)\n add_graph (visualizing a Caffe2 model as a graph)\n\nNOTE: lmdb must be installed and enabled with -DUSE_LMDB=ON for this demo to work.\n\"\"\"\n\n# If you would like to see some really detailed initializations,\n# you can change --caffe2_log_level=0 to --caffe2_log_level=-1\ncore.GlobalInit(['caffe2', '--caffe2_log_level=0'])\nprint(\"Necessities imported!\")\n\n\n# This section preps your image and test set in a lmdb database\ndef DownloadResource(url, path):\n '''Downloads resources from s3 by url and unzips them to the provided path'''\n import requests\n from six import BytesIO\n import zipfile\n print(\"Downloading... {} to {}\".format(url, path))\n r = requests.get(url, stream=True)\n z = zipfile.ZipFile(BytesIO(r.content))\n z.extractall(path)\n print(\"Completed download and extraction.\")\n\n\ncurrent_folder = os.path.join(os.path.expanduser('~'), 'caffe2_notebooks')\ndata_folder = os.path.join(current_folder, 'tutorial_data', 'mnist')\nroot_folder = os.path.join(current_folder, 'tutorial_files', 'tutorial_mnist')\ndb_missing = False\n\nif not os.path.exists(data_folder):\n os.makedirs(data_folder)\n print(\"Your data folder was not found!! This was generated: {}\".format(data_folder))\n\n# Look for existing database: lmdb\nif os.path.exists(os.path.join(data_folder, \"mnist-train-nchw-lmdb\")):\n print(\"lmdb train db found!\")\nelse:\n db_missing = True\n\nif os.path.exists(os.path.join(data_folder, \"mnist-test-nchw-lmdb\")):\n print(\"lmdb test db found!\")\nelse:\n db_missing = True\n\n# attempt the download of the db if either was missing\nif db_missing:\n print(\"one or both of the MNIST lmbd dbs not found!!\")\n db_url = \"http://download.caffe2.ai/databases/mnist-lmdb.zip\"\n try:\n DownloadResource(db_url, data_folder)\n except Exception as ex:\n print(\n \"Failed to download dataset. Please download it manually from {}\".format(db_url))\n print(\"Unzip it and place the two database folders here: {}\".format(data_folder))\n raise ex\n\nif os.path.exists(root_folder):\n print(\"Looks like you ran this before, so we need to cleanup those old files...\")\n shutil.rmtree(root_folder)\n\nos.makedirs(root_folder)\nworkspace.ResetWorkspace(root_folder)\n\nprint(\"training data folder:\" + data_folder)\nprint(\"workspace root folder:\" + root_folder)\n\n# END DATA PREPARATION #\n\n# Create TorchVis in preparation for writing. Default format is 'tensorboard'\ntv = TorchVis()\n\n\ndef AddInput(model, batch_size, db, db_type):\n # load the data\n data_uint8, label = model.TensorProtosDBInput(\n [], [\"data_uint8\", \"label\"], batch_size=batch_size,\n db=db, db_type=db_type)\n # cast the data to float\n data = model.Cast(data_uint8, \"data\", to=core.DataType.FLOAT)\n # scale data from [0,255] down to [0,1]\n data = model.Scale(data, data, scale=float(1. / 256))\n # don't need the gradient for the backward pass\n data = model.StopGradient(data, data)\n return data, label\n\n\ndef AddLeNetModel(model, data):\n '''\n This part is the standard LeNet model: from data to the softmax prediction.\n\n For each convolutional layer we specify dim_in - number of input channels\n and dim_out - number or output channels. Also each Conv and MaxPool layer changes the\n image size. For example, kernel of size 5 reduces each side of an image by 4.\n\n While when we have kernel and stride sizes equal 2 in a MaxPool layer, it divides\n each side in half.\n '''\n # Image size: 28 x 28 -> 24 x 24\n conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=20, kernel=5)\n # Image size: 24 x 24 -> 12 x 12\n pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2)\n # Image size: 12 x 12 -> 8 x 8\n conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=100, kernel=5)\n # Image size: 8 x 8 -> 4 x 4\n pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2)\n # 50 * 4 * 4 stands for dim_out from previous layer multiplied by the\n # image size\n fc3 = brew.fc(model, pool2, 'fc3', dim_in=100 * 4 * 4, dim_out=500)\n relu = brew.relu(model, fc3, fc3)\n pred = brew.fc(model, relu, 'pred', 500, 10)\n softmax = brew.softmax(model, pred, 'softmax')\n return softmax\n\n\ndef AddAccuracy(model, softmax, label):\n \"\"\"Adds an accuracy op to the model\"\"\"\n accuracy = brew.accuracy(model, [softmax, label], \"accuracy\")\n return accuracy\n\n\ndef AddTrainingOperators(model, softmax, label):\n \"\"\"Adds training operators to the model.\"\"\"\n xent = model.LabelCrossEntropy([softmax, label], 'xent')\n # compute the expected loss\n loss = model.AveragedLoss(xent, \"loss\")\n # track the accuracy of the model\n AddAccuracy(model, softmax, label)\n # use the average loss we just computed to add gradient operators to the\n # model\n model.AddGradientOperators([loss])\n # do a simple stochastic gradient descent\n ITER = brew.iter(model, \"iter\")\n # set the learning rate schedule\n LR = model.LearningRate(\n ITER, \"LR\", base_lr=-0.1, policy=\"step\", stepsize=1, gamma=0.999)\n # ONE is a constant value that is used in the gradient update. We only need\n # to create it once, so it is explicitly placed in param_init_net.\n ONE = model.param_init_net.ConstantFill([], \"ONE\", shape=[1], value=1.0)\n # Now, for each parameter, we do the gradient updates.\n for param in model.params:\n # Note how we get the gradient of each parameter - ModelHelper keeps\n # track of that.\n param_grad = model.param_to_grad[param]\n # The update is a simple weighted sum: param = param + param_grad * LR\n model.WeightedSum([param, ONE, param_grad, LR], param)\n\n\ndef AddBookkeepingOperators(model):\n \"\"\"This adds a few bookkeeping operators that we can inspect later.\n\n These operators do not affect the training procedure: they only collect\n statistics and prints them to file or to logs.\n \"\"\"\n # Print basically prints out the content of the blob. to_file=1 routes the\n # printed output to a file. The file is going to be stored under\n # root_folder/[blob name]\n model.Print('accuracy', [], to_file=1)\n model.Print('loss', [], to_file=1)\n # Summarizes the parameters. Different from Print, Summarize gives some\n # statistics of the parameter, such as mean, std, min and max.\n for param in model.params:\n model.Summarize(param, [], to_file=1)\n model.Summarize(model.param_to_grad[param], [], to_file=1)\n # Now, if we really want to be verbose, we can summarize EVERY blob\n # that the model produces; it is probably not a good idea, because that\n # is going to take time - summarization do not come for free. For this\n # demo, we will only show how to summarize the parameters and their\n # gradients.\n\n\narg_scope = {\"order\": \"NCHW\"}\ntrain_model = model_helper.ModelHelper(name=\"mnist_train\", arg_scope=arg_scope)\ndata, label = AddInput(\n train_model, batch_size=64,\n db=os.path.join(data_folder, 'mnist-train-nchw-lmdb'),\n db_type='lmdb')\nsoftmax = AddLeNetModel(train_model, data)\nAddTrainingOperators(train_model, softmax, label)\nAddBookkeepingOperators(train_model)\n\n# Visualize the Caffe2 model in Tensorboard\ntv.add_graph(train_model, data)\n\n# Testing model. We will set the batch size to 100, so that the testing\n# pass is 100 iterations (10,000 images in total).\n# For the testing model, we need the data input part, the main LeNetModel\n# part, and an accuracy part. Note that init_params is set False because\n# we will be using the parameters obtained from the train model.\ntest_model = model_helper.ModelHelper(\n name=\"mnist_test\", arg_scope=arg_scope, init_params=False)\ndata, label = AddInput(\n test_model, batch_size=100,\n db=os.path.join(data_folder, 'mnist-test-nchw-lmdb'),\n db_type='lmdb')\nsoftmax = AddLeNetModel(test_model, data)\nAddAccuracy(test_model, softmax, label)\n\n# Deployment model. We simply need the main LeNetModel part.\ndeploy_model = model_helper.ModelHelper(\n name=\"mnist_deploy\", arg_scope=arg_scope, init_params=False)\nAddLeNetModel(deploy_model, \"data\")\n# You may wonder what happens with the param_init_net part of the deploy_model.\n# No, we will not use them, since during deployment time we will not randomly\n# initialize the parameters, but load the parameters from the db.\n\nwith open(os.path.join(root_folder, \"train_net.pbtxt\"), 'w') as fid:\n fid.write(str(train_model.net.Proto()))\nwith open(os.path.join(root_folder, \"train_init_net.pbtxt\"), 'w') as fid:\n fid.write(str(train_model.param_init_net.Proto()))\nwith open(os.path.join(root_folder, \"test_net.pbtxt\"), 'w') as fid:\n fid.write(str(test_model.net.Proto()))\nwith open(os.path.join(root_folder, \"test_init_net.pbtxt\"), 'w') as fid:\n fid.write(str(test_model.param_init_net.Proto()))\nwith open(os.path.join(root_folder, \"deploy_net.pbtxt\"), 'w') as fid:\n fid.write(str(deploy_model.net.Proto()))\nprint(\"Protocol buffers files have been created in your root folder: \" + root_folder)\n\n# The parameter initialization network only needs to be run once.\nworkspace.RunNetOnce(train_model.param_init_net)\n# creating the network\nworkspace.CreateNet(train_model.net, overwrite=True)\n# set the number of iterations and track the accuracy & loss\ntotal_iters = 200\naccuracy = np.zeros(total_iters)\nloss = np.zeros(total_iters)\n# Now, we will manually run the network for 200 iterations.\nfor i in range(total_iters):\n workspace.RunNet(train_model.net)\n accuracy[i] = workspace.FetchBlob('accuracy')\n loss[i] = workspace.FetchBlob('loss')\n scalar_dict_raw = {'accuracy': accuracy[i], 'loss': loss[i]}\n scalar_dict_blobname = {'accuracy': 'accuracy', 'loss': 'loss'}\n # Can pass raw numerical data\n tv.add_scalars('training_raw', scalar_dict_raw, i)\n # Can also pass blobname corresponding to data, for fetching\n tv.add_scalars('training_blobname', scalar_dict_blobname, i)\n\ndata = workspace.FetchBlob('data')\nsoftmax = workspace.FetchBlob('softmax')\n\n# Convolutions for this mini-batch\nconv = workspace.FetchBlob('conv1')\nshape = list(conv.shape)\nshape[1] = 1\n# We can look into any channel. This of it as a feature model learned\nconv = conv[:, 15, :, :].reshape(shape)\n\n# run a test pass on the test net\nworkspace.RunNetOnce(test_model.param_init_net)\nworkspace.CreateNet(test_model.net, overwrite=True)\ntest_accuracy = np.zeros(100)\nfor i in range(100):\n workspace.RunNet(test_model.net.Proto().name)\n test_accuracy[i] = workspace.FetchBlob('accuracy')\n tv.add_scalar('test_accuracy_raw', test_accuracy[i], i)\n tv.add_scalar('test_accuracy_blobname', 'accuracy', i)\n# After the execution is done, let's plot the values.\nprint('test_accuracy: %f' % test_accuracy.mean())\n" }, { "path": "tensorboardX/examples/demo_custom_scalars.py", "content": "from numpy.random import rand\nfrom tensorboardX import SummaryWriter\nimport time\n\n\nwith SummaryWriter() as writer:\n for n_iter in range(100):\n writer.add_scalar('twse/0050', rand(), n_iter)\n writer.add_scalar('twse/2330', rand(), n_iter)\n t = rand()\n writer.add_scalar('dow/aaa', t, n_iter)\n writer.add_scalar('dow/bbb', t - 1, n_iter)\n writer.add_scalar('dow/ccc', t + 1, n_iter)\n writer.add_scalar('nasdaq/aaa', rand(), n_iter)\n writer.add_scalar('nasdaq/bbb', rand(), n_iter)\n writer.add_scalar('nasdaq/ccc', rand(), n_iter)\n\n layout = {'Taiwan': {'twse': ['Multiline', ['twse/0050', 'twse/2330']]},\n 'USA': {'dow': ['Margin', ['dow/aaa', 'dow/bbb', 'dow/ccc']],\n 'nasdaq': ['Margin', ['nasdaq/aaa', 'nasdaq/bbb', 'nasdaq/ccc']]}}\n writer.add_custom_scalars(layout)\n# writer.add_custom_scalars(layout) second call has no effect\n\ntime.sleep(1)\n\nwith SummaryWriter() as writer:\n for n_iter in range(100):\n writer.add_scalar('twse/0050', rand(), n_iter)\n writer.add_scalar('twse/2330', rand(), n_iter)\n\n writer.add_custom_scalars_multilinechart(['twse/0050', 'twse/2330'])\n\ntime.sleep(1)\n\nwith SummaryWriter() as writer:\n for n_iter in range(100):\n t = rand()\n writer.add_scalar('dow/aaa', t, n_iter)\n writer.add_scalar('dow/bbb', t - 1, n_iter)\n writer.add_scalar('dow/ccc', t + 1, n_iter)\n\n writer.add_custom_scalars_marginchart(['dow/aaa', 'dow/bbb', 'dow/ccc'])\n" }, { "path": "tensorboardX/examples/demo_embedding.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport os\nfrom torch.autograd.variable import Variable\nfrom tensorboardX import SummaryWriter\nfrom torch.utils.data import TensorDataset, DataLoader\n\n# EMBEDDING VISUALIZATION FOR A TWO-CLASSES PROBLEM\n\n# just a bunch of layers\n\n\nclass M(nn.Module):\n def __init__(self):\n super(M, self).__init__()\n self.cn1 = nn.Conv2d(in_channels=1, out_channels=64, kernel_size=3)\n self.cn2 = nn.Conv2d(in_channels=64, out_channels=32, kernel_size=3)\n self.fc1 = nn.Linear(in_features=128, out_features=2)\n\n def forward(self, i):\n i = self.cn1(i)\n i = F.relu(i)\n i = F.max_pool2d(i, 2)\n i = self.cn2(i)\n i = F.relu(i)\n i = F.max_pool2d(i, 2)\n i = i.view(len(i), -1)\n i = self.fc1(i)\n i = F.log_softmax(i, dim=1)\n return i\n\n# get some random data around value\n\n\ndef get_data(value, shape):\n data = torch.ones(shape) * value\n # add some noise\n data += torch.randn(shape)**2\n return data\n\n\n# dataset\n# cat some data with different values\ndata = torch.cat(\n (get_data(\n 0, (100, 1, 14, 14)), get_data(\n 0.5, (100, 1, 14, 14))), 0)\n# labels\nlabels = torch.cat((torch.zeros(100), torch.ones(100)), 0)\n# generator\ngen = DataLoader(TensorDataset(data, labels), batch_size=25, shuffle=True)\n# network\nm = M()\n#loss and optim\nloss = nn.NLLLoss()\noptimizer = torch.optim.Adam(params=m.parameters())\n# settings for train and log\nnum_epochs = 20\nembedding_log = 5\nwriter = SummaryWriter(comment='mnist_embedding_training')\n\n# TRAIN\nfor epoch in range(num_epochs):\n for j, sample in enumerate(gen):\n n_iter = (epoch * len(gen)) + j\n # reset grad\n m.zero_grad()\n optimizer.zero_grad()\n # get batch data\n data_batch = Variable(sample[0], requires_grad=True).float()\n label_batch = Variable(sample[1], requires_grad=False).long()\n # FORWARD\n out = m(data_batch)\n loss_value = loss(out, label_batch)\n # BACKWARD\n loss_value.backward()\n optimizer.step()\n # LOGGING\n writer.add_scalar('loss', loss_value.data.item(), n_iter)\n\n if j % embedding_log == 0:\n print(\"loss_value:{}\".format(loss_value.data.item()))\n # we need 3 dimension for tensor to visualize it!\n out = torch.cat((out.data, torch.ones(len(out), 1)), 1)\n writer.add_embedding(\n out,\n metadata=label_batch.data,\n label_img=data_batch.data,\n global_step=n_iter)\n\nwriter.close()\n\n# tensorboard --logdir runs\n# you should now see a dropdown list with all the timestep,\n# last timestep should have a visible separation between the two classes\n" }, { "path": "tensorboardX/examples/demo_graph.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nfrom torch.autograd import Variable\nfrom tensorboardX import SummaryWriter\n\ndummy_input = (torch.zeros(1, 3),)\n\n\nclass LinearInLinear(nn.Module):\n def __init__(self):\n super(LinearInLinear, self).__init__()\n self.l = nn.Linear(3, 5)\n\n def forward(self, x):\n return self.l(x)\n\nwith SummaryWriter(comment='LinearInLinear') as w:\n w.add_graph(LinearInLinear(), dummy_input, True)\n\n\nclass MultipleInput(nn.Module):\n def __init__(self):\n super(MultipleInput, self).__init__()\n self.Linear_1 = nn.Linear(3, 5)\n\n\n def forward(self, x, y):\n return self.Linear_1(x+y)\n\nwith SummaryWriter(comment='MultipleInput') as w:\n w.add_graph(MultipleInput(), (torch.zeros(1, 3), torch.zeros(1, 3)), True)\n\nclass MultipleOutput(nn.Module):\n def __init__(self):\n super(MultipleOutput, self).__init__()\n self.Linear_1 = nn.Linear(3, 5)\n self.Linear_2 = nn.Linear(3, 7)\n\n def forward(self, x):\n return self.Linear_1(x), self.Linear_2(x)\n\nwith SummaryWriter(comment='MultipleOutput') as w:\n w.add_graph(MultipleOutput(), dummy_input, True)\n\n\nclass MultipleOutput_shared(nn.Module):\n def __init__(self):\n super(MultipleOutput_shared, self).__init__()\n self.Linear_1 = nn.Linear(3, 5)\n\n def forward(self, x):\n return self.Linear_1(x), self.Linear_1(x)\n\nwith SummaryWriter(comment='MultipleOutput_shared') as w:\n w.add_graph(MultipleOutput_shared(), dummy_input, True)\n\n\nclass SimpleModel(nn.Module):\n def __init__(self):\n super(SimpleModel, self).__init__()\n\n def forward(self, x):\n return x * 2\n\n\nmodel = SimpleModel()\ndummy_input = (torch.zeros(1, 2, 3),)\n\nwith SummaryWriter(comment='constantModel') as w:\n w.add_graph(model, dummy_input, True)\n\n\ndef conv3x3(in_planes, out_planes, stride=1):\n \"\"\"3x3 convolution with padding\"\"\"\n return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n padding=1, bias=False)\n\n\nclass BasicBlock(nn.Module):\n expansion = 1\n\n def __init__(self, inplanes, planes, stride=1, downsample=None):\n super(BasicBlock, self).__init__()\n self.conv1 = conv3x3(inplanes, planes, stride)\n self.bn1 = nn.BatchNorm2d(planes)\n # self.relu = nn.ReLU(inplace=True)\n self.conv2 = conv3x3(planes, planes)\n self.bn2 = nn.BatchNorm2d(planes)\n self.stride = stride\n\n def forward(self, x):\n residual = x\n\n out = self.conv1(x)\n out = self.bn1(out)\n out = F.relu(out)\n out = self.conv2(out)\n out = self.bn2(out)\n out += residual\n out = F.relu(out)\n return out\n\n\ndummy_input = torch.rand(1, 3, 224, 224)\n\nwith SummaryWriter(comment='basicblock') as w:\n model = BasicBlock(3, 3)\n w.add_graph(model, (dummy_input, ), verbose=True)\n\n\n\n\nclass Net1(nn.Module):\n def __init__(self):\n super(Net1, self).__init__()\n self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n self.conv2_drop = nn.Dropout2d()\n self.fc1 = nn.Linear(320, 50)\n self.fc2 = nn.Linear(50, 10)\n self.bn = nn.BatchNorm2d(20)\n\n def forward(self, x):\n x = F.max_pool2d(self.conv1(x), 2)\n x = F.relu(x) + F.relu(-x)\n x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))\n x = self.bn(x)\n x = x.view(-1, 320)\n x = F.relu(self.fc1(x))\n x = F.dropout(x, training=self.training)\n x = self.fc2(x)\n x = F.softmax(x, dim=1)\n return x\n\n\nclass Net2(nn.Module):\n def __init__(self):\n super(Net2, self).__init__()\n self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n self.conv2_drop = nn.Dropout2d()\n self.fc1 = nn.Linear(320, 50)\n self.fc2 = nn.Linear(50, 10)\n\n def forward(self, x):\n x = F.relu(F.max_pool2d(self.conv1(x), 2))\n x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))\n x = x.view(-1, 320)\n x = F.relu(self.fc1(x))\n x = F.dropout(x, training=self.training)\n x = self.fc2(x)\n x = F.log_softmax(x, dim=1)\n return x\n\n\ndummy_input = Variable(torch.rand(13, 1, 28, 28))\n\nmodel = Net1()\nwith SummaryWriter(comment='Net1') as w:\n w.add_graph(model, (dummy_input, ))\n\nmodel = Net2()\nwith SummaryWriter(comment='Net2') as w:\n w.add_graph(model, (dummy_input, ))\n\n\nclass SiameseNetwork(nn.Module):\n def __init__(self):\n super(SiameseNetwork, self).__init__()\n self.cnn1 = Net1()\n\n def forward_once(self, x):\n output = self.cnn1(x)\n return output\n\n def forward(self, input1, input2):\n output1 = self.forward_once(input1)\n output2 = self.forward_once(input2)\n return output1, output2\n\nmodel = SiameseNetwork()\nwith SummaryWriter(comment='SiameseNetwork') as w:\n w.add_graph(model, (dummy_input, dummy_input))\n\n\ndummy_input = torch.Tensor(1, 3, 224, 224)\n\nwith SummaryWriter(comment='alexnet') as w:\n model = torchvision.models.alexnet()\n w.add_graph(model, (dummy_input, ))\n\nwith SummaryWriter(comment='vgg19') as w:\n model = torchvision.models.vgg19()\n w.add_graph(model, (dummy_input, ))\n\nwith SummaryWriter(comment='densenet121') as w:\n model = torchvision.models.densenet121()\n w.add_graph(model, (dummy_input, ))\n\nwith SummaryWriter(comment='resnet18') as w:\n model = torchvision.models.resnet18()\n w.add_graph(model, (dummy_input, ))\n\n\n\nclass RNN(nn.Module):\n def __init__(self, input_size, hidden_size, output_size):\n super(RNN, self).__init__()\n self.hidden_size = hidden_size\n self.i2h = nn.Linear(\n n_categories +\n input_size +\n hidden_size,\n hidden_size)\n self.i2o = nn.Linear(\n n_categories +\n input_size +\n hidden_size,\n output_size)\n self.o2o = nn.Linear(hidden_size + output_size, output_size)\n self.dropout = nn.Dropout(0.1)\n self.softmax = nn.LogSoftmax(dim=1)\n\n def forward(self, category, input, hidden):\n input_combined = torch.cat((category, input, hidden), 1)\n hidden = self.i2h(input_combined)\n output = self.i2o(input_combined)\n output_combined = torch.cat((hidden, output), 1)\n output = self.o2o(output_combined)\n output = self.dropout(output)\n output = self.softmax(output)\n return output, hidden, input\n\n def initHidden(self):\n return torch.zeros(1, self.hidden_size)\n\n\nn_letters = 100\nn_hidden = 128\nn_categories = 10\nrnn = RNN(n_letters, n_hidden, n_categories)\ncat = torch.Tensor(1, n_categories)\ndummy_input = torch.Tensor(1, n_letters)\nhidden = torch.Tensor(1, n_hidden)\n\n\nout, hidden, input = rnn(cat, dummy_input, hidden)\nwith SummaryWriter(comment='RNN') as w:\n w.add_graph(rnn, (cat, dummy_input, hidden), verbose=False)\n\n\n\nlstm = torch.nn.LSTM(3, 3) # Input dim is 3, output dim is 3\ninputs = [torch.randn(1, 3) for _ in range(5)] # make a sequence of length 5\n\n# initialize the hidden state.\nhidden = (torch.randn(1, 1, 3),\n torch.randn(1, 1, 3))\nfor i in inputs:\n out, hidden = lstm(i.view(1, 1, -1), hidden)\n\nwith SummaryWriter(comment='lstm') as w:\n w.add_graph(lstm, (torch.randn(1, 3).view(1, 1, -1), hidden), verbose=True)\n\n\nimport pytest\nprint('expect error here:')\nwith pytest.raises(Exception) as e_info:\n dummy_input = torch.rand(1, 1, 224, 224)\n with SummaryWriter(comment='basicblock_error') as w:\n w.add_graph(model, (dummy_input, )) # error\n" }, { "path": "tensorboardX/examples/demo_hparams.py", "content": "from tensorboardX import SummaryWriter\nimport time\nimport random\n\n\nhparam = {'lr': [0.1, 0.01, 0.001],\n 'bsize': [1, 2, 4],\n 'n_hidden': [100, 200]}\n\nmetrics = {'accuracy', 'loss'}\n\ndef train(lr, bsize, n_hidden):\n x = random.random()\n return x, x*5\n\nwith SummaryWriter() as w:\n for lr in hparam['lr']:\n for bsize in hparam['bsize']:\n for n_hidden in hparam['n_hidden']:\n accu, loss = train(lr, bsize, n_hidden)\n \n w.add_hparams({'lr': lr, 'bsize': bsize, 'n_hidden': n_hidden},\n {'accuracy': accu, 'loss': loss})\n\n" }, { "path": "tensorboardX/examples/demo_matplotlib.py", "content": "import matplotlib.pyplot as plt\nplt.switch_backend('agg')\n\nfig = plt.figure()\n\nc1 = plt.Circle((0.2, 0.5), 0.2, color='r')\nc2 = plt.Circle((0.8, 0.5), 0.2, color='r')\n\nax = plt.gca()\nax.add_patch(c1)\nax.add_patch(c2)\nplt.axis('scaled')\n\n\nfrom tensorboardX import SummaryWriter\nwriter = SummaryWriter()\nwriter.add_figure('matplotlib', fig)\nwriter.close()\n" }, { "path": "tensorboardX/examples/demo_multiple_embedding.py", "content": "import math\nimport numpy as np\nfrom tensorboardX import SummaryWriter\n\n\ndef main():\n degrees = np.linspace(0, 3600 * math.pi / 180.0, 3600)\n degrees = degrees.reshape(3600, 1)\n labels = [\"%d\" % (i) for i in range(0, 3600)]\n\n with SummaryWriter() as writer:\n # Maybe make a bunch of data that's always shifted in some\n # way, and that will be hard for PCA to turn into a sphere?\n\n for epoch in range(0, 16):\n shift = epoch * 2 * math.pi / 16.0\n mat = np.concatenate([\n np.sin(shift + degrees * 2 * math.pi / 180.0),\n np.sin(shift + degrees * 3 * math.pi / 180.0),\n np.sin(shift + degrees * 5 * math.pi / 180.0),\n np.sin(shift + degrees * 7 * math.pi / 180.0),\n np.sin(shift + degrees * 11 * math.pi / 180.0)\n ], axis=1)\n writer.add_embedding(\n mat=mat,\n metadata=labels,\n tag=\"sin\",\n global_step=epoch)\n\n mat = np.concatenate([\n np.cos(shift + degrees * 2 * math.pi / 180.0),\n np.cos(shift + degrees * 3 * math.pi / 180.0),\n np.cos(shift + degrees * 5 * math.pi / 180.0),\n np.cos(shift + degrees * 7 * math.pi / 180.0),\n np.cos(shift + degrees * 11 * math.pi / 180.0)\n ], axis=1)\n writer.add_embedding(\n mat=mat,\n metadata=labels,\n tag=\"cos\",\n global_step=epoch)\n\n mat = np.concatenate([\n np.tan(shift + degrees * 2 * math.pi / 180.0),\n np.tan(shift + degrees * 3 * math.pi / 180.0),\n np.tan(shift + degrees * 5 * math.pi / 180.0),\n np.tan(shift + degrees * 7 * math.pi / 180.0),\n np.tan(shift + degrees * 11 * math.pi / 180.0)\n ], axis=1)\n writer.add_embedding(\n mat=mat,\n metadata=labels,\n tag=\"tan\",\n global_step=epoch)\n\n\nif __name__ == \"__main__\":\n main()\n\n# tensorboard --logdir runs\n# Under \"Projection, you should see\n# 48 tensor found named\n# cos:cos-00000 to cos:cos-00016\n# sin:sin-00000 to sin:sin-00016\n# tan:tan-00000 to tan:tan-00016\n" }, { "path": "tensorboardX/examples/demo_nvidia_smi.py", "content": "\"\"\"\nwrite gpu and (gpu) memory usage of nvidia cards as scalar\n\"\"\"\nfrom tensorboardX import SummaryWriter\nimport time\nimport torch\ntry:\n import nvidia_smi\n nvidia_smi.nvmlInit()\n handle = nvidia_smi.nvmlDeviceGetHandleByIndex(0) # gpu0\nexcept ImportError:\n print('This demo needs nvidia-ml-py or nvidia-ml-py3')\n exit()\n\n\nwith SummaryWriter() as writer:\n x = []\n for n_iter in range(50):\n x.append(torch.Tensor(1000, 1000).cuda())\n res = nvidia_smi.nvmlDeviceGetUtilizationRates(handle)\n writer.add_scalar('nv/gpu', res.gpu, n_iter)\n res = nvidia_smi.nvmlDeviceGetMemoryInfo(handle)\n writer.add_scalar('nv/gpu_mem', res.used, n_iter)\n time.sleep(0.1)\n" }, { "path": "tensorboardX/examples/demo_onnx.py", "content": "from tensorboardX import SummaryWriter\n\nimport subprocess\nzoo_address = 'https://onnxzoo.blob.core.windows.net/models/opset_8/mnist/mnist.tar.gz'\n\nres = subprocess.call(['wget', '-nc', zoo_address])\nassert res == 0, 'cannot download example onnx model from the zoo'\nres = subprocess.call(['tar', 'xf', 'mnist.tar.gz', '-C', 'examples/', 'mnist/model.onnx'])\n\n\n\nwith SummaryWriter() as w:\n w.add_onnx_graph('examples/mnist/model.onnx')\n # w.add_onnx_graph('/Users/dexter/Downloads/resnet50/model.onnx')\n" }, { "path": "tensorboardX/examples/demo_purge.py", "content": "from time import sleep\nfrom tensorboardX import SummaryWriter\n\nwith SummaryWriter(logdir='runs/purge') as w:\n for i in range(100):\n w.add_scalar('purgetest', i, i)\n\nsleep(1.0)\n\nwith SummaryWriter(logdir='runs/purge', purge_step=42) as w:\n # event 42~99 are removed (inclusively)\n for i in range(42, 100):\n w.add_scalar('purgetest', 42, i)\n" }, { "path": "tensorboardX/setup.cfg", "content": "[metadata]\nlicense_file = LICENSE\n\n[bdist_wheel]\nuniversal = 1\n" }, { "path": "tensorboardX/setup.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\nimport subprocess\nimport os\nfrom setuptools import setup, find_packages\nfrom setuptools.command.develop import develop\nfrom setuptools.command.install import install\n\n# Dynamically compile protos\ndef compileProtoBuf():\n res = subprocess.call(['bash', './compile.sh'])\n assert res == 0, 'cannot compile protobuf'\n\nclass PostDevelopCommand(develop):\n \"\"\"Post-installation for development mode.\"\"\"\n def run(self):\n compileProtoBuf()\n develop.run(self)\n\n\nclass PostInstallCommand(install):\n \"\"\"Post-installation for installation mode.\"\"\"\n def run(self):\n compileProtoBuf()\n import os\n os.system(\"pip install protobuf numpy six\")\n install.run(self)\n\nwith open('HISTORY.rst') as history_file:\n history = history_file.read()\n\npreparing_PyPI_package = False\nversion_git = version = '1.8'\n\nif not preparing_PyPI_package:\n if os.path.exists('.git'):\n sha = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('ascii').strip()\n version_git = version_git + '+' + sha[:7]\n\n with open('tensorboardX/__init__.py', 'a') as f:\n f.write('\\n__version__ = \"{}\"\\n'.format(version_git))\n\nrequirements = [\n 'numpy',\n 'protobuf >= 3.6.1',\n 'six',\n]\n\ntest_requirements = [\n 'pytest',\n 'matplotlib',\n 'crc32c',\n]\n\nsetup(\n name='tensorboardX',\n version=version_git,\n description='TensorBoardX lets you watch Tensors Flow without Tensorflow',\n long_description=history,\n author='Tzu-Wei Huang',\n author_email='huang.dexter@gmail.com',\n url='https://github.com/lanpa/tensorboardX',\n packages=['tensorboardX'],\n include_package_data=True,\n install_requires=requirements,\n license='MIT license',\n zip_safe=False,\n classifiers=[\n 'Development Status :: 2 - Pre-Alpha',\n 'Intended Audience :: Developers',\n 'License :: OSI Approved :: MIT License',\n 'Natural Language :: English',\n 'Programming Language :: Python :: 2',\n 'Programming Language :: Python :: 2.7',\n 'Programming Language :: Python :: 3',\n 'Programming Language :: Python :: 3.4',\n 'Programming Language :: Python :: 3.5',\n 'Programming Language :: Python :: 3.6',\n ],\n cmdclass={\n 'develop': PostDevelopCommand,\n 'install': PostInstallCommand,\n },\n test_suite='tests',\n tests_require=test_requirements\n)\n\n\n# checklist: update History.rst readme.md\n# change preparing_PyPI_package to True\n# remove __version__ = \"1.old\" in __init__.py\n# commit\n# add tag\n# python setup.py sdist bdist_wheel --universal\n# twine upload dist/*\n# push commit" }, { "path": "tensorboardX/tensorboardX/__init__.py", "content": "\"\"\"A module for visualization with tensorboard\n\"\"\"\n\nfrom .record_writer import RecordWriter\nfrom .torchvis import TorchVis\nfrom .writer import FileWriter, SummaryWriter\n\n__version__ = \"1.8\" # will be overwritten if run setup.py\n" }, { "path": "tensorboardX/tensorboardX/beholder/__init__.py", "content": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom .beholder import Beholder\nfrom .beholder import BeholderHook\n" }, { "path": "tensorboardX/tensorboardX/beholder/beholder.py", "content": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom ..proto.summary_pb2 import Summary\nfrom ..proto.summary_pb2 import SummaryMetadata\nfrom ..proto.tensor_pb2 import TensorProto\nfrom ..proto.tensor_shape_pb2 import TensorShapeProto\n\nimport os\nimport time\n\nimport numpy as np\n# import tensorflow as tf\n\n# from tensorboard.plugins.beholder import im_util\n# from . import im_util\nfrom .file_system_tools import read_pickle,\\\n write_pickle, write_file\nfrom .shared_config import PLUGIN_NAME, TAG_NAME,\\\n SUMMARY_FILENAME, DEFAULT_CONFIG, CONFIG_FILENAME, SUMMARY_COLLECTION_KEY_NAME, SECTION_INFO_FILENAME\nfrom . import video_writing\n# from .visualizer import Visualizer\n\n\nclass Beholder(object):\n\n def __init__(self, logdir):\n self.PLUGIN_LOGDIR = logdir + '/plugins/' + PLUGIN_NAME\n\n self.is_recording = False\n self.video_writer = video_writing.VideoWriter(\n self.PLUGIN_LOGDIR,\n outputs=[video_writing.FFmpegVideoOutput, video_writing.PNGVideoOutput])\n\n self.last_image_shape = []\n self.last_update_time = time.time()\n self.config_last_modified_time = -1\n self.previous_config = dict(DEFAULT_CONFIG)\n\n if not os.path.exists(self.PLUGIN_LOGDIR + '/config.pkl'):\n os.makedirs(self.PLUGIN_LOGDIR)\n write_pickle(DEFAULT_CONFIG,\n '{}/{}'.format(self.PLUGIN_LOGDIR, CONFIG_FILENAME))\n\n # self.visualizer = Visualizer(self.PLUGIN_LOGDIR)\n def _get_config(self):\n '''Reads the config file from disk or creates a new one.'''\n filename = '{}/{}'.format(self.PLUGIN_LOGDIR, CONFIG_FILENAME)\n modified_time = os.path.getmtime(filename)\n\n if modified_time != self.config_last_modified_time:\n config = read_pickle(filename, default=self.previous_config)\n self.previous_config = config\n else:\n config = self.previous_config\n\n self.config_last_modified_time = modified_time\n return config\n\n def _write_summary(self, frame):\n '''Writes the frame to disk as a tensor summary.'''\n path = '{}/{}'.format(self.PLUGIN_LOGDIR, SUMMARY_FILENAME)\n smd = SummaryMetadata()\n tensor = TensorProto(\n dtype='DT_FLOAT',\n float_val=frame.reshape(-1).tolist(),\n tensor_shape=TensorShapeProto(\n dim=[TensorShapeProto.Dim(size=frame.shape[0]),\n TensorShapeProto.Dim(size=frame.shape[1]),\n TensorShapeProto.Dim(size=frame.shape[2])]\n )\n )\n summary = Summary(value=[Summary.Value(\n tag=TAG_NAME, metadata=smd, tensor=tensor)]).SerializeToString()\n write_file(summary, path)\n\n @staticmethod\n def stats(tensor_and_name):\n imgstats = []\n for (img, name) in tensor_and_name:\n immax = img.max()\n immin = img.min()\n imgstats.append(\n {\n 'height': img.shape[0],\n 'max': str(immax),\n 'mean': str(img.mean()),\n 'min': str(immin),\n 'name': name,\n 'range': str(immax - immin),\n 'shape': str((img.shape[1], img.shape[2]))\n })\n return imgstats\n\n def _get_final_image(self, config, trainable=None, arrays=None, frame=None):\n if config['values'] == 'frames':\n # print('===frames===')\n final_image = frame\n elif config['values'] == 'arrays':\n # print('===arrays===')\n final_image = np.concatenate([arr for arr, _ in arrays])\n stat = self.stats(arrays)\n write_pickle(\n stat, '{}/{}'.format(self.PLUGIN_LOGDIR, SECTION_INFO_FILENAME))\n elif config['values'] == 'trainable_variables':\n # print('===trainable===')\n final_image = np.concatenate([arr for arr, _ in trainable])\n stat = self.stats(trainable)\n write_pickle(\n stat, '{}/{}'.format(self.PLUGIN_LOGDIR, SECTION_INFO_FILENAME))\n if len(final_image.shape) == 2: # Map grayscale images to 3D tensors.\n final_image = np.expand_dims(final_image, -1)\n\n return final_image\n\n def _enough_time_has_passed(self, FPS):\n '''For limiting how often frames are computed.'''\n if FPS == 0:\n return False\n else:\n earliest_time = self.last_update_time + (1.0 / FPS)\n return time.time() >= earliest_time\n\n def _update_frame(self, trainable, arrays, frame, config):\n final_image = self._get_final_image(config, trainable, arrays, frame)\n self._write_summary(final_image)\n self.last_image_shape = final_image.shape\n\n return final_image\n\n def _update_recording(self, frame, config):\n '''Adds a frame to the current video output.'''\n # pylint: disable=redefined-variable-type\n should_record = config['is_recording']\n\n if should_record:\n if not self.is_recording:\n self.is_recording = True\n print('Starting recording using %s',\n self.video_writer.current_output().name())\n self.video_writer.write_frame(frame)\n elif self.is_recording:\n self.is_recording = False\n self.video_writer.finish()\n print('Finished recording')\n\n # TODO: blanket try and except for production? I don't someone's script to die\n # after weeks of running because of a visualization.\n def update(self, trainable=None, arrays=None, frame=None):\n '''Creates a frame and writes it to disk.\n\n Args:\n trainable: a list of namedtuple (tensors, name).\n arrays: a list of namedtuple (tensors, name).\n frame: lalala\n '''\n\n new_config = self._get_config()\n if True or self._enough_time_has_passed(self.previous_config['FPS']):\n # self.visualizer.update(new_config)\n self.last_update_time = time.time()\n final_image = self._update_frame(\n trainable, arrays, frame, new_config)\n self._update_recording(final_image, new_config)\n\n ##############################################################################\n # @staticmethod\n # def gradient_helper(optimizer, loss, var_list=None):\n # '''A helper to get the gradients out at each step.\n\n # Args:\n # optimizer: the optimizer op.\n # loss: the op that computes your loss value.\n\n # Returns: the gradient tensors and the train_step op.\n # '''\n # if var_list is None:\n # var_list = tf.trainable_variables()\n\n # grads_and_vars = optimizer.compute_gradients(loss, var_list=var_list)\n # grads = [pair[0] for pair in grads_and_vars]\n\n # return grads, optimizer.apply_gradients(grads_and_vars)\n\n\n# implements pytorch backward later\nclass BeholderHook():\n pass\n # \"\"\"SessionRunHook implementation that runs Beholder every step.\n\n # Convenient when using tf.train.MonitoredSession:\n # ```python\n # beholder_hook = BeholderHook(LOG_DIRECTORY)\n # with MonitoredSession(..., hooks=[beholder_hook]) as sess:\n # sess.run(train_op)\n # ```\n # \"\"\"\n # def __init__(self, logdir):\n # \"\"\"Creates new Hook instance\n\n # Args:\n # logdir: Directory where Beholder should write data.\n # \"\"\"\n # self._logdir = logdir\n # self.beholder = None\n\n # def begin(self):\n # self.beholder = Beholder(self._logdir)\n\n # def after_run(self, run_context, unused_run_values):\n # self.beholder.update(run_context.session)\n" }, { "path": "tensorboardX/tensorboardX/beholder/file_system_tools.py", "content": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport pickle\n\n# import tensorflow as tf\n# from google.protobuf import message\n\n\ndef write_file(contents, path, mode='wb'):\n with open(path, mode) as new_file:\n new_file.write(contents)\n\n\ndef write_pickle(obj, path):\n with open(path, 'wb') as new_file:\n pickle.dump(obj, new_file)\n\n\ndef read_pickle(path, default=None):\n with open(path, 'rb') as pickle_file:\n result = pickle.load(pickle_file)\n return result\n" }, { "path": "tensorboardX/tensorboardX/beholder/shared_config.py", "content": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nPLUGIN_NAME = 'beholder'\nTAG_NAME = 'beholder-frame'\nSUMMARY_FILENAME = 'frame.summary'\nCONFIG_FILENAME = 'config.pkl'\nSECTION_INFO_FILENAME = 'section-info.pkl'\nSUMMARY_COLLECTION_KEY_NAME = 'summaries_beholder'\n\nDEFAULT_CONFIG = {\n 'values': 'trainable_variables',\n 'mode': 'variance',\n 'scaling': 'layer',\n 'window_size': 15,\n 'FPS': 10,\n 'is_recording': False,\n 'show_all': False,\n 'colormap': 'magma'\n}\n\nSECTION_HEIGHT = 128\nIMAGE_WIDTH = 512 + 256\n\nTB_WHITE = 245\n" }, { "path": "tensorboardX/tensorboardX/beholder/video_writing.py", "content": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport abc\nimport os\nimport subprocess\nimport time\n\nimport numpy as np\n\n\nclass VideoWriter(object):\n \"\"\"Video file writer that can use different output types.\n\n Each VideoWriter instance writes video files to a specified directory, using\n the first available VideoOutput from the provided list.\n \"\"\"\n\n def __init__(self, directory, outputs):\n self.directory = directory\n # Filter to the available outputs\n self.outputs = [out for out in outputs if out.available()]\n if not self.outputs:\n raise IOError('No available video outputs')\n self.output_index = 0\n self.output = None\n self.frame_shape = None\n\n def current_output(self):\n return self.outputs[self.output_index]\n\n def write_frame(self, np_array):\n # Reset whenever we encounter a new frame shape.\n if self.frame_shape != np_array.shape:\n if self.output:\n self.output.close()\n self.output = None\n self.frame_shape = np_array.shape\n print('Starting video with frame shape: %s', self.frame_shape)\n # Write the frame, advancing across output types as necessary.\n original_output_index = self.output_index\n for self.output_index in range(original_output_index, len(self.outputs)):\n try:\n if not self.output:\n new_output = self.outputs[self.output_index]\n if self.output_index > original_output_index:\n print('Falling back to video output %s',\n new_output.name())\n self.output = new_output(self.directory, self.frame_shape)\n self.output.emit_frame(np_array)\n return\n except (IOError, OSError) as e:\n print('Video output type %s not available: %s',\n self.current_output().name(), str(e))\n if self.output:\n self.output.close()\n self.output = None\n raise IOError('Exhausted available video outputs')\n\n def finish(self):\n if self.output:\n self.output.close()\n self.output = None\n self.frame_shape = None\n # Reconsider failed outputs when video is manually restarted.\n self.output_index = 0\n\n\nclass VideoOutput(object):\n \"\"\"Base class for video outputs supported by VideoWriter.\"\"\"\n\n __metaclass__ = abc.ABCMeta\n\n # Would add @abc.abstractmethod in python 3.3+\n @classmethod\n def available(cls):\n raise NotImplementedError()\n\n @classmethod\n def name(cls):\n return cls.__name__\n\n @abc.abstractmethod\n def emit_frame(self, np_array):\n raise NotImplementedError()\n\n @abc.abstractmethod\n def close(self):\n raise NotImplementedError()\n\n\nclass PNGVideoOutput(VideoOutput):\n \"\"\"Video output implemented by writing individual PNGs to disk.\"\"\"\n\n @classmethod\n def available(cls):\n return True\n\n def __init__(self, directory, frame_shape):\n del frame_shape # unused\n self.directory = directory + '/video-frames-{}'.format(time.time())\n self.frame_num = 0\n os.makedirs(self.directory)\n\n def emit_frame(self, np_array):\n filename = self.directory + '/{:05}.png'.format(self.frame_num)\n self._write_image(np_array.astype(np.uint8), filename)\n self.frame_num += 1\n\n def _write_image(self, im, filename):\n from PIL import Image\n Image.fromarray(im).save(filename)\n\n def close(self):\n pass\n\n\nclass FFmpegVideoOutput(VideoOutput):\n \"\"\"Video output implemented by streaming to FFmpeg with .mp4 output.\"\"\"\n\n @classmethod\n def available(cls):\n # Silently check if ffmpeg is available.\n try:\n with open(os.devnull, 'wb') as devnull:\n subprocess.check_call(\n ['ffmpeg', '-version'], stdout=devnull, stderr=devnull)\n return True\n except (OSError, subprocess.CalledProcessError):\n return False\n\n def __init__(self, directory, frame_shape):\n self.filename = directory + '/video-{}.webm'.format(time.time())\n if len(frame_shape) != 3:\n raise ValueError(\n 'Expected rank-3 array for frame, got %s' % str(frame_shape))\n # Set input pixel format based on channel count.\n if frame_shape[2] == 1:\n pix_fmt = 'gray'\n elif frame_shape[2] == 3:\n pix_fmt = 'rgb24'\n else:\n raise ValueError('Unsupported channel count %d' % frame_shape[2])\n\n command = [\n 'ffmpeg',\n '-y', # Overwite output\n # Input options - raw video file format and codec.\n '-f', 'rawvideo',\n '-vcodec', 'rawvideo',\n # Width x height.\n '-s', '%dx%d' % (frame_shape[1], frame_shape[0]),\n '-pix_fmt', pix_fmt,\n '-r', '15', # Frame rate: arbitrarily use 15 frames per second.\n '-i', '-', # Use stdin.\n '-an', # No audio.\n # Output options - use lossless VP9 codec inside .webm.\n '-vcodec', 'libvpx-vp9',\n '-lossless', '1',\n # Using YUV is most compatible, though conversion from RGB skews colors.\n '-pix_fmt', 'yuv420p',\n self.filename\n ]\n PIPE = subprocess.PIPE\n self.ffmpeg = subprocess.Popen(\n command, stdin=PIPE, stdout=PIPE, stderr=PIPE)\n\n def _handle_error(self):\n _, stderr = self.ffmpeg.communicate()\n bar = '=' * 40\n print('Error writing to FFmpeg:\\n{}\\n{}\\n{}',\n bar, stderr, bar)\n\n def emit_frame(self, np_array):\n try:\n self.ffmpeg.stdin.write(np_array.tobytes())\n self.ffmpeg.stdin.flush()\n except IOError:\n self._handle_error()\n raise IOError('Failure invoking FFmpeg')\n\n def close(self):\n if self.ffmpeg.poll() is None:\n # Close stdin and consume and discard stderr/stdout.\n self.ffmpeg.communicate()\n self.ffmpeg = None\n" }, { "path": "tensorboardX/tensorboardX/caffe2_graph.py", "content": "from __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\n\nimport copy\nimport logging\nimport os\nimport re\nimport six\n\nfrom builtins import bytes\nfrom caffe2.proto import caffe2_pb2\nfrom caffe2.python import core, workspace\n\nfrom .proto.graph_pb2 import GraphDef\nfrom .proto.node_def_pb2 import NodeDef\nfrom .proto.tensor_shape_pb2 import TensorShapeProto\n\n\ndef _make_unique_name(seen, name, min_version=0):\n '''\n Make the name unique by appending a unique number to the name. Used for SSA.\n\n Args:\n seen (set): Set of names that have already been used (with respect to\n some context).\n name (string): The name to make unique\n min_version (number): Starting index. Is incremented continually until\n it can make the resulting name unique relative to 'seen'.\n\n Returns:\n x (string): A version of name that is not in seen.\n '''\n assert name is not None\n i = min_version\n x = '%s_%d' % (name, i) if i else name\n while x in seen:\n i += 1\n x = '%s_%d' % (name, i)\n seen.add(x)\n return x\n\n\ndef _rename_tensorflow_style(shapes, blob_name_tracker, ops):\n '''\n Convert some of the common names in Caffe2 to tensorflow.\n NOTE: The common names in both Caffe2 and Tensorflow are currently\n hardcoded, if either side changes at some point, then this code should\n change as well.\n\n Args:\n shapes: Dictionary mapping blob names to their shapes/dimensions.\n blob_name_tracker: Dictionary of all unique blob names (with respect to\n some context).\n ops: List of Caffe2 operators\n\n Returns:\n None. The _rename_all() call modifies blob_name_tracker and ops in-place.\n '''\n WEIGHT = re.compile(r\"(_w)$\")\n WEIGHT_ = re.compile(r\"(_w_)\")\n BN = re.compile(r\"(_bn)$\")\n BN_ = re.compile(r\"(_bn_)\")\n BIAS = re.compile(r\"(_b)$\")\n BIAS_ = re.compile(r\"(_b_)\")\n SCALE = re.compile(r\"(_s)$\")\n SCALE_ = re.compile(r\"(_s_)\")\n SUM = re.compile(r\"(_sum)$\")\n SUM_ = re.compile(r\"(_sum_)\")\n BRANCH = re.compile(r\"(_branch)\")\n\n def f(name):\n inter_name = WEIGHT_.sub('/weight_', WEIGHT.sub('/weight', name))\n inter_name = BN_.sub('/batchnorm_', BN.sub('/batchnorm', inter_name))\n inter_name = BIAS_.sub('/bias_', BIAS.sub('/bias', inter_name))\n inter_name = SCALE_.sub('/scale_', SCALE.sub('/scale', inter_name))\n inter_name = SUM_.sub('/sum_', SUM.sub('/sum', inter_name))\n new_name = BRANCH.sub('/branch', inter_name)\n return new_name\n _rename_all(shapes, blob_name_tracker, ops, f)\n\n\ndef _convert_to_ssa(shapes, blob_name_tracker, ops):\n '''\n Convert an operator graph to SSA (i.e. out-of-place).\n i.e. blobs will be renamed so that each blob is produced only once.\n\n Args:\n shapes: Dictionary mapping blob names to their shapes/dimensions.\n blob_name_tracker: Dictionary of all unique blob names (with respect to\n some context).\n ops: List of Caffe2 operators\n\n Returns:\n None. Modifies blob_name_tracker and ops in-place.\n '''\n ir = core.IR(ops)\n seen = set()\n versioned = {}\n new_shapes = {}\n new_blob_name_tracker = {}\n\n def ssa_name(name, versions):\n assert name in versions\n version = versions[name]\n if (name, version) in versioned:\n return versioned[(name, version)]\n # Always setting name2 = `{name}_{version}` would work, but we also try\n # to avoid a trailing `_0`, so we have to be careful not to introduce\n # name collisions, such as (foo_1, 0) = foo_1 = (foo, 1).\n # Note: operator names (if any) will be handled later.\n new_name = _make_unique_name(seen, name, min_version=version)\n versioned[(name, version)] = new_name\n # Transfer shape.\n if name in shapes:\n new_shapes[new_name] = shapes[name]\n if blob_name_tracker and name in blob_name_tracker:\n new_blob_name_tracker[new_name] = blob_name_tracker[name]\n return new_name\n\n for (op, ssa) in zip(ops, ir.ssa):\n assert op is ssa.op\n inputs = list(op.input)\n outputs = list(op.output)\n del op.input[:]\n del op.output[:]\n op.input.extend(ssa_name(name, ssa.in_versions) for name in inputs)\n op.output.extend(ssa_name(name, ssa.out_versions) for name in outputs)\n\n shapes.clear()\n shapes.update(new_shapes)\n if blob_name_tracker:\n blob_name_tracker.clear()\n blob_name_tracker.update(new_blob_name_tracker)\n\n\ndef _get_blob_names(ops):\n '''\n Get all the operator input and output blobs and perform dedup on their names.\n\n Args:\n ops: List of Caffe2 operators to extract inputs and outputs from\n\n Returns:\n set containing distinct inputs and outputs from 'ops'\n '''\n names = set()\n for op in ops:\n names.update(op.input)\n names.update(op.output)\n return {name: name for name in names}\n\n\ndef _remap_keys(old_dict, rename_fn):\n '''\n Rename keys of 'old_dict' according to 'rename_fn'.\n\n Args:\n old_dict: Dictionary (i.e. containing blob_name -> blob_name\n relationships.)\n remap_fn: Function string -> string for renaming.\n\n Returns:\n None. Modifies old_dict in-place.\n '''\n new_dict = {rename_fn(key): value for key,\n value in six.iteritems(old_dict)}\n old_dict.clear()\n old_dict.update(new_dict)\n\n\ndef _rename_all(shapes, blob_name_tracker, ops, rename_fn):\n '''\n Rename all the names in the operators.\n\n Args:\n shapes: Dictionary mapping blob names to their shapes/dimensions.\n blob_name_tracker: Dictionary of all unique blob names (with respect to\n some context).\n ops: List of Caffe2 operators\n rename_fn: Function string -> string that specifies how to rename\n\n Returns:\n None. Modifies shapes, blob_name_tracker and ops in-place using the\n specified 'rename_fn'.\n '''\n seen = set()\n renamed = {}\n\n def g(name):\n \"\"\" Collision-free version of f.\n \"\"\"\n if name is None:\n return None\n if name in renamed:\n return renamed[name]\n new_name = _make_unique_name(seen, rename_fn(name))\n renamed[name] = new_name\n return new_name\n\n for op in ops:\n inputs = list(op.input)\n outputs = list(op.output)\n del op.input[:]\n del op.output[:]\n op.input.extend(g(name) for name in inputs)\n op.output.extend(g(name) for name in outputs)\n\n _remap_keys(shapes, g)\n if blob_name_tracker:\n _remap_keys(blob_name_tracker, g)\n # Rename all operator names (if any) independently so that the\n # unique-fication happens only once in _fill_missing_operator_names().\n seen.clear()\n renamed.clear()\n for op in ops:\n op.name = g(op.name)\n\n\ndef _add_gradient_scope(shapes, blob_name_tracker, ops):\n \"\"\"\n For all operators or blobs with name containing \"_grad\", add a\n \"GRADIENTS/\" scope.\n Note: breaks graph execution since the blob -> gradient mapping is\n hardcoded.\n\n Args:\n shapes: Dictionary mapping blob names to their shapes/dimensions.\n blob_name_tracker: Dictionary of all unique blob names (with respect to\n some context).\n ops: List of Caffe2 operators\n\n Returns:\n None. Modifies shapes, blob_name_tracker and ops in-place by renaming.\n \"\"\"\n def f(name):\n if '_grad' in name:\n return 'GRADIENTS/{}'.format(name)\n else:\n return name\n _rename_all(shapes, blob_name_tracker, ops, f)\n\n\ndef _replace_colons(shapes, blob_name_tracker, ops, repl):\n '''\n `:i` has a special meaning in Tensorflow. This function replaces all colons\n with $ to avoid any possible conflicts.\n\n Args:\n shapes: Dictionary mapping blob names to their shapes/dimensions.\n blob_name_tracker: Dictionary of all unique blob names (with respect to\n some context).\n ops: List of Caffe2 operators\n repl: String representing the text to replace ':' with. Usually this is\n '$'.\n\n Returns:\n None. Modifies blob_name_tracker in-place.\n\n '''\n def f(name):\n return name.replace(':', repl)\n _rename_all(shapes, blob_name_tracker, ops, f)\n\n\ndef _fill_missing_operator_names(ops):\n '''\n Give missing operators a name.\n We expect C2 operators to be generally unnamed. This gives them a scope\n (inferred from their outputs) and a name after their type. Duplicates will\n be postfixed by an index.\n\n Args:\n ops: List of Caffe2 operators to assign names to.\n\n Returns:\n None: Modifies 'ops' in-place.\n '''\n seen = set()\n for op in ops:\n # Make sure operator names don't collide with blobs.\n seen.update(op.input)\n seen.update(op.output)\n for op in ops:\n if op.name:\n name = op.name\n elif op.output or op.input:\n name_list = [os.path.dirname(name)\n for name in op.output or op.input]\n scope = os.path.commonprefix(name_list)\n name = os.path.join(scope, op.type)\n else:\n name = op.type\n assert(name)\n op.name = _make_unique_name(seen, name)\n\n\ndef _tf_device(device_option):\n '''\n Handle the devices.\n\n Args:\n device_option (caffe2_pb2.DeviceOption): DeviceOption protobuf,\n associated to an operator, that contains information such as\n device_type (optional), cuda_gpu_id (optional), node_name (optional,\n tells which node the operator should execute on). See caffe2.proto\n in caffe2/proto for the full list.\n\n Returns:\n Formatted string representing device information contained in\n device_option.\n '''\n if not device_option.HasField(\"device_type\"):\n return \"\"\n if device_option.device_type == caffe2_pb2.CPU or device_option.device_type == caffe2_pb2.MKLDNN:\n return \"/cpu:*\"\n if device_option.device_type == caffe2_pb2.CUDA:\n return \"/gpu:{}\".format(device_option.device_id)\n raise Exception(\"Unhandled device\", device_option)\n\n\ndef _add_tf_shape(attr_dict, ints):\n '''\n Converts a list of ints to a TensorShapeProto representing the dimensions of\n a blob/object.\n\n Args:\n attr_dict: Dictionary to update (usually attributes of a Node)\n ints: List of integers representing dimensions of some object.\n\n Returns:\n None. Modifies attr_dict in-place.\n '''\n shape_proto = TensorShapeProto()\n for i in ints:\n dim = TensorShapeProto.Dim()\n dim.size = i\n shape_proto.dim.extend([dim])\n attr_dict['_output_shapes'].list.shape.extend([shape_proto])\n\n\ndef _set_tf_attr(attr_dict, arg):\n '''\n Add attributes to a node. Key is the arg.name, and values can be shape,\n floats, strings, ints or an empty list.\n\n Args:\n attr_dict: Dictionary to update (usually attributes of a Node)\n arg: Object with name and data fields.\n\n Returns:\n None. Modifies attr_dict in-place.\n '''\n k = arg.name\n if k == 'shape' and arg.ints:\n _add_tf_shape(attr_dict, arg.ints)\n return\n # Float\n if arg.HasField(\"f\"):\n attr_dict[k].f = arg.f\n return\n # Integer\n if arg.HasField(\"i\"):\n attr_dict[k].i = arg.i\n return\n # String\n if arg.HasField(\"s\"):\n attr_dict[k].s = (\n arg.s if isinstance(arg.s, bytes) else str(arg.s).encode('utf-8')\n )\n return\n if arg.floats:\n attr_dict[k].list.f.extend(arg.floats)\n return\n if arg.ints:\n attr_dict[k].list.i.extend(arg.ints)\n return\n if arg.strings:\n attr_dict[k].list.s.extend(\n s if isinstance(s, bytes) else str(s).encode('utf-8')\n for s in arg.strings\n )\n return\n # The value is an empty list.\n attr_dict[k].list.s.extend([])\n\n\ndef _operator_to_node(shapes, op):\n '''\n Converts an operator to a node in a TF graph.\n\n Args:\n shapes: Dictionary mapping blob names to their shapes/dimensions.\n op: The Caffe2 operator to convert to a TF graph node.\n\n Returns:\n n: The TF graph node created from op.\n '''\n assert op.name, op\n n = NodeDef()\n n.name = op.name\n n.input.extend(op.input)\n n.op = op.type\n n.device = _tf_device(op.device_option)\n if shapes:\n # Add shapes in order.\n for output in op.output:\n if output not in shapes:\n break\n _add_tf_shape(n.attr, shapes[output])\n for arg in op.arg:\n _set_tf_attr(n.attr, arg)\n return n\n\n\ndef _operator_to_node_simp(op, inter_blobs, seen):\n '''\n Convert the operators to nodes.\n\n Args:\n op: Caffe2 operator to convert to node\n inter_blobs: Set of intermediate blobs\n seen: Names that have already been used and are not unique\n\n Returns:\n nodes: Nodes representing 'op' and the outputs of 'op'\n '''\n assert op\n nodes = []\n outputs = [o for o in op.output if o not in inter_blobs]\n seen.update(outputs)\n len_outputs = len(outputs)\n if len_outputs == 1:\n n = NodeDef()\n n.name = outputs[0]\n # Here we are sure the name is unique.\n n.input.extend(op.input)\n n.op = op.type\n n.device = _tf_device(op.device_option)\n for arg in op.arg:\n _set_tf_attr(n.attr, arg)\n nodes.append(n)\n elif len_outputs > 1:\n # Create a name that is likely unique\n if op.name:\n name = op.name\n else:\n name_list = [name for name in outputs]\n scope = os.path.commonprefix(name_list)\n name = os.path.join(scope, op.type)\n assert(name)\n op.name = _make_unique_name(seen, name)\n device = _tf_device(op.device_option)\n\n # Create additional output nodes\n for output in outputs:\n n = NodeDef()\n n.name = output\n n.input.extend([op.name])\n n.op = 'Blob'\n n.device = device\n nodes.append(n)\n\n # Node for the current op\n n = NodeDef()\n n.name = op.name\n n.input.extend(op.input)\n n.op = op.type\n n.device = device\n for arg in op.arg:\n _set_tf_attr(n.attr, arg)\n nodes.append(n)\n\n return nodes\n\n\ndef _blob_to_node(producing_ops, shapes, name):\n '''\n Converts a blob (operator input or output) to a node in a TF graph.\n\n Args:\n producing_ops: Dictionary of blob name to list of\n (producing_op, blob_index within producing_op.output) mapping.\n shapes: Dictionary mapping blob names to their shapes/dimensions.\n name: String representing the name of this blob.\n\n Returns:\n n: The TF graph node created from this blob.\n '''\n assert name\n n = NodeDef()\n n.name = name\n # Get all ops that have the blob corresponding to 'name' as one of their\n # outputs. See _operators_to_graph_def.\n produced_by = producing_ops.get(name, [])\n if len(produced_by) > 0:\n n.op = 'Blob'\n else:\n # This blob is not produced but is instead a TF Placeholder where a\n # value is passed in.\n n.op = 'Placeholder'\n n.input.extend('%s:%d' % (p_op.name, i) for p_op, i in produced_by)\n if produced_by:\n device = produced_by[0][0].device_option\n if (all(producer[0].device_option == device for producer in produced_by)):\n n.device = _tf_device(device)\n if shapes and name in shapes:\n _add_tf_shape(n.attr, shapes[name])\n return n\n\n\ndef _clear_debug_info(ops, perform_clear):\n '''\n Removes debug information from operators, they are copious.\n\n Args:\n ops: List of Caffe2 operators\n perform_clear: Boolean passed from _operators_to_graph_def specifying\n whether to remove the debug information. This boolean is passed into\n this function to reduce the complexity of _operators_to_graph_def.\n\n Returns:\n None. Modifies the list of Caffe2 operators in-place and removes the\n 'debug_info' field.\n\n '''\n if not perform_clear:\n return\n\n for op in ops:\n if op.HasField('debug_info'):\n op.ClearField('debug_info')\n\n\ndef _check_if_forward(blob):\n '''\n Blobs with names containing '_m' or 'grad' are part of the backward pass.\n This function references facebookresearch/Detectron/detectron/utils/net.py.\n\n Args:\n blob: The blob to inspect\n\n Returns:\n Boolean representing whether this blob is part of the forward pass\n '''\n #\n return (blob.find('__m') < 0 or blob.find('grad') < 0)\n\n\ndef _check_if_cpu(blob):\n '''\n Check if the blob's name starts with '_gpu'.\n\n Args:\n blob: The blob to inspect\n\n Returns:\n Boolean representing whether this blob is associated with a gpu\n '''\n return not blob.startswith('_gpu')\n\n\ndef _compute_in_out(ops):\n '''\n Find the input, intermediate and output nodes of a set of operators.\n\n Args:\n ops: List of Caffe2 operators to look through\n\n Returns:\n input_blobs: The input nodes of the set of operators\n inter_blobs: The intermediate nodes of the set of operators\n output_blobs: The output nodes of the set of operators\n '''\n in_blobs = set()\n out_blobs = set()\n\n for op in ops:\n for input_blob in op.input:\n in_blobs.add(input_blob)\n for output_blob in op.output:\n out_blobs.add(output_blob)\n\n input_blobs = list(in_blobs.difference(out_blobs))\n output_blobs = list(out_blobs.difference(in_blobs))\n inter_blobs = {b for b in output_blobs if b.startswith('_')}\n output_blobs = [b for b in output_blobs if b not in inter_blobs]\n\n return input_blobs, inter_blobs, output_blobs\n\n\ndef _filter_ops(ops, filter_fn, perform_filter):\n '''\n Filter unwanted operators based on criteria in 'filter_fn'.\n\n Args:\n ops: List of Caffe2 operators to filter\n filter_fn: Criteria function for whether inputs/outputs in an operator\n should be filtered.\n perform_filter: Boolean passed from _operators_to_graph_def specifying\n whether to filter operators\n\n Returns:\n new_ops: Subset of ops containing a subset of their inputs and outputs.\n '''\n if not perform_filter:\n return ops\n\n new_ops = []\n for op in ops:\n inputs = list(op.input)\n outputs = list(op.output)\n del op.input[:]\n del op.output[:]\n new_inputs = [i for i in inputs if filter_fn(i)]\n new_outputs = [o for o in outputs if filter_fn(o)]\n\n # Only add the op if output is not empty\n if new_outputs:\n op.input.extend(new_inputs)\n op.output.extend(new_outputs)\n new_ops.append(op)\n\n return new_ops\n\n\ndef _operators_to_graph_def(\n shapes,\n ops,\n colon_replacement='$',\n with_ssa=True,\n with_gradient_scope=True,\n blob_name_tracker=None,\n show_simplified=False,\n custom_rename=None\n):\n '''\n Main function to convert set of operators to a graph.\n\n Args:\n shapes: Dictionary mapping blob names to their shapes/dimensions.\n ops: List of Caffe2 operators, representing some computation graph\n ### **kwargs (model_to_graph_def, nets_to_graph_def, protos_to_graph_def) ###\n colon_replacement: Symbol to replace ':' with. ':i' in TF has a special\n meaning, so we need to replace it with a non-conflicting symbol.\n with_ssa: Boolean\n with_gradient_scope: Boolean\n blob_name_tracker: Dictionary tracking names of blobs (inputs/outputs\n from operators)\n show_simplified: Whether to show a simplified version of the model graph\n Sets all of the following values:\n clear_debug_info: Boolean representing whether to silence debug\n info (which can be very verbose)\n show_forward_only: Boolean representing whether to only show\n blobs involved in the forward pass\n show_cpu_only: Boolean representing whether to only show blobs\n that are not associated with a gpu\n use_tensorflow_naming: Boolean representing whether to convert\n some common Caffe2 naming conventions to their Tensorflow\n counterparts\n custom_rename: Function string -> string that defines a custom\n renaming function to use.\n\n Returns:\n current_graph: GraphDef representing the computation graph formed by the\n set of operators.\n '''\n if blob_name_tracker is not None:\n blob_name_tracker.clear()\n else:\n blob_name_tracker = {}\n\n blob_name_tracker.update(_get_blob_names(ops))\n\n _clear_debug_info(ops, show_simplified) # clear_debug_info\n ops = _filter_ops(ops, _check_if_forward,\n show_simplified) # show_forward_only\n ops = _filter_ops(ops, _check_if_cpu, show_simplified) # show_cpu_only\n if custom_rename:\n _rename_all(shapes, blob_name_tracker, ops, custom_rename)\n if colon_replacement:\n _replace_colons(shapes, blob_name_tracker, ops, colon_replacement)\n if with_ssa:\n _convert_to_ssa(shapes, blob_name_tracker, ops)\n if with_gradient_scope:\n _add_gradient_scope(shapes, blob_name_tracker, ops)\n _fill_missing_operator_names(ops)\n if show_simplified: # use_tensorflow_naming\n _rename_tensorflow_style(shapes, blob_name_tracker, ops)\n producing_ops = {}\n blobs = []\n input_blobs, inter_blobs, _ = _compute_in_out(ops)\n current_graph = GraphDef()\n seen = set(input_blobs)\n for op in ops:\n nodes_from_op = _operator_to_node_simp(op, inter_blobs, seen) if \\\n show_simplified else \\\n [_operator_to_node(shapes, op)] # .extend() expects an iterable\n current_graph.node.extend(nodes_from_op)\n for input_blob in op.input:\n blobs.append(input_blob)\n for i, output_blob in enumerate(op.output):\n blobs.append(output_blob)\n producing_ops.setdefault(output_blob, []).append((op, i))\n\n if show_simplified:\n # Show a cleaner, easier-to-interpret version of the model graph\n blobs = input_blobs\n\n for blob in blobs:\n current_graph.node.extend([_blob_to_node(producing_ops, {}, blob)])\n\n return current_graph\n\n\ndef _propagate_device_option(net_def):\n '''\n Propagate the device options from net to operators.\n\n Args:\n net_def: A caffe2_pb2.NetDef representing a computation graph. The graph\n consists of Caffe2 operators.\n\n Returns:\n None. Iterates through all ops contained within the net. For each op,\n modifies the op device_option in-place to be the net device_option\n if the op has no pre-existing device_option, and leaves the op as-is\n if it already has a device_option.\n '''\n if not net_def.HasField(\"device_option\"):\n return\n for op in net_def.op:\n if not op.HasField(\"device_option\"):\n op.device_option.CopyFrom(net_def.device_option)\n\n\ndef _try_get_shapes(nets):\n '''\n Get missing shapes for all blobs contained in the nets.\n\n Args:\n nets: List of core.Net to extract blob shape information from.\n\n Returns:\n Dictionary containing blob name to shape/dimensions mapping. The net\n is a computation graph that is composed of operators, and the\n operators have input and output blobs, each with their own dims.\n '''\n try:\n # Note: this will inspect the workspace for better or worse.\n # We don't care about the types, only the shapes\n shapes, _ = workspace.InferShapesAndTypes(nets)\n return shapes\n except Exception as e:\n logging.warning('Failed to compute shapes: %s', e)\n return {}\n\n\ndef model_to_graph_def(model, **kwargs):\n '''\n Convert a Caffe2 model to a Tensorflow graph. This function extracts\n 'param_init_net' and 'net' from the model and passes it to nets_to_graph()\n for further processing.\n\n Args:\n model (cnn.CNNModelHelper, model_helper.ModelHelper): The model to\n extract the nets (instances of core.Net) from.\n\n Returns:\n Call to nets_to_graph_def() with extracted 'param_init_net', 'net' and\n **kwargs. See _operators_to_graph_def for detailed **kwargs.\n '''\n nets = [model.param_init_net, model.net]\n return nets_to_graph_def(nets, **kwargs)\n\n\ndef nets_to_graph_def(nets, shapes=None, **kwargs):\n '''\n Convert a set of Caffe2 nets to a Tensorflow graph.\n\n Args:\n nets: List of core.Nets. core.Net is a wrapper around a NetDef protobuf.\n The corresponding protobuf can be extracted using .Proto().\n shapes: Dictionary mapping blob names to their shapes/dimensions.\n\n Returns:\n Call to protos_to_graph_def() with the extracted NetDef protobufs and\n **kwargs. See _operators_to_graph_def for detailed **kwargs.\n '''\n # if shapes is None:\n # shapes = _try_get_shapes(nets)\n # _try_get_shapes(nets) depends on workspace.InferShapesAndTypes(nets),\n # which is currently broken (segfault). We omit the shapes for now.\n shapes = {}\n nets = [copy.deepcopy(net.Proto()) for net in nets]\n shapes = copy.deepcopy(shapes)\n return protos_to_graph_def(nets, shapes, **kwargs)\n\n\ndef protos_to_graph_def(net_defs, shapes=None, **kwargs):\n '''\n Convert a set of Caffe2 net definitions to a Tensorflow graph.\n\n Args:\n net_defs: List of caffe2_pb2.NetDef protobufs representing computation\n graphs.\n shapes: Dictionary mapping blob names to their shapes/dimensions.\n\n Returns:\n Call to _operators_to_graph_def() with the extracted operators from the\n NetDefs and **kwargs. See _operators_to_graph_def for detailed\n **kwargs.\n '''\n for net in net_defs:\n _propagate_device_option(net)\n shapes = copy.deepcopy(shapes or {})\n ops = [op for net_def in net_defs for op in net_def.op]\n return _operators_to_graph_def(shapes, ops, **kwargs)\n" }, { "path": "tensorboardX/tensorboardX/crc32c.py", "content": "# https://www.ietf.org/rfc/rfc3309.txt\nimport array\nimport os\n\ntry:\n if os.environ.get('CRC32C_SW_MODE', None) is None:\n os.environ['CRC32C_SW_MODE'] = 'auto'\n from crc32c import crc32 as _crc32c_native\nexcept ImportError:\n _crc32c_native = None\n\n\nCRC_TABLE = (\n 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4,\n 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb,\n 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b,\n 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24,\n 0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b,\n 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384,\n 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54,\n 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b,\n 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a,\n 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35,\n 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5,\n 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa,\n 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45,\n 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a,\n 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a,\n 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595,\n 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48,\n 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957,\n 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687,\n 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198,\n 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927,\n 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38,\n 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8,\n 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7,\n 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096,\n 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789,\n 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859,\n 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46,\n 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9,\n 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6,\n 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36,\n 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829,\n 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c,\n 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93,\n 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043,\n 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c,\n 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3,\n 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc,\n 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c,\n 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033,\n 0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652,\n 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d,\n 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d,\n 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982,\n 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d,\n 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622,\n 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2,\n 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed,\n 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530,\n 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f,\n 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff,\n 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0,\n 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f,\n 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540,\n 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90,\n 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f,\n 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee,\n 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1,\n 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321,\n 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e,\n 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81,\n 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e,\n 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e,\n 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351,\n)\n\nCRC_INIT = 0\n\n_MASK = 0xFFFFFFFF\n\n\ndef crc_update(crc, data):\n \"\"\"Update CRC-32C checksum with data.\n\n Args:\n crc: 32-bit checksum to update as long.\n data: byte array, string or iterable over bytes.\n\n Returns:\n 32-bit updated CRC-32C as long.\n \"\"\"\n\n if type(data) != array.array or data.itemsize != 1:\n buf = array.array(\"B\", data)\n else:\n buf = data\n\n crc ^= _MASK\n for b in buf:\n table_index = (crc ^ b) & 0xff\n crc = (CRC_TABLE[table_index] ^ (crc >> 8)) & _MASK\n return crc ^ _MASK\n\n\ndef crc_finalize(crc):\n \"\"\"Finalize CRC-32C checksum.\n\n This function should be called as last step of crc calculation.\n\n Args:\n crc: 32-bit checksum as long.\n\n Returns:\n finalized 32-bit checksum as long\n \"\"\"\n return crc & _MASK\n\n\ndef _crc32c(data):\n \"\"\"Compute CRC-32C checksum of the data.\n\n Args:\n data: byte array, string or iterable over bytes.\n\n Returns:\n 32-bit CRC-32C checksum of data as long.\n \"\"\"\n return crc_finalize(crc_update(CRC_INIT, data))\n\n\ncrc32c = _crc32c if _crc32c_native is None else _crc32c_native\n" }, { "path": "tensorboardX/tensorboardX/embedding.py", "content": "import os\n\n\ndef make_tsv(metadata, save_path, metadata_header=None):\n if not metadata_header:\n metadata = [str(x) for x in metadata]\n else:\n assert len(metadata_header) == len(metadata[0]), \\\n 'len of header must be equal to the number of columns in metadata'\n metadata = ['\\t'.join(str(e) for e in l)\n for l in [metadata_header] + metadata]\n import sys\n if sys.version_info[0] == 3:\n with open(os.path.join(save_path, 'metadata.tsv'), 'w', encoding='utf8') as f:\n for x in metadata:\n f.write(x + '\\n')\n else:\n with open(os.path.join(save_path, 'metadata.tsv'), 'wb') as f:\n for x in metadata:\n f.write((x + '\\n').encode('utf-8'))\n\n\n# https://github.com/tensorflow/tensorboard/issues/44 image label will be squared\ndef make_sprite(label_img, save_path):\n import math\n import numpy as np\n from .x2num import make_np\n from .utils import make_grid\n from PIL import Image\n # this ensures the sprite image has correct dimension as described in\n # https://www.tensorflow.org/get_started/embedding_viz\n # There are some constraints for the sprite image:\n # 1. The sprite image should be square.\n # 2. Each image patch in the sprite image should be square.\n # 2. The content is row major order, so we can padding the image on the\n # bottom, but not on the right, otherwise, TB will treat some padded location\n # as images to be shown.\n # args: label_img: tensor in NCHW\n\n assert label_img.shape[2] == label_img.shape[3], 'Image should be square, see tensorflow/tensorboard#670'\n total_pixels = label_img.shape[0] * label_img.shape[2] * label_img.shape[3]\n pixels_one_side = total_pixels ** 0.5\n number_of_images_per_row = int(math.ceil(pixels_one_side / label_img.shape[3]))\n arranged_img_CHW = make_grid(make_np(label_img), ncols=number_of_images_per_row)\n arranged_img_HWC = arranged_img_CHW.transpose(1, 2, 0) # chw -> hwc\n\n arranged_augment_square_HWC = np.ndarray((arranged_img_CHW.shape[2], arranged_img_CHW.shape[2], 3))\n arranged_augment_square_HWC[:arranged_img_HWC.shape[0], :, :] = arranged_img_HWC\n im = Image.fromarray(np.uint8((arranged_augment_square_HWC * 255).clip(0, 255)))\n im.save(os.path.join(save_path, 'sprite.png'))\n\n\ndef append_pbtxt(metadata, label_img, save_path, subdir, global_step, tag):\n from posixpath import join\n with open(os.path.join(save_path, 'projector_config.pbtxt'), 'a') as f:\n # step = os.path.split(save_path)[-1]\n f.write('embeddings {\\n')\n f.write('tensor_name: \"{}:{}\"\\n'.format(\n tag, str(global_step).zfill(5)))\n f.write('tensor_path: \"{}\"\\n'.format(join(subdir, 'tensors.tsv')))\n if metadata is not None:\n f.write('metadata_path: \"{}\"\\n'.format(\n join(subdir, 'metadata.tsv')))\n if label_img is not None:\n f.write('sprite {\\n')\n f.write('image_path: \"{}\"\\n'.format(join(subdir, 'sprite.png')))\n f.write('single_image_dim: {}\\n'.format(label_img.shape[3]))\n f.write('single_image_dim: {}\\n'.format(label_img.shape[2]))\n f.write('}\\n')\n f.write('}\\n')\n\n\ndef make_mat(matlist, save_path):\n with open(os.path.join(save_path, 'tensors.tsv'), 'w') as f:\n for x in matlist:\n x = [str(i.item()) for i in x]\n f.write('\\t'.join(x) + '\\n')\n" }, { "path": "tensorboardX/tensorboardX/event_file_writer.py", "content": "# Copyright 2015 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ==============================================================================\n\"\"\"Writes events to disk in a logdir.\"\"\"\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport os\nimport socket\nimport threading\nimport time\n\nimport six\n\nfrom .proto import event_pb2\nfrom .record_writer import RecordWriter, directory_check\n\n\nclass EventsWriter(object):\n '''Writes `Event` protocol buffers to an event file.'''\n\n def __init__(self, file_prefix, filename_suffix=''):\n '''\n Events files have a name of the form\n '/some/file/path/events.out.tfevents.[timestamp].[hostname]'\n '''\n self._file_name = file_prefix + \".out.tfevents.\" + str(time.time())[:10] + \".\" +\\\n socket.gethostname() + filename_suffix\n self._num_outstanding_events = 0\n self._py_recordio_writer = RecordWriter(self._file_name)\n # Initialize an event instance.\n self._event = event_pb2.Event()\n self._event.wall_time = time.time()\n self._event.file_version = 'brain.Event:2'\n self._lock = threading.Lock()\n self.write_event(self._event)\n\n def write_event(self, event):\n '''Append \"event\" to the file.'''\n\n # Check if event is of type event_pb2.Event proto.\n if not isinstance(event, event_pb2.Event):\n raise TypeError(\"Expected an event_pb2.Event proto, \"\n \" but got %s\" % type(event))\n return self._write_serialized_event(event.SerializeToString())\n\n def _write_serialized_event(self, event_str):\n with self._lock:\n self._num_outstanding_events += 1\n self._py_recordio_writer.write(event_str)\n\n def flush(self):\n '''Flushes the event file to disk.'''\n with self._lock:\n self._num_outstanding_events = 0\n self._py_recordio_writer.flush()\n return True\n\n def close(self):\n '''Call self.flush().'''\n return_value = self.flush()\n with self._lock:\n self._py_recordio_writer.close()\n return return_value\n\n\nclass EventFileWriter(object):\n \"\"\"Writes `Event` protocol buffers to an event file.\n\n The `EventFileWriter` class creates an event file in the specified directory,\n and asynchronously writes Event protocol buffers to the file. The Event file\n is encoded using the tfrecord format, which is similar to RecordIO.\n \"\"\"\n\n def __init__(self, logdir, max_queue_size=10, flush_secs=120, filename_suffix=''):\n \"\"\"Creates a `EventFileWriter` and an event file to write to.\n\n On construction the summary writer creates a new event file in `logdir`.\n This event file will contain `Event` protocol buffers, which are written to\n disk via the add_event method.\n The other arguments to the constructor control the asynchronous writes to\n the event file:\n\n Args:\n logdir: A string. Directory where event file will be written.\n max_queue_size: Integer. Size of the queue for pending events and summaries.\n flush_secs: Number. How often, in seconds, to flush the\n pending events and summaries to disk.\n \"\"\"\n self._logdir = logdir\n directory_check(self._logdir)\n self._event_queue = six.moves.queue.Queue(max_queue_size)\n self._ev_writer = EventsWriter(os.path.join(\n self._logdir, \"events\"), filename_suffix)\n self._flush_secs = flush_secs\n self._closed = False\n self._worker = _EventLoggerThread(self._event_queue, self._ev_writer,\n flush_secs)\n\n self._worker.start()\n\n def get_logdir(self):\n \"\"\"Returns the directory where event file will be written.\"\"\"\n return self._logdir\n\n def reopen(self):\n \"\"\"Reopens the EventFileWriter.\n Can be called after `close()` to add more events in the same directory.\n The events will go into a new events file and a new write/flush worker\n is created. Does nothing if the EventFileWriter was not closed.\n \"\"\"\n if self._closed:\n self._closed = False\n self._worker = _EventLoggerThread(\n self._event_queue, self._ev_writer, self._flush_secs\n )\n self._worker.start()\n\n def add_event(self, event):\n \"\"\"Adds an event to the event file.\n\n Args:\n event: An `Event` protocol buffer.\n \"\"\"\n if not self._closed:\n self._event_queue.put(event)\n\n def flush(self):\n \"\"\"Flushes the event file to disk.\n\n Call this method to make sure that all pending events have been written to\n disk.\n \"\"\"\n if not self._closed:\n self._event_queue.join()\n self._ev_writer.flush()\n\n def close(self):\n \"\"\"Performs a final flush of the event file to disk, stops the\n write/flush worker and closes the file. Call this method when you do not\n need the summary writer anymore.\n \"\"\"\n if not self._closed:\n self.flush()\n self._worker.stop()\n self._ev_writer.close()\n self._closed = True\n\n\nclass _EventLoggerThread(threading.Thread):\n \"\"\"Thread that logs events.\"\"\"\n\n def __init__(self, queue, record_writer, flush_secs):\n \"\"\"Creates an _EventLoggerThread.\n Args:\n queue: A Queue from which to dequeue data.\n record_writer: An data writer. Used to log brain events for\n the visualizer.\n flush_secs: How often, in seconds, to flush the\n pending file to disk.\n \"\"\"\n threading.Thread.__init__(self)\n self.daemon = True\n self._queue = queue\n self._record_writer = record_writer\n self._flush_secs = flush_secs\n # The first data will be flushed immediately.\n self._next_flush_time = 0\n self._has_pending_data = False\n self._shutdown_signal = object()\n\n def stop(self):\n self._queue.put(self._shutdown_signal)\n self.join()\n\n def run(self):\n # Here wait on the queue until an data appears, or till the next\n # time to flush the writer, whichever is earlier. If we have an\n # data, write it. If not, an empty queue exception will be raised\n # and we can proceed to flush the writer.\n while True:\n now = time.time()\n queue_wait_duration = self._next_flush_time - now\n data = None\n try:\n if queue_wait_duration > 0:\n data = self._queue.get(True, queue_wait_duration)\n else:\n data = self._queue.get(False)\n\n if data == self._shutdown_signal:\n return\n self._record_writer.write_event(data)\n self._has_pending_data = True\n except six.moves.queue.Empty:\n pass\n finally:\n if data:\n self._queue.task_done()\n\n now = time.time()\n if now > self._next_flush_time:\n if self._has_pending_data:\n # Small optimization - if there are no pending data,\n # there's no need to flush, since each flush can be\n # expensive (e.g. uploading a new file to a server).\n self._record_writer.flush()\n self._has_pending_data = False\n # Do it again in flush_secs.\n self._next_flush_time = now + self._flush_secs\n" }, { "path": "tensorboardX/tensorboardX/onnx_graph.py", "content": "from .proto.graph_pb2 import GraphDef\nfrom .proto.node_def_pb2 import NodeDef\nfrom .proto.versions_pb2 import VersionDef\nfrom .proto.attr_value_pb2 import AttrValue\nfrom .proto.tensor_shape_pb2 import TensorShapeProto\n\n\ndef load_onnx_graph(fname):\n import onnx\n m = onnx.load(fname)\n g = m.graph\n return parse(g)\n\n\ndef parse(graph):\n nodes_proto = []\n nodes = []\n import itertools\n for node in itertools.chain(graph.input, graph.output):\n nodes_proto.append(node)\n\n for node in nodes_proto:\n print(node.name)\n shapeproto = TensorShapeProto(\n dim=[TensorShapeProto.Dim(size=d.dim_value) for d in node.type.tensor_type.shape.dim])\n nodes.append(NodeDef(\n name=node.name.encode(encoding='utf_8'),\n op='Variable',\n input=[],\n attr={\n 'dtype': AttrValue(type=node.type.tensor_type.elem_type),\n 'shape': AttrValue(shape=shapeproto),\n })\n )\n\n for node in graph.node:\n attr = []\n for s in node.attribute:\n attr.append(' = '.join([str(f[1]) for f in s.ListFields()]))\n attr = ', '.join(attr).encode(encoding='utf_8')\n print(node.output[0])\n nodes.append(NodeDef(\n name=node.output[0].encode(encoding='utf_8'),\n op=node.op_type,\n input=node.input,\n attr={'parameters': AttrValue(s=attr)},\n ))\n\n # two pass token replacement, appends opname to object id\n mapping = {}\n for node in nodes:\n mapping[node.name] = node.op + '_' + node.name\n\n return GraphDef(node=nodes, versions=VersionDef(producer=22))\n" }, { "path": "tensorboardX/tensorboardX/proto/__init__.py", "content": "" }, { "path": "tensorboardX/tensorboardX/proto/api.proto", "content": "/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n==============================================================================*/\n\n// Defines a proto3-based REST API that the HParams web-component of the plugin\n// would use to read data from a hyperparameter-tuning experiment.\n// This file defines the message types (resources) used\n// to pass information into and out of the API methods. These messages will be\n// transmitted using proto3 native JSON encoding. See http_api.md for a\n// description of the actual HTTP API.\n\n// General note: in what follows we use the field 'name' of a message to\n// stores its id. We avoid calling this field 'id' since it is a reserved word\n// in Python, as well as to be more compliant with the API style guide\n// detailed in https://cloud.google.com/apis/design/.\n\n// IMPORTANT: If you change any of the messages here, make sure to also update\n// api.d.ts accordingly.\n\nsyntax = \"proto3\";\n\nimport \"google/protobuf/struct.proto\";\n\npackage tensorboardX.hparam;\n\n// Represents a single experiment.\n// An experiment consists of multiple \"sessions\". Typically, in each session\n// a model is trained for a given set of hyperparameter values. In each session\n// a training program may generate one or more series of real numbers--each\n// containing the evaluation of some metric on the model at different training\n// steps.\n//\n// Note that Sessions can consist of multiple Tensorboard \"runs\", since in\n// a distributed Tensorflow deployment, training can be accomplished using\n// several cooporating processes, each one emitting Summary data to a different\n// log directory or run. For example, in a single session one process could\n// periodically compute the loss on the validation set, and another could\n// compute the loss on the training set.\n// NEXT_TAG: 7\nmessage Experiment {\n // -- Experiments are scoped by a global name.\n // Currently, Tensorboard supports displaying data for a single experiment.\n string name = 6;\n\n // A description. May contain markdown.\n string description = 1;\n\n // An id for the owning user or group.\n string user = 2;\n\n // The time the experiment was created. In seconds since the UNIX epoch.\n double time_created_secs = 3;\n\n // Information about each hyperparameter used in the experiment.\n repeated HParamInfo hparam_infos = 4;\n\n // Information about each metric used in the experiment.\n repeated MetricInfo metric_infos = 5;\n}\n\n// NEXT_TAG: 7\nmessage HParamInfo {\n // An id for the hyperparameter.\n string name = 1;\n\n // A string used to display the hyperparameter in the UI. If empty, the UI\n // will display the 'name' field.\n string display_name = 2;\n\n // A description. May contain markdown.\n string description = 3;\n\n // The data type of this hyperparameter.\n DataType type = 4;\n\n // Specifies the set of values this hyperparameter can hold. The UI assumes\n // every instance of this hyperparameter will hold a value from this set. It\n // is used by the UI to allow filtering so that only session groups (see\n // below) whose associated hyperparameter value \"passes\" the filter are\n // displayed. If this is not populated, the domain is assumed to be the\n // entire domain of the type of the hyperparameter.\n oneof domain {\n // A discrete set of the values this hyperparameter can hold.\n google.protobuf.ListValue domain_discrete = 5;\n // Numeric data type only. The (real) interval from which values of this\n // hyperparameter are taken.\n Interval domain_interval = 6;\n }\n}\n\nenum DataType {\n DATA_TYPE_UNSET = 0;\n DATA_TYPE_STRING = 1;\n DATA_TYPE_BOOL = 2;\n DATA_TYPE_FLOAT64 = 3;\n}\n\n// Represents the closed interval [min_value, max_value] of the real line.\n// NEXT_TAG: 3\nmessage Interval {\n double min_value = 1;\n double max_value = 2;\n}\n\n// NEXT_TAG: 3\nmessage MetricName {\n // An identifier for a metric. A metric is a real-valued function of the\n // model. The UI can plot metrics for a session evaluated on the model at\n // different training steps.\n //\n // We identify a metric by a (group, tag) pair of strings. The UI treats\n // both of these as opaque strings. The only requirement is that the pair\n // uniquely identifies a metric in the experiment.\n //\n // We use a pair so the UI could allow the user to group metrics for a\n // single session by either group or tag to be displayed in the same chart.\n // For instance, one can set the metric group to correspond to the dataset\n // on which the model was evaluated, and the UI can then display different\n // metrics describing the same underlying computation and using different\n // datasets, on the same chart.\n //\n // When exporting summaries from Tensorflow, in a typical setup, a\n // training session exports evaluations of metrics at different training steps\n // as Scalar-plugin summaries--each having a run of the form\n // \"/\", and some associated tag. The same\n // metric for different sessions would use the same sub_dir and tag, but\n // would have a different session_base_log_dir. For example, a session\n // computing two metrics: model loss on the validation set and model loss on\n // the training set, can export these as scalar summaries with the tag \"loss\"\n // and runs session_base_log_dir/validation and session_base_log_dir/training,\n // respectively. In this setup, the 'group' field can be populated with\n // the \"sub_dir\" associated with the metric, and the 'tag' field can be\n // populated with the tag: \"loss\".\n string group = 1;\n string tag = 2;\n}\n\n// NEXT_TAG: 6\nmessage MetricInfo {\n MetricName name = 1;\n\n // A string used to display the metric in the UI. If empty, the UI\n // will display the 'name' field.\n string display_name = 3;\n\n // A description. May contain markdown.\n string description = 4;\n\n // The dataset type (validation, training) on which the metric is computed.\n DatasetType dataset_type = 5;\n}\n\nenum DatasetType {\n DATASET_UNKNOWN = 0;\n DATASET_TRAINING = 1;\n DATASET_VALIDATION = 2;\n}\n\n// In some experiments, the user trains a model with the same set of\n// hyperparameters multiple times to get the distribution of metric\n// evaluations, when the computation (such as the training algorithm, or metric\n// evaluation) is non-deterministic. To make the UI aware of this, sessions\n// are partitioned into groups: each group consists of all training sessions\n// which share the same values for the hyperparameters. In experiments with no\n// repeated executions, each group consists of exactly one session.\n// NEXT_TAG: 6\nmessage SessionGroup {\n string name = 1;\n\n // Stores the hyperparameters for sessions within this group as a mapping\n // from the hyperparameter name to its value.\n map hparams = 2;\n\n // A list of pairs (metric, value)--one for each metric in the experiment.\n // The value denotes the evaluation of the corresponding metric on\n // the model aggregated across the sessions in this group. The exact method\n // of aggregation is specified in the comments of ListSessionGroupsRequest.\n // Unfortunately, we can't store these as a map, since proto maps can't have\n // message keys.\n repeated MetricValue metric_values = 3;\n\n // The sessions belonging to this group.\n repeated Session sessions = 4;\n\n // An optional link to a web page monitoring the session group.\n string monitor_url = 5;\n}\n\n// NEXT_TAG: 5\nmessage MetricValue {\n MetricName name = 1;\n\n double value = 2;\n\n // The training step at which this value is computed.\n int32 training_step = 3;\n\n // The wall time in seconds since UNIX epoch at which this value is computed.\n double wall_time_secs = 4;\n}\n\n// NEXT_TAG: 8\nmessage Session {\n // An id for the session. Unique within an experiment (not just the group).\n string name = 1;\n\n // In seconds since the UNIX epoch.\n double start_time_secs = 2;\n\n // In seconds since the UNIX epoch.\n // May be 0 if unavailable or the session has not finished yet.\n double end_time_secs = 3;\n\n // May be STATUS_UNKNOWN if unavailable.\n Status status = 4;\n\n // A URI for a resource that will allow the user to reconstruct the model for\n // this session. E.g., in Tensorflow this could point to a directory where the\n // checkpoints are stored. Currently, this is treated opaquely by the UI\n // and only displayed to the user as it is passed here.\n string model_uri = 5;\n\n // Stores each metric evaluation on the model at the current training step.\n // Unfortunately, we can't store these as a map, since proto maps can't have\n // message keys.\n repeated MetricValue metric_values = 6;\n\n // An optional link to a web page monitoring the session.\n string monitor_url = 7;\n}\n\n// Represents the status of a Session.\nenum Status {\n STATUS_UNKNOWN = 0;\n STATUS_SUCCESS = 1;\n STATUS_FAILURE = 2;\n STATUS_RUNNING = 3;\n}\n\n// Parameters for a GetExperiment API call.\n// Each experiment is scoped by a unique global id.\n// NEXT_TAG: 2\nmessage GetExperimentRequest {\n // REQUIRED\n string experiment_name = 1;\n}\n\n// Parameters for a ListSessionGroups API call.\n// Computes a list of the current session groups allowing for filtering and\n// sorting by metrics and hyperparameter values. Returns a \"slice\" of\n// that list specified by start_index and slice_size.\n// NEXT_TAG: 8\nmessage ListSessionGroupsRequest {\n string experiment_name = 6;\n\n // Filters the set of sessions (from which the session groups are formed) to\n // contain only these sessions whose status is contained in\n // 'allowed_statuses'.\n repeated Status allowed_statuses = 7;\n\n // A list of ColParams messages--one for each \"column\" of a session group. A\n // session group column contains either a metric evaluated at the current\n // reported computation step or a hyperparameter value. In addition to\n // 'regular' values, a column may take on a special 'missing-value' which\n // denotes that the hyperparameter or metric is not available\n // for the session group (for example, if the metric is not used in the\n // group).\n //\n // The ColParams messages in the repeated field below configure filtering and\n // sorting of the resulting collection of session groups. See the comments of\n // the fields in the ColParam message below for more details.\n repeated ColParams col_params = 1;\n\n // Fields controlling how to aggregate metrics across sessions within a\n // session group.\n // If aggregation_type is AGGREGATION_AVG, each metric value of the\n // session group is the average of the values of the metric across the\n // sessions.\n // Otherwise, the session group metric values are taken directly from a\n // \"representative\" session in the group, selected as a session for which\n // primary_metric takes on its minimum, maximum, or median value, as\n // specified by the choice of aggregation_type (for median, if the number of\n // sessions in the group is even, a session with a lower \"middle\" value is\n // chosen as the representative session).\n AggregationType aggregation_type = 2;\n\n // See comment for 'aggregation_type' above.\n MetricName aggregation_metric = 3;\n\n // The next two parameters determine the \"slice\" of the full list of\n // session groups--sorted and filtered by the parameters above--to return.\n // The 0-based index of the first session group to return.\n int32 start_index = 4;\n\n // The number of session groups to return starting at the session group\n // indexed by 'start_index'. The actual number of session groups returned\n // is min{slice_size, total_size - start_index}, where\n // total_size is the number of session groups in the full list\n // sorted and filtered by the parameters above (if start_index > total_size\n // no session groups are returned).\n int32 slice_size = 5;\n}\n\n// Defines parmeters for a ListSessionGroupsRequest for a specific column.\n// See the comment for \"ListSessionGroupsRequest\" above for more details.\n// NEXT_TAG: 9\nmessage ColParams {\n oneof name {\n MetricName metric = 1;\n string hparam = 2;\n }\n\n // Sorting.\n // The final order of session groups in the response is defined by the sub\n // collection of ColParams messages (out of the\n // ListSessionGroupsRequest.col_params repeated field) whose 'order' field\n // (below) is not ORDER_UNSPECIFIED. In each of the messages in this\n // sub-collection, the next two fields specify the ordering of the values\n // and missing_values in the associated column of the session group. The\n // order of the ColParams messages themselves within the sub-collection\n // determines the \"significance\" of the associated column as a sorting key:\n // with the first being the primary sorting key, the second being the\n // secondary sorting key, etc.\n // Note: The 'session group name' is added as a least significant sorting\n // key to the keys defined by the user, so the order in the response is always\n // deterministic.\n SortOrder order = 3;\n // This field is ignored if order is ORDER_UNSPECIFIED.\n // Otherwise, if true, missing values are ordered before every other value in\n // the column; if false they are ordered after every other value in the\n // column.\n bool missing_values_first = 4;\n\n // Filtering.\n // The 'filter' oneof specifies a subset of the domain of the values a column\n // may take. Only session groups with each of their column values belonging\n // to this subset are included in the response. If this field is not\n // specified, the subset is taken to be the entire column domain.\n oneof filter {\n // Only valid for string-valued hyperparameter columns. The subset is\n // the set of all strings matching the regular expression stored\n // in 'regexp' as a partial match (use '^$' to have a full\n // match against regexp).\n string filter_regexp = 5;\n\n // Only valid for numeric-valued columns. The subset is the given interval.\n Interval filter_interval = 6;\n\n // Valid for all data types. The subset is defined explicitly.\n google.protobuf.ListValue filter_discrete = 7;\n }\n // Specifies whether to exclude session groups whose column value is missing\n // from the response.\n bool exclude_missing_values = 8;\n}\n\nenum SortOrder {\n ORDER_UNSPECIFIED = 0;\n ORDER_ASC = 1;\n ORDER_DESC = 2;\n}\n\nenum AggregationType {\n AGGREGATION_UNSET = 0;\n AGGREGATION_AVG = 1;\n AGGREGATION_MEDIAN = 2;\n AGGREGATION_MIN = 3;\n AGGREGATION_MAX = 4;\n}\n\n// See ListSessionGroups in http_api.md.\n// NEXT_TAG: 4\nmessage ListSessionGroupsResponse {\n repeated SessionGroup session_groups = 1;\n\n // Denotes the total number of session groups in the full filtered list.\n // (Recall that this response may only be a slice).\n // It is used by the UI to calculate total number of pages and can be\n // set here to -1 to mean \"unknown\".\n int32 total_size = 3;\n}\n\n// See ListMetricEvalsRequest in http_api.md.\n// NEXT_TAG: 4\nmessage ListMetricEvalsRequest {\n string experiment_name = 3;\n string session_name = 1;\n MetricName metric_name = 2;\n}\n" }, { "path": "tensorboardX/tensorboardX/proto/api_pb2.py", "content": "# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: tensorboardX/proto/api.proto\n\nimport sys\n_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))\nfrom google.protobuf.internal import enum_type_wrapper\nfrom google.protobuf import descriptor as _descriptor\nfrom google.protobuf import message as _message\nfrom google.protobuf import reflection as _reflection\nfrom google.protobuf import symbol_database as _symbol_database\n# @@protoc_insertion_point(imports)\n\n_sym_db = _symbol_database.Default()\n\n\nfrom google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2\n\n\nDESCRIPTOR = _descriptor.FileDescriptor(\n name='tensorboardX/proto/api.proto',\n package='tensorboardX.hparam',\n syntax='proto3',\n serialized_options=None,\n serialized_pb=_b('\\n\\x1ctensorboardX/proto/api.proto\\x12\\x13tensorboardX.hparam\\x1a\\x1cgoogle/protobuf/struct.proto\\\"\\xc6\\x01\\n\\nExperiment\\x12\\x0c\\n\\x04name\\x18\\x06 \\x01(\\t\\x12\\x13\\n\\x0b\\x64\\x65scription\\x18\\x01 \\x01(\\t\\x12\\x0c\\n\\x04user\\x18\\x02 \\x01(\\t\\x12\\x19\\n\\x11time_created_secs\\x18\\x03 \\x01(\\x01\\x12\\x35\\n\\x0chparam_infos\\x18\\x04 \\x03(\\x0b\\x32\\x1f.tensorboardX.hparam.HParamInfo\\x12\\x35\\n\\x0cmetric_infos\\x18\\x05 \\x03(\\x0b\\x32\\x1f.tensorboardX.hparam.MetricInfo\\\"\\xed\\x01\\n\\nHParamInfo\\x12\\x0c\\n\\x04name\\x18\\x01 \\x01(\\t\\x12\\x14\\n\\x0c\\x64isplay_name\\x18\\x02 \\x01(\\t\\x12\\x13\\n\\x0b\\x64\\x65scription\\x18\\x03 \\x01(\\t\\x12+\\n\\x04type\\x18\\x04 \\x01(\\x0e\\x32\\x1d.tensorboardX.hparam.DataType\\x12\\x35\\n\\x0f\\x64omain_discrete\\x18\\x05 \\x01(\\x0b\\x32\\x1a.google.protobuf.ListValueH\\x00\\x12\\x38\\n\\x0f\\x64omain_interval\\x18\\x06 \\x01(\\x0b\\x32\\x1d.tensorboardX.hparam.IntervalH\\x00\\x42\\x08\\n\\x06\\x64omain\\\"0\\n\\x08Interval\\x12\\x11\\n\\tmin_value\\x18\\x01 \\x01(\\x01\\x12\\x11\\n\\tmax_value\\x18\\x02 \\x01(\\x01\\\"(\\n\\nMetricName\\x12\\r\\n\\x05group\\x18\\x01 \\x01(\\t\\x12\\x0b\\n\\x03tag\\x18\\x02 \\x01(\\t\\\"\\x9e\\x01\\n\\nMetricInfo\\x12-\\n\\x04name\\x18\\x01 \\x01(\\x0b\\x32\\x1f.tensorboardX.hparam.MetricName\\x12\\x14\\n\\x0c\\x64isplay_name\\x18\\x03 \\x01(\\t\\x12\\x13\\n\\x0b\\x64\\x65scription\\x18\\x04 \\x01(\\t\\x12\\x36\\n\\x0c\\x64\\x61taset_type\\x18\\x05 \\x01(\\x0e\\x32 .tensorboardX.hparam.DatasetType\\\"\\xa3\\x02\\n\\x0cSessionGroup\\x12\\x0c\\n\\x04name\\x18\\x01 \\x01(\\t\\x12?\\n\\x07hparams\\x18\\x02 \\x03(\\x0b\\x32..tensorboardX.hparam.SessionGroup.HparamsEntry\\x12\\x37\\n\\rmetric_values\\x18\\x03 \\x03(\\x0b\\x32 .tensorboardX.hparam.MetricValue\\x12.\\n\\x08sessions\\x18\\x04 \\x03(\\x0b\\x32\\x1c.tensorboardX.hparam.Session\\x12\\x13\\n\\x0bmonitor_url\\x18\\x05 \\x01(\\t\\x1a\\x46\\n\\x0cHparamsEntry\\x12\\x0b\\n\\x03key\\x18\\x01 \\x01(\\t\\x12%\\n\\x05value\\x18\\x02 \\x01(\\x0b\\x32\\x16.google.protobuf.Value:\\x02\\x38\\x01\\\"z\\n\\x0bMetricValue\\x12-\\n\\x04name\\x18\\x01 \\x01(\\x0b\\x32\\x1f.tensorboardX.hparam.MetricName\\x12\\r\\n\\x05value\\x18\\x02 \\x01(\\x01\\x12\\x15\\n\\rtraining_step\\x18\\x03 \\x01(\\x05\\x12\\x16\\n\\x0ewall_time_secs\\x18\\x04 \\x01(\\x01\\\"\\xd5\\x01\\n\\x07Session\\x12\\x0c\\n\\x04name\\x18\\x01 \\x01(\\t\\x12\\x17\\n\\x0fstart_time_secs\\x18\\x02 \\x01(\\x01\\x12\\x15\\n\\rend_time_secs\\x18\\x03 \\x01(\\x01\\x12+\\n\\x06status\\x18\\x04 \\x01(\\x0e\\x32\\x1b.tensorboardX.hparam.Status\\x12\\x11\\n\\tmodel_uri\\x18\\x05 \\x01(\\t\\x12\\x37\\n\\rmetric_values\\x18\\x06 \\x03(\\x0b\\x32 .tensorboardX.hparam.MetricValue\\x12\\x13\\n\\x0bmonitor_url\\x18\\x07 \\x01(\\t\\\"/\\n\\x14GetExperimentRequest\\x12\\x17\\n\\x0f\\x65xperiment_name\\x18\\x01 \\x01(\\t\\\"\\xc4\\x02\\n\\x18ListSessionGroupsRequest\\x12\\x17\\n\\x0f\\x65xperiment_name\\x18\\x06 \\x01(\\t\\x12\\x35\\n\\x10\\x61llowed_statuses\\x18\\x07 \\x03(\\x0e\\x32\\x1b.tensorboardX.hparam.Status\\x12\\x32\\n\\ncol_params\\x18\\x01 \\x03(\\x0b\\x32\\x1e.tensorboardX.hparam.ColParams\\x12>\\n\\x10\\x61ggregation_type\\x18\\x02 \\x01(\\x0e\\x32$.tensorboardX.hparam.AggregationType\\x12;\\n\\x12\\x61ggregation_metric\\x18\\x03 \\x01(\\x0b\\x32\\x1f.tensorboardX.hparam.MetricName\\x12\\x13\\n\\x0bstart_index\\x18\\x04 \\x01(\\x05\\x12\\x12\\n\\nslice_size\\x18\\x05 \\x01(\\x05\\\"\\xd9\\x02\\n\\tColParams\\x12\\x31\\n\\x06metric\\x18\\x01 \\x01(\\x0b\\x32\\x1f.tensorboardX.hparam.MetricNameH\\x00\\x12\\x10\\n\\x06hparam\\x18\\x02 \\x01(\\tH\\x00\\x12-\\n\\x05order\\x18\\x03 \\x01(\\x0e\\x32\\x1e.tensorboardX.hparam.SortOrder\\x12\\x1c\\n\\x14missing_values_first\\x18\\x04 \\x01(\\x08\\x12\\x17\\n\\rfilter_regexp\\x18\\x05 \\x01(\\tH\\x01\\x12\\x38\\n\\x0f\\x66ilter_interval\\x18\\x06 \\x01(\\x0b\\x32\\x1d.tensorboardX.hparam.IntervalH\\x01\\x12\\x35\\n\\x0f\\x66ilter_discrete\\x18\\x07 \\x01(\\x0b\\x32\\x1a.google.protobuf.ListValueH\\x01\\x12\\x1e\\n\\x16\\x65xclude_missing_values\\x18\\x08 \\x01(\\x08\\x42\\x06\\n\\x04nameB\\x08\\n\\x06\\x66ilter\\\"j\\n\\x19ListSessionGroupsResponse\\x12\\x39\\n\\x0esession_groups\\x18\\x01 \\x03(\\x0b\\x32!.tensorboardX.hparam.SessionGroup\\x12\\x12\\n\\ntotal_size\\x18\\x03 \\x01(\\x05\\\"}\\n\\x16ListMetricEvalsRequest\\x12\\x17\\n\\x0f\\x65xperiment_name\\x18\\x03 \\x01(\\t\\x12\\x14\\n\\x0csession_name\\x18\\x01 \\x01(\\t\\x12\\x34\\n\\x0bmetric_name\\x18\\x02 \\x01(\\x0b\\x32\\x1f.tensorboardX.hparam.MetricName*`\\n\\x08\\x44\\x61taType\\x12\\x13\\n\\x0f\\x44\\x41TA_TYPE_UNSET\\x10\\x00\\x12\\x14\\n\\x10\\x44\\x41TA_TYPE_STRING\\x10\\x01\\x12\\x12\\n\\x0e\\x44\\x41TA_TYPE_BOOL\\x10\\x02\\x12\\x15\\n\\x11\\x44\\x41TA_TYPE_FLOAT64\\x10\\x03*P\\n\\x0b\\x44\\x61tasetType\\x12\\x13\\n\\x0f\\x44\\x41TASET_UNKNOWN\\x10\\x00\\x12\\x14\\n\\x10\\x44\\x41TASET_TRAINING\\x10\\x01\\x12\\x16\\n\\x12\\x44\\x41TASET_VALIDATION\\x10\\x02*X\\n\\x06Status\\x12\\x12\\n\\x0eSTATUS_UNKNOWN\\x10\\x00\\x12\\x12\\n\\x0eSTATUS_SUCCESS\\x10\\x01\\x12\\x12\\n\\x0eSTATUS_FAILURE\\x10\\x02\\x12\\x12\\n\\x0eSTATUS_RUNNING\\x10\\x03*A\\n\\tSortOrder\\x12\\x15\\n\\x11ORDER_UNSPECIFIED\\x10\\x00\\x12\\r\\n\\tORDER_ASC\\x10\\x01\\x12\\x0e\\n\\nORDER_DESC\\x10\\x02*\\x7f\\n\\x0f\\x41ggregationType\\x12\\x15\\n\\x11\\x41GGREGATION_UNSET\\x10\\x00\\x12\\x13\\n\\x0f\\x41GGREGATION_AVG\\x10\\x01\\x12\\x16\\n\\x12\\x41GGREGATION_MEDIAN\\x10\\x02\\x12\\x13\\n\\x0f\\x41GGREGATION_MIN\\x10\\x03\\x12\\x13\\n\\x0f\\x41GGREGATION_MAX\\x10\\x04\\x62\\x06proto3')\n ,\n dependencies=[google_dot_protobuf_dot_struct__pb2.DESCRIPTOR,])\n\n_DATATYPE = _descriptor.EnumDescriptor(\n name='DataType',\n full_name='tensorboardX.hparam.DataType',\n filename=None,\n file=DESCRIPTOR,\n values=[\n _descriptor.EnumValueDescriptor(\n name='DATA_TYPE_UNSET', index=0, number=0,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DATA_TYPE_STRING', index=1, number=1,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DATA_TYPE_BOOL', index=2, number=2,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DATA_TYPE_FLOAT64', index=3, number=3,\n serialized_options=None,\n type=None),\n ],\n containing_type=None,\n serialized_options=None,\n serialized_start=2370,\n serialized_end=2466,\n)\n_sym_db.RegisterEnumDescriptor(_DATATYPE)\n\nDataType = enum_type_wrapper.EnumTypeWrapper(_DATATYPE)\n_DATASETTYPE = _descriptor.EnumDescriptor(\n name='DatasetType',\n full_name='tensorboardX.hparam.DatasetType',\n filename=None,\n file=DESCRIPTOR,\n values=[\n _descriptor.EnumValueDescriptor(\n name='DATASET_UNKNOWN', index=0, number=0,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DATASET_TRAINING', index=1, number=1,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DATASET_VALIDATION', index=2, number=2,\n serialized_options=None,\n type=None),\n ],\n containing_type=None,\n serialized_options=None,\n serialized_start=2468,\n serialized_end=2548,\n)\n_sym_db.RegisterEnumDescriptor(_DATASETTYPE)\n\nDatasetType = enum_type_wrapper.EnumTypeWrapper(_DATASETTYPE)\n_STATUS = _descriptor.EnumDescriptor(\n name='Status',\n full_name='tensorboardX.hparam.Status',\n filename=None,\n file=DESCRIPTOR,\n values=[\n _descriptor.EnumValueDescriptor(\n name='STATUS_UNKNOWN', index=0, number=0,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='STATUS_SUCCESS', index=1, number=1,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='STATUS_FAILURE', index=2, number=2,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='STATUS_RUNNING', index=3, number=3,\n serialized_options=None,\n type=None),\n ],\n containing_type=None,\n serialized_options=None,\n serialized_start=2550,\n serialized_end=2638,\n)\n_sym_db.RegisterEnumDescriptor(_STATUS)\n\nStatus = enum_type_wrapper.EnumTypeWrapper(_STATUS)\n_SORTORDER = _descriptor.EnumDescriptor(\n name='SortOrder',\n full_name='tensorboardX.hparam.SortOrder',\n filename=None,\n file=DESCRIPTOR,\n values=[\n _descriptor.EnumValueDescriptor(\n name='ORDER_UNSPECIFIED', index=0, number=0,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='ORDER_ASC', index=1, number=1,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='ORDER_DESC', index=2, number=2,\n serialized_options=None,\n type=None),\n ],\n containing_type=None,\n serialized_options=None,\n serialized_start=2640,\n serialized_end=2705,\n)\n_sym_db.RegisterEnumDescriptor(_SORTORDER)\n\nSortOrder = enum_type_wrapper.EnumTypeWrapper(_SORTORDER)\n_AGGREGATIONTYPE = _descriptor.EnumDescriptor(\n name='AggregationType',\n full_name='tensorboardX.hparam.AggregationType',\n filename=None,\n file=DESCRIPTOR,\n values=[\n _descriptor.EnumValueDescriptor(\n name='AGGREGATION_UNSET', index=0, number=0,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='AGGREGATION_AVG', index=1, number=1,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='AGGREGATION_MEDIAN', index=2, number=2,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='AGGREGATION_MIN', index=3, number=3,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='AGGREGATION_MAX', index=4, number=4,\n serialized_options=None,\n type=None),\n ],\n containing_type=None,\n serialized_options=None,\n serialized_start=2707,\n serialized_end=2834,\n)\n_sym_db.RegisterEnumDescriptor(_AGGREGATIONTYPE)\n\nAggregationType = enum_type_wrapper.EnumTypeWrapper(_AGGREGATIONTYPE)\nDATA_TYPE_UNSET = 0\nDATA_TYPE_STRING = 1\nDATA_TYPE_BOOL = 2\nDATA_TYPE_FLOAT64 = 3\nDATASET_UNKNOWN = 0\nDATASET_TRAINING = 1\nDATASET_VALIDATION = 2\nSTATUS_UNKNOWN = 0\nSTATUS_SUCCESS = 1\nSTATUS_FAILURE = 2\nSTATUS_RUNNING = 3\nORDER_UNSPECIFIED = 0\nORDER_ASC = 1\nORDER_DESC = 2\nAGGREGATION_UNSET = 0\nAGGREGATION_AVG = 1\nAGGREGATION_MEDIAN = 2\nAGGREGATION_MIN = 3\nAGGREGATION_MAX = 4\n\n\n\n_EXPERIMENT = _descriptor.Descriptor(\n name='Experiment',\n full_name='tensorboardX.hparam.Experiment',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='name', full_name='tensorboardX.hparam.Experiment.name', index=0,\n number=6, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='description', full_name='tensorboardX.hparam.Experiment.description', index=1,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='user', full_name='tensorboardX.hparam.Experiment.user', index=2,\n number=2, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='time_created_secs', full_name='tensorboardX.hparam.Experiment.time_created_secs', index=3,\n number=3, type=1, cpp_type=5, label=1,\n has_default_value=False, default_value=float(0),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='hparam_infos', full_name='tensorboardX.hparam.Experiment.hparam_infos', index=4,\n number=4, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='metric_infos', full_name='tensorboardX.hparam.Experiment.metric_infos', index=5,\n number=5, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=84,\n serialized_end=282,\n)\n\n\n_HPARAMINFO = _descriptor.Descriptor(\n name='HParamInfo',\n full_name='tensorboardX.hparam.HParamInfo',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='name', full_name='tensorboardX.hparam.HParamInfo.name', index=0,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='display_name', full_name='tensorboardX.hparam.HParamInfo.display_name', index=1,\n number=2, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='description', full_name='tensorboardX.hparam.HParamInfo.description', index=2,\n number=3, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='type', full_name='tensorboardX.hparam.HParamInfo.type', index=3,\n number=4, type=14, cpp_type=8, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='domain_discrete', full_name='tensorboardX.hparam.HParamInfo.domain_discrete', index=4,\n number=5, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='domain_interval', full_name='tensorboardX.hparam.HParamInfo.domain_interval', index=5,\n number=6, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n _descriptor.OneofDescriptor(\n name='domain', full_name='tensorboardX.hparam.HParamInfo.domain',\n index=0, containing_type=None, fields=[]),\n ],\n serialized_start=285,\n serialized_end=522,\n)\n\n\n_INTERVAL = _descriptor.Descriptor(\n name='Interval',\n full_name='tensorboardX.hparam.Interval',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='min_value', full_name='tensorboardX.hparam.Interval.min_value', index=0,\n number=1, type=1, cpp_type=5, label=1,\n has_default_value=False, default_value=float(0),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='max_value', full_name='tensorboardX.hparam.Interval.max_value', index=1,\n number=2, type=1, cpp_type=5, label=1,\n has_default_value=False, default_value=float(0),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=524,\n serialized_end=572,\n)\n\n\n_METRICNAME = _descriptor.Descriptor(\n name='MetricName',\n full_name='tensorboardX.hparam.MetricName',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='group', full_name='tensorboardX.hparam.MetricName.group', index=0,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='tag', full_name='tensorboardX.hparam.MetricName.tag', index=1,\n number=2, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=574,\n serialized_end=614,\n)\n\n\n_METRICINFO = _descriptor.Descriptor(\n name='MetricInfo',\n full_name='tensorboardX.hparam.MetricInfo',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='name', full_name='tensorboardX.hparam.MetricInfo.name', index=0,\n number=1, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='display_name', full_name='tensorboardX.hparam.MetricInfo.display_name', index=1,\n number=3, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='description', full_name='tensorboardX.hparam.MetricInfo.description', index=2,\n number=4, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='dataset_type', full_name='tensorboardX.hparam.MetricInfo.dataset_type', index=3,\n number=5, type=14, cpp_type=8, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=617,\n serialized_end=775,\n)\n\n\n_SESSIONGROUP_HPARAMSENTRY = _descriptor.Descriptor(\n name='HparamsEntry',\n full_name='tensorboardX.hparam.SessionGroup.HparamsEntry',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='key', full_name='tensorboardX.hparam.SessionGroup.HparamsEntry.key', index=0,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='value', full_name='tensorboardX.hparam.SessionGroup.HparamsEntry.value', index=1,\n number=2, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=_b('8\\001'),\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=999,\n serialized_end=1069,\n)\n\n_SESSIONGROUP = _descriptor.Descriptor(\n name='SessionGroup',\n full_name='tensorboardX.hparam.SessionGroup',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='name', full_name='tensorboardX.hparam.SessionGroup.name', index=0,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='hparams', full_name='tensorboardX.hparam.SessionGroup.hparams', index=1,\n number=2, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='metric_values', full_name='tensorboardX.hparam.SessionGroup.metric_values', index=2,\n number=3, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='sessions', full_name='tensorboardX.hparam.SessionGroup.sessions', index=3,\n number=4, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='monitor_url', full_name='tensorboardX.hparam.SessionGroup.monitor_url', index=4,\n number=5, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[_SESSIONGROUP_HPARAMSENTRY, ],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=778,\n serialized_end=1069,\n)\n\n\n_METRICVALUE = _descriptor.Descriptor(\n name='MetricValue',\n full_name='tensorboardX.hparam.MetricValue',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='name', full_name='tensorboardX.hparam.MetricValue.name', index=0,\n number=1, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='value', full_name='tensorboardX.hparam.MetricValue.value', index=1,\n number=2, type=1, cpp_type=5, label=1,\n has_default_value=False, default_value=float(0),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='training_step', full_name='tensorboardX.hparam.MetricValue.training_step', index=2,\n number=3, type=5, cpp_type=1, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='wall_time_secs', full_name='tensorboardX.hparam.MetricValue.wall_time_secs', index=3,\n number=4, type=1, cpp_type=5, label=1,\n has_default_value=False, default_value=float(0),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=1071,\n serialized_end=1193,\n)\n\n\n_SESSION = _descriptor.Descriptor(\n name='Session',\n full_name='tensorboardX.hparam.Session',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='name', full_name='tensorboardX.hparam.Session.name', index=0,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='start_time_secs', full_name='tensorboardX.hparam.Session.start_time_secs', index=1,\n number=2, type=1, cpp_type=5, label=1,\n has_default_value=False, default_value=float(0),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='end_time_secs', full_name='tensorboardX.hparam.Session.end_time_secs', index=2,\n number=3, type=1, cpp_type=5, label=1,\n has_default_value=False, default_value=float(0),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='status', full_name='tensorboardX.hparam.Session.status', index=3,\n number=4, type=14, cpp_type=8, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='model_uri', full_name='tensorboardX.hparam.Session.model_uri', index=4,\n number=5, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='metric_values', full_name='tensorboardX.hparam.Session.metric_values', index=5,\n number=6, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='monitor_url', full_name='tensorboardX.hparam.Session.monitor_url', index=6,\n number=7, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=1196,\n serialized_end=1409,\n)\n\n\n_GETEXPERIMENTREQUEST = _descriptor.Descriptor(\n name='GetExperimentRequest',\n full_name='tensorboardX.hparam.GetExperimentRequest',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='experiment_name', full_name='tensorboardX.hparam.GetExperimentRequest.experiment_name', index=0,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=1411,\n serialized_end=1458,\n)\n\n\n_LISTSESSIONGROUPSREQUEST = _descriptor.Descriptor(\n name='ListSessionGroupsRequest',\n full_name='tensorboardX.hparam.ListSessionGroupsRequest',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='experiment_name', full_name='tensorboardX.hparam.ListSessionGroupsRequest.experiment_name', index=0,\n number=6, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='allowed_statuses', full_name='tensorboardX.hparam.ListSessionGroupsRequest.allowed_statuses', index=1,\n number=7, type=14, cpp_type=8, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='col_params', full_name='tensorboardX.hparam.ListSessionGroupsRequest.col_params', index=2,\n number=1, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='aggregation_type', full_name='tensorboardX.hparam.ListSessionGroupsRequest.aggregation_type', index=3,\n number=2, type=14, cpp_type=8, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='aggregation_metric', full_name='tensorboardX.hparam.ListSessionGroupsRequest.aggregation_metric', index=4,\n number=3, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='start_index', full_name='tensorboardX.hparam.ListSessionGroupsRequest.start_index', index=5,\n number=4, type=5, cpp_type=1, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='slice_size', full_name='tensorboardX.hparam.ListSessionGroupsRequest.slice_size', index=6,\n number=5, type=5, cpp_type=1, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=1461,\n serialized_end=1785,\n)\n\n\n_COLPARAMS = _descriptor.Descriptor(\n name='ColParams',\n full_name='tensorboardX.hparam.ColParams',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='metric', full_name='tensorboardX.hparam.ColParams.metric', index=0,\n number=1, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='hparam', full_name='tensorboardX.hparam.ColParams.hparam', index=1,\n number=2, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='order', full_name='tensorboardX.hparam.ColParams.order', index=2,\n number=3, type=14, cpp_type=8, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='missing_values_first', full_name='tensorboardX.hparam.ColParams.missing_values_first', index=3,\n number=4, type=8, cpp_type=7, label=1,\n has_default_value=False, default_value=False,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='filter_regexp', full_name='tensorboardX.hparam.ColParams.filter_regexp', index=4,\n number=5, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='filter_interval', full_name='tensorboardX.hparam.ColParams.filter_interval', index=5,\n number=6, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='filter_discrete', full_name='tensorboardX.hparam.ColParams.filter_discrete', index=6,\n number=7, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='exclude_missing_values', full_name='tensorboardX.hparam.ColParams.exclude_missing_values', index=7,\n number=8, type=8, cpp_type=7, label=1,\n has_default_value=False, default_value=False,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n _descriptor.OneofDescriptor(\n name='name', full_name='tensorboardX.hparam.ColParams.name',\n index=0, containing_type=None, fields=[]),\n _descriptor.OneofDescriptor(\n name='filter', full_name='tensorboardX.hparam.ColParams.filter',\n index=1, containing_type=None, fields=[]),\n ],\n serialized_start=1788,\n serialized_end=2133,\n)\n\n\n_LISTSESSIONGROUPSRESPONSE = _descriptor.Descriptor(\n name='ListSessionGroupsResponse',\n full_name='tensorboardX.hparam.ListSessionGroupsResponse',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='session_groups', full_name='tensorboardX.hparam.ListSessionGroupsResponse.session_groups', index=0,\n number=1, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='total_size', full_name='tensorboardX.hparam.ListSessionGroupsResponse.total_size', index=1,\n number=3, type=5, cpp_type=1, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=2135,\n serialized_end=2241,\n)\n\n\n_LISTMETRICEVALSREQUEST = _descriptor.Descriptor(\n name='ListMetricEvalsRequest',\n full_name='tensorboardX.hparam.ListMetricEvalsRequest',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='experiment_name', full_name='tensorboardX.hparam.ListMetricEvalsRequest.experiment_name', index=0,\n number=3, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='session_name', full_name='tensorboardX.hparam.ListMetricEvalsRequest.session_name', index=1,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='metric_name', full_name='tensorboardX.hparam.ListMetricEvalsRequest.metric_name', index=2,\n number=2, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=2243,\n serialized_end=2368,\n)\n\n_EXPERIMENT.fields_by_name['hparam_infos'].message_type = _HPARAMINFO\n_EXPERIMENT.fields_by_name['metric_infos'].message_type = _METRICINFO\n_HPARAMINFO.fields_by_name['type'].enum_type = _DATATYPE\n_HPARAMINFO.fields_by_name['domain_discrete'].message_type = google_dot_protobuf_dot_struct__pb2._LISTVALUE\n_HPARAMINFO.fields_by_name['domain_interval'].message_type = _INTERVAL\n_HPARAMINFO.oneofs_by_name['domain'].fields.append(\n _HPARAMINFO.fields_by_name['domain_discrete'])\n_HPARAMINFO.fields_by_name['domain_discrete'].containing_oneof = _HPARAMINFO.oneofs_by_name['domain']\n_HPARAMINFO.oneofs_by_name['domain'].fields.append(\n _HPARAMINFO.fields_by_name['domain_interval'])\n_HPARAMINFO.fields_by_name['domain_interval'].containing_oneof = _HPARAMINFO.oneofs_by_name['domain']\n_METRICINFO.fields_by_name['name'].message_type = _METRICNAME\n_METRICINFO.fields_by_name['dataset_type'].enum_type = _DATASETTYPE\n_SESSIONGROUP_HPARAMSENTRY.fields_by_name['value'].message_type = google_dot_protobuf_dot_struct__pb2._VALUE\n_SESSIONGROUP_HPARAMSENTRY.containing_type = _SESSIONGROUP\n_SESSIONGROUP.fields_by_name['hparams'].message_type = _SESSIONGROUP_HPARAMSENTRY\n_SESSIONGROUP.fields_by_name['metric_values'].message_type = _METRICVALUE\n_SESSIONGROUP.fields_by_name['sessions'].message_type = _SESSION\n_METRICVALUE.fields_by_name['name'].message_type = _METRICNAME\n_SESSION.fields_by_name['status'].enum_type = _STATUS\n_SESSION.fields_by_name['metric_values'].message_type = _METRICVALUE\n_LISTSESSIONGROUPSREQUEST.fields_by_name['allowed_statuses'].enum_type = _STATUS\n_LISTSESSIONGROUPSREQUEST.fields_by_name['col_params'].message_type = _COLPARAMS\n_LISTSESSIONGROUPSREQUEST.fields_by_name['aggregation_type'].enum_type = _AGGREGATIONTYPE\n_LISTSESSIONGROUPSREQUEST.fields_by_name['aggregation_metric'].message_type = _METRICNAME\n_COLPARAMS.fields_by_name['metric'].message_type = _METRICNAME\n_COLPARAMS.fields_by_name['order'].enum_type = _SORTORDER\n_COLPARAMS.fields_by_name['filter_interval'].message_type = _INTERVAL\n_COLPARAMS.fields_by_name['filter_discrete'].message_type = google_dot_protobuf_dot_struct__pb2._LISTVALUE\n_COLPARAMS.oneofs_by_name['name'].fields.append(\n _COLPARAMS.fields_by_name['metric'])\n_COLPARAMS.fields_by_name['metric'].containing_oneof = _COLPARAMS.oneofs_by_name['name']\n_COLPARAMS.oneofs_by_name['name'].fields.append(\n _COLPARAMS.fields_by_name['hparam'])\n_COLPARAMS.fields_by_name['hparam'].containing_oneof = _COLPARAMS.oneofs_by_name['name']\n_COLPARAMS.oneofs_by_name['filter'].fields.append(\n _COLPARAMS.fields_by_name['filter_regexp'])\n_COLPARAMS.fields_by_name['filter_regexp'].containing_oneof = _COLPARAMS.oneofs_by_name['filter']\n_COLPARAMS.oneofs_by_name['filter'].fields.append(\n _COLPARAMS.fields_by_name['filter_interval'])\n_COLPARAMS.fields_by_name['filter_interval'].containing_oneof = _COLPARAMS.oneofs_by_name['filter']\n_COLPARAMS.oneofs_by_name['filter'].fields.append(\n _COLPARAMS.fields_by_name['filter_discrete'])\n_COLPARAMS.fields_by_name['filter_discrete'].containing_oneof = _COLPARAMS.oneofs_by_name['filter']\n_LISTSESSIONGROUPSRESPONSE.fields_by_name['session_groups'].message_type = _SESSIONGROUP\n_LISTMETRICEVALSREQUEST.fields_by_name['metric_name'].message_type = _METRICNAME\nDESCRIPTOR.message_types_by_name['Experiment'] = _EXPERIMENT\nDESCRIPTOR.message_types_by_name['HParamInfo'] = _HPARAMINFO\nDESCRIPTOR.message_types_by_name['Interval'] = _INTERVAL\nDESCRIPTOR.message_types_by_name['MetricName'] = _METRICNAME\nDESCRIPTOR.message_types_by_name['MetricInfo'] = _METRICINFO\nDESCRIPTOR.message_types_by_name['SessionGroup'] = _SESSIONGROUP\nDESCRIPTOR.message_types_by_name['MetricValue'] = _METRICVALUE\nDESCRIPTOR.message_types_by_name['Session'] = _SESSION\nDESCRIPTOR.message_types_by_name['GetExperimentRequest'] = _GETEXPERIMENTREQUEST\nDESCRIPTOR.message_types_by_name['ListSessionGroupsRequest'] = _LISTSESSIONGROUPSREQUEST\nDESCRIPTOR.message_types_by_name['ColParams'] = _COLPARAMS\nDESCRIPTOR.message_types_by_name['ListSessionGroupsResponse'] = _LISTSESSIONGROUPSRESPONSE\nDESCRIPTOR.message_types_by_name['ListMetricEvalsRequest'] = _LISTMETRICEVALSREQUEST\nDESCRIPTOR.enum_types_by_name['DataType'] = _DATATYPE\nDESCRIPTOR.enum_types_by_name['DatasetType'] = _DATASETTYPE\nDESCRIPTOR.enum_types_by_name['Status'] = _STATUS\nDESCRIPTOR.enum_types_by_name['SortOrder'] = _SORTORDER\nDESCRIPTOR.enum_types_by_name['AggregationType'] = _AGGREGATIONTYPE\n_sym_db.RegisterFileDescriptor(DESCRIPTOR)\n\nExperiment = _reflection.GeneratedProtocolMessageType('Experiment', (_message.Message,), dict(\n DESCRIPTOR = _EXPERIMENT,\n __module__ = 'tensorboardX.proto.api_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.hparam.Experiment)\n ))\n_sym_db.RegisterMessage(Experiment)\n\nHParamInfo = _reflection.GeneratedProtocolMessageType('HParamInfo', (_message.Message,), dict(\n DESCRIPTOR = _HPARAMINFO,\n __module__ = 'tensorboardX.proto.api_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.hparam.HParamInfo)\n ))\n_sym_db.RegisterMessage(HParamInfo)\n\nInterval = _reflection.GeneratedProtocolMessageType('Interval', (_message.Message,), dict(\n DESCRIPTOR = _INTERVAL,\n __module__ = 'tensorboardX.proto.api_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.hparam.Interval)\n ))\n_sym_db.RegisterMessage(Interval)\n\nMetricName = _reflection.GeneratedProtocolMessageType('MetricName', (_message.Message,), dict(\n DESCRIPTOR = _METRICNAME,\n __module__ = 'tensorboardX.proto.api_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.hparam.MetricName)\n ))\n_sym_db.RegisterMessage(MetricName)\n\nMetricInfo = _reflection.GeneratedProtocolMessageType('MetricInfo', (_message.Message,), dict(\n DESCRIPTOR = _METRICINFO,\n __module__ = 'tensorboardX.proto.api_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.hparam.MetricInfo)\n ))\n_sym_db.RegisterMessage(MetricInfo)\n\nSessionGroup = _reflection.GeneratedProtocolMessageType('SessionGroup', (_message.Message,), dict(\n\n HparamsEntry = _reflection.GeneratedProtocolMessageType('HparamsEntry', (_message.Message,), dict(\n DESCRIPTOR = _SESSIONGROUP_HPARAMSENTRY,\n __module__ = 'tensorboardX.proto.api_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.hparam.SessionGroup.HparamsEntry)\n ))\n ,\n DESCRIPTOR = _SESSIONGROUP,\n __module__ = 'tensorboardX.proto.api_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.hparam.SessionGroup)\n ))\n_sym_db.RegisterMessage(SessionGroup)\n_sym_db.RegisterMessage(SessionGroup.HparamsEntry)\n\nMetricValue = _reflection.GeneratedProtocolMessageType('MetricValue', (_message.Message,), dict(\n DESCRIPTOR = _METRICVALUE,\n __module__ = 'tensorboardX.proto.api_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.hparam.MetricValue)\n ))\n_sym_db.RegisterMessage(MetricValue)\n\nSession = _reflection.GeneratedProtocolMessageType('Session', (_message.Message,), dict(\n DESCRIPTOR = _SESSION,\n __module__ = 'tensorboardX.proto.api_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.hparam.Session)\n ))\n_sym_db.RegisterMessage(Session)\n\nGetExperimentRequest = _reflection.GeneratedProtocolMessageType('GetExperimentRequest', (_message.Message,), dict(\n DESCRIPTOR = _GETEXPERIMENTREQUEST,\n __module__ = 'tensorboardX.proto.api_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.hparam.GetExperimentRequest)\n ))\n_sym_db.RegisterMessage(GetExperimentRequest)\n\nListSessionGroupsRequest = _reflection.GeneratedProtocolMessageType('ListSessionGroupsRequest', (_message.Message,), dict(\n DESCRIPTOR = _LISTSESSIONGROUPSREQUEST,\n __module__ = 'tensorboardX.proto.api_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.hparam.ListSessionGroupsRequest)\n ))\n_sym_db.RegisterMessage(ListSessionGroupsRequest)\n\nColParams = _reflection.GeneratedProtocolMessageType('ColParams', (_message.Message,), dict(\n DESCRIPTOR = _COLPARAMS,\n __module__ = 'tensorboardX.proto.api_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.hparam.ColParams)\n ))\n_sym_db.RegisterMessage(ColParams)\n\nListSessionGroupsResponse = _reflection.GeneratedProtocolMessageType('ListSessionGroupsResponse', (_message.Message,), dict(\n DESCRIPTOR = _LISTSESSIONGROUPSRESPONSE,\n __module__ = 'tensorboardX.proto.api_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.hparam.ListSessionGroupsResponse)\n ))\n_sym_db.RegisterMessage(ListSessionGroupsResponse)\n\nListMetricEvalsRequest = _reflection.GeneratedProtocolMessageType('ListMetricEvalsRequest', (_message.Message,), dict(\n DESCRIPTOR = _LISTMETRICEVALSREQUEST,\n __module__ = 'tensorboardX.proto.api_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.hparam.ListMetricEvalsRequest)\n ))\n_sym_db.RegisterMessage(ListMetricEvalsRequest)\n\n\n_SESSIONGROUP_HPARAMSENTRY._options = None\n# @@protoc_insertion_point(module_scope)\n" }, { "path": "tensorboardX/tensorboardX/proto/attr_value.proto", "content": "syntax = \"proto3\";\n\npackage tensorboardX;\noption cc_enable_arenas = true;\noption java_outer_classname = \"AttrValueProtos\";\noption java_multiple_files = true;\noption java_package = \"org.tensorflow.framework\";\n\nimport \"tensorboardX/proto/tensor.proto\";\nimport \"tensorboardX/proto/tensor_shape.proto\";\nimport \"tensorboardX/proto/types.proto\";\n\n// Protocol buffer representing the value for an attr used to configure an Op.\n// Comment indicates the corresponding attr type. Only the field matching the\n// attr type may be filled.\nmessage AttrValue {\n // LINT.IfChange\n message ListValue {\n repeated bytes s = 2; // \"list(string)\"\n repeated int64 i = 3 [packed = true]; // \"list(int)\"\n repeated float f = 4 [packed = true]; // \"list(float)\"\n repeated bool b = 5 [packed = true]; // \"list(bool)\"\n repeated DataType type = 6 [packed = true]; // \"list(type)\"\n repeated TensorShapeProto shape = 7; // \"list(shape)\"\n repeated TensorProto tensor = 8; // \"list(tensor)\"\n repeated NameAttrList func = 9; // \"list(attr)\"\n }\n // LINT.ThenChange(https://www.tensorflow.org/code/tensorflow/c/c_api.cc)\n\n oneof value {\n bytes s = 2; // \"string\"\n int64 i = 3; // \"int\"\n float f = 4; // \"float\"\n bool b = 5; // \"bool\"\n DataType type = 6; // \"type\"\n TensorShapeProto shape = 7; // \"shape\"\n TensorProto tensor = 8; // \"tensor\"\n ListValue list = 1; // any \"list(...)\"\n\n // \"func\" represents a function. func.name is a function's name or\n // a primitive op's name. func.attr.first is the name of an attr\n // defined for that function. func.attr.second is the value for\n // that attr in the instantiation.\n NameAttrList func = 10;\n\n // This is a placeholder only used in nodes defined inside a\n // function. It indicates the attr value will be supplied when\n // the function is instantiated. For example, let us suppose a\n // node \"N\" in function \"FN\". \"N\" has an attr \"A\" with value\n // placeholder = \"foo\". When FN is instantiated with attr \"foo\"\n // set to \"bar\", the instantiated node N's attr A will have been\n // given the value \"bar\".\n string placeholder = 9;\n }\n}\n\n// A list of attr names and their values. The whole list is attached\n// with a string name. E.g., MatMul[T=float].\nmessage NameAttrList {\n string name = 1;\n map attr = 2;\n}\n" }, { "path": "tensorboardX/tensorboardX/proto/attr_value_pb2.py", "content": "# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: tensorboardX/proto/attr_value.proto\n\nimport sys\n_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))\nfrom google.protobuf import descriptor as _descriptor\nfrom google.protobuf import message as _message\nfrom google.protobuf import reflection as _reflection\nfrom google.protobuf import symbol_database as _symbol_database\n# @@protoc_insertion_point(imports)\n\n_sym_db = _symbol_database.Default()\n\n\nfrom tensorboardX.proto import tensor_pb2 as tensorboardX_dot_proto_dot_tensor__pb2\nfrom tensorboardX.proto import tensor_shape_pb2 as tensorboardX_dot_proto_dot_tensor__shape__pb2\nfrom tensorboardX.proto import types_pb2 as tensorboardX_dot_proto_dot_types__pb2\n\n\nDESCRIPTOR = _descriptor.FileDescriptor(\n name='tensorboardX/proto/attr_value.proto',\n package='tensorboardX',\n syntax='proto3',\n serialized_options=_b('\\n\\030org.tensorflow.frameworkB\\017AttrValueProtosP\\001\\370\\001\\001'),\n serialized_pb=_b('\\n#tensorboardX/proto/attr_value.proto\\x12\\x0ctensorboardX\\x1a\\x1ftensorboardX/proto/tensor.proto\\x1a%tensorboardX/proto/tensor_shape.proto\\x1a\\x1etensorboardX/proto/types.proto\\\"\\xb8\\x04\\n\\tAttrValue\\x12\\x0b\\n\\x01s\\x18\\x02 \\x01(\\x0cH\\x00\\x12\\x0b\\n\\x01i\\x18\\x03 \\x01(\\x03H\\x00\\x12\\x0b\\n\\x01\\x66\\x18\\x04 \\x01(\\x02H\\x00\\x12\\x0b\\n\\x01\\x62\\x18\\x05 \\x01(\\x08H\\x00\\x12&\\n\\x04type\\x18\\x06 \\x01(\\x0e\\x32\\x16.tensorboardX.DataTypeH\\x00\\x12/\\n\\x05shape\\x18\\x07 \\x01(\\x0b\\x32\\x1e.tensorboardX.TensorShapeProtoH\\x00\\x12+\\n\\x06tensor\\x18\\x08 \\x01(\\x0b\\x32\\x19.tensorboardX.TensorProtoH\\x00\\x12\\x31\\n\\x04list\\x18\\x01 \\x01(\\x0b\\x32!.tensorboardX.AttrValue.ListValueH\\x00\\x12*\\n\\x04\\x66unc\\x18\\n \\x01(\\x0b\\x32\\x1a.tensorboardX.NameAttrListH\\x00\\x12\\x15\\n\\x0bplaceholder\\x18\\t \\x01(\\tH\\x00\\x1a\\xf1\\x01\\n\\tListValue\\x12\\t\\n\\x01s\\x18\\x02 \\x03(\\x0c\\x12\\r\\n\\x01i\\x18\\x03 \\x03(\\x03\\x42\\x02\\x10\\x01\\x12\\r\\n\\x01\\x66\\x18\\x04 \\x03(\\x02\\x42\\x02\\x10\\x01\\x12\\r\\n\\x01\\x62\\x18\\x05 \\x03(\\x08\\x42\\x02\\x10\\x01\\x12(\\n\\x04type\\x18\\x06 \\x03(\\x0e\\x32\\x16.tensorboardX.DataTypeB\\x02\\x10\\x01\\x12-\\n\\x05shape\\x18\\x07 \\x03(\\x0b\\x32\\x1e.tensorboardX.TensorShapeProto\\x12)\\n\\x06tensor\\x18\\x08 \\x03(\\x0b\\x32\\x19.tensorboardX.TensorProto\\x12(\\n\\x04\\x66unc\\x18\\t \\x03(\\x0b\\x32\\x1a.tensorboardX.NameAttrListB\\x07\\n\\x05value\\\"\\x96\\x01\\n\\x0cNameAttrList\\x12\\x0c\\n\\x04name\\x18\\x01 \\x01(\\t\\x12\\x32\\n\\x04\\x61ttr\\x18\\x02 \\x03(\\x0b\\x32$.tensorboardX.NameAttrList.AttrEntry\\x1a\\x44\\n\\tAttrEntry\\x12\\x0b\\n\\x03key\\x18\\x01 \\x01(\\t\\x12&\\n\\x05value\\x18\\x02 \\x01(\\x0b\\x32\\x17.tensorboardX.AttrValue:\\x02\\x38\\x01\\x42\\x30\\n\\x18org.tensorflow.frameworkB\\x0f\\x41ttrValueProtosP\\x01\\xf8\\x01\\x01\\x62\\x06proto3')\n ,\n dependencies=[tensorboardX_dot_proto_dot_tensor__pb2.DESCRIPTOR,tensorboardX_dot_proto_dot_tensor__shape__pb2.DESCRIPTOR,tensorboardX_dot_proto_dot_types__pb2.DESCRIPTOR,])\n\n\n\n\n_ATTRVALUE_LISTVALUE = _descriptor.Descriptor(\n name='ListValue',\n full_name='tensorboardX.AttrValue.ListValue',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='s', full_name='tensorboardX.AttrValue.ListValue.s', index=0,\n number=2, type=12, cpp_type=9, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='i', full_name='tensorboardX.AttrValue.ListValue.i', index=1,\n number=3, type=3, cpp_type=2, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=_b('\\020\\001'), file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='f', full_name='tensorboardX.AttrValue.ListValue.f', index=2,\n number=4, type=2, cpp_type=6, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=_b('\\020\\001'), file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='b', full_name='tensorboardX.AttrValue.ListValue.b', index=3,\n number=5, type=8, cpp_type=7, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=_b('\\020\\001'), file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='type', full_name='tensorboardX.AttrValue.ListValue.type', index=4,\n number=6, type=14, cpp_type=8, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=_b('\\020\\001'), file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='shape', full_name='tensorboardX.AttrValue.ListValue.shape', index=5,\n number=7, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='tensor', full_name='tensorboardX.AttrValue.ListValue.tensor', index=6,\n number=8, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='func', full_name='tensorboardX.AttrValue.ListValue.func', index=7,\n number=9, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=476,\n serialized_end=717,\n)\n\n_ATTRVALUE = _descriptor.Descriptor(\n name='AttrValue',\n full_name='tensorboardX.AttrValue',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='s', full_name='tensorboardX.AttrValue.s', index=0,\n number=2, type=12, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\"),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='i', full_name='tensorboardX.AttrValue.i', index=1,\n number=3, type=3, cpp_type=2, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='f', full_name='tensorboardX.AttrValue.f', index=2,\n number=4, type=2, cpp_type=6, label=1,\n has_default_value=False, default_value=float(0),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='b', full_name='tensorboardX.AttrValue.b', index=3,\n number=5, type=8, cpp_type=7, label=1,\n has_default_value=False, default_value=False,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='type', full_name='tensorboardX.AttrValue.type', index=4,\n number=6, type=14, cpp_type=8, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='shape', full_name='tensorboardX.AttrValue.shape', index=5,\n number=7, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='tensor', full_name='tensorboardX.AttrValue.tensor', index=6,\n number=8, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='list', full_name='tensorboardX.AttrValue.list', index=7,\n number=1, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='func', full_name='tensorboardX.AttrValue.func', index=8,\n number=10, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='placeholder', full_name='tensorboardX.AttrValue.placeholder', index=9,\n number=9, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[_ATTRVALUE_LISTVALUE, ],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n _descriptor.OneofDescriptor(\n name='value', full_name='tensorboardX.AttrValue.value',\n index=0, containing_type=None, fields=[]),\n ],\n serialized_start=158,\n serialized_end=726,\n)\n\n\n_NAMEATTRLIST_ATTRENTRY = _descriptor.Descriptor(\n name='AttrEntry',\n full_name='tensorboardX.NameAttrList.AttrEntry',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='key', full_name='tensorboardX.NameAttrList.AttrEntry.key', index=0,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='value', full_name='tensorboardX.NameAttrList.AttrEntry.value', index=1,\n number=2, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=_b('8\\001'),\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=811,\n serialized_end=879,\n)\n\n_NAMEATTRLIST = _descriptor.Descriptor(\n name='NameAttrList',\n full_name='tensorboardX.NameAttrList',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='name', full_name='tensorboardX.NameAttrList.name', index=0,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='attr', full_name='tensorboardX.NameAttrList.attr', index=1,\n number=2, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[_NAMEATTRLIST_ATTRENTRY, ],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=729,\n serialized_end=879,\n)\n\n_ATTRVALUE_LISTVALUE.fields_by_name['type'].enum_type = tensorboardX_dot_proto_dot_types__pb2._DATATYPE\n_ATTRVALUE_LISTVALUE.fields_by_name['shape'].message_type = tensorboardX_dot_proto_dot_tensor__shape__pb2._TENSORSHAPEPROTO\n_ATTRVALUE_LISTVALUE.fields_by_name['tensor'].message_type = tensorboardX_dot_proto_dot_tensor__pb2._TENSORPROTO\n_ATTRVALUE_LISTVALUE.fields_by_name['func'].message_type = _NAMEATTRLIST\n_ATTRVALUE_LISTVALUE.containing_type = _ATTRVALUE\n_ATTRVALUE.fields_by_name['type'].enum_type = tensorboardX_dot_proto_dot_types__pb2._DATATYPE\n_ATTRVALUE.fields_by_name['shape'].message_type = tensorboardX_dot_proto_dot_tensor__shape__pb2._TENSORSHAPEPROTO\n_ATTRVALUE.fields_by_name['tensor'].message_type = tensorboardX_dot_proto_dot_tensor__pb2._TENSORPROTO\n_ATTRVALUE.fields_by_name['list'].message_type = _ATTRVALUE_LISTVALUE\n_ATTRVALUE.fields_by_name['func'].message_type = _NAMEATTRLIST\n_ATTRVALUE.oneofs_by_name['value'].fields.append(\n _ATTRVALUE.fields_by_name['s'])\n_ATTRVALUE.fields_by_name['s'].containing_oneof = _ATTRVALUE.oneofs_by_name['value']\n_ATTRVALUE.oneofs_by_name['value'].fields.append(\n _ATTRVALUE.fields_by_name['i'])\n_ATTRVALUE.fields_by_name['i'].containing_oneof = _ATTRVALUE.oneofs_by_name['value']\n_ATTRVALUE.oneofs_by_name['value'].fields.append(\n _ATTRVALUE.fields_by_name['f'])\n_ATTRVALUE.fields_by_name['f'].containing_oneof = _ATTRVALUE.oneofs_by_name['value']\n_ATTRVALUE.oneofs_by_name['value'].fields.append(\n _ATTRVALUE.fields_by_name['b'])\n_ATTRVALUE.fields_by_name['b'].containing_oneof = _ATTRVALUE.oneofs_by_name['value']\n_ATTRVALUE.oneofs_by_name['value'].fields.append(\n _ATTRVALUE.fields_by_name['type'])\n_ATTRVALUE.fields_by_name['type'].containing_oneof = _ATTRVALUE.oneofs_by_name['value']\n_ATTRVALUE.oneofs_by_name['value'].fields.append(\n _ATTRVALUE.fields_by_name['shape'])\n_ATTRVALUE.fields_by_name['shape'].containing_oneof = _ATTRVALUE.oneofs_by_name['value']\n_ATTRVALUE.oneofs_by_name['value'].fields.append(\n _ATTRVALUE.fields_by_name['tensor'])\n_ATTRVALUE.fields_by_name['tensor'].containing_oneof = _ATTRVALUE.oneofs_by_name['value']\n_ATTRVALUE.oneofs_by_name['value'].fields.append(\n _ATTRVALUE.fields_by_name['list'])\n_ATTRVALUE.fields_by_name['list'].containing_oneof = _ATTRVALUE.oneofs_by_name['value']\n_ATTRVALUE.oneofs_by_name['value'].fields.append(\n _ATTRVALUE.fields_by_name['func'])\n_ATTRVALUE.fields_by_name['func'].containing_oneof = _ATTRVALUE.oneofs_by_name['value']\n_ATTRVALUE.oneofs_by_name['value'].fields.append(\n _ATTRVALUE.fields_by_name['placeholder'])\n_ATTRVALUE.fields_by_name['placeholder'].containing_oneof = _ATTRVALUE.oneofs_by_name['value']\n_NAMEATTRLIST_ATTRENTRY.fields_by_name['value'].message_type = _ATTRVALUE\n_NAMEATTRLIST_ATTRENTRY.containing_type = _NAMEATTRLIST\n_NAMEATTRLIST.fields_by_name['attr'].message_type = _NAMEATTRLIST_ATTRENTRY\nDESCRIPTOR.message_types_by_name['AttrValue'] = _ATTRVALUE\nDESCRIPTOR.message_types_by_name['NameAttrList'] = _NAMEATTRLIST\n_sym_db.RegisterFileDescriptor(DESCRIPTOR)\n\nAttrValue = _reflection.GeneratedProtocolMessageType('AttrValue', (_message.Message,), dict(\n\n ListValue = _reflection.GeneratedProtocolMessageType('ListValue', (_message.Message,), dict(\n DESCRIPTOR = _ATTRVALUE_LISTVALUE,\n __module__ = 'tensorboardX.proto.attr_value_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.AttrValue.ListValue)\n ))\n ,\n DESCRIPTOR = _ATTRVALUE,\n __module__ = 'tensorboardX.proto.attr_value_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.AttrValue)\n ))\n_sym_db.RegisterMessage(AttrValue)\n_sym_db.RegisterMessage(AttrValue.ListValue)\n\nNameAttrList = _reflection.GeneratedProtocolMessageType('NameAttrList', (_message.Message,), dict(\n\n AttrEntry = _reflection.GeneratedProtocolMessageType('AttrEntry', (_message.Message,), dict(\n DESCRIPTOR = _NAMEATTRLIST_ATTRENTRY,\n __module__ = 'tensorboardX.proto.attr_value_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.NameAttrList.AttrEntry)\n ))\n ,\n DESCRIPTOR = _NAMEATTRLIST,\n __module__ = 'tensorboardX.proto.attr_value_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.NameAttrList)\n ))\n_sym_db.RegisterMessage(NameAttrList)\n_sym_db.RegisterMessage(NameAttrList.AttrEntry)\n\n\nDESCRIPTOR._options = None\n_ATTRVALUE_LISTVALUE.fields_by_name['i']._options = None\n_ATTRVALUE_LISTVALUE.fields_by_name['f']._options = None\n_ATTRVALUE_LISTVALUE.fields_by_name['b']._options = None\n_ATTRVALUE_LISTVALUE.fields_by_name['type']._options = None\n_NAMEATTRLIST_ATTRENTRY._options = None\n# @@protoc_insertion_point(module_scope)\n" }, { "path": "tensorboardX/tensorboardX/proto/event.proto", "content": "syntax = \"proto3\";\n\npackage tensorboardX;\noption cc_enable_arenas = true;\noption java_outer_classname = \"EventProtos\";\noption java_multiple_files = true;\noption java_package = \"org.tensorflow.util\";\n\nimport \"tensorboardX/proto/summary.proto\";\n\n// Protocol buffer representing an event that happened during\n// the execution of a Brain model.\nmessage Event {\n // Timestamp of the event.\n double wall_time = 1;\n\n // Global step of the event.\n int64 step = 2;\n\n oneof what {\n // An event file was started, with the specified version.\n // This is use to identify the contents of the record IO files\n // easily. Current version is \"brain.Event:2\". All versions\n // start with \"brain.Event:\".\n string file_version = 3;\n // An encoded version of a GraphDef.\n bytes graph_def = 4;\n // A summary was generated.\n Summary summary = 5;\n // The user output a log message. Not all messages are logged, only ones\n // generated via the Python tensorboard_logging module.\n LogMessage log_message = 6;\n // The state of the session which can be used for restarting after crashes.\n SessionLog session_log = 7;\n // The metadata returned by running a session.run() call.\n TaggedRunMetadata tagged_run_metadata = 8;\n // An encoded version of a MetaGraphDef.\n bytes meta_graph_def = 9;\n }\n}\n\n// Protocol buffer used for logging messages to the events file.\nmessage LogMessage {\n enum Level {\n UNKNOWN = 0;\n DEBUG = 10;\n INFO = 20;\n WARN = 30;\n ERROR = 40;\n FATAL = 50;\n }\n Level level = 1;\n string message = 2;\n}\n\n// Protocol buffer used for logging session state.\nmessage SessionLog {\n enum SessionStatus {\n STATUS_UNSPECIFIED = 0;\n START = 1;\n STOP = 2;\n CHECKPOINT = 3;\n }\n\n SessionStatus status = 1;\n // This checkpoint_path contains both the path and filename.\n string checkpoint_path = 2;\n string msg = 3;\n}\n\n// For logging the metadata output for a single session.run() call.\nmessage TaggedRunMetadata {\n // Tag name associated with this metadata.\n string tag = 1;\n // Byte-encoded version of the `RunMetadata` proto in order to allow lazy\n // deserialization.\n bytes run_metadata = 2;\n}\n" }, { "path": "tensorboardX/tensorboardX/proto/event_pb2.py", "content": "# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: tensorboardX/proto/event.proto\n\nimport sys\n_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))\nfrom google.protobuf import descriptor as _descriptor\nfrom google.protobuf import message as _message\nfrom google.protobuf import reflection as _reflection\nfrom google.protobuf import symbol_database as _symbol_database\n# @@protoc_insertion_point(imports)\n\n_sym_db = _symbol_database.Default()\n\n\nfrom tensorboardX.proto import summary_pb2 as tensorboardX_dot_proto_dot_summary__pb2\n\n\nDESCRIPTOR = _descriptor.FileDescriptor(\n name='tensorboardX/proto/event.proto',\n package='tensorboardX',\n syntax='proto3',\n serialized_options=_b('\\n\\023org.tensorflow.utilB\\013EventProtosP\\001\\370\\001\\001'),\n serialized_pb=_b('\\n\\x1etensorboardX/proto/event.proto\\x12\\x0ctensorboardX\\x1a tensorboardX/proto/summary.proto\\\"\\xc3\\x02\\n\\x05\\x45vent\\x12\\x11\\n\\twall_time\\x18\\x01 \\x01(\\x01\\x12\\x0c\\n\\x04step\\x18\\x02 \\x01(\\x03\\x12\\x16\\n\\x0c\\x66ile_version\\x18\\x03 \\x01(\\tH\\x00\\x12\\x13\\n\\tgraph_def\\x18\\x04 \\x01(\\x0cH\\x00\\x12(\\n\\x07summary\\x18\\x05 \\x01(\\x0b\\x32\\x15.tensorboardX.SummaryH\\x00\\x12/\\n\\x0blog_message\\x18\\x06 \\x01(\\x0b\\x32\\x18.tensorboardX.LogMessageH\\x00\\x12/\\n\\x0bsession_log\\x18\\x07 \\x01(\\x0b\\x32\\x18.tensorboardX.SessionLogH\\x00\\x12>\\n\\x13tagged_run_metadata\\x18\\x08 \\x01(\\x0b\\x32\\x1f.tensorboardX.TaggedRunMetadataH\\x00\\x12\\x18\\n\\x0emeta_graph_def\\x18\\t \\x01(\\x0cH\\x00\\x42\\x06\\n\\x04what\\\"\\x97\\x01\\n\\nLogMessage\\x12-\\n\\x05level\\x18\\x01 \\x01(\\x0e\\x32\\x1e.tensorboardX.LogMessage.Level\\x12\\x0f\\n\\x07message\\x18\\x02 \\x01(\\t\\\"I\\n\\x05Level\\x12\\x0b\\n\\x07UNKNOWN\\x10\\x00\\x12\\t\\n\\x05\\x44\\x45\\x42UG\\x10\\n\\x12\\x08\\n\\x04INFO\\x10\\x14\\x12\\x08\\n\\x04WARN\\x10\\x1e\\x12\\t\\n\\x05\\x45RROR\\x10(\\x12\\t\\n\\x05\\x46\\x41TAL\\x10\\x32\\\"\\xb8\\x01\\n\\nSessionLog\\x12\\x36\\n\\x06status\\x18\\x01 \\x01(\\x0e\\x32&.tensorboardX.SessionLog.SessionStatus\\x12\\x17\\n\\x0f\\x63heckpoint_path\\x18\\x02 \\x01(\\t\\x12\\x0b\\n\\x03msg\\x18\\x03 \\x01(\\t\\\"L\\n\\rSessionStatus\\x12\\x16\\n\\x12STATUS_UNSPECIFIED\\x10\\x00\\x12\\t\\n\\x05START\\x10\\x01\\x12\\x08\\n\\x04STOP\\x10\\x02\\x12\\x0e\\n\\nCHECKPOINT\\x10\\x03\\\"6\\n\\x11TaggedRunMetadata\\x12\\x0b\\n\\x03tag\\x18\\x01 \\x01(\\t\\x12\\x14\\n\\x0crun_metadata\\x18\\x02 \\x01(\\x0c\\x42\\'\\n\\x13org.tensorflow.utilB\\x0b\\x45ventProtosP\\x01\\xf8\\x01\\x01\\x62\\x06proto3')\n ,\n dependencies=[tensorboardX_dot_proto_dot_summary__pb2.DESCRIPTOR,])\n\n\n\n_LOGMESSAGE_LEVEL = _descriptor.EnumDescriptor(\n name='Level',\n full_name='tensorboardX.LogMessage.Level',\n filename=None,\n file=DESCRIPTOR,\n values=[\n _descriptor.EnumValueDescriptor(\n name='UNKNOWN', index=0, number=0,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DEBUG', index=1, number=10,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='INFO', index=2, number=20,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='WARN', index=3, number=30,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='ERROR', index=4, number=40,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='FATAL', index=5, number=50,\n serialized_options=None,\n type=None),\n ],\n containing_type=None,\n serialized_options=None,\n serialized_start=487,\n serialized_end=560,\n)\n_sym_db.RegisterEnumDescriptor(_LOGMESSAGE_LEVEL)\n\n_SESSIONLOG_SESSIONSTATUS = _descriptor.EnumDescriptor(\n name='SessionStatus',\n full_name='tensorboardX.SessionLog.SessionStatus',\n filename=None,\n file=DESCRIPTOR,\n values=[\n _descriptor.EnumValueDescriptor(\n name='STATUS_UNSPECIFIED', index=0, number=0,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='START', index=1, number=1,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='STOP', index=2, number=2,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='CHECKPOINT', index=3, number=3,\n serialized_options=None,\n type=None),\n ],\n containing_type=None,\n serialized_options=None,\n serialized_start=671,\n serialized_end=747,\n)\n_sym_db.RegisterEnumDescriptor(_SESSIONLOG_SESSIONSTATUS)\n\n\n_EVENT = _descriptor.Descriptor(\n name='Event',\n full_name='tensorboardX.Event',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='wall_time', full_name='tensorboardX.Event.wall_time', index=0,\n number=1, type=1, cpp_type=5, label=1,\n has_default_value=False, default_value=float(0),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='step', full_name='tensorboardX.Event.step', index=1,\n number=2, type=3, cpp_type=2, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='file_version', full_name='tensorboardX.Event.file_version', index=2,\n number=3, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='graph_def', full_name='tensorboardX.Event.graph_def', index=3,\n number=4, type=12, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\"),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='summary', full_name='tensorboardX.Event.summary', index=4,\n number=5, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='log_message', full_name='tensorboardX.Event.log_message', index=5,\n number=6, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='session_log', full_name='tensorboardX.Event.session_log', index=6,\n number=7, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='tagged_run_metadata', full_name='tensorboardX.Event.tagged_run_metadata', index=7,\n number=8, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='meta_graph_def', full_name='tensorboardX.Event.meta_graph_def', index=8,\n number=9, type=12, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\"),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n _descriptor.OneofDescriptor(\n name='what', full_name='tensorboardX.Event.what',\n index=0, containing_type=None, fields=[]),\n ],\n serialized_start=83,\n serialized_end=406,\n)\n\n\n_LOGMESSAGE = _descriptor.Descriptor(\n name='LogMessage',\n full_name='tensorboardX.LogMessage',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='level', full_name='tensorboardX.LogMessage.level', index=0,\n number=1, type=14, cpp_type=8, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='message', full_name='tensorboardX.LogMessage.message', index=1,\n number=2, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n _LOGMESSAGE_LEVEL,\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=409,\n serialized_end=560,\n)\n\n\n_SESSIONLOG = _descriptor.Descriptor(\n name='SessionLog',\n full_name='tensorboardX.SessionLog',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='status', full_name='tensorboardX.SessionLog.status', index=0,\n number=1, type=14, cpp_type=8, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='checkpoint_path', full_name='tensorboardX.SessionLog.checkpoint_path', index=1,\n number=2, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='msg', full_name='tensorboardX.SessionLog.msg', index=2,\n number=3, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n _SESSIONLOG_SESSIONSTATUS,\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=563,\n serialized_end=747,\n)\n\n\n_TAGGEDRUNMETADATA = _descriptor.Descriptor(\n name='TaggedRunMetadata',\n full_name='tensorboardX.TaggedRunMetadata',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='tag', full_name='tensorboardX.TaggedRunMetadata.tag', index=0,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='run_metadata', full_name='tensorboardX.TaggedRunMetadata.run_metadata', index=1,\n number=2, type=12, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\"),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=749,\n serialized_end=803,\n)\n\n_EVENT.fields_by_name['summary'].message_type = tensorboardX_dot_proto_dot_summary__pb2._SUMMARY\n_EVENT.fields_by_name['log_message'].message_type = _LOGMESSAGE\n_EVENT.fields_by_name['session_log'].message_type = _SESSIONLOG\n_EVENT.fields_by_name['tagged_run_metadata'].message_type = _TAGGEDRUNMETADATA\n_EVENT.oneofs_by_name['what'].fields.append(\n _EVENT.fields_by_name['file_version'])\n_EVENT.fields_by_name['file_version'].containing_oneof = _EVENT.oneofs_by_name['what']\n_EVENT.oneofs_by_name['what'].fields.append(\n _EVENT.fields_by_name['graph_def'])\n_EVENT.fields_by_name['graph_def'].containing_oneof = _EVENT.oneofs_by_name['what']\n_EVENT.oneofs_by_name['what'].fields.append(\n _EVENT.fields_by_name['summary'])\n_EVENT.fields_by_name['summary'].containing_oneof = _EVENT.oneofs_by_name['what']\n_EVENT.oneofs_by_name['what'].fields.append(\n _EVENT.fields_by_name['log_message'])\n_EVENT.fields_by_name['log_message'].containing_oneof = _EVENT.oneofs_by_name['what']\n_EVENT.oneofs_by_name['what'].fields.append(\n _EVENT.fields_by_name['session_log'])\n_EVENT.fields_by_name['session_log'].containing_oneof = _EVENT.oneofs_by_name['what']\n_EVENT.oneofs_by_name['what'].fields.append(\n _EVENT.fields_by_name['tagged_run_metadata'])\n_EVENT.fields_by_name['tagged_run_metadata'].containing_oneof = _EVENT.oneofs_by_name['what']\n_EVENT.oneofs_by_name['what'].fields.append(\n _EVENT.fields_by_name['meta_graph_def'])\n_EVENT.fields_by_name['meta_graph_def'].containing_oneof = _EVENT.oneofs_by_name['what']\n_LOGMESSAGE.fields_by_name['level'].enum_type = _LOGMESSAGE_LEVEL\n_LOGMESSAGE_LEVEL.containing_type = _LOGMESSAGE\n_SESSIONLOG.fields_by_name['status'].enum_type = _SESSIONLOG_SESSIONSTATUS\n_SESSIONLOG_SESSIONSTATUS.containing_type = _SESSIONLOG\nDESCRIPTOR.message_types_by_name['Event'] = _EVENT\nDESCRIPTOR.message_types_by_name['LogMessage'] = _LOGMESSAGE\nDESCRIPTOR.message_types_by_name['SessionLog'] = _SESSIONLOG\nDESCRIPTOR.message_types_by_name['TaggedRunMetadata'] = _TAGGEDRUNMETADATA\n_sym_db.RegisterFileDescriptor(DESCRIPTOR)\n\nEvent = _reflection.GeneratedProtocolMessageType('Event', (_message.Message,), dict(\n DESCRIPTOR = _EVENT,\n __module__ = 'tensorboardX.proto.event_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.Event)\n ))\n_sym_db.RegisterMessage(Event)\n\nLogMessage = _reflection.GeneratedProtocolMessageType('LogMessage', (_message.Message,), dict(\n DESCRIPTOR = _LOGMESSAGE,\n __module__ = 'tensorboardX.proto.event_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.LogMessage)\n ))\n_sym_db.RegisterMessage(LogMessage)\n\nSessionLog = _reflection.GeneratedProtocolMessageType('SessionLog', (_message.Message,), dict(\n DESCRIPTOR = _SESSIONLOG,\n __module__ = 'tensorboardX.proto.event_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.SessionLog)\n ))\n_sym_db.RegisterMessage(SessionLog)\n\nTaggedRunMetadata = _reflection.GeneratedProtocolMessageType('TaggedRunMetadata', (_message.Message,), dict(\n DESCRIPTOR = _TAGGEDRUNMETADATA,\n __module__ = 'tensorboardX.proto.event_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.TaggedRunMetadata)\n ))\n_sym_db.RegisterMessage(TaggedRunMetadata)\n\n\nDESCRIPTOR._options = None\n# @@protoc_insertion_point(module_scope)\n" }, { "path": "tensorboardX/tensorboardX/proto/graph.proto", "content": "syntax = \"proto3\";\n\npackage tensorboardX;\noption cc_enable_arenas = true;\noption java_outer_classname = \"GraphProtos\";\noption java_multiple_files = true;\noption java_package = \"org.tensorflow.framework\";\n\nimport \"tensorboardX/proto/node_def.proto\";\n//import \"tensorflow/core/framework/function.proto\";\nimport \"tensorboardX/proto/versions.proto\";\n\n// Represents the graph of operations\nmessage GraphDef {\n repeated NodeDef node = 1;\n\n // Compatibility versions of the graph. See core/public/version.h for version\n // history. The GraphDef version is distinct from the TensorFlow version, and\n // each release of TensorFlow will support a range of GraphDef versions.\n VersionDef versions = 4;\n\n // Deprecated single version field; use versions above instead. Since all\n // GraphDef changes before \"versions\" was introduced were forward\n // compatible, this field is entirely ignored.\n int32 version = 3 [deprecated = true];\n\n // EXPERIMENTAL. DO NOT USE OR DEPEND ON THIS YET.\n //\n // \"library\" provides user-defined functions.\n //\n // Naming:\n // * library.function.name are in a flat namespace.\n // NOTE: We may need to change it to be hierarchical to support\n // different orgs. E.g.,\n // { \"/google/nn\", { ... }},\n // { \"/google/vision\", { ... }}\n // { \"/org_foo/module_bar\", { ... }}\n // map named_lib;\n // * If node[i].op is the name of one function in \"library\",\n // node[i] is deemed as a function call. Otherwise, node[i].op\n // must be a primitive operation supported by the runtime.\n //\n //\n // Function call semantics:\n //\n // * The callee may start execution as soon as some of its inputs\n // are ready. The caller may want to use Tuple() mechanism to\n // ensure all inputs are ready in the same time.\n //\n // * The consumer of return values may start executing as soon as\n // the return values the consumer depends on are ready. The\n // consumer may want to use Tuple() mechanism to ensure the\n // consumer does not start until all return values of the callee\n // function are ready.\n //FunctionDefLibrary library = 2;\n};\n" }, { "path": "tensorboardX/tensorboardX/proto/graph_pb2.py", "content": "# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: tensorboardX/proto/graph.proto\n\nimport sys\n_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))\nfrom google.protobuf import descriptor as _descriptor\nfrom google.protobuf import message as _message\nfrom google.protobuf import reflection as _reflection\nfrom google.protobuf import symbol_database as _symbol_database\n# @@protoc_insertion_point(imports)\n\n_sym_db = _symbol_database.Default()\n\n\nfrom tensorboardX.proto import node_def_pb2 as tensorboardX_dot_proto_dot_node__def__pb2\nfrom tensorboardX.proto import versions_pb2 as tensorboardX_dot_proto_dot_versions__pb2\n\n\nDESCRIPTOR = _descriptor.FileDescriptor(\n name='tensorboardX/proto/graph.proto',\n package='tensorboardX',\n syntax='proto3',\n serialized_options=_b('\\n\\030org.tensorflow.frameworkB\\013GraphProtosP\\001\\370\\001\\001'),\n serialized_pb=_b('\\n\\x1etensorboardX/proto/graph.proto\\x12\\x0ctensorboardX\\x1a!tensorboardX/proto/node_def.proto\\x1a!tensorboardX/proto/versions.proto\\\"p\\n\\x08GraphDef\\x12#\\n\\x04node\\x18\\x01 \\x03(\\x0b\\x32\\x15.tensorboardX.NodeDef\\x12*\\n\\x08versions\\x18\\x04 \\x01(\\x0b\\x32\\x18.tensorboardX.VersionDef\\x12\\x13\\n\\x07version\\x18\\x03 \\x01(\\x05\\x42\\x02\\x18\\x01\\x42,\\n\\x18org.tensorflow.frameworkB\\x0bGraphProtosP\\x01\\xf8\\x01\\x01\\x62\\x06proto3')\n ,\n dependencies=[tensorboardX_dot_proto_dot_node__def__pb2.DESCRIPTOR,tensorboardX_dot_proto_dot_versions__pb2.DESCRIPTOR,])\n\n\n\n\n_GRAPHDEF = _descriptor.Descriptor(\n name='GraphDef',\n full_name='tensorboardX.GraphDef',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='node', full_name='tensorboardX.GraphDef.node', index=0,\n number=1, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='versions', full_name='tensorboardX.GraphDef.versions', index=1,\n number=4, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='version', full_name='tensorboardX.GraphDef.version', index=2,\n number=3, type=5, cpp_type=1, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=_b('\\030\\001'), file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=118,\n serialized_end=230,\n)\n\n_GRAPHDEF.fields_by_name['node'].message_type = tensorboardX_dot_proto_dot_node__def__pb2._NODEDEF\n_GRAPHDEF.fields_by_name['versions'].message_type = tensorboardX_dot_proto_dot_versions__pb2._VERSIONDEF\nDESCRIPTOR.message_types_by_name['GraphDef'] = _GRAPHDEF\n_sym_db.RegisterFileDescriptor(DESCRIPTOR)\n\nGraphDef = _reflection.GeneratedProtocolMessageType('GraphDef', (_message.Message,), dict(\n DESCRIPTOR = _GRAPHDEF,\n __module__ = 'tensorboardX.proto.graph_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.GraphDef)\n ))\n_sym_db.RegisterMessage(GraphDef)\n\n\nDESCRIPTOR._options = None\n_GRAPHDEF.fields_by_name['version']._options = None\n# @@protoc_insertion_point(module_scope)\n" }, { "path": "tensorboardX/tensorboardX/proto/layout.proto", "content": "/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n==============================================================================*/\n\nsyntax = \"proto3\";\n\npackage tensorboardX;\n\n\n/**\n * Encapsulates information on a single chart. Many charts appear in a category.\n */\nmessage Chart {\n // The title shown atop this chart. Optional. Defaults to 'untitled'.\n string title = 1;\n\n // The content of the chart. This depends on the type of the chart.\n oneof content {\n MultilineChartContent multiline = 2;\n MarginChartContent margin = 3;\n }\n}\n\n/**\n * Encapsulates information on a single line chart. This line chart may have\n * lines associated with several tags.\n */\nmessage MultilineChartContent {\n // A list of regular expressions for tags that should appear in this chart.\n // Tags are matched from beginning to end. Each regex captures a set of tags.\n repeated string tag = 1;\n}\n\n/**\n * Encapsulates information on a single margin chart. A margin chart uses fill\n * area to visualize lower and upper bounds that surround a value.\n */\nmessage MarginChartContent {\n /**\n * Encapsulates a tag of data for the chart.\n */\n message Series {\n // The exact tag string associated with the scalar summaries making up the\n // main value between the bounds.\n string value = 1;\n\n // The exact tag string associated with the scalar summaries making up the\n // lower bound.\n string lower = 2;\n\n // The exact tag string associated with the scalar summaries making up the\n // upper bound.\n string upper = 3;\n }\n\n // A list of data series to include within this margin chart.\n repeated Series series = 1;\n}\n\n/**\n * A category contains a group of charts. Each category maps to a collapsible\n * within the dashboard.\n */\nmessage Category {\n // This string appears atop each grouping of charts within the dashboard.\n string title = 1;\n\n // Encapsulates data on charts to be shown in the category.\n repeated Chart chart = 2;\n\n // Whether this category should be initially closed. False by default.\n bool closed = 3;\n}\n\n/**\n * A layout encapsulates how charts are laid out within the custom scalars\n * dashboard.\n */\nmessage Layout {\n // Version `0` is the only supported version.\n int32 version = 1;\n\n // The categories here are rendered from top to bottom.\n repeated Category category = 2;\n}\n" }, { "path": "tensorboardX/tensorboardX/proto/layout_pb2.py", "content": "# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: tensorboardX/proto/layout.proto\n\nimport sys\n_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))\nfrom google.protobuf import descriptor as _descriptor\nfrom google.protobuf import message as _message\nfrom google.protobuf import reflection as _reflection\nfrom google.protobuf import symbol_database as _symbol_database\n# @@protoc_insertion_point(imports)\n\n_sym_db = _symbol_database.Default()\n\n\n\n\nDESCRIPTOR = _descriptor.FileDescriptor(\n name='tensorboardX/proto/layout.proto',\n package='tensorboardX',\n syntax='proto3',\n serialized_options=None,\n serialized_pb=_b('\\n\\x1ftensorboardX/proto/layout.proto\\x12\\x0ctensorboardX\\\"\\x8f\\x01\\n\\x05\\x43hart\\x12\\r\\n\\x05title\\x18\\x01 \\x01(\\t\\x12\\x38\\n\\tmultiline\\x18\\x02 \\x01(\\x0b\\x32#.tensorboardX.MultilineChartContentH\\x00\\x12\\x32\\n\\x06margin\\x18\\x03 \\x01(\\x0b\\x32 .tensorboardX.MarginChartContentH\\x00\\x42\\t\\n\\x07\\x63ontent\\\"$\\n\\x15MultilineChartContent\\x12\\x0b\\n\\x03tag\\x18\\x01 \\x03(\\t\\\"\\x84\\x01\\n\\x12MarginChartContent\\x12\\x37\\n\\x06series\\x18\\x01 \\x03(\\x0b\\x32\\'.tensorboardX.MarginChartContent.Series\\x1a\\x35\\n\\x06Series\\x12\\r\\n\\x05value\\x18\\x01 \\x01(\\t\\x12\\r\\n\\x05lower\\x18\\x02 \\x01(\\t\\x12\\r\\n\\x05upper\\x18\\x03 \\x01(\\t\\\"M\\n\\x08\\x43\\x61tegory\\x12\\r\\n\\x05title\\x18\\x01 \\x01(\\t\\x12\\\"\\n\\x05\\x63hart\\x18\\x02 \\x03(\\x0b\\x32\\x13.tensorboardX.Chart\\x12\\x0e\\n\\x06\\x63losed\\x18\\x03 \\x01(\\x08\\\"C\\n\\x06Layout\\x12\\x0f\\n\\x07version\\x18\\x01 \\x01(\\x05\\x12(\\n\\x08\\x63\\x61tegory\\x18\\x02 \\x03(\\x0b\\x32\\x16.tensorboardX.Categoryb\\x06proto3')\n)\n\n\n\n\n_CHART = _descriptor.Descriptor(\n name='Chart',\n full_name='tensorboardX.Chart',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='title', full_name='tensorboardX.Chart.title', index=0,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='multiline', full_name='tensorboardX.Chart.multiline', index=1,\n number=2, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='margin', full_name='tensorboardX.Chart.margin', index=2,\n number=3, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n _descriptor.OneofDescriptor(\n name='content', full_name='tensorboardX.Chart.content',\n index=0, containing_type=None, fields=[]),\n ],\n serialized_start=50,\n serialized_end=193,\n)\n\n\n_MULTILINECHARTCONTENT = _descriptor.Descriptor(\n name='MultilineChartContent',\n full_name='tensorboardX.MultilineChartContent',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='tag', full_name='tensorboardX.MultilineChartContent.tag', index=0,\n number=1, type=9, cpp_type=9, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=195,\n serialized_end=231,\n)\n\n\n_MARGINCHARTCONTENT_SERIES = _descriptor.Descriptor(\n name='Series',\n full_name='tensorboardX.MarginChartContent.Series',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='value', full_name='tensorboardX.MarginChartContent.Series.value', index=0,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='lower', full_name='tensorboardX.MarginChartContent.Series.lower', index=1,\n number=2, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='upper', full_name='tensorboardX.MarginChartContent.Series.upper', index=2,\n number=3, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=313,\n serialized_end=366,\n)\n\n_MARGINCHARTCONTENT = _descriptor.Descriptor(\n name='MarginChartContent',\n full_name='tensorboardX.MarginChartContent',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='series', full_name='tensorboardX.MarginChartContent.series', index=0,\n number=1, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[_MARGINCHARTCONTENT_SERIES, ],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=234,\n serialized_end=366,\n)\n\n\n_CATEGORY = _descriptor.Descriptor(\n name='Category',\n full_name='tensorboardX.Category',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='title', full_name='tensorboardX.Category.title', index=0,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='chart', full_name='tensorboardX.Category.chart', index=1,\n number=2, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='closed', full_name='tensorboardX.Category.closed', index=2,\n number=3, type=8, cpp_type=7, label=1,\n has_default_value=False, default_value=False,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=368,\n serialized_end=445,\n)\n\n\n_LAYOUT = _descriptor.Descriptor(\n name='Layout',\n full_name='tensorboardX.Layout',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='version', full_name='tensorboardX.Layout.version', index=0,\n number=1, type=5, cpp_type=1, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='category', full_name='tensorboardX.Layout.category', index=1,\n number=2, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=447,\n serialized_end=514,\n)\n\n_CHART.fields_by_name['multiline'].message_type = _MULTILINECHARTCONTENT\n_CHART.fields_by_name['margin'].message_type = _MARGINCHARTCONTENT\n_CHART.oneofs_by_name['content'].fields.append(\n _CHART.fields_by_name['multiline'])\n_CHART.fields_by_name['multiline'].containing_oneof = _CHART.oneofs_by_name['content']\n_CHART.oneofs_by_name['content'].fields.append(\n _CHART.fields_by_name['margin'])\n_CHART.fields_by_name['margin'].containing_oneof = _CHART.oneofs_by_name['content']\n_MARGINCHARTCONTENT_SERIES.containing_type = _MARGINCHARTCONTENT\n_MARGINCHARTCONTENT.fields_by_name['series'].message_type = _MARGINCHARTCONTENT_SERIES\n_CATEGORY.fields_by_name['chart'].message_type = _CHART\n_LAYOUT.fields_by_name['category'].message_type = _CATEGORY\nDESCRIPTOR.message_types_by_name['Chart'] = _CHART\nDESCRIPTOR.message_types_by_name['MultilineChartContent'] = _MULTILINECHARTCONTENT\nDESCRIPTOR.message_types_by_name['MarginChartContent'] = _MARGINCHARTCONTENT\nDESCRIPTOR.message_types_by_name['Category'] = _CATEGORY\nDESCRIPTOR.message_types_by_name['Layout'] = _LAYOUT\n_sym_db.RegisterFileDescriptor(DESCRIPTOR)\n\nChart = _reflection.GeneratedProtocolMessageType('Chart', (_message.Message,), dict(\n DESCRIPTOR = _CHART,\n __module__ = 'tensorboardX.proto.layout_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.Chart)\n ))\n_sym_db.RegisterMessage(Chart)\n\nMultilineChartContent = _reflection.GeneratedProtocolMessageType('MultilineChartContent', (_message.Message,), dict(\n DESCRIPTOR = _MULTILINECHARTCONTENT,\n __module__ = 'tensorboardX.proto.layout_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.MultilineChartContent)\n ))\n_sym_db.RegisterMessage(MultilineChartContent)\n\nMarginChartContent = _reflection.GeneratedProtocolMessageType('MarginChartContent', (_message.Message,), dict(\n\n Series = _reflection.GeneratedProtocolMessageType('Series', (_message.Message,), dict(\n DESCRIPTOR = _MARGINCHARTCONTENT_SERIES,\n __module__ = 'tensorboardX.proto.layout_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.MarginChartContent.Series)\n ))\n ,\n DESCRIPTOR = _MARGINCHARTCONTENT,\n __module__ = 'tensorboardX.proto.layout_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.MarginChartContent)\n ))\n_sym_db.RegisterMessage(MarginChartContent)\n_sym_db.RegisterMessage(MarginChartContent.Series)\n\nCategory = _reflection.GeneratedProtocolMessageType('Category', (_message.Message,), dict(\n DESCRIPTOR = _CATEGORY,\n __module__ = 'tensorboardX.proto.layout_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.Category)\n ))\n_sym_db.RegisterMessage(Category)\n\nLayout = _reflection.GeneratedProtocolMessageType('Layout', (_message.Message,), dict(\n DESCRIPTOR = _LAYOUT,\n __module__ = 'tensorboardX.proto.layout_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.Layout)\n ))\n_sym_db.RegisterMessage(Layout)\n\n\n# @@protoc_insertion_point(module_scope)\n" }, { "path": "tensorboardX/tensorboardX/proto/node_def.proto", "content": "syntax = \"proto3\";\n\npackage tensorboardX;\noption cc_enable_arenas = true;\noption java_outer_classname = \"NodeProto\";\noption java_multiple_files = true;\noption java_package = \"org.tensorflow.framework\";\n\nimport \"tensorboardX/proto/attr_value.proto\";\n\nmessage NodeDef {\n // The name given to this operator. Used for naming inputs,\n // logging, visualization, etc. Unique within a single GraphDef.\n // Must match the regexp \"[A-Za-z0-9.][A-Za-z0-9_./]*\".\n string name = 1;\n\n // The operation name. There may be custom parameters in attrs.\n // Op names starting with an underscore are reserved for internal use.\n string op = 2;\n\n // Each input is \"node:src_output\" with \"node\" being a string name and\n // \"src_output\" indicating which output tensor to use from \"node\". If\n // \"src_output\" is 0 the \":0\" suffix can be omitted. Regular inputs\n // may optionally be followed by control inputs that have the format\n // \"^node\".\n repeated string input = 3;\n\n // A (possibly partial) specification for the device on which this\n // node should be placed.\n // The expected syntax for this string is as follows:\n //\n // DEVICE_SPEC ::= PARTIAL_SPEC\n //\n // PARTIAL_SPEC ::= (\"/\" CONSTRAINT) *\n // CONSTRAINT ::= (\"job:\" JOB_NAME)\n // | (\"replica:\" [1-9][0-9]*)\n // | (\"task:\" [1-9][0-9]*)\n // | ( (\"gpu\" | \"cpu\") \":\" ([1-9][0-9]* | \"*\") )\n //\n // Valid values for this string include:\n // * \"/job:worker/replica:0/task:1/gpu:3\" (full specification)\n // * \"/job:worker/gpu:3\" (partial specification)\n // * \"\" (no specification)\n //\n // If the constraints do not resolve to a single device (or if this\n // field is empty or not present), the runtime will attempt to\n // choose a device automatically.\n string device = 4;\n\n // Operation-specific graph-construction-time configuration.\n // Note that this should include all attrs defined in the\n // corresponding OpDef, including those with a value matching\n // the default -- this allows the default to change and makes\n // NodeDefs easier to interpret on their own. However, if\n // an attr with a default is not specified in this list, the\n // default will be used.\n // The \"names\" (keys) must match the regexp \"[a-z][a-z0-9_]+\" (and\n // one of the names from the corresponding OpDef's attr field).\n // The values must have a type matching the corresponding OpDef\n // attr's type field.\n // TODO(josh11b): Add some examples here showing best practices.\n map attr = 5;\n};\n" }, { "path": "tensorboardX/tensorboardX/proto/node_def_pb2.py", "content": "# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: tensorboardX/proto/node_def.proto\n\nimport sys\n_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))\nfrom google.protobuf import descriptor as _descriptor\nfrom google.protobuf import message as _message\nfrom google.protobuf import reflection as _reflection\nfrom google.protobuf import symbol_database as _symbol_database\n# @@protoc_insertion_point(imports)\n\n_sym_db = _symbol_database.Default()\n\n\nfrom tensorboardX.proto import attr_value_pb2 as tensorboardX_dot_proto_dot_attr__value__pb2\n\n\nDESCRIPTOR = _descriptor.FileDescriptor(\n name='tensorboardX/proto/node_def.proto',\n package='tensorboardX',\n syntax='proto3',\n serialized_options=_b('\\n\\030org.tensorflow.frameworkB\\tNodeProtoP\\001\\370\\001\\001'),\n serialized_pb=_b('\\n!tensorboardX/proto/node_def.proto\\x12\\x0ctensorboardX\\x1a#tensorboardX/proto/attr_value.proto\\\"\\xb7\\x01\\n\\x07NodeDef\\x12\\x0c\\n\\x04name\\x18\\x01 \\x01(\\t\\x12\\n\\n\\x02op\\x18\\x02 \\x01(\\t\\x12\\r\\n\\x05input\\x18\\x03 \\x03(\\t\\x12\\x0e\\n\\x06\\x64\\x65vice\\x18\\x04 \\x01(\\t\\x12-\\n\\x04\\x61ttr\\x18\\x05 \\x03(\\x0b\\x32\\x1f.tensorboardX.NodeDef.AttrEntry\\x1a\\x44\\n\\tAttrEntry\\x12\\x0b\\n\\x03key\\x18\\x01 \\x01(\\t\\x12&\\n\\x05value\\x18\\x02 \\x01(\\x0b\\x32\\x17.tensorboardX.AttrValue:\\x02\\x38\\x01\\x42*\\n\\x18org.tensorflow.frameworkB\\tNodeProtoP\\x01\\xf8\\x01\\x01\\x62\\x06proto3')\n ,\n dependencies=[tensorboardX_dot_proto_dot_attr__value__pb2.DESCRIPTOR,])\n\n\n\n\n_NODEDEF_ATTRENTRY = _descriptor.Descriptor(\n name='AttrEntry',\n full_name='tensorboardX.NodeDef.AttrEntry',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='key', full_name='tensorboardX.NodeDef.AttrEntry.key', index=0,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='value', full_name='tensorboardX.NodeDef.AttrEntry.value', index=1,\n number=2, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=_b('8\\001'),\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=204,\n serialized_end=272,\n)\n\n_NODEDEF = _descriptor.Descriptor(\n name='NodeDef',\n full_name='tensorboardX.NodeDef',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='name', full_name='tensorboardX.NodeDef.name', index=0,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='op', full_name='tensorboardX.NodeDef.op', index=1,\n number=2, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='input', full_name='tensorboardX.NodeDef.input', index=2,\n number=3, type=9, cpp_type=9, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='device', full_name='tensorboardX.NodeDef.device', index=3,\n number=4, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='attr', full_name='tensorboardX.NodeDef.attr', index=4,\n number=5, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[_NODEDEF_ATTRENTRY, ],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=89,\n serialized_end=272,\n)\n\n_NODEDEF_ATTRENTRY.fields_by_name['value'].message_type = tensorboardX_dot_proto_dot_attr__value__pb2._ATTRVALUE\n_NODEDEF_ATTRENTRY.containing_type = _NODEDEF\n_NODEDEF.fields_by_name['attr'].message_type = _NODEDEF_ATTRENTRY\nDESCRIPTOR.message_types_by_name['NodeDef'] = _NODEDEF\n_sym_db.RegisterFileDescriptor(DESCRIPTOR)\n\nNodeDef = _reflection.GeneratedProtocolMessageType('NodeDef', (_message.Message,), dict(\n\n AttrEntry = _reflection.GeneratedProtocolMessageType('AttrEntry', (_message.Message,), dict(\n DESCRIPTOR = _NODEDEF_ATTRENTRY,\n __module__ = 'tensorboardX.proto.node_def_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.NodeDef.AttrEntry)\n ))\n ,\n DESCRIPTOR = _NODEDEF,\n __module__ = 'tensorboardX.proto.node_def_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.NodeDef)\n ))\n_sym_db.RegisterMessage(NodeDef)\n_sym_db.RegisterMessage(NodeDef.AttrEntry)\n\n\nDESCRIPTOR._options = None\n_NODEDEF_ATTRENTRY._options = None\n# @@protoc_insertion_point(module_scope)\n" }, { "path": "tensorboardX/tensorboardX/proto/plugin_hparams.proto", "content": "/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n==============================================================================*/\n\n// Defines protos for storing a hypertuning experiment data inside Summary tags.\n//\n// A hypertuning-experiment data consists of metadata that's constant\n// throughout the experiment and evolving metric data for each training session\n// in the experiment. The HParams plugin assumes the following organization of\n// this entire data set. Experiment metadata is recorded in the empty run in a\n// tag (named by the Python constant) metadata.EXPERIMENT_TAG. Within the\n// experiment, for a session named by its metadata is recorded\n// in the run in the tags metadata.SESSION_START_INFO and\n// metadata.SESSION_END_INFO. Finally, the session's metric data for a metric\n// with a (, ) name (see MetricName in api.proto), is recorded\n// in a Scalar-plugin summary with tag in the run .\n\nsyntax = \"proto3\";\n\nimport \"tensorboardX/proto/api.proto\";\nimport \"google/protobuf/struct.proto\";\n\npackage tensorboardX.hparam;\n\n// HParam summaries created by `tensorboard.plugins.hparams.summary`\n// module will include `SummaryMetadata` whose `plugin_data` field has\n// as `content` a serialized HParamsPluginData message.\nmessage HParamsPluginData {\n // The version of the plugin data schema.\n int32 version = 1;\n oneof data {\n Experiment experiment = 2;\n SessionStartInfo session_start_info = 3;\n SessionEndInfo session_end_info = 4;\n }\n}\n\nmessage SessionStartInfo {\n // A map describing the hyperparameter values for the session.\n // Maps each hyperparameter name to its value.\n // Currently only scalars are supported.\n map hparams = 1;\n\n // A URI for where checkpoints are saved.\n string model_uri = 2;\n\n // An optional URL to a website monitoring the session.\n string monitor_url = 3;\n\n // The name of the session group containing this session. If empty, the\n // group name is taken to be the session id (so this session is the only\n // member of its group).\n string group_name = 4;\n\n // The time the session started in seconds since epoch.\n double start_time_secs = 5;\n}\n\nmessage SessionEndInfo {\n Status status = 1;\n\n // The time the session ended in seconds since epoch.\n double end_time_secs = 2;\n}\n" }, { "path": "tensorboardX/tensorboardX/proto/plugin_hparams_pb2.py", "content": "# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: tensorboardX/proto/plugin_hparams.proto\n\nimport sys\n_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))\nfrom google.protobuf import descriptor as _descriptor\nfrom google.protobuf import message as _message\nfrom google.protobuf import reflection as _reflection\nfrom google.protobuf import symbol_database as _symbol_database\n# @@protoc_insertion_point(imports)\n\n_sym_db = _symbol_database.Default()\n\n\nfrom tensorboardX.proto import api_pb2 as tensorboardX_dot_proto_dot_api__pb2\nfrom google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2\n\n\nDESCRIPTOR = _descriptor.FileDescriptor(\n name='tensorboardX/proto/plugin_hparams.proto',\n package='tensorboardX.hparam',\n syntax='proto3',\n serialized_options=None,\n serialized_pb=_b('\\n\\'tensorboardX/proto/plugin_hparams.proto\\x12\\x13tensorboardX.hparam\\x1a\\x1ctensorboardX/proto/api.proto\\x1a\\x1cgoogle/protobuf/struct.proto\\\"\\xe9\\x01\\n\\x11HParamsPluginData\\x12\\x0f\\n\\x07version\\x18\\x01 \\x01(\\x05\\x12\\x35\\n\\nexperiment\\x18\\x02 \\x01(\\x0b\\x32\\x1f.tensorboardX.hparam.ExperimentH\\x00\\x12\\x43\\n\\x12session_start_info\\x18\\x03 \\x01(\\x0b\\x32%.tensorboardX.hparam.SessionStartInfoH\\x00\\x12?\\n\\x10session_end_info\\x18\\x04 \\x01(\\x0b\\x32#.tensorboardX.hparam.SessionEndInfoH\\x00\\x42\\x06\\n\\x04\\x64\\x61ta\\\"\\xf4\\x01\\n\\x10SessionStartInfo\\x12\\x43\\n\\x07hparams\\x18\\x01 \\x03(\\x0b\\x32\\x32.tensorboardX.hparam.SessionStartInfo.HparamsEntry\\x12\\x11\\n\\tmodel_uri\\x18\\x02 \\x01(\\t\\x12\\x13\\n\\x0bmonitor_url\\x18\\x03 \\x01(\\t\\x12\\x12\\n\\ngroup_name\\x18\\x04 \\x01(\\t\\x12\\x17\\n\\x0fstart_time_secs\\x18\\x05 \\x01(\\x01\\x1a\\x46\\n\\x0cHparamsEntry\\x12\\x0b\\n\\x03key\\x18\\x01 \\x01(\\t\\x12%\\n\\x05value\\x18\\x02 \\x01(\\x0b\\x32\\x16.google.protobuf.Value:\\x02\\x38\\x01\\\"T\\n\\x0eSessionEndInfo\\x12+\\n\\x06status\\x18\\x01 \\x01(\\x0e\\x32\\x1b.tensorboardX.hparam.Status\\x12\\x15\\n\\rend_time_secs\\x18\\x02 \\x01(\\x01\\x62\\x06proto3')\n ,\n dependencies=[tensorboardX_dot_proto_dot_api__pb2.DESCRIPTOR,google_dot_protobuf_dot_struct__pb2.DESCRIPTOR,])\n\n\n\n\n_HPARAMSPLUGINDATA = _descriptor.Descriptor(\n name='HParamsPluginData',\n full_name='tensorboardX.hparam.HParamsPluginData',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='version', full_name='tensorboardX.hparam.HParamsPluginData.version', index=0,\n number=1, type=5, cpp_type=1, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='experiment', full_name='tensorboardX.hparam.HParamsPluginData.experiment', index=1,\n number=2, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='session_start_info', full_name='tensorboardX.hparam.HParamsPluginData.session_start_info', index=2,\n number=3, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='session_end_info', full_name='tensorboardX.hparam.HParamsPluginData.session_end_info', index=3,\n number=4, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n _descriptor.OneofDescriptor(\n name='data', full_name='tensorboardX.hparam.HParamsPluginData.data',\n index=0, containing_type=None, fields=[]),\n ],\n serialized_start=125,\n serialized_end=358,\n)\n\n\n_SESSIONSTARTINFO_HPARAMSENTRY = _descriptor.Descriptor(\n name='HparamsEntry',\n full_name='tensorboardX.hparam.SessionStartInfo.HparamsEntry',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='key', full_name='tensorboardX.hparam.SessionStartInfo.HparamsEntry.key', index=0,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='value', full_name='tensorboardX.hparam.SessionStartInfo.HparamsEntry.value', index=1,\n number=2, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=_b('8\\001'),\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=535,\n serialized_end=605,\n)\n\n_SESSIONSTARTINFO = _descriptor.Descriptor(\n name='SessionStartInfo',\n full_name='tensorboardX.hparam.SessionStartInfo',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='hparams', full_name='tensorboardX.hparam.SessionStartInfo.hparams', index=0,\n number=1, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='model_uri', full_name='tensorboardX.hparam.SessionStartInfo.model_uri', index=1,\n number=2, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='monitor_url', full_name='tensorboardX.hparam.SessionStartInfo.monitor_url', index=2,\n number=3, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='group_name', full_name='tensorboardX.hparam.SessionStartInfo.group_name', index=3,\n number=4, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='start_time_secs', full_name='tensorboardX.hparam.SessionStartInfo.start_time_secs', index=4,\n number=5, type=1, cpp_type=5, label=1,\n has_default_value=False, default_value=float(0),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[_SESSIONSTARTINFO_HPARAMSENTRY, ],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=361,\n serialized_end=605,\n)\n\n\n_SESSIONENDINFO = _descriptor.Descriptor(\n name='SessionEndInfo',\n full_name='tensorboardX.hparam.SessionEndInfo',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='status', full_name='tensorboardX.hparam.SessionEndInfo.status', index=0,\n number=1, type=14, cpp_type=8, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='end_time_secs', full_name='tensorboardX.hparam.SessionEndInfo.end_time_secs', index=1,\n number=2, type=1, cpp_type=5, label=1,\n has_default_value=False, default_value=float(0),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=607,\n serialized_end=691,\n)\n\n_HPARAMSPLUGINDATA.fields_by_name['experiment'].message_type = tensorboardX_dot_proto_dot_api__pb2._EXPERIMENT\n_HPARAMSPLUGINDATA.fields_by_name['session_start_info'].message_type = _SESSIONSTARTINFO\n_HPARAMSPLUGINDATA.fields_by_name['session_end_info'].message_type = _SESSIONENDINFO\n_HPARAMSPLUGINDATA.oneofs_by_name['data'].fields.append(\n _HPARAMSPLUGINDATA.fields_by_name['experiment'])\n_HPARAMSPLUGINDATA.fields_by_name['experiment'].containing_oneof = _HPARAMSPLUGINDATA.oneofs_by_name['data']\n_HPARAMSPLUGINDATA.oneofs_by_name['data'].fields.append(\n _HPARAMSPLUGINDATA.fields_by_name['session_start_info'])\n_HPARAMSPLUGINDATA.fields_by_name['session_start_info'].containing_oneof = _HPARAMSPLUGINDATA.oneofs_by_name['data']\n_HPARAMSPLUGINDATA.oneofs_by_name['data'].fields.append(\n _HPARAMSPLUGINDATA.fields_by_name['session_end_info'])\n_HPARAMSPLUGINDATA.fields_by_name['session_end_info'].containing_oneof = _HPARAMSPLUGINDATA.oneofs_by_name['data']\n_SESSIONSTARTINFO_HPARAMSENTRY.fields_by_name['value'].message_type = google_dot_protobuf_dot_struct__pb2._VALUE\n_SESSIONSTARTINFO_HPARAMSENTRY.containing_type = _SESSIONSTARTINFO\n_SESSIONSTARTINFO.fields_by_name['hparams'].message_type = _SESSIONSTARTINFO_HPARAMSENTRY\n_SESSIONENDINFO.fields_by_name['status'].enum_type = tensorboardX_dot_proto_dot_api__pb2._STATUS\nDESCRIPTOR.message_types_by_name['HParamsPluginData'] = _HPARAMSPLUGINDATA\nDESCRIPTOR.message_types_by_name['SessionStartInfo'] = _SESSIONSTARTINFO\nDESCRIPTOR.message_types_by_name['SessionEndInfo'] = _SESSIONENDINFO\n_sym_db.RegisterFileDescriptor(DESCRIPTOR)\n\nHParamsPluginData = _reflection.GeneratedProtocolMessageType('HParamsPluginData', (_message.Message,), dict(\n DESCRIPTOR = _HPARAMSPLUGINDATA,\n __module__ = 'tensorboardX.proto.plugin_hparams_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.hparam.HParamsPluginData)\n ))\n_sym_db.RegisterMessage(HParamsPluginData)\n\nSessionStartInfo = _reflection.GeneratedProtocolMessageType('SessionStartInfo', (_message.Message,), dict(\n\n HparamsEntry = _reflection.GeneratedProtocolMessageType('HparamsEntry', (_message.Message,), dict(\n DESCRIPTOR = _SESSIONSTARTINFO_HPARAMSENTRY,\n __module__ = 'tensorboardX.proto.plugin_hparams_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.hparam.SessionStartInfo.HparamsEntry)\n ))\n ,\n DESCRIPTOR = _SESSIONSTARTINFO,\n __module__ = 'tensorboardX.proto.plugin_hparams_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.hparam.SessionStartInfo)\n ))\n_sym_db.RegisterMessage(SessionStartInfo)\n_sym_db.RegisterMessage(SessionStartInfo.HparamsEntry)\n\nSessionEndInfo = _reflection.GeneratedProtocolMessageType('SessionEndInfo', (_message.Message,), dict(\n DESCRIPTOR = _SESSIONENDINFO,\n __module__ = 'tensorboardX.proto.plugin_hparams_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.hparam.SessionEndInfo)\n ))\n_sym_db.RegisterMessage(SessionEndInfo)\n\n\n_SESSIONSTARTINFO_HPARAMSENTRY._options = None\n# @@protoc_insertion_point(module_scope)\n" }, { "path": "tensorboardX/tensorboardX/proto/plugin_mesh.proto", "content": "syntax = \"proto3\";\n\npackage tensorboardX.mesh;\n\n// A MeshPluginData encapsulates information on which plugins are able to make\n// use of a certain summary value.\nmessage MeshPluginData {\n enum ContentType {\n UNDEFINED = 0;\n VERTEX = 1;\n FACE = 2; // Triangle face.\n COLOR = 3;\n }\n\n // Version `0` is the only supported version.\n int32 version = 1;\n\n // The name of the mesh summary this particular summary belongs to.\n string name = 2;\n\n // Type of data in the summary.\n ContentType content_type = 3;\n\n // JSON-serialized dictionary of ThreeJS classes configuration.\n string json_config = 5;\n\n // Shape of underlying data. Cache it here for performance reasons.\n repeated int32 shape = 6;\n}\n" }, { "path": "tensorboardX/tensorboardX/proto/plugin_mesh_pb2.py", "content": "# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: tensorboardX/proto/plugin_mesh.proto\n\nimport sys\n_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))\nfrom google.protobuf import descriptor as _descriptor\nfrom google.protobuf import message as _message\nfrom google.protobuf import reflection as _reflection\nfrom google.protobuf import symbol_database as _symbol_database\n# @@protoc_insertion_point(imports)\n\n_sym_db = _symbol_database.Default()\n\n\n\n\nDESCRIPTOR = _descriptor.FileDescriptor(\n name='tensorboardX/proto/plugin_mesh.proto',\n package='tensorboardX.mesh',\n syntax='proto3',\n serialized_options=None,\n serialized_pb=_b('\\n$tensorboardX/proto/plugin_mesh.proto\\x12\\x11tensorboardX.mesh\\\"\\xd7\\x01\\n\\x0eMeshPluginData\\x12\\x0f\\n\\x07version\\x18\\x01 \\x01(\\x05\\x12\\x0c\\n\\x04name\\x18\\x02 \\x01(\\t\\x12\\x43\\n\\x0c\\x63ontent_type\\x18\\x03 \\x01(\\x0e\\x32-.tensorboardX.mesh.MeshPluginData.ContentType\\x12\\x13\\n\\x0bjson_config\\x18\\x05 \\x01(\\t\\x12\\r\\n\\x05shape\\x18\\x06 \\x03(\\x05\\\"=\\n\\x0b\\x43ontentType\\x12\\r\\n\\tUNDEFINED\\x10\\x00\\x12\\n\\n\\x06VERTEX\\x10\\x01\\x12\\x08\\n\\x04\\x46\\x41\\x43\\x45\\x10\\x02\\x12\\t\\n\\x05\\x43OLOR\\x10\\x03\\x62\\x06proto3')\n)\n\n\n\n_MESHPLUGINDATA_CONTENTTYPE = _descriptor.EnumDescriptor(\n name='ContentType',\n full_name='tensorboardX.mesh.MeshPluginData.ContentType',\n filename=None,\n file=DESCRIPTOR,\n values=[\n _descriptor.EnumValueDescriptor(\n name='UNDEFINED', index=0, number=0,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='VERTEX', index=1, number=1,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='FACE', index=2, number=2,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='COLOR', index=3, number=3,\n serialized_options=None,\n type=None),\n ],\n containing_type=None,\n serialized_options=None,\n serialized_start=214,\n serialized_end=275,\n)\n_sym_db.RegisterEnumDescriptor(_MESHPLUGINDATA_CONTENTTYPE)\n\n\n_MESHPLUGINDATA = _descriptor.Descriptor(\n name='MeshPluginData',\n full_name='tensorboardX.mesh.MeshPluginData',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='version', full_name='tensorboardX.mesh.MeshPluginData.version', index=0,\n number=1, type=5, cpp_type=1, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='name', full_name='tensorboardX.mesh.MeshPluginData.name', index=1,\n number=2, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='content_type', full_name='tensorboardX.mesh.MeshPluginData.content_type', index=2,\n number=3, type=14, cpp_type=8, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='json_config', full_name='tensorboardX.mesh.MeshPluginData.json_config', index=3,\n number=5, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='shape', full_name='tensorboardX.mesh.MeshPluginData.shape', index=4,\n number=6, type=5, cpp_type=1, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n _MESHPLUGINDATA_CONTENTTYPE,\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=60,\n serialized_end=275,\n)\n\n_MESHPLUGINDATA.fields_by_name['content_type'].enum_type = _MESHPLUGINDATA_CONTENTTYPE\n_MESHPLUGINDATA_CONTENTTYPE.containing_type = _MESHPLUGINDATA\nDESCRIPTOR.message_types_by_name['MeshPluginData'] = _MESHPLUGINDATA\n_sym_db.RegisterFileDescriptor(DESCRIPTOR)\n\nMeshPluginData = _reflection.GeneratedProtocolMessageType('MeshPluginData', (_message.Message,), dict(\n DESCRIPTOR = _MESHPLUGINDATA,\n __module__ = 'tensorboardX.proto.plugin_mesh_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.mesh.MeshPluginData)\n ))\n_sym_db.RegisterMessage(MeshPluginData)\n\n\n# @@protoc_insertion_point(module_scope)\n" }, { "path": "tensorboardX/tensorboardX/proto/plugin_pr_curve.proto", "content": "/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n==============================================================================*/\n\nsyntax = \"proto3\";\n\npackage tensorboardX;\n\nmessage PrCurvePluginData {\n // Version `0` is the only supported version.\n int32 version = 1;\n\n uint32 num_thresholds = 2;\n}\n" }, { "path": "tensorboardX/tensorboardX/proto/plugin_pr_curve_pb2.py", "content": "# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: tensorboardX/proto/plugin_pr_curve.proto\n\nimport sys\n_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))\nfrom google.protobuf import descriptor as _descriptor\nfrom google.protobuf import message as _message\nfrom google.protobuf import reflection as _reflection\nfrom google.protobuf import symbol_database as _symbol_database\n# @@protoc_insertion_point(imports)\n\n_sym_db = _symbol_database.Default()\n\n\n\n\nDESCRIPTOR = _descriptor.FileDescriptor(\n name='tensorboardX/proto/plugin_pr_curve.proto',\n package='tensorboardX',\n syntax='proto3',\n serialized_options=None,\n serialized_pb=_b('\\n(tensorboardX/proto/plugin_pr_curve.proto\\x12\\x0ctensorboardX\\\"<\\n\\x11PrCurvePluginData\\x12\\x0f\\n\\x07version\\x18\\x01 \\x01(\\x05\\x12\\x16\\n\\x0enum_thresholds\\x18\\x02 \\x01(\\rb\\x06proto3')\n)\n\n\n\n\n_PRCURVEPLUGINDATA = _descriptor.Descriptor(\n name='PrCurvePluginData',\n full_name='tensorboardX.PrCurvePluginData',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='version', full_name='tensorboardX.PrCurvePluginData.version', index=0,\n number=1, type=5, cpp_type=1, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='num_thresholds', full_name='tensorboardX.PrCurvePluginData.num_thresholds', index=1,\n number=2, type=13, cpp_type=3, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=58,\n serialized_end=118,\n)\n\nDESCRIPTOR.message_types_by_name['PrCurvePluginData'] = _PRCURVEPLUGINDATA\n_sym_db.RegisterFileDescriptor(DESCRIPTOR)\n\nPrCurvePluginData = _reflection.GeneratedProtocolMessageType('PrCurvePluginData', (_message.Message,), dict(\n DESCRIPTOR = _PRCURVEPLUGINDATA,\n __module__ = 'tensorboardX.proto.plugin_pr_curve_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.PrCurvePluginData)\n ))\n_sym_db.RegisterMessage(PrCurvePluginData)\n\n\n# @@protoc_insertion_point(module_scope)\n" }, { "path": "tensorboardX/tensorboardX/proto/plugin_text.proto", "content": "/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n==============================================================================*/\n\nsyntax = \"proto3\";\n\npackage tensorboardX;\n\n// Text summaries created by the `tensorboard.plugins.text.summary`\n// module will include `SummaryMetadata` whose `plugin_data` field has\n// as `content` a binary string that is the encoding of an\n// `TextPluginData` proto.\nmessage TextPluginData {\n // Version `0` is the only supported version.\n int32 version = 1;\n}\n" }, { "path": "tensorboardX/tensorboardX/proto/plugin_text_pb2.py", "content": "# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: tensorboardX/proto/plugin_text.proto\n\nimport sys\n_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))\nfrom google.protobuf import descriptor as _descriptor\nfrom google.protobuf import message as _message\nfrom google.protobuf import reflection as _reflection\nfrom google.protobuf import symbol_database as _symbol_database\n# @@protoc_insertion_point(imports)\n\n_sym_db = _symbol_database.Default()\n\n\n\n\nDESCRIPTOR = _descriptor.FileDescriptor(\n name='tensorboardX/proto/plugin_text.proto',\n package='tensorboardX',\n syntax='proto3',\n serialized_options=None,\n serialized_pb=_b('\\n$tensorboardX/proto/plugin_text.proto\\x12\\x0ctensorboardX\\\"!\\n\\x0eTextPluginData\\x12\\x0f\\n\\x07version\\x18\\x01 \\x01(\\x05\\x62\\x06proto3')\n)\n\n\n\n\n_TEXTPLUGINDATA = _descriptor.Descriptor(\n name='TextPluginData',\n full_name='tensorboardX.TextPluginData',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='version', full_name='tensorboardX.TextPluginData.version', index=0,\n number=1, type=5, cpp_type=1, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=54,\n serialized_end=87,\n)\n\nDESCRIPTOR.message_types_by_name['TextPluginData'] = _TEXTPLUGINDATA\n_sym_db.RegisterFileDescriptor(DESCRIPTOR)\n\nTextPluginData = _reflection.GeneratedProtocolMessageType('TextPluginData', (_message.Message,), dict(\n DESCRIPTOR = _TEXTPLUGINDATA,\n __module__ = 'tensorboardX.proto.plugin_text_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.TextPluginData)\n ))\n_sym_db.RegisterMessage(TextPluginData)\n\n\n# @@protoc_insertion_point(module_scope)\n" }, { "path": "tensorboardX/tensorboardX/proto/resource_handle.proto", "content": "syntax = \"proto3\";\n\npackage tensorboardX;\noption cc_enable_arenas = true;\noption java_outer_classname = \"ResourceHandle\";\noption java_multiple_files = true;\noption java_package = \"org.tensorflow.framework\";\n\n// Protocol buffer representing a handle to a tensorflow resource. Handles are\n// not valid across executions, but can be serialized back and forth from within\n// a single run.\nmessage ResourceHandleProto {\n // Unique name for the device containing the resource.\n string device = 1;\n\n // Container in which this resource is placed.\n string container = 2;\n\n // Unique name of this resource.\n string name = 3;\n\n // Hash code for the type of the resource. Is only valid in the same device\n // and in the same execution.\n uint64 hash_code = 4;\n\n // For debug-only, the name of the type pointed to by this handle, if\n // available.\n string maybe_type_name = 5;\n};\n" }, { "path": "tensorboardX/tensorboardX/proto/resource_handle_pb2.py", "content": "# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: tensorboardX/proto/resource_handle.proto\n\nimport sys\n_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))\nfrom google.protobuf import descriptor as _descriptor\nfrom google.protobuf import message as _message\nfrom google.protobuf import reflection as _reflection\nfrom google.protobuf import symbol_database as _symbol_database\n# @@protoc_insertion_point(imports)\n\n_sym_db = _symbol_database.Default()\n\n\n\n\nDESCRIPTOR = _descriptor.FileDescriptor(\n name='tensorboardX/proto/resource_handle.proto',\n package='tensorboardX',\n syntax='proto3',\n serialized_options=_b('\\n\\030org.tensorflow.frameworkB\\016ResourceHandleP\\001\\370\\001\\001'),\n serialized_pb=_b('\\n(tensorboardX/proto/resource_handle.proto\\x12\\x0ctensorboardX\\\"r\\n\\x13ResourceHandleProto\\x12\\x0e\\n\\x06\\x64\\x65vice\\x18\\x01 \\x01(\\t\\x12\\x11\\n\\tcontainer\\x18\\x02 \\x01(\\t\\x12\\x0c\\n\\x04name\\x18\\x03 \\x01(\\t\\x12\\x11\\n\\thash_code\\x18\\x04 \\x01(\\x04\\x12\\x17\\n\\x0fmaybe_type_name\\x18\\x05 \\x01(\\tB/\\n\\x18org.tensorflow.frameworkB\\x0eResourceHandleP\\x01\\xf8\\x01\\x01\\x62\\x06proto3')\n)\n\n\n\n\n_RESOURCEHANDLEPROTO = _descriptor.Descriptor(\n name='ResourceHandleProto',\n full_name='tensorboardX.ResourceHandleProto',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='device', full_name='tensorboardX.ResourceHandleProto.device', index=0,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='container', full_name='tensorboardX.ResourceHandleProto.container', index=1,\n number=2, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='name', full_name='tensorboardX.ResourceHandleProto.name', index=2,\n number=3, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='hash_code', full_name='tensorboardX.ResourceHandleProto.hash_code', index=3,\n number=4, type=4, cpp_type=4, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='maybe_type_name', full_name='tensorboardX.ResourceHandleProto.maybe_type_name', index=4,\n number=5, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=58,\n serialized_end=172,\n)\n\nDESCRIPTOR.message_types_by_name['ResourceHandleProto'] = _RESOURCEHANDLEPROTO\n_sym_db.RegisterFileDescriptor(DESCRIPTOR)\n\nResourceHandleProto = _reflection.GeneratedProtocolMessageType('ResourceHandleProto', (_message.Message,), dict(\n DESCRIPTOR = _RESOURCEHANDLEPROTO,\n __module__ = 'tensorboardX.proto.resource_handle_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.ResourceHandleProto)\n ))\n_sym_db.RegisterMessage(ResourceHandleProto)\n\n\nDESCRIPTOR._options = None\n# @@protoc_insertion_point(module_scope)\n" }, { "path": "tensorboardX/tensorboardX/proto/step_stats.proto", "content": "syntax = \"proto3\";\n\npackage tensorboardX;\noption cc_enable_arenas = true;\noption java_outer_classname = \"StepStatsProtos\";\noption java_multiple_files = true;\noption java_package = \"org.tensorflow.framework\";\noption go_package = \"github.com/tensorflow/tensorflow/tensorflow/go/core/framework\";\n//import \"tensorflow/core/framework/allocation_description.proto\";\n//import \"tensorflow/core/framework/tensor_description.proto\";\n\n// An allocation/de-allocation operation performed by the allocator.\nmessage AllocationRecord {\n // The timestamp of the operation.\n int64 alloc_micros = 1;\n // Number of bytes allocated, or de-allocated if negative.\n int64 alloc_bytes = 2;\n}\n\nmessage AllocatorMemoryUsed {\n string allocator_name = 1;\n // These are per-node allocator memory stats.\n int64 total_bytes = 2;\n int64 peak_bytes = 3;\n // The bytes that are not deallocated.\n int64 live_bytes = 4;\n // The allocation and deallocation timeline.\n repeated AllocationRecord allocation_records = 6;\n\n // These are snapshots of the overall allocator memory stats.\n // The number of live bytes currently allocated by the allocator.\n int64 allocator_bytes_in_use = 5;\n}\n\n// Output sizes recorded for a single execution of a graph node.\nmessage NodeOutput {\n int32 slot = 1;\n // TensorDescription tensor_description = 3;\n};\n\n// For memory tracking.\nmessage MemoryStats {\n int64 temp_memory_size = 1;\n int64 persistent_memory_size = 3;\n repeated int64 persistent_tensor_alloc_ids = 5;\n\n int64 device_temp_memory_size = 2 [deprecated = true];\n int64 device_persistent_memory_size = 4 [deprecated = true];\n repeated int64 device_persistent_tensor_alloc_ids = 6 [deprecated = true];\n}\n\n// Time/size stats recorded for a single execution of a graph node.\nmessage NodeExecStats {\n // TODO(tucker): Use some more compact form of node identity than\n // the full string name. Either all processes should agree on a\n // global id (cost_id?) for each node, or we should use a hash of\n // the name.\n string node_name = 1;\n int64 all_start_micros = 2;\n int64 op_start_rel_micros = 3;\n int64 op_end_rel_micros = 4;\n int64 all_end_rel_micros = 5;\n repeated AllocatorMemoryUsed memory = 6;\n repeated NodeOutput output = 7;\n string timeline_label = 8;\n int64 scheduled_micros = 9;\n uint32 thread_id = 10;\n // repeated AllocationDescription referenced_tensor = 11;\n MemoryStats memory_stats = 12;\n};\n\nmessage DeviceStepStats {\n string device = 1;\n repeated NodeExecStats node_stats = 2;\n}\n\nmessage StepStats {\n repeated DeviceStepStats dev_stats = 1;\n};\n\n\n// lanpa, copied from config.proto\n// Metadata output (i.e., non-Tensor) for a single Run() call.\nmessage RunMetadata {\n // Statistics traced for this step. Populated if tracing is turned on via the\n // \"RunOptions\" proto.\n // EXPERIMENTAL: The format and set of events may change in future versions.\n StepStats step_stats = 1;\n\n // The cost graph for the computation defined by the run call.\n // CostGraphDef cost_graph = 2;\n\n // Graphs of the partitions executed by executors.\n // repeated GraphDef partition_graphs = 3;\n}\n" }, { "path": "tensorboardX/tensorboardX/proto/step_stats_pb2.py", "content": "# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: tensorboardX/proto/step_stats.proto\n\nimport sys\n_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))\nfrom google.protobuf import descriptor as _descriptor\nfrom google.protobuf import message as _message\nfrom google.protobuf import reflection as _reflection\nfrom google.protobuf import symbol_database as _symbol_database\n# @@protoc_insertion_point(imports)\n\n_sym_db = _symbol_database.Default()\n\n\n\n\nDESCRIPTOR = _descriptor.FileDescriptor(\n name='tensorboardX/proto/step_stats.proto',\n package='tensorboardX',\n syntax='proto3',\n serialized_options=_b('\\n\\030org.tensorflow.frameworkB\\017StepStatsProtosP\\001Z=github.com/tensorflow/tensorflow/tensorflow/go/core/framework\\370\\001\\001'),\n serialized_pb=_b('\\n#tensorboardX/proto/step_stats.proto\\x12\\x0ctensorboardX\\\"=\\n\\x10\\x41llocationRecord\\x12\\x14\\n\\x0c\\x61lloc_micros\\x18\\x01 \\x01(\\x03\\x12\\x13\\n\\x0b\\x61lloc_bytes\\x18\\x02 \\x01(\\x03\\\"\\xc6\\x01\\n\\x13\\x41llocatorMemoryUsed\\x12\\x16\\n\\x0e\\x61llocator_name\\x18\\x01 \\x01(\\t\\x12\\x13\\n\\x0btotal_bytes\\x18\\x02 \\x01(\\x03\\x12\\x12\\n\\npeak_bytes\\x18\\x03 \\x01(\\x03\\x12\\x12\\n\\nlive_bytes\\x18\\x04 \\x01(\\x03\\x12:\\n\\x12\\x61llocation_records\\x18\\x06 \\x03(\\x0b\\x32\\x1e.tensorboardX.AllocationRecord\\x12\\x1e\\n\\x16\\x61llocator_bytes_in_use\\x18\\x05 \\x01(\\x03\\\"\\x1a\\n\\nNodeOutput\\x12\\x0c\\n\\x04slot\\x18\\x01 \\x01(\\x05\\\"\\xec\\x01\\n\\x0bMemoryStats\\x12\\x18\\n\\x10temp_memory_size\\x18\\x01 \\x01(\\x03\\x12\\x1e\\n\\x16persistent_memory_size\\x18\\x03 \\x01(\\x03\\x12#\\n\\x1bpersistent_tensor_alloc_ids\\x18\\x05 \\x03(\\x03\\x12#\\n\\x17\\x64\\x65vice_temp_memory_size\\x18\\x02 \\x01(\\x03\\x42\\x02\\x18\\x01\\x12)\\n\\x1d\\x64\\x65vice_persistent_memory_size\\x18\\x04 \\x01(\\x03\\x42\\x02\\x18\\x01\\x12.\\n\\\"device_persistent_tensor_alloc_ids\\x18\\x06 \\x03(\\x03\\x42\\x02\\x18\\x01\\\"\\xe3\\x02\\n\\rNodeExecStats\\x12\\x11\\n\\tnode_name\\x18\\x01 \\x01(\\t\\x12\\x18\\n\\x10\\x61ll_start_micros\\x18\\x02 \\x01(\\x03\\x12\\x1b\\n\\x13op_start_rel_micros\\x18\\x03 \\x01(\\x03\\x12\\x19\\n\\x11op_end_rel_micros\\x18\\x04 \\x01(\\x03\\x12\\x1a\\n\\x12\\x61ll_end_rel_micros\\x18\\x05 \\x01(\\x03\\x12\\x31\\n\\x06memory\\x18\\x06 \\x03(\\x0b\\x32!.tensorboardX.AllocatorMemoryUsed\\x12(\\n\\x06output\\x18\\x07 \\x03(\\x0b\\x32\\x18.tensorboardX.NodeOutput\\x12\\x16\\n\\x0etimeline_label\\x18\\x08 \\x01(\\t\\x12\\x18\\n\\x10scheduled_micros\\x18\\t \\x01(\\x03\\x12\\x11\\n\\tthread_id\\x18\\n \\x01(\\r\\x12/\\n\\x0cmemory_stats\\x18\\x0c \\x01(\\x0b\\x32\\x19.tensorboardX.MemoryStats\\\"R\\n\\x0f\\x44\\x65viceStepStats\\x12\\x0e\\n\\x06\\x64\\x65vice\\x18\\x01 \\x01(\\t\\x12/\\n\\nnode_stats\\x18\\x02 \\x03(\\x0b\\x32\\x1b.tensorboardX.NodeExecStats\\\"=\\n\\tStepStats\\x12\\x30\\n\\tdev_stats\\x18\\x01 \\x03(\\x0b\\x32\\x1d.tensorboardX.DeviceStepStats\\\":\\n\\x0bRunMetadata\\x12+\\n\\nstep_stats\\x18\\x01 \\x01(\\x0b\\x32\\x17.tensorboardX.StepStatsBo\\n\\x18org.tensorflow.frameworkB\\x0fStepStatsProtosP\\x01Z=github.com/tensorflow/tensorflow/tensorflow/go/core/framework\\xf8\\x01\\x01\\x62\\x06proto3')\n)\n\n\n\n\n_ALLOCATIONRECORD = _descriptor.Descriptor(\n name='AllocationRecord',\n full_name='tensorboardX.AllocationRecord',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='alloc_micros', full_name='tensorboardX.AllocationRecord.alloc_micros', index=0,\n number=1, type=3, cpp_type=2, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='alloc_bytes', full_name='tensorboardX.AllocationRecord.alloc_bytes', index=1,\n number=2, type=3, cpp_type=2, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=53,\n serialized_end=114,\n)\n\n\n_ALLOCATORMEMORYUSED = _descriptor.Descriptor(\n name='AllocatorMemoryUsed',\n full_name='tensorboardX.AllocatorMemoryUsed',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='allocator_name', full_name='tensorboardX.AllocatorMemoryUsed.allocator_name', index=0,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='total_bytes', full_name='tensorboardX.AllocatorMemoryUsed.total_bytes', index=1,\n number=2, type=3, cpp_type=2, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='peak_bytes', full_name='tensorboardX.AllocatorMemoryUsed.peak_bytes', index=2,\n number=3, type=3, cpp_type=2, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='live_bytes', full_name='tensorboardX.AllocatorMemoryUsed.live_bytes', index=3,\n number=4, type=3, cpp_type=2, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='allocation_records', full_name='tensorboardX.AllocatorMemoryUsed.allocation_records', index=4,\n number=6, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='allocator_bytes_in_use', full_name='tensorboardX.AllocatorMemoryUsed.allocator_bytes_in_use', index=5,\n number=5, type=3, cpp_type=2, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=117,\n serialized_end=315,\n)\n\n\n_NODEOUTPUT = _descriptor.Descriptor(\n name='NodeOutput',\n full_name='tensorboardX.NodeOutput',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='slot', full_name='tensorboardX.NodeOutput.slot', index=0,\n number=1, type=5, cpp_type=1, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=317,\n serialized_end=343,\n)\n\n\n_MEMORYSTATS = _descriptor.Descriptor(\n name='MemoryStats',\n full_name='tensorboardX.MemoryStats',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='temp_memory_size', full_name='tensorboardX.MemoryStats.temp_memory_size', index=0,\n number=1, type=3, cpp_type=2, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='persistent_memory_size', full_name='tensorboardX.MemoryStats.persistent_memory_size', index=1,\n number=3, type=3, cpp_type=2, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='persistent_tensor_alloc_ids', full_name='tensorboardX.MemoryStats.persistent_tensor_alloc_ids', index=2,\n number=5, type=3, cpp_type=2, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='device_temp_memory_size', full_name='tensorboardX.MemoryStats.device_temp_memory_size', index=3,\n number=2, type=3, cpp_type=2, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=_b('\\030\\001'), file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='device_persistent_memory_size', full_name='tensorboardX.MemoryStats.device_persistent_memory_size', index=4,\n number=4, type=3, cpp_type=2, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=_b('\\030\\001'), file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='device_persistent_tensor_alloc_ids', full_name='tensorboardX.MemoryStats.device_persistent_tensor_alloc_ids', index=5,\n number=6, type=3, cpp_type=2, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=_b('\\030\\001'), file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=346,\n serialized_end=582,\n)\n\n\n_NODEEXECSTATS = _descriptor.Descriptor(\n name='NodeExecStats',\n full_name='tensorboardX.NodeExecStats',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='node_name', full_name='tensorboardX.NodeExecStats.node_name', index=0,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='all_start_micros', full_name='tensorboardX.NodeExecStats.all_start_micros', index=1,\n number=2, type=3, cpp_type=2, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='op_start_rel_micros', full_name='tensorboardX.NodeExecStats.op_start_rel_micros', index=2,\n number=3, type=3, cpp_type=2, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='op_end_rel_micros', full_name='tensorboardX.NodeExecStats.op_end_rel_micros', index=3,\n number=4, type=3, cpp_type=2, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='all_end_rel_micros', full_name='tensorboardX.NodeExecStats.all_end_rel_micros', index=4,\n number=5, type=3, cpp_type=2, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='memory', full_name='tensorboardX.NodeExecStats.memory', index=5,\n number=6, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='output', full_name='tensorboardX.NodeExecStats.output', index=6,\n number=7, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='timeline_label', full_name='tensorboardX.NodeExecStats.timeline_label', index=7,\n number=8, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='scheduled_micros', full_name='tensorboardX.NodeExecStats.scheduled_micros', index=8,\n number=9, type=3, cpp_type=2, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='thread_id', full_name='tensorboardX.NodeExecStats.thread_id', index=9,\n number=10, type=13, cpp_type=3, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='memory_stats', full_name='tensorboardX.NodeExecStats.memory_stats', index=10,\n number=12, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=585,\n serialized_end=940,\n)\n\n\n_DEVICESTEPSTATS = _descriptor.Descriptor(\n name='DeviceStepStats',\n full_name='tensorboardX.DeviceStepStats',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='device', full_name='tensorboardX.DeviceStepStats.device', index=0,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='node_stats', full_name='tensorboardX.DeviceStepStats.node_stats', index=1,\n number=2, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=942,\n serialized_end=1024,\n)\n\n\n_STEPSTATS = _descriptor.Descriptor(\n name='StepStats',\n full_name='tensorboardX.StepStats',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='dev_stats', full_name='tensorboardX.StepStats.dev_stats', index=0,\n number=1, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=1026,\n serialized_end=1087,\n)\n\n\n_RUNMETADATA = _descriptor.Descriptor(\n name='RunMetadata',\n full_name='tensorboardX.RunMetadata',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='step_stats', full_name='tensorboardX.RunMetadata.step_stats', index=0,\n number=1, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=1089,\n serialized_end=1147,\n)\n\n_ALLOCATORMEMORYUSED.fields_by_name['allocation_records'].message_type = _ALLOCATIONRECORD\n_NODEEXECSTATS.fields_by_name['memory'].message_type = _ALLOCATORMEMORYUSED\n_NODEEXECSTATS.fields_by_name['output'].message_type = _NODEOUTPUT\n_NODEEXECSTATS.fields_by_name['memory_stats'].message_type = _MEMORYSTATS\n_DEVICESTEPSTATS.fields_by_name['node_stats'].message_type = _NODEEXECSTATS\n_STEPSTATS.fields_by_name['dev_stats'].message_type = _DEVICESTEPSTATS\n_RUNMETADATA.fields_by_name['step_stats'].message_type = _STEPSTATS\nDESCRIPTOR.message_types_by_name['AllocationRecord'] = _ALLOCATIONRECORD\nDESCRIPTOR.message_types_by_name['AllocatorMemoryUsed'] = _ALLOCATORMEMORYUSED\nDESCRIPTOR.message_types_by_name['NodeOutput'] = _NODEOUTPUT\nDESCRIPTOR.message_types_by_name['MemoryStats'] = _MEMORYSTATS\nDESCRIPTOR.message_types_by_name['NodeExecStats'] = _NODEEXECSTATS\nDESCRIPTOR.message_types_by_name['DeviceStepStats'] = _DEVICESTEPSTATS\nDESCRIPTOR.message_types_by_name['StepStats'] = _STEPSTATS\nDESCRIPTOR.message_types_by_name['RunMetadata'] = _RUNMETADATA\n_sym_db.RegisterFileDescriptor(DESCRIPTOR)\n\nAllocationRecord = _reflection.GeneratedProtocolMessageType('AllocationRecord', (_message.Message,), dict(\n DESCRIPTOR = _ALLOCATIONRECORD,\n __module__ = 'tensorboardX.proto.step_stats_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.AllocationRecord)\n ))\n_sym_db.RegisterMessage(AllocationRecord)\n\nAllocatorMemoryUsed = _reflection.GeneratedProtocolMessageType('AllocatorMemoryUsed', (_message.Message,), dict(\n DESCRIPTOR = _ALLOCATORMEMORYUSED,\n __module__ = 'tensorboardX.proto.step_stats_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.AllocatorMemoryUsed)\n ))\n_sym_db.RegisterMessage(AllocatorMemoryUsed)\n\nNodeOutput = _reflection.GeneratedProtocolMessageType('NodeOutput', (_message.Message,), dict(\n DESCRIPTOR = _NODEOUTPUT,\n __module__ = 'tensorboardX.proto.step_stats_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.NodeOutput)\n ))\n_sym_db.RegisterMessage(NodeOutput)\n\nMemoryStats = _reflection.GeneratedProtocolMessageType('MemoryStats', (_message.Message,), dict(\n DESCRIPTOR = _MEMORYSTATS,\n __module__ = 'tensorboardX.proto.step_stats_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.MemoryStats)\n ))\n_sym_db.RegisterMessage(MemoryStats)\n\nNodeExecStats = _reflection.GeneratedProtocolMessageType('NodeExecStats', (_message.Message,), dict(\n DESCRIPTOR = _NODEEXECSTATS,\n __module__ = 'tensorboardX.proto.step_stats_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.NodeExecStats)\n ))\n_sym_db.RegisterMessage(NodeExecStats)\n\nDeviceStepStats = _reflection.GeneratedProtocolMessageType('DeviceStepStats', (_message.Message,), dict(\n DESCRIPTOR = _DEVICESTEPSTATS,\n __module__ = 'tensorboardX.proto.step_stats_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.DeviceStepStats)\n ))\n_sym_db.RegisterMessage(DeviceStepStats)\n\nStepStats = _reflection.GeneratedProtocolMessageType('StepStats', (_message.Message,), dict(\n DESCRIPTOR = _STEPSTATS,\n __module__ = 'tensorboardX.proto.step_stats_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.StepStats)\n ))\n_sym_db.RegisterMessage(StepStats)\n\nRunMetadata = _reflection.GeneratedProtocolMessageType('RunMetadata', (_message.Message,), dict(\n DESCRIPTOR = _RUNMETADATA,\n __module__ = 'tensorboardX.proto.step_stats_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.RunMetadata)\n ))\n_sym_db.RegisterMessage(RunMetadata)\n\n\nDESCRIPTOR._options = None\n_MEMORYSTATS.fields_by_name['device_temp_memory_size']._options = None\n_MEMORYSTATS.fields_by_name['device_persistent_memory_size']._options = None\n_MEMORYSTATS.fields_by_name['device_persistent_tensor_alloc_ids']._options = None\n# @@protoc_insertion_point(module_scope)\n" }, { "path": "tensorboardX/tensorboardX/proto/summary.proto", "content": "syntax = \"proto3\";\n\npackage tensorboardX;\noption cc_enable_arenas = true;\noption java_outer_classname = \"SummaryProtos\";\noption java_multiple_files = true;\noption java_package = \"org.tensorflow.framework\";\n\nimport \"tensorboardX/proto/tensor.proto\";\n\n// Metadata associated with a series of Summary data\nmessage SummaryDescription {\n // Hint on how plugins should process the data in this series.\n // Supported values include \"scalar\", \"histogram\", \"image\", \"audio\"\n string type_hint = 1;\n}\n\n// Serialization format for histogram module in\n// core/lib/histogram/histogram.h\nmessage HistogramProto {\n double min = 1;\n double max = 2;\n double num = 3;\n double sum = 4;\n double sum_squares = 5;\n\n // Parallel arrays encoding the bucket boundaries and the bucket values.\n // bucket(i) is the count for the bucket i. The range for\n // a bucket is:\n // i == 0: -DBL_MAX .. bucket_limit(0)\n // i != 0: bucket_limit(i-1) .. bucket_limit(i)\n repeated double bucket_limit = 6 [packed = true];\n repeated double bucket = 7 [packed = true];\n};\n\n// A SummaryMetadata encapsulates information on which plugins are able to make\n// use of a certain summary value.\nmessage SummaryMetadata {\n message PluginData {\n // The name of the plugin this data pertains to.\n string plugin_name = 1;\n\n // The content to store for the plugin. The best practice is for this to be\n // a binary serialized protocol buffer.\n bytes content = 2;\n }\n\n // Data that associates a summary with a certain plugin.\n PluginData plugin_data = 1;\n\n // Display name for viewing in TensorBoard.\n string display_name = 2;\n\n // Longform readable description of the summary sequence. Markdown supported.\n string summary_description = 3;\n};\n\n// A Summary is a set of named values to be displayed by the\n// visualizer.\n//\n// Summaries are produced regularly during training, as controlled by\n// the \"summary_interval_secs\" attribute of the training operation.\n// Summaries are also produced at the end of an evaluation.\nmessage Summary {\n message Image {\n // Dimensions of the image.\n int32 height = 1;\n int32 width = 2;\n // Valid colorspace values are\n // 1 - grayscale\n // 2 - grayscale + alpha\n // 3 - RGB\n // 4 - RGBA\n // 5 - DIGITAL_YUV\n // 6 - BGRA\n int32 colorspace = 3;\n // Image data in encoded format. All image formats supported by\n // image_codec::CoderUtil can be stored here.\n bytes encoded_image_string = 4;\n }\n\n message Audio {\n // Sample rate of the audio in Hz.\n float sample_rate = 1;\n // Number of channels of audio.\n int64 num_channels = 2;\n // Length of the audio in frames (samples per channel).\n int64 length_frames = 3;\n // Encoded audio data and its associated RFC 2045 content type (e.g.\n // \"audio/wav\").\n bytes encoded_audio_string = 4;\n string content_type = 5;\n }\n\n message Value {\n // This field is deprecated and will not be set.\n string node_name = 7;\n\n // Tag name for the data. Used by TensorBoard plugins to organize data. Tags\n // are often organized by scope (which contains slashes to convey\n // hierarchy). For example: foo/bar/0\n string tag = 1;\n\n // Contains metadata on the summary value such as which plugins may use it.\n // Take note that many summary values may lack a metadata field. This is\n // because the FileWriter only keeps a metadata object on the first summary\n // value with a certain tag for each tag. TensorBoard then remembers which\n // tags are associated with which plugins. This saves space.\n SummaryMetadata metadata = 9;\n\n // Value associated with the tag.\n oneof value {\n float simple_value = 2;\n bytes obsolete_old_style_histogram = 3;\n Image image = 4;\n HistogramProto histo = 5;\n Audio audio = 6;\n TensorProto tensor = 8;\n }\n }\n\n // Set of values for the summary.\n repeated Value value = 1;\n}\n" }, { "path": "tensorboardX/tensorboardX/proto/summary_pb2.py", "content": "# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: tensorboardX/proto/summary.proto\n\nimport sys\n_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))\nfrom google.protobuf import descriptor as _descriptor\nfrom google.protobuf import message as _message\nfrom google.protobuf import reflection as _reflection\nfrom google.protobuf import symbol_database as _symbol_database\n# @@protoc_insertion_point(imports)\n\n_sym_db = _symbol_database.Default()\n\n\nfrom tensorboardX.proto import tensor_pb2 as tensorboardX_dot_proto_dot_tensor__pb2\n\n\nDESCRIPTOR = _descriptor.FileDescriptor(\n name='tensorboardX/proto/summary.proto',\n package='tensorboardX',\n syntax='proto3',\n serialized_options=_b('\\n\\030org.tensorflow.frameworkB\\rSummaryProtosP\\001\\370\\001\\001'),\n serialized_pb=_b('\\n tensorboardX/proto/summary.proto\\x12\\x0ctensorboardX\\x1a\\x1ftensorboardX/proto/tensor.proto\\\"\\'\\n\\x12SummaryDescription\\x12\\x11\\n\\ttype_hint\\x18\\x01 \\x01(\\t\\\"\\x87\\x01\\n\\x0eHistogramProto\\x12\\x0b\\n\\x03min\\x18\\x01 \\x01(\\x01\\x12\\x0b\\n\\x03max\\x18\\x02 \\x01(\\x01\\x12\\x0b\\n\\x03num\\x18\\x03 \\x01(\\x01\\x12\\x0b\\n\\x03sum\\x18\\x04 \\x01(\\x01\\x12\\x13\\n\\x0bsum_squares\\x18\\x05 \\x01(\\x01\\x12\\x18\\n\\x0c\\x62ucket_limit\\x18\\x06 \\x03(\\x01\\x42\\x02\\x10\\x01\\x12\\x12\\n\\x06\\x62ucket\\x18\\x07 \\x03(\\x01\\x42\\x02\\x10\\x01\\\"\\xb7\\x01\\n\\x0fSummaryMetadata\\x12=\\n\\x0bplugin_data\\x18\\x01 \\x01(\\x0b\\x32(.tensorboardX.SummaryMetadata.PluginData\\x12\\x14\\n\\x0c\\x64isplay_name\\x18\\x02 \\x01(\\t\\x12\\x1b\\n\\x13summary_description\\x18\\x03 \\x01(\\t\\x1a\\x32\\n\\nPluginData\\x12\\x13\\n\\x0bplugin_name\\x18\\x01 \\x01(\\t\\x12\\x0f\\n\\x07\\x63ontent\\x18\\x02 \\x01(\\x0c\\\"\\xea\\x04\\n\\x07Summary\\x12*\\n\\x05value\\x18\\x01 \\x03(\\x0b\\x32\\x1b.tensorboardX.Summary.Value\\x1aX\\n\\x05Image\\x12\\x0e\\n\\x06height\\x18\\x01 \\x01(\\x05\\x12\\r\\n\\x05width\\x18\\x02 \\x01(\\x05\\x12\\x12\\n\\ncolorspace\\x18\\x03 \\x01(\\x05\\x12\\x1c\\n\\x14\\x65ncoded_image_string\\x18\\x04 \\x01(\\x0c\\x1a}\\n\\x05\\x41udio\\x12\\x13\\n\\x0bsample_rate\\x18\\x01 \\x01(\\x02\\x12\\x14\\n\\x0cnum_channels\\x18\\x02 \\x01(\\x03\\x12\\x15\\n\\rlength_frames\\x18\\x03 \\x01(\\x03\\x12\\x1c\\n\\x14\\x65ncoded_audio_string\\x18\\x04 \\x01(\\x0c\\x12\\x14\\n\\x0c\\x63ontent_type\\x18\\x05 \\x01(\\t\\x1a\\xd9\\x02\\n\\x05Value\\x12\\x11\\n\\tnode_name\\x18\\x07 \\x01(\\t\\x12\\x0b\\n\\x03tag\\x18\\x01 \\x01(\\t\\x12/\\n\\x08metadata\\x18\\t \\x01(\\x0b\\x32\\x1d.tensorboardX.SummaryMetadata\\x12\\x16\\n\\x0csimple_value\\x18\\x02 \\x01(\\x02H\\x00\\x12&\\n\\x1cobsolete_old_style_histogram\\x18\\x03 \\x01(\\x0cH\\x00\\x12,\\n\\x05image\\x18\\x04 \\x01(\\x0b\\x32\\x1b.tensorboardX.Summary.ImageH\\x00\\x12-\\n\\x05histo\\x18\\x05 \\x01(\\x0b\\x32\\x1c.tensorboardX.HistogramProtoH\\x00\\x12,\\n\\x05\\x61udio\\x18\\x06 \\x01(\\x0b\\x32\\x1b.tensorboardX.Summary.AudioH\\x00\\x12+\\n\\x06tensor\\x18\\x08 \\x01(\\x0b\\x32\\x19.tensorboardX.TensorProtoH\\x00\\x42\\x07\\n\\x05valueB.\\n\\x18org.tensorflow.frameworkB\\rSummaryProtosP\\x01\\xf8\\x01\\x01\\x62\\x06proto3')\n ,\n dependencies=[tensorboardX_dot_proto_dot_tensor__pb2.DESCRIPTOR,])\n\n\n\n\n_SUMMARYDESCRIPTION = _descriptor.Descriptor(\n name='SummaryDescription',\n full_name='tensorboardX.SummaryDescription',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='type_hint', full_name='tensorboardX.SummaryDescription.type_hint', index=0,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=83,\n serialized_end=122,\n)\n\n\n_HISTOGRAMPROTO = _descriptor.Descriptor(\n name='HistogramProto',\n full_name='tensorboardX.HistogramProto',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='min', full_name='tensorboardX.HistogramProto.min', index=0,\n number=1, type=1, cpp_type=5, label=1,\n has_default_value=False, default_value=float(0),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='max', full_name='tensorboardX.HistogramProto.max', index=1,\n number=2, type=1, cpp_type=5, label=1,\n has_default_value=False, default_value=float(0),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='num', full_name='tensorboardX.HistogramProto.num', index=2,\n number=3, type=1, cpp_type=5, label=1,\n has_default_value=False, default_value=float(0),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='sum', full_name='tensorboardX.HistogramProto.sum', index=3,\n number=4, type=1, cpp_type=5, label=1,\n has_default_value=False, default_value=float(0),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='sum_squares', full_name='tensorboardX.HistogramProto.sum_squares', index=4,\n number=5, type=1, cpp_type=5, label=1,\n has_default_value=False, default_value=float(0),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='bucket_limit', full_name='tensorboardX.HistogramProto.bucket_limit', index=5,\n number=6, type=1, cpp_type=5, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=_b('\\020\\001'), file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='bucket', full_name='tensorboardX.HistogramProto.bucket', index=6,\n number=7, type=1, cpp_type=5, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=_b('\\020\\001'), file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=125,\n serialized_end=260,\n)\n\n\n_SUMMARYMETADATA_PLUGINDATA = _descriptor.Descriptor(\n name='PluginData',\n full_name='tensorboardX.SummaryMetadata.PluginData',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='plugin_name', full_name='tensorboardX.SummaryMetadata.PluginData.plugin_name', index=0,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='content', full_name='tensorboardX.SummaryMetadata.PluginData.content', index=1,\n number=2, type=12, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\"),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=396,\n serialized_end=446,\n)\n\n_SUMMARYMETADATA = _descriptor.Descriptor(\n name='SummaryMetadata',\n full_name='tensorboardX.SummaryMetadata',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='plugin_data', full_name='tensorboardX.SummaryMetadata.plugin_data', index=0,\n number=1, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='display_name', full_name='tensorboardX.SummaryMetadata.display_name', index=1,\n number=2, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='summary_description', full_name='tensorboardX.SummaryMetadata.summary_description', index=2,\n number=3, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[_SUMMARYMETADATA_PLUGINDATA, ],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=263,\n serialized_end=446,\n)\n\n\n_SUMMARY_IMAGE = _descriptor.Descriptor(\n name='Image',\n full_name='tensorboardX.Summary.Image',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='height', full_name='tensorboardX.Summary.Image.height', index=0,\n number=1, type=5, cpp_type=1, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='width', full_name='tensorboardX.Summary.Image.width', index=1,\n number=2, type=5, cpp_type=1, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='colorspace', full_name='tensorboardX.Summary.Image.colorspace', index=2,\n number=3, type=5, cpp_type=1, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='encoded_image_string', full_name='tensorboardX.Summary.Image.encoded_image_string', index=3,\n number=4, type=12, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\"),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=504,\n serialized_end=592,\n)\n\n_SUMMARY_AUDIO = _descriptor.Descriptor(\n name='Audio',\n full_name='tensorboardX.Summary.Audio',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='sample_rate', full_name='tensorboardX.Summary.Audio.sample_rate', index=0,\n number=1, type=2, cpp_type=6, label=1,\n has_default_value=False, default_value=float(0),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='num_channels', full_name='tensorboardX.Summary.Audio.num_channels', index=1,\n number=2, type=3, cpp_type=2, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='length_frames', full_name='tensorboardX.Summary.Audio.length_frames', index=2,\n number=3, type=3, cpp_type=2, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='encoded_audio_string', full_name='tensorboardX.Summary.Audio.encoded_audio_string', index=3,\n number=4, type=12, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\"),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='content_type', full_name='tensorboardX.Summary.Audio.content_type', index=4,\n number=5, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=594,\n serialized_end=719,\n)\n\n_SUMMARY_VALUE = _descriptor.Descriptor(\n name='Value',\n full_name='tensorboardX.Summary.Value',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='node_name', full_name='tensorboardX.Summary.Value.node_name', index=0,\n number=7, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='tag', full_name='tensorboardX.Summary.Value.tag', index=1,\n number=1, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='metadata', full_name='tensorboardX.Summary.Value.metadata', index=2,\n number=9, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='simple_value', full_name='tensorboardX.Summary.Value.simple_value', index=3,\n number=2, type=2, cpp_type=6, label=1,\n has_default_value=False, default_value=float(0),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='obsolete_old_style_histogram', full_name='tensorboardX.Summary.Value.obsolete_old_style_histogram', index=4,\n number=3, type=12, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\"),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='image', full_name='tensorboardX.Summary.Value.image', index=5,\n number=4, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='histo', full_name='tensorboardX.Summary.Value.histo', index=6,\n number=5, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='audio', full_name='tensorboardX.Summary.Value.audio', index=7,\n number=6, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='tensor', full_name='tensorboardX.Summary.Value.tensor', index=8,\n number=8, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n _descriptor.OneofDescriptor(\n name='value', full_name='tensorboardX.Summary.Value.value',\n index=0, containing_type=None, fields=[]),\n ],\n serialized_start=722,\n serialized_end=1067,\n)\n\n_SUMMARY = _descriptor.Descriptor(\n name='Summary',\n full_name='tensorboardX.Summary',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='value', full_name='tensorboardX.Summary.value', index=0,\n number=1, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[_SUMMARY_IMAGE, _SUMMARY_AUDIO, _SUMMARY_VALUE, ],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=449,\n serialized_end=1067,\n)\n\n_SUMMARYMETADATA_PLUGINDATA.containing_type = _SUMMARYMETADATA\n_SUMMARYMETADATA.fields_by_name['plugin_data'].message_type = _SUMMARYMETADATA_PLUGINDATA\n_SUMMARY_IMAGE.containing_type = _SUMMARY\n_SUMMARY_AUDIO.containing_type = _SUMMARY\n_SUMMARY_VALUE.fields_by_name['metadata'].message_type = _SUMMARYMETADATA\n_SUMMARY_VALUE.fields_by_name['image'].message_type = _SUMMARY_IMAGE\n_SUMMARY_VALUE.fields_by_name['histo'].message_type = _HISTOGRAMPROTO\n_SUMMARY_VALUE.fields_by_name['audio'].message_type = _SUMMARY_AUDIO\n_SUMMARY_VALUE.fields_by_name['tensor'].message_type = tensorboardX_dot_proto_dot_tensor__pb2._TENSORPROTO\n_SUMMARY_VALUE.containing_type = _SUMMARY\n_SUMMARY_VALUE.oneofs_by_name['value'].fields.append(\n _SUMMARY_VALUE.fields_by_name['simple_value'])\n_SUMMARY_VALUE.fields_by_name['simple_value'].containing_oneof = _SUMMARY_VALUE.oneofs_by_name['value']\n_SUMMARY_VALUE.oneofs_by_name['value'].fields.append(\n _SUMMARY_VALUE.fields_by_name['obsolete_old_style_histogram'])\n_SUMMARY_VALUE.fields_by_name['obsolete_old_style_histogram'].containing_oneof = _SUMMARY_VALUE.oneofs_by_name['value']\n_SUMMARY_VALUE.oneofs_by_name['value'].fields.append(\n _SUMMARY_VALUE.fields_by_name['image'])\n_SUMMARY_VALUE.fields_by_name['image'].containing_oneof = _SUMMARY_VALUE.oneofs_by_name['value']\n_SUMMARY_VALUE.oneofs_by_name['value'].fields.append(\n _SUMMARY_VALUE.fields_by_name['histo'])\n_SUMMARY_VALUE.fields_by_name['histo'].containing_oneof = _SUMMARY_VALUE.oneofs_by_name['value']\n_SUMMARY_VALUE.oneofs_by_name['value'].fields.append(\n _SUMMARY_VALUE.fields_by_name['audio'])\n_SUMMARY_VALUE.fields_by_name['audio'].containing_oneof = _SUMMARY_VALUE.oneofs_by_name['value']\n_SUMMARY_VALUE.oneofs_by_name['value'].fields.append(\n _SUMMARY_VALUE.fields_by_name['tensor'])\n_SUMMARY_VALUE.fields_by_name['tensor'].containing_oneof = _SUMMARY_VALUE.oneofs_by_name['value']\n_SUMMARY.fields_by_name['value'].message_type = _SUMMARY_VALUE\nDESCRIPTOR.message_types_by_name['SummaryDescription'] = _SUMMARYDESCRIPTION\nDESCRIPTOR.message_types_by_name['HistogramProto'] = _HISTOGRAMPROTO\nDESCRIPTOR.message_types_by_name['SummaryMetadata'] = _SUMMARYMETADATA\nDESCRIPTOR.message_types_by_name['Summary'] = _SUMMARY\n_sym_db.RegisterFileDescriptor(DESCRIPTOR)\n\nSummaryDescription = _reflection.GeneratedProtocolMessageType('SummaryDescription', (_message.Message,), dict(\n DESCRIPTOR = _SUMMARYDESCRIPTION,\n __module__ = 'tensorboardX.proto.summary_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.SummaryDescription)\n ))\n_sym_db.RegisterMessage(SummaryDescription)\n\nHistogramProto = _reflection.GeneratedProtocolMessageType('HistogramProto', (_message.Message,), dict(\n DESCRIPTOR = _HISTOGRAMPROTO,\n __module__ = 'tensorboardX.proto.summary_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.HistogramProto)\n ))\n_sym_db.RegisterMessage(HistogramProto)\n\nSummaryMetadata = _reflection.GeneratedProtocolMessageType('SummaryMetadata', (_message.Message,), dict(\n\n PluginData = _reflection.GeneratedProtocolMessageType('PluginData', (_message.Message,), dict(\n DESCRIPTOR = _SUMMARYMETADATA_PLUGINDATA,\n __module__ = 'tensorboardX.proto.summary_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.SummaryMetadata.PluginData)\n ))\n ,\n DESCRIPTOR = _SUMMARYMETADATA,\n __module__ = 'tensorboardX.proto.summary_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.SummaryMetadata)\n ))\n_sym_db.RegisterMessage(SummaryMetadata)\n_sym_db.RegisterMessage(SummaryMetadata.PluginData)\n\nSummary = _reflection.GeneratedProtocolMessageType('Summary', (_message.Message,), dict(\n\n Image = _reflection.GeneratedProtocolMessageType('Image', (_message.Message,), dict(\n DESCRIPTOR = _SUMMARY_IMAGE,\n __module__ = 'tensorboardX.proto.summary_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.Summary.Image)\n ))\n ,\n\n Audio = _reflection.GeneratedProtocolMessageType('Audio', (_message.Message,), dict(\n DESCRIPTOR = _SUMMARY_AUDIO,\n __module__ = 'tensorboardX.proto.summary_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.Summary.Audio)\n ))\n ,\n\n Value = _reflection.GeneratedProtocolMessageType('Value', (_message.Message,), dict(\n DESCRIPTOR = _SUMMARY_VALUE,\n __module__ = 'tensorboardX.proto.summary_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.Summary.Value)\n ))\n ,\n DESCRIPTOR = _SUMMARY,\n __module__ = 'tensorboardX.proto.summary_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.Summary)\n ))\n_sym_db.RegisterMessage(Summary)\n_sym_db.RegisterMessage(Summary.Image)\n_sym_db.RegisterMessage(Summary.Audio)\n_sym_db.RegisterMessage(Summary.Value)\n\n\nDESCRIPTOR._options = None\n_HISTOGRAMPROTO.fields_by_name['bucket_limit']._options = None\n_HISTOGRAMPROTO.fields_by_name['bucket']._options = None\n# @@protoc_insertion_point(module_scope)\n" }, { "path": "tensorboardX/tensorboardX/proto/tensor.proto", "content": "syntax = \"proto3\";\n\npackage tensorboardX;\noption cc_enable_arenas = true;\noption java_outer_classname = \"TensorProtos\";\noption java_multiple_files = true;\noption java_package = \"org.tensorflow.framework\";\n\nimport \"tensorboardX/proto/resource_handle.proto\";\nimport \"tensorboardX/proto/tensor_shape.proto\";\nimport \"tensorboardX/proto/types.proto\";\n\n// Protocol buffer representing a tensor.\nmessage TensorProto {\n DataType dtype = 1;\n\n // Shape of the tensor. TODO(touts): sort out the 0-rank issues.\n TensorShapeProto tensor_shape = 2;\n\n // Only one of the representations below is set, one of \"tensor_contents\" and\n // the \"xxx_val\" attributes. We are not using oneof because as oneofs cannot\n // contain repeated fields it would require another extra set of messages.\n\n // Version number.\n //\n // In version 0, if the \"repeated xxx\" representations contain only one\n // element, that element is repeated to fill the shape. This makes it easy\n // to represent a constant Tensor with a single value.\n int32 version_number = 3;\n\n // Serialized raw tensor content from either Tensor::AsProtoTensorContent or\n // memcpy in tensorflow::grpc::EncodeTensorToByteBuffer. This representation\n // can be used for all tensor types. The purpose of this representation is to\n // reduce serialization overhead during RPC call by avoiding serialization of\n // many repeated small items.\n bytes tensor_content = 4;\n\n // Type specific representations that make it easy to create tensor protos in\n // all languages. Only the representation corresponding to \"dtype\" can\n // be set. The values hold the flattened representation of the tensor in\n // row major order.\n\n // DT_HALF. Note that since protobuf has no int16 type, we'll have some\n // pointless zero padding for each value here.\n repeated int32 half_val = 13 [packed = true];\n\n // DT_FLOAT.\n repeated float float_val = 5 [packed = true];\n\n // DT_DOUBLE.\n repeated double double_val = 6 [packed = true];\n\n // DT_INT32, DT_INT16, DT_INT8, DT_UINT8.\n repeated int32 int_val = 7 [packed = true];\n\n // DT_STRING\n repeated bytes string_val = 8;\n\n // DT_COMPLEX64. scomplex_val(2*i) and scomplex_val(2*i+1) are real\n // and imaginary parts of i-th single precision complex.\n repeated float scomplex_val = 9 [packed = true];\n\n // DT_INT64\n repeated int64 int64_val = 10 [packed = true];\n\n // DT_BOOL\n repeated bool bool_val = 11 [packed = true];\n\n // DT_COMPLEX128. dcomplex_val(2*i) and dcomplex_val(2*i+1) are real\n // and imaginary parts of i-th double precision complex.\n repeated double dcomplex_val = 12 [packed = true];\n\n // DT_RESOURCE\n repeated ResourceHandleProto resource_handle_val = 14;\n};\n" }, { "path": "tensorboardX/tensorboardX/proto/tensor_pb2.py", "content": "# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: tensorboardX/proto/tensor.proto\n\nimport sys\n_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))\nfrom google.protobuf import descriptor as _descriptor\nfrom google.protobuf import message as _message\nfrom google.protobuf import reflection as _reflection\nfrom google.protobuf import symbol_database as _symbol_database\n# @@protoc_insertion_point(imports)\n\n_sym_db = _symbol_database.Default()\n\n\nfrom tensorboardX.proto import resource_handle_pb2 as tensorboardX_dot_proto_dot_resource__handle__pb2\nfrom tensorboardX.proto import tensor_shape_pb2 as tensorboardX_dot_proto_dot_tensor__shape__pb2\nfrom tensorboardX.proto import types_pb2 as tensorboardX_dot_proto_dot_types__pb2\n\n\nDESCRIPTOR = _descriptor.FileDescriptor(\n name='tensorboardX/proto/tensor.proto',\n package='tensorboardX',\n syntax='proto3',\n serialized_options=_b('\\n\\030org.tensorflow.frameworkB\\014TensorProtosP\\001\\370\\001\\001'),\n serialized_pb=_b('\\n\\x1ftensorboardX/proto/tensor.proto\\x12\\x0ctensorboardX\\x1a(tensorboardX/proto/resource_handle.proto\\x1a%tensorboardX/proto/tensor_shape.proto\\x1a\\x1etensorboardX/proto/types.proto\\\"\\xa9\\x03\\n\\x0bTensorProto\\x12%\\n\\x05\\x64type\\x18\\x01 \\x01(\\x0e\\x32\\x16.tensorboardX.DataType\\x12\\x34\\n\\x0ctensor_shape\\x18\\x02 \\x01(\\x0b\\x32\\x1e.tensorboardX.TensorShapeProto\\x12\\x16\\n\\x0eversion_number\\x18\\x03 \\x01(\\x05\\x12\\x16\\n\\x0etensor_content\\x18\\x04 \\x01(\\x0c\\x12\\x14\\n\\x08half_val\\x18\\r \\x03(\\x05\\x42\\x02\\x10\\x01\\x12\\x15\\n\\tfloat_val\\x18\\x05 \\x03(\\x02\\x42\\x02\\x10\\x01\\x12\\x16\\n\\ndouble_val\\x18\\x06 \\x03(\\x01\\x42\\x02\\x10\\x01\\x12\\x13\\n\\x07int_val\\x18\\x07 \\x03(\\x05\\x42\\x02\\x10\\x01\\x12\\x12\\n\\nstring_val\\x18\\x08 \\x03(\\x0c\\x12\\x18\\n\\x0cscomplex_val\\x18\\t \\x03(\\x02\\x42\\x02\\x10\\x01\\x12\\x15\\n\\tint64_val\\x18\\n \\x03(\\x03\\x42\\x02\\x10\\x01\\x12\\x14\\n\\x08\\x62ool_val\\x18\\x0b \\x03(\\x08\\x42\\x02\\x10\\x01\\x12\\x18\\n\\x0c\\x64\\x63omplex_val\\x18\\x0c \\x03(\\x01\\x42\\x02\\x10\\x01\\x12>\\n\\x13resource_handle_val\\x18\\x0e \\x03(\\x0b\\x32!.tensorboardX.ResourceHandleProtoB-\\n\\x18org.tensorflow.frameworkB\\x0cTensorProtosP\\x01\\xf8\\x01\\x01\\x62\\x06proto3')\n ,\n dependencies=[tensorboardX_dot_proto_dot_resource__handle__pb2.DESCRIPTOR,tensorboardX_dot_proto_dot_tensor__shape__pb2.DESCRIPTOR,tensorboardX_dot_proto_dot_types__pb2.DESCRIPTOR,])\n\n\n\n\n_TENSORPROTO = _descriptor.Descriptor(\n name='TensorProto',\n full_name='tensorboardX.TensorProto',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='dtype', full_name='tensorboardX.TensorProto.dtype', index=0,\n number=1, type=14, cpp_type=8, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='tensor_shape', full_name='tensorboardX.TensorProto.tensor_shape', index=1,\n number=2, type=11, cpp_type=10, label=1,\n has_default_value=False, default_value=None,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='version_number', full_name='tensorboardX.TensorProto.version_number', index=2,\n number=3, type=5, cpp_type=1, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='tensor_content', full_name='tensorboardX.TensorProto.tensor_content', index=3,\n number=4, type=12, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\"),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='half_val', full_name='tensorboardX.TensorProto.half_val', index=4,\n number=13, type=5, cpp_type=1, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=_b('\\020\\001'), file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='float_val', full_name='tensorboardX.TensorProto.float_val', index=5,\n number=5, type=2, cpp_type=6, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=_b('\\020\\001'), file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='double_val', full_name='tensorboardX.TensorProto.double_val', index=6,\n number=6, type=1, cpp_type=5, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=_b('\\020\\001'), file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='int_val', full_name='tensorboardX.TensorProto.int_val', index=7,\n number=7, type=5, cpp_type=1, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=_b('\\020\\001'), file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='string_val', full_name='tensorboardX.TensorProto.string_val', index=8,\n number=8, type=12, cpp_type=9, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='scomplex_val', full_name='tensorboardX.TensorProto.scomplex_val', index=9,\n number=9, type=2, cpp_type=6, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=_b('\\020\\001'), file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='int64_val', full_name='tensorboardX.TensorProto.int64_val', index=10,\n number=10, type=3, cpp_type=2, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=_b('\\020\\001'), file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='bool_val', full_name='tensorboardX.TensorProto.bool_val', index=11,\n number=11, type=8, cpp_type=7, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=_b('\\020\\001'), file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='dcomplex_val', full_name='tensorboardX.TensorProto.dcomplex_val', index=12,\n number=12, type=1, cpp_type=5, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=_b('\\020\\001'), file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='resource_handle_val', full_name='tensorboardX.TensorProto.resource_handle_val', index=13,\n number=14, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=163,\n serialized_end=588,\n)\n\n_TENSORPROTO.fields_by_name['dtype'].enum_type = tensorboardX_dot_proto_dot_types__pb2._DATATYPE\n_TENSORPROTO.fields_by_name['tensor_shape'].message_type = tensorboardX_dot_proto_dot_tensor__shape__pb2._TENSORSHAPEPROTO\n_TENSORPROTO.fields_by_name['resource_handle_val'].message_type = tensorboardX_dot_proto_dot_resource__handle__pb2._RESOURCEHANDLEPROTO\nDESCRIPTOR.message_types_by_name['TensorProto'] = _TENSORPROTO\n_sym_db.RegisterFileDescriptor(DESCRIPTOR)\n\nTensorProto = _reflection.GeneratedProtocolMessageType('TensorProto', (_message.Message,), dict(\n DESCRIPTOR = _TENSORPROTO,\n __module__ = 'tensorboardX.proto.tensor_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.TensorProto)\n ))\n_sym_db.RegisterMessage(TensorProto)\n\n\nDESCRIPTOR._options = None\n_TENSORPROTO.fields_by_name['half_val']._options = None\n_TENSORPROTO.fields_by_name['float_val']._options = None\n_TENSORPROTO.fields_by_name['double_val']._options = None\n_TENSORPROTO.fields_by_name['int_val']._options = None\n_TENSORPROTO.fields_by_name['scomplex_val']._options = None\n_TENSORPROTO.fields_by_name['int64_val']._options = None\n_TENSORPROTO.fields_by_name['bool_val']._options = None\n_TENSORPROTO.fields_by_name['dcomplex_val']._options = None\n# @@protoc_insertion_point(module_scope)\n" }, { "path": "tensorboardX/tensorboardX/proto/tensor_shape.proto", "content": "// Protocol buffer representing the shape of tensors.\n\nsyntax = \"proto3\";\noption cc_enable_arenas = true;\noption java_outer_classname = \"TensorShapeProtos\";\noption java_multiple_files = true;\noption java_package = \"org.tensorflow.framework\";\n\npackage tensorboardX;\n\n// Dimensions of a tensor.\nmessage TensorShapeProto {\n // One dimension of the tensor.\n message Dim {\n // Size of the tensor in that dimension.\n // This value must be >= -1, but values of -1 are reserved for \"unknown\"\n // shapes (values of -1 mean \"unknown\" dimension). Certain wrappers\n // that work with TensorShapeProto may fail at runtime when deserializing\n // a TensorShapeProto containing a dim value of -1.\n int64 size = 1;\n\n // Optional name of the tensor dimension.\n string name = 2;\n };\n\n // Dimensions of the tensor, such as {\"input\", 30}, {\"output\", 40}\n // for a 30 x 40 2D tensor. If an entry has size -1, this\n // corresponds to a dimension of unknown size. The names are\n // optional.\n //\n // The order of entries in \"dim\" matters: It indicates the layout of the\n // values in the tensor in-memory representation.\n //\n // The first entry in \"dim\" is the outermost dimension used to layout the\n // values, the last entry is the innermost dimension. This matches the\n // in-memory layout of RowMajor Eigen tensors.\n //\n // If \"dim.size()\" > 0, \"unknown_rank\" must be false.\n repeated Dim dim = 2;\n\n // If true, the number of dimensions in the shape is unknown.\n //\n // If true, \"dim.size()\" must be 0.\n bool unknown_rank = 3;\n};\n" }, { "path": "tensorboardX/tensorboardX/proto/tensor_shape_pb2.py", "content": "# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: tensorboardX/proto/tensor_shape.proto\n\nimport sys\n_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))\nfrom google.protobuf import descriptor as _descriptor\nfrom google.protobuf import message as _message\nfrom google.protobuf import reflection as _reflection\nfrom google.protobuf import symbol_database as _symbol_database\n# @@protoc_insertion_point(imports)\n\n_sym_db = _symbol_database.Default()\n\n\n\n\nDESCRIPTOR = _descriptor.FileDescriptor(\n name='tensorboardX/proto/tensor_shape.proto',\n package='tensorboardX',\n syntax='proto3',\n serialized_options=_b('\\n\\030org.tensorflow.frameworkB\\021TensorShapeProtosP\\001\\370\\001\\001'),\n serialized_pb=_b('\\n%tensorboardX/proto/tensor_shape.proto\\x12\\x0ctensorboardX\\\"|\\n\\x10TensorShapeProto\\x12/\\n\\x03\\x64im\\x18\\x02 \\x03(\\x0b\\x32\\\".tensorboardX.TensorShapeProto.Dim\\x12\\x14\\n\\x0cunknown_rank\\x18\\x03 \\x01(\\x08\\x1a!\\n\\x03\\x44im\\x12\\x0c\\n\\x04size\\x18\\x01 \\x01(\\x03\\x12\\x0c\\n\\x04name\\x18\\x02 \\x01(\\tB2\\n\\x18org.tensorflow.frameworkB\\x11TensorShapeProtosP\\x01\\xf8\\x01\\x01\\x62\\x06proto3')\n)\n\n\n\n\n_TENSORSHAPEPROTO_DIM = _descriptor.Descriptor(\n name='Dim',\n full_name='tensorboardX.TensorShapeProto.Dim',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='size', full_name='tensorboardX.TensorShapeProto.Dim.size', index=0,\n number=1, type=3, cpp_type=2, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='name', full_name='tensorboardX.TensorShapeProto.Dim.name', index=1,\n number=2, type=9, cpp_type=9, label=1,\n has_default_value=False, default_value=_b(\"\").decode('utf-8'),\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=146,\n serialized_end=179,\n)\n\n_TENSORSHAPEPROTO = _descriptor.Descriptor(\n name='TensorShapeProto',\n full_name='tensorboardX.TensorShapeProto',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='dim', full_name='tensorboardX.TensorShapeProto.dim', index=0,\n number=2, type=11, cpp_type=10, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='unknown_rank', full_name='tensorboardX.TensorShapeProto.unknown_rank', index=1,\n number=3, type=8, cpp_type=7, label=1,\n has_default_value=False, default_value=False,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[_TENSORSHAPEPROTO_DIM, ],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=55,\n serialized_end=179,\n)\n\n_TENSORSHAPEPROTO_DIM.containing_type = _TENSORSHAPEPROTO\n_TENSORSHAPEPROTO.fields_by_name['dim'].message_type = _TENSORSHAPEPROTO_DIM\nDESCRIPTOR.message_types_by_name['TensorShapeProto'] = _TENSORSHAPEPROTO\n_sym_db.RegisterFileDescriptor(DESCRIPTOR)\n\nTensorShapeProto = _reflection.GeneratedProtocolMessageType('TensorShapeProto', (_message.Message,), dict(\n\n Dim = _reflection.GeneratedProtocolMessageType('Dim', (_message.Message,), dict(\n DESCRIPTOR = _TENSORSHAPEPROTO_DIM,\n __module__ = 'tensorboardX.proto.tensor_shape_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.TensorShapeProto.Dim)\n ))\n ,\n DESCRIPTOR = _TENSORSHAPEPROTO,\n __module__ = 'tensorboardX.proto.tensor_shape_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.TensorShapeProto)\n ))\n_sym_db.RegisterMessage(TensorShapeProto)\n_sym_db.RegisterMessage(TensorShapeProto.Dim)\n\n\nDESCRIPTOR._options = None\n# @@protoc_insertion_point(module_scope)\n" }, { "path": "tensorboardX/tensorboardX/proto/types.proto", "content": "syntax = \"proto3\";\n\npackage tensorboardX;\noption cc_enable_arenas = true;\noption java_outer_classname = \"TypesProtos\";\noption java_multiple_files = true;\noption java_package = \"org.tensorflow.framework\";\n\n// LINT.IfChange\nenum DataType {\n // Not a legal value for DataType. Used to indicate a DataType field\n // has not been set.\n DT_INVALID = 0;\n\n // Data types that all computation devices are expected to be\n // capable to support.\n DT_FLOAT = 1;\n DT_DOUBLE = 2;\n DT_INT32 = 3;\n DT_UINT8 = 4;\n DT_INT16 = 5;\n DT_INT8 = 6;\n DT_STRING = 7;\n DT_COMPLEX64 = 8; // Single-precision complex\n DT_INT64 = 9;\n DT_BOOL = 10;\n DT_QINT8 = 11; // Quantized int8\n DT_QUINT8 = 12; // Quantized uint8\n DT_QINT32 = 13; // Quantized int32\n DT_BFLOAT16 = 14; // Float32 truncated to 16 bits. Only for cast ops.\n DT_QINT16 = 15; // Quantized int16\n DT_QUINT16 = 16; // Quantized uint16\n DT_UINT16 = 17;\n DT_COMPLEX128 = 18; // Double-precision complex\n DT_HALF = 19;\n DT_RESOURCE = 20;\n\n // TODO(josh11b): DT_GENERIC_PROTO = ??;\n // TODO(jeff,josh11b): DT_UINT64? DT_UINT32?\n\n // Do not use! These are only for parameters. Every enum above\n // should have a corresponding value below (verified by types_test).\n DT_FLOAT_REF = 101;\n DT_DOUBLE_REF = 102;\n DT_INT32_REF = 103;\n DT_UINT8_REF = 104;\n DT_INT16_REF = 105;\n DT_INT8_REF = 106;\n DT_STRING_REF = 107;\n DT_COMPLEX64_REF = 108;\n DT_INT64_REF = 109;\n DT_BOOL_REF = 110;\n DT_QINT8_REF = 111;\n DT_QUINT8_REF = 112;\n DT_QINT32_REF = 113;\n DT_BFLOAT16_REF = 114;\n DT_QINT16_REF = 115;\n DT_QUINT16_REF = 116;\n DT_UINT16_REF = 117;\n DT_COMPLEX128_REF = 118;\n DT_HALF_REF = 119;\n DT_RESOURCE_REF = 120;\n}\n// LINT.ThenChange(https://www.tensorflow.org/code/tensorflow/c/c_api.h,https://www.tensorflow.org/code/tensorflow/go/tensor.go)\n" }, { "path": "tensorboardX/tensorboardX/proto/types_pb2.py", "content": "# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: tensorboardX/proto/types.proto\n\nimport sys\n_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))\nfrom google.protobuf.internal import enum_type_wrapper\nfrom google.protobuf import descriptor as _descriptor\nfrom google.protobuf import message as _message\nfrom google.protobuf import reflection as _reflection\nfrom google.protobuf import symbol_database as _symbol_database\n# @@protoc_insertion_point(imports)\n\n_sym_db = _symbol_database.Default()\n\n\n\n\nDESCRIPTOR = _descriptor.FileDescriptor(\n name='tensorboardX/proto/types.proto',\n package='tensorboardX',\n syntax='proto3',\n serialized_options=_b('\\n\\030org.tensorflow.frameworkB\\013TypesProtosP\\001\\370\\001\\001'),\n serialized_pb=_b('\\n\\x1etensorboardX/proto/types.proto\\x12\\x0ctensorboardX*\\xc2\\x05\\n\\x08\\x44\\x61taType\\x12\\x0e\\n\\nDT_INVALID\\x10\\x00\\x12\\x0c\\n\\x08\\x44T_FLOAT\\x10\\x01\\x12\\r\\n\\tDT_DOUBLE\\x10\\x02\\x12\\x0c\\n\\x08\\x44T_INT32\\x10\\x03\\x12\\x0c\\n\\x08\\x44T_UINT8\\x10\\x04\\x12\\x0c\\n\\x08\\x44T_INT16\\x10\\x05\\x12\\x0b\\n\\x07\\x44T_INT8\\x10\\x06\\x12\\r\\n\\tDT_STRING\\x10\\x07\\x12\\x10\\n\\x0c\\x44T_COMPLEX64\\x10\\x08\\x12\\x0c\\n\\x08\\x44T_INT64\\x10\\t\\x12\\x0b\\n\\x07\\x44T_BOOL\\x10\\n\\x12\\x0c\\n\\x08\\x44T_QINT8\\x10\\x0b\\x12\\r\\n\\tDT_QUINT8\\x10\\x0c\\x12\\r\\n\\tDT_QINT32\\x10\\r\\x12\\x0f\\n\\x0b\\x44T_BFLOAT16\\x10\\x0e\\x12\\r\\n\\tDT_QINT16\\x10\\x0f\\x12\\x0e\\n\\nDT_QUINT16\\x10\\x10\\x12\\r\\n\\tDT_UINT16\\x10\\x11\\x12\\x11\\n\\rDT_COMPLEX128\\x10\\x12\\x12\\x0b\\n\\x07\\x44T_HALF\\x10\\x13\\x12\\x0f\\n\\x0b\\x44T_RESOURCE\\x10\\x14\\x12\\x10\\n\\x0c\\x44T_FLOAT_REF\\x10\\x65\\x12\\x11\\n\\rDT_DOUBLE_REF\\x10\\x66\\x12\\x10\\n\\x0c\\x44T_INT32_REF\\x10g\\x12\\x10\\n\\x0c\\x44T_UINT8_REF\\x10h\\x12\\x10\\n\\x0c\\x44T_INT16_REF\\x10i\\x12\\x0f\\n\\x0b\\x44T_INT8_REF\\x10j\\x12\\x11\\n\\rDT_STRING_REF\\x10k\\x12\\x14\\n\\x10\\x44T_COMPLEX64_REF\\x10l\\x12\\x10\\n\\x0c\\x44T_INT64_REF\\x10m\\x12\\x0f\\n\\x0b\\x44T_BOOL_REF\\x10n\\x12\\x10\\n\\x0c\\x44T_QINT8_REF\\x10o\\x12\\x11\\n\\rDT_QUINT8_REF\\x10p\\x12\\x11\\n\\rDT_QINT32_REF\\x10q\\x12\\x13\\n\\x0f\\x44T_BFLOAT16_REF\\x10r\\x12\\x11\\n\\rDT_QINT16_REF\\x10s\\x12\\x12\\n\\x0e\\x44T_QUINT16_REF\\x10t\\x12\\x11\\n\\rDT_UINT16_REF\\x10u\\x12\\x15\\n\\x11\\x44T_COMPLEX128_REF\\x10v\\x12\\x0f\\n\\x0b\\x44T_HALF_REF\\x10w\\x12\\x13\\n\\x0f\\x44T_RESOURCE_REF\\x10xB,\\n\\x18org.tensorflow.frameworkB\\x0bTypesProtosP\\x01\\xf8\\x01\\x01\\x62\\x06proto3')\n)\n\n_DATATYPE = _descriptor.EnumDescriptor(\n name='DataType',\n full_name='tensorboardX.DataType',\n filename=None,\n file=DESCRIPTOR,\n values=[\n _descriptor.EnumValueDescriptor(\n name='DT_INVALID', index=0, number=0,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_FLOAT', index=1, number=1,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_DOUBLE', index=2, number=2,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_INT32', index=3, number=3,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_UINT8', index=4, number=4,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_INT16', index=5, number=5,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_INT8', index=6, number=6,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_STRING', index=7, number=7,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_COMPLEX64', index=8, number=8,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_INT64', index=9, number=9,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_BOOL', index=10, number=10,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_QINT8', index=11, number=11,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_QUINT8', index=12, number=12,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_QINT32', index=13, number=13,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_BFLOAT16', index=14, number=14,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_QINT16', index=15, number=15,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_QUINT16', index=16, number=16,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_UINT16', index=17, number=17,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_COMPLEX128', index=18, number=18,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_HALF', index=19, number=19,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_RESOURCE', index=20, number=20,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_FLOAT_REF', index=21, number=101,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_DOUBLE_REF', index=22, number=102,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_INT32_REF', index=23, number=103,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_UINT8_REF', index=24, number=104,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_INT16_REF', index=25, number=105,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_INT8_REF', index=26, number=106,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_STRING_REF', index=27, number=107,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_COMPLEX64_REF', index=28, number=108,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_INT64_REF', index=29, number=109,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_BOOL_REF', index=30, number=110,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_QINT8_REF', index=31, number=111,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_QUINT8_REF', index=32, number=112,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_QINT32_REF', index=33, number=113,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_BFLOAT16_REF', index=34, number=114,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_QINT16_REF', index=35, number=115,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_QUINT16_REF', index=36, number=116,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_UINT16_REF', index=37, number=117,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_COMPLEX128_REF', index=38, number=118,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_HALF_REF', index=39, number=119,\n serialized_options=None,\n type=None),\n _descriptor.EnumValueDescriptor(\n name='DT_RESOURCE_REF', index=40, number=120,\n serialized_options=None,\n type=None),\n ],\n containing_type=None,\n serialized_options=None,\n serialized_start=49,\n serialized_end=755,\n)\n_sym_db.RegisterEnumDescriptor(_DATATYPE)\n\nDataType = enum_type_wrapper.EnumTypeWrapper(_DATATYPE)\nDT_INVALID = 0\nDT_FLOAT = 1\nDT_DOUBLE = 2\nDT_INT32 = 3\nDT_UINT8 = 4\nDT_INT16 = 5\nDT_INT8 = 6\nDT_STRING = 7\nDT_COMPLEX64 = 8\nDT_INT64 = 9\nDT_BOOL = 10\nDT_QINT8 = 11\nDT_QUINT8 = 12\nDT_QINT32 = 13\nDT_BFLOAT16 = 14\nDT_QINT16 = 15\nDT_QUINT16 = 16\nDT_UINT16 = 17\nDT_COMPLEX128 = 18\nDT_HALF = 19\nDT_RESOURCE = 20\nDT_FLOAT_REF = 101\nDT_DOUBLE_REF = 102\nDT_INT32_REF = 103\nDT_UINT8_REF = 104\nDT_INT16_REF = 105\nDT_INT8_REF = 106\nDT_STRING_REF = 107\nDT_COMPLEX64_REF = 108\nDT_INT64_REF = 109\nDT_BOOL_REF = 110\nDT_QINT8_REF = 111\nDT_QUINT8_REF = 112\nDT_QINT32_REF = 113\nDT_BFLOAT16_REF = 114\nDT_QINT16_REF = 115\nDT_QUINT16_REF = 116\nDT_UINT16_REF = 117\nDT_COMPLEX128_REF = 118\nDT_HALF_REF = 119\nDT_RESOURCE_REF = 120\n\n\nDESCRIPTOR.enum_types_by_name['DataType'] = _DATATYPE\n_sym_db.RegisterFileDescriptor(DESCRIPTOR)\n\n\nDESCRIPTOR._options = None\n# @@protoc_insertion_point(module_scope)\n" }, { "path": "tensorboardX/tensorboardX/proto/versions.proto", "content": "syntax = \"proto3\";\n\npackage tensorboardX;\noption cc_enable_arenas = true;\noption java_outer_classname = \"VersionsProtos\";\noption java_multiple_files = true;\noption java_package = \"org.tensorflow.framework\";\n\n// Version information for a piece of serialized data\n//\n// There are different types of versions for each type of data\n// (GraphDef, etc.), but they all have the same common shape\n// described here.\n//\n// Each consumer has \"consumer\" and \"min_producer\" versions (specified\n// elsewhere). A consumer is allowed to consume this data if\n//\n// producer >= min_producer\n// consumer >= min_consumer\n// consumer not in bad_consumers\n//\nmessage VersionDef {\n // The version of the code that produced this data.\n int32 producer = 1;\n\n // Any consumer below this version is not allowed to consume this data.\n int32 min_consumer = 2;\n\n // Specific consumer versions which are disallowed (e.g. due to bugs).\n repeated int32 bad_consumers = 3;\n};\n" }, { "path": "tensorboardX/tensorboardX/proto/versions_pb2.py", "content": "# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: tensorboardX/proto/versions.proto\n\nimport sys\n_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))\nfrom google.protobuf import descriptor as _descriptor\nfrom google.protobuf import message as _message\nfrom google.protobuf import reflection as _reflection\nfrom google.protobuf import symbol_database as _symbol_database\n# @@protoc_insertion_point(imports)\n\n_sym_db = _symbol_database.Default()\n\n\n\n\nDESCRIPTOR = _descriptor.FileDescriptor(\n name='tensorboardX/proto/versions.proto',\n package='tensorboardX',\n syntax='proto3',\n serialized_options=_b('\\n\\030org.tensorflow.frameworkB\\016VersionsProtosP\\001\\370\\001\\001'),\n serialized_pb=_b('\\n!tensorboardX/proto/versions.proto\\x12\\x0ctensorboardX\\\"K\\n\\nVersionDef\\x12\\x10\\n\\x08producer\\x18\\x01 \\x01(\\x05\\x12\\x14\\n\\x0cmin_consumer\\x18\\x02 \\x01(\\x05\\x12\\x15\\n\\rbad_consumers\\x18\\x03 \\x03(\\x05\\x42/\\n\\x18org.tensorflow.frameworkB\\x0eVersionsProtosP\\x01\\xf8\\x01\\x01\\x62\\x06proto3')\n)\n\n\n\n\n_VERSIONDEF = _descriptor.Descriptor(\n name='VersionDef',\n full_name='tensorboardX.VersionDef',\n filename=None,\n file=DESCRIPTOR,\n containing_type=None,\n fields=[\n _descriptor.FieldDescriptor(\n name='producer', full_name='tensorboardX.VersionDef.producer', index=0,\n number=1, type=5, cpp_type=1, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='min_consumer', full_name='tensorboardX.VersionDef.min_consumer', index=1,\n number=2, type=5, cpp_type=1, label=1,\n has_default_value=False, default_value=0,\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n _descriptor.FieldDescriptor(\n name='bad_consumers', full_name='tensorboardX.VersionDef.bad_consumers', index=2,\n number=3, type=5, cpp_type=1, label=3,\n has_default_value=False, default_value=[],\n message_type=None, enum_type=None, containing_type=None,\n is_extension=False, extension_scope=None,\n serialized_options=None, file=DESCRIPTOR),\n ],\n extensions=[\n ],\n nested_types=[],\n enum_types=[\n ],\n serialized_options=None,\n is_extendable=False,\n syntax='proto3',\n extension_ranges=[],\n oneofs=[\n ],\n serialized_start=51,\n serialized_end=126,\n)\n\nDESCRIPTOR.message_types_by_name['VersionDef'] = _VERSIONDEF\n_sym_db.RegisterFileDescriptor(DESCRIPTOR)\n\nVersionDef = _reflection.GeneratedProtocolMessageType('VersionDef', (_message.Message,), dict(\n DESCRIPTOR = _VERSIONDEF,\n __module__ = 'tensorboardX.proto.versions_pb2'\n # @@protoc_insertion_point(class_scope:tensorboardX.VersionDef)\n ))\n_sym_db.RegisterMessage(VersionDef)\n\n\nDESCRIPTOR._options = None\n# @@protoc_insertion_point(module_scope)\n" }, { "path": "tensorboardX/tensorboardX/proto_graph.py", "content": "from .proto.graph_pb2 import GraphDef\nfrom .proto.node_def_pb2 import NodeDef\nfrom .proto.versions_pb2 import VersionDef\nfrom .proto.attr_value_pb2 import AttrValue\nfrom .proto.tensor_shape_pb2 import TensorShapeProto\n\n\ndef attr_value_proto(dtype, shape, s):\n \"\"\"Creates a dict of objects matching\n https://github.com/tensorflow/tensorboard/blob/master/tensorboard/compat/proto/attr_value.proto\n specifically designed for a NodeDef. The values have been\n reverse engineered from standard TensorBoard logged data.\n \"\"\"\n attr = {}\n if s is not None:\n attr['attr'] = AttrValue(s=s.encode(encoding='utf_8'))\n if shape is not None:\n shapeproto = tensor_shape_proto(shape)\n attr['_output_shapes'] = AttrValue(list=AttrValue.ListValue(shape=[shapeproto]))\n return attr\n\n\ndef tensor_shape_proto(outputsize):\n \"\"\"Creates an object matching\n https://github.com/tensorflow/tensorboard/blob/master/tensorboard/compat/proto/tensor_shape.proto\n \"\"\"\n return TensorShapeProto(dim=[TensorShapeProto.Dim(size=d) for d in outputsize])\n\n\ndef node_proto(name,\n op='UnSpecified',\n input=None,\n dtype=None,\n shape=None, # type: tuple\n outputsize=None,\n attributes=''\n ):\n \"\"\"Creates an object matching\n https://github.com/tensorflow/tensorboard/blob/master/tensorboard/compat/proto/node_def.proto\n \"\"\"\n if input is None:\n input = []\n if not isinstance(input, list):\n input = [input]\n return NodeDef(\n name=name.encode(encoding='utf_8'),\n op=op,\n input=input,\n attr=attr_value_proto(dtype, outputsize, attributes)\n )\n" }, { "path": "tensorboardX/tensorboardX/pytorch_graph.py", "content": "import logging\nimport time\nfrom collections import OrderedDict\nfrom .proto.attr_value_pb2 import AttrValue\nfrom .proto.graph_pb2 import GraphDef\nfrom .proto.node_def_pb2 import NodeDef\nfrom .proto.step_stats_pb2 import RunMetadata, StepStats, DeviceStepStats, NodeExecStats, AllocatorMemoryUsed\nfrom .proto.tensor_shape_pb2 import TensorShapeProto\nfrom .proto.versions_pb2 import VersionDef\nfrom .proto_graph import node_proto\n\nmethods_OP = ['attributeNames', 'hasMultipleOutputs', 'hasUses', 'inputs',\n 'kind', 'outputs', 'outputsSize', 'scopeName']\nmethods_IO = ['node', 'offset', 'debugName'] # 'unique' , 'type' >\n\nbackward_mode = False\n\nclass NodeBase(object):\n def __init__(self,\n debugName=None,\n inputs=None,\n scope=None,\n tensor_size=None,\n op_type='UnSpecified',\n attributes=''):\n self.debugName = debugName\n self.inputs = inputs\n self.tensor_size = tensor_size\n self.kind = op_type\n self.attributes = attributes\n if scope is not None:\n self.scope = scope\n\n def __repr__(self):\n repr = []\n repr.append(str(type(self)))\n for m in dir(self):\n if '__' not in m:\n repr.append(m + ': ' + str(getattr(self, m)) + str(type(getattr(self, m))))\n return '\\n'.join(repr) + '\\n\\n'\n\n\nclass NodePy(NodeBase):\n def __init__(self, node_cpp, valid_methods):\n super(NodePy, self).__init__(node_cpp)\n valid_methods = valid_methods[:]\n self.inputs = []\n global backward_mode\n for m in valid_methods:\n if m == 'inputs' or m == 'outputs':\n list_of_node = list(getattr(node_cpp, m)())\n io_unique_names = []\n io_tensor_sizes = []\n for n in list_of_node:\n if backward_mode:\n io_unique_names.append(n.uniqueName())\n else:\n io_unique_names.append(n.debugName())\n\n if n.type().kind() == 'CompleteTensorType':\n io_tensor_sizes.append(n.type().sizes())\n else:\n io_tensor_sizes.append(None)\n\n setattr(self, m, io_unique_names)\n setattr(self, m + 'tensor_size', io_tensor_sizes)\n\n else:\n if m == 'debugName' and backward_mode:\n setattr(self, m, getattr(node_cpp, 'uniqueName')())\n else:\n setattr(self, m, getattr(node_cpp, m)())\n\n\nclass NodePyIO(NodePy):\n def __init__(self, node_cpp, input_or_output=None):\n super(NodePyIO, self).__init__(node_cpp, methods_IO)\n try:\n tensor_size = node_cpp.type().sizes()\n except RuntimeError:\n tensor_size = [1, ] # fail when constant model is used.\n self.tensor_size = tensor_size\n # Kind attribute string is purely descriptive and will be shown\n # in detailed information for the node in TensorBoard's graph plugin.\n #\n # NodePyOP nodes get this from their kind() method.\n self.kind = 'Parameter'\n if input_or_output:\n self.input_or_output = input_or_output\n self.kind = 'IO Node'\n\n\nclass NodePyOP(NodePy):\n def __init__(self, node_cpp):\n super(NodePyOP, self).__init__(node_cpp, methods_OP)\n # Replace single quote which causes strange behavior in TensorBoard\n # TODO: See if we can remove this in the future\n self.attributes = str({k: node_cpp[k] for k in node_cpp.attributeNames()}).replace(\"'\", ' ')\n self.kind = node_cpp.kind()\n\n\nclass GraphPy(object):\n \"\"\"Helper class to convert torch.nn.Module to GraphDef proto and visualization\n with TensorBoard.\n\n GraphDef generation operates in two passes:\n\n In the first pass, all nodes are read and saved to two lists.\n One list is for input/output nodes (nodes_io), which only have inbound\n or outbound connections, but not both. Another list is for internal\n operator nodes (nodes_op). The first pass also saves all scope name\n appeared in the nodes in scope_name_appeared list for later processing.\n\n In the second pass, scope names are fully applied to all nodes.\n debugNameToScopedName is a mapping from a node's ID to its fully qualified\n scope name. e.g. Net1/Linear[0]/1. Unfortunately torch.jit doesn't have\n totally correct scope output, so this is nontrivial. The function\n populate_namespace_from_OP_to_IO and find_common_root are used to\n assign scope name to a node based on the connection between nodes\n in a heuristic kind of way. Bookkeeping is done with shallowest_scope_name\n and scope_name_appeared.\n \"\"\"\n def __init__(self):\n self.nodes_op = []\n self.nodes_io = OrderedDict()\n self.unique_name_to_scoped_name = {}\n self.shallowest_scope_name = 'default'\n self.scope_name_appeared = []\n\n def append(self, x):\n if isinstance(x, NodePyIO):\n self.nodes_io[x.debugName] = x\n if isinstance(x, NodePyOP):\n self.nodes_op.append(x)\n for node_output, outputSize in zip(x.outputs, x.outputstensor_size):\n self.scope_name_appeared.append(x.scopeName)\n self.nodes_io[node_output] = NodeBase(node_output,\n x.inputs,\n x.scopeName,\n outputSize,\n op_type=x.kind,\n attributes=x.attributes)\n\n def printall(self):\n print('all nodes')\n for node in self.nodes_op:\n print(node)\n for key in self.nodes_io:\n print(self.nodes_io[key])\n\n def find_common_root(self):\n for fullscope in self.scope_name_appeared:\n if fullscope:\n self.shallowest_scope_name = fullscope.split('/')[0]\n\n def populate_namespace_from_OP_to_IO(self):\n for node in self.nodes_op:\n for input_node_id in node.inputs:\n self.unique_name_to_scoped_name[input_node_id] = node.scopeName + '/' + input_node_id\n\n for key, node in self.nodes_io.items():\n if type(node) == NodeBase:\n self.unique_name_to_scoped_name[key] = node.scope + '/' + node.debugName\n if hasattr(node, 'input_or_output'):\n self.unique_name_to_scoped_name[key] = node.input_or_output + '/' + node.debugName\n if hasattr(node, 'scope'):\n if node.scope == '' and self.shallowest_scope_name:\n self.unique_name_to_scoped_name[node.debugName] = \\\n self.shallowest_scope_name + '/' + node.debugName\n\n # replace name\n for key, node in self.nodes_io.items():\n self.nodes_io[key].inputs = \\\n [self.unique_name_to_scoped_name[node_input_id] for node_input_id in node.inputs]\n if node.debugName in self.unique_name_to_scoped_name:\n self.nodes_io[key].debugName = self.unique_name_to_scoped_name[node.debugName]\n\n def to_proto(self):\n \"\"\"\n Converts graph representation of GraphPy object to TensorBoard\n required format.\n \"\"\"\n # TODO: compute correct memory usage and CPU time once\n # PyTorch supports it\n import numpy as np\n nodes = []\n node_stats = []\n for v in self.nodes_io.values():\n nodes.append(node_proto(v.debugName,\n input=v.inputs,\n outputsize=v.tensor_size,\n op=v.kind,\n attributes=v.attributes))\n\n if v.tensor_size and len(v.tensor_size) > 0: # assume data is float32, only parameter is counted\n node_stats.append(\n NodeExecStats(node_name=v.debugName,\n all_start_micros=int(time.time() * 1e7),\n all_end_rel_micros=42,\n memory=[AllocatorMemoryUsed(allocator_name=\"cpu\",\n total_bytes=int(np.prod(v.tensor_size)) * 4)]))\n\n return nodes, node_stats\n\n\n# one argument: 'hasAttribute', 'hasAttributes',\ndef parse(graph, args=None, omit_useless_nodes=True):\n \"\"\"This method parses an optimized PyTorch model graph and produces\n a list of nodes and node stats for eventual conversion to TensorBoard\n protobuf format.\n\n Args:\n graph (PyTorch module): The model to be parsed.\n args (tuple): input tensor[s] for the model.\n omit_useless_nodes (boolean): Whether to remove nodes from the graph.\n \"\"\"\n import torch\n n_inputs = len(args) # not sure...\n\n nodes_py = GraphPy()\n for i, node in enumerate(graph.inputs()):\n global backward_mode\n if not backward_mode:\n try:\n node.debugName()\n except:\n backward_mode = True\n if omit_useless_nodes:\n if len(node.uses()) == 0: # number of user of the node (= number of outputs/ fanout)\n continue\n\n if i < n_inputs:\n nodes_py.append(NodePyIO(node, 'input'))\n else:\n nodes_py.append(NodePyIO(node)) # parameter\n\n for node in graph.nodes():\n nodes_py.append(NodePyOP(node))\n\n for node in graph.outputs(): # must place last.\n NodePyIO(node, 'output')\n nodes_py.find_common_root()\n nodes_py.populate_namespace_from_OP_to_IO()\n return nodes_py.to_proto()\n\n\ndef graph(model, args, verbose=False, **kwargs):\n \"\"\"\n This method processes a PyTorch model and produces a `GraphDef` proto\n that can be logged to TensorBoard.\n\n Args:\n model (PyTorch module): The model to be parsed.\n args (tuple): input tensor[s] for the model.\n verbose (bool): Whether to print out verbose information while\n processing.\n \"\"\"\n import torch\n\n with torch.onnx.set_training(model, False): # TODO: move outside of torch.onnx\n try:\n trace = torch.jit.trace(model, args)\n graph = trace.graph\n\n except RuntimeError as e:\n print(e)\n print('Error occurs, No graph saved')\n raise e\n # Create an object matching\n # https://github.com/tensorflow/tensorboard/blob/master/tensorboard/compat/proto/graph.proto\n # The producer version has been reverse engineered from standard\n # TensorBoard logged data.\n\n if verbose:\n print(graph)\n list_of_nodes, node_stats = parse(graph, args)\n # We are hardcoding that this was run on CPU even though it might have actually\n # run on GPU. Note this is what is shown in TensorBoard and has no bearing\n # on actual execution.\n # TODO: See if we can extract GPU vs CPU information from the PyTorch model\n # and pass it correctly to TensorBoard.\n #\n # Definition of StepStats and DeviceStepStats can be found at\n # https://github.com/tensorflow/tensorboard/blob/master/tensorboard/plugins/graph/tf_graph_common/test/graph-test.ts\n # and\n # https://github.com/tensorflow/tensorboard/blob/master/tensorboard/compat/proto/step_stats.proto\n stepstats = RunMetadata(step_stats=StepStats(dev_stats=[DeviceStepStats(device=\"/device:CPU:0\",\n node_stats=node_stats)]))\n return GraphDef(node=list_of_nodes, versions=VersionDef(producer=22)), stepstats\n" }, { "path": "tensorboardX/tensorboardX/record_writer.py", "content": "\"\"\"\nTo write tf_record into file. Here we use it for tensorboard's event writting.\nThe code was borrowed from https://github.com/TeamHG-Memex/tensorboard_logger\n\"\"\"\n\nimport copy\nimport io\nimport os.path\nimport re\nimport struct\ntry:\n import boto3\n S3_ENABLED = True\nexcept ImportError:\n S3_ENABLED = False\n\nfrom .crc32c import crc32c\n\n\n_VALID_OP_NAME_START = re.compile('^[A-Za-z0-9.]')\n_VALID_OP_NAME_PART = re.compile('[A-Za-z0-9_.\\\\-/]+')\n\n# Registry of writer factories by prefix backends.\n#\n# Currently supports \"s3://\" URLs for S3 based on boto and falls\n# back to local filesystem.\nREGISTERED_FACTORIES = {}\n\n\ndef register_writer_factory(prefix, factory):\n if ':' in prefix:\n raise ValueError('prefix cannot contain a :')\n REGISTERED_FACTORIES[prefix] = factory\n\n\ndef directory_check(path):\n '''Initialize the directory for log files.'''\n try:\n prefix = path.split(':')[0]\n factory = REGISTERED_FACTORIES[prefix]\n return factory.directory_check(path)\n except KeyError:\n if not os.path.exists(path):\n os.makedirs(path)\n\n\ndef open_file(path):\n '''Open a writer for outputting event files.'''\n try:\n prefix = path.split(':')[0]\n factory = REGISTERED_FACTORIES[prefix]\n return factory.open(path)\n except KeyError:\n return open(path, 'wb')\n\n\nclass S3RecordWriter(object):\n \"\"\"Writes tensorboard protocol buffer files to S3.\"\"\"\n\n def __init__(self, path):\n if not S3_ENABLED:\n raise ImportError(\"boto3 must be installed for S3 support.\")\n self.path = path\n self.buffer = io.BytesIO()\n\n def __del__(self):\n self.close()\n\n def bucket_and_path(self):\n path = self.path\n if path.startswith(\"s3://\"):\n path = path[len(\"s3://\"):]\n bp = path.split(\"/\")\n bucket = bp[0]\n path = path[1 + len(bucket):]\n return bucket, path\n\n def write(self, val):\n self.buffer.write(val)\n\n def flush(self):\n s3 = boto3.client('s3')\n bucket, path = self.bucket_and_path()\n upload_buffer = copy.copy(self.buffer)\n upload_buffer.seek(0)\n s3.upload_fileobj(upload_buffer, bucket, path)\n\n def close(self):\n self.flush()\n\n\nclass S3RecordWriterFactory(object):\n \"\"\"Factory for event protocol buffer files to S3.\"\"\"\n\n def open(self, path):\n return S3RecordWriter(path)\n\n def directory_check(self, path):\n # S3 doesn't need directories created before files are added\n # so we can just skip this check\n pass\n\n\nregister_writer_factory(\"s3\", S3RecordWriterFactory())\n\n\nclass RecordWriter(object):\n def __init__(self, path):\n self._name_to_tf_name = {}\n self._tf_names = set()\n self.path = path\n self._writer = None\n self._writer = open_file(path)\n\n def write(self, data):\n w = self._writer.write\n header = struct.pack('Q', len(data))\n w(header)\n w(struct.pack('I', masked_crc32c(header)))\n w(data)\n w(struct.pack('I', masked_crc32c(data)))\n\n def flush(self):\n self._writer.flush()\n\n def close(self):\n self._writer.close()\n\n\ndef masked_crc32c(data):\n x = u32(crc32c(data))\n return u32(((x >> 15) | u32(x << 17)) + 0xa282ead8)\n\n\ndef u32(x):\n return x & 0xffffffff\n\n\ndef make_valid_tf_name(name):\n if not _VALID_OP_NAME_START.match(name):\n # Must make it valid somehow, but don't want to remove stuff\n name = '.' + name\n return '_'.join(_VALID_OP_NAME_PART.findall(name))\n" }, { "path": "tensorboardX/tensorboardX/summary.py", "content": "from __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport logging\nimport numpy as np\nimport os\nimport re as _re\n\n# pylint: disable=unused-import\nfrom six.moves import range\n\nfrom .proto.summary_pb2 import Summary\nfrom .proto.summary_pb2 import HistogramProto\nfrom .proto.summary_pb2 import SummaryMetadata\nfrom .proto.tensor_pb2 import TensorProto\nfrom .proto.tensor_shape_pb2 import TensorShapeProto\nfrom .proto.plugin_pr_curve_pb2 import PrCurvePluginData\nfrom .proto.plugin_text_pb2 import TextPluginData\nfrom .proto.plugin_mesh_pb2 import MeshPluginData\nfrom .proto import layout_pb2\nfrom .x2num import make_np\nfrom .utils import _prepare_video, convert_to_HWC\n\n_INVALID_TAG_CHARACTERS = _re.compile(r'[^-/\\w\\.]')\n\n\ndef _clean_tag(name):\n # In the past, the first argument to summary ops was a tag, which allowed\n # arbitrary characters. Now we are changing the first argument to be the node\n # name. This has a number of advantages (users of summary ops now can\n # take advantage of the tf name scope system) but risks breaking existing\n # usage, because a much smaller set of characters are allowed in node names.\n # This function replaces all illegal characters with _s, and logs a warning.\n # It also strips leading slashes from the name.\n if name is not None:\n new_name = _INVALID_TAG_CHARACTERS.sub('_', name)\n new_name = new_name.lstrip('/') # Remove leading slashes\n if new_name != name:\n logging.info(\n 'Summary name %s is illegal; using %s instead.' % (name, new_name))\n name = new_name\n return name\n\n\ndef _draw_single_box(image, xmin, ymin, xmax, ymax, display_str, color='black', color_text='black', thickness=2):\n from PIL import ImageDraw, ImageFont\n font = ImageFont.load_default()\n draw = ImageDraw.Draw(image)\n (left, right, top, bottom) = (xmin, xmax, ymin, ymax)\n draw.line([(left, top), (left, bottom), (right, bottom),\n (right, top), (left, top)], width=thickness, fill=color)\n if display_str:\n text_bottom = bottom\n # Reverse list and print from bottom to top.\n text_width, text_height = font.getsize(display_str)\n margin = np.ceil(0.05 * text_height)\n draw.rectangle(\n [(left, text_bottom - text_height - 2 * margin),\n (left + text_width, text_bottom)], fill=color\n )\n draw.text(\n (left + margin, text_bottom - text_height - margin),\n display_str, fill=color_text, font=font\n )\n return image\n\n\ndef hparams(hparam_dict=None, metric_dict=None):\n from tensorboardX.proto.plugin_hparams_pb2 import HParamsPluginData, SessionEndInfo, SessionStartInfo\n from tensorboardX.proto.api_pb2 import Experiment, HParamInfo, MetricInfo, MetricName, Status\n from six import string_types\n\n PLUGIN_NAME = 'hparams'\n PLUGIN_DATA_VERSION = 0\n\n EXPERIMENT_TAG = '_hparams_/experiment'\n SESSION_START_INFO_TAG = '_hparams_/session_start_info'\n SESSION_END_INFO_TAG = '_hparams_/session_end_info'\n\n # TODO: expose other parameters in the future.\n # hp = HParamInfo(name='lr',display_name='learning rate', type=DataType.DATA_TYPE_FLOAT64, domain_interval=Interval(min_value=10, max_value=100)) # noqa E501\n # mt = MetricInfo(name=MetricName(tag='accuracy'), display_name='accuracy', description='', dataset_type=DatasetType.DATASET_VALIDATION) # noqa E501\n # exp = Experiment(name='123', description='456', time_created_secs=100.0, hparam_infos=[hp], metric_infos=[mt], user='tw') # noqa E501\n\n hps = [HParamInfo(name=k) for k in hparam_dict.keys()]\n mts = [MetricInfo(name=MetricName(tag=k)) for k in metric_dict.keys()]\n\n exp = Experiment(hparam_infos=hps, metric_infos=mts)\n\n content = HParamsPluginData(experiment=exp, version=PLUGIN_DATA_VERSION)\n smd = SummaryMetadata(plugin_data=SummaryMetadata.PluginData(plugin_name=PLUGIN_NAME,\n content=content.SerializeToString()))\n exp = Summary(value=[Summary.Value(tag=EXPERIMENT_TAG, metadata=smd)])\n\n ssi = SessionStartInfo()\n for k, v in hparam_dict.items():\n if isinstance(v, string_types):\n ssi.hparams[k].string_value = v\n continue\n\n if isinstance(v, bool):\n ssi.hparams[k].bool_value = v\n continue\n\n if not isinstance(v, int) or not isinstance(v, float):\n v = make_np(v)[0]\n ssi.hparams[k].number_value = v\n\n content = HParamsPluginData(session_start_info=ssi, version=PLUGIN_DATA_VERSION)\n smd = SummaryMetadata(plugin_data=SummaryMetadata.PluginData(plugin_name=PLUGIN_NAME,\n content=content.SerializeToString()))\n ssi = Summary(value=[Summary.Value(tag=SESSION_START_INFO_TAG, metadata=smd)])\n\n sei = SessionEndInfo(status=Status.STATUS_SUCCESS)\n content = HParamsPluginData(session_end_info=sei, version=PLUGIN_DATA_VERSION)\n smd = SummaryMetadata(plugin_data=SummaryMetadata.PluginData(plugin_name=PLUGIN_NAME,\n content=content.SerializeToString()))\n sei = Summary(value=[Summary.Value(tag=SESSION_END_INFO_TAG, metadata=smd)])\n\n return exp, ssi, sei\n\n\ndef scalar(name, scalar, collections=None):\n \"\"\"Outputs a `Summary` protocol buffer containing a single scalar value.\n The generated Summary has a Tensor.proto containing the input Tensor.\n Args:\n name: A name for the generated node. Will also serve as the series name in\n TensorBoard.\n tensor: A real numeric Tensor containing a single value.\n collections: Optional list of graph collections keys. The new summary op is\n added to these collections. Defaults to `[GraphKeys.SUMMARIES]`.\n Returns:\n A scalar `Tensor` of type `string`. Which contains a `Summary` protobuf.\n Raises:\n ValueError: If tensor has the wrong shape or type.\n \"\"\"\n name = _clean_tag(name)\n scalar = make_np(scalar)\n assert(scalar.squeeze().ndim == 0), 'scalar should be 0D'\n scalar = float(scalar)\n return Summary(value=[Summary.Value(tag=name, simple_value=scalar)])\n\n\ndef histogram_raw(name, min, max, num, sum, sum_squares, bucket_limits, bucket_counts):\n # pylint: disable=line-too-long\n \"\"\"Outputs a `Summary` protocol buffer with a histogram.\n The generated\n [`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto)\n has one summary value containing a histogram for `values`.\n Args:\n name: A name for the generated node. Will also serve as a series name in\n TensorBoard.\n min: A float or int min value\n max: A float or int max value\n num: Int number of values\n sum: Float or int sum of all values\n sum_squares: Float or int sum of squares for all values\n bucket_limits: A numeric `Tensor` with upper value per bucket\n bucket_counts: A numeric `Tensor` with number of values per bucket\n Returns:\n A scalar `Tensor` of type `string`. The serialized `Summary` protocol\n buffer.\n \"\"\"\n hist = HistogramProto(min=min,\n max=max,\n num=num,\n sum=sum,\n sum_squares=sum_squares,\n bucket_limit=bucket_limits,\n bucket=bucket_counts)\n return Summary(value=[Summary.Value(tag=name, histo=hist)])\n\n\ndef histogram(name, values, bins, max_bins=None):\n # pylint: disable=line-too-long\n \"\"\"Outputs a `Summary` protocol buffer with a histogram.\n The generated\n [`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto)\n has one summary value containing a histogram for `values`.\n This op reports an `InvalidArgument` error if any value is not finite.\n Args:\n name: A name for the generated node. Will also serve as a series name in\n TensorBoard.\n values: A real numeric `Tensor`. Any shape. Values to use to\n build the histogram.\n Returns:\n A scalar `Tensor` of type `string`. The serialized `Summary` protocol\n buffer.\n \"\"\"\n name = _clean_tag(name)\n values = make_np(values)\n hist = make_histogram(values.astype(float), bins, max_bins)\n return Summary(value=[Summary.Value(tag=name, histo=hist)])\n\n\ndef make_histogram(values, bins, max_bins=None):\n \"\"\"Convert values into a histogram proto using logic from histogram.cc.\"\"\"\n if values.size == 0:\n raise ValueError('The input has no element.')\n values = values.reshape(-1)\n counts, limits = np.histogram(values, bins=bins)\n num_bins = len(counts)\n if max_bins is not None and num_bins > max_bins:\n subsampling = num_bins // max_bins\n subsampling_remainder = num_bins % subsampling\n if subsampling_remainder != 0:\n counts = np.pad(counts, pad_width=[[0, subsampling - subsampling_remainder]],\n mode=\"constant\", constant_values=0)\n counts = counts.reshape(-1, subsampling).sum(axis=-1)\n new_limits = np.empty((counts.size + 1,), limits.dtype)\n new_limits[:-1] = limits[:-1:subsampling]\n new_limits[-1] = limits[-1]\n limits = new_limits\n\n # Find the first and the last bin defining the support of the histogram:\n cum_counts = np.cumsum(np.greater(counts, 0, dtype=np.int32))\n start, end = np.searchsorted(cum_counts, [0, cum_counts[-1] - 1], side=\"right\")\n start = int(start)\n end = int(end) + 1\n del cum_counts\n\n # TensorBoard only includes the right bin limits. To still have the leftmost limit\n # included, we include an empty bin left.\n # If start == 0, we need to add an empty one left, otherwise we can just include the bin left to the\n # first nonzero-count bin:\n counts = counts[start - 1:end] if start > 0 else np.concatenate([[0], counts[:end]])\n limits = limits[start:end + 1]\n\n if counts.size == 0 or limits.size == 0:\n raise ValueError('The histogram is empty, please file a bug report.')\n\n sum_sq = values.dot(values)\n return HistogramProto(min=values.min(),\n max=values.max(),\n num=len(values),\n sum=values.sum(),\n sum_squares=sum_sq,\n bucket_limit=limits.tolist(),\n bucket=counts.tolist())\n\n\ndef image(tag, tensor, rescale=1, dataformats='CHW'):\n \"\"\"Outputs a `Summary` protocol buffer with images.\n The summary has up to `max_images` summary values containing images. The\n images are built from `tensor` which must be 3-D with shape `[height, width,\n channels]` and where `channels` can be:\n * 1: `tensor` is interpreted as Grayscale.\n * 3: `tensor` is interpreted as RGB.\n * 4: `tensor` is interpreted as RGBA.\n\n Args:\n tag: A name for the generated node. Will also serve as a series name in\n TensorBoard.\n tensor: A 3-D `uint8` or `float32` `Tensor` of shape `[height, width,\n channels]` where `channels` is 1, 3, or 4.\n 'tensor' can either have values in [0, 1] (float32) or [0, 255] (uint8).\n The image() function will scale the image values to [0, 255] by applying\n a scale factor of either 1 (uint8) or 255 (float32).\n Returns:\n A scalar `Tensor` of type `string`. The serialized `Summary` protocol\n buffer.\n \"\"\"\n tag = _clean_tag(tag)\n tensor = make_np(tensor)\n tensor = convert_to_HWC(tensor, dataformats)\n # Do not assume that user passes in values in [0, 255], use data type to detect\n if tensor.dtype != np.uint8:\n tensor = (tensor * 255.0).astype(np.uint8)\n\n image = make_image(tensor, rescale=rescale)\n return Summary(value=[Summary.Value(tag=tag, image=image)])\n\n\ndef image_boxes(tag, tensor_image, tensor_boxes, rescale=1, dataformats='CHW', labels=None):\n '''Outputs a `Summary` protocol buffer with images.'''\n tensor_image = make_np(tensor_image)\n tensor_image = convert_to_HWC(tensor_image, dataformats)\n tensor_boxes = make_np(tensor_boxes)\n\n if tensor_image.dtype != np.uint8:\n tensor_image = (tensor_image * 255.0).astype(np.uint8)\n\n image = make_image(tensor_image,\n rescale=rescale,\n rois=tensor_boxes, labels=labels)\n return Summary(value=[Summary.Value(tag=tag, image=image)])\n\n\ndef draw_boxes(disp_image, boxes, labels=None):\n # xyxy format\n num_boxes = boxes.shape[0]\n list_gt = range(num_boxes)\n for i in list_gt:\n disp_image = _draw_single_box(disp_image,\n boxes[i, 0],\n boxes[i, 1],\n boxes[i, 2],\n boxes[i, 3],\n display_str=None if labels is None else labels[i],\n color='Red')\n return disp_image\n\n\ndef make_image(tensor, rescale=1, rois=None, labels=None):\n \"\"\"Convert an numpy representation image to Image protobuf\"\"\"\n from PIL import Image\n height, width, channel = tensor.shape\n scaled_height = int(height * rescale)\n scaled_width = int(width * rescale)\n image = Image.fromarray(tensor)\n if rois is not None:\n image = draw_boxes(image, rois, labels=labels)\n image = image.resize((scaled_width, scaled_height), Image.ANTIALIAS)\n import io\n output = io.BytesIO()\n image.save(output, format='PNG')\n image_string = output.getvalue()\n output.close()\n return Summary.Image(height=height,\n width=width,\n colorspace=channel,\n encoded_image_string=image_string)\n\n\ndef video(tag, tensor, fps=4):\n tag = _clean_tag(tag)\n tensor = make_np(tensor)\n tensor = _prepare_video(tensor)\n # If user passes in uint8, then we don't need to rescale by 255\n if tensor.dtype != np.uint8:\n tensor = (tensor * 255.0).astype(np.uint8)\n\n video = make_video(tensor, fps)\n return Summary(value=[Summary.Value(tag=tag, image=video)])\n\n\ndef make_video(tensor, fps):\n try:\n import moviepy # noqa: F401\n except ImportError:\n print('add_video needs package moviepy')\n return\n try:\n from moviepy import editor as mpy\n except ImportError:\n print(\"moviepy is installed, but can't import moviepy.editor.\",\n \"Some packages could be missing [imageio, requests]\")\n return\n import tempfile\n\n t, h, w, c = tensor.shape\n\n # encode sequence of images into gif string\n clip = mpy.ImageSequenceClip(list(tensor), fps=fps)\n\n filename = tempfile.NamedTemporaryFile(suffix='.gif', delete=False).name\n try: # older version of moviepy does not support progress_bar argument.\n clip.write_gif(filename, verbose=False, progress_bar=False)\n except TypeError:\n clip.write_gif(filename, verbose=False)\n\n with open(filename, 'rb') as f:\n tensor_string = f.read()\n\n try:\n os.remove(filename)\n except OSError:\n logging.warning('The temporary file used by moviepy cannot be deleted.')\n\n return Summary.Image(height=h, width=w, colorspace=c, encoded_image_string=tensor_string)\n\n\ndef audio(tag, tensor, sample_rate=44100):\n tensor = make_np(tensor)\n if abs(tensor).max() > 1:\n print('warning: audio amplitude out of range, auto clipped.')\n tensor = tensor.clip(-1, 1)\n assert(tensor.ndim == 2), 'input tensor should be 2 dimensional.'\n length_frames, num_channels = tensor.shape\n assert num_channels == 1 or num_channels == 2, f'Expected 1/2 channels, got {num_channels}'\n import soundfile\n import io\n with io.BytesIO() as fio:\n soundfile.write(fio, tensor, samplerate=sample_rate, format='wav')\n audio_string = fio.getvalue()\n audio = Summary.Audio(sample_rate=sample_rate,\n num_channels=num_channels,\n length_frames=length_frames,\n encoded_audio_string=audio_string,\n content_type='audio/wav')\n return Summary(value=[Summary.Value(tag=tag, audio=audio)])\n\ndef custom_scalars(layout):\n categoriesnames = layout.keys()\n categories = []\n layouts = []\n for k, v in layout.items():\n charts = []\n for chart_name, chart_meatadata in v.items():\n tags = chart_meatadata[1]\n if chart_meatadata[0] == 'Margin':\n assert len(tags) == 3\n mgcc = layout_pb2.MarginChartContent(series=[layout_pb2.MarginChartContent.Series(value=tags[0],\n lower=tags[1],\n upper=tags[2])])\n chart = layout_pb2.Chart(title=chart_name, margin=mgcc)\n else:\n mlcc = layout_pb2.MultilineChartContent(tag=tags)\n chart = layout_pb2.Chart(title=chart_name, multiline=mlcc)\n charts.append(chart)\n categories.append(layout_pb2.Category(title=k, chart=charts))\n\n layout = layout_pb2.Layout(category=categories)\n PluginData = SummaryMetadata.PluginData(plugin_name='custom_scalars')\n smd = SummaryMetadata(plugin_data=PluginData)\n tensor = TensorProto(dtype='DT_STRING',\n string_val=[layout.SerializeToString()],\n tensor_shape=TensorShapeProto())\n return Summary(value=[Summary.Value(tag='custom_scalars__config__', tensor=tensor, metadata=smd)])\n\n\ndef text(tag, text):\n import json\n PluginData = SummaryMetadata.PluginData(\n plugin_name='text', content=TextPluginData(version=0).SerializeToString())\n smd = SummaryMetadata(plugin_data=PluginData)\n tensor = TensorProto(dtype='DT_STRING',\n string_val=[text.encode(encoding='utf_8')],\n tensor_shape=TensorShapeProto(dim=[TensorShapeProto.Dim(size=1)]))\n return Summary(value=[Summary.Value(tag=tag + '/text_summary', metadata=smd, tensor=tensor)])\n\n\ndef pr_curve_raw(tag, tp, fp, tn, fn, precision, recall, num_thresholds=127, weights=None):\n if num_thresholds > 127: # weird, value > 127 breaks protobuf\n num_thresholds = 127\n data = np.stack((tp, fp, tn, fn, precision, recall))\n pr_curve_plugin_data = PrCurvePluginData(\n version=0, num_thresholds=num_thresholds).SerializeToString()\n PluginData = SummaryMetadata.PluginData(\n plugin_name='pr_curves', content=pr_curve_plugin_data)\n smd = SummaryMetadata(plugin_data=PluginData)\n tensor = TensorProto(dtype='DT_FLOAT',\n float_val=data.reshape(-1).tolist(),\n tensor_shape=TensorShapeProto(\n dim=[TensorShapeProto.Dim(size=data.shape[0]), TensorShapeProto.Dim(size=data.shape[1])]))\n return Summary(value=[Summary.Value(tag=tag, metadata=smd, tensor=tensor)])\n\n\ndef pr_curve(tag, labels, predictions, num_thresholds=127, weights=None):\n # weird, value > 127 breaks protobuf\n num_thresholds = min(num_thresholds, 127)\n data = compute_curve(labels, predictions,\n num_thresholds=num_thresholds, weights=weights)\n pr_curve_plugin_data = PrCurvePluginData(\n version=0, num_thresholds=num_thresholds).SerializeToString()\n PluginData = SummaryMetadata.PluginData(\n plugin_name='pr_curves', content=pr_curve_plugin_data)\n smd = SummaryMetadata(plugin_data=PluginData)\n tensor = TensorProto(dtype='DT_FLOAT',\n float_val=data.reshape(-1).tolist(),\n tensor_shape=TensorShapeProto(\n dim=[TensorShapeProto.Dim(size=data.shape[0]), TensorShapeProto.Dim(size=data.shape[1])]))\n return Summary(value=[Summary.Value(tag=tag, metadata=smd, tensor=tensor)])\n\n\n# https://github.com/tensorflow/tensorboard/blob/master/tensorboard/plugins/pr_curve/summary.py\ndef compute_curve(labels, predictions, num_thresholds=None, weights=None):\n _MINIMUM_COUNT = 1e-7\n\n if weights is None:\n weights = 1.0\n\n # Compute bins of true positives and false positives.\n bucket_indices = np.int32(np.floor(predictions * (num_thresholds - 1)))\n float_labels = labels.astype(np.float)\n histogram_range = (0, num_thresholds - 1)\n tp_buckets, _ = np.histogram(\n bucket_indices,\n bins=num_thresholds,\n range=histogram_range,\n weights=float_labels * weights)\n fp_buckets, _ = np.histogram(\n bucket_indices,\n bins=num_thresholds,\n range=histogram_range,\n weights=(1.0 - float_labels) * weights)\n\n # Obtain the reverse cumulative sum.\n tp = np.cumsum(tp_buckets[::-1])[::-1]\n fp = np.cumsum(fp_buckets[::-1])[::-1]\n tn = fp[0] - fp\n fn = tp[0] - tp\n precision = tp / np.maximum(_MINIMUM_COUNT, tp + fp)\n recall = tp / np.maximum(_MINIMUM_COUNT, tp + fn)\n return np.stack((tp, fp, tn, fn, precision, recall))\n\n\ndef _get_tensor_summary(tag, tensor, content_type, json_config):\n mesh_plugin_data = MeshPluginData(\n version=0,\n name=tag,\n content_type=content_type,\n json_config=json_config,\n shape=tensor.shape,\n )\n content = mesh_plugin_data.SerializeToString()\n smd = SummaryMetadata(\n plugin_data=SummaryMetadata.PluginData(\n plugin_name='mesh',\n content=content))\n\n tensor = TensorProto(dtype='DT_FLOAT',\n float_val=tensor.reshape(-1).tolist(),\n tensor_shape=TensorShapeProto(dim=[\n TensorShapeProto.Dim(size=tensor.shape[0]),\n TensorShapeProto.Dim(size=tensor.shape[1]),\n TensorShapeProto.Dim(size=tensor.shape[2]),\n ]))\n tensor_summary = Summary.Value(\n tag='{}_{}'.format(tag, content_type),\n tensor=tensor,\n metadata=smd,\n )\n return tensor_summary\n\n\ndef mesh(tag, vertices, colors, faces, config_dict=None):\n\n import json\n summaries = []\n tensors = [\n (vertices, 1),\n (faces, 2),\n (colors, 3)\n ]\n\n for tensor, content_type in tensors:\n if tensor is None:\n continue\n summaries.append(\n _get_tensor_summary(tag, make_np(tensor), content_type, json.dumps(config_dict, sort_keys=True)))\n\n return Summary(value=summaries)\n" }, { "path": "tensorboardX/tensorboardX/torchvis.py", "content": "from __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\n\nimport gc\nimport six\nimport time\n\nfrom functools import wraps\nfrom .writer import SummaryWriter\nfrom .visdom_writer import VisdomWriter\n\n\n# Supports both TensorBoard and Visdom (no embedding or graph visualization with Visdom)\nvis_formats = {'tensorboard': SummaryWriter, 'visdom': VisdomWriter}\n\n\nclass TorchVis:\n def __init__(self, *args, **init_kwargs):\n \"\"\"\n Args:\n args (list of strings): The name of the visualization target(s).\n Accepted targets are 'tensorboard' and 'visdom'.\n init_kwargs: Additional keyword parameters for the visdom writer (For example, server IP).\n See https://github.com/facebookresearch/visdom/blob/master/README.md#visdom-arguments-python-only\n for more.\n \"\"\"\n self.subscribers = {}\n self.register(*args, **init_kwargs)\n\n def register(self, *args, **init_kwargs):\n # Sets tensorboard as the default visualization format if not specified\n formats = ['tensorboard'] if not args else args\n for format in formats:\n if self.subscribers.get(format) is None and format in vis_formats.keys():\n self.subscribers[format] = vis_formats[format](**init_kwargs.get(format, {}))\n\n def unregister(self, *args):\n for format in args:\n self.subscribers[format].close()\n del self.subscribers[format]\n gc.collect()\n\n def __getattr__(self, attr):\n for _, subscriber in six.iteritems(self.subscribers):\n def wrapper(*args, **kwargs):\n for _, subscriber in six.iteritems(self.subscribers):\n if hasattr(subscriber, attr):\n getattr(subscriber, attr)(*args, **kwargs)\n return wrapper\n raise AttributeError\n\n # Handle writer management (open/close) for the user\n def __del__(self):\n for _, subscriber in six.iteritems(self.subscribers):\n subscriber.close()\n" }, { "path": "tensorboardX/tensorboardX/utils.py", "content": "# Functions for converting\ndef figure_to_image(figures, close=True):\n \"\"\"Render matplotlib figure to numpy format.\n\n Note that this requires the ``matplotlib`` package.\n\n Args:\n figure (matplotlib.pyplot.figure) or list of figures: figure or a list of figures\n close (bool): Flag to automatically close the figure\n\n Returns:\n numpy.array: image in [CHW] order\n \"\"\"\n import numpy as np\n try:\n import matplotlib.pyplot as plt\n import matplotlib.backends.backend_agg as plt_backend_agg\n except ModuleNotFoundError:\n print('please install matplotlib')\n\n def render_to_rgb(figure):\n canvas = plt_backend_agg.FigureCanvasAgg(figure)\n canvas.draw()\n data = np.frombuffer(canvas.buffer_rgba(), dtype=np.uint8)\n w, h = figure.canvas.get_width_height()\n image_hwc = data.reshape([h, w, 4])[:, :, 0:3]\n image_chw = np.moveaxis(image_hwc, source=2, destination=0)\n if close:\n plt.close(figure)\n return image_chw\n\n if isinstance(figures, list):\n images = [render_to_rgb(figure) for figure in figures]\n return np.stack(images)\n else:\n image = render_to_rgb(figures)\n return image\n\n\ndef graphviz_to_image():\n pass\n\n\ndef _prepare_video(V):\n import numpy as np\n b, t, c, h, w = V.shape\n\n if V.dtype == np.uint8:\n V = np.float32(V) / 255.\n\n def is_power2(num):\n return num != 0 and ((num & (num - 1)) == 0)\n\n # pad to nearest power of 2, all at once\n if not is_power2(V.shape[0]):\n len_addition = int(2**V.shape[0].bit_length() - V.shape[0])\n V = np.concatenate(\n (V, np.zeros(shape=(len_addition, t, c, h, w))), axis=0)\n\n n_rows = 2**((b.bit_length() - 1) // 2)\n n_cols = V.shape[0] // n_rows\n\n V = np.reshape(V, newshape=(n_rows, n_cols, t, c, h, w))\n V = np.transpose(V, axes=(2, 0, 4, 1, 5, 3))\n V = np.reshape(V, newshape=(t, n_rows * h, n_cols * w, c))\n\n return V\n\n\ndef make_grid(I, ncols=8):\n # I: N1HW or N3HW\n import numpy as np\n assert isinstance(\n I, np.ndarray), 'plugin error, should pass numpy array here'\n if I.shape[1] == 1:\n I = np.concatenate([I, I, I], 1)\n assert I.ndim == 4 and I.shape[1] == 3 or I.shape[1] == 4\n nimg = I.shape[0]\n H = I.shape[2]\n W = I.shape[3]\n ncols = min(nimg, ncols)\n nrows = int(np.ceil(float(nimg) / ncols))\n canvas = np.zeros((I.shape[1], H * nrows, W * ncols))\n i = 0\n for y in range(nrows):\n for x in range(ncols):\n if i >= nimg:\n break\n canvas[:, y * H:(y + 1) * H, x * W:(x + 1) * W] = I[i]\n i = i + 1\n return canvas\n\n # if modality == 'IMG':\n # if x.dtype == np.uint8:\n # x = x.astype(np.float32) / 255.0\n\n\ndef convert_to_HWC(tensor, input_format): # tensor: numpy array\n import numpy as np\n assert(len(set(input_format)) == len(input_format)), \"You can not use the same dimension shordhand twice. \\\n input_format: {}\".format(input_format)\n assert(len(tensor.shape) == len(input_format)), \"size of input tensor and input format are different. \\\n tensor shape: {}, input_format: {}\".format(tensor.shape, input_format)\n input_format = input_format.upper()\n\n if len(input_format) == 4:\n index = [input_format.find(c) for c in 'NCHW']\n tensor_NCHW = tensor.transpose(index)\n tensor_CHW = make_grid(tensor_NCHW)\n return tensor_CHW.transpose(1, 2, 0)\n\n if len(input_format) == 3:\n index = [input_format.find(c) for c in 'HWC']\n tensor_HWC = tensor.transpose(index)\n if tensor_HWC.shape[2] == 1:\n tensor_HWC = np.concatenate([tensor_HWC, tensor_HWC, tensor_HWC], 2)\n return tensor_HWC\n\n if len(input_format) == 2:\n index = [input_format.find(c) for c in 'HW']\n tensor = tensor.transpose(index)\n tensor = np.stack([tensor, tensor, tensor], 2)\n return tensor\n" }, { "path": "tensorboardX/tensorboardX/visdom_writer.py", "content": "import gc\nimport numpy as np\nimport math\nimport json\nimport time\n\nfrom .summary import compute_curve\nfrom .utils import figure_to_image\nfrom .x2num import make_np\n\n\n# Decorator that checks if there is a Visdom connection\ndef _check_connection(fn):\n def wrapper(self, *args, **kwargs):\n if not self.server_connected:\n print('ERROR: No Visdom server currently connected')\n self._try_connect()\n return\n fn(self, *args, **kwargs)\n return wrapper\n\n\nclass VisdomWriter:\n def __init__(self, *args, **kwargs):\n try:\n from visdom import Visdom\n except ImportError:\n raise ImportError(\n \"Visdom visualization requires installation of Visdom\")\n\n self.scalar_dict = {}\n self.server_connected = False\n self.vis = Visdom(*args, **kwargs)\n self.windows = {}\n\n self._try_connect()\n\n def _try_connect(self):\n startup_sec = 1\n self.server_connected = self.vis.check_connection()\n while not self.server_connected and startup_sec > 0:\n time.sleep(0.1)\n startup_sec -= 0.1\n self.server_connected = self.vis.check_connection()\n assert self.server_connected, 'No connection could be formed quickly'\n\n @_check_connection\n def add_scalar(self, tag, scalar_value, global_step=None, main_tag='default'):\n \"\"\"Add scalar data to Visdom. Plots the values in a plot titled\n {main_tag}-{tag}.\n\n Args:\n tag (string): Data identifier\n scalar_value (float or string/blobname): Value to save\n global_step (int): Global step value to record\n main_tag (string): Data group identifier\n \"\"\"\n if self.scalar_dict.get(main_tag) is None:\n self.scalar_dict[main_tag] = {}\n exists = self.scalar_dict[main_tag].get(tag) is not None\n self.scalar_dict[main_tag][tag] = self.scalar_dict[main_tag][tag] + \\\n [scalar_value] if exists else [scalar_value]\n plot_name = '{}-{}'.format(main_tag, tag)\n # If there is no global_step provided, follow sequential order\n x_val = len(self.scalar_dict[main_tag][tag]\n ) if not global_step else global_step\n if exists:\n # Update our existing Visdom window\n self.vis.line(\n X=make_np(x_val),\n Y=make_np(scalar_value),\n name=plot_name,\n update='append',\n win=self.windows[plot_name],\n )\n else:\n # Save the window if we are creating this graph for the first time\n self.windows[plot_name] = self.vis.line(\n X=make_np(x_val),\n Y=make_np(scalar_value),\n name=plot_name,\n opts={\n 'title': plot_name,\n 'xlabel': 'timestep',\n 'ylabel': tag,\n },\n )\n\n @_check_connection\n def add_scalars(self, main_tag, tag_scalar_dict, global_step=None):\n \"\"\"Adds many scalar data to summary.\n\n Note that this function also keeps logged scalars in memory. In extreme case it explodes your RAM.\n\n Args:\n tag (string): Data identifier\n main_tag (string): Data group identifier\n tag_scalar_dict (dict): Key-value pair storing the tag and corresponding values\n global_step (int): Global step value to record\n\n Examples::\n\n writer.add_scalars('run_14h',{'xsinx':i*np.sin(i/r),\n 'xcosx':i*np.cos(i/r),\n 'arctanx': numsteps*np.arctan(i/r)}, i)\n This function adds three plots:\n 'run_14h-xsinx',\n 'run_14h-xcosx',\n 'run_14h-arctanx'\n with the corresponding values.\n \"\"\"\n for key in tag_scalar_dict.keys():\n self.add_scalar(key, tag_scalar_dict[key], global_step, main_tag)\n\n @_check_connection\n def export_scalars_to_json(self, path):\n \"\"\"Exports to the given 'path' an ASCII file containing all the scalars written\n so far by this instance, with the following format:\n {writer_id : [[timestamp, step, value], ...], ...}\n\n The scalars saved by ``add_scalars()`` will be flushed after export.\n \"\"\"\n with open(path, \"w\") as f:\n json.dump(self.scalar_dict, f)\n self.scalar_dict = {}\n\n @_check_connection\n def add_histogram(self, tag, values, global_step=None, bins='tensorflow'):\n \"\"\"Add histogram to summary.\n\n Args:\n tag (string): Data identifier\n values (torch.Tensor, numpy.array, or string/blobname): Values to build histogram\n global_step (int): Global step value to record\n bins (string): one of {'tensorflow', 'auto', 'fd', ...}, this determines how the bins are made. You can find\n other options in: https://docs.scipy.org/doc/numpy/reference/generated/numpy.histogram.html\n \"\"\"\n values = make_np(values)\n self.vis.histogram(make_np(values), opts={'title': tag})\n\n @_check_connection\n def add_image(self, tag, img_tensor, global_step=None, caption=None):\n \"\"\"Add image data to summary.\n\n Note that this requires the ``pillow`` package.\n\n Args:\n tag (string): Data identifier\n img_tensor (torch.Tensor, numpy.array, or string/blobname): Image data\n global_step (int): Global step value to record\n Shape:\n img_tensor: :math:`(C, H, W)`. Use ``torchvision.utils.make_grid()`` to prepare it is a good idea.\n C = colors (can be 1 - grayscale, 3 - RGB, 4 - RGBA)\n \"\"\"\n img_tensor = make_np(img_tensor)\n self.vis.image(img_tensor, opts={'title': tag, 'caption': caption})\n\n @_check_connection\n def add_figure(self, tag, figure, global_step=None, close=True):\n \"\"\"Render matplotlib figure into an image and add it to summary.\n\n Note that this requires the ``matplotlib`` package.\n\n Args:\n tag (string): Data identifier\n figure (matplotlib.pyplot.figure) or list of figures: figure or a list of figures\n global_step (int): Global step value to record\n close (bool): Flag to automatically close the figure\n \"\"\"\n self.add_image(tag, figure_to_image(figure, close), global_step)\n\n @_check_connection\n def add_video(self, tag, vid_tensor, global_step=None, fps=4):\n \"\"\"Add video data to summary.\n\n Note that this requires the ``moviepy`` package.\n\n Args:\n tag (string): Data identifier\n vid_tensor (torch.Tensor): Video data\n global_step (int): Global step value to record\n fps (float or int): Frames per second\n Shape:\n vid_tensor: :math:`(B, C, T, H, W)`. (if following tensorboardX format)\n vid_tensor: :math:`(T, H, W, C)`. (if following visdom format)\n B = batches, C = colors (1, 3, or 4), T = time frames, H = height, W = width\n \"\"\"\n shape = vid_tensor.shape\n # A batch of videos (tensorboardX format) is a 5D tensor\n if len(shape) > 4:\n for i in range(shape[0]):\n # Reshape each video to Visdom's (T x H x W x C) and write each video\n # TODO: reverse the logic here, shoudl do the permutation in numpy\n if isinstance(vid_tensor, np.ndarray):\n import torch\n ind_vid = torch.from_numpy(\n vid_tensor[i, :, :, :, :]).permute(1, 2, 3, 0)\n else:\n ind_vid = vid_tensor[i, :, :, :, :].permute(1, 2, 3, 0)\n scale_factor = 255 if np.any(\n (ind_vid > 0) & (ind_vid < 1)) else 1\n # Visdom looks for .ndim attr, this is something raw Tensors don't have\n # Cast to Numpy array to get .ndim attr\n ind_vid = ind_vid.numpy()\n ind_vid = (ind_vid * scale_factor).astype(np.uint8)\n assert ind_vid.shape[3] in [1, 3, 4], \\\n 'Visdom requires the last dimension to be color, which can be 1 (grayscale), 3 (RGB) or 4 (RGBA)'\n self.vis.video(tensor=ind_vid, opts={'fps': fps})\n else:\n self.vis.video(tensor=vid_tensor, opts={'fps': fps})\n\n @_check_connection\n def add_audio(self, tag, snd_tensor, global_step=None, sample_rate=44100):\n \"\"\"Add audio data to summary.\n\n Args:\n tag (string): Data identifier\n snd_tensor (torch.Tensor, numpy.array, or string/blobname): Sound data\n global_step (int): Global step value to record\n sample_rate (int): sample rate in Hz\n\n Shape:\n snd_tensor: :math:`(1, L)`. The values should lie between [-1, 1].\n \"\"\"\n snd_tensor = make_np(snd_tensor)\n self.vis.audio(tensor=snd_tensor, opts={\n 'sample_frequency': sample_rate})\n\n @_check_connection\n def add_text(self, tag, text_string, global_step=None):\n \"\"\"Add text data to summary.\n\n Args:\n tag (string): Data identifier\n text_string (string): String to save\n global_step (int): Global step value to record\n Examples::\n writer.add_text('lstm', 'This is an lstm', 0)\n writer.add_text('rnn', 'This is an rnn', 10)\n \"\"\"\n if text_string is None:\n # Visdom doesn't support tags, write the tag as the text_string\n text_string = tag\n self.vis.text(text_string)\n\n @_check_connection\n def add_onnx_graph(self, prototxt):\n # TODO: Visdom doesn't support graph visualization yet, so this is a no-op\n return\n\n @_check_connection\n def add_graph(self, model, input_to_model=None, verbose=False, **kwargs):\n # TODO: Visdom doesn't support graph visualization yet, so this is a no-op\n return\n\n @_check_connection\n def add_embedding(self, mat, metadata=None, label_img=None, global_step=None, tag='default', metadata_header=None):\n # TODO: Visdom doesn't support embeddings yet, so this is a no-op\n return\n\n @_check_connection\n def add_pr_curve(self, tag, labels, predictions, global_step=None, num_thresholds=127, weights=None):\n \"\"\"Adds precision recall curve.\n\n Args:\n tag (string): Data identifier\n labels (torch.Tensor, numpy.array, or string/blobname): Ground truth data. Binary label for each element.\n predictions (torch.Tensor, numpy.array, or string/blobname):\n The probability that an element be classified as true. Value should in [0, 1]\n global_step (int): Global step value to record\n num_thresholds (int): Number of thresholds used to draw the curve.\n\n \"\"\"\n labels, predictions = make_np(labels), make_np(predictions)\n raw_data = compute_curve(labels, predictions, num_thresholds, weights)\n\n # compute_curve returns np.stack((tp, fp, tn, fn, precision, recall))\n # We want to access 'precision' and 'recall'\n precision, recall = raw_data[4, :], raw_data[5, :]\n\n self.vis.line(\n X=recall,\n Y=precision,\n name=tag,\n opts={\n 'title': 'PR Curve for {}'.format(tag),\n 'xlabel': 'recall',\n 'ylabel': 'precision',\n },\n )\n\n @_check_connection\n def add_pr_curve_raw(self, tag, true_positive_counts,\n false_positive_counts,\n true_negative_counts,\n false_negative_counts,\n precision,\n recall, global_step=None, num_thresholds=127, weights=None):\n \"\"\"Adds precision recall curve with raw data.\n\n Args:\n tag (string): Data identifier\n true_positive_counts (torch.Tensor, numpy.array, or string/blobname): true positive counts\n false_positive_counts (torch.Tensor, numpy.array, or string/blobname): false positive counts\n true_negative_counts (torch.Tensor, numpy.array, or string/blobname): true negative counts\n false_negative_counts (torch.Tensor, numpy.array, or string/blobname): false negative counts\n precision (torch.Tensor, numpy.array, or string/blobname): precision\n recall (torch.Tensor, numpy.array, or string/blobname): recall\n global_step (int): Global step value to record\n num_thresholds (int): Number of thresholds used to draw the curve.\n see: https://github.com/tensorflow/tensorboard/blob/master/tensorboard/plugins/pr_curve/README.md\n \"\"\"\n precision, recall = make_np(precision), make_np(recall)\n self.vis.line(\n X=recall,\n Y=precision,\n name=tag,\n opts={\n 'title': 'PR Curve for {}'.format(tag),\n 'xlabel': 'recall',\n 'ylabel': 'precision',\n },\n )\n\n def close(self):\n del self.vis\n del self.scalar_dict\n gc.collect()\n" }, { "path": "tensorboardX/tensorboardX/writer.py", "content": "\"\"\"Provides an API for writing protocol buffers to event files to be\nconsumed by TensorBoard for visualization.\"\"\"\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport json\nimport os\nimport six\nimport time\nimport logging\n\nfrom .embedding import make_mat, make_sprite, make_tsv, append_pbtxt\nfrom .event_file_writer import EventFileWriter\nfrom .onnx_graph import load_onnx_graph\nfrom .pytorch_graph import graph\nfrom .proto import event_pb2\nfrom .proto import summary_pb2\nfrom .proto.event_pb2 import SessionLog, Event\nfrom .utils import figure_to_image\nfrom .summary import (\n scalar, histogram, histogram_raw, image, audio, text,\n pr_curve, pr_curve_raw, video, custom_scalars, image_boxes, mesh, hparams\n)\n\n\nclass DummyFileWriter(object):\n \"\"\"A fake file writer that writes nothing to the disk.\n \"\"\"\n def __init__(self, logdir):\n self._logdir = logdir\n\n def get_logdir(self):\n \"\"\"Returns the directory where event file will be written.\"\"\"\n return self._logdir\n\n def add_event(self, event, step=None, walltime=None):\n return\n\n def add_summary(self, summary, global_step=None, walltime=None):\n return\n\n def add_graph(self, graph_profile, walltime=None):\n return\n\n def add_onnx_graph(self, graph, walltime=None):\n return\n\n def flush(self):\n return\n\n def close(self):\n return\n\n def reopen(self):\n return\n\n\nclass FileWriter(object):\n \"\"\"Writes protocol buffers to event files to be consumed by TensorBoard.\n\n The `FileWriter` class provides a mechanism to create an event file in a\n given directory and add summaries and events to it. The class updates the\n file contents asynchronously. This allows a training program to call methods\n to add data to the file directly from the training loop, without slowing down\n training.\n \"\"\"\n\n def __init__(self, logdir, max_queue=10, flush_secs=120, filename_suffix=''):\n \"\"\"Creates a `FileWriter` and an event file.\n On construction the writer creates a new event file in `logdir`.\n The other arguments to the constructor control the asynchronous writes to\n the event file.\n\n Args:\n logdir: A string. Directory where event file will be written.\n max_queue: Integer. Size of the queue for pending events and\n summaries before one of the 'add' calls forces a flush to disk.\n Default is ten items.\n flush_secs: Number. How often, in seconds, to flush the\n pending events and summaries to disk. Default is every two minutes.\n filename_suffix: A string. Suffix added to all event filenames\n in the logdir directory. More details on filename construction in\n tensorboard.summary.writer.event_file_writer.EventFileWriter.\n \"\"\"\n # Sometimes PosixPath is passed in and we need to coerce it to\n # a string in all cases\n # TODO: See if we can remove this in the future if we are\n # actually the ones passing in a PosixPath\n logdir = str(logdir)\n self.event_writer = EventFileWriter(\n logdir, max_queue, flush_secs, filename_suffix)\n\n def get_logdir(self):\n \"\"\"Returns the directory where event file will be written.\"\"\"\n return self.event_writer.get_logdir()\n\n def add_event(self, event, step=None, walltime=None):\n \"\"\"Adds an event to the event file.\n Args:\n event: An `Event` protocol buffer.\n step: Number. Optional global step value for training process\n to record with the event.\n walltime: float. Optional walltime to override the default (current)\n walltime (from time.time())\n \"\"\"\n event.wall_time = time.time() if walltime is None else walltime\n if step is not None:\n # Make sure step is converted from numpy or other formats\n # since protobuf might not convert depending on version\n event.step = int(step)\n self.event_writer.add_event(event)\n\n def add_summary(self, summary, global_step=None, walltime=None):\n \"\"\"Adds a `Summary` protocol buffer to the event file.\n This method wraps the provided summary in an `Event` protocol buffer\n and adds it to the event file.\n\n Args:\n summary: A `Summary` protocol buffer.\n global_step: Number. Optional global step value for training process\n to record with the summary.\n walltime: float. Optional walltime to override the default (current)\n walltime (from time.time())\n \"\"\"\n event = event_pb2.Event(summary=summary)\n self.add_event(event, global_step, walltime)\n\n def add_graph(self, graph_profile, walltime=None):\n \"\"\"Adds a `Graph` and step stats protocol buffer to the event file.\n\n Args:\n graph_profile: A `Graph` and step stats protocol buffer.\n walltime: float. Optional walltime to override the default (current)\n walltime (from time.time()) seconds after epoch\n \"\"\"\n graph = graph_profile[0]\n stepstats = graph_profile[1]\n event = event_pb2.Event(graph_def=graph.SerializeToString())\n self.add_event(event, None, walltime)\n\n trm = event_pb2.TaggedRunMetadata(\n tag='step1', run_metadata=stepstats.SerializeToString())\n event = event_pb2.Event(tagged_run_metadata=trm)\n self.add_event(event, None, walltime)\n\n def add_onnx_graph(self, graph, walltime=None):\n \"\"\"Adds a `Graph` protocol buffer to the event file.\n\n Args:\n graph: A `Graph` protocol buffer.\n walltime: float. Optional walltime to override the default (current)\n _get_file_writerfrom time.time())\n \"\"\"\n event = event_pb2.Event(graph_def=graph.SerializeToString())\n self.add_event(event, None, walltime)\n\n def flush(self):\n \"\"\"Flushes the event file to disk.\n Call this method to make sure that all pending events have been written to\n disk.\n \"\"\"\n self.event_writer.flush()\n\n def close(self):\n \"\"\"Flushes the event file to disk and close the file.\n Call this method when you do not need the summary writer anymore.\n \"\"\"\n self.event_writer.close()\n\n def reopen(self):\n \"\"\"Reopens the EventFileWriter.\n Can be called after `close()` to add more events in the same directory.\n The events will go into a new events file.\n Does nothing if the EventFileWriter was not closed.\n \"\"\"\n self.event_writer.reopen()\n\n\nclass SummaryWriter(object):\n \"\"\"Writes entries directly to event files in the logdir to be\n consumed by TensorBoard.\n\n The `SummaryWriter` class provides a high-level API to create an event file\n in a given directory and add summaries and events to it. The class updates the\n file contents asynchronously. This allows a training program to call methods\n to add data to the file directly from the training loop, without slowing down\n training.\n \"\"\"\n\n def __init__(self, logdir=None, comment='', purge_step=None, max_queue=10,\n flush_secs=120, filename_suffix='', write_to_disk=True, log_dir=None, **kwargs):\n \"\"\"Creates a `SummaryWriter` that will write out events and summaries\n to the event file.\n\n Args:\n logdir (string): Save directory location. Default is\n runs/**CURRENT_DATETIME_HOSTNAME**, which changes after each run.\n Use hierarchical folder structure to compare\n between runs easily. e.g. pass in 'runs/exp1', 'runs/exp2', etc.\n for each new experiment to compare across them.\n comment (string): Comment logdir suffix appended to the default\n ``logdir``. If ``logdir`` is assigned, this argument has no effect.\n purge_step (int):\n When logging crashes at step :math:`T+X` and restarts at step :math:`T`,\n any events whose global_step larger or equal to :math:`T` will be\n purged and hidden from TensorBoard.\n Note that crashed and resumed experiments should have the same ``logdir``.\n max_queue (int): Size of the queue for pending events and\n summaries before one of the 'add' calls forces a flush to disk.\n Default is ten items.\n flush_secs (int): How often, in seconds, to flush the\n pending events and summaries to disk. Default is every two minutes.\n filename_suffix (string): Suffix added to all event filenames in\n the logdir directory. More details on filename construction in\n tensorboard.summary.writer.event_file_writer.EventFileWriter.\n write_to_disk (boolean):\n If pass `False`, SummaryWriter will not write to disk.\n\n Examples::\n\n from tensorboardX import SummaryWriter\n\n # create a summary writer with automatically generated folder name.\n writer = SummaryWriter()\n # folder location: runs/May04_22-14-54_s-MacBook-Pro.local/\n\n # create a summary writer using the specified folder name.\n writer = SummaryWriter(\"my_experiment\")\n # folder location: my_experiment\n\n # create a summary writer with comment appended.\n writer = SummaryWriter(comment=\"LR_0.1_BATCH_16\")\n # folder location: runs/May04_22-14-54_s-MacBook-Pro.localLR_0.1_BATCH_16/\n\n \"\"\"\n if log_dir is not None and logdir is None:\n logdir = log_dir\n if not logdir:\n import socket\n from datetime import datetime\n current_time = datetime.now().strftime('%b%d_%H-%M-%S')\n logdir = os.path.join(\n 'runs', current_time + '_' + socket.gethostname() + comment)\n self.logdir = logdir\n self.purge_step = purge_step\n self._max_queue = max_queue\n self._flush_secs = flush_secs\n self._filename_suffix = filename_suffix\n self._write_to_disk = write_to_disk\n self.kwargs = kwargs\n\n # Initialize the file writers, but they can be cleared out on close\n # and recreated later as needed.\n self.file_writer = self.all_writers = None\n self._get_file_writer()\n\n # Create default bins for histograms, see generate_testdata.py in tensorflow/tensorboard\n v = 1E-12\n buckets = []\n neg_buckets = []\n while v < 1E20:\n buckets.append(v)\n neg_buckets.append(-v)\n v *= 1.1\n self.default_bins = neg_buckets[::-1] + [0] + buckets\n\n self.scalar_dict = {}\n\n def __append_to_scalar_dict(self, tag, scalar_value, global_step,\n timestamp):\n \"\"\"This adds an entry to the self.scalar_dict datastructure with format\n {writer_id : [[timestamp, step, value], ...], ...}.\n \"\"\"\n from .x2num import make_np\n if tag not in self.scalar_dict.keys():\n self.scalar_dict[tag] = []\n self.scalar_dict[tag].append(\n [timestamp, global_step, float(make_np(scalar_value))])\n\n def _check_caffe2_blob(self, item):\n \"\"\"\n Caffe2 users have the option of passing a string representing the name of\n a blob in the workspace instead of passing the actual Tensor/array containing\n the numeric values. Thus, we need to check if we received a string as input\n instead of an actual Tensor/array, and if so, we need to fetch the Blob\n from the workspace corresponding to that name. Fetching can be done with the\n following:\n\n from caffe2.python import workspace (if not already imported)\n workspace.FetchBlob(blob_name)\n workspace.FetchBlobs([blob_name1, blob_name2, ...])\n \"\"\"\n return isinstance(item, six.string_types)\n\n def _get_file_writer(self):\n \"\"\"Returns the default FileWriter instance. Recreates it if closed.\"\"\"\n if not self._write_to_disk:\n self.file_writer = DummyFileWriter(logdir=self.logdir)\n self.all_writers = {self.file_writer.get_logdir(): self.file_writer}\n return self.file_writer\n\n if self.all_writers is None or self.file_writer is None:\n if 'purge_step' in self.kwargs.keys():\n most_recent_step = self.kwargs.pop('purge_step')\n self.file_writer = FileWriter(logdir=self.logdir,\n max_queue=self._max_queue,\n flush_secs=self._flush_secs,\n filename_suffix=self._filename_suffix,\n **self.kwargs)\n self.file_writer.add_event(\n Event(step=most_recent_step, file_version='brain.Event:2'))\n self.file_writer.add_event(\n Event(step=most_recent_step, session_log=SessionLog(status=SessionLog.START)))\n else:\n self.file_writer = FileWriter(logdir=self.logdir,\n max_queue=self._max_queue,\n flush_secs=self._flush_secs,\n filename_suffix=self._filename_suffix,\n **self.kwargs)\n self.all_writers = {self.file_writer.get_logdir(): self.file_writer}\n return self.file_writer\n\n def add_hparams(self, hparam_dict=None, metric_dict=None):\n \"\"\"Add a set of hyperparameters to be compared in tensorboard.\n\n Args:\n hparam_dict (dictionary): Each key-value pair in the dictionary is the\n name of the hyper parameter and it's corresponding value.\n metric_dict (dictionary): Each key-value pair in the dictionary is the\n name of the metric and it's corresponding value. Note that the key used\n here should be unique in the tensorboard record. Otherwise the value\n you added by `add_scalar` will be displayed in hparam plugin. In most\n cases, this is unwanted.\n\n Examples::\n\n from tensorboardX import SummaryWriter\n with SummaryWriter() as w:\n for i in range(5):\n w.add_hparams({'lr': 0.1*i, 'bsize': i},\n {'hparam/accuracy': 10*i, 'hparam/loss': 10*i})\n\n Expected result:\n\n .. image:: _static/img/tensorboard/add_hparam.png\n :scale: 50 %\n \"\"\"\n if type(hparam_dict) is not dict or type(metric_dict) is not dict:\n raise TypeError('hparam_dict and metric_dict should be dictionary.')\n exp, ssi, sei = hparams(hparam_dict, metric_dict)\n\n with SummaryWriter(logdir=os.path.join(self.file_writer.get_logdir(), str(time.time()))) as w_hp:\n w_hp.file_writer.add_summary(exp)\n w_hp.file_writer.add_summary(ssi)\n w_hp.file_writer.add_summary(sei)\n for k, v in metric_dict.items():\n w_hp.add_scalar(k, v)\n\n def add_scalar(self, tag, scalar_value, global_step=None, walltime=None):\n \"\"\"Add scalar data to summary.\n\n Args:\n tag (string): Data identifier\n scalar_value (float or string/blobname): Value to save\n global_step (int): Global step value to record\n walltime (float): Optional override default walltime (time.time()) of event\n\n Examples::\n\n from tensorboardX import SummaryWriter\n writer = SummaryWriter()\n x = range(100)\n for i in x:\n writer.add_scalar('y=2x', i * 2, i)\n writer.close()\n\n Expected result:\n\n .. image:: _static/img/tensorboard/add_scalar.png\n :scale: 50 %\n\n \"\"\"\n if self._check_caffe2_blob(scalar_value):\n scalar_value = workspace.FetchBlob(scalar_value)\n self._get_file_writer().add_summary(\n scalar(tag, scalar_value), global_step, walltime)\n\n def add_scalars(self, main_tag, tag_scalar_dict, global_step=None, walltime=None):\n \"\"\"Adds many scalar data to summary.\n\n Note that this function also keeps logged scalars in memory. In extreme case it explodes your RAM.\n\n Args:\n main_tag (string): The parent name for the tags\n tag_scalar_dict (dict): Key-value pair storing the tag and corresponding values\n global_step (int): Global step value to record\n walltime (float): Optional override default walltime (time.time()) of event\n\n Examples::\n\n from tensorboardX import SummaryWriter\n writer = SummaryWriter()\n r = 5\n for i in range(100):\n writer.add_scalars('run_14h', {'xsinx':i*np.sin(i/r),\n 'xcosx':i*np.cos(i/r),\n 'tanx': np.tan(i/r)}, i)\n writer.close()\n # This call adds three values to the same scalar plot with the tag\n # 'run_14h' in TensorBoard's scalar section.\n\n Expected result:\n\n .. image:: _static/img/tensorboard/add_scalars.png\n :scale: 50 %\n\n \"\"\"\n walltime = time.time() if walltime is None else walltime\n fw_logdir = self._get_file_writer().get_logdir()\n for tag, scalar_value in tag_scalar_dict.items():\n fw_tag = fw_logdir + \"/\" + main_tag + \"/\" + tag\n if fw_tag in self.all_writers.keys():\n fw = self.all_writers[fw_tag]\n else:\n fw = FileWriter(logdir=fw_tag)\n self.all_writers[fw_tag] = fw\n if self._check_caffe2_blob(scalar_value):\n scalar_value = workspace.FetchBlob(scalar_value)\n fw.add_summary(scalar(main_tag, scalar_value),\n global_step, walltime)\n self.__append_to_scalar_dict(\n fw_tag, scalar_value, global_step, walltime)\n\n def export_scalars_to_json(self, path):\n \"\"\"Exports to the given path an ASCII file containing all the scalars written\n so far by this instance, with the following format:\n {writer_id : [[timestamp, step, value], ...], ...}\n\n The scalars saved by ``add_scalars()`` will be flushed after export.\n \"\"\"\n with open(path, \"w\") as f:\n json.dump(self.scalar_dict, f)\n self.scalar_dict = {}\n\n def add_histogram(self, tag, values, global_step=None, bins='tensorflow', walltime=None, max_bins=None):\n \"\"\"Add histogram to summary.\n\n Args:\n tag (string): Data identifier\n values (torch.Tensor, numpy.array, or string/blobname): Values to build histogram\n global_step (int): Global step value to record\n bins (string): One of {'tensorflow','auto', 'fd', ...}. This determines how the bins are made. You can find\n other options in: https://docs.scipy.org/doc/numpy/reference/generated/numpy.histogram.html\n walltime (float): Optional override default walltime (time.time()) of event\n\n Examples::\n\n from tensorboardX import SummaryWriter\n import numpy as np\n writer = SummaryWriter()\n for i in range(10):\n x = np.random.random(1000)\n writer.add_histogram('distribution centers', x + i, i)\n writer.close()\n\n Expected result:\n\n .. image:: _static/img/tensorboard/add_histogram.png\n :scale: 50 %\n\n \"\"\"\n if self._check_caffe2_blob(values):\n values = workspace.FetchBlob(values)\n if isinstance(bins, six.string_types) and bins == 'tensorflow':\n bins = self.default_bins\n self._get_file_writer().add_summary(\n histogram(tag, values, bins, max_bins=max_bins), global_step, walltime)\n\n def add_histogram_raw(self, tag, min, max, num, sum, sum_squares,\n bucket_limits, bucket_counts, global_step=None,\n walltime=None):\n \"\"\"Adds histogram with raw data.\n\n Args:\n tag (string): Data identifier\n min (float or int): Min value\n max (float or int): Max value\n num (int): Number of values\n sum (float or int): Sum of all values\n sum_squares (float or int): Sum of squares for all values\n bucket_limits (torch.Tensor, numpy.array): Upper value per\n bucket, note that the bucket_limits returned from `np.histogram`\n has one more element. See the comment in the following example.\n bucket_counts (torch.Tensor, numpy.array): Number of values per bucket\n global_step (int): Global step value to record\n walltime (float): Optional override default walltime (time.time()) of event\n\n Examples::\n\n import numpy as np\n dummy_data = []\n for idx, value in enumerate(range(30)):\n dummy_data += [idx + 0.001] * value\n values = np.array(dummy_data).astype(float).reshape(-1)\n counts, limits = np.histogram(values)\n sum_sq = values.dot(values)\n with SummaryWriter() as summary_writer:\n summary_writer.add_histogram_raw(\n tag='hist_dummy_data',\n min=values.min(),\n max=values.max(),\n num=len(values),\n sum=values.sum(),\n sum_squares=sum_sq,\n bucket_limits=limits[1:].tolist(), # <- note here.\n bucket_counts=counts.tolist(),\n global_step=0)\n\n \"\"\"\n if len(bucket_limits) != len(bucket_counts):\n raise ValueError('len(bucket_limits) != len(bucket_counts), see the document.')\n self._get_file_writer().add_summary(\n histogram_raw(tag,\n min,\n max,\n num,\n sum,\n sum_squares,\n bucket_limits,\n bucket_counts),\n global_step,\n walltime)\n\n def add_image(self, tag, img_tensor, global_step=None, walltime=None, dataformats='CHW'):\n \"\"\"Add image data to summary.\n\n Note that this requires the ``pillow`` package.\n\n Args:\n tag (string): Data identifier\n img_tensor (torch.Tensor, numpy.array, or string/blobname): An `uint8` or `float`\n Tensor of shape `[channel, height, width]` where `channel` is 1, 3, or 4.\n The elements in img_tensor can either have values in [0, 1] (float32) or [0, 255] (uint8).\n Users are responsible to scale the data in the correct range/type.\n global_step (int): Global step value to record\n walltime (float): Optional override default walltime (time.time()) of event.\n dataformats (string): This parameter specifies the meaning of each dimension of the input tensor.\n Shape:\n img_tensor: Default is :math:`(3, H, W)`. You can use ``torchvision.utils.make_grid()`` to\n convert a batch of tensor into 3xHxW format or use ``add_images()`` and let us do the job.\n Tensor with :math:`(1, H, W)`, :math:`(H, W)`, :math:`(H, W, 3)` is also suitible as long as\n corresponding ``dataformats`` argument is passed. e.g. CHW, HWC, HW.\n\n Examples::\n\n from tensorboardX import SummaryWriter\n import numpy as np\n img = np.zeros((3, 100, 100))\n img[0] = np.arange(0, 10000).reshape(100, 100) / 10000\n img[1] = 1 - np.arange(0, 10000).reshape(100, 100) / 10000\n\n img_HWC = np.zeros((100, 100, 3))\n img_HWC[:, :, 0] = np.arange(0, 10000).reshape(100, 100) / 10000\n img_HWC[:, :, 1] = 1 - np.arange(0, 10000).reshape(100, 100) / 10000\n\n writer = SummaryWriter()\n writer.add_image('my_image', img, 0)\n\n # If you have non-default dimension setting, set the dataformats argument.\n writer.add_image('my_image_HWC', img_HWC, 0, dataformats='HWC')\n writer.close()\n\n Expected result:\n\n .. image:: _static/img/tensorboard/add_image.png\n :scale: 50 %\n\n \"\"\"\n if self._check_caffe2_blob(img_tensor):\n img_tensor = workspace.FetchBlob(img_tensor)\n self._get_file_writer().add_summary(\n image(tag, img_tensor, dataformats=dataformats), global_step, walltime)\n\n def add_images(self, tag, img_tensor, global_step=None, walltime=None, dataformats='NCHW'):\n \"\"\"Add batched (4D) image data to summary.\n Besides passing 4D (NCHW) tensor, you can also pass a list of tensors of the same size.\n In this case, the ``dataformats`` should be `CHW` or `HWC`.\n Note that this requires the ``pillow`` package.\n\n Args:\n tag (string): Data identifier\n img_tensor (torch.Tensor, numpy.array, or string/blobname): Image data\n The elements in img_tensor can either have values in [0, 1] (float32) or [0, 255] (uint8).\n Users are responsible to scale the data in the correct range/type.\n global_step (int): Global step value to record\n walltime (float): Optional override default walltime (time.time()) of event\n Shape:\n img_tensor: Default is :math:`(N, 3, H, W)`. If ``dataformats`` is specified, other shape will be\n accepted. e.g. NCHW or NHWC.\n\n Examples::\n\n from tensorboardX import SummaryWriter\n import numpy as np\n\n img_batch = np.zeros((16, 3, 100, 100))\n for i in range(16):\n img_batch[i, 0] = np.arange(0, 10000).reshape(100, 100) / 10000 / 16 * i\n img_batch[i, 1] = (1 - np.arange(0, 10000).reshape(100, 100) / 10000) / 16 * i\n\n writer = SummaryWriter()\n writer.add_images('my_image_batch', img_batch, 0)\n writer.close()\n\n Expected result:\n\n .. image:: _static/img/tensorboard/add_images.png\n :scale: 30 %\n\n \"\"\"\n if self._check_caffe2_blob(img_tensor):\n img_tensor = workspace.FetchBlob(img_tensor)\n if isinstance(img_tensor, list): # a list of tensors in CHW or HWC\n if dataformats.upper() != 'CHW' and dataformats.upper() != 'HWC':\n print('A list of image is passed, but the dataformat is neither CHW nor HWC.')\n print('Nothing is written.')\n return\n import torch\n try:\n img_tensor = torch.stack(img_tensor, 0)\n except TypeError as e:\n import numpy as np\n img_tensor = np.stack(img_tensor, 0)\n\n dataformats = 'N' + dataformats\n\n self._get_file_writer().add_summary(\n image(tag, img_tensor, dataformats=dataformats), global_step, walltime)\n\n def add_image_with_boxes(self, tag, img_tensor, box_tensor, global_step=None,\n walltime=None, dataformats='CHW', labels=None, **kwargs):\n \"\"\"Add image and draw bounding boxes on the image.\n\n Args:\n tag (string): Data identifier\n img_tensor (torch.Tensor, numpy.array, or string/blobname): Image data\n box_tensor (torch.Tensor, numpy.array, or string/blobname): Box data (for detected objects)\n box should be represented as [x1, y1, x2, y2].\n global_step (int): Global step value to record\n walltime (float): Optional override default walltime (time.time()) of event\n labels (list of string): The strings to be show on each bounding box.\n Shape:\n img_tensor: Default is :math:`(3, H, W)`. It can be specified with ``dataformat`` agrument.\n e.g. CHW or HWC\n\n box_tensor: (torch.Tensor, numpy.array, or string/blobname): NX4, where N is the number of\n boxes and each 4 elememts in a row represents (xmin, ymin, xmax, ymax).\n \"\"\"\n if self._check_caffe2_blob(img_tensor):\n img_tensor = workspace.FetchBlob(img_tensor)\n if self._check_caffe2_blob(box_tensor):\n box_tensor = workspace.FetchBlob(box_tensor)\n if labels is not None:\n if isinstance(labels, str):\n labels = [labels]\n if len(labels) != box_tensor.shape[0]:\n logging.warning('Number of labels do not equal to number of box, skip the labels.')\n labels = None\n self._get_file_writer().add_summary(image_boxes(\n tag, img_tensor, box_tensor, dataformats=dataformats, labels=labels, **kwargs), global_step, walltime)\n\n def add_figure(self, tag, figure, global_step=None, close=True, walltime=None):\n \"\"\"Render matplotlib figure into an image and add it to summary.\n\n Note that this requires the ``matplotlib`` package.\n\n Args:\n tag (string): Data identifier\n figure (matplotlib.pyplot.figure) or list of figures: Figure or a list of figures\n global_step (int): Global step value to record\n close (bool): Flag to automatically close the figure\n walltime (float): Optional override default walltime (time.time()) of event\n \"\"\"\n if isinstance(figure, list):\n self.add_image(tag, figure_to_image(figure, close), global_step, walltime, dataformats='NCHW')\n else:\n self.add_image(tag, figure_to_image(figure, close), global_step, walltime, dataformats='CHW')\n\n def add_video(self, tag, vid_tensor, global_step=None, fps=4, walltime=None):\n \"\"\"Add video data to summary.\n\n Note that this requires the ``moviepy`` package.\n\n Args:\n tag (string): Data identifier\n vid_tensor (torch.Tensor): Video data\n global_step (int): Global step value to record\n fps (float or int): Frames per second\n walltime (float): Optional override default walltime (time.time()) of event\n Shape:\n vid_tensor: :math:`(N, T, C, H, W)`. The values should lie in [0, 255] for type\n `uint8` or [0, 1] for type `float`.\n \"\"\"\n self._get_file_writer().add_summary(\n video(tag, vid_tensor, fps), global_step, walltime)\n\n def add_audio(self, tag, snd_tensor, global_step=None, sample_rate=44100, walltime=None):\n \"\"\"Add audio data to summary.\n\n Args:\n tag (string): Data identifier\n snd_tensor (torch.Tensor): Sound data\n global_step (int): Global step value to record\n sample_rate (int): sample rate in Hz\n walltime (float): Optional override default walltime (time.time()) of event\n Shape:\n snd_tensor: :math:`(L, c)`. The values should lie between [-1, 1].\n \"\"\"\n if self._check_caffe2_blob(snd_tensor):\n snd_tensor = workspace.FetchBlob(snd_tensor)\n self._get_file_writer().add_summary(\n audio(tag, snd_tensor, sample_rate=sample_rate), global_step, walltime)\n\n def add_text(self, tag, text_string, global_step=None, walltime=None):\n \"\"\"Add text data to summary.\n\n Args:\n tag (string): Data identifier\n text_string (string): String to save\n global_step (int): Global step value to record\n walltime (float): Optional override default walltime (time.time()) of event\n Examples::\n\n writer.add_text('lstm', 'This is an lstm', 0)\n writer.add_text('rnn', 'This is an rnn', 10)\n \"\"\"\n self._get_file_writer().add_summary(\n text(tag, text_string), global_step, walltime)\n\n def add_onnx_graph(self, prototxt):\n self._get_file_writer().add_onnx_graph(load_onnx_graph(prototxt))\n\n def add_graph(self, model, input_to_model=None, verbose=False, **kwargs):\n # prohibit second call?\n # no, let tensorboard handle it and show its warning message.\n \"\"\"Add graph data to summary.\n\n Args:\n model (torch.nn.Module): Model to draw.\n input_to_model (torch.Tensor or list of torch.Tensor): A variable or a tuple of\n variables to be fed.\n verbose (bool): Whether to print graph structure in console.\n omit_useless_nodes (bool): Default to ``true``, which eliminates unused nodes.\n operator_export_type (string): One of: ``\"ONNX\"``, ``\"RAW\"``. This determines\n the optimization level of the graph. If error happens during exporting\n the graph, using ``\"RAW\"`` might help.\n\n \"\"\"\n if hasattr(model, 'forward'):\n # A valid PyTorch model should have a 'forward' method\n import torch\n from distutils.version import LooseVersion\n if LooseVersion(torch.__version__) >= LooseVersion(\"0.3.1\"):\n pass\n else:\n if LooseVersion(torch.__version__) >= LooseVersion(\"0.3.0\"):\n print('You are using PyTorch==0.3.0, use add_onnx_graph()')\n return\n if not hasattr(torch.autograd.Variable, 'grad_fn'):\n print('add_graph() only supports PyTorch v0.2.')\n return\n self._get_file_writer().add_graph(graph(model, input_to_model, verbose, **kwargs))\n else:\n # Caffe2 models do not have the 'forward' method\n from caffe2.proto import caffe2_pb2\n from caffe2.python import core\n from .caffe2_graph import (\n model_to_graph_def, nets_to_graph_def, protos_to_graph_def\n )\n if isinstance(model, list):\n if isinstance(model[0], core.Net):\n current_graph = nets_to_graph_def(\n model, **kwargs)\n elif isinstance(model[0], caffe2_pb2.NetDef):\n current_graph = protos_to_graph_def(\n model, **kwargs)\n else:\n # Handles cnn.CNNModelHelper, model_helper.ModelHelper\n current_graph = model_to_graph_def(\n model, **kwargs)\n event = event_pb2.Event(\n graph_def=current_graph.SerializeToString())\n self._get_file_writer().add_event(event)\n\n @staticmethod\n def _encode(rawstr):\n # I'd use urllib but, I'm unsure about the differences from python3 to python2, etc.\n retval = rawstr\n retval = retval.replace(\"%\", \"%%%02x\" % (ord(\"%\")))\n retval = retval.replace(\"/\", \"%%%02x\" % (ord(\"/\")))\n retval = retval.replace(\"\\\\\", \"%%%02x\" % (ord(\"\\\\\")))\n return retval\n\n def add_embedding(self, mat, metadata=None, label_img=None, global_step=None, tag='default', metadata_header=None):\n \"\"\"Add embedding projector data to summary.\n\n Args:\n mat (torch.Tensor or numpy.array): A matrix which each row is the feature vector of the data point\n metadata (list): A list of labels, each element will be convert to string\n label_img (torch.Tensor or numpy.array): Images correspond to each data point. Each image should be square.\n global_step (int): Global step value to record\n tag (string): Name for the embedding\n Shape:\n mat: :math:`(N, D)`, where N is number of data and D is feature dimension\n\n label_img: :math:`(N, C, H, W)`, where `Height` should be equal to `Width`.\n\n Examples::\n\n import keyword\n import torch\n meta = []\n while len(meta)<100:\n meta = meta+keyword.kwlist # get some strings\n meta = meta[:100]\n\n for i, v in enumerate(meta):\n meta[i] = v+str(i)\n\n label_img = torch.rand(100, 3, 32, 32)\n for i in range(100):\n label_img[i]*=i/100.0\n\n writer.add_embedding(torch.randn(100, 5), metadata=meta, label_img=label_img)\n writer.add_embedding(torch.randn(100, 5), label_img=label_img)\n writer.add_embedding(torch.randn(100, 5), metadata=meta)\n \"\"\"\n from .x2num import make_np\n mat = make_np(mat)\n if global_step is None:\n global_step = 0\n # clear pbtxt?\n # Maybe we should encode the tag so slashes don't trip us up?\n # I don't think this will mess us up, but better safe than sorry.\n subdir = \"%s/%s\" % (str(global_step).zfill(5), self._encode(tag))\n save_path = os.path.join(self._get_file_writer().get_logdir(), subdir)\n try:\n os.makedirs(save_path)\n except OSError:\n print(\n 'warning: Embedding dir exists, did you set global_step for add_embedding()?')\n if metadata is not None:\n assert mat.shape[0] == len(\n metadata), '#labels should equal with #data points'\n make_tsv(metadata, save_path, metadata_header=metadata_header)\n if label_img is not None:\n assert mat.shape[0] == label_img.shape[0], '#images should equal with #data points'\n assert label_img.shape[2] == label_img.shape[3], 'Image should be square, see tensorflow/tensorboard#670'\n make_sprite(label_img, save_path)\n assert mat.ndim == 2, 'mat should be 2D, where mat.size(0) is the number of data points'\n make_mat(mat, save_path)\n # new funcion to append to the config file a new embedding\n append_pbtxt(metadata, label_img,\n self._get_file_writer().get_logdir(), subdir, global_step, tag)\n\n def add_pr_curve(self, tag, labels, predictions, global_step=None,\n num_thresholds=127, weights=None, walltime=None):\n \"\"\"Adds precision recall curve.\n Plotting a precision-recall curve lets you understand your model's\n performance under different threshold settings. With this function,\n you provide the ground truth labeling (T/F) and prediction confidence\n (usually the output of your model) for each target. The TensorBoard UI\n will let you choose the threshold interactively.\n\n Args:\n tag (string): Data identifier\n labels (torch.Tensor, numpy.array, or string/blobname):\n Ground truth data. Binary label for each element.\n predictions (torch.Tensor, numpy.array, or string/blobname):\n The probability that an element be classified as true.\n Value should in [0, 1]\n global_step (int): Global step value to record\n num_thresholds (int): Number of thresholds used to draw the curve.\n walltime (float): Optional override default walltime (time.time()) of event\n\n Examples::\n\n from tensorboardX import SummaryWriter\n import numpy as np\n labels = np.random.randint(2, size=100) # binary label\n predictions = np.random.rand(100)\n writer = SummaryWriter()\n writer.add_pr_curve('pr_curve', labels, predictions, 0)\n writer.close()\n\n \"\"\"\n from .x2num import make_np\n labels, predictions = make_np(labels), make_np(predictions)\n self._get_file_writer().add_summary(\n pr_curve(tag, labels, predictions, num_thresholds, weights),\n global_step, walltime)\n\n def add_pr_curve_raw(self, tag, true_positive_counts,\n false_positive_counts,\n true_negative_counts,\n false_negative_counts,\n precision,\n recall,\n global_step=None,\n num_thresholds=127,\n weights=None,\n walltime=None):\n \"\"\"Adds precision recall curve with raw data.\n\n Args:\n tag (string): Data identifier\n true_positive_counts (torch.Tensor, numpy.array, or string/blobname): true positive counts\n false_positive_counts (torch.Tensor, numpy.array, or string/blobname): false positive counts\n true_negative_counts (torch.Tensor, numpy.array, or string/blobname): true negative counts\n false_negative_counts (torch.Tensor, numpy.array, or string/blobname): false negative counts\n precision (torch.Tensor, numpy.array, or string/blobname): precision\n recall (torch.Tensor, numpy.array, or string/blobname): recall\n global_step (int): Global step value to record\n num_thresholds (int): Number of thresholds used to draw the curve.\n walltime (float): Optional override default walltime (time.time()) of event\n see: https://github.com/tensorflow/tensorboard/blob/master/tensorboard/plugins/pr_curve/README.md\n \"\"\"\n self._get_file_writer().add_summary(\n pr_curve_raw(tag,\n true_positive_counts,\n false_positive_counts,\n true_negative_counts,\n false_negative_counts,\n precision,\n recall,\n num_thresholds,\n weights),\n global_step,\n walltime)\n\n def add_custom_scalars_multilinechart(self, tags, category='default', title='untitled'):\n \"\"\"Shorthand for creating multilinechart. Similar to ``add_custom_scalars()``, but the only necessary argument\n is *tags*.\n\n Args:\n tags (list): list of tags that have been used in ``add_scalar()``\n\n Examples::\n\n writer.add_custom_scalars_multilinechart(['twse/0050', 'twse/2330'])\n \"\"\"\n layout = {category: {title: ['Multiline', tags]}}\n self._get_file_writer().add_summary(custom_scalars(layout))\n\n def add_custom_scalars_marginchart(self, tags, category='default', title='untitled'):\n \"\"\"Shorthand for creating marginchart. Similar to ``add_custom_scalars()``, but the only necessary argument\n is *tags*, which should have exactly 3 elements.\n\n Args:\n tags (list): list of tags that have been used in ``add_scalar()``\n\n Examples::\n\n writer.add_custom_scalars_marginchart(['twse/0050', 'twse/2330', 'twse/2006'])\n \"\"\"\n assert len(tags) == 3\n layout = {category: {title: ['Margin', tags]}}\n self._get_file_writer().add_summary(custom_scalars(layout))\n\n def add_custom_scalars(self, layout):\n \"\"\"Create special chart by collecting charts tags in 'scalars'. Note that this function can only be called once\n for each SummaryWriter() object. Because it only provides metadata to tensorboard, the function can be called\n before or after the training loop. See ``examples/demo_custom_scalars.py`` for more.\n\n Args:\n layout (dict): {categoryName: *charts*}, where *charts* is also a dictionary\n {chartName: *ListOfProperties*}. The first element in *ListOfProperties* is the chart's type\n (one of **Multiline** or **Margin**) and the second element should be a list containing the tags\n you have used in add_scalar function, which will be collected into the new chart.\n\n Examples::\n\n layout = {'Taiwan':{'twse':['Multiline',['twse/0050', 'twse/2330']]},\n 'USA':{ 'dow':['Margin', ['dow/aaa', 'dow/bbb', 'dow/ccc']],\n 'nasdaq':['Margin', ['nasdaq/aaa', 'nasdaq/bbb', 'nasdaq/ccc']]}}\n\n writer.add_custom_scalars(layout)\n \"\"\"\n self._get_file_writer().add_summary(custom_scalars(layout))\n\n def add_mesh(self, tag, vertices, colors=None, faces=None, config_dict=None, global_step=None, walltime=None):\n \"\"\"Add meshes or 3D point clouds to TensorBoard. The visualization is based on Three.js,\n so it allows users to interact with the rendered object. Besides the basic definitions\n such as vertices, faces, users can further provide camera parameter, lighting condition, etc.\n Please check https://threejs.org/docs/index.html#manual/en/introduction/Creating-a-scene for\n advanced usage. Note that currently this depends on tb-nightly to show.\n\n Args:\n tag (string): Data identifier\n vertices (torch.Tensor): List of the 3D coordinates of vertices.\n colors (torch.Tensor): Colors for each vertex\n faces (torch.Tensor): Indices of vertices within each triangle. (Optional)\n config_dict: Dictionary with ThreeJS classes names and configuration.\n global_step (int): Global step value to record\n walltime (float): Optional override default walltime (time.time())\n seconds after epoch of event\n\n Shape:\n vertices: :math:`(B, N, 3)`. (batch, number_of_vertices, channels). If you see nothing on\n tensorboard, try normalizing the values to [-1, 1].\n\n colors: :math:`(B, N, 3)`. The values should lie in [0, 255].\n\n faces: :math:`(B, N, 3)`. The values should lie in [0, number_of_vertices] for type `uint8`.\n\n Examples::\n\n from tensorboardX import SummaryWriter\n vertices_tensor = np.array([[\n [1, 1, 1],\n [-1, -1, 1],\n [1, -1, -1],\n [-1, 1, -1],\n ]], dtype=float)\n colors_tensor = np.array([[\n [255, 0, 0],\n [0, 255, 0],\n [0, 0, 255],\n [255, 0, 255],\n ]], dtype=int)\n faces_tensor = np.array([[\n [0, 2, 3],\n [0, 3, 1],\n [0, 1, 2],\n [1, 3, 2],\n ]], dtype=int)\n\n writer = SummaryWriter()\n writer.add_mesh('my_mesh', vertices=vertices_tensor, colors=colors_tensor, faces=faces_tensor)\n\n writer.close()\n \"\"\"\n self._get_file_writer().add_summary(mesh(tag, vertices, colors, faces, config_dict), global_step, walltime)\n\n def close(self):\n if self.all_writers is None:\n return # ignore double close\n for writer in self.all_writers.values():\n writer.flush()\n writer.close()\n self.file_writer = self.all_writers = None\n\n def flush(self):\n if self.all_writers is None:\n return # ignore double close\n for writer in self.all_writers.values():\n writer.flush()\n\n def __enter__(self):\n return self\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n self.close()\n" }, { "path": "tensorboardX/tensorboardX/x2num.py", "content": "# DO NOT alter/distruct/free input object !\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport logging\nimport numpy as np\nimport six\n\n\ndef check_nan(array):\n tmp = np.sum(array)\n if np.isnan(tmp) or np.isinf(tmp):\n logging.warning('NaN or Inf found in input tensor.')\n return array\n\n\ndef make_np(x):\n if isinstance(x, list):\n return check_nan(np.array(x))\n if isinstance(x, np.ndarray):\n return check_nan(x)\n if isinstance(x, six.string_types): # Caffe2 will pass name of blob(s) to fetch\n return check_nan(prepare_caffe2(x))\n if np.isscalar(x):\n return check_nan(np.array([x]))\n if 'torch' in str(type(x)):\n return check_nan(prepare_pytorch(x))\n if 'chainer' in str(type(x)):\n return check_nan(prepare_chainer(x))\n if 'mxnet' in str(type(x)):\n return check_nan(prepare_mxnet(x))\n raise NotImplementedError(\n 'Got {}, but expected numpy array or torch tensor.'.format(type(x)))\n\n\ndef prepare_pytorch(x):\n import torch\n if isinstance(x, torch.autograd.Variable):\n x = x.data\n x = x.cpu().numpy()\n return x\n\n\ndef prepare_theano(x):\n import theano\n pass\n\n\ndef prepare_caffe2(x):\n from caffe2.python import workspace\n x = workspace.FetchBlob(x)\n return x\n\n\ndef prepare_mxnet(x):\n x = x.asnumpy()\n return x\n\n\ndef prepare_chainer(x):\n import chainer\n x = chainer.cuda.to_cpu(x.data)\n return x\n" }, { "path": "tensorboardX/tensorboardX.patch", "content": "diff --git a/tensorboardX/summary.py b/tensorboardX/summary.py\nindex 27d99ea..f5bf234 100644\n--- a/tensorboardX/summary.py\n+++ b/tensorboardX/summary.py\n@@ -373,36 +373,24 @@ def make_video(tensor, fps):\n \n def audio(tag, tensor, sample_rate=44100):\n tensor = make_np(tensor)\n- tensor = tensor.squeeze()\n if abs(tensor).max() > 1:\n print('warning: audio amplitude out of range, auto clipped.')\n tensor = tensor.clip(-1, 1)\n- assert(tensor.ndim == 1), 'input tensor should be 1 dimensional.'\n-\n- tensor_list = [int(32767.0 * x) for x in tensor]\n+ assert(tensor.ndim == 2), 'input tensor should be 2 dimensional.'\n+ length_frames, num_channels = tensor.shape\n+ assert num_channels == 1 or num_channels == 2, f'Expected 1/2 channels, got {num_channels}'\n+ import soundfile\n import io\n- import wave\n- import struct\n- fio = io.BytesIO()\n- Wave_write = wave.open(fio, 'wb')\n- Wave_write.setnchannels(1)\n- Wave_write.setsampwidth(2)\n- Wave_write.setframerate(sample_rate)\n- tensor_enc = b''\n- tensor_enc += struct.pack(\"<\" + \"h\" * len(tensor_list), *tensor_list)\n-\n- Wave_write.writeframes(tensor_enc)\n- Wave_write.close()\n- audio_string = fio.getvalue()\n- fio.close()\n+ with io.BytesIO() as fio:\n+ soundfile.write(fio, tensor, samplerate=sample_rate, format='wav')\n+ audio_string = fio.getvalue()\n audio = Summary.Audio(sample_rate=sample_rate,\n- num_channels=1,\n- length_frames=len(tensor_list),\n+ num_channels=num_channels,\n+ length_frames=length_frames,\n encoded_audio_string=audio_string,\n content_type='audio/wav')\n return Summary(value=[Summary.Value(tag=tag, audio=audio)])\n \n-\n def custom_scalars(layout):\n categoriesnames = layout.keys()\n categories = []\ndiff --git a/tensorboardX/writer.py b/tensorboardX/writer.py\nindex 06337a7..58d57a1 100644\n--- a/tensorboardX/writer.py\n+++ b/tensorboardX/writer.py\n@@ -716,7 +716,7 @@ class SummaryWriter(object):\n sample_rate (int): sample rate in Hz\n walltime (float): Optional override default walltime (time.time()) of event\n Shape:\n- snd_tensor: :math:`(1, L)`. The values should lie between [-1, 1].\n+ snd_tensor: :math:`(L, c)`. The values should lie between [-1, 1].\n \"\"\"\n if self._check_caffe2_blob(snd_tensor):\n snd_tensor = workspace.FetchBlob(snd_tensor)\n" }, { "path": "tensorboardX/tests/__init__.py", "content": "import torch\nimport tensorboardX.proto\n" }, { "path": "tensorboardX/tests/event_file_writer_test.py", "content": "# Copyright 2019 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ==============================================================================\n\n# \"\"\"Tests for EventFileWriter and _AsyncWriter\"\"\"\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\n\nimport glob\nimport os\nfrom tensorboardX.event_file_writer import EventFileWriter\nfrom tensorboardX.event_file_writer import EventFileWriter as _AsyncWriter\n\n\nfrom tensorboardX.proto import event_pb2\nfrom tensorboardX.proto.summary_pb2 import Summary\n\nfrom tensorboard.compat.tensorflow_stub.pywrap_tensorflow import PyRecordReader_New\nimport unittest\n\n\nclass EventFileWriterTest(unittest.TestCase):\n def get_temp_dir(self):\n import tempfile\n return tempfile.mkdtemp()\n\n def test_event_file_writer_roundtrip(self):\n _TAGNAME = 'dummy'\n _DUMMY_VALUE = 42\n logdir = self.get_temp_dir()\n w = EventFileWriter(logdir)\n summary = Summary(value=[Summary.Value(tag=_TAGNAME, simple_value=_DUMMY_VALUE)])\n fakeevent = event_pb2.Event(summary=summary)\n w.add_event(fakeevent)\n w.close()\n event_files = sorted(glob.glob(os.path.join(logdir, '*')))\n self.assertEqual(len(event_files), 1)\n r = PyRecordReader_New(event_files[0])\n r.GetNext() # meta data, so skip\n r.GetNext()\n self.assertEqual(fakeevent.SerializeToString(), r.record())\n\n def test_setting_filename_suffix_works(self):\n logdir = self.get_temp_dir()\n\n w = EventFileWriter(logdir, filename_suffix='.event_horizon')\n w.close()\n event_files = sorted(glob.glob(os.path.join(logdir, '*')))\n self.assertEqual(event_files[0].split('.')[-1], 'event_horizon')\n\n def test_async_writer_without_write(self):\n logdir = self.get_temp_dir()\n w = EventFileWriter(logdir)\n w.close()\n event_files = sorted(glob.glob(os.path.join(logdir, '*')))\n r = PyRecordReader_New(event_files[0])\n r.GetNext()\n s = event_pb2.Event.FromString(r.record())\n self.assertEqual(s.file_version, \"brain.Event:2\")\n\n\n# skip the test, because tensorboard's implementaion of filewriter\n# writes raw data while that in tensorboardX writes event protobuf.\nclass AsyncWriterTest(): #unittest.TestCase):\n def get_temp_dir(self):\n import tempfile\n return tempfile.mkdtemp()\n\n def test_async_writer_write_once(self):\n foldername = os.path.join(self.get_temp_dir(), \"async_writer_write_once\")\n w = _AsyncWriter(foldername)\n filename = w._ev_writer._file_name\n bytes_to_write = b\"hello world\"\n w.add_event(bytes_to_write)\n w.close()\n with open(filename, 'rb') as f:\n self.assertEqual(f.read(), bytes_to_write)\n\n def test_async_writer_write_queue_full(self):\n filename = os.path.join(self.get_temp_dir(), \"async_writer_write_queue_full\")\n w = _AsyncWriter(filename)\n bytes_to_write = b\"hello world\"\n repeat = 100\n for i in range(repeat):\n w.write(bytes_to_write)\n w.close()\n with open(filename, 'rb') as f:\n self.assertEqual(f.read(), bytes_to_write * repeat)\n\n def test_async_writer_write_one_slot_queue(self):\n filename = os.path.join(self.get_temp_dir(), \"async_writer_write_one_slot_queue\")\n w = _AsyncWriter(filename, max_queue_size=1)\n bytes_to_write = b\"hello world\"\n repeat = 10 # faster\n for i in range(repeat):\n w.write(bytes_to_write)\n w.close()\n with open(filename, 'rb') as f:\n self.assertEqual(f.read(), bytes_to_write * repeat)\n\n def test_async_writer_close_triggers_flush(self):\n filename = os.path.join(self.get_temp_dir(), \"async_writer_close_triggers_flush\")\n w = _AsyncWriter(filename)\n bytes_to_write = b\"x\" * 64\n w.write(bytes_to_write)\n w.close()\n with open(filename, 'rb') as f:\n self.assertEqual(f.read(), bytes_to_write)\n\n def test_write_after_async_writer_closed(self):\n filename = os.path.join(self.get_temp_dir(), \"write_after_async_writer_closed\")\n w = _AsyncWriter(filename)\n bytes_to_write = b\"x\" * 64\n w.write(bytes_to_write)\n w.close()\n\n with self.assertRaises(IOError):\n w.write(bytes_to_write)\n # nothing is written to the file after close\n with open(filename, 'rb') as f:\n self.assertEqual(f.read(), bytes_to_write)\n\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "tensorboardX/tests/expect/caffe_mnist.expect", "content": "node {\n name: \"conv1/XavierFill\"\n op: \"XavierFill\"\n device: \"/gpu:0\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 20\n }\n dim {\n size: 1\n }\n dim {\n size: 5\n }\n dim {\n size: 5\n }\n }\n }\n }\n }\n}\nnode {\n name: \"conv1/ConstantFill\"\n op: \"ConstantFill\"\n device: \"/gpu:0\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 20\n }\n }\n }\n }\n }\n}\nnode {\n name: \"conv1/XavierFill_1\"\n op: \"XavierFill\"\n device: \"/gpu:0\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 100\n }\n dim {\n size: 20\n }\n dim {\n size: 5\n }\n dim {\n size: 5\n }\n }\n }\n }\n }\n}\nnode {\n name: \"conv1/ConstantFill_1\"\n op: \"ConstantFill\"\n device: \"/gpu:0\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 100\n }\n }\n }\n }\n }\n}\nnode {\n name: \"classifier/XavierFill\"\n op: \"XavierFill\"\n device: \"/gpu:0\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 500\n }\n dim {\n size: 1600\n }\n }\n }\n }\n }\n}\nnode {\n name: \"classifier/ConstantFill\"\n op: \"ConstantFill\"\n device: \"/gpu:0\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 500\n }\n }\n }\n }\n }\n}\nnode {\n name: \"classifier/XavierFill_1\"\n op: \"XavierFill\"\n device: \"/gpu:0\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 10\n }\n dim {\n size: 500\n }\n }\n }\n }\n }\n}\nnode {\n name: \"classifier/ConstantFill_1\"\n op: \"ConstantFill\"\n device: \"/gpu:0\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 10\n }\n }\n }\n }\n }\n}\nnode {\n name: \"ImageInput\"\n op: \"ImageInput\"\n input: \"db\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"is_test\"\n value {\n i: 0\n }\n }\n attr {\n key: \"use_cudnn\"\n value {\n i: 1\n }\n }\n}\nnode {\n name: \"NHWC2NCHW\"\n op: \"NHWC2NCHW\"\n input: \"data_nhwc\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"conv1/Conv\"\n op: \"Conv\"\n input: \"data\"\n input: \"conv1/conv1_w\"\n input: \"conv1/conv1_b\"\n device: \"/gpu:0\"\n attr {\n key: \"exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"kernel\"\n value {\n i: 5\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n}\nnode {\n name: \"conv1/MaxPool\"\n op: \"MaxPool\"\n input: \"conv1/conv1\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"kernel\"\n value {\n i: 2\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"stride\"\n value {\n i: 2\n }\n }\n}\nnode {\n name: \"conv1/Conv_1\"\n op: \"Conv\"\n input: \"conv1/pool1\"\n input: \"conv1/conv2_w\"\n input: \"conv1/conv2_b\"\n device: \"/gpu:0\"\n attr {\n key: \"exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"kernel\"\n value {\n i: 5\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n}\nnode {\n name: \"conv1/MaxPool_1\"\n op: \"MaxPool\"\n input: \"conv1/conv2\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"kernel\"\n value {\n i: 2\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"stride\"\n value {\n i: 2\n }\n }\n}\nnode {\n name: \"classifier/FC\"\n op: \"FC\"\n input: \"conv1/pool2\"\n input: \"classifier/fc3_w\"\n input: \"classifier/fc3_b\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"use_cudnn\"\n value {\n i: 1\n }\n }\n}\nnode {\n name: \"classifier/Relu\"\n op: \"Relu\"\n input: \"classifier/fc3\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n}\nnode {\n name: \"classifier/FC_1\"\n op: \"FC\"\n input: \"classifier/fc3_1\"\n input: \"classifier/pred_w\"\n input: \"classifier/pred_b\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"use_cudnn\"\n value {\n i: 1\n }\n }\n}\nnode {\n name: \"classifier/Softmax\"\n op: \"Softmax\"\n input: \"classifier/pred\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n}\nnode {\n name: \"classifier/LabelCrossEntropy\"\n op: \"LabelCrossEntropy\"\n input: \"classifier/softmax\"\n input: \"label\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"classifier/AveragedLoss\"\n op: \"AveragedLoss\"\n input: \"classifier/xent\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/ConstantFill\"\n op: \"ConstantFill\"\n input: \"classifier/loss\"\n device: \"/gpu:0\"\n attr {\n key: \"value\"\n value {\n f: 1.0\n }\n }\n}\nnode {\n name: \"GRADIENTS/classifier/AveragedLossGradient\"\n op: \"AveragedLossGradient\"\n input: \"classifier/xent\"\n input: \"GRADIENTS/classifier/loss_autogen_grad\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/LabelCrossEntropyGradient\"\n op: \"LabelCrossEntropyGradient\"\n input: \"classifier/softmax\"\n input: \"label\"\n input: \"GRADIENTS/classifier/xent_grad\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/SoftmaxGradient\"\n op: \"SoftmaxGradient\"\n input: \"classifier/softmax\"\n input: \"GRADIENTS/classifier/softmax_grad\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n}\nnode {\n name: \"GRADIENTS/classifier/FCGradient\"\n op: \"FCGradient\"\n input: \"classifier/fc3_1\"\n input: \"classifier/pred_w\"\n input: \"GRADIENTS/classifier/pred_grad\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"use_cudnn\"\n value {\n i: 1\n }\n }\n}\nnode {\n name: \"GRADIENTS/classifier/ReluGradient\"\n op: \"ReluGradient\"\n input: \"classifier/fc3_1\"\n input: \"GRADIENTS/classifier/fc3_grad\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n}\nnode {\n name: \"GRADIENTS/c/FCGradient\"\n op: \"FCGradient\"\n input: \"conv1/pool2\"\n input: \"classifier/fc3_w\"\n input: \"GRADIENTS/classifier/fc3_grad_1\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"use_cudnn\"\n value {\n i: 1\n }\n }\n}\nnode {\n name: \"GRADIENTS/conv1/MaxPoolGradient\"\n op: \"MaxPoolGradient\"\n input: \"conv1/conv2\"\n input: \"conv1/pool2\"\n input: \"GRADIENTS/conv1/pool2_grad\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"kernel\"\n value {\n i: 2\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"stride\"\n value {\n i: 2\n }\n }\n}\nnode {\n name: \"GRADIENTS/conv1/ConvGradient\"\n op: \"ConvGradient\"\n input: \"conv1/pool1\"\n input: \"conv1/conv2_w\"\n input: \"GRADIENTS/conv1/conv2_grad\"\n device: \"/gpu:0\"\n attr {\n key: \"exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"kernel\"\n value {\n i: 5\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n}\nnode {\n name: \"GRADIENTS/conv1/MaxPoolGradient_1\"\n op: \"MaxPoolGradient\"\n input: \"conv1/conv1\"\n input: \"conv1/pool1\"\n input: \"GRADIENTS/conv1/pool1_grad\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"kernel\"\n value {\n i: 2\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"stride\"\n value {\n i: 2\n }\n }\n}\nnode {\n name: \"GRADIENTS/ConvGradient\"\n op: \"ConvGradient\"\n input: \"data\"\n input: \"conv1/conv1_w\"\n input: \"GRADIENTS/conv1/conv1_grad\"\n device: \"/gpu:0\"\n attr {\n key: \"exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"kernel\"\n value {\n i: 5\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n}\nnode {\n name: \"GRADIENTS/NCHW2NHWC\"\n op: \"NCHW2NHWC\"\n input: \"GRADIENTS/data_grad\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/fc3_grad_1\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/ReluGradient:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/xent_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/AveragedLossGradient:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/pred_w_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/FCGradient:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/data_nhwc_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/NCHW2NHWC:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/fc3_w_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/c/FCGradient:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/conv1/conv1_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/conv1/MaxPoolGradient_1:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/conv1/conv1_b_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/ConvGradient:1\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/conv1/conv2_w_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/conv1/ConvGradient:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"classifier/pred\"\n op: \"Blob\"\n input: \"classifier/FC_1:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/conv1/pool2_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/c/FCGradient:2\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/conv1/conv1_w_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/ConvGradient:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"data\"\n op: \"Blob\"\n input: \"NHWC2NCHW:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"classifier/xent\"\n op: \"Blob\"\n input: \"classifier/LabelCrossEntropy:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/conv1/pool1_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/conv1/ConvGradient:2\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"db\"\n op: \"Placeholder\"\n}\nnode {\n name: \"classifier/fc3_b\"\n op: \"Blob\"\n input: \"classifier/ConstantFill:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"classifier/pred_b\"\n op: \"Blob\"\n input: \"classifier/ConstantFill_1:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"classifier/softmax\"\n op: \"Blob\"\n input: \"classifier/Softmax:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/data_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/ConvGradient:2\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/pred_b_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/FCGradient:1\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"label\"\n op: \"Blob\"\n input: \"ImageInput:1\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"conv1/pool1\"\n op: \"Blob\"\n input: \"conv1/MaxPool:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"data_nhwc\"\n op: \"Blob\"\n input: \"ImageInput:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"conv1/conv2\"\n op: \"Blob\"\n input: \"conv1/Conv_1:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/conv1/conv2_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/conv1/MaxPoolGradient:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"conv1/conv2_b\"\n op: \"Blob\"\n input: \"conv1/ConstantFill_1:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"conv1/conv1_b\"\n op: \"Blob\"\n input: \"conv1/ConstantFill:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"classifier/fc3_w\"\n op: \"Blob\"\n input: \"classifier/XavierFill:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/fc3_b_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/c/FCGradient:1\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"classifier/pred_w\"\n op: \"Blob\"\n input: \"classifier/XavierFill_1:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"conv1/pool2\"\n op: \"Blob\"\n input: \"conv1/MaxPool_1:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/conv1/conv2_b_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/conv1/ConvGradient:1\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"classifier/fc3_1\"\n op: \"Blob\"\n input: \"classifier/Relu:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"classifier/loss\"\n op: \"Blob\"\n input: \"classifier/AveragedLoss:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/fc3_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/FCGradient:2\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"conv1/conv1_w\"\n op: \"Blob\"\n input: \"conv1/XavierFill:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"conv1/conv1\"\n op: \"Blob\"\n input: \"conv1/Conv:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/loss_autogen_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/ConstantFill:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"classifier/fc3\"\n op: \"Blob\"\n input: \"classifier/FC:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/pred_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/SoftmaxGradient:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/softmax_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/LabelCrossEntropyGradient:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"conv1/conv2_w\"\n op: \"Blob\"\n input: \"conv1/XavierFill_1:0\"\n device: \"/gpu:0\"\n}" }, { "path": "tensorboardX/tests/expect/caffe_overfeat.expect", "content": "node {\n name: \"conv1/XavierFill\"\n op: \"XavierFill\"\n device: \"/gpu:0\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 96\n }\n dim {\n size: 3\n }\n dim {\n size: 11\n }\n dim {\n size: 11\n }\n }\n }\n }\n }\n}\nnode {\n name: \"conv1/ConstantFill\"\n op: \"ConstantFill\"\n device: \"/gpu:0\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 96\n }\n }\n }\n }\n }\n}\nnode {\n name: \"classifier/XavierFill\"\n op: \"XavierFill\"\n device: \"/gpu:0\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 1000\n }\n dim {\n size: 4096\n }\n }\n }\n }\n }\n}\nnode {\n name: \"classifier/ConstantFill\"\n op: \"ConstantFill\"\n device: \"/gpu:0\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 1000\n }\n }\n }\n }\n }\n}\nnode {\n name: \"ImageInput\"\n op: \"ImageInput\"\n input: \"db\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"is_test\"\n value {\n i: 0\n }\n }\n attr {\n key: \"use_cudnn\"\n value {\n i: 1\n }\n }\n}\nnode {\n name: \"NHWC2NCHW\"\n op: \"NHWC2NCHW\"\n input: \"data_nhwc\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"conv1/Conv\"\n op: \"Conv\"\n input: \"data\"\n input: \"conv1/conv1_w\"\n input: \"conv1/conv1_b\"\n device: \"/gpu:0\"\n attr {\n key: \"exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"kernel\"\n value {\n i: 11\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"stride\"\n value {\n i: 4\n }\n }\n}\nnode {\n name: \"conv1/Relu\"\n op: \"Relu\"\n input: \"conv1/conv1\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n}\nnode {\n name: \"conv1/MaxPool\"\n op: \"MaxPool\"\n input: \"conv1/conv1_1\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"kernel\"\n value {\n i: 2\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"stride\"\n value {\n i: 2\n }\n }\n}\nnode {\n name: \"classifier/FC\"\n op: \"FC\"\n input: \"conv1/pool1\"\n input: \"classifier/fc_w\"\n input: \"classifier/fc_b\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"use_cudnn\"\n value {\n i: 1\n }\n }\n}\nnode {\n name: \"classifier/Softmax\"\n op: \"Softmax\"\n input: \"classifier/fc\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n}\nnode {\n name: \"classifier/LabelCrossEntropy\"\n op: \"LabelCrossEntropy\"\n input: \"classifier/pred\"\n input: \"label\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"classifier/AveragedLoss\"\n op: \"AveragedLoss\"\n input: \"classifier/xent\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/ConstantFill\"\n op: \"ConstantFill\"\n input: \"classifier/loss\"\n device: \"/gpu:0\"\n attr {\n key: \"value\"\n value {\n f: 1.0\n }\n }\n}\nnode {\n name: \"GRADIENTS/classifier/AveragedLossGradient\"\n op: \"AveragedLossGradient\"\n input: \"classifier/xent\"\n input: \"GRADIENTS/classifier/loss_autogen_grad\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/LabelCrossEntropyGradient\"\n op: \"LabelCrossEntropyGradient\"\n input: \"classifier/pred\"\n input: \"label\"\n input: \"GRADIENTS/classifier/xent_grad\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/SoftmaxGradient\"\n op: \"SoftmaxGradient\"\n input: \"classifier/pred\"\n input: \"GRADIENTS/classifier/pred_grad\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n}\nnode {\n name: \"GRADIENTS/c/FCGradient\"\n op: \"FCGradient\"\n input: \"conv1/pool1\"\n input: \"classifier/fc_w\"\n input: \"GRADIENTS/classifier/fc_grad\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"use_cudnn\"\n value {\n i: 1\n }\n }\n}\nnode {\n name: \"GRADIENTS/conv1/MaxPoolGradient\"\n op: \"MaxPoolGradient\"\n input: \"conv1/conv1_1\"\n input: \"conv1/pool1\"\n input: \"GRADIENTS/conv1/pool1_grad\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"kernel\"\n value {\n i: 2\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"stride\"\n value {\n i: 2\n }\n }\n}\nnode {\n name: \"GRADIENTS/conv1/ReluGradient\"\n op: \"ReluGradient\"\n input: \"conv1/conv1_1\"\n input: \"GRADIENTS/conv1/conv1_grad\"\n device: \"/gpu:0\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n}\nnode {\n name: \"GRADIENTS/ConvGradient\"\n op: \"ConvGradient\"\n input: \"data\"\n input: \"conv1/conv1_w\"\n input: \"GRADIENTS/conv1/conv1_grad_1\"\n device: \"/gpu:0\"\n attr {\n key: \"exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"kernel\"\n value {\n i: 11\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"stride\"\n value {\n i: 4\n }\n }\n}\nnode {\n name: \"GRADIENTS/NCHW2NHWC\"\n op: \"NCHW2NHWC\"\n input: \"GRADIENTS/data_grad\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"conv1/conv1_w\"\n op: \"Blob\"\n input: \"conv1/XavierFill:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"classifier/fc\"\n op: \"Blob\"\n input: \"classifier/FC:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"data_nhwc\"\n op: \"Blob\"\n input: \"ImageInput:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/conv1/conv1_b_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/ConvGradient:1\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/pred_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/LabelCrossEntropyGradient:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/fc_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/SoftmaxGradient:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"conv1/conv1_b\"\n op: \"Blob\"\n input: \"conv1/ConstantFill:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/fc_b_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/c/FCGradient:1\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/fc_w_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/c/FCGradient:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"label\"\n op: \"Blob\"\n input: \"ImageInput:1\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/data_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/ConvGradient:2\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"classifier/loss\"\n op: \"Blob\"\n input: \"classifier/AveragedLoss:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"conv1/conv1\"\n op: \"Blob\"\n input: \"conv1/Conv:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/conv1/conv1_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/conv1/MaxPoolGradient:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"classifier/xent\"\n op: \"Blob\"\n input: \"classifier/LabelCrossEntropy:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/loss_autogen_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/ConstantFill:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"classifier/fc_w\"\n op: \"Blob\"\n input: \"classifier/XavierFill:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"conv1/conv1_1\"\n op: \"Blob\"\n input: \"conv1/Relu:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"db\"\n op: \"Placeholder\"\n}\nnode {\n name: \"classifier/pred\"\n op: \"Blob\"\n input: \"classifier/Softmax:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"classifier/fc_b\"\n op: \"Blob\"\n input: \"classifier/ConstantFill:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/classifier/xent_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/AveragedLossGradient:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"data\"\n op: \"Blob\"\n input: \"NHWC2NCHW:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/conv1/conv1_w_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/ConvGradient:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/conv1/conv1_grad_1\"\n op: \"Blob\"\n input: \"GRADIENTS/conv1/ReluGradient:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/data_nhwc_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/NCHW2NHWC:0\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"GRADIENTS/conv1/pool1_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/c/FCGradient:2\"\n device: \"/gpu:0\"\n}\nnode {\n name: \"conv1/pool1\"\n op: \"Blob\"\n input: \"conv1/MaxPool:0\"\n device: \"/gpu:0\"\n}\n\"\"\"\n\n" }, { "path": "tensorboardX/tests/expect/test_caffe2.test_simple_cnnmodel.expect", "content": "node {\n name: \"conv1/XavierFill\"\n op: \"XavierFill\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 96\n }\n dim {\n size: 3\n }\n dim {\n size: 11\n }\n dim {\n size: 11\n }\n }\n }\n }\n }\n}\nnode {\n name: \"conv1/ConstantFill\"\n op: \"ConstantFill\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 96\n }\n }\n }\n }\n }\n}\nnode {\n name: \"classifier/XavierFill\"\n op: \"XavierFill\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 1000\n }\n dim {\n size: 4096\n }\n }\n }\n }\n }\n}\nnode {\n name: \"classifier/ConstantFill\"\n op: \"ConstantFill\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 1000\n }\n }\n }\n }\n }\n}\nnode {\n name: \"conv1/Conv\"\n op: \"Conv\"\n input: \"conv1/data\"\n input: \"conv1/conv1_w\"\n input: \"conv1/conv1_b\"\n attr {\n key: \"exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"kernel\"\n value {\n i: 11\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"stride\"\n value {\n i: 4\n }\n }\n}\nnode {\n name: \"conv1/Relu\"\n op: \"Relu\"\n input: \"conv1/conv1\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n}\nnode {\n name: \"conv1/MaxPool\"\n op: \"MaxPool\"\n input: \"conv1/conv1_1\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"kernel\"\n value {\n i: 2\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"stride\"\n value {\n i: 2\n }\n }\n}\nnode {\n name: \"classifier/FC\"\n op: \"FC\"\n input: \"conv1/pool1\"\n input: \"classifier/fc_w\"\n input: \"classifier/fc_b\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"use_cudnn\"\n value {\n i: 1\n }\n }\n}\nnode {\n name: \"classifier/Softmax\"\n op: \"Softmax\"\n input: \"classifier/fc\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n}\nnode {\n name: \"classifier/LabelCrossEntropy\"\n op: \"LabelCrossEntropy\"\n input: \"classifier/pred\"\n input: \"classifier/label\"\n}\nnode {\n name: \"classifier/AveragedLoss\"\n op: \"AveragedLoss\"\n input: \"classifier/xent\"\n}\nnode {\n name: \"conv1/conv1_w\"\n op: \"Blob\"\n input: \"conv1/XavierFill:0\"\n}\nnode {\n name: \"conv1/conv1_b\"\n op: \"Blob\"\n input: \"conv1/ConstantFill:0\"\n}\nnode {\n name: \"classifier/fc_w\"\n op: \"Blob\"\n input: \"classifier/XavierFill:0\"\n}\nnode {\n name: \"classifier/fc_b\"\n op: \"Blob\"\n input: \"classifier/ConstantFill:0\"\n}\nnode {\n name: \"conv1/data\"\n op: \"Placeholder\"\n}\nnode {\n name: \"conv1/conv1_w\"\n op: \"Blob\"\n input: \"conv1/XavierFill:0\"\n}\nnode {\n name: \"conv1/conv1_b\"\n op: \"Blob\"\n input: \"conv1/ConstantFill:0\"\n}\nnode {\n name: \"conv1/conv1\"\n op: \"Blob\"\n input: \"conv1/Conv:0\"\n}\nnode {\n name: \"conv1/conv1\"\n op: \"Blob\"\n input: \"conv1/Conv:0\"\n}\nnode {\n name: \"conv1/conv1_1\"\n op: \"Blob\"\n input: \"conv1/Relu:0\"\n}\nnode {\n name: \"conv1/conv1_1\"\n op: \"Blob\"\n input: \"conv1/Relu:0\"\n}\nnode {\n name: \"conv1/pool1\"\n op: \"Blob\"\n input: \"conv1/MaxPool:0\"\n}\nnode {\n name: \"conv1/pool1\"\n op: \"Blob\"\n input: \"conv1/MaxPool:0\"\n}\nnode {\n name: \"classifier/fc_w\"\n op: \"Blob\"\n input: \"classifier/XavierFill:0\"\n}\nnode {\n name: \"classifier/fc_b\"\n op: \"Blob\"\n input: \"classifier/ConstantFill:0\"\n}\nnode {\n name: \"classifier/fc\"\n op: \"Blob\"\n input: \"classifier/FC:0\"\n}\nnode {\n name: \"classifier/fc\"\n op: \"Blob\"\n input: \"classifier/FC:0\"\n}\nnode {\n name: \"classifier/pred\"\n op: \"Blob\"\n input: \"classifier/Softmax:0\"\n}\nnode {\n name: \"classifier/pred\"\n op: \"Blob\"\n input: \"classifier/Softmax:0\"\n}\nnode {\n name: \"classifier/label\"\n op: \"Placeholder\"\n}\nnode {\n name: \"classifier/xent\"\n op: \"Blob\"\n input: \"classifier/LabelCrossEntropy:0\"\n}\nnode {\n name: \"classifier/xent\"\n op: \"Blob\"\n input: \"classifier/LabelCrossEntropy:0\"\n}\nnode {\n name: \"classifier/loss\"\n op: \"Blob\"\n input: \"classifier/AveragedLoss:0\"\n}\n" }, { "path": "tensorboardX/tests/expect/test_caffe2.test_simple_model.expect", "content": "node {\n name: \"conv1/XavierFill\"\n op: \"XavierFill\"\n device: \"/cpu:*\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 20\n }\n dim {\n size: 1\n }\n dim {\n size: 5\n }\n dim {\n size: 5\n }\n }\n }\n }\n }\n}\nnode {\n name: \"conv1/ConstantFill\"\n op: \"ConstantFill\"\n device: \"/cpu:*\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 20\n }\n }\n }\n }\n }\n}\nnode {\n name: \"conv1/XavierFill_1\"\n op: \"XavierFill\"\n device: \"/cpu:*\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 100\n }\n dim {\n size: 20\n }\n dim {\n size: 5\n }\n dim {\n size: 5\n }\n }\n }\n }\n }\n}\nnode {\n name: \"conv1/ConstantFill_1\"\n op: \"ConstantFill\"\n device: \"/cpu:*\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 100\n }\n }\n }\n }\n }\n}\nnode {\n name: \"classifier/XavierFill\"\n op: \"XavierFill\"\n device: \"/cpu:*\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 500\n }\n dim {\n size: 1600\n }\n }\n }\n }\n }\n}\nnode {\n name: \"classifier/ConstantFill\"\n op: \"ConstantFill\"\n device: \"/cpu:*\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 500\n }\n }\n }\n }\n }\n}\nnode {\n name: \"classifier/XavierFill_1\"\n op: \"XavierFill\"\n device: \"/cpu:*\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 10\n }\n dim {\n size: 500\n }\n }\n }\n }\n }\n}\nnode {\n name: \"classifier/ConstantFill_1\"\n op: \"ConstantFill\"\n device: \"/cpu:*\"\n attr {\n key: \"_output_shapes\"\n value {\n list {\n shape {\n dim {\n size: 10\n }\n }\n }\n }\n }\n}\nnode {\n name: \"conv1/Conv\"\n op: \"Conv\"\n input: \"conv1/data\"\n input: \"conv1/conv1_w\"\n input: \"conv1/conv1_b\"\n device: \"/cpu:*\"\n attr {\n key: \"exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"kernel\"\n value {\n i: 5\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n}\nnode {\n name: \"conv1/MaxPool\"\n op: \"MaxPool\"\n input: \"conv1/conv1\"\n device: \"/cpu:*\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"kernel\"\n value {\n i: 2\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"stride\"\n value {\n i: 2\n }\n }\n}\nnode {\n name: \"conv1/Conv_1\"\n op: \"Conv\"\n input: \"conv1/pool1\"\n input: \"conv1/conv2_w\"\n input: \"conv1/conv2_b\"\n device: \"/cpu:*\"\n attr {\n key: \"exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"kernel\"\n value {\n i: 5\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n}\nnode {\n name: \"conv1/MaxPool_1\"\n op: \"MaxPool\"\n input: \"conv1/conv2\"\n device: \"/cpu:*\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"kernel\"\n value {\n i: 2\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"stride\"\n value {\n i: 2\n }\n }\n}\nnode {\n name: \"classifier/FC\"\n op: \"FC\"\n input: \"conv1/pool2\"\n input: \"classifier/fc3_w\"\n input: \"classifier/fc3_b\"\n device: \"/cpu:*\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"use_cudnn\"\n value {\n i: 1\n }\n }\n}\nnode {\n name: \"classifier/Relu\"\n op: \"Relu\"\n input: \"classifier/fc3\"\n device: \"/cpu:*\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n}\nnode {\n name: \"classifier/FC_1\"\n op: \"FC\"\n input: \"classifier/fc3_1\"\n input: \"classifier/pred_w\"\n input: \"classifier/pred_b\"\n device: \"/cpu:*\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"use_cudnn\"\n value {\n i: 1\n }\n }\n}\nnode {\n name: \"classifier/Softmax\"\n op: \"Softmax\"\n input: \"classifier/pred\"\n device: \"/cpu:*\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n}\nnode {\n name: \"classifier/LabelCrossEntropy\"\n op: \"LabelCrossEntropy\"\n input: \"classifier/softmax\"\n input: \"classifier/label\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/AveragedLoss\"\n op: \"AveragedLoss\"\n input: \"classifier/xent\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/classifier/ConstantFill\"\n op: \"ConstantFill\"\n input: \"classifier/loss\"\n device: \"/cpu:*\"\n attr {\n key: \"value\"\n value {\n f: 1.0\n }\n }\n}\nnode {\n name: \"GRADIENTS/classifier/AveragedLossGradient\"\n op: \"AveragedLossGradient\"\n input: \"classifier/xent\"\n input: \"GRADIENTS/classifier/loss_autogen_grad\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/classifier/LabelCrossEntropyGradient\"\n op: \"LabelCrossEntropyGradient\"\n input: \"classifier/softmax\"\n input: \"classifier/label\"\n input: \"GRADIENTS/classifier/xent_grad\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/classifier/SoftmaxGradient\"\n op: \"SoftmaxGradient\"\n input: \"classifier/softmax\"\n input: \"GRADIENTS/classifier/softmax_grad\"\n device: \"/cpu:*\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n}\nnode {\n name: \"GRADIENTS/classifier/FCGradient\"\n op: \"FCGradient\"\n input: \"classifier/fc3_1\"\n input: \"classifier/pred_w\"\n input: \"GRADIENTS/classifier/pred_grad\"\n device: \"/cpu:*\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"use_cudnn\"\n value {\n i: 1\n }\n }\n}\nnode {\n name: \"GRADIENTS/classifier/ReluGradient\"\n op: \"ReluGradient\"\n input: \"classifier/fc3_1\"\n input: \"GRADIENTS/classifier/fc3_grad\"\n device: \"/cpu:*\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n}\nnode {\n name: \"GRADIENTS/c/FCGradient\"\n op: \"FCGradient\"\n input: \"conv1/pool2\"\n input: \"classifier/fc3_w\"\n input: \"GRADIENTS/classifier/fc3_grad_1\"\n device: \"/cpu:*\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"use_cudnn\"\n value {\n i: 1\n }\n }\n}\nnode {\n name: \"GRADIENTS/conv1/MaxPoolGradient\"\n op: \"MaxPoolGradient\"\n input: \"conv1/conv2\"\n input: \"conv1/pool2\"\n input: \"GRADIENTS/conv1/pool2_grad\"\n device: \"/cpu:*\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"kernel\"\n value {\n i: 2\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"stride\"\n value {\n i: 2\n }\n }\n}\nnode {\n name: \"GRADIENTS/conv1/ConvGradient\"\n op: \"ConvGradient\"\n input: \"conv1/pool1\"\n input: \"conv1/conv2_w\"\n input: \"GRADIENTS/conv1/conv2_grad\"\n device: \"/cpu:*\"\n attr {\n key: \"exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"kernel\"\n value {\n i: 5\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n}\nnode {\n name: \"GRADIENTS/conv1/MaxPoolGradient_1\"\n op: \"MaxPoolGradient\"\n input: \"conv1/conv1\"\n input: \"conv1/pool1\"\n input: \"GRADIENTS/conv1/pool1_grad\"\n device: \"/cpu:*\"\n attr {\n key: \"cudnn_exhaustive_search\"\n value {\n i: 0\n }\n }\n attr {\n key: \"kernel\"\n value {\n i: 2\n }\n }\n attr {\n key: \"order\"\n value {\n s: \"NCHW\"\n }\n }\n attr {\n key: \"stride\"\n value {\n i: 2\n }\n }\n}\nnode {\n name: \"conv1/conv1_w\"\n op: \"Blob\"\n input: \"conv1/XavierFill:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/conv1_b\"\n op: \"Blob\"\n input: \"conv1/ConstantFill:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/conv2_w\"\n op: \"Blob\"\n input: \"conv1/XavierFill_1:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/conv2_b\"\n op: \"Blob\"\n input: \"conv1/ConstantFill_1:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/fc3_w\"\n op: \"Blob\"\n input: \"classifier/XavierFill:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/fc3_b\"\n op: \"Blob\"\n input: \"classifier/ConstantFill:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/pred_w\"\n op: \"Blob\"\n input: \"classifier/XavierFill_1:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/pred_b\"\n op: \"Blob\"\n input: \"classifier/ConstantFill_1:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/data\"\n op: \"Placeholder\"\n}\nnode {\n name: \"conv1/conv1_w\"\n op: \"Blob\"\n input: \"conv1/XavierFill:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/conv1_b\"\n op: \"Blob\"\n input: \"conv1/ConstantFill:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/conv1\"\n op: \"Blob\"\n input: \"conv1/Conv:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/conv1\"\n op: \"Blob\"\n input: \"conv1/Conv:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/pool1\"\n op: \"Blob\"\n input: \"conv1/MaxPool:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/pool1\"\n op: \"Blob\"\n input: \"conv1/MaxPool:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/conv2_w\"\n op: \"Blob\"\n input: \"conv1/XavierFill_1:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/conv2_b\"\n op: \"Blob\"\n input: \"conv1/ConstantFill_1:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/conv2\"\n op: \"Blob\"\n input: \"conv1/Conv_1:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/conv2\"\n op: \"Blob\"\n input: \"conv1/Conv_1:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/pool2\"\n op: \"Blob\"\n input: \"conv1/MaxPool_1:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/pool2\"\n op: \"Blob\"\n input: \"conv1/MaxPool_1:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/fc3_w\"\n op: \"Blob\"\n input: \"classifier/XavierFill:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/fc3_b\"\n op: \"Blob\"\n input: \"classifier/ConstantFill:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/fc3\"\n op: \"Blob\"\n input: \"classifier/FC:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/fc3\"\n op: \"Blob\"\n input: \"classifier/FC:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/fc3_1\"\n op: \"Blob\"\n input: \"classifier/Relu:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/fc3_1\"\n op: \"Blob\"\n input: \"classifier/Relu:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/pred_w\"\n op: \"Blob\"\n input: \"classifier/XavierFill_1:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/pred_b\"\n op: \"Blob\"\n input: \"classifier/ConstantFill_1:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/pred\"\n op: \"Blob\"\n input: \"classifier/FC_1:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/pred\"\n op: \"Blob\"\n input: \"classifier/FC_1:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/softmax\"\n op: \"Blob\"\n input: \"classifier/Softmax:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/softmax\"\n op: \"Blob\"\n input: \"classifier/Softmax:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/label\"\n op: \"Placeholder\"\n}\nnode {\n name: \"classifier/xent\"\n op: \"Blob\"\n input: \"classifier/LabelCrossEntropy:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/xent\"\n op: \"Blob\"\n input: \"classifier/LabelCrossEntropy:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/loss\"\n op: \"Blob\"\n input: \"classifier/AveragedLoss:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/loss\"\n op: \"Blob\"\n input: \"classifier/AveragedLoss:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/classifier/loss_autogen_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/ConstantFill:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/xent\"\n op: \"Blob\"\n input: \"classifier/LabelCrossEntropy:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/classifier/loss_autogen_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/ConstantFill:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/classifier/xent_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/AveragedLossGradient:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/softmax\"\n op: \"Blob\"\n input: \"classifier/Softmax:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/label\"\n op: \"Placeholder\"\n}\nnode {\n name: \"GRADIENTS/classifier/xent_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/AveragedLossGradient:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/classifier/softmax_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/LabelCrossEntropyGradient:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/softmax\"\n op: \"Blob\"\n input: \"classifier/Softmax:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/classifier/softmax_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/LabelCrossEntropyGradient:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/classifier/pred_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/SoftmaxGradient:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/fc3_1\"\n op: \"Blob\"\n input: \"classifier/Relu:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/pred_w\"\n op: \"Blob\"\n input: \"classifier/XavierFill_1:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/classifier/pred_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/SoftmaxGradient:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/classifier/pred_w_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/FCGradient:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/classifier/pred_b_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/FCGradient:1\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/classifier/fc3_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/FCGradient:2\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/fc3_1\"\n op: \"Blob\"\n input: \"classifier/Relu:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/classifier/fc3_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/FCGradient:2\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/classifier/fc3_grad_1\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/ReluGradient:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/pool2\"\n op: \"Blob\"\n input: \"conv1/MaxPool_1:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"classifier/fc3_w\"\n op: \"Blob\"\n input: \"classifier/XavierFill:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/classifier/fc3_grad_1\"\n op: \"Blob\"\n input: \"GRADIENTS/classifier/ReluGradient:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/classifier/fc3_w_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/c/FCGradient:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/classifier/fc3_b_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/c/FCGradient:1\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/conv1/pool2_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/c/FCGradient:2\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/conv2\"\n op: \"Blob\"\n input: \"conv1/Conv_1:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/pool2\"\n op: \"Blob\"\n input: \"conv1/MaxPool_1:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/conv1/pool2_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/c/FCGradient:2\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/conv1/conv2_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/conv1/MaxPoolGradient:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/pool1\"\n op: \"Blob\"\n input: \"conv1/MaxPool:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/conv2_w\"\n op: \"Blob\"\n input: \"conv1/XavierFill_1:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/conv1/conv2_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/conv1/MaxPoolGradient:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/conv1/conv2_w_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/conv1/ConvGradient:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/conv1/conv2_b_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/conv1/ConvGradient:1\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/conv1/pool1_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/conv1/ConvGradient:2\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/conv1\"\n op: \"Blob\"\n input: \"conv1/Conv:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"conv1/pool1\"\n op: \"Blob\"\n input: \"conv1/MaxPool:0\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/conv1/pool1_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/conv1/ConvGradient:2\"\n device: \"/cpu:*\"\n}\nnode {\n name: \"GRADIENTS/conv1/conv1_grad\"\n op: \"Blob\"\n input: \"GRADIENTS/conv1/MaxPoolGradient_1:0\"\n device: \"/cpu:*\"\n}\n" }, { "path": "tensorboardX/tests/expect/test_pr_curve.test_pr_purve.expect", "content": "value {\n tag: \"tag\"\n tensor {\n dtype: DT_FLOAT\n tensor_shape {\n dim {\n size: 6\n }\n dim {\n size: 1\n }\n }\n float_val: 57.0\n float_val: 43.0\n float_val: 0.0\n float_val: 0.0\n float_val: 0.57\n float_val: 1.0\n }\n metadata {\n plugin_data {\n plugin_name: \"pr_curves\"\n content: \"\\020\\001\"\n }\n }\n}\n" }, { "path": "tensorboardX/tests/expect/test_pr_curve.test_pr_purve_raw.expect", "content": "value {\n tag: \"prcurve with raw data\"\n tensor {\n dtype: DT_FLOAT\n tensor_shape {\n dim {\n size: 6\n }\n dim {\n size: 5\n }\n }\n float_val: 75.0\n float_val: 64.0\n float_val: 21.0\n float_val: 5.0\n float_val: 0.0\n float_val: 150.0\n float_val: 105.0\n float_val: 18.0\n float_val: 0.0\n float_val: 0.0\n float_val: 0.0\n float_val: 45.0\n float_val: 132.0\n float_val: 150.0\n float_val: 150.0\n float_val: 0.0\n float_val: 11.0\n float_val: 54.0\n float_val: 70.0\n float_val: 75.0\n float_val: 0.3333333\n float_val: 0.3786982\n float_val: 0.5384616\n float_val: 1.0\n float_val: 0.0\n float_val: 1.0\n float_val: 0.8533334\n float_val: 0.28\n float_val: 0.0666667\n float_val: 0.0\n }\n metadata {\n plugin_data {\n plugin_name: \"pr_curves\"\n content: \"\\020\\001\"\n }\n }\n}\n" }, { "path": "tensorboardX/tests/expect/test_summary.test_audio.expect", "content": "value {\n tag: \"dummy\"\n audio {\n sample_rate: 44100.0\n num_channels: 1\n length_frames: 42\n encoded_audio_string: \"RIFFx\\000\\000\\000WAVEfmt \\020\\000\\000\\000\\001\\000\\001\\000D\\254\\000\\000\\210X\\001\\000\\002\\000\\020\\000dataT\\000\\000\\000\\000\\000\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\\377\\177\"\n content_type: \"audio/wav\"\n }\n}\n" }, { "path": "tensorboardX/tests/expect/test_summary.test_custom_scalars.expect", "content": "value {\n tag: \"custom_scalars__config__\"\n tensor {\n dtype: DT_STRING\n tensor_shape {\n }\n string_val: \"\\022(\\n\\006Taiwan\\022\\036\\n\\004twse\\022\\026\\n\\ttwse/0050\\n\\ttwse/2330\\022]\\n\\003USA\\022$\\n\\003dow\\032\\035\\n\\033\\n\\007dow/aaa\\022\\007dow/bbb\\032\\007dow/ccc\\0220\\n\\006nasdaq\\032&\\n$\\n\\nnasdaq/aaa\\022\\nnasdaq/bbb\\032\\nnasdaq/ccc\"\n }\n metadata {\n plugin_data {\n plugin_name: \"custom_scalars\"\n }\n }\n}\n" }, { "path": "tensorboardX/tests/expect/test_summary.test_float32_image.expect", "content": "value {\n tag: \"dummy\"\n image {\n height: 32\n width: 32\n colorspace: 3\n encoded_image_string: \"\\211PNG\\r\\n\\032\\n\\000\\000\\000\\rIHDR\\000\\000\\000 \\000\\000\\000 \\010\\002\\000\\000\\000\\374\\030\\355\\243\\000\\000\\000DIDATx\\234cd``\\370OK\\300\\370\\340\\301\\003\\232Z\\3002j\\301\\360\\267\\200QAA\\201\\266\\026\\214\\346\\203Q\\013\\006\\277\\005\\243\\371\\200 \\030\\372\\2210\\204b\\311\\233\\305/\\344G\\000\\334\\236\\021Uu\\005R\\000\\377\\007\\244\\224\\342\\013||\\007\\2655\\330BfP\\215\\337S`>:{_l\\020\\335\\242\\tX6-\\000\\032r\\007G\\316\\000\\2561\\226\\201\\244\\252/\\005V\\357\\026\\271\\003\\033\\0149\\000\\232\\270\\003+\\260\\301\\220\\003\\240y\\000T\\221\\324V\\250_v\\320\\000\\000\\000\\000IEND\\256B`\\202\"\n }\n}\n" }, { "path": "tensorboardX/tests/expect/test_summary.test_image_with_four_channel.expect", "content": "value {\n tag: \"dummy\"\n image {\n height: 8\n width: 8\n colorspace: 4\n encoded_image_string: \"\\211PNG\\r\\n\\032\\n\\000\\000\\000\\rIHDR\\000\\000\\000\\010\\000\\000\\000\\010\\010\\006\\000\\000\\000\\304\\017\\276\\213\\000\\000\\000\\036IDATx\\234cd8\\320\\340\\360\\037\\017`\\371\\361\\343\\307\\217\\037\\204\\024\\0204a\\260+\\000\\000\\240\\302\\373\\327\\246\\231O\\'\\000\\000\\000\\000IEND\\256B`\\202\"\n }\n}\n" }, { "path": "tensorboardX/tests/expect/test_summary.test_image_with_four_channel_batched.expect", "content": "value {\n tag: \"dummy\"\n image {\n height: 8\n width: 16\n colorspace: 4\n encoded_image_string: \"\\211PNG\\r\\n\\032\\n\\000\\000\\000\\rIHDR\\000\\000\\000\\020\\000\\000\\000\\010\\010\\006\\000\\000\\000\\360v\\177\\227\\000\\000\\000-IDATx\\234cd8\\320\\340\\360\\037\\017`ggg\\307\\'\\317\\362\\343\\307\\217\\037?\\360(\\370\\001\\305x\\r\\300g\\003!0j\\000\\025\\014\\000\\000\\356b\\366\\370\\366\\336\\316\\301\\000\\000\\000\\000IEND\\256B`\\202\"\n }\n}\n" }, { "path": "tensorboardX/tests/expect/test_summary.test_image_with_one_channel.expect", "content": "value {\n tag: \"dummy\"\n image {\n height: 8\n width: 8\n colorspace: 3\n encoded_image_string: \"\\211PNG\\r\\n\\032\\n\\000\\000\\000\\rIHDR\\000\\000\\000\\010\\000\\000\\000\\010\\010\\002\\000\\000\\000Km)\\334\\000\\000\\000\\031IDATx\\234cd``\\370\\217\\r0\\376\\370\\361\\003\\253\\004\\313\\240\\224\\000\\000;\\267\\273\\313%\\020=\\255\\000\\000\\000\\000IEND\\256B`\\202\"\n }\n}\n" }, { "path": "tensorboardX/tests/expect/test_summary.test_image_with_one_channel_batched.expect", "content": "value {\n tag: \"dummy\"\n image {\n height: 8\n width: 16\n colorspace: 3\n encoded_image_string: \"\\211PNG\\r\\n\\032\\n\\000\\000\\000\\rIHDR\\000\\000\\000\\020\\000\\000\\000\\010\\010\\002\\000\\000\\000\\177\\024\\350\\300\\000\\000\\000(IDATx\\234cd``\\370\\217\\r\\034?~\\034\\2538\\313\\217\\037?~\\374\\370\\201)\\201U\\020\\252\\001\\253\\304\\250\\006$\\000\\000\\230\\346y\\315\\204l;t\\000\\000\\000\\000IEND\\256B`\\202\"\n }\n}\n" }, { "path": "tensorboardX/tests/expect/test_summary.test_image_without_channel.expect", "content": "value {\n tag: \"dummy\"\n image {\n height: 8\n width: 8\n colorspace: 3\n encoded_image_string: \"\\211PNG\\r\\n\\032\\n\\000\\000\\000\\rIHDR\\000\\000\\000\\010\\000\\000\\000\\010\\010\\002\\000\\000\\000Km)\\334\\000\\000\\000\\031IDATx\\234cd``\\370\\217\\r0\\376\\370\\361\\003\\253\\004\\313\\240\\224\\000\\000;\\267\\273\\313%\\020=\\255\\000\\000\\000\\000IEND\\256B`\\202\"\n }\n}\n" }, { "path": "tensorboardX/tests/expect/test_summary.test_mesh.expect", "content": "value {\n tag: \"my_mesh_1\"\n tensor {\n dtype: DT_FLOAT\n tensor_shape {\n dim {\n size: 1\n }\n dim {\n size: 4\n }\n dim {\n size: 3\n }\n }\n float_val: 1.0\n float_val: 1.0\n float_val: 1.0\n float_val: -1.0\n float_val: -1.0\n float_val: 1.0\n float_val: 1.0\n float_val: -1.0\n float_val: -1.0\n float_val: -1.0\n float_val: 1.0\n float_val: -1.0\n }\n metadata {\n plugin_data {\n plugin_name: \"mesh\"\n content: \"\\022\\007my_mesh\\030\\001*\\004null2\\003\\001\\004\\003\"\n }\n }\n}\nvalue {\n tag: \"my_mesh_2\"\n tensor {\n dtype: DT_FLOAT\n tensor_shape {\n dim {\n size: 1\n }\n dim {\n size: 4\n }\n dim {\n size: 3\n }\n }\n float_val: 0.0\n float_val: 2.0\n float_val: 3.0\n float_val: 0.0\n float_val: 3.0\n float_val: 1.0\n float_val: 0.0\n float_val: 1.0\n float_val: 2.0\n float_val: 1.0\n float_val: 3.0\n float_val: 2.0\n }\n metadata {\n plugin_data {\n plugin_name: \"mesh\"\n content: \"\\022\\007my_mesh\\030\\002*\\004null2\\003\\001\\004\\003\"\n }\n }\n}\nvalue {\n tag: \"my_mesh_3\"\n tensor {\n dtype: DT_FLOAT\n tensor_shape {\n dim {\n size: 1\n }\n dim {\n size: 4\n }\n dim {\n size: 3\n }\n }\n float_val: 255.0\n float_val: 0.0\n float_val: 0.0\n float_val: 0.0\n float_val: 255.0\n float_val: 0.0\n float_val: 0.0\n float_val: 0.0\n float_val: 255.0\n float_val: 255.0\n float_val: 0.0\n float_val: 255.0\n }\n metadata {\n plugin_data {\n plugin_name: \"mesh\"\n content: \"\\022\\007my_mesh\\030\\003*\\004null2\\003\\001\\004\\003\"\n }\n }\n}\n" }, { "path": "tensorboardX/tests/expect/test_summary.test_text.expect", "content": "value {\n tag: \"dummy/text_summary\"\n tensor {\n dtype: DT_STRING\n tensor_shape {\n dim {\n size: 1\n }\n }\n string_val: \"text 123\"\n }\n metadata {\n plugin_data {\n plugin_name: \"text\"\n }\n }\n}\n" }, { "path": "tensorboardX/tests/expect/test_summary.test_uint8_image.expect", "content": "value {\n tag: \"dummy\"\n image {\n height: 32\n width: 32\n colorspace: 3\n encoded_image_string: \"\\211PNG\\r\\n\\032\\n\\000\\000\\000\\rIHDR\\000\\000\\000 \\000\\000\\000 \\010\\002\\000\\000\\000\\374\\030\\355\\243\\000\\000\\000CIDATx\\234cd```\\244)PPP\\240\\251\\371,\\243\\026\\014\\177\\013\\030\\037>>===<<<;;;:::999888777666555444333222111000///...---,,,+++***)))(((\\'\\'\\'&&&%%%$$$###\\\"\\\"\\\"!!! \\037\\037\\037\\036\\036\\036\\035\\035\\035\\034\\034\\034\\033\\033\\033\\032\\032\\032\\031\\031\\031\\030\\030\\030\\027\\027\\027\\026\\026\\026\\025\\025\\025\\024\\024\\024\\023\\023\\023\\022\\022\\022\\021\\021\\021\\020\\020\\020\\017\\017\\017\\016\\016\\016\\r\\r\\r\\014\\014\\014\\013\\013\\013\\n\\n\\n\\t\\t\\t\\010\\010\\010\\007\\007\\007\\006\\006\\006\\005\\005\\005\\004\\004\\004\\003\\003\\003\\002\\002\\002\\001\\001\\001\\000\\000\\000!\\377\\013NETSCAPE2.0\\003\\001\\377\\377\\000!\\371\\004\\010\\031\\000\\000\\000,\\000\\000\\000\\000\\020\\000\\020\\000\\000\\010\\377\\000\\377\\001\\010 `\\000\\201\\002\\006~\\001\\013&l\\030\\261b\\306\\016 H\\240`\\001\\203\\006\\016\\216!K\\246l\\031\\263f\\316\\036@\\210 a\\002\\205\\n\\026\\236A\\213&m\\032\\265j\\326.`\\310\\240a\\003\\207\\016\\036\\256a\\313\\246m\\033\\267n\\336>\\200\\010!b\\004\\211\\022&\\276\\201\\013\\'n\\034\\271r\\346N\\240H\\241b\\005\\213\\026.\\316\\241K\\247n\\035\\273v\\356^\\300\\210!c\\006\\215\\0326\\336\\301\\213\\'o\\036\\275z\\366n\\340\\310\\241c\\007\\217\\036>\\356\\341\\313\\247o\\037\\277~\\376\\376\\000\\n$h\\020\\241B\\206~\\000\\t\\\"d\\010\\221\\\"F\\016!J\\244h\\021\\243F\\216\\216 I\\242d\\t\\223&N\\036A\\212$i\\022\\245J\\226\\236@\\211\\\"e\\n\\225*V.a\\312\\244i\\023\\247N\\236\\256`\\311\\242e\\013\\227.^>\\201\\n%j\\024\\251R\\246\\276\\200\\t#f\\014\\2312fN\\241J\\245j\\025\\253V\\256\\316\\240I\\243f\\r\\2336n^\\301%\\212%k\\026\\255Z\\266\\336\\300\\211#g\\016\\235:vn\\341\\312\\245k\\027\\257^\\276\\356\\340\\311\\243g\\017\\037\\200}\\374\\004\\004\\000!\\371\\004\\010\\031\\000\\000\\000,\\000\\000\\000\\000\\020\\000\\020\\000\\000\\010\\377\\000\\177\\000\\t\\\"d\\010\\221\\\"F\\376\\001\\010 `\\000\\201\\002\\006\\216 I\\242d\\t\\223&N\\016 H\\240`\\001\\203\\006\\016\\236@\\211\\\"e\\n\\225*V\\036@\\210 a\\002\\205\\n\\026\\256`\\311\\242e\\013\\227.^.`\\310\\240a\\003\\207\\016\\036\\276\\200\\t#f\\014\\2312f>\\200\\010!b\\004\\211\\022&\\316\\240I\\243f\\r\\2336nN\\240H\\241b\\005\\213\\026.\\336\\300\\211#g\\016\\235:v^\\300\\210!c\\006\\215\\0326\\356\\340\\311\\243g\\017\\237>~n\\340\\310\\241c\\007\\217\\036>~\\001\\013&l\\030\\261b\\306\\376\\000\\n$h\\020\\241B\\206\\216!K\\246l\\031\\263f\\316\\016!J\\244h\\021\\243F\\216\\236A\\213&m\\032\\265j\\326\\036A\\212$i\\022\\245J\\226\\256a\\313\\246m\\033\\267n\\336.a\\312\\244i\\023\\247N\\236\\276\\201\\013\\'n\\034\\271r\\346>\\201\\n%j\\024\\251R\\246\\316\\241K\\247n\\035\\273v\\356N\\241J\\245j\\025\\253V\\256\\336\\301%\\213\\'o\\036\\275z\\366^\\301\\212%k\\026\\255Z\\266\\356\\341\\313\\247o\\037\\277~\\376n\\341\\312\\245k\\027/\\200\\275|\\005\\004\\000!\\371\\004\\010\\031\\000\\000\\000,\\000\\000\\000\\000\\020\\000\\020\\000\\000\\010\\377\\000\\377\\000\\n$h\\020\\241B\\206~\\000\\t\\\"d\\010\\221\\\"F\\016!J\\244h\\021\\243F\\216\\216 I\\242d\\t\\223&N\\036A\\212$i\\022\\245J\\226\\236@\\211\\\"e\\n\\225*V.a\\312\\244i\\023\\247N\\236\\256`\\311\\242e\\013\\227.^>\\201\\n%j\\024\\251R\\246\\276\\200\\t#f\\014\\2312fN\\241J\\245j\\025\\253V\\256\\316\\240I\\243f\\r\\2336n^\\301\\212%k\\026\\255Z\\266\\336\\300\\211#g\\016\\235:vn\\341\\312\\245k\\027\\257^\\276\\356\\340\\311\\243g\\017\\237>~\\376\\001\\010 `\\000\\201\\002\\006~\\001\\013&l\\030\\261b\\306\\016 H\\240`\\001\\203\\006\\016\\216!K\\246l\\031\\263f\\316\\036@\\210 a\\002\\205\\n\\026\\236A\\213&m\\032\\265j\\326.`\\310\\240a\\003\\207\\016\\036\\256a\\313\\246m\\033\\267n\\336>\\200\\010!b\\004\\211\\022&\\276\\201\\013\\'n\\034\\271r\\346N\\240H\\241b\\005\\213\\026.\\316\\241K\\247n\\035\\273v\\356^\\300%\\210!c\\006\\215\\0326\\336\\301\\213\\'o\\036\\275z\\366n\\340\\310\\241c\\007\\217\\036>\\356\\341\\313\\247o\\037?\\200\\375\\374\\005\\004\\000;\"\n }\n}\n" }, { "path": "tensorboardX/tests/expect_reader.py", "content": "from __future__ import absolute_import, division, print_function, unicode_literals\nimport os\nimport sys\n\n\ndef removeWhiteChar(string):\n return string.replace(' ', '').replace('\\t', '').replace('\\n', '')\n\n\ndef compare_proto(str_to_compare, function_ptr):\n module_id = function_ptr.__class__.__module__\n functionName = function_ptr.id().split('.')[-1]\n test_file = os.path.realpath(sys.modules[module_id].__file__)\n expected_file = os.path.join(os.path.dirname(test_file),\n \"expect\",\n module_id.split('.')[-1] + '.' + functionName + \".expect\")\n print(\"expected_file: %s\" % expected_file)\n assert os.path.exists(expected_file)\n with open(expected_file) as f:\n expected = f.read()\n str_to_compare = str(str_to_compare)\n print(\"str_to_compare:\", removeWhiteChar(str_to_compare))\n print(\"expected:\", removeWhiteChar(expected))\n assert removeWhiteChar(str_to_compare) == removeWhiteChar(expected)\n\n\ndef write_proto(str_to_compare, function_ptr):\n module_id = function_ptr.__class__.__module__\n functionName = function_ptr.id().split('.')[-1]\n test_file = os.path.realpath(sys.modules[module_id].__file__)\n expected_file = os.path.join(os.path.dirname(test_file),\n \"expect\",\n module_id.split('.')[-1] + '.' + functionName + \".expect\")\n print(expected_file)\n with open(expected_file, 'w') as f:\n f.write(str(str_to_compare))\n" }, { "path": "tensorboardX/tests/record_writer_test.py", "content": "# Copyright 2019 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ==============================================================================\n\n# \"\"\"Tests for RecordWriter\"\"\"\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport six\nimport os\nfrom tensorboardX.record_writer import RecordWriter\nfrom tensorboard.compat.tensorflow_stub.pywrap_tensorflow import PyRecordReader_New\nimport unittest\n\n\nclass RecordWriterTest(unittest.TestCase):\n def get_temp_dir(self):\n import tempfile\n return tempfile.mkdtemp()\n\n def test_expect_bytes_written(self):\n filename = os.path.join(self.get_temp_dir(), \"expect_bytes_written\")\n byte_len = 64\n w = RecordWriter(filename)\n bytes_to_write = b\"x\" * byte_len\n w.write(bytes_to_write)\n w.close()\n with open(filename, 'rb') as f:\n self.assertEqual(len(f.read()), (8 + 4 + byte_len + 4)) # uint64+uint32+data+uint32\n\n def test_empty_record(self):\n filename = os.path.join(self.get_temp_dir(), \"empty_record\")\n w = RecordWriter(filename)\n bytes_to_write = b\"\"\n w.write(bytes_to_write)\n w.close()\n r = PyRecordReader_New(filename)\n r.GetNext()\n self.assertEqual(r.record(), bytes_to_write)\n\n def test_record_writer_roundtrip(self):\n filename = os.path.join(self.get_temp_dir(), \"record_writer_roundtrip\")\n w = RecordWriter(filename)\n bytes_to_write = b\"hello world\"\n times_to_test = 50\n for _ in range(times_to_test):\n w.write(bytes_to_write)\n w.close()\n\n r = PyRecordReader_New(filename)\n for i in range(times_to_test):\n r.GetNext()\n self.assertEqual(r.record(), bytes_to_write)\n\n # def test_expect_bytes_written_bytes_IO(self):\n # byte_len = 64\n # Bytes_io = six.BytesIO()\n # w = RecordWriter(Bytes_io)\n # bytes_to_write = b\"x\" * byte_len\n # w.write(bytes_to_write)\n # self.assertEqual(len(Bytes_io.getvalue()), (8 + 4 + byte_len + 4)) # uint64+uint32+data+uint32\n\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "tensorboardX/tests/test_beholder.py", "content": "from __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nfrom tensorboardX import SummaryWriter\nimport numpy as np\nimport pytest\nimport unittest\nimport tensorboardX.beholder as beholder_lib\nimport tensorboardX.beholder.file_system_tools as fio\nfrom collections import namedtuple\n\n\nclass BeholderTest(unittest.TestCase):\n def test_beholder(self):\n LOG_DIRECTORY = '/tmp/beholder-demo'\n tensor_and_name = namedtuple('tensor_and_name', 'tensor, name')\n fake_param = [tensor_and_name(np.random.randn(128, 768, 3), 'test' + str(i)) for i in range(5)]\n arrays = [tensor_and_name(np.random.randn(128, 768, 3), 'test' + str(i)) for i in range(5)]\n beholder = beholder_lib.Beholder(logdir=LOG_DIRECTORY)\n beholder.update(\n trainable=fake_param,\n arrays=arrays,\n frame=np.random.randn(128, 128),\n )\n\n def test_beholder_video(self):\n LOG_DIRECTORY = '/tmp/beholder-demo-recording'\n tensor_and_name = namedtuple('tensor_and_name', 'tensor, name')\n fake_param = [tensor_and_name(np.random.randn(128, 768, 3), 'test' + str(i)) for i in range(5)]\n arrays = [tensor_and_name(np.random.randn(128, 768, 3), 'test' + str(i)) for i in range(5)]\n beholder = beholder_lib.Beholder(logdir=LOG_DIRECTORY)\n pkl = fio.read_pickle(LOG_DIRECTORY + '/plugins/beholder/config.pkl')\n pkl['is_recording'] = True\n fio.write_pickle(pkl, LOG_DIRECTORY + '/plugins/beholder/config.pkl')\n for i in range(3):\n if i == 2:\n pkl = fio.read_pickle(LOG_DIRECTORY + '/plugins/beholder/config.pkl')\n pkl['is_recording'] = False\n fio.write_pickle(pkl, LOG_DIRECTORY + '/plugins/beholder/config.pkl')\n beholder.update(\n trainable=fake_param,\n arrays=arrays,\n frame=np.random.randn(128, 128),\n )\n" }, { "path": "tensorboardX/tests/test_caffe2.py", "content": "from __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\n\nfrom tensorboardX import SummaryWriter\nimport os\nimport unittest\n\n# try:\nimport numpy as np\nimport caffe2.python.brew as brew\nimport caffe2.python.cnn as cnn\nimport caffe2.python.core as core\nimport caffe2.python.model_helper as model_helper\nfrom caffe2.proto import caffe2_pb2\nfrom caffe2.python import workspace\nimport tensorboardX.caffe2_graph as tb\nfrom tensorboardX import x2num\nfrom .expect_reader import compare_proto, write_proto\n\n\nclass Caffe2Test(unittest.TestCase):\n def test_caffe2_np(self):\n workspace.FeedBlob(\"testBlob\", np.random.randn(1, 3, 64, 64).astype(np.float32))\n assert isinstance(x2num.make_np('testBlob'), np.ndarray)\n # assert isinstance(x2num.make_np('testBlob', 'IMG'), np.ndarray)\n\n def test_that_operators_gets_non_colliding_names(self):\n op = caffe2_pb2.OperatorDef()\n op.type = 'foo'\n op.input.extend(['foo'])\n tb._fill_missing_operator_names([op])\n self.assertEqual(op.input[0], 'foo')\n self.assertEqual(op.name, 'foo_1')\n\n def test_that_replacing_colons_gives_non_colliding_names(self):\n # .. and update shapes\n op = caffe2_pb2.OperatorDef()\n op.name = 'foo:0'\n op.input.extend(['foo:0', 'foo$0'])\n shapes = {'foo:0': [1]}\n blob_name_tracker = tb._get_blob_names([op])\n tb._replace_colons(shapes, blob_name_tracker, [op], '$')\n self.assertEqual(op.input[0], 'foo$0')\n self.assertEqual(op.input[1], 'foo$0_1')\n # Collision but blobs and op names are handled later by\n # _fill_missing_operator_names.\n self.assertEqual(op.name, 'foo$0')\n self.assertEqual(len(shapes), 1)\n self.assertEqual(shapes['foo$0'], [1])\n self.assertEqual(len(blob_name_tracker), 2)\n self.assertEqual(blob_name_tracker['foo$0'], 'foo:0')\n self.assertEqual(blob_name_tracker['foo$0_1'], 'foo$0')\n\n def test_that_adding_gradient_scope_does_no_fancy_renaming(self):\n # because it cannot create collisions\n op = caffe2_pb2.OperatorDef()\n op.name = 'foo_grad'\n op.input.extend(['foo_grad', 'foo_grad_1'])\n shapes = {'foo_grad': [1]}\n blob_name_tracker = tb._get_blob_names([op])\n tb._add_gradient_scope(shapes, blob_name_tracker, [op])\n self.assertEqual(op.input[0], 'GRADIENTS/foo_grad')\n self.assertEqual(op.input[1], 'GRADIENTS/foo_grad_1')\n self.assertEqual(op.name, 'GRADIENTS/foo_grad')\n self.assertEqual(len(shapes), 1)\n self.assertEqual(shapes['GRADIENTS/foo_grad'], [1])\n self.assertEqual(len(blob_name_tracker), 2)\n self.assertEqual(\n blob_name_tracker['GRADIENTS/foo_grad'], 'foo_grad')\n self.assertEqual(\n blob_name_tracker['GRADIENTS/foo_grad_1'], 'foo_grad_1')\n\n def test_that_auto_ssa_gives_non_colliding_names(self):\n op1 = caffe2_pb2.OperatorDef()\n op1.output.extend(['foo'])\n op2 = caffe2_pb2.OperatorDef()\n op2.input.extend(['foo'])\n op2.output.extend(['foo'])\n op2.output.extend(['foo_1'])\n shapes = {'foo': [1], 'foo_1': [2]}\n blob_name_tracker = tb._get_blob_names([op1, op2])\n tb._convert_to_ssa(shapes, blob_name_tracker, [op1, op2])\n self.assertEqual(op1.output[0], 'foo')\n self.assertEqual(op2.input[0], 'foo')\n self.assertEqual(op2.output[0], 'foo_1')\n # Unfortunate name but we do not parse original `_` for now.\n self.assertEqual(op2.output[1], 'foo_1_1')\n self.assertEqual(len(shapes), 3)\n self.assertEqual(shapes['foo'], [1])\n self.assertEqual(shapes['foo_1'], [1])\n self.assertEqual(shapes['foo_1_1'], [2])\n self.assertEqual(len(blob_name_tracker), 3)\n self.assertEqual(blob_name_tracker['foo'], 'foo')\n self.assertEqual(blob_name_tracker['foo_1'], 'foo')\n self.assertEqual(blob_name_tracker['foo_1_1'], 'foo_1')\n\n def test_renaming_tensorflow_style(self):\n # Construct some dummy operators here\n # NOTE: '_w', '_bn', etc without the postfix '_' are only renamed when\n # they are at the very end of the name.\n # Test that '_w', '_w_' are renamed to '/weight', '/weight_', resp.\n op1 = caffe2_pb2.OperatorDef()\n op1.input.extend(['foo_w'])\n op1.output.extend(['foo_w_2'])\n # Test that '_bn', '_bn_' are renamed to '/batchnorm', '/batchnorm_',\n # respectively.\n op2 = caffe2_pb2.OperatorDef()\n op2.input.extend(['foo_bn'])\n op2.output.extend(['foo_bn_2'])\n # Test that '_b', '_b_', are renamed to '/bias', '/bias_', resp.\n op3 = caffe2_pb2.OperatorDef()\n op3.input.extend(['foo_b'])\n op3.output.extend(['foo_b_2'])\n # Test that '_s', '_s_', are renamed to '/scale', '/scale_', resp.\n op4 = caffe2_pb2.OperatorDef()\n op4.input.extend(['foo_s'])\n op4.output.extend(['foo_s_2'])\n # Test that '_sum', '_sum_', are renamed to '/sum', '/sum_', resp.\n op5 = caffe2_pb2.OperatorDef()\n op5.input.extend(['foo_sum'])\n op5.output.extend(['foo_sum_2'])\n # Test that '_branch', '_branch_', are renamed to '/branch', '/branch_',\n # respectively. Multiple inputs/outputs are also tested in this case.\n op6 = caffe2_pb2.OperatorDef()\n op6.input.extend(['foo_branch'])\n op6.input.extend(['test_branch_2'])\n op6.output.extend(['foo_branch_3'])\n op6.output.extend(['test_branch4'])\n shapes = {\n 'foo_w': [1], 'foo_w_2': [2], 'foo_bn': [3], 'foo_bn_2': [4],\n 'foo_b': [5], 'foo_b_2': [6], 'foo_s': [7], 'foo_s_2': [8],\n 'foo_sum': [9], 'foo_sum_2': [10], 'foo_branch': [11],\n 'test_branch_2': [12], 'foo_branch_3': [13], 'test_branch4': [14],\n }\n ops = [op1, op2, op3, op4, op5, op6]\n blob_name_tracker = tb._get_blob_names(ops)\n tb._rename_tensorflow_style(shapes, blob_name_tracker, ops)\n # Testing that keys in blob name tracker were renamed correctly\n self.assertEqual(blob_name_tracker['foo/weight'], 'foo_w')\n self.assertEqual(blob_name_tracker['foo/weight_2'], 'foo_w_2')\n self.assertEqual(blob_name_tracker['foo/batchnorm'], 'foo_bn')\n self.assertEqual(blob_name_tracker['foo/batchnorm_2'], 'foo_bn_2')\n self.assertEqual(blob_name_tracker['foo/bias'], 'foo_b')\n self.assertEqual(blob_name_tracker['foo/bias_2'], 'foo_b_2')\n self.assertEqual(blob_name_tracker['foo/scale'], 'foo_s')\n self.assertEqual(blob_name_tracker['foo/scale_2'], 'foo_s_2')\n self.assertEqual(blob_name_tracker['foo/sum'], 'foo_sum')\n self.assertEqual(blob_name_tracker['foo/sum_2'], 'foo_sum_2')\n self.assertEqual(blob_name_tracker['foo/branch'], 'foo_branch')\n self.assertEqual(blob_name_tracker['test/branch_2'], 'test_branch_2')\n self.assertEqual(blob_name_tracker['foo/branch_3'], 'foo_branch_3')\n self.assertEqual(blob_name_tracker['test/branch4'], 'test_branch4')\n # Testing that keys in shapes were renamed correctly\n self.assertEqual(shapes['foo/weight'], [1])\n self.assertEqual(shapes['foo/batchnorm_2'], [4])\n self.assertEqual(shapes['foo/sum'], [9])\n self.assertEqual(shapes['test/branch_2'], [12])\n # Testing that the ops were renamed correctly\n self.assertEqual(op1.input[0], 'foo/weight')\n self.assertEqual(op1.output[0], 'foo/weight_2')\n self.assertEqual(op2.input[0], 'foo/batchnorm')\n self.assertEqual(op2.output[0], 'foo/batchnorm_2')\n self.assertEqual(op3.input[0], 'foo/bias')\n self.assertEqual(op3.output[0], 'foo/bias_2')\n self.assertEqual(op4.input[0], 'foo/scale')\n self.assertEqual(op4.output[0], 'foo/scale_2')\n self.assertEqual(op5.input[0], 'foo/sum')\n self.assertEqual(op5.output[0], 'foo/sum_2')\n self.assertEqual(op6.input[0], 'foo/branch')\n self.assertEqual(op6.input[1], 'test/branch_2')\n self.assertEqual(op6.output[0], 'foo/branch_3')\n self.assertEqual(op6.output[1], 'test/branch4')\n\n def test_filter_ops(self):\n op1 = caffe2_pb2.OperatorDef()\n op1.input.extend(['remove_this'])\n op1.output.extend(['random_output'])\n op2 = caffe2_pb2.OperatorDef()\n op2.input.extend(['leave_this'])\n op2.output.extend(['leave_this_also'])\n op3 = caffe2_pb2.OperatorDef()\n op3.input.extend(['random_input'])\n op3.output.extend(['remove_this_also'])\n\n def filter_fn(blob):\n # Filter all blobs with names containing 'remove'\n return 'remove' not in str(blob)\n\n op_set1 = [op1, op2, op3]\n op_set2 = [op1, op2, op3]\n\n # Test case for when perform_filter = True.\n result_ops1 = tb._filter_ops(op_set1, filter_fn, True)\n new_op1, new_op2 = result_ops1[0], result_ops1[1]\n # input named 'remove_this' should have been filtered\n self.assertEqual(len(new_op1.input), 0)\n self.assertEqual(new_op1.output, ['random_output'])\n self.assertEqual(new_op2.input, ['leave_this'])\n self.assertEqual(new_op2.output, ['leave_this_also'])\n # output named 'remove_this_also' should have been filtered as well.\n # This should have also removed op3 as the filter function excludes ops\n # with no outputs.\n self.assertEqual(len(result_ops1), 2)\n\n # Test case for when perform_filter = False. op_set2 should remain\n # unchanged.\n result_ops2 = tb._filter_ops(op_set2, filter_fn, False)\n self.assertEqual(result_ops2, op_set2)\n\n # Use show_simplified=False. This shows the original style of graph\n # visualization from caffe2.contrib.tensorboard.\n # TODO: Add test for show_simplified=True.\n def test_simple_cnnmodel(self):\n model = cnn.CNNModelHelper(\"NCHW\", name=\"overfeat\")\n workspace.FeedBlob(\"data\", np.random.randn(1, 3, 64, 64).astype(np.float32))\n workspace.FeedBlob(\"label\", np.random.randn(1, 1000).astype(np.int))\n with core.NameScope(\"conv1\"):\n conv1 = model.Conv(\"data\", \"conv1\", 3, 96, 11, stride=4)\n relu1 = model.Relu(conv1, conv1)\n pool1 = model.MaxPool(relu1, \"pool1\", kernel=2, stride=2)\n with core.NameScope(\"classifier\"):\n fc = model.FC(pool1, \"fc\", 4096, 1000)\n pred = model.Softmax(fc, \"pred\")\n xent = model.LabelCrossEntropy([pred, \"label\"], \"xent\")\n loss = model.AveragedLoss(xent, \"loss\")\n\n blob_name_tracker = {}\n graph = tb.model_to_graph_def(\n model,\n blob_name_tracker=blob_name_tracker,\n shapes={},\n show_simplified=False,\n )\n\n compare_proto(graph, self)\n\n # cnn.CNNModelHelper is deprecated, so we also test with\n # model_helper.ModelHelper. The model used in this test is taken from the\n # Caffe2 MNIST tutorial. Also use show_simplified=False here.\n def test_simple_model(self):\n model = model_helper.ModelHelper(name=\"mnist\")\n # how come those inputs don't break the forward pass =.=a\n workspace.FeedBlob(\"data\", np.random.randn(1, 3, 64, 64).astype(np.float32))\n workspace.FeedBlob(\"label\", np.random.randn(1, 1000).astype(np.int))\n\n with core.NameScope(\"conv1\"):\n conv1 = brew.conv(model, \"data\", 'conv1', dim_in=1, dim_out=20, kernel=5)\n # Image size: 24 x 24 -> 12 x 12\n pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2)\n # Image size: 12 x 12 -> 8 x 8\n conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=100, kernel=5)\n # Image size: 8 x 8 -> 4 x 4\n pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2)\n with core.NameScope(\"classifier\"):\n # 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size\n fc3 = brew.fc(model, pool2, 'fc3', dim_in=100 * 4 * 4, dim_out=500)\n relu = brew.relu(model, fc3, fc3)\n pred = brew.fc(model, relu, 'pred', 500, 10)\n softmax = brew.softmax(model, pred, 'softmax')\n xent = model.LabelCrossEntropy([softmax, \"label\"], 'xent')\n # compute the expected loss\n loss = model.AveragedLoss(xent, \"loss\")\n model.net.RunAllOnMKL()\n model.param_init_net.RunAllOnMKL()\n model.AddGradientOperators([loss], skip=1)\n blob_name_tracker = {}\n graph = tb.model_to_graph_def(\n model,\n blob_name_tracker=blob_name_tracker,\n shapes={},\n show_simplified=False,\n )\n\n compare_proto(graph, self)\n\n\nif __name__ == \"__main__\":\n unittest.main()\n" }, { "path": "tensorboardX/tests/test_chainer_np.py", "content": "from __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\n\nfrom tensorboardX import x2num, SummaryWriter\ntry:\n import chainer\n chainer_installed = True\nexcept ImportError:\n print('Chainer is not installed, skipping test')\n chainer_installed = False\nimport numpy as np\nimport unittest\n\n\nif chainer_installed:\n chainer.Variable\n tensors = [chainer.Variable(np.random.rand(3, 10, 10)),\n chainer.Variable(np.random.rand(1)),\n chainer.Variable(np.random.rand(1, 2, 3, 4, 5))]\n\n class ChainerTest(unittest.TestCase):\n def test_chainer_np(self):\n for tensor in tensors:\n # regular variable\n assert isinstance(x2num.make_np(tensor), np.ndarray)\n\n # python primitive type\n assert(isinstance(x2num.make_np(0), np.ndarray))\n assert(isinstance(x2num.make_np(0.1), np.ndarray))\n\n def test_chainer_img(self):\n shapes = [(77, 3, 13, 7), (77, 1, 13, 7), (3, 13, 7), (1, 13, 7), (13, 7)]\n for s in shapes:\n x = chainer.Variable(np.random.random_sample(s))\n # assert x2num.make_np(x, 'IMG').shape[2] == 3\n\n def test_chainer_write(self):\n with SummaryWriter() as w:\n w.add_scalar('scalar', chainer.Variable(np.random.rand(1)), 0)\n" }, { "path": "tensorboardX/tests/test_crc32c.py", "content": "import unittest\nfrom tensorboardX.crc32c import _crc32c, _crc32c_native, crc32c\n\n\nclass CRC32CTest(unittest.TestCase):\n def test_crc32c(self):\n data = b'abcd'\n assert crc32c(data) == 0x92c80a31\n\n def test_crc32c_python(self):\n data = b'abcd'\n assert _crc32c(data) == 0x92c80a31\n\n def test_crc32c_native(self):\n if _crc32c_native is None:\n return\n data = b'abcd'\n assert _crc32c_native(data) == 0x92c80a31\n" }, { "path": "tensorboardX/tests/test_embedding.py", "content": "import unittest\nimport torch\nfrom tensorboardX import SummaryWriter\n\n\nclass EmbeddingTest(unittest.TestCase):\n def test_embedding(self):\n w = SummaryWriter()\n all_features = torch.Tensor([[1, 2, 3], [5, 4, 1], [3, 7, 7]])\n all_labels = torch.Tensor([33, 44, 55])\n all_images = torch.zeros(3, 3, 5, 5)\n\n w.add_embedding(all_features,\n metadata=all_labels,\n label_img=all_images,\n global_step=2)\n\n dataset_label = ['test'] * 2 + ['train'] * 2\n all_labels = list(zip(all_labels, dataset_label))\n w.add_embedding(all_features,\n metadata=all_labels,\n label_img=all_images,\n metadata_header=['digit', 'dataset'],\n global_step=2)\n # assert...\n\n def test_embedding_64(self):\n w = SummaryWriter()\n all_features = torch.Tensor([[1, 2, 3], [5, 4, 1], [3, 7, 7]])\n all_labels = torch.Tensor([33, 44, 55])\n all_images = torch.zeros((3, 3, 5, 5), dtype=torch.float64)\n\n w.add_embedding(all_features,\n metadata=all_labels,\n label_img=all_images,\n global_step=2)\n\n dataset_label = ['test'] * 2 + ['train'] * 2\n all_labels = list(zip(all_labels, dataset_label))\n w.add_embedding(all_features,\n metadata=all_labels,\n label_img=all_images,\n metadata_header=['digit', 'dataset'],\n global_step=2)\n\n def test_embedding_square(self):\n w = SummaryWriter(comment='sq')\n all_features = torch.rand(228,256)\n all_images = torch.rand(228, 3, 32, 32)\n for i in range(all_images.shape[0]):\n all_images[i] *= (float(i)+60)/(all_images.shape[0]+60)\n w.add_embedding(all_features,\n label_img=all_images,\n global_step=2)\n\n def test_embedding_fail(self):\n with self.assertRaises(AssertionError):\n w = SummaryWriter(comment='shouldfail')\n all_features = torch.rand(228,256)\n all_images = torch.rand(228, 3, 16, 32)\n for i in range(all_images.shape[0]):\n all_images[i] *= (float(i)+60)/(all_images.shape[0]+60)\n w.add_embedding(all_features,\n label_img=all_images,\n global_step=2)\n" }, { "path": "tensorboardX/tests/test_figure.py", "content": "from __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\n\nimport matplotlib.pyplot as plt\nimport unittest\n\nfrom tensorboardX import SummaryWriter\n\n\nclass FigureTest(unittest.TestCase):\n def test_figure(self):\n writer = SummaryWriter()\n\n figure, axes = plt.figure(), plt.gca()\n circle1 = plt.Circle((0.2, 0.5), 0.2, color='r')\n circle2 = plt.Circle((0.8, 0.5), 0.2, color='g')\n axes.add_patch(circle1)\n axes.add_patch(circle2)\n plt.axis('scaled')\n plt.tight_layout()\n\n writer.add_figure(\"add_figure/figure\", figure, 0, close=False)\n assert plt.fignum_exists(figure.number) is True\n\n writer.add_figure(\"add_figure/figure\", figure, 1)\n assert plt.fignum_exists(figure.number) is False\n\n writer.close()\n\n def test_figure_list(self):\n writer = SummaryWriter()\n\n figures = []\n for i in range(5):\n figure = plt.figure()\n plt.plot([i * 1, i * 2, i * 3], label=\"Plot \" + str(i))\n plt.xlabel(\"X\")\n plt.xlabel(\"Y\")\n plt.legend()\n plt.tight_layout()\n figures.append(figure)\n\n writer.add_figure(\"add_figure/figure_list\", figures, 0, close=False)\n assert all([plt.fignum_exists(figure.number) is True for figure in figures])\n\n writer.add_figure(\"add_figure/figure_list\", figures, 1)\n assert all([plt.fignum_exists(figure.number) is False for figure in figures])\n\n writer.close()\n" }, { "path": "tensorboardX/tests/test_numpy.py", "content": "from __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\n\nimport numpy as np\nimport unittest\n\nfrom tensorboardX import x2num\n\n\nclass NumpyTest(unittest.TestCase):\n def test_scalar(self):\n res = x2num.make_np(1.1)\n assert isinstance(res, np.ndarray) and res.shape == (1,)\n res = x2num.make_np(1 << 64 - 1) # uint64_max\n assert isinstance(res, np.ndarray) and res.shape == (1,)\n res = x2num.make_np(np.float16(1.00000087))\n assert isinstance(res, np.ndarray) and res.shape == (1,)\n res = x2num.make_np(np.float128(1.00008 + 9))\n assert isinstance(res, np.ndarray) and res.shape == (1,)\n res = x2num.make_np(np.int64(100000000000))\n assert isinstance(res, np.ndarray) and res.shape == (1,)\n\n def test_make_grid(self):\n pass\n\n def test_numpy_vid(self):\n shapes = [(16, 3, 30, 28, 28), (19, 3, 30, 28, 28), (19, 3, 29, 23, 19)]\n for s in shapes:\n x = np.random.random_sample(s)\n # assert x2num.make_np(x, 'VID').shape[3] == 3\n\n def test_numpy_vid_uint8(self):\n x = np.random.randint(0, 256, (16, 3, 30, 28, 28)).astype(np.uint8)\n # x2num.make_np(x, 'VID').shape[3] == 3\n" }, { "path": "tensorboardX/tests/test_onnx_graph.py", "content": "import unittest\nimport torch\nfrom tensorboardX import SummaryWriter\n\n\nclass ONNXGraphTest(unittest.TestCase):\n def test_onnx_graph(self):\n import subprocess\n zoo_address = 'https://onnxzoo.blob.core.windows.net/models/opset_8/mnist/mnist.tar.gz'\n\n res = subprocess.call(['wget', '-nc', zoo_address])\n assert res == 0, 'cannot download example onnx model from the zoo'\n res = subprocess.call(['tar', 'xf', 'mnist.tar.gz', '-C', 'examples/', 'mnist/model.onnx'])\n\n with SummaryWriter() as w:\n w.add_onnx_graph('examples/mnist/model.onnx')\n" }, { "path": "tensorboardX/tests/test_pr_curve.py", "content": "import unittest\nimport torch\nimport numpy as np\nfrom tensorboardX import SummaryWriter\nfrom tensorboardX import summary\nfrom .expect_reader import compare_proto\n\nnp.random.seed(0)\ntrue_positive_counts = [75, 64, 21, 5, 0]\nfalse_positive_counts = [150, 105, 18, 0, 0]\ntrue_negative_counts = [0, 45, 132, 150, 150]\nfalse_negative_counts = [0, 11, 54, 70, 75]\nprecision = [0.3333333, 0.3786982, 0.5384616, 1.0, 0.0]\nrecall = [1.0, 0.8533334, 0.28, 0.0666667, 0.0]\n\n\nclass PRCurveTest(unittest.TestCase):\n def test_smoke(self):\n with SummaryWriter() as writer:\n writer.add_pr_curve('xoxo', np.random.randint(2, size=100), np.random.rand(\n 100), 1)\n writer.add_pr_curve_raw('prcurve with raw data',\n true_positive_counts,\n false_positive_counts,\n true_negative_counts,\n false_negative_counts,\n precision,\n recall,\n 1)\n\n def test_pr_purve(self):\n random_labels = np.array([0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1,\n 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0,\n 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1,\n 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0,\n 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0])\n random_probs = np.array([0.33327776, 0.30032885, 0.79012837, 0.04306813, 0.65221544,\n 0.58481968, 0.28305522, 0.53795795, 0.00729739, 0.52266951,\n 0.22464247, 0.11262435, 0.41573075, 0.92493992, 0.73066758,\n 0.43867735, 0.27955449, 0.56975382, 0.53933028, 0.34392824,\n 0.30312509, 0.81732807, 0.55408544, 0.3969487 , 0.31768033,\n 0.24353266, 0.47198005, 0.19999122, 0.05788022, 0.24046305,\n 0.04651082, 0.30061738, 0.78321545, 0.82670207, 0.49200517,\n 0.80904619, 0.96711993, 0.3160946 , 0.01049424, 0.60108337,\n 0.56508792, 0.83729429, 0.9717386 , 0.46306053, 0.80232138,\n 0.24166823, 0.7393237 , 0.50820418, 0.04944932, 0.53854157,\n 0.10765172, 0.84723855, 0.20518299, 0.3143431 , 0.51299074,\n 0.47065695, 0.54267833, 0.1812676 , 0.06265177, 0.34110327,\n 0.30915171, 0.91870169, 0.91309447, 0.31395817, 0.36780571,\n 0.98297986, 0.00594547, 0.52839042, 0.70229202, 0.37779588,\n 0.15207045, 0.59759632, 0.72397032, 0.71502195, 0.90135725,\n 0.43970107, 0.17123532, 0.08785938, 0.04986818, 0.62702444,\n 0.69171023, 0.30537792, 0.30285433, 0.27124347, 0.27693729,\n 0.7136039 , 0.48022489, 0.20916285, 0.2018599 , 0.92401008,\n 0.30189681, 0.46862626, 0.96353024, 0.30468533, 0.68281294,\n 0.30623562, 0.40795975, 0.76824531, 0.89824215, 0.69845035], dtype=np.float16)\n compare_proto(summary.pr_curve('tag', random_labels, random_probs, 1), self)\n\n def test_pr_purve_raw(self):\n compare_proto(summary.pr_curve_raw('prcurve with raw data',\n true_positive_counts,\n false_positive_counts,\n true_negative_counts,\n false_negative_counts,\n precision,\n recall,\n 1),\n self)\n" }, { "path": "tensorboardX/tests/test_pytorch_graph.py", "content": "from __future__ import absolute_import, division, print_function, unicode_literals\nimport unittest\nimport torch\nfrom tensorboardX import SummaryWriter\n\n\nclass PytorchGraphTest(unittest.TestCase):\n def test_pytorch_graph(self):\n dummy_input = (torch.zeros(1, 3),)\n\n class myLinear(torch.nn.Module):\n def __init__(self):\n super(myLinear, self).__init__()\n self.linear = torch.nn.Linear(3, 5)\n\n def forward(self, x):\n return self.linear(x)\n\n with SummaryWriter(comment='LinearModel') as w:\n w.add_graph(myLinear(), dummy_input, True)\n\n def test_wrong_input_size(self):\n print('expect error here:')\n with self.assertRaises(RuntimeError):\n dummy_input = torch.rand(1, 9)\n model = torch.nn.Linear(3, 5)\n with SummaryWriter(comment='expect_error') as w:\n w.add_graph(model, dummy_input) # error\n" }, { "path": "tensorboardX/tests/test_pytorch_np.py", "content": "from __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\n\nfrom tensorboardX import x2num, SummaryWriter\nimport torch\nimport numpy as np\nimport unittest\n\n\nclass PyTorchNumpyTest(unittest.TestCase):\n def test_pytorch_np(self):\n tensors = [torch.rand(3, 10, 10), torch.rand(1), torch.rand(1, 2, 3, 4, 5)]\n for tensor in tensors:\n # regular tensor\n assert isinstance(x2num.make_np(tensor), np.ndarray)\n\n # CUDA tensor\n if torch.cuda.device_count() > 0:\n assert isinstance(x2num.make_np(tensor.cuda()), np.ndarray)\n\n # regular variable\n assert isinstance(x2num.make_np(torch.autograd.Variable(tensor)), np.ndarray)\n\n # CUDA variable\n if torch.cuda.device_count() > 0:\n assert isinstance(x2num.make_np(torch.autograd.Variable(tensor).cuda()), np.ndarray)\n\n # python primitive type\n assert(isinstance(x2num.make_np(0), np.ndarray))\n assert(isinstance(x2num.make_np(0.1), np.ndarray))\n\n def test_pytorch_write(self):\n with SummaryWriter() as w:\n w.add_scalar('scalar', torch.autograd.Variable(torch.rand(1)), 0)\n\n def test_pytorch_histogram(self):\n with SummaryWriter() as w:\n w.add_histogram('float histogram', torch.rand((50,)))\n w.add_histogram('int histogram', torch.randint(0, 100, (50,)))\n\n def test_pytorch_histogram_raw(self):\n with SummaryWriter() as w:\n num = 50\n floats = x2num.make_np(torch.rand((num,)))\n bins = [0.0, 0.25, 0.5, 0.75, 1.0]\n counts, limits = np.histogram(floats, bins)\n sum_sq = floats.dot(floats).item()\n w.add_histogram_raw('float histogram raw',\n min=floats.min().item(),\n max=floats.max().item(),\n num=num,\n sum=floats.sum().item(),\n sum_squares=sum_sq,\n bucket_limits=limits[1:].tolist(),\n bucket_counts=counts.tolist())\n\n ints = x2num.make_np(torch.randint(0, 100, (num,)))\n bins = [0, 25, 50, 75, 100]\n counts, limits = np.histogram(ints, bins)\n sum_sq = ints.dot(ints).item()\n w.add_histogram_raw('int histogram raw',\n min=ints.min().item(),\n max=ints.max().item(),\n num=num,\n sum=ints.sum().item(),\n sum_squares=sum_sq,\n bucket_limits=limits[1:].tolist(),\n bucket_counts=counts.tolist())\n" }, { "path": "tensorboardX/tests/test_record_writer.py", "content": "from tensorboardX import SummaryWriter\nimport unittest\nfrom tensorboardX.record_writer import S3RecordWriter, make_valid_tf_name\nimport os\nimport boto3\nfrom moto import mock_s3\n\nos.environ.setdefault(\"AWS_ACCESS_KEY_ID\", \"foobar_key\")\nos.environ.setdefault(\"AWS_SECRET_ACCESS_KEY\", \"foobar_secret\")\n\n\nclass RecordWriterTest(unittest.TestCase):\n @mock_s3\n def test_record_writer_s3(self):\n client = boto3.client('s3', region_name='us-east-1')\n client.create_bucket(Bucket='this')\n writer = S3RecordWriter('s3://this/is/apen')\n bucket, path = writer.bucket_and_path()\n assert bucket == 'this'\n assert path == 'is/apen'\n writer.write(bytes(42))\n writer.flush()\n\n def test_make_valid_tf_name(self):\n newname = make_valid_tf_name('$ave/&sound')\n assert newname == '._ave/_sound'\n" }, { "path": "tensorboardX/tests/test_summary.py", "content": "from __future__ import absolute_import, division, print_function, unicode_literals\nfrom tensorboardX import summary\nfrom .expect_reader import compare_proto, write_proto\nimport numpy as np\nimport pytest\nimport unittest\n# compare_proto = write_proto # massive update expect\n\ndef tensor_N(shape, dtype=float):\n numel = np.prod(shape)\n x = (np.arange(numel, dtype=dtype)).reshape(shape)\n return x\n\nclass SummaryTest(unittest.TestCase):\n def test_uint8_image(self):\n '''\n Tests that uint8 image (pixel values in [0, 255]) is not changed\n '''\n test_image = tensor_N(shape=(3, 32, 32), dtype=np.uint8)\n compare_proto(summary.image('dummy', test_image), self)\n\n def test_float32_image(self):\n '''\n Tests that float32 image (pixel values in [0, 1]) are scaled correctly\n to [0, 255]\n '''\n test_image = tensor_N(shape=(3, 32, 32))\n compare_proto(summary.image('dummy', test_image), self)\n\n def test_float_1_converts_to_uint8_255(self):\n green_uint8 = np.array([[[0, 255, 0]]], dtype='uint8') \n green_float32 = np.array([[[0, 1, 0]]], dtype='float32') \n\n a = summary.image(tensor=green_uint8, tag='')\n b = summary.image(tensor=green_float32, tag='')\n self.assertEqual(a, b)\n\n def test_list_input(self):\n with pytest.raises(Exception):\n summary.histogram('dummy', [1, 3, 4, 5, 6], 'tensorflow')\n\n def test_empty_input(self):\n print('expect error here:')\n with pytest.raises(Exception):\n summary.histogram('dummy', np.ndarray(0), 'tensorflow')\n\n def test_image_with_boxes(self):\n compare_proto(summary.image_boxes('dummy',\n tensor_N(shape=(3, 32, 32)),\n np.array([[10, 10, 40, 40]])), self)\n\n def test_image_with_one_channel(self):\n compare_proto(summary.image('dummy', tensor_N(shape=(1, 8, 8)), dataformats='CHW'), self)\n\n def test_image_with_four_channel(self):\n compare_proto(summary.image('dummy', tensor_N(shape=(4, 8, 8)), dataformats='CHW'), self)\n\n def test_image_with_one_channel_batched(self):\n compare_proto(summary.image('dummy', tensor_N(shape=(2, 1, 8, 8)), dataformats='NCHW'), self)\n\n def test_image_with_3_channel_batched(self):\n compare_proto(summary.image('dummy', tensor_N(shape=(2, 3, 8, 8)), dataformats='NCHW'), self)\n\n def test_image_with_four_channel_batched(self):\n compare_proto(summary.image('dummy', tensor_N(shape=(2, 4, 8, 8)), dataformats='NCHW'), self)\n\n def test_image_without_channel(self):\n compare_proto(summary.image('dummy', tensor_N(shape=(8, 8)), dataformats='HW'), self)\n\n def test_video(self):\n try:\n import moviepy\n except ImportError:\n return\n compare_proto(summary.video('dummy', tensor_N(shape=(4, 3, 1, 8, 8))), self)\n summary.video('dummy', tensor_N(shape=(16, 48, 1, 28, 28)))\n summary.video('dummy', tensor_N(shape=(20, 7, 1, 8, 8)))\n\n def test_audio(self):\n compare_proto(summary.audio('dummy', tensor_N(shape=(42,))), self)\n\n def test_text(self):\n compare_proto(summary.text('dummy', 'text 123'), self)\n\n def test_histogram_auto(self):\n compare_proto(summary.histogram('dummy', tensor_N(shape=(1024,)), bins='auto', max_bins=5), self)\n\n def test_histogram_fd(self):\n compare_proto(summary.histogram('dummy', tensor_N(shape=(1024,)), bins='fd', max_bins=5), self)\n\n def test_histogram_doane(self):\n compare_proto(summary.histogram('dummy', tensor_N(shape=(1024,)), bins='doane', max_bins=5), self)\n\n def test_custom_scalars(self):\n layout = {'Taiwan': {'twse': ['Multiline', ['twse/0050', 'twse/2330']]},\n 'USA': {'dow': ['Margin', ['dow/aaa', 'dow/bbb', 'dow/ccc']],\n 'nasdaq': ['Margin', ['nasdaq/aaa', 'nasdaq/bbb', 'nasdaq/ccc']]}}\n summary.custom_scalars(layout) # smoke test only.\n\n def test_mesh(self):\n vertices_tensor = np.array([[\n [1, 1, 1],\n [-1, -1, 1],\n [1, -1, -1],\n [-1, 1, -1],\n ]], dtype=float)\n colors_tensor = np.array([[\n [255, 0, 0],\n [0, 255, 0],\n [0, 0, 255],\n [255, 0, 255],\n ]], dtype=int)\n faces_tensor = np.array([[\n [0, 2, 3],\n [0, 3, 1],\n [0, 1, 2],\n [1, 3, 2],\n ]], dtype=int)\n compare_proto(summary.mesh('my_mesh', vertices=vertices_tensor, colors=colors_tensor, faces=faces_tensor), self)\n\n # It's hard to get dictionary sorted with same result in various envs. So only use one.\n def test_hparams(self):\n hp = {'lr': 0.1}\n mt = {'accuracy': 0.1}\n compare_proto(summary.hparams(hp, mt), self)\n\n def test_hparams_smoke(self):\n hp = {'lr': 0.1, 'bsize': 4}\n mt = {'accuracy': 0.1, 'loss': 10}\n summary.hparams(hp, mt)\n \n hp = {'string': \"1b\", 'use magic': True}\n summary.hparams(hp, mt)\n" }, { "path": "tensorboardX/tests/test_summary_writer.py", "content": "from tensorboardX import SummaryWriter\nimport unittest\n\n\nclass SummaryWriterTest(unittest.TestCase):\n def test_summary_writer_ctx(self):\n # after using a SummaryWriter as a ctx it should be closed\n with SummaryWriter(filename_suffix='.test') as writer:\n writer.add_scalar('test', 1)\n assert writer.file_writer is None\n\n def test_summary_writer_backcomapt(self):\n with SummaryWriter(log_dir='/tmp/tbxtest') as writer:\n writer.add_scalar('test', 1)\n\n def test_summary_writer_close(self):\n # Opening and closing SummaryWriter a lot should not run into\n # OSError: [Errno 24] Too many open files\n passed = True\n try:\n writer = SummaryWriter()\n writer.close()\n except OSError:\n passed = False\n\n assert passed\n\n def test_windowsPath(self):\n dummyPath = \"C:\\\\Downloads\\\\fjoweifj02utj43tj430\"\n with SummaryWriter(dummyPath) as writer:\n writer.add_scalar('test', 1)\n import shutil\n shutil.rmtree(dummyPath)\n\n def test_pathlib(self):\n import sys\n if sys.version_info.major == 2:\n import pathlib2 as pathlib\n else:\n import pathlib\n p = pathlib.Path('./pathlibtest')\n with SummaryWriter(p) as writer:\n writer.add_scalar('test', 1)\n import shutil\n shutil.rmtree(str(p))\n" }, { "path": "tensorboardX/tests/test_test.py", "content": "def test_linting():\n import subprocess\n # subprocess.check_output(['flake8', 'tensorboardX'])\n" }, { "path": "tensorboardX/tests/test_utils.py", "content": "from tensorboardX import summary\nfrom tensorboardX.utils import make_grid, _prepare_video, convert_to_HWC\nimport numpy as np\nimport pytest\nimport unittest\n\n\nclass UtilsTest(unittest.TestCase):\n def test_to_HWC(self):\n np.random.seed(1)\n test_image = np.random.randint(0, 256, size=(3, 32, 32), dtype=np.uint8)\n converted = convert_to_HWC(test_image, 'chw')\n assert converted.shape == (32, 32, 3)\n test_image = np.random.randint(0, 256, size=(16, 3, 32, 32), dtype=np.uint8)\n converted = convert_to_HWC(test_image, 'nchw')\n assert converted.shape == (64, 256, 3)\n test_image = np.random.randint(0, 256, size=(32, 32), dtype=np.uint8)\n converted = convert_to_HWC(test_image, 'hw')\n assert converted.shape == (32, 32, 3)\n\n def test_prepare_video(self):\n # at each timestep the sum over all other dimensions of the video should stay the same\n np.random.seed(1)\n V_before = np.random.random((4, 10, 3, 20, 20))\n V_after = _prepare_video(np.copy(V_before))\n V_before = np.swapaxes(V_before, 0, 1)\n V_before = np.reshape(V_before, newshape=(10, -1))\n V_after = np.reshape(V_after, newshape=(10, -1))\n np.testing.assert_array_almost_equal(np.sum(V_before, axis=1), np.sum(V_after, axis=1))\n" }, { "path": "tensorboardX/tests/test_visdom.py", "content": "from tensorboardX import TorchVis\n\nimport numpy as np\nimport pytest\nimport unittest\n\ntrue_positive_counts = [75, 64, 21, 5, 0]\nfalse_positive_counts = [150, 105, 18, 0, 0]\ntrue_negative_counts = [0, 45, 132, 150, 150]\nfalse_negative_counts = [0, 11, 54, 70, 75]\nprecision = [0.3333333, 0.3786982, 0.5384616, 1.0, 0.0]\nrecall = [1.0, 0.8533334, 0.28, 0.0666667, 0.0]\n\n\nclass VisdomTest(unittest.TestCase):\n def test_TorchVis(self):\n w = TorchVis('visdom')\n w.add_scalar('scalar_visdom', 1, 0)\n w.add_scalar('scalar_visdom', 2, 1)\n w.add_histogram('histogram_visdom', np.array([1, 2, 3, 4, 5]), 1)\n w.add_image('image_visdom', np.ndarray((3, 20, 20)), 2)\n # w.add_video('video_visdom', np.ndarray((1, 3, 10, 20, 20)), 3)\n w.add_audio('audio_visdom', [1, 2, 3, 4, 5])\n w.add_text('text_visdom', 'mystring')\n w.add_pr_curve('pr_curve_visdom', np.random.randint(2, size=100), np.random.rand(100), 10)\n w.add_pr_curve_raw('prcurve with raw data',\n true_positive_counts,\n false_positive_counts,\n true_negative_counts,\n false_negative_counts,\n precision,\n recall, 20)\n del w\n" }, { "path": "tensorboardX/tests/test_writer.py", "content": "from tensorboardX import SummaryWriter\nfrom tensorboard.compat.tensorflow_stub.pywrap_tensorflow import PyRecordReader_New\nfrom tensorboardX.proto import event_pb2\n\nimport numpy as np\nimport pytest\nimport unittest\nimport time\nfreqs = [262, 294, 330, 349, 392, 440, 440, 440, 440, 440, 440]\n\ntrue_positive_counts = [75, 64, 21, 5, 0]\nfalse_positive_counts = [150, 105, 18, 0, 0]\ntrue_negative_counts = [0, 45, 132, 150, 150]\nfalse_negative_counts = [0, 11, 54, 70, 75]\nprecision = [0.3333333, 0.3786982, 0.5384616, 1.0, 0.0]\nrecall = [1.0, 0.8533334, 0.28, 0.0666667, 0.0]\n\n\nclass WriterTest(unittest.TestCase):\n def test_flush(self):\n N_TEST = 5\n w = SummaryWriter(flush_secs=1)\n f = w.file_writer.event_writer._ev_writer._file_name\n for i in range(N_TEST):\n w.add_scalar('a', i)\n time.sleep(2)\n r = PyRecordReader_New(f)\n r.GetNext() # meta data, so skip\n for _ in range(N_TEST): # all of the data should be flushed\n r.GetNext()\n\n def test_flush_timer_is_long_so_data_is_not_there(self):\n with self.assertRaises(BaseException):\n N_TEST = 5\n w = SummaryWriter(flush_secs=20)\n f = w.file_writer.event_writer._ev_writer._file_name\n for i in range(N_TEST):\n w.add_scalar('a', i)\n time.sleep(2)\n r = PyRecordReader_New(f)\n r.GetNext() # meta data, so skip\n for _ in range(N_TEST): # missing data\n r.GetNext()\n\n def test_flush_after_close(self):\n N_TEST = 5\n w = SummaryWriter(flush_secs=20)\n f = w.file_writer.event_writer._ev_writer._file_name\n for i in range(N_TEST):\n w.add_scalar('a', i)\n time.sleep(2)\n w.close()\n r = PyRecordReader_New(f)\n r.GetNext() # meta data, so skip\n for _ in range(N_TEST): # all of the data should be flushed\n r.GetNext()\n\n def test_flush(self):\n N_TEST = 5\n w = SummaryWriter(flush_secs=20)\n f = w.file_writer.event_writer._ev_writer._file_name\n for i in range(N_TEST):\n w.add_scalar('a', i)\n time.sleep(2)\n w.flush()\n r = PyRecordReader_New(f)\n r.GetNext() # meta data, so skip\n for _ in range(N_TEST): # all of the data should be flushed\n r.GetNext()\n\n def test_auto_close(self):\n pass\n\n def test_writer(self):\n with SummaryWriter() as writer:\n sample_rate = 44100\n\n n_iter = 0\n writer.add_scalar('data/scalar_systemtime', 0.1, n_iter)\n writer.add_scalar('data/scalar_customtime', 0.2, n_iter, walltime=n_iter)\n writer.add_scalars('data/scalar_group', {\"xsinx\": n_iter * np.sin(n_iter),\n \"xcosx\": n_iter * np.cos(n_iter),\n \"arctanx\": np.arctan(n_iter)}, n_iter)\n x = np.zeros((32, 3, 64, 64)) # output from network\n writer.add_images('Image', x, n_iter) # Tensor\n writer.add_image_with_boxes('imagebox',\n np.zeros((3, 64, 64)),\n np.array([[10, 10, 40, 40], [40, 40, 60, 60]]),\n n_iter)\n x = np.zeros(sample_rate * 2)\n\n writer.add_audio('myAudio', x, n_iter)\n writer.add_video('myVideo', np.random.rand(16, 48, 1, 28, 28).astype(np.float32), n_iter)\n writer.add_text('Text', 'text logged at step:' + str(n_iter), n_iter)\n writer.add_text('markdown Text', '''a|b\\n-|-\\nc|d''', n_iter)\n writer.add_histogram('hist', np.random.rand(100, 100), n_iter)\n writer.add_pr_curve('xoxo', np.random.randint(2, size=100), np.random.rand(\n 100), n_iter) # needs tensorboard 0.4RC or later\n writer.add_pr_curve_raw('prcurve with raw data', true_positive_counts,\n false_positive_counts,\n true_negative_counts,\n false_negative_counts,\n precision,\n recall, n_iter)\n # export scalar data to JSON for external processing\n writer.export_scalars_to_json(\"./all_scalars.json\")\n imgs = []\n for i in range(5):\n imgs.append(np.ones((3, 100, 110)))\n with SummaryWriter() as w:\n w.add_images('img_list', imgs, dataformats='CHW')" } ]