Repository: pytorch/examples Branch: main Commit: acc295dc7b90 Files: 223 Total size: 884.1 KB Directory structure: gitextract_f9lneout/ ├── .github/ │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_template.md │ │ ├── doc_template.md │ │ └── feature_template.md │ ├── PULL_REQUEST_TEMPLATE/ │ │ └── pull_request_template.md │ └── workflows/ │ ├── doc-build.yml │ ├── main_cpp.yml │ ├── main_distributed.yaml │ └── main_python.yml ├── .gitignore ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── cpp/ │ ├── .clang-format │ ├── autograd/ │ │ ├── CMakeLists.txt │ │ ├── README.md │ │ └── autograd.cpp │ ├── custom-dataset/ │ │ ├── CMakeLists.txt │ │ ├── README.md │ │ ├── custom-dataset.cpp │ │ └── info.txt │ ├── dcgan/ │ │ ├── CMakeLists.txt │ │ ├── README.md │ │ ├── dcgan.cpp │ │ └── display_samples.py │ ├── distributed/ │ │ ├── CMakeLists.txt │ │ ├── README.md │ │ └── dist-mnist.cpp │ ├── mnist/ │ │ ├── CMakeLists.txt │ │ ├── README.md │ │ └── mnist.cpp │ ├── regression/ │ │ ├── CMakeLists.txt │ │ ├── README.md │ │ └── regression.cpp │ ├── tools/ │ │ ├── InstallingOpenCV.md │ │ └── download_mnist.py │ └── transfer-learning/ │ ├── CMakeLists.txt │ ├── README.md │ ├── classify.cpp │ ├── convert.py │ ├── main.cpp │ └── main.h ├── dcgan/ │ ├── README.md │ ├── main.py │ └── requirements.txt ├── distributed/ │ ├── FSDP/ │ │ ├── .gitignore │ │ ├── README.md │ │ ├── T5_training.py │ │ ├── configs/ │ │ │ ├── __init__.py │ │ │ ├── fsdp.py │ │ │ └── training.py │ │ ├── download_dataset.sh │ │ ├── model_checkpointing/ │ │ │ ├── __init__.py │ │ │ └── checkpoint_handler.py │ │ ├── policies/ │ │ │ ├── __init__.py │ │ │ ├── activation_checkpointing_functions.py │ │ │ ├── mixed_precision.py │ │ │ └── wrapping.py │ │ ├── requirements.txt │ │ ├── summarization_dataset.py │ │ └── utils/ │ │ ├── __init__.py │ │ ├── environment.py │ │ └── train_utils.py │ ├── FSDP2/ │ │ ├── README.md │ │ ├── checkpoint.py │ │ ├── example.py │ │ ├── model.py │ │ ├── requirements.txt │ │ ├── run_example.sh │ │ └── utils.py │ ├── ddp/ │ │ ├── README.md │ │ ├── example.py │ │ ├── requirements.txt │ │ └── run_example.sh │ ├── ddp-tutorial-series/ │ │ ├── README.md │ │ ├── datautils.py │ │ ├── multigpu.py │ │ ├── multigpu_torchrun.py │ │ ├── multinode.py │ │ ├── requirements.txt │ │ ├── single_gpu.py │ │ └── slurm/ │ │ ├── config.yaml.template │ │ ├── sbatch_run.sh │ │ └── setup_pcluster_slurm.md │ ├── minGPT-ddp/ │ │ ├── README.md │ │ ├── mingpt/ │ │ │ ├── char_dataset.py │ │ │ ├── gpt2_train_cfg.yaml │ │ │ ├── main.py │ │ │ ├── model.py │ │ │ ├── slurm/ │ │ │ │ ├── config.yaml.template │ │ │ │ ├── sbatch_run.sh │ │ │ │ └── setup_pcluster_slurm.md │ │ │ └── trainer.py │ │ ├── requirements.txt │ │ └── run_example.sh │ ├── rpc/ │ │ ├── batch/ │ │ │ ├── README.md │ │ │ ├── parameter_server.py │ │ │ ├── reinforce.py │ │ │ └── requirements.txt │ │ ├── ddp_rpc/ │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── requirements.txt │ │ ├── parameter_server/ │ │ │ ├── README.md │ │ │ ├── requirements.txt │ │ │ └── rpc_parameter_server.py │ │ ├── pipeline/ │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── requirements.txt │ │ ├── rl/ │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── requirements.txt │ │ └── rnn/ │ │ ├── README.md │ │ ├── main.py │ │ ├── requirements.txt │ │ └── rnn.py │ └── tensor_parallelism/ │ ├── README.md │ ├── fsdp_tp_example.py │ ├── llama2_model.py │ ├── log_utils.py │ ├── requirements.txt │ ├── run_example.sh │ ├── sequence_parallel_example.py │ └── tensor_parallel_example.py ├── docs/ │ ├── Makefile │ ├── make.bat │ ├── requirements.txt │ └── source/ │ ├── _static/ │ │ └── .gitkeep │ ├── conf.py │ └── index.rst ├── fast_neural_style/ │ ├── README.md │ ├── download_saved_models.py │ ├── neural_style/ │ │ ├── __init__.py │ │ ├── neural_style.py │ │ ├── transformer_net.py │ │ ├── utils.py │ │ └── vgg.py │ └── requirements.txt ├── fx/ │ ├── README.md │ ├── custom_tracer.py │ ├── inline_function.py │ ├── invert.py │ ├── module_tracer.py │ ├── native_interpreter/ │ │ ├── CMakeLists.txt │ │ ├── README.md │ │ ├── interpreter.cpp │ │ └── use_interpreter.py │ ├── primitive_library.py │ ├── profiling_tracer.py │ ├── proxy_based_graph_creation.py │ ├── replace_op.py │ ├── requirements.txt │ ├── subgraph_rewriter_basic_use.py │ └── wrap_output_dynamically.py ├── gat/ │ ├── README.md │ ├── main.py │ └── requirements.txt ├── gcn/ │ ├── README.md │ ├── main.py │ └── requirements.txt ├── imagenet/ │ ├── README.md │ ├── extract_ILSVRC.sh │ ├── main.py │ └── requirements.txt ├── language_translation/ │ ├── README.md │ ├── main.py │ ├── requirements.txt │ └── src/ │ ├── data.py │ └── model.py ├── legacy/ │ └── snli/ │ ├── README.md │ ├── model.py │ ├── requirements.txt │ ├── train.py │ └── util.py ├── mnist/ │ ├── README.md │ ├── main.py │ └── requirements.txt ├── mnist_forward_forward/ │ ├── README.md │ ├── main.py │ └── requirements.txt ├── mnist_hogwild/ │ ├── README.md │ ├── main.py │ ├── requirements.txt │ └── train.py ├── mnist_rnn/ │ ├── README.md │ ├── main.py │ └── requirements.txt ├── regression/ │ ├── README.md │ ├── main.py │ └── requirements.txt ├── reinforcement_learning/ │ ├── README.md │ ├── actor_critic.py │ ├── reinforce.py │ └── requirements.txt ├── run_cpp_examples.sh ├── run_distributed_examples.sh ├── run_python_examples.sh ├── runtime.txt ├── siamese_network/ │ ├── README.md │ ├── main.py │ └── requirements.txt ├── super_resolution/ │ ├── README.md │ ├── data.py │ ├── dataset.py │ ├── main.py │ ├── model.py │ ├── requirements.txt │ └── super_resolve.py ├── time_sequence_prediction/ │ ├── README.md │ ├── generate_sine_wave.py │ ├── requirements.txt │ └── train.py ├── utils.sh ├── vae/ │ ├── README.md │ ├── main.py │ ├── requirements.txt │ └── results/ │ └── .gitignore └── word_language_model/ ├── README.md ├── data.py ├── generate.py ├── main.py ├── model.py └── requirements.txt ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/ISSUE_TEMPLATE/bug_template.md ================================================ --- name: "\U0001F41B Bug report" about: Create a report to help us improve --- Your issue may already be reported! Please search on the [issue tracker](https://github.com/pytorch/examples/issues) before creating one. ## Context * Pytorch version: * Operating System and version: ## Your Environment * Installed using source? [yes/no]: * Are you planning to deploy it using docker container? [yes/no]: * Is it a CPU or GPU environment?: * Which example are you using: * Link to code or data to repro [if any]: ## Expected Behavior ## Current Behavior ## Possible Solution ## Steps to Reproduce 1. 2. ... ## Failure Logs [if any] ================================================ FILE: .github/ISSUE_TEMPLATE/doc_template.md ================================================ --- name: "\U0001F4DA Documentation" about: Report a documentation related issue --- ## 📚 Documentation ## Is your feature request related to a problem? Please describe. ## Describe the solution ## Describe alternatives solution ================================================ FILE: .github/PULL_REQUEST_TEMPLATE/pull_request_template.md ================================================ --- name: "\U0001F41B Pull Request" about: Fix a bug or create new example --- ## Description Please include a summary of the newly proposed example or issue being fixed. Please also include relevant motivation, context. If this is a new example, how is your example different enough from the remaining examples in the repo. If this is a bug fix please link the issue you are fixing. Fixes #(issue) ## Type of change Please delete options that are not relevant. - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New Example (new example contribution) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] This change requires a documentation update ## Feature/Issue validation/testing Please describe the tests [UT/IT] that you ran to verify your changes and relevant result summary. If this is a bug fix please run `run_python_examples.sh` before and after your change locally to make sure it works and add the logs here. - [ ] Logs before change - [ ] Logs after change - Logs If this is a new example please add a corresponding test in `run_python_examples.sh` - [ ] Test Added ## Checklist: - [ ] Have you added tests that prove your fix is effective or that this example works? - [ ] Has code been commented, particularly in hard-to-understand areas? - [ ] Have you made corresponding changes to the documentation? ================================================ FILE: .github/workflows/doc-build.yml ================================================ name: Doc Build on: push: branches: - main workflow_dispatch: jobs: build_docs_job: runs-on: ubuntu-latest # Grant write permission here so that the doc can be pushed to gh-pages branch permissions: contents: write strategy: matrix: python-version: [3.9] steps: - name: Checkout uses: actions/checkout@v4 id: build - name: Build the docset run: | cd docs pip install -r requirements.txt make html - name: Get output time run: echo "The time was ${{ steps.build.outputs.time }}" - name: Deploy uses: JamesIves/github-pages-deploy-action@releases/v3 with: ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }} BRANCH: gh-pages # The branch the action should deploy to. FOLDER: ./docs/build/html # The folder the action should deploy. ================================================ FILE: .github/workflows/main_cpp.yml ================================================ name: Run CPP Examples on: push: branches: [ main ] pull_request: branches: [ main ] schedule: # Every day at 3:00am - cron: '0 3 * * *' jobs: test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - name: Set up Python 3.11 uses: actions/setup-python@v2 with: python-version: 3.11 - name: Install Cmake, Make, g++, MKL run: | sudo apt update && sudo apt upgrade sudo apt install cmake g++ make sudo apt-get -y install intel-mkl - name: Install OpenCV run: | sudo apt -y install libtbb-dev sudo apt install libopencv-dev - name: Install argparse run: | git clone https://github.com/p-ranav/argparse cd argparse mkdir build cd build cmake -DARGPARSE_BUILD_SAMPLES=off -DARGPARSE_BUILD_TESTS=off .. sudo make install # Alternatively, you can install OpenCV from source # - name: Install OpenCV from source # run: | # wget -O opencv.zip https://github.com/opencv/opencv/archive/4.x.zip # unzip opencv.zip # mkdir -p build && cd build # cmake ../opencv-4.x # cmake --build . # sudo make install - name: Run Cpp Tests run: | chmod +x ./run_cpp_examples.sh ./run_cpp_examples.sh "get_libtorch,run_all,clean" ================================================ FILE: .github/workflows/main_distributed.yaml ================================================ name: Run Distributed Examples on: push: branches: [ main ] pull_request: branches: [ main ] schedule: # Every day at 3:00am - cron: '0 3 * * *' jobs: test: runs-on: 4-core-ubuntu-gpu-t4 steps: - uses: actions/checkout@v4 - name: Set up Python 3.10 uses: actions/setup-python@v5 with: python-version: '3.10' - name: Install PyTorch uses: astral-sh/setup-uv@v6 - name: Run Tests env: USE_CUDA: 'True' VIRTUAL_ENV: '.venv' PIP_INSTALL_ARGS: '--pre -f https://download.pytorch.org/whl/nightly/cu118/torch_nightly.html' run: | ./run_distributed_examples.sh - name: Open issue on failure if: ${{ failure() && github.event_name == 'schedule' }} uses: rishabhgupta/git-action-issue@v2 with: token: ${{ secrets.GITHUB_TOKEN }} title: Daily CI failed body: Commit ${{ github.sha }} daily scheduled [CI run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) failed, please check why assignees: '' ================================================ FILE: .github/workflows/main_python.yml ================================================ name: Run Python Examples on: push: branches: [ main ] pull_request: branches: [ main ] schedule: # Every day at 3:00am - cron: '0 3 * * *' jobs: test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Python 3.10 uses: actions/setup-python@v5 with: python-version: '3.10' - name: Install uv uses: astral-sh/setup-uv@v6 - name: Run Tests env: VIRTUAL_ENV: '.venv' PIP_INSTAL_ARGS: '--pre -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html' run: | ./run_python_examples.sh - name: Open issue on failure if: ${{ failure() && github.event_name == 'schedule' }} uses: rishabhgupta/git-action-issue@v2 with: token: ${{ secrets.GITHUB_TOKEN }} title: Daily CI failed body: Commit ${{ github.sha }} daily scheduled [CI run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) failed, please check why assignees: '' ================================================ FILE: .gitignore ================================================ dcgan/data data *.pyc OpenNMT/data cpp/mnist/build cpp/dcgan/build dcgan/*.png dcgan/*.pth snli/.data snli/.vector_cache snli/results word_language_model/model.pt fast_neural_style/saved_models fast_neural_style/saved_models.zip gcn/cora/ gat/cora/ docs/build docs/venv # vi backups *~ # development .vscode **/.DS_Store ================================================ FILE: CODEOWNERS ================================================ # This is a comment. # Each line is a file pattern followed by one or more owners. # Github Actions, tests and CI ./github/ @msaroufim run_python_examples.sh @msaroufim # Distributed examples # Can also add the distributed oncall ./distributed/ @mrshenli @pritamdamania87 @rohan-varma @H-Huang ./mnist_hogwild/ @mrshenli @pritamdamania87 @rohan-varma @H-Huang # FX examples ./fx/ @jamesr66a @Chillee # Domain Examples ./reinforcement_learning/ @msaroufim ./word_language_model/ @msaroufim # Need an owner ./regression/ ./mnist/ ./imagenet/ ./super_resolution/ ./time_sequence_prediction/ ./vae/ # Legacy examples ./cpp/ ./legacy/snli/ ================================================ FILE: CODE_OF_CONDUCT.md ================================================ # Code of Conduct ## Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to make participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. ## Our Standards Examples of behavior that contributes to creating a positive environment include: - Using welcoming and inclusive language - Being respectful of differing viewpoints and experiences - Gracefully accepting constructive criticism - Focusing on what is best for the community - Showing empathy towards other community members Examples of unacceptable behavior by participants include: - The use of sexualized language or imagery and unwelcome sexual attention or advances - Trolling, insulting/derogatory comments, and personal or political attacks - Public or private harassment - Publishing other's private information, such as physical or electronic address, without explicit permission - Other conduct which could reasonably be considered inappropriate in a professional setting ## Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. ## Scope This Code of Conduct applies within all project spaces, and it also applies when an individual is representing the project or its community in public spaces. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at . All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html [homepage]: https://www.contributor-covenant.org For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing to examples We want to make contributing to this project as easy and transparent as possible. ## Pull Requests We actively welcome your pull requests. If you're new, we encourage you to take a look at issues tagged with [good first issue](https://github.com/pytorch/examples/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) ### For new examples 0. Create a GitHub issue proposing a new example and make sure it's substantially different from an existing one. 1. Fork the repo and create your branch from `main`. 2. If you've added code that should be tested, add tests to `run_python_examples.sh`. 3. Create a `README.md`. 4. Add a card with a brief description of your example and link to the repo to the `docs/source/index.rst` file and build the docs by running: ``` cd docs virtualenv venv source venv/bin/activate pip install -r requirements.txt make html ``` When done working with `virtualenv`, run `deactivate`. 5. Verify that there are no issues in your doc build. You can check the preview locally by installing [sphinx-serve](https://pypi.org/project/sphinx-serve/) then running `sphinx-serve -b build`. 6. Ensure your test passes locally. 7. If you haven't already, complete the Contributor License Agreement ("CLA"). 8. Address any feedback in code review promptly. ## For bug fixes 1. Fork the repo and create your branch from `main`. 2. Make sure you have a GPU-enabled machine, either locally or in the cloud. `g4dn.4xlarge` is a good starting point on AWS. 3. Make your code change. 4. Install `uv`. 5. Then, make sure that `VIRTUAL_ENV=.venv ./run_python_examples.sh` passes locally by running the script end to end. 6. If you haven't already, complete the Contributor License Agreement ("CLA"). 7. Address any feedback in code review promptly. ## Contributor License Agreement ("CLA") To accept your pull request, we need you to submit a CLA. You only need to do this once to work on any of Facebook's open source projects. Complete your CLA here: ## Issues We use GitHub issues to track public bugs. Please ensure your description is clear and has sufficient instructions to be able to reproduce the issue. ## License By contributing to examples, you agree that your contributions will be licensed under the LICENSE file in the root directory of this source tree. ================================================ FILE: LICENSE ================================================ BSD 3-Clause License Copyright (c) 2017, Pytorch contributors All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================ FILE: README.md ================================================ # PyTorch Examples https://pytorch.org/examples/ `pytorch/examples` is a repository showcasing examples of using [PyTorch](https://github.com/pytorch/pytorch). The goal is to have curated, short, few/no dependencies _high quality_ examples that are substantially different from each other that can be emulated in your existing work. - For tutorials: https://github.com/pytorch/tutorials - For changes to pytorch.org: https://github.com/pytorch/pytorch.github.io - For a general model hub: https://pytorch.org/hub/ or https://huggingface.co/models - For recipes on how to run PyTorch in production: https://github.com/facebookresearch/recipes - For general Q&A and support: https://discuss.pytorch.org/ ## Available models - [Image classification (MNIST) using Convnets](./mnist/README.md) - [Word-level Language Modeling using RNN and Transformer](./word_language_model/README.md) - [Training Imagenet Classifiers with Popular Networks](./imagenet/README.md) - [Generative Adversarial Networks (DCGAN)](./dcgan/README.md) - [Variational Auto-Encoders](./vae/README.md) - [Superresolution using an efficient sub-pixel convolutional neural network](./super_resolution/README.md) - [Hogwild training of shared ConvNets across multiple processes on MNIST](mnist_hogwild) - [Training a CartPole to balance with actor-critic](./reinforcement_learning/README.md) - [Natural Language Inference (SNLI) with GloVe vectors, LSTMs, and torchtext](snli) - [Time sequence prediction - use an LSTM to learn Sine waves](./time_sequence_prediction/README.md) - [Implement the Neural Style Transfer algorithm on images](./fast_neural_style/README.md) - [Reinforcement Learning with Actor Critic and REINFORCE algorithms on OpenAI gym](./reinforcement_learning/README.md) - [PyTorch Module Transformations using fx](./fx/README.md) - Distributed PyTorch examples with [Distributed Data Parallel](./distributed/ddp/README.md) and [RPC](./distributed/rpc) - [Several examples illustrating the C++ Frontend](cpp) - [Image Classification Using Forward-Forward](./mnist_forward_forward/README.md) - [Language Translation using Transformers](./language_translation/README.md) Additionally, a list of good examples hosted in their own repositories: - [Neural Machine Translation using sequence-to-sequence RNN with attention (OpenNMT)](https://github.com/OpenNMT/OpenNMT-py) ## Contributing If you'd like to contribute your own example or fix a bug please make sure to take a look at [CONTRIBUTING.md](CONTRIBUTING.md). ================================================ FILE: cpp/.clang-format ================================================ --- AccessModifierOffset: -1 AlignAfterOpenBracket: AlwaysBreak AlignConsecutiveAssignments: false AlignConsecutiveDeclarations: false AlignEscapedNewlinesLeft: true AlignOperands: false AlignTrailingComments: false AllowAllParametersOfDeclarationOnNextLine: false AllowShortBlocksOnASingleLine: false AllowShortCaseLabelsOnASingleLine: false AllowShortFunctionsOnASingleLine: Empty AllowShortIfStatementsOnASingleLine: false AllowShortLoopsOnASingleLine: false AlwaysBreakAfterReturnType: None AlwaysBreakBeforeMultilineStrings: true AlwaysBreakTemplateDeclarations: true BinPackArguments: false BinPackParameters: false BraceWrapping: AfterClass: false AfterControlStatement: false AfterEnum: false AfterFunction: false AfterNamespace: false AfterObjCDeclaration: false AfterStruct: false AfterUnion: false BeforeCatch: false BeforeElse: false IndentBraces: false BreakBeforeBinaryOperators: None BreakBeforeBraces: Attach BreakBeforeTernaryOperators: true BreakConstructorInitializersBeforeComma: false BreakAfterJavaFieldAnnotations: false BreakStringLiterals: false ColumnLimit: 80 CommentPragmas: '^ IWYU pragma:' CompactNamespaces: false ConstructorInitializerAllOnOneLineOrOnePerLine: true ConstructorInitializerIndentWidth: 4 ContinuationIndentWidth: 4 Cpp11BracedListStyle: true DerivePointerAlignment: false DisableFormat: false ForEachMacros: [ FOR_EACH_RANGE, FOR_EACH, ] IncludeCategories: - Regex: '^<.*\.h(pp)?>' Priority: 1 - Regex: '^<.*' Priority: 2 - Regex: '.*' Priority: 3 IndentCaseLabels: true IndentWidth: 2 IndentWrappedFunctionNames: false KeepEmptyLinesAtTheStartOfBlocks: false MacroBlockBegin: '' MacroBlockEnd: '' MaxEmptyLinesToKeep: 1 NamespaceIndentation: None ObjCBlockIndentWidth: 2 ObjCSpaceAfterProperty: false ObjCSpaceBeforeProtocolList: false PenaltyBreakBeforeFirstCallParameter: 1 PenaltyBreakComment: 300 PenaltyBreakFirstLessLess: 120 PenaltyBreakString: 1000 PenaltyExcessCharacter: 1000000 PenaltyReturnTypeOnItsOwnLine: 2000000 PointerAlignment: Left ReflowComments: true SortIncludes: true SpaceAfterCStyleCast: false SpaceBeforeAssignmentOperators: true SpaceBeforeParens: ControlStatements SpaceInEmptyParentheses: false SpacesBeforeTrailingComments: 1 SpacesInAngles: false SpacesInContainerLiterals: true SpacesInCStyleCastParentheses: false SpacesInParentheses: false SpacesInSquareBrackets: false Standard: Cpp11 TabWidth: 8 UseTab: Never ... ================================================ FILE: cpp/autograd/CMakeLists.txt ================================================ cmake_minimum_required(VERSION 3.5) project(autograd) set(CMAKE_CXX_STANDARD 17) find_package(Torch REQUIRED) add_executable(${PROJECT_NAME} "autograd.cpp") target_link_libraries(${PROJECT_NAME} "${TORCH_LIBRARIES}") # The following code block is suggested to be used on Windows. # According to https://github.com/pytorch/pytorch/issues/25457, # the DLLs need to be copied to avoid memory errors. if (MSVC) file(GLOB TORCH_DLLS "${TORCH_INSTALL_PREFIX}/lib/*.dll") add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different ${TORCH_DLLS} $) endif (MSVC) ================================================ FILE: cpp/autograd/README.md ================================================ # C++ autograd example `autograd.cpp` contains several examples of doing autograd in PyTorch C++ frontend. To build the code, run the following commands from your terminal: ```shell $ cd autograd $ mkdir build $ cd build $ cmake -DCMAKE_PREFIX_PATH=/path/to/libtorch .. $ make ``` where `/path/to/libtorch` should be the path to the unzipped _LibTorch_ distribution, which you can get from the [PyTorch homepage](https://pytorch.org/get-started/locally/). Execute the compiled binary to run: ```shell $ ./autograd ====== Running: "Basic autograd operations" ====== 1 1 1 1 [ CPUFloatType{2,2} ] 3 3 3 3 [ CPUFloatType{2,2} ] AddBackward1 27 27 27 27 [ CPUFloatType{2,2} ] MulBackward1 27 [ CPUFloatType{} ] MeanBackward0 false true SumBackward0 4.5000 4.5000 4.5000 4.5000 [ CPUFloatType{2,2} ] 813.6625 1015.0142 -664.8849 [ CPUFloatType{3} ] MulBackward1 204.8000 2048.0000 0.2048 [ CPUFloatType{3} ] true true false true false true ====== Running "Computing higher-order gradients in C++" ====== 0.0025 0.0946 0.1474 0.1387 0.0238 -0.0018 0.0259 0.0094 0.0513 -0.0549 -0.0604 0.0210 [ CPUFloatType{3,4} ] ====== Running "Using custom autograd function in C++" ====== -3.5513 3.7160 3.6477 -3.5513 3.7160 3.6477 [ CPUFloatType{2,3} ] 0.3095 1.4035 -0.0349 0.3095 1.4035 -0.0349 0.3095 1.4035 -0.0349 0.3095 1.4035 -0.0349 [ CPUFloatType{4,3} ] 5.5000 5.5000 [ CPUFloatType{2} ] ``` ================================================ FILE: cpp/autograd/autograd.cpp ================================================ #include #include using namespace torch::autograd; void basic_autograd_operations_example() { std::cout << "====== Running: \"Basic autograd operations\" ======" << std::endl; // Create a tensor and set ``torch::requires_grad()`` to track computation with it auto x = torch::ones({2, 2}, torch::requires_grad()); std::cout << x << std::endl; // Do a tensor operation: auto y = x + 2; std::cout << y << std::endl; // ``y`` was created as a result of an operation, so it has a ``grad_fn``. std::cout << y.grad_fn()->name() << std::endl; // Do more operations on ``y`` auto z = y * y * 3; auto out = z.mean(); std::cout << z << std::endl; std::cout << z.grad_fn()->name() << std::endl; std::cout << out << std::endl; std::cout << out.grad_fn()->name() << std::endl; // ``.requires_grad_( ... )`` changes an existing tensor's ``requires_grad`` flag in-place. auto a = torch::randn({2, 2}); a = ((a * 3) / (a - 1)); std::cout << a.requires_grad() << std::endl; a.requires_grad_(true); std::cout << a.requires_grad() << std::endl; auto b = (a * a).sum(); std::cout << b.grad_fn()->name() << std::endl; // Let's backprop now. Because ``out`` contains a single scalar, ``out.backward()`` // is equivalent to ``out.backward(torch::tensor(1.))``. out.backward(); // Print gradients d(out)/dx std::cout << x.grad() << std::endl; // Now let's take a look at an example of vector-Jacobian product: x = torch::randn(3, torch::requires_grad()); y = x * 2; while (y.norm().item() < 1000) { y = y * 2; } std::cout << y << std::endl; std::cout << y.grad_fn()->name() << std::endl; // If we want the vector-Jacobian product, pass the vector to ``backward`` as argument: auto v = torch::tensor({0.1, 1.0, 0.0001}, torch::kFloat); y.backward(v); std::cout << x.grad() << std::endl; // You can also stop autograd from tracking history on tensors that require gradients // either by putting ``torch::NoGradGuard`` in a code block std::cout << x.requires_grad() << std::endl; std::cout << x.pow(2).requires_grad() << std::endl; { torch::NoGradGuard no_grad; std::cout << x.pow(2).requires_grad() << std::endl; } // Or by using ``.detach()`` to get a new tensor with the same content but that does // not require gradients: std::cout << x.requires_grad() << std::endl; y = x.detach(); std::cout << y.requires_grad() << std::endl; std::cout << x.eq(y).all().item() << std::endl; } void compute_higher_order_gradients_example() { std::cout << "====== Running \"Computing higher-order gradients in C++\" ======" << std::endl; // One of the applications of higher-order gradients is calculating gradient penalty. // Let's see an example of it using ``torch::autograd::grad``: auto model = torch::nn::Linear(4, 3); auto input = torch::randn({3, 4}).requires_grad_(true); auto output = model(input); // Calculate loss auto target = torch::randn({3, 3}); auto loss = torch::nn::MSELoss()(output, target); // Use norm of gradients as penalty auto grad_output = torch::ones_like(output); auto gradient = torch::autograd::grad({output}, {input}, /*grad_outputs=*/{grad_output}, /*create_graph=*/true)[0]; auto gradient_penalty = torch::pow((gradient.norm(2, /*dim=*/1) - 1), 2).mean(); // Add gradient penalty to loss auto combined_loss = loss + gradient_penalty; combined_loss.backward(); std::cout << input.grad() << std::endl; } // Inherit from Function class LinearFunction : public Function { public: // Note that both forward and backward are static functions // bias is an optional argument static torch::Tensor forward( AutogradContext *ctx, torch::Tensor input, torch::Tensor weight, torch::Tensor bias = torch::Tensor()) { ctx->save_for_backward({input, weight, bias}); auto output = input.mm(weight.t()); if (bias.defined()) { output += bias.unsqueeze(0).expand_as(output); } return output; } static tensor_list backward(AutogradContext *ctx, tensor_list grad_outputs) { auto saved = ctx->get_saved_variables(); auto input = saved[0]; auto weight = saved[1]; auto bias = saved[2]; auto grad_output = grad_outputs[0]; auto grad_input = grad_output.mm(weight); auto grad_weight = grad_output.t().mm(input); auto grad_bias = torch::Tensor(); if (bias.defined()) { grad_bias = grad_output.sum(0); } return {grad_input, grad_weight, grad_bias}; } }; class MulConstant : public Function { public: static torch::Tensor forward(AutogradContext *ctx, torch::Tensor tensor, double constant) { // ctx is a context object that can be used to stash information // for backward computation ctx->saved_data["constant"] = constant; return tensor * constant; } static tensor_list backward(AutogradContext *ctx, tensor_list grad_outputs) { // We return as many input gradients as there were arguments. // Gradients of non-tensor arguments to forward must be `torch::Tensor()`. return {grad_outputs[0] * ctx->saved_data["constant"].toDouble(), torch::Tensor()}; } }; void custom_autograd_function_example() { std::cout << "====== Running \"Using custom autograd function in C++\" ======" << std::endl; { auto x = torch::randn({2, 3}).requires_grad_(); auto weight = torch::randn({4, 3}).requires_grad_(); auto y = LinearFunction::apply(x, weight); y.sum().backward(); std::cout << x.grad() << std::endl; std::cout << weight.grad() << std::endl; } { auto x = torch::randn({2}).requires_grad_(); auto y = MulConstant::apply(x, 5.5); y.sum().backward(); std::cout << x.grad() << std::endl; } } int main() { std::cout << std::boolalpha; basic_autograd_operations_example(); std::cout << "\n"; compute_higher_order_gradients_example(); std::cout << "\n"; custom_autograd_function_example(); } ================================================ FILE: cpp/custom-dataset/CMakeLists.txt ================================================ cmake_minimum_required(VERSION 3.5) project(custom-dataset) set(CMAKE_CXX_STANDARD 17) find_package(Torch REQUIRED) find_package(OpenCV REQUIRED COMPONENTS core imgproc imgcodecs) message(STATUS "OpenCV include dirs: ${OpenCV_INCLUDE_DIRS}") message(STATUS "OpenCV libraries: ${OpenCV_LIBS}") include_directories(${OpenCV_INCLUDE_DIRS}) add_executable(${PROJECT_NAME} "custom-dataset.cpp") target_link_libraries(${PROJECT_NAME} "${OpenCV_LIBS}") target_link_libraries(${PROJECT_NAME} "${TORCH_LIBRARIES}") configure_file("info.txt" "info.txt" COPYONLY) # The following code block is suggested to be used on Windows. # According to https://github.com/pytorch/pytorch/issues/25457, # the DLLs need to be copied to avoid memory errors. if (MSVC) file(GLOB TORCH_DLLS "${TORCH_INSTALL_PREFIX}/lib/*.dll") add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different ${TORCH_DLLS} $) endif (MSVC) ================================================ FILE: cpp/custom-dataset/README.md ================================================ # Custom Dataset Example with the PyTorch C++ Frontend This folder contains an example of loading a custom image dataset with OpenCV and training a model to label images, using the PyTorch C++ frontend. The dataset used here is [Caltech 101](https://data.caltech.edu/records/mzrjq-6wc02) dataset. The entire training code is contained in custom-data.cpp. You can find instructions on how to install OpenCV [here](../tools/InstallingOpenCV.md). To build the code, run the following commands from your terminal: ```shell $ cd custom-dataset $ mkdir build $ cd build $ cmake -DCMAKE_PREFIX_PATH=/path/to/libtorch .. $ make ``` where /path/to/libtorch should be the path to the unzipped LibTorch distribution, which you can get from the [PyTorch homepage](https://pytorch.org/get-started/locally/). If you see an error like `undefined reference to cv::imread(std::string const&, int)` when running the `make` command, you should build LibTorch from source using the instructions [here](https://github.com/pytorch/pytorch#from-source), and then set `CMAKE_PREFIX_PATH` to that PyTorch source directory. An alternative solution is to use `libtorch-cxx11-abi-shared-with-deps` instead of `libtorch-shared-with-deps` as the latter is not compatible with openCV (reported [here](https://discuss.pytorch.org/t/library-conflict-between-libtorch-and-opencv/64489)). The build directory should look like this: ``` . ├── custom-dataset ├── dataset │   ├── accordion │   │   ├── image_0001.jpg │   │   ├── ... │   ├── airplanes │   │   ├── ... │   ├── ... ├── info.txt └── Makefile └── ... ``` `info.txt` file gets copied from source directory during build. Execute the compiled binary to train the model: ```shell ./custom-dataset Running on: CUDA Train Epoch: 1 16/7281 Loss: 0.314655 Acc: 0 Train Epoch: 1 176/7281 Loss: 0.532111 Acc: 0.0681818 Train Epoch: 1 336/7281 Loss: 0.538482 Acc: 0.0714286 Train Epoch: 1 496/7281 Loss: 0.535302 Acc: 0.0705645 Train Epoch: 1 656/7281 Loss: 0.536113 Acc: 0.0716463 Train Epoch: 1 816/7281 Loss: 0.537626 Acc: 0.0784314 Train Epoch: 1 976/7281 Loss: 0.537055 Acc: 0.079918 ... ``` ================================================ FILE: cpp/custom-dataset/custom-dataset.cpp ================================================ #include #include #include #include #include #include #include #include struct Options { int image_size = 224; size_t train_batch_size = 8; size_t test_batch_size = 200; size_t iterations = 10; size_t log_interval = 20; // path must end in delimiter std::string datasetPath = "./dataset/"; std::string infoFilePath = "info.txt"; torch::DeviceType device = torch::kCPU; }; static Options options; using Data = std::vector>; class CustomDataset : public torch::data::datasets::Dataset { using Example = torch::data::Example<>; const Data data; public: CustomDataset(const Data& data) : data(data) {} Example get(size_t index) { std::string path = options.datasetPath + data[index].first; auto mat = cv::imread(path); assert(!mat.empty()); cv::resize(mat, mat, cv::Size(options.image_size, options.image_size)); std::vector channels(3); cv::split(mat, channels); auto R = torch::from_blob( channels[2].ptr(), {options.image_size, options.image_size}, torch::kUInt8); auto G = torch::from_blob( channels[1].ptr(), {options.image_size, options.image_size}, torch::kUInt8); auto B = torch::from_blob( channels[0].ptr(), {options.image_size, options.image_size}, torch::kUInt8); auto tdata = torch::cat({R, G, B}) .view({3, options.image_size, options.image_size}) .to(torch::kFloat); auto tlabel = torch::tensor(data[index].second, torch::kLong); return {tdata, tlabel}; } torch::optional size() const { return data.size(); } }; std::pair readInfo() { std::random_device randomDevice; std::mt19937 mersenneTwisterGenerator(randomDevice()); Data train, test; std::ifstream stream(options.infoFilePath); assert(stream.is_open()); long label; std::string path, type; while (true) { stream >> path >> label >> type; if (type == "train") train.push_back(std::make_pair(path, label)); else if (type == "test") test.push_back(std::make_pair(path, label)); else assert(false); if (stream.eof()) break; } std::shuffle(train.begin(), train.end(), mersenneTwisterGenerator); std::shuffle(test.begin(), test.end(), mersenneTwisterGenerator); return std::make_pair(train, test); } struct NetworkImpl : torch::nn::SequentialImpl { NetworkImpl() { using namespace torch::nn; auto stride = torch::ExpandingArray<2>({2, 2}); torch::ExpandingArray<2> shape({-1, 256 * 6 * 6}); push_back(Conv2d(Conv2dOptions(3, 64, 11).stride(4).padding(2))); push_back(Functional(torch::relu)); push_back(Functional(torch::max_pool2d, 3, stride, 0, 1, false)); push_back(Conv2d(Conv2dOptions(64, 192, 5).padding(2))); push_back(Functional(torch::relu)); push_back(Functional(torch::max_pool2d, 3, stride, 0, 1, false)); push_back(Conv2d(Conv2dOptions(192, 384, 3).padding(1))); push_back(Functional(torch::relu)); push_back(Conv2d(Conv2dOptions(384, 256, 3).padding(1))); push_back(Functional(torch::relu)); push_back(Conv2d(Conv2dOptions(256, 256, 3).padding(1))); push_back(Functional(torch::relu)); push_back(Functional(torch::max_pool2d, 3, stride, 0, 1, false)); push_back(Functional(torch::reshape, shape)); push_back(Dropout()); push_back(Linear(256 * 6 * 6, 4096)); push_back(Functional(torch::relu)); push_back(Dropout()); push_back(Linear(4096, 4096)); push_back(Functional(torch::relu)); push_back(Linear(4096, 102)); push_back(Functional( [](torch::Tensor input) { return torch::log_softmax(input, 1); })); } }; TORCH_MODULE(Network); template void train( Network& network, DataLoader& loader, torch::optim::Optimizer& optimizer, size_t epoch, size_t data_size) { size_t index = 0; network->train(); float Loss = 0, Acc = 0; for (auto& batch : loader) { auto data = batch.data.to(options.device); auto targets = batch.target.to(options.device).view({-1}); auto output = network->forward(data); auto loss = torch::nll_loss(output, targets); assert(!std::isnan(loss.template item())); auto acc = output.argmax(1).eq(targets).sum(); optimizer.zero_grad(); loss.backward(); optimizer.step(); Loss += loss.template item(); Acc += acc.template item(); if (index++ % options.log_interval == 0) { auto end = std::min(data_size, (index + 1) * options.train_batch_size); std::cout << "Train Epoch: " << epoch << " " << end << "/" << data_size << "\tLoss: " << Loss / end << "\tAcc: " << Acc / end << std::endl; } } } template void test(Network& network, DataLoader& loader, size_t data_size) { size_t index = 0; network->eval(); torch::NoGradGuard no_grad; float Loss = 0, Acc = 0; for (const auto& batch : loader) { auto data = batch.data.to(options.device); auto targets = batch.target.to(options.device).view({-1}); auto output = network->forward(data); auto loss = torch::nll_loss(output, targets); assert(!std::isnan(loss.template item())); auto acc = output.argmax(1).eq(targets).sum(); Loss += loss.template item(); Acc += acc.template item(); } if (index++ % options.log_interval == 0) std::cout << "Test Loss: " << Loss / data_size << "\tAcc: " << Acc / data_size << std::endl; } int main() { torch::manual_seed(1); if (torch::cuda::is_available()) options.device = torch::kCUDA; std::cout << "Running on: " << (options.device == torch::kCUDA ? "CUDA" : "CPU") << std::endl; const auto data = readInfo(); auto train_set = CustomDataset(data.first).map(torch::data::transforms::Stack<>()); auto train_size = train_set.size().value(); auto train_loader = torch::data::make_data_loader( std::move(train_set), options.train_batch_size); auto test_set = CustomDataset(data.second).map(torch::data::transforms::Stack<>()); auto test_size = test_set.size().value(); auto test_loader = torch::data::make_data_loader( std::move(test_set), options.test_batch_size); Network network; network->to(options.device); torch::optim::SGD optimizer( network->parameters(), torch::optim::SGDOptions(0.001).momentum(0.5)); for (size_t i = 0; i < options.iterations; ++i) { train(network, *train_loader, optimizer, i + 1, train_size); std::cout << std::endl; test(network, *test_loader, test_size); std::cout << std::endl; } return 0; } ================================================ FILE: cpp/custom-dataset/info.txt ================================================ airplanes/image_0316.jpg 0 train airplanes/image_0716.jpg 0 train airplanes/image_0258.jpg 0 train airplanes/image_0426.jpg 0 train airplanes/image_0511.jpg 0 train airplanes/image_0676.jpg 0 train airplanes/image_0528.jpg 0 train airplanes/image_0720.jpg 0 train airplanes/image_0556.jpg 0 train airplanes/image_0302.jpg 0 train airplanes/image_0171.jpg 0 train airplanes/image_0062.jpg 0 train airplanes/image_0504.jpg 0 train airplanes/image_0346.jpg 0 train airplanes/image_0546.jpg 0 train airplanes/image_0435.jpg 0 train airplanes/image_0341.jpg 0 train airplanes/image_0776.jpg 0 train airplanes/image_0340.jpg 0 train airplanes/image_0331.jpg 0 train airplanes/image_0037.jpg 0 train airplanes/image_0737.jpg 0 train airplanes/image_0786.jpg 0 train airplanes/image_0560.jpg 0 train airplanes/image_0472.jpg 0 train airplanes/image_0049.jpg 0 train airplanes/image_0337.jpg 0 train airplanes/image_0779.jpg 0 train airplanes/image_0317.jpg 0 train airplanes/image_0357.jpg 0 train airplanes/image_0083.jpg 0 train airplanes/image_0101.jpg 0 train airplanes/image_0051.jpg 0 train airplanes/image_0006.jpg 0 train airplanes/image_0142.jpg 0 train airplanes/image_0681.jpg 0 train airplanes/image_0250.jpg 0 train airplanes/image_0427.jpg 0 train airplanes/image_0486.jpg 0 train airplanes/image_0396.jpg 0 train airplanes/image_0733.jpg 0 train airplanes/image_0332.jpg 0 train airplanes/image_0448.jpg 0 train airplanes/image_0335.jpg 0 train airplanes/image_0752.jpg 0 train airplanes/image_0201.jpg 0 train airplanes/image_0505.jpg 0 train airplanes/image_0016.jpg 0 train airplanes/image_0373.jpg 0 train airplanes/image_0772.jpg 0 train airplanes/image_0565.jpg 0 train airplanes/image_0513.jpg 0 train airplanes/image_0709.jpg 0 train airplanes/image_0176.jpg 0 train airplanes/image_0411.jpg 0 train airplanes/image_0155.jpg 0 train airplanes/image_0030.jpg 0 train airplanes/image_0091.jpg 0 train airplanes/image_0664.jpg 0 train airplanes/image_0563.jpg 0 train airplanes/image_0397.jpg 0 train airplanes/image_0618.jpg 0 train airplanes/image_0012.jpg 0 train airplanes/image_0066.jpg 0 train airplanes/image_0657.jpg 0 train airplanes/image_0765.jpg 0 train airplanes/image_0585.jpg 0 train airplanes/image_0612.jpg 0 train airplanes/image_0544.jpg 0 train airplanes/image_0703.jpg 0 train airplanes/image_0465.jpg 0 train airplanes/image_0728.jpg 0 train airplanes/image_0464.jpg 0 train airplanes/image_0283.jpg 0 train airplanes/image_0551.jpg 0 train airplanes/image_0318.jpg 0 train airplanes/image_0224.jpg 0 train airplanes/image_0771.jpg 0 train airplanes/image_0148.jpg 0 train airplanes/image_0501.jpg 0 train airplanes/image_0496.jpg 0 train airplanes/image_0077.jpg 0 train airplanes/image_0129.jpg 0 train airplanes/image_0339.jpg 0 train airplanes/image_0554.jpg 0 train airplanes/image_0025.jpg 0 train airplanes/image_0328.jpg 0 train airplanes/image_0549.jpg 0 train airplanes/image_0659.jpg 0 train airplanes/image_0540.jpg 0 train airplanes/image_0518.jpg 0 train airplanes/image_0479.jpg 0 train airplanes/image_0539.jpg 0 train airplanes/image_0527.jpg 0 train airplanes/image_0783.jpg 0 train airplanes/image_0621.jpg 0 train airplanes/image_0476.jpg 0 train airplanes/image_0139.jpg 0 train airplanes/image_0519.jpg 0 train airplanes/image_0425.jpg 0 train airplanes/image_0292.jpg 0 train airplanes/image_0348.jpg 0 train airplanes/image_0499.jpg 0 train airplanes/image_0115.jpg 0 train airplanes/image_0123.jpg 0 train airplanes/image_0119.jpg 0 train airplanes/image_0670.jpg 0 train airplanes/image_0721.jpg 0 train airplanes/image_0639.jpg 0 train airplanes/image_0060.jpg 0 train airplanes/image_0269.jpg 0 train airplanes/image_0052.jpg 0 train airplanes/image_0108.jpg 0 train airplanes/image_0239.jpg 0 train airplanes/image_0442.jpg 0 train airplanes/image_0684.jpg 0 train airplanes/image_0370.jpg 0 train airplanes/image_0491.jpg 0 train airplanes/image_0469.jpg 0 train airplanes/image_0326.jpg 0 train airplanes/image_0223.jpg 0 train airplanes/image_0172.jpg 0 train airplanes/image_0372.jpg 0 train airplanes/image_0704.jpg 0 train airplanes/image_0344.jpg 0 train airplanes/image_0390.jpg 0 train airplanes/image_0007.jpg 0 train airplanes/image_0213.jpg 0 train airplanes/image_0739.jpg 0 train airplanes/image_0312.jpg 0 train airplanes/image_0463.jpg 0 train airplanes/image_0063.jpg 0 train airplanes/image_0019.jpg 0 train airplanes/image_0407.jpg 0 train airplanes/image_0192.jpg 0 train airplanes/image_0490.jpg 0 train airplanes/image_0533.jpg 0 train airplanes/image_0743.jpg 0 train airplanes/image_0384.jpg 0 train airplanes/image_0195.jpg 0 train airplanes/image_0113.jpg 0 train airplanes/image_0677.jpg 0 train airplanes/image_0127.jpg 0 train airplanes/image_0215.jpg 0 train airplanes/image_0389.jpg 0 train airplanes/image_0031.jpg 0 train airplanes/image_0349.jpg 0 train airplanes/image_0228.jpg 0 train airplanes/image_0380.jpg 0 train airplanes/image_0219.jpg 0 train airplanes/image_0796.jpg 0 train airplanes/image_0688.jpg 0 train airplanes/image_0597.jpg 0 train airplanes/image_0353.jpg 0 train airplanes/image_0011.jpg 0 train airplanes/image_0562.jpg 0 train airplanes/image_0233.jpg 0 train airplanes/image_0447.jpg 0 train airplanes/image_0747.jpg 0 train airplanes/image_0477.jpg 0 train airplanes/image_0680.jpg 0 train airplanes/image_0061.jpg 0 train airplanes/image_0444.jpg 0 train airplanes/image_0090.jpg 0 train airplanes/image_0745.jpg 0 train airplanes/image_0351.jpg 0 train airplanes/image_0261.jpg 0 train airplanes/image_0229.jpg 0 train airplanes/image_0126.jpg 0 train airplanes/image_0761.jpg 0 train airplanes/image_0775.jpg 0 train airplanes/image_0794.jpg 0 train airplanes/image_0240.jpg 0 train airplanes/image_0467.jpg 0 train airplanes/image_0768.jpg 0 train airplanes/image_0455.jpg 0 train airplanes/image_0441.jpg 0 train airplanes/image_0305.jpg 0 train airplanes/image_0572.jpg 0 train airplanes/image_0334.jpg 0 train airplanes/image_0602.jpg 0 train airplanes/image_0354.jpg 0 train airplanes/image_0542.jpg 0 train airplanes/image_0218.jpg 0 train airplanes/image_0573.jpg 0 train airplanes/image_0366.jpg 0 train airplanes/image_0288.jpg 0 train airplanes/image_0583.jpg 0 train airplanes/image_0216.jpg 0 train airplanes/image_0459.jpg 0 train airplanes/image_0429.jpg 0 train airplanes/image_0277.jpg 0 train airplanes/image_0623.jpg 0 train airplanes/image_0358.jpg 0 train airplanes/image_0588.jpg 0 train airplanes/image_0050.jpg 0 train airplanes/image_0246.jpg 0 train airplanes/image_0361.jpg 0 train airplanes/image_0177.jpg 0 train airplanes/image_0356.jpg 0 train airplanes/image_0648.jpg 0 train airplanes/image_0789.jpg 0 train airplanes/image_0196.jpg 0 train airplanes/image_0293.jpg 0 train airplanes/image_0406.jpg 0 train airplanes/image_0596.jpg 0 train airplanes/image_0640.jpg 0 train airplanes/image_0575.jpg 0 train airplanes/image_0625.jpg 0 train airplanes/image_0347.jpg 0 train airplanes/image_0521.jpg 0 train airplanes/image_0443.jpg 0 train airplanes/image_0187.jpg 0 train airplanes/image_0558.jpg 0 train airplanes/image_0234.jpg 0 train airplanes/image_0045.jpg 0 train airplanes/image_0303.jpg 0 train airplanes/image_0363.jpg 0 train airplanes/image_0679.jpg 0 train airplanes/image_0017.jpg 0 train airplanes/image_0131.jpg 0 train airplanes/image_0762.jpg 0 train airplanes/image_0492.jpg 0 train airplanes/image_0296.jpg 0 train airplanes/image_0555.jpg 0 train airplanes/image_0735.jpg 0 train airplanes/image_0150.jpg 0 train airplanes/image_0018.jpg 0 train airplanes/image_0350.jpg 0 train airplanes/image_0795.jpg 0 train airplanes/image_0209.jpg 0 train airplanes/image_0013.jpg 0 train airplanes/image_0570.jpg 0 train airplanes/image_0748.jpg 0 train airplanes/image_0569.jpg 0 train airplanes/image_0696.jpg 0 train airplanes/image_0614.jpg 0 train airplanes/image_0054.jpg 0 train airplanes/image_0629.jpg 0 train airplanes/image_0094.jpg 0 train airplanes/image_0613.jpg 0 train airplanes/image_0082.jpg 0 train airplanes/image_0097.jpg 0 train airplanes/image_0130.jpg 0 train airplanes/image_0457.jpg 0 train airplanes/image_0488.jpg 0 train airplanes/image_0382.jpg 0 train airplanes/image_0484.jpg 0 train airplanes/image_0141.jpg 0 train airplanes/image_0700.jpg 0 train airplanes/image_0144.jpg 0 train airplanes/image_0109.jpg 0 train airplanes/image_0645.jpg 0 train airplanes/image_0468.jpg 0 train airplanes/image_0515.jpg 0 train airplanes/image_0079.jpg 0 train airplanes/image_0306.jpg 0 train airplanes/image_0398.jpg 0 train airplanes/image_0055.jpg 0 train airplanes/image_0367.jpg 0 train airplanes/image_0365.jpg 0 train airplanes/image_0566.jpg 0 train airplanes/image_0217.jpg 0 train airplanes/image_0343.jpg 0 train airplanes/image_0432.jpg 0 train airplanes/image_0538.jpg 0 train airplanes/image_0600.jpg 0 train airplanes/image_0502.jpg 0 train airplanes/image_0661.jpg 0 train airplanes/image_0731.jpg 0 train airplanes/image_0071.jpg 0 train airplanes/image_0590.jpg 0 train airplanes/image_0282.jpg 0 train airplanes/image_0393.jpg 0 train airplanes/image_0287.jpg 0 train airplanes/image_0198.jpg 0 train airplanes/image_0262.jpg 0 train airplanes/image_0686.jpg 0 train airplanes/image_0462.jpg 0 train airplanes/image_0698.jpg 0 train airplanes/image_0168.jpg 0 train airplanes/image_0368.jpg 0 train airplanes/image_0420.jpg 0 train airplanes/image_0650.jpg 0 train airplanes/image_0763.jpg 0 train airplanes/image_0211.jpg 0 train airplanes/image_0537.jpg 0 train airplanes/image_0580.jpg 0 train airplanes/image_0627.jpg 0 train airplanes/image_0270.jpg 0 train airplanes/image_0594.jpg 0 train airplanes/image_0103.jpg 0 train airplanes/image_0643.jpg 0 train airplanes/image_0591.jpg 0 train airplanes/image_0024.jpg 0 train airplanes/image_0787.jpg 0 train airplanes/image_0297.jpg 0 train airplanes/image_0188.jpg 0 train airplanes/image_0792.jpg 0 train airplanes/image_0105.jpg 0 train airplanes/image_0074.jpg 0 train airplanes/image_0294.jpg 0 train airplanes/image_0440.jpg 0 train airplanes/image_0391.jpg 0 train airplanes/image_0189.jpg 0 train airplanes/image_0003.jpg 0 train airplanes/image_0009.jpg 0 train airplanes/image_0387.jpg 0 train airplanes/image_0275.jpg 0 train airplanes/image_0226.jpg 0 train airplanes/image_0628.jpg 0 train airplanes/image_0718.jpg 0 train airplanes/image_0714.jpg 0 train airplanes/image_0658.jpg 0 train airplanes/image_0601.jpg 0 train airplanes/image_0185.jpg 0 train airplanes/image_0790.jpg 0 train airplanes/image_0656.jpg 0 train airplanes/image_0567.jpg 0 train airplanes/image_0485.jpg 0 train airplanes/image_0541.jpg 0 train airplanes/image_0118.jpg 0 train airplanes/image_0652.jpg 0 train airplanes/image_0304.jpg 0 train airplanes/image_0278.jpg 0 train airplanes/image_0512.jpg 0 train airplanes/image_0230.jpg 0 train airplanes/image_0474.jpg 0 train airplanes/image_0553.jpg 0 train airplanes/image_0404.jpg 0 train airplanes/image_0487.jpg 0 train airplanes/image_0531.jpg 0 train airplanes/image_0638.jpg 0 train airplanes/image_0525.jpg 0 train airplanes/image_0482.jpg 0 train airplanes/image_0522.jpg 0 train airplanes/image_0460.jpg 0 train airplanes/image_0608.jpg 0 train airplanes/image_0134.jpg 0 train airplanes/image_0249.jpg 0 train airplanes/image_0042.jpg 0 train airplanes/image_0730.jpg 0 train airplanes/image_0352.jpg 0 train airplanes/image_0333.jpg 0 train airplanes/image_0221.jpg 0 train airplanes/image_0093.jpg 0 train airplanes/image_0266.jpg 0 train airplanes/image_0412.jpg 0 train airplanes/image_0759.jpg 0 train airplanes/image_0470.jpg 0 train airplanes/image_0497.jpg 0 train airplanes/image_0751.jpg 0 train airplanes/image_0301.jpg 0 train airplanes/image_0392.jpg 0 train airplanes/image_0473.jpg 0 train airplanes/image_0254.jpg 0 train airplanes/image_0561.jpg 0 train airplanes/image_0530.jpg 0 train airplanes/image_0321.jpg 0 train airplanes/image_0557.jpg 0 train airplanes/image_0386.jpg 0 train airplanes/image_0232.jpg 0 train airplanes/image_0324.jpg 0 train airplanes/image_0040.jpg 0 train airplanes/image_0067.jpg 0 train airplanes/image_0769.jpg 0 train airplanes/image_0310.jpg 0 train airplanes/image_0330.jpg 0 train airplanes/image_0548.jpg 0 train airplanes/image_0122.jpg 0 train airplanes/image_0671.jpg 0 train airplanes/image_0205.jpg 0 train airplanes/image_0161.jpg 0 train airplanes/image_0146.jpg 0 train airplanes/image_0300.jpg 0 train airplanes/image_0369.jpg 0 train airplanes/image_0451.jpg 0 train airplanes/image_0461.jpg 0 train airplanes/image_0204.jpg 0 train airplanes/image_0125.jpg 0 train airplanes/image_0047.jpg 0 train airplanes/image_0285.jpg 0 train airplanes/image_0160.jpg 0 train airplanes/image_0678.jpg 0 train airplanes/image_0422.jpg 0 train airplanes/image_0579.jpg 0 train airplanes/image_0064.jpg 0 train airplanes/image_0070.jpg 0 train airplanes/image_0552.jpg 0 train airplanes/image_0264.jpg 0 train airplanes/image_0112.jpg 0 train airplanes/image_0078.jpg 0 train airplanes/image_0637.jpg 0 train airplanes/image_0068.jpg 0 train airplanes/image_0506.jpg 0 train airplanes/image_0433.jpg 0 train airplanes/image_0547.jpg 0 train airplanes/image_0085.jpg 0 train airplanes/image_0410.jpg 0 train airplanes/image_0110.jpg 0 train airplanes/image_0722.jpg 0 train airplanes/image_0593.jpg 0 train airplanes/image_0259.jpg 0 train airplanes/image_0143.jpg 0 train airplanes/image_0043.jpg 0 train airplanes/image_0162.jpg 0 train airplanes/image_0132.jpg 0 train airplanes/image_0157.jpg 0 train airplanes/image_0475.jpg 0 train airplanes/image_0206.jpg 0 train airplanes/image_0376.jpg 0 train airplanes/image_0644.jpg 0 train airplanes/image_0319.jpg 0 train airplanes/image_0371.jpg 0 train airplanes/image_0044.jpg 0 train airplanes/image_0199.jpg 0 train airplanes/image_0729.jpg 0 train airplanes/image_0691.jpg 0 train airplanes/image_0033.jpg 0 train airplanes/image_0136.jpg 0 train airplanes/image_0180.jpg 0 train airplanes/image_0260.jpg 0 train airplanes/image_0604.jpg 0 train airplanes/image_0184.jpg 0 train airplanes/image_0719.jpg 0 train airplanes/image_0672.jpg 0 train airplanes/image_0784.jpg 0 train airplanes/image_0073.jpg 0 train airplanes/image_0395.jpg 0 train airplanes/image_0257.jpg 0 train airplanes/image_0364.jpg 0 train airplanes/image_0756.jpg 0 train airplanes/image_0001.jpg 0 train airplanes/image_0167.jpg 0 train airplanes/image_0284.jpg 0 train airplanes/image_0375.jpg 0 train airplanes/image_0080.jpg 0 train airplanes/image_0665.jpg 0 train airplanes/image_0156.jpg 0 train airplanes/image_0021.jpg 0 train airplanes/image_0571.jpg 0 train airplanes/image_0325.jpg 0 train airplanes/image_0517.jpg 0 train airplanes/image_0713.jpg 0 train airplanes/image_0516.jpg 0 train airplanes/image_0574.jpg 0 train airplanes/image_0471.jpg 0 train airplanes/image_0767.jpg 0 train airplanes/image_0581.jpg 0 train airplanes/image_0267.jpg 0 train airplanes/image_0299.jpg 0 train airplanes/image_0421.jpg 0 train airplanes/image_0495.jpg 0 train airplanes/image_0529.jpg 0 train airplanes/image_0436.jpg 0 train airplanes/image_0360.jpg 0 train airplanes/image_0649.jpg 0 train airplanes/image_0095.jpg 0 train airplanes/image_0057.jpg 0 train airplanes/image_0707.jpg 0 train airplanes/image_0797.jpg 0 train airplanes/image_0403.jpg 0 train airplanes/image_0114.jpg 0 train airplanes/image_0641.jpg 0 train airplanes/image_0595.jpg 0 train airplanes/image_0620.jpg 0 train airplanes/image_0153.jpg 0 train airplanes/image_0697.jpg 0 train airplanes/image_0654.jpg 0 train airplanes/image_0564.jpg 0 train airplanes/image_0416.jpg 0 train airplanes/image_0607.jpg 0 train airplanes/image_0098.jpg 0 train airplanes/image_0256.jpg 0 train airplanes/image_0402.jpg 0 train airplanes/image_0788.jpg 0 train airplanes/image_0203.jpg 0 train airplanes/image_0577.jpg 0 train airplanes/image_0764.jpg 0 train airplanes/image_0338.jpg 0 train airplanes/image_0582.jpg 0 train airplanes/image_0265.jpg 0 train airplanes/image_0147.jpg 0 train airplanes/image_0536.jpg 0 train airplanes/image_0008.jpg 0 train airplanes/image_0446.jpg 0 train airplanes/image_0514.jpg 0 train airplanes/image_0617.jpg 0 train airplanes/image_0619.jpg 0 train airplanes/image_0727.jpg 0 train airplanes/image_0589.jpg 0 train airplanes/image_0207.jpg 0 train airplanes/image_0208.jpg 0 train airplanes/image_0222.jpg 0 train airplanes/image_0320.jpg 0 train airplanes/image_0781.jpg 0 train airplanes/image_0532.jpg 0 train airplanes/image_0227.jpg 0 train airplanes/image_0651.jpg 0 train airplanes/image_0286.jpg 0 train airplanes/image_0035.jpg 0 train airplanes/image_0399.jpg 0 train airplanes/image_0166.jpg 0 train airplanes/image_0711.jpg 0 train airplanes/image_0669.jpg 0 train airplanes/image_0169.jpg 0 train airplanes/image_0191.jpg 0 train airplanes/image_0128.jpg 0 train airplanes/image_0493.jpg 0 train airplanes/image_0414.jpg 0 train airplanes/image_0606.jpg 0 train airplanes/image_0734.jpg 0 train airplanes/image_0683.jpg 0 train airplanes/image_0102.jpg 0 train airplanes/image_0523.jpg 0 train airplanes/image_0434.jpg 0 train airplanes/image_0699.jpg 0 train airplanes/image_0178.jpg 0 train airplanes/image_0431.jpg 0 train airplanes/image_0243.jpg 0 train airplanes/image_0005.jpg 0 train airplanes/image_0263.jpg 0 train airplanes/image_0780.jpg 0 train airplanes/image_0173.jpg 0 train airplanes/image_0622.jpg 0 train airplanes/image_0692.jpg 0 train airplanes/image_0087.jpg 0 train airplanes/image_0417.jpg 0 train airplanes/image_0089.jpg 0 train airplanes/image_0799.jpg 0 train airplanes/image_0059.jpg 0 train airplanes/image_0673.jpg 0 train airplanes/image_0253.jpg 0 train airplanes/image_0159.jpg 0 train airplanes/image_0225.jpg 0 train airplanes/image_0445.jpg 0 train airplanes/image_0186.jpg 0 train airplanes/image_0706.jpg 0 train airplanes/image_0388.jpg 0 train airplanes/image_0694.jpg 0 train airplanes/image_0069.jpg 0 train airplanes/image_0758.jpg 0 train airplanes/image_0315.jpg 0 train airplanes/image_0295.jpg 0 train airplanes/image_0746.jpg 0 train airplanes/image_0200.jpg 0 train airplanes/image_0099.jpg 0 train airplanes/image_0738.jpg 0 train airplanes/image_0336.jpg 0 train airplanes/image_0034.jpg 0 train airplanes/image_0740.jpg 0 train airplanes/image_0712.jpg 0 train airplanes/image_0559.jpg 0 train airplanes/image_0409.jpg 0 train airplanes/image_0791.jpg 0 train airplanes/image_0635.jpg 0 train airplanes/image_0668.jpg 0 train airplanes/image_0314.jpg 0 train airplanes/image_0276.jpg 0 train airplanes/image_0121.jpg 0 train airplanes/image_0438.jpg 0 train airplanes/image_0766.jpg 0 train airplanes/image_0428.jpg 0 train airplanes/image_0568.jpg 0 train airplanes/image_0489.jpg 0 train airplanes/image_0181.jpg 0 train airplanes/image_0742.jpg 0 train airplanes/image_0280.jpg 0 train airplanes/image_0183.jpg 0 train airplanes/image_0605.jpg 0 train airplanes/image_0383.jpg 0 train airplanes/image_0175.jpg 0 train airplanes/image_0002.jpg 0 train airplanes/image_0291.jpg 0 train airplanes/image_0646.jpg 0 train airplanes/image_0165.jpg 0 train airplanes/image_0154.jpg 0 train airplanes/image_0271.jpg 0 train airplanes/image_0193.jpg 0 train airplanes/image_0048.jpg 0 train airplanes/image_0437.jpg 0 train airplanes/image_0480.jpg 0 train airplanes/image_0723.jpg 0 train airplanes/image_0053.jpg 0 train airplanes/image_0255.jpg 0 train airplanes/image_0450.jpg 0 train airplanes/image_0736.jpg 0 train airplanes/image_0616.jpg 0 train airplanes/image_0084.jpg 0 train airplanes/image_0313.jpg 0 train airplanes/image_0757.jpg 0 train airplanes/image_0020.jpg 0 train airplanes/image_0586.jpg 0 train airplanes/image_0494.jpg 0 train airplanes/image_0633.jpg 0 train airplanes/image_0251.jpg 0 train airplanes/image_0631.jpg 0 train airplanes/image_0046.jpg 0 train airplanes/image_0041.jpg 0 train airplanes/image_0027.jpg 0 train airplanes/image_0655.jpg 0 train airplanes/image_0322.jpg 0 train airplanes/image_0458.jpg 0 train airplanes/image_0014.jpg 0 train airplanes/image_0732.jpg 0 train airplanes/image_0058.jpg 0 train airplanes/image_0717.jpg 0 train airplanes/image_0408.jpg 0 train airplanes/image_0329.jpg 0 train airplanes/image_0212.jpg 0 train airplanes/image_0483.jpg 0 train airplanes/image_0611.jpg 0 train airplanes/image_0615.jpg 0 train airplanes/image_0741.jpg 0 train airplanes/image_0163.jpg 0 train airplanes/image_0202.jpg 0 train airplanes/image_0690.jpg 0 train airplanes/image_0104.jpg 0 train airplanes/image_0509.jpg 0 train airplanes/image_0081.jpg 0 train airplanes/image_0272.jpg 0 train airplanes/image_0689.jpg 0 train airplanes/image_0534.jpg 0 train airplanes/image_0400.jpg 0 train airplanes/image_0237.jpg 0 train airplanes/image_0543.jpg 0 train airplanes/image_0667.jpg 0 train airplanes/image_0308.jpg 0 train airplanes/image_0149.jpg 0 train airplanes/image_0500.jpg 0 train airplanes/image_0088.jpg 0 train airplanes/image_0309.jpg 0 train airplanes/image_0026.jpg 0 train airplanes/image_0231.jpg 0 train airplanes/image_0004.jpg 0 train airplanes/image_0015.jpg 0 train airplanes/image_0634.jpg 0 train airplanes/image_0675.jpg 0 train airplanes/image_0770.jpg 0 train airplanes/image_0355.jpg 0 train ant/image_0037.jpg 1 train ant/image_0006.jpg 1 train ant/image_0016.jpg 1 train ant/image_0030.jpg 1 train ant/image_0012.jpg 1 train ant/image_0025.jpg 1 train ant/image_0007.jpg 1 train ant/image_0019.jpg 1 train ant/image_0031.jpg 1 train ant/image_0011.jpg 1 train ant/image_0017.jpg 1 train ant/image_0018.jpg 1 train ant/image_0013.jpg 1 train ant/image_0024.jpg 1 train ant/image_0003.jpg 1 train ant/image_0009.jpg 1 train ant/image_0042.jpg 1 train ant/image_0040.jpg 1 train ant/image_0033.jpg 1 train ant/image_0001.jpg 1 train ant/image_0021.jpg 1 train ant/image_0008.jpg 1 train ant/image_0035.jpg 1 train ant/image_0005.jpg 1 train ant/image_0034.jpg 1 train ant/image_0002.jpg 1 train ant/image_0020.jpg 1 train ant/image_0041.jpg 1 train ant/image_0027.jpg 1 train ant/image_0014.jpg 1 train ant/image_0026.jpg 1 train ant/image_0004.jpg 1 train ant/image_0015.jpg 1 train Faces/image_0316.jpg 2 train Faces/image_0258.jpg 2 train Faces/image_0426.jpg 2 train Faces/image_0302.jpg 2 train Faces/image_0171.jpg 2 train Faces/image_0062.jpg 2 train Faces/image_0346.jpg 2 train Faces/image_0435.jpg 2 train Faces/image_0341.jpg 2 train Faces/image_0340.jpg 2 train Faces/image_0331.jpg 2 train Faces/image_0037.jpg 2 train Faces/image_0049.jpg 2 train Faces/image_0337.jpg 2 train Faces/image_0317.jpg 2 train Faces/image_0357.jpg 2 train Faces/image_0083.jpg 2 train Faces/image_0101.jpg 2 train Faces/image_0051.jpg 2 train Faces/image_0006.jpg 2 train Faces/image_0142.jpg 2 train Faces/image_0250.jpg 2 train Faces/image_0427.jpg 2 train Faces/image_0396.jpg 2 train Faces/image_0332.jpg 2 train Faces/image_0335.jpg 2 train Faces/image_0201.jpg 2 train Faces/image_0016.jpg 2 train Faces/image_0373.jpg 2 train Faces/image_0176.jpg 2 train Faces/image_0411.jpg 2 train Faces/image_0155.jpg 2 train Faces/image_0030.jpg 2 train Faces/image_0091.jpg 2 train Faces/image_0397.jpg 2 train Faces/image_0012.jpg 2 train Faces/image_0066.jpg 2 train Faces/image_0283.jpg 2 train Faces/image_0318.jpg 2 train Faces/image_0224.jpg 2 train Faces/image_0148.jpg 2 train Faces/image_0077.jpg 2 train Faces/image_0129.jpg 2 train Faces/image_0339.jpg 2 train Faces/image_0025.jpg 2 train Faces/image_0328.jpg 2 train Faces/image_0139.jpg 2 train Faces/image_0425.jpg 2 train Faces/image_0292.jpg 2 train Faces/image_0348.jpg 2 train Faces/image_0115.jpg 2 train Faces/image_0123.jpg 2 train Faces/image_0119.jpg 2 train Faces/image_0060.jpg 2 train Faces/image_0269.jpg 2 train Faces/image_0052.jpg 2 train Faces/image_0108.jpg 2 train Faces/image_0239.jpg 2 train Faces/image_0370.jpg 2 train Faces/image_0326.jpg 2 train Faces/image_0223.jpg 2 train Faces/image_0172.jpg 2 train Faces/image_0372.jpg 2 train Faces/image_0344.jpg 2 train Faces/image_0390.jpg 2 train Faces/image_0007.jpg 2 train Faces/image_0213.jpg 2 train Faces/image_0312.jpg 2 train Faces/image_0063.jpg 2 train Faces/image_0019.jpg 2 train Faces/image_0407.jpg 2 train Faces/image_0192.jpg 2 train Faces/image_0384.jpg 2 train Faces/image_0195.jpg 2 train Faces/image_0113.jpg 2 train Faces/image_0127.jpg 2 train Faces/image_0215.jpg 2 train Faces/image_0389.jpg 2 train Faces/image_0031.jpg 2 train Faces/image_0349.jpg 2 train Faces/image_0228.jpg 2 train Faces/image_0380.jpg 2 train Faces/image_0219.jpg 2 train Faces/image_0353.jpg 2 train Faces/image_0011.jpg 2 train Faces/image_0233.jpg 2 train Faces/image_0061.jpg 2 train Faces/image_0090.jpg 2 train Faces/image_0351.jpg 2 train Faces/image_0261.jpg 2 train Faces/image_0229.jpg 2 train Faces/image_0126.jpg 2 train Faces/image_0240.jpg 2 train Faces/image_0305.jpg 2 train Faces/image_0334.jpg 2 train Faces/image_0354.jpg 2 train Faces/image_0218.jpg 2 train Faces/image_0366.jpg 2 train Faces/image_0288.jpg 2 train Faces/image_0216.jpg 2 train Faces/image_0429.jpg 2 train Faces/image_0277.jpg 2 train Faces/image_0358.jpg 2 train Faces/image_0050.jpg 2 train Faces/image_0246.jpg 2 train Faces/image_0361.jpg 2 train Faces/image_0177.jpg 2 train Faces/image_0356.jpg 2 train Faces/image_0196.jpg 2 train Faces/image_0293.jpg 2 train Faces/image_0406.jpg 2 train Faces/image_0347.jpg 2 train Faces/image_0187.jpg 2 train Faces/image_0234.jpg 2 train Faces/image_0045.jpg 2 train Faces/image_0303.jpg 2 train Faces/image_0363.jpg 2 train Faces/image_0017.jpg 2 train Faces/image_0131.jpg 2 train Faces/image_0296.jpg 2 train Faces/image_0150.jpg 2 train Faces/image_0018.jpg 2 train Faces/image_0350.jpg 2 train Faces/image_0209.jpg 2 train Faces/image_0013.jpg 2 train Faces/image_0054.jpg 2 train Faces/image_0094.jpg 2 train Faces/image_0082.jpg 2 train Faces/image_0097.jpg 2 train Faces/image_0130.jpg 2 train Faces/image_0382.jpg 2 train Faces/image_0141.jpg 2 train Faces/image_0144.jpg 2 train Faces/image_0109.jpg 2 train Faces/image_0079.jpg 2 train Faces/image_0306.jpg 2 train Faces/image_0398.jpg 2 train Faces/image_0055.jpg 2 train Faces/image_0367.jpg 2 train Faces/image_0365.jpg 2 train Faces/image_0217.jpg 2 train Faces/image_0343.jpg 2 train Faces/image_0432.jpg 2 train Faces/image_0071.jpg 2 train Faces/image_0282.jpg 2 train Faces/image_0393.jpg 2 train Faces/image_0287.jpg 2 train Faces/image_0198.jpg 2 train Faces/image_0262.jpg 2 train Faces/image_0168.jpg 2 train Faces/image_0368.jpg 2 train Faces/image_0420.jpg 2 train Faces/image_0211.jpg 2 train Faces/image_0270.jpg 2 train Faces/image_0103.jpg 2 train Faces/image_0024.jpg 2 train Faces/image_0297.jpg 2 train Faces/image_0188.jpg 2 train Faces/image_0105.jpg 2 train Faces/image_0074.jpg 2 train Faces/image_0294.jpg 2 train Faces/image_0391.jpg 2 train Faces/image_0189.jpg 2 train Faces/image_0003.jpg 2 train Faces/image_0009.jpg 2 train Faces/image_0387.jpg 2 train Faces/image_0275.jpg 2 train Faces/image_0226.jpg 2 train Faces/image_0185.jpg 2 train Faces/image_0118.jpg 2 train Faces/image_0304.jpg 2 train Faces/image_0278.jpg 2 train Faces/image_0230.jpg 2 train Faces/image_0404.jpg 2 train Faces/image_0134.jpg 2 train Faces/image_0249.jpg 2 train Faces/image_0042.jpg 2 train Faces/image_0352.jpg 2 train Faces/image_0333.jpg 2 train Faces/image_0221.jpg 2 train Faces/image_0093.jpg 2 train Faces/image_0266.jpg 2 train Faces/image_0412.jpg 2 train Faces/image_0301.jpg 2 train Faces/image_0392.jpg 2 train Faces/image_0254.jpg 2 train Faces/image_0321.jpg 2 train Faces/image_0386.jpg 2 train Faces/image_0232.jpg 2 train Faces/image_0324.jpg 2 train Faces/image_0040.jpg 2 train Faces/image_0067.jpg 2 train Faces/image_0310.jpg 2 train Faces/image_0330.jpg 2 train Faces/image_0122.jpg 2 train Faces/image_0205.jpg 2 train Faces/image_0161.jpg 2 train Faces/image_0146.jpg 2 train Faces/image_0300.jpg 2 train Faces/image_0369.jpg 2 train Faces/image_0204.jpg 2 train Faces/image_0125.jpg 2 train Faces/image_0047.jpg 2 train Faces/image_0285.jpg 2 train Faces/image_0160.jpg 2 train Faces/image_0422.jpg 2 train Faces/image_0064.jpg 2 train Faces/image_0070.jpg 2 train Faces/image_0264.jpg 2 train Faces/image_0112.jpg 2 train Faces/image_0078.jpg 2 train Faces/image_0068.jpg 2 train Faces/image_0433.jpg 2 train Faces/image_0085.jpg 2 train Faces/image_0410.jpg 2 train Faces/image_0110.jpg 2 train Faces/image_0259.jpg 2 train Faces/image_0143.jpg 2 train Faces/image_0043.jpg 2 train Faces/image_0162.jpg 2 train Faces/image_0132.jpg 2 train Faces/image_0157.jpg 2 train Faces/image_0206.jpg 2 train Faces/image_0376.jpg 2 train Faces/image_0319.jpg 2 train Faces/image_0371.jpg 2 train Faces/image_0044.jpg 2 train Faces/image_0199.jpg 2 train Faces/image_0033.jpg 2 train Faces/image_0136.jpg 2 train Faces/image_0180.jpg 2 train Faces/image_0260.jpg 2 train Faces/image_0184.jpg 2 train Faces/image_0073.jpg 2 train Faces/image_0395.jpg 2 train Faces/image_0257.jpg 2 train Faces/image_0364.jpg 2 train Faces/image_0001.jpg 2 train Faces/image_0167.jpg 2 train Faces/image_0284.jpg 2 train Faces/image_0375.jpg 2 train Faces/image_0080.jpg 2 train Faces/image_0156.jpg 2 train Faces/image_0021.jpg 2 train Faces/image_0325.jpg 2 train Faces/image_0267.jpg 2 train Faces/image_0299.jpg 2 train Faces/image_0421.jpg 2 train Faces/image_0360.jpg 2 train Faces/image_0095.jpg 2 train Faces/image_0057.jpg 2 train Faces/image_0403.jpg 2 train Faces/image_0114.jpg 2 train Faces/image_0153.jpg 2 train Faces/image_0416.jpg 2 train Faces/image_0098.jpg 2 train Faces/image_0256.jpg 2 train Faces/image_0402.jpg 2 train Faces/image_0203.jpg 2 train Faces/image_0338.jpg 2 train Faces/image_0265.jpg 2 train Faces/image_0147.jpg 2 train Faces/image_0008.jpg 2 train Faces/image_0207.jpg 2 train Faces/image_0208.jpg 2 train Faces/image_0222.jpg 2 train Faces/image_0320.jpg 2 train Faces/image_0227.jpg 2 train Faces/image_0286.jpg 2 train Faces/image_0035.jpg 2 train Faces/image_0399.jpg 2 train Faces/image_0166.jpg 2 train Faces/image_0169.jpg 2 train Faces/image_0191.jpg 2 train Faces/image_0128.jpg 2 train Faces/image_0414.jpg 2 train Faces/image_0102.jpg 2 train Faces/image_0434.jpg 2 train Faces/image_0178.jpg 2 train Faces/image_0431.jpg 2 train Faces/image_0243.jpg 2 train Faces/image_0005.jpg 2 train Faces/image_0263.jpg 2 train Faces/image_0173.jpg 2 train Faces/image_0087.jpg 2 train Faces/image_0417.jpg 2 train Faces/image_0089.jpg 2 train Faces/image_0059.jpg 2 train Faces/image_0253.jpg 2 train Faces/image_0159.jpg 2 train Faces/image_0225.jpg 2 train Faces/image_0186.jpg 2 train Faces/image_0388.jpg 2 train Faces/image_0069.jpg 2 train Faces/image_0315.jpg 2 train Faces/image_0295.jpg 2 train Faces/image_0200.jpg 2 train Faces/image_0099.jpg 2 train Faces/image_0336.jpg 2 train Faces/image_0034.jpg 2 train Faces/image_0409.jpg 2 train Faces/image_0314.jpg 2 train Faces/image_0276.jpg 2 train Faces/image_0121.jpg 2 train Faces/image_0428.jpg 2 train Faces/image_0181.jpg 2 train Faces/image_0280.jpg 2 train Faces/image_0183.jpg 2 train Faces/image_0383.jpg 2 train Faces/image_0175.jpg 2 train Faces/image_0002.jpg 2 train Faces/image_0291.jpg 2 train Faces/image_0165.jpg 2 train Faces/image_0154.jpg 2 train Faces/image_0271.jpg 2 train Faces/image_0193.jpg 2 train Faces/image_0048.jpg 2 train Faces/image_0053.jpg 2 train Faces/image_0255.jpg 2 train Faces/image_0084.jpg 2 train Faces/image_0313.jpg 2 train Faces/image_0020.jpg 2 train Faces/image_0251.jpg 2 train Faces/image_0046.jpg 2 train Faces/image_0041.jpg 2 train Faces/image_0027.jpg 2 train Faces/image_0322.jpg 2 train Faces/image_0014.jpg 2 train Faces/image_0058.jpg 2 train Faces/image_0408.jpg 2 train Faces/image_0329.jpg 2 train Faces/image_0212.jpg 2 train Faces/image_0163.jpg 2 train Faces/image_0202.jpg 2 train Faces/image_0104.jpg 2 train Faces/image_0081.jpg 2 train Faces/image_0272.jpg 2 train Faces/image_0400.jpg 2 train Faces/image_0237.jpg 2 train Faces/image_0308.jpg 2 train Faces/image_0149.jpg 2 train Faces/image_0088.jpg 2 train Faces/image_0309.jpg 2 train Faces/image_0026.jpg 2 train Faces/image_0231.jpg 2 train Faces/image_0004.jpg 2 train Faces/image_0015.jpg 2 train Faces/image_0355.jpg 2 train rooster/image_0037.jpg 3 train rooster/image_0049.jpg 3 train rooster/image_0006.jpg 3 train rooster/image_0016.jpg 3 train rooster/image_0030.jpg 3 train rooster/image_0012.jpg 3 train rooster/image_0025.jpg 3 train rooster/image_0007.jpg 3 train rooster/image_0019.jpg 3 train rooster/image_0031.jpg 3 train rooster/image_0011.jpg 3 train rooster/image_0045.jpg 3 train rooster/image_0017.jpg 3 train rooster/image_0018.jpg 3 train rooster/image_0013.jpg 3 train rooster/image_0024.jpg 3 train rooster/image_0003.jpg 3 train rooster/image_0009.jpg 3 train rooster/image_0042.jpg 3 train rooster/image_0040.jpg 3 train rooster/image_0047.jpg 3 train rooster/image_0043.jpg 3 train rooster/image_0044.jpg 3 train rooster/image_0033.jpg 3 train rooster/image_0001.jpg 3 train rooster/image_0021.jpg 3 train rooster/image_0008.jpg 3 train rooster/image_0035.jpg 3 train rooster/image_0005.jpg 3 train rooster/image_0034.jpg 3 train rooster/image_0002.jpg 3 train rooster/image_0048.jpg 3 train rooster/image_0020.jpg 3 train rooster/image_0046.jpg 3 train rooster/image_0041.jpg 3 train rooster/image_0027.jpg 3 train rooster/image_0014.jpg 3 train rooster/image_0026.jpg 3 train rooster/image_0004.jpg 3 train water_lilly/image_0037.jpg 4 train water_lilly/image_0006.jpg 4 train water_lilly/image_0016.jpg 4 train water_lilly/image_0030.jpg 4 train water_lilly/image_0012.jpg 4 train water_lilly/image_0025.jpg 4 train water_lilly/image_0007.jpg 4 train water_lilly/image_0019.jpg 4 train water_lilly/image_0031.jpg 4 train water_lilly/image_0011.jpg 4 train water_lilly/image_0017.jpg 4 train water_lilly/image_0018.jpg 4 train water_lilly/image_0013.jpg 4 train water_lilly/image_0024.jpg 4 train water_lilly/image_0003.jpg 4 train water_lilly/image_0009.jpg 4 train water_lilly/image_0033.jpg 4 train water_lilly/image_0001.jpg 4 train water_lilly/image_0021.jpg 4 train water_lilly/image_0008.jpg 4 train water_lilly/image_0035.jpg 4 train water_lilly/image_0005.jpg 4 train water_lilly/image_0034.jpg 4 train water_lilly/image_0002.jpg 4 train water_lilly/image_0020.jpg 4 train water_lilly/image_0027.jpg 4 train water_lilly/image_0014.jpg 4 train water_lilly/image_0026.jpg 4 train water_lilly/image_0004.jpg 4 train elephant/image_0062.jpg 5 train elephant/image_0037.jpg 5 train elephant/image_0049.jpg 5 train elephant/image_0051.jpg 5 train elephant/image_0006.jpg 5 train elephant/image_0016.jpg 5 train elephant/image_0030.jpg 5 train elephant/image_0012.jpg 5 train elephant/image_0025.jpg 5 train elephant/image_0060.jpg 5 train elephant/image_0052.jpg 5 train elephant/image_0007.jpg 5 train elephant/image_0063.jpg 5 train elephant/image_0019.jpg 5 train elephant/image_0031.jpg 5 train elephant/image_0011.jpg 5 train elephant/image_0061.jpg 5 train elephant/image_0050.jpg 5 train elephant/image_0045.jpg 5 train elephant/image_0017.jpg 5 train elephant/image_0018.jpg 5 train elephant/image_0013.jpg 5 train elephant/image_0054.jpg 5 train elephant/image_0055.jpg 5 train elephant/image_0024.jpg 5 train elephant/image_0003.jpg 5 train elephant/image_0009.jpg 5 train elephant/image_0042.jpg 5 train elephant/image_0040.jpg 5 train elephant/image_0047.jpg 5 train elephant/image_0064.jpg 5 train elephant/image_0043.jpg 5 train elephant/image_0044.jpg 5 train elephant/image_0033.jpg 5 train elephant/image_0001.jpg 5 train elephant/image_0021.jpg 5 train elephant/image_0057.jpg 5 train elephant/image_0008.jpg 5 train elephant/image_0035.jpg 5 train elephant/image_0005.jpg 5 train elephant/image_0059.jpg 5 train elephant/image_0034.jpg 5 train elephant/image_0002.jpg 5 train elephant/image_0048.jpg 5 train elephant/image_0053.jpg 5 train elephant/image_0020.jpg 5 train elephant/image_0046.jpg 5 train elephant/image_0041.jpg 5 train elephant/image_0027.jpg 5 train elephant/image_0014.jpg 5 train elephant/image_0058.jpg 5 train umbrella/image_0062.jpg 6 train umbrella/image_0037.jpg 6 train umbrella/image_0049.jpg 6 train umbrella/image_0051.jpg 6 train umbrella/image_0006.jpg 6 train umbrella/image_0016.jpg 6 train umbrella/image_0030.jpg 6 train umbrella/image_0012.jpg 6 train umbrella/image_0066.jpg 6 train umbrella/image_0025.jpg 6 train umbrella/image_0060.jpg 6 train umbrella/image_0052.jpg 6 train umbrella/image_0007.jpg 6 train umbrella/image_0063.jpg 6 train umbrella/image_0019.jpg 6 train umbrella/image_0031.jpg 6 train umbrella/image_0011.jpg 6 train umbrella/image_0061.jpg 6 train umbrella/image_0050.jpg 6 train umbrella/image_0045.jpg 6 train umbrella/image_0017.jpg 6 train umbrella/image_0018.jpg 6 train umbrella/image_0013.jpg 6 train umbrella/image_0054.jpg 6 train umbrella/image_0055.jpg 6 train umbrella/image_0071.jpg 6 train umbrella/image_0024.jpg 6 train umbrella/image_0074.jpg 6 train umbrella/image_0003.jpg 6 train umbrella/image_0009.jpg 6 train umbrella/image_0042.jpg 6 train umbrella/image_0040.jpg 6 train umbrella/image_0067.jpg 6 train umbrella/image_0047.jpg 6 train umbrella/image_0064.jpg 6 train umbrella/image_0070.jpg 6 train umbrella/image_0068.jpg 6 train umbrella/image_0043.jpg 6 train umbrella/image_0044.jpg 6 train umbrella/image_0033.jpg 6 train umbrella/image_0073.jpg 6 train umbrella/image_0001.jpg 6 train umbrella/image_0021.jpg 6 train umbrella/image_0057.jpg 6 train umbrella/image_0008.jpg 6 train umbrella/image_0035.jpg 6 train umbrella/image_0005.jpg 6 train umbrella/image_0059.jpg 6 train umbrella/image_0069.jpg 6 train umbrella/image_0034.jpg 6 train umbrella/image_0002.jpg 6 train umbrella/image_0048.jpg 6 train umbrella/image_0053.jpg 6 train umbrella/image_0020.jpg 6 train umbrella/image_0046.jpg 6 train umbrella/image_0041.jpg 6 train umbrella/image_0027.jpg 6 train umbrella/image_0014.jpg 6 train umbrella/image_0058.jpg 6 train umbrella/image_0026.jpg 6 train dolphin/image_0062.jpg 7 train dolphin/image_0037.jpg 7 train dolphin/image_0049.jpg 7 train dolphin/image_0051.jpg 7 train dolphin/image_0006.jpg 7 train dolphin/image_0016.jpg 7 train dolphin/image_0030.jpg 7 train dolphin/image_0012.jpg 7 train dolphin/image_0025.jpg 7 train dolphin/image_0060.jpg 7 train dolphin/image_0052.jpg 7 train dolphin/image_0007.jpg 7 train dolphin/image_0063.jpg 7 train dolphin/image_0019.jpg 7 train dolphin/image_0031.jpg 7 train dolphin/image_0011.jpg 7 train dolphin/image_0061.jpg 7 train dolphin/image_0050.jpg 7 train dolphin/image_0045.jpg 7 train dolphin/image_0017.jpg 7 train dolphin/image_0018.jpg 7 train dolphin/image_0013.jpg 7 train dolphin/image_0054.jpg 7 train dolphin/image_0055.jpg 7 train dolphin/image_0024.jpg 7 train dolphin/image_0003.jpg 7 train dolphin/image_0009.jpg 7 train dolphin/image_0042.jpg 7 train dolphin/image_0040.jpg 7 train dolphin/image_0047.jpg 7 train dolphin/image_0064.jpg 7 train dolphin/image_0043.jpg 7 train dolphin/image_0044.jpg 7 train dolphin/image_0033.jpg 7 train dolphin/image_0001.jpg 7 train dolphin/image_0021.jpg 7 train dolphin/image_0057.jpg 7 train dolphin/image_0008.jpg 7 train dolphin/image_0035.jpg 7 train dolphin/image_0005.jpg 7 train dolphin/image_0059.jpg 7 train dolphin/image_0034.jpg 7 train dolphin/image_0002.jpg 7 train dolphin/image_0048.jpg 7 train dolphin/image_0053.jpg 7 train dolphin/image_0020.jpg 7 train dolphin/image_0046.jpg 7 train dolphin/image_0041.jpg 7 train dolphin/image_0027.jpg 7 train dolphin/image_0014.jpg 7 train dolphin/image_0058.jpg 7 train dolphin/image_0026.jpg 7 train gerenuk/image_0006.jpg 8 train gerenuk/image_0016.jpg 8 train gerenuk/image_0030.jpg 8 train gerenuk/image_0012.jpg 8 train gerenuk/image_0025.jpg 8 train gerenuk/image_0007.jpg 8 train gerenuk/image_0019.jpg 8 train gerenuk/image_0031.jpg 8 train gerenuk/image_0011.jpg 8 train gerenuk/image_0017.jpg 8 train gerenuk/image_0018.jpg 8 train gerenuk/image_0013.jpg 8 train gerenuk/image_0024.jpg 8 train gerenuk/image_0003.jpg 8 train gerenuk/image_0009.jpg 8 train gerenuk/image_0033.jpg 8 train gerenuk/image_0001.jpg 8 train gerenuk/image_0021.jpg 8 train gerenuk/image_0008.jpg 8 train gerenuk/image_0005.jpg 8 train gerenuk/image_0034.jpg 8 train gerenuk/image_0002.jpg 8 train gerenuk/image_0020.jpg 8 train gerenuk/image_0027.jpg 8 train gerenuk/image_0014.jpg 8 train gerenuk/image_0026.jpg 8 train gerenuk/image_0004.jpg 8 train dragonfly/image_0062.jpg 9 train dragonfly/image_0037.jpg 9 train dragonfly/image_0049.jpg 9 train dragonfly/image_0051.jpg 9 train dragonfly/image_0006.jpg 9 train dragonfly/image_0016.jpg 9 train dragonfly/image_0030.jpg 9 train dragonfly/image_0012.jpg 9 train dragonfly/image_0066.jpg 9 train dragonfly/image_0025.jpg 9 train dragonfly/image_0060.jpg 9 train dragonfly/image_0052.jpg 9 train dragonfly/image_0007.jpg 9 train dragonfly/image_0063.jpg 9 train dragonfly/image_0019.jpg 9 train dragonfly/image_0031.jpg 9 train dragonfly/image_0011.jpg 9 train dragonfly/image_0061.jpg 9 train dragonfly/image_0050.jpg 9 train dragonfly/image_0045.jpg 9 train dragonfly/image_0017.jpg 9 train dragonfly/image_0018.jpg 9 train dragonfly/image_0013.jpg 9 train dragonfly/image_0054.jpg 9 train dragonfly/image_0055.jpg 9 train dragonfly/image_0024.jpg 9 train dragonfly/image_0003.jpg 9 train dragonfly/image_0009.jpg 9 train dragonfly/image_0042.jpg 9 train dragonfly/image_0040.jpg 9 train dragonfly/image_0067.jpg 9 train dragonfly/image_0047.jpg 9 train dragonfly/image_0064.jpg 9 train dragonfly/image_0068.jpg 9 train dragonfly/image_0043.jpg 9 train dragonfly/image_0044.jpg 9 train dragonfly/image_0033.jpg 9 train dragonfly/image_0001.jpg 9 train dragonfly/image_0021.jpg 9 train dragonfly/image_0057.jpg 9 train dragonfly/image_0008.jpg 9 train dragonfly/image_0035.jpg 9 train dragonfly/image_0005.jpg 9 train dragonfly/image_0059.jpg 9 train dragonfly/image_0034.jpg 9 train dragonfly/image_0002.jpg 9 train dragonfly/image_0048.jpg 9 train dragonfly/image_0053.jpg 9 train dragonfly/image_0020.jpg 9 train dragonfly/image_0046.jpg 9 train dragonfly/image_0041.jpg 9 train dragonfly/image_0027.jpg 9 train dragonfly/image_0014.jpg 9 train dragonfly/image_0058.jpg 9 train yin_yang/image_0037.jpg 10 train yin_yang/image_0049.jpg 10 train yin_yang/image_0051.jpg 10 train yin_yang/image_0006.jpg 10 train yin_yang/image_0016.jpg 10 train yin_yang/image_0030.jpg 10 train yin_yang/image_0012.jpg 10 train yin_yang/image_0025.jpg 10 train yin_yang/image_0060.jpg 10 train yin_yang/image_0052.jpg 10 train yin_yang/image_0007.jpg 10 train yin_yang/image_0019.jpg 10 train yin_yang/image_0031.jpg 10 train yin_yang/image_0011.jpg 10 train yin_yang/image_0050.jpg 10 train yin_yang/image_0045.jpg 10 train yin_yang/image_0017.jpg 10 train yin_yang/image_0018.jpg 10 train yin_yang/image_0013.jpg 10 train yin_yang/image_0054.jpg 10 train yin_yang/image_0055.jpg 10 train yin_yang/image_0024.jpg 10 train yin_yang/image_0003.jpg 10 train yin_yang/image_0009.jpg 10 train yin_yang/image_0042.jpg 10 train yin_yang/image_0040.jpg 10 train yin_yang/image_0047.jpg 10 train yin_yang/image_0043.jpg 10 train yin_yang/image_0044.jpg 10 train yin_yang/image_0033.jpg 10 train yin_yang/image_0001.jpg 10 train yin_yang/image_0021.jpg 10 train yin_yang/image_0057.jpg 10 train yin_yang/image_0008.jpg 10 train yin_yang/image_0035.jpg 10 train yin_yang/image_0005.jpg 10 train yin_yang/image_0059.jpg 10 train yin_yang/image_0034.jpg 10 train yin_yang/image_0002.jpg 10 train yin_yang/image_0048.jpg 10 train yin_yang/image_0053.jpg 10 train yin_yang/image_0020.jpg 10 train yin_yang/image_0046.jpg 10 train yin_yang/image_0041.jpg 10 train yin_yang/image_0027.jpg 10 train yin_yang/image_0014.jpg 10 train yin_yang/image_0058.jpg 10 train yin_yang/image_0026.jpg 10 train starfish/image_0062.jpg 11 train starfish/image_0037.jpg 11 train starfish/image_0049.jpg 11 train starfish/image_0083.jpg 11 train starfish/image_0051.jpg 11 train starfish/image_0006.jpg 11 train starfish/image_0016.jpg 11 train starfish/image_0030.jpg 11 train starfish/image_0012.jpg 11 train starfish/image_0066.jpg 11 train starfish/image_0077.jpg 11 train starfish/image_0025.jpg 11 train starfish/image_0060.jpg 11 train starfish/image_0052.jpg 11 train starfish/image_0007.jpg 11 train starfish/image_0063.jpg 11 train starfish/image_0019.jpg 11 train starfish/image_0031.jpg 11 train starfish/image_0011.jpg 11 train starfish/image_0061.jpg 11 train starfish/image_0050.jpg 11 train starfish/image_0045.jpg 11 train starfish/image_0017.jpg 11 train starfish/image_0018.jpg 11 train starfish/image_0013.jpg 11 train starfish/image_0054.jpg 11 train starfish/image_0082.jpg 11 train starfish/image_0079.jpg 11 train starfish/image_0055.jpg 11 train starfish/image_0071.jpg 11 train starfish/image_0024.jpg 11 train starfish/image_0074.jpg 11 train starfish/image_0003.jpg 11 train starfish/image_0009.jpg 11 train starfish/image_0042.jpg 11 train starfish/image_0040.jpg 11 train starfish/image_0067.jpg 11 train starfish/image_0047.jpg 11 train starfish/image_0064.jpg 11 train starfish/image_0070.jpg 11 train starfish/image_0078.jpg 11 train starfish/image_0068.jpg 11 train starfish/image_0085.jpg 11 train starfish/image_0043.jpg 11 train starfish/image_0044.jpg 11 train starfish/image_0033.jpg 11 train starfish/image_0073.jpg 11 train starfish/image_0001.jpg 11 train starfish/image_0080.jpg 11 train starfish/image_0021.jpg 11 train starfish/image_0057.jpg 11 train starfish/image_0008.jpg 11 train starfish/image_0035.jpg 11 train starfish/image_0005.jpg 11 train starfish/image_0059.jpg 11 train starfish/image_0069.jpg 11 train starfish/image_0034.jpg 11 train starfish/image_0002.jpg 11 train starfish/image_0048.jpg 11 train starfish/image_0053.jpg 11 train starfish/image_0084.jpg 11 train starfish/image_0020.jpg 11 train starfish/image_0046.jpg 11 train starfish/image_0041.jpg 11 train starfish/image_0027.jpg 11 train starfish/image_0014.jpg 11 train starfish/image_0058.jpg 11 train starfish/image_0081.jpg 11 train ceiling_fan/image_0037.jpg 12 train ceiling_fan/image_0006.jpg 12 train ceiling_fan/image_0016.jpg 12 train ceiling_fan/image_0030.jpg 12 train ceiling_fan/image_0012.jpg 12 train ceiling_fan/image_0025.jpg 12 train ceiling_fan/image_0007.jpg 12 train ceiling_fan/image_0019.jpg 12 train ceiling_fan/image_0031.jpg 12 train ceiling_fan/image_0011.jpg 12 train ceiling_fan/image_0045.jpg 12 train ceiling_fan/image_0017.jpg 12 train ceiling_fan/image_0018.jpg 12 train ceiling_fan/image_0013.jpg 12 train ceiling_fan/image_0024.jpg 12 train ceiling_fan/image_0003.jpg 12 train ceiling_fan/image_0009.jpg 12 train ceiling_fan/image_0042.jpg 12 train ceiling_fan/image_0040.jpg 12 train ceiling_fan/image_0047.jpg 12 train ceiling_fan/image_0043.jpg 12 train ceiling_fan/image_0044.jpg 12 train ceiling_fan/image_0033.jpg 12 train ceiling_fan/image_0001.jpg 12 train ceiling_fan/image_0021.jpg 12 train ceiling_fan/image_0008.jpg 12 train ceiling_fan/image_0035.jpg 12 train ceiling_fan/image_0005.jpg 12 train ceiling_fan/image_0034.jpg 12 train ceiling_fan/image_0002.jpg 12 train ceiling_fan/image_0020.jpg 12 train ceiling_fan/image_0046.jpg 12 train ceiling_fan/image_0041.jpg 12 train ceiling_fan/image_0027.jpg 12 train ceiling_fan/image_0014.jpg 12 train ceiling_fan/image_0026.jpg 12 train ceiling_fan/image_0004.jpg 12 train soccer_ball/image_0062.jpg 13 train soccer_ball/image_0037.jpg 13 train soccer_ball/image_0049.jpg 13 train soccer_ball/image_0051.jpg 13 train soccer_ball/image_0006.jpg 13 train soccer_ball/image_0016.jpg 13 train soccer_ball/image_0030.jpg 13 train soccer_ball/image_0012.jpg 13 train soccer_ball/image_0025.jpg 13 train soccer_ball/image_0060.jpg 13 train soccer_ball/image_0052.jpg 13 train soccer_ball/image_0007.jpg 13 train soccer_ball/image_0063.jpg 13 train soccer_ball/image_0019.jpg 13 train soccer_ball/image_0031.jpg 13 train soccer_ball/image_0011.jpg 13 train soccer_ball/image_0061.jpg 13 train soccer_ball/image_0050.jpg 13 train soccer_ball/image_0045.jpg 13 train soccer_ball/image_0017.jpg 13 train soccer_ball/image_0018.jpg 13 train soccer_ball/image_0013.jpg 13 train soccer_ball/image_0054.jpg 13 train soccer_ball/image_0055.jpg 13 train soccer_ball/image_0024.jpg 13 train soccer_ball/image_0003.jpg 13 train soccer_ball/image_0009.jpg 13 train soccer_ball/image_0042.jpg 13 train soccer_ball/image_0040.jpg 13 train soccer_ball/image_0047.jpg 13 train soccer_ball/image_0064.jpg 13 train soccer_ball/image_0043.jpg 13 train soccer_ball/image_0044.jpg 13 train soccer_ball/image_0033.jpg 13 train soccer_ball/image_0001.jpg 13 train soccer_ball/image_0021.jpg 13 train soccer_ball/image_0057.jpg 13 train soccer_ball/image_0008.jpg 13 train soccer_ball/image_0035.jpg 13 train soccer_ball/image_0005.jpg 13 train soccer_ball/image_0059.jpg 13 train soccer_ball/image_0034.jpg 13 train soccer_ball/image_0002.jpg 13 train soccer_ball/image_0048.jpg 13 train soccer_ball/image_0053.jpg 13 train soccer_ball/image_0020.jpg 13 train soccer_ball/image_0046.jpg 13 train soccer_ball/image_0041.jpg 13 train soccer_ball/image_0027.jpg 13 train soccer_ball/image_0014.jpg 13 train soccer_ball/image_0058.jpg 13 train Leopards/image_0171.jpg 14 train Leopards/image_0062.jpg 14 train Leopards/image_0037.jpg 14 train Leopards/image_0049.jpg 14 train Leopards/image_0083.jpg 14 train Leopards/image_0101.jpg 14 train Leopards/image_0051.jpg 14 train Leopards/image_0006.jpg 14 train Leopards/image_0142.jpg 14 train Leopards/image_0016.jpg 14 train Leopards/image_0176.jpg 14 train Leopards/image_0155.jpg 14 train Leopards/image_0030.jpg 14 train Leopards/image_0091.jpg 14 train Leopards/image_0012.jpg 14 train Leopards/image_0066.jpg 14 train Leopards/image_0148.jpg 14 train Leopards/image_0077.jpg 14 train Leopards/image_0129.jpg 14 train Leopards/image_0025.jpg 14 train Leopards/image_0139.jpg 14 train Leopards/image_0115.jpg 14 train Leopards/image_0123.jpg 14 train Leopards/image_0119.jpg 14 train Leopards/image_0060.jpg 14 train Leopards/image_0052.jpg 14 train Leopards/image_0108.jpg 14 train Leopards/image_0172.jpg 14 train Leopards/image_0007.jpg 14 train Leopards/image_0063.jpg 14 train Leopards/image_0019.jpg 14 train Leopards/image_0192.jpg 14 train Leopards/image_0195.jpg 14 train Leopards/image_0113.jpg 14 train Leopards/image_0127.jpg 14 train Leopards/image_0031.jpg 14 train Leopards/image_0011.jpg 14 train Leopards/image_0061.jpg 14 train Leopards/image_0090.jpg 14 train Leopards/image_0126.jpg 14 train Leopards/image_0050.jpg 14 train Leopards/image_0177.jpg 14 train Leopards/image_0196.jpg 14 train Leopards/image_0187.jpg 14 train Leopards/image_0045.jpg 14 train Leopards/image_0017.jpg 14 train Leopards/image_0131.jpg 14 train Leopards/image_0150.jpg 14 train Leopards/image_0018.jpg 14 train Leopards/image_0013.jpg 14 train Leopards/image_0054.jpg 14 train Leopards/image_0094.jpg 14 train Leopards/image_0082.jpg 14 train Leopards/image_0097.jpg 14 train Leopards/image_0130.jpg 14 train Leopards/image_0141.jpg 14 train Leopards/image_0144.jpg 14 train Leopards/image_0109.jpg 14 train Leopards/image_0079.jpg 14 train Leopards/image_0055.jpg 14 train Leopards/image_0071.jpg 14 train Leopards/image_0198.jpg 14 train Leopards/image_0168.jpg 14 train Leopards/image_0103.jpg 14 train Leopards/image_0024.jpg 14 train Leopards/image_0188.jpg 14 train Leopards/image_0105.jpg 14 train Leopards/image_0074.jpg 14 train Leopards/image_0189.jpg 14 train Leopards/image_0003.jpg 14 train Leopards/image_0009.jpg 14 train Leopards/image_0185.jpg 14 train Leopards/image_0118.jpg 14 train Leopards/image_0134.jpg 14 train Leopards/image_0042.jpg 14 train Leopards/image_0093.jpg 14 train Leopards/image_0040.jpg 14 train Leopards/image_0067.jpg 14 train Leopards/image_0122.jpg 14 train Leopards/image_0161.jpg 14 train Leopards/image_0146.jpg 14 train Leopards/image_0125.jpg 14 train Leopards/image_0047.jpg 14 train Leopards/image_0160.jpg 14 train Leopards/image_0064.jpg 14 train Leopards/image_0070.jpg 14 train Leopards/image_0112.jpg 14 train Leopards/image_0078.jpg 14 train Leopards/image_0068.jpg 14 train Leopards/image_0085.jpg 14 train Leopards/image_0110.jpg 14 train Leopards/image_0143.jpg 14 train Leopards/image_0043.jpg 14 train Leopards/image_0162.jpg 14 train Leopards/image_0132.jpg 14 train Leopards/image_0157.jpg 14 train Leopards/image_0044.jpg 14 train Leopards/image_0199.jpg 14 train Leopards/image_0033.jpg 14 train Leopards/image_0136.jpg 14 train Leopards/image_0180.jpg 14 train Leopards/image_0184.jpg 14 train Leopards/image_0073.jpg 14 train Leopards/image_0001.jpg 14 train Leopards/image_0167.jpg 14 train Leopards/image_0080.jpg 14 train Leopards/image_0156.jpg 14 train Leopards/image_0021.jpg 14 train Leopards/image_0095.jpg 14 train Leopards/image_0057.jpg 14 train Leopards/image_0114.jpg 14 train Leopards/image_0153.jpg 14 train Leopards/image_0098.jpg 14 train Leopards/image_0147.jpg 14 train Leopards/image_0008.jpg 14 train Leopards/image_0035.jpg 14 train Leopards/image_0166.jpg 14 train Leopards/image_0169.jpg 14 train Leopards/image_0191.jpg 14 train Leopards/image_0128.jpg 14 train Leopards/image_0102.jpg 14 train Leopards/image_0178.jpg 14 train Leopards/image_0005.jpg 14 train Leopards/image_0173.jpg 14 train Leopards/image_0087.jpg 14 train Leopards/image_0089.jpg 14 train Leopards/image_0059.jpg 14 train Leopards/image_0159.jpg 14 train Leopards/image_0186.jpg 14 train Leopards/image_0069.jpg 14 train Leopards/image_0200.jpg 14 train Leopards/image_0099.jpg 14 train Leopards/image_0034.jpg 14 train Leopards/image_0121.jpg 14 train Leopards/image_0181.jpg 14 train Leopards/image_0183.jpg 14 train Leopards/image_0175.jpg 14 train Leopards/image_0002.jpg 14 train Leopards/image_0165.jpg 14 train Leopards/image_0154.jpg 14 train Leopards/image_0193.jpg 14 train Leopards/image_0048.jpg 14 train Leopards/image_0053.jpg 14 train Leopards/image_0084.jpg 14 train Leopards/image_0020.jpg 14 train Leopards/image_0046.jpg 14 train Leopards/image_0041.jpg 14 train Leopards/image_0027.jpg 14 train Leopards/image_0014.jpg 14 train Leopards/image_0058.jpg 14 train Leopards/image_0163.jpg 14 train Leopards/image_0104.jpg 14 train Leopards/image_0081.jpg 14 train Leopards/image_0149.jpg 14 train Leopards/image_0088.jpg 14 train Leopards/image_0026.jpg 14 train Leopards/image_0004.jpg 14 train Leopards/image_0015.jpg 14 train Leopards/image_0029.jpg 14 train Leopards/image_0120.jpg 14 train scorpion/image_0062.jpg 15 train scorpion/image_0037.jpg 15 train scorpion/image_0049.jpg 15 train scorpion/image_0083.jpg 15 train scorpion/image_0051.jpg 15 train scorpion/image_0006.jpg 15 train scorpion/image_0016.jpg 15 train scorpion/image_0030.jpg 15 train scorpion/image_0012.jpg 15 train scorpion/image_0066.jpg 15 train scorpion/image_0077.jpg 15 train scorpion/image_0025.jpg 15 train scorpion/image_0060.jpg 15 train scorpion/image_0052.jpg 15 train scorpion/image_0007.jpg 15 train scorpion/image_0063.jpg 15 train scorpion/image_0019.jpg 15 train scorpion/image_0031.jpg 15 train scorpion/image_0011.jpg 15 train scorpion/image_0061.jpg 15 train scorpion/image_0050.jpg 15 train scorpion/image_0045.jpg 15 train scorpion/image_0017.jpg 15 train scorpion/image_0018.jpg 15 train scorpion/image_0013.jpg 15 train scorpion/image_0054.jpg 15 train scorpion/image_0082.jpg 15 train scorpion/image_0079.jpg 15 train scorpion/image_0055.jpg 15 train scorpion/image_0071.jpg 15 train scorpion/image_0024.jpg 15 train scorpion/image_0074.jpg 15 train scorpion/image_0003.jpg 15 train scorpion/image_0009.jpg 15 train scorpion/image_0042.jpg 15 train scorpion/image_0040.jpg 15 train scorpion/image_0067.jpg 15 train scorpion/image_0047.jpg 15 train scorpion/image_0064.jpg 15 train scorpion/image_0070.jpg 15 train scorpion/image_0078.jpg 15 train scorpion/image_0068.jpg 15 train scorpion/image_0043.jpg 15 train scorpion/image_0044.jpg 15 train scorpion/image_0033.jpg 15 train scorpion/image_0073.jpg 15 train scorpion/image_0001.jpg 15 train scorpion/image_0080.jpg 15 train scorpion/image_0021.jpg 15 train scorpion/image_0057.jpg 15 train scorpion/image_0008.jpg 15 train scorpion/image_0035.jpg 15 train scorpion/image_0005.jpg 15 train scorpion/image_0059.jpg 15 train scorpion/image_0069.jpg 15 train scorpion/image_0034.jpg 15 train scorpion/image_0002.jpg 15 train scorpion/image_0048.jpg 15 train scorpion/image_0053.jpg 15 train scorpion/image_0084.jpg 15 train scorpion/image_0020.jpg 15 train scorpion/image_0046.jpg 15 train scorpion/image_0041.jpg 15 train scorpion/image_0027.jpg 15 train scorpion/image_0014.jpg 15 train scorpion/image_0058.jpg 15 train scorpion/image_0081.jpg 15 train llama/image_0062.jpg 16 train llama/image_0037.jpg 16 train llama/image_0049.jpg 16 train llama/image_0051.jpg 16 train llama/image_0006.jpg 16 train llama/image_0016.jpg 16 train llama/image_0030.jpg 16 train llama/image_0012.jpg 16 train llama/image_0066.jpg 16 train llama/image_0077.jpg 16 train llama/image_0025.jpg 16 train llama/image_0060.jpg 16 train llama/image_0052.jpg 16 train llama/image_0007.jpg 16 train llama/image_0063.jpg 16 train llama/image_0019.jpg 16 train llama/image_0031.jpg 16 train llama/image_0011.jpg 16 train llama/image_0061.jpg 16 train llama/image_0050.jpg 16 train llama/image_0045.jpg 16 train llama/image_0017.jpg 16 train llama/image_0018.jpg 16 train llama/image_0013.jpg 16 train llama/image_0054.jpg 16 train llama/image_0055.jpg 16 train llama/image_0071.jpg 16 train llama/image_0024.jpg 16 train llama/image_0074.jpg 16 train llama/image_0003.jpg 16 train llama/image_0009.jpg 16 train llama/image_0042.jpg 16 train llama/image_0040.jpg 16 train llama/image_0067.jpg 16 train llama/image_0047.jpg 16 train llama/image_0064.jpg 16 train llama/image_0070.jpg 16 train llama/image_0078.jpg 16 train llama/image_0068.jpg 16 train llama/image_0043.jpg 16 train llama/image_0044.jpg 16 train llama/image_0033.jpg 16 train llama/image_0073.jpg 16 train llama/image_0001.jpg 16 train llama/image_0021.jpg 16 train llama/image_0057.jpg 16 train llama/image_0008.jpg 16 train llama/image_0035.jpg 16 train llama/image_0005.jpg 16 train llama/image_0059.jpg 16 train llama/image_0069.jpg 16 train llama/image_0034.jpg 16 train llama/image_0002.jpg 16 train llama/image_0048.jpg 16 train llama/image_0053.jpg 16 train llama/image_0020.jpg 16 train llama/image_0046.jpg 16 train llama/image_0041.jpg 16 train llama/image_0027.jpg 16 train llama/image_0014.jpg 16 train llama/image_0058.jpg 16 train llama/image_0026.jpg 16 train wild_cat/image_0006.jpg 17 train wild_cat/image_0016.jpg 17 train wild_cat/image_0030.jpg 17 train wild_cat/image_0012.jpg 17 train wild_cat/image_0025.jpg 17 train wild_cat/image_0007.jpg 17 train wild_cat/image_0019.jpg 17 train wild_cat/image_0031.jpg 17 train wild_cat/image_0011.jpg 17 train wild_cat/image_0017.jpg 17 train wild_cat/image_0018.jpg 17 train wild_cat/image_0013.jpg 17 train wild_cat/image_0024.jpg 17 train wild_cat/image_0003.jpg 17 train wild_cat/image_0009.jpg 17 train wild_cat/image_0033.jpg 17 train wild_cat/image_0001.jpg 17 train wild_cat/image_0021.jpg 17 train wild_cat/image_0008.jpg 17 train wild_cat/image_0005.jpg 17 train wild_cat/image_0034.jpg 17 train wild_cat/image_0002.jpg 17 train wild_cat/image_0020.jpg 17 train wild_cat/image_0027.jpg 17 train wild_cat/image_0014.jpg 17 train wild_cat/image_0026.jpg 17 train wild_cat/image_0004.jpg 17 train lamp/image_0037.jpg 18 train lamp/image_0049.jpg 18 train lamp/image_0051.jpg 18 train lamp/image_0006.jpg 18 train lamp/image_0016.jpg 18 train lamp/image_0030.jpg 18 train lamp/image_0012.jpg 18 train lamp/image_0025.jpg 18 train lamp/image_0060.jpg 18 train lamp/image_0052.jpg 18 train lamp/image_0007.jpg 18 train lamp/image_0019.jpg 18 train lamp/image_0031.jpg 18 train lamp/image_0011.jpg 18 train lamp/image_0061.jpg 18 train lamp/image_0050.jpg 18 train lamp/image_0045.jpg 18 train lamp/image_0017.jpg 18 train lamp/image_0018.jpg 18 train lamp/image_0013.jpg 18 train lamp/image_0054.jpg 18 train lamp/image_0055.jpg 18 train lamp/image_0024.jpg 18 train lamp/image_0003.jpg 18 train lamp/image_0009.jpg 18 train lamp/image_0042.jpg 18 train lamp/image_0040.jpg 18 train lamp/image_0047.jpg 18 train lamp/image_0043.jpg 18 train lamp/image_0044.jpg 18 train lamp/image_0033.jpg 18 train lamp/image_0001.jpg 18 train lamp/image_0021.jpg 18 train lamp/image_0057.jpg 18 train lamp/image_0008.jpg 18 train lamp/image_0035.jpg 18 train lamp/image_0005.jpg 18 train lamp/image_0059.jpg 18 train lamp/image_0034.jpg 18 train lamp/image_0002.jpg 18 train lamp/image_0048.jpg 18 train lamp/image_0053.jpg 18 train lamp/image_0020.jpg 18 train lamp/image_0046.jpg 18 train lamp/image_0041.jpg 18 train lamp/image_0027.jpg 18 train lamp/image_0014.jpg 18 train lamp/image_0058.jpg 18 train lotus/image_0062.jpg 19 train lotus/image_0037.jpg 19 train lotus/image_0049.jpg 19 train lotus/image_0051.jpg 19 train lotus/image_0006.jpg 19 train lotus/image_0016.jpg 19 train lotus/image_0030.jpg 19 train lotus/image_0012.jpg 19 train lotus/image_0066.jpg 19 train lotus/image_0025.jpg 19 train lotus/image_0060.jpg 19 train lotus/image_0052.jpg 19 train lotus/image_0007.jpg 19 train lotus/image_0063.jpg 19 train lotus/image_0019.jpg 19 train lotus/image_0031.jpg 19 train lotus/image_0011.jpg 19 train lotus/image_0061.jpg 19 train lotus/image_0050.jpg 19 train lotus/image_0045.jpg 19 train lotus/image_0017.jpg 19 train lotus/image_0018.jpg 19 train lotus/image_0013.jpg 19 train lotus/image_0054.jpg 19 train lotus/image_0055.jpg 19 train lotus/image_0024.jpg 19 train lotus/image_0003.jpg 19 train lotus/image_0009.jpg 19 train lotus/image_0042.jpg 19 train lotus/image_0040.jpg 19 train lotus/image_0047.jpg 19 train lotus/image_0064.jpg 19 train lotus/image_0043.jpg 19 train lotus/image_0044.jpg 19 train lotus/image_0033.jpg 19 train lotus/image_0001.jpg 19 train lotus/image_0021.jpg 19 train lotus/image_0057.jpg 19 train lotus/image_0008.jpg 19 train lotus/image_0035.jpg 19 train lotus/image_0005.jpg 19 train lotus/image_0059.jpg 19 train lotus/image_0034.jpg 19 train lotus/image_0002.jpg 19 train lotus/image_0048.jpg 19 train lotus/image_0053.jpg 19 train lotus/image_0020.jpg 19 train lotus/image_0046.jpg 19 train lotus/image_0041.jpg 19 train lotus/image_0027.jpg 19 train lotus/image_0014.jpg 19 train lotus/image_0058.jpg 19 train crocodile_head/image_0037.jpg 20 train crocodile_head/image_0049.jpg 20 train crocodile_head/image_0051.jpg 20 train crocodile_head/image_0006.jpg 20 train crocodile_head/image_0016.jpg 20 train crocodile_head/image_0030.jpg 20 train crocodile_head/image_0012.jpg 20 train crocodile_head/image_0025.jpg 20 train crocodile_head/image_0007.jpg 20 train crocodile_head/image_0019.jpg 20 train crocodile_head/image_0031.jpg 20 train crocodile_head/image_0011.jpg 20 train crocodile_head/image_0050.jpg 20 train crocodile_head/image_0045.jpg 20 train crocodile_head/image_0017.jpg 20 train crocodile_head/image_0018.jpg 20 train crocodile_head/image_0013.jpg 20 train crocodile_head/image_0024.jpg 20 train crocodile_head/image_0003.jpg 20 train crocodile_head/image_0009.jpg 20 train crocodile_head/image_0042.jpg 20 train crocodile_head/image_0040.jpg 20 train crocodile_head/image_0047.jpg 20 train crocodile_head/image_0043.jpg 20 train crocodile_head/image_0044.jpg 20 train crocodile_head/image_0033.jpg 20 train crocodile_head/image_0001.jpg 20 train crocodile_head/image_0021.jpg 20 train crocodile_head/image_0008.jpg 20 train crocodile_head/image_0035.jpg 20 train crocodile_head/image_0005.jpg 20 train crocodile_head/image_0034.jpg 20 train crocodile_head/image_0002.jpg 20 train crocodile_head/image_0048.jpg 20 train crocodile_head/image_0020.jpg 20 train crocodile_head/image_0046.jpg 20 train crocodile_head/image_0041.jpg 20 train crocodile_head/image_0027.jpg 20 train crocodile_head/image_0014.jpg 20 train crocodile_head/image_0026.jpg 20 train butterfly/image_0062.jpg 21 train butterfly/image_0037.jpg 21 train butterfly/image_0049.jpg 21 train butterfly/image_0083.jpg 21 train butterfly/image_0051.jpg 21 train butterfly/image_0006.jpg 21 train butterfly/image_0016.jpg 21 train butterfly/image_0030.jpg 21 train butterfly/image_0091.jpg 21 train butterfly/image_0012.jpg 21 train butterfly/image_0066.jpg 21 train butterfly/image_0077.jpg 21 train butterfly/image_0025.jpg 21 train butterfly/image_0060.jpg 21 train butterfly/image_0052.jpg 21 train butterfly/image_0007.jpg 21 train butterfly/image_0063.jpg 21 train butterfly/image_0019.jpg 21 train butterfly/image_0031.jpg 21 train butterfly/image_0011.jpg 21 train butterfly/image_0061.jpg 21 train butterfly/image_0090.jpg 21 train butterfly/image_0050.jpg 21 train butterfly/image_0045.jpg 21 train butterfly/image_0017.jpg 21 train butterfly/image_0018.jpg 21 train butterfly/image_0013.jpg 21 train butterfly/image_0054.jpg 21 train butterfly/image_0082.jpg 21 train butterfly/image_0079.jpg 21 train butterfly/image_0055.jpg 21 train butterfly/image_0071.jpg 21 train butterfly/image_0024.jpg 21 train butterfly/image_0074.jpg 21 train butterfly/image_0003.jpg 21 train butterfly/image_0009.jpg 21 train butterfly/image_0042.jpg 21 train butterfly/image_0040.jpg 21 train butterfly/image_0067.jpg 21 train butterfly/image_0047.jpg 21 train butterfly/image_0064.jpg 21 train butterfly/image_0070.jpg 21 train butterfly/image_0078.jpg 21 train butterfly/image_0068.jpg 21 train butterfly/image_0085.jpg 21 train butterfly/image_0043.jpg 21 train butterfly/image_0044.jpg 21 train butterfly/image_0033.jpg 21 train butterfly/image_0073.jpg 21 train butterfly/image_0001.jpg 21 train butterfly/image_0080.jpg 21 train butterfly/image_0021.jpg 21 train butterfly/image_0057.jpg 21 train butterfly/image_0008.jpg 21 train butterfly/image_0035.jpg 21 train butterfly/image_0005.jpg 21 train butterfly/image_0087.jpg 21 train butterfly/image_0089.jpg 21 train butterfly/image_0059.jpg 21 train butterfly/image_0069.jpg 21 train butterfly/image_0034.jpg 21 train butterfly/image_0002.jpg 21 train butterfly/image_0048.jpg 21 train butterfly/image_0053.jpg 21 train butterfly/image_0084.jpg 21 train butterfly/image_0020.jpg 21 train butterfly/image_0046.jpg 21 train butterfly/image_0041.jpg 21 train butterfly/image_0027.jpg 21 train butterfly/image_0014.jpg 21 train butterfly/image_0058.jpg 21 train butterfly/image_0081.jpg 21 train windsor_chair/image_0037.jpg 22 train windsor_chair/image_0049.jpg 22 train windsor_chair/image_0051.jpg 22 train windsor_chair/image_0006.jpg 22 train windsor_chair/image_0016.jpg 22 train windsor_chair/image_0030.jpg 22 train windsor_chair/image_0012.jpg 22 train windsor_chair/image_0025.jpg 22 train windsor_chair/image_0052.jpg 22 train windsor_chair/image_0007.jpg 22 train windsor_chair/image_0019.jpg 22 train windsor_chair/image_0031.jpg 22 train windsor_chair/image_0011.jpg 22 train windsor_chair/image_0050.jpg 22 train windsor_chair/image_0045.jpg 22 train windsor_chair/image_0017.jpg 22 train windsor_chair/image_0018.jpg 22 train windsor_chair/image_0013.jpg 22 train windsor_chair/image_0054.jpg 22 train windsor_chair/image_0055.jpg 22 train windsor_chair/image_0024.jpg 22 train windsor_chair/image_0003.jpg 22 train windsor_chair/image_0009.jpg 22 train windsor_chair/image_0042.jpg 22 train windsor_chair/image_0040.jpg 22 train windsor_chair/image_0047.jpg 22 train windsor_chair/image_0043.jpg 22 train windsor_chair/image_0044.jpg 22 train windsor_chair/image_0033.jpg 22 train windsor_chair/image_0001.jpg 22 train windsor_chair/image_0021.jpg 22 train windsor_chair/image_0008.jpg 22 train windsor_chair/image_0035.jpg 22 train windsor_chair/image_0005.jpg 22 train windsor_chair/image_0034.jpg 22 train windsor_chair/image_0002.jpg 22 train windsor_chair/image_0048.jpg 22 train windsor_chair/image_0053.jpg 22 train windsor_chair/image_0020.jpg 22 train windsor_chair/image_0046.jpg 22 train windsor_chair/image_0041.jpg 22 train windsor_chair/image_0027.jpg 22 train windsor_chair/image_0014.jpg 22 train windsor_chair/image_0026.jpg 22 train metronome/image_0006.jpg 23 train metronome/image_0016.jpg 23 train metronome/image_0030.jpg 23 train metronome/image_0012.jpg 23 train metronome/image_0025.jpg 23 train metronome/image_0007.jpg 23 train metronome/image_0019.jpg 23 train metronome/image_0031.jpg 23 train metronome/image_0011.jpg 23 train metronome/image_0017.jpg 23 train metronome/image_0018.jpg 23 train metronome/image_0013.jpg 23 train metronome/image_0024.jpg 23 train metronome/image_0003.jpg 23 train metronome/image_0009.jpg 23 train metronome/image_0001.jpg 23 train metronome/image_0021.jpg 23 train metronome/image_0008.jpg 23 train metronome/image_0005.jpg 23 train metronome/image_0002.jpg 23 train metronome/image_0020.jpg 23 train metronome/image_0027.jpg 23 train metronome/image_0014.jpg 23 train metronome/image_0026.jpg 23 train metronome/image_0004.jpg 23 train revolver/image_0062.jpg 24 train revolver/image_0037.jpg 24 train revolver/image_0049.jpg 24 train revolver/image_0051.jpg 24 train revolver/image_0006.jpg 24 train revolver/image_0016.jpg 24 train revolver/image_0030.jpg 24 train revolver/image_0012.jpg 24 train revolver/image_0066.jpg 24 train revolver/image_0077.jpg 24 train revolver/image_0025.jpg 24 train revolver/image_0060.jpg 24 train revolver/image_0052.jpg 24 train revolver/image_0007.jpg 24 train revolver/image_0063.jpg 24 train revolver/image_0019.jpg 24 train revolver/image_0031.jpg 24 train revolver/image_0011.jpg 24 train revolver/image_0061.jpg 24 train revolver/image_0050.jpg 24 train revolver/image_0045.jpg 24 train revolver/image_0017.jpg 24 train revolver/image_0018.jpg 24 train revolver/image_0013.jpg 24 train revolver/image_0054.jpg 24 train revolver/image_0082.jpg 24 train revolver/image_0079.jpg 24 train revolver/image_0055.jpg 24 train revolver/image_0071.jpg 24 train revolver/image_0024.jpg 24 train revolver/image_0074.jpg 24 train revolver/image_0003.jpg 24 train revolver/image_0009.jpg 24 train revolver/image_0042.jpg 24 train revolver/image_0040.jpg 24 train revolver/image_0067.jpg 24 train revolver/image_0047.jpg 24 train revolver/image_0064.jpg 24 train revolver/image_0070.jpg 24 train revolver/image_0078.jpg 24 train revolver/image_0068.jpg 24 train revolver/image_0043.jpg 24 train revolver/image_0044.jpg 24 train revolver/image_0033.jpg 24 train revolver/image_0073.jpg 24 train revolver/image_0001.jpg 24 train revolver/image_0080.jpg 24 train revolver/image_0021.jpg 24 train revolver/image_0057.jpg 24 train revolver/image_0008.jpg 24 train revolver/image_0035.jpg 24 train revolver/image_0005.jpg 24 train revolver/image_0059.jpg 24 train revolver/image_0069.jpg 24 train revolver/image_0034.jpg 24 train revolver/image_0002.jpg 24 train revolver/image_0048.jpg 24 train revolver/image_0053.jpg 24 train revolver/image_0020.jpg 24 train revolver/image_0046.jpg 24 train revolver/image_0041.jpg 24 train revolver/image_0027.jpg 24 train revolver/image_0014.jpg 24 train revolver/image_0058.jpg 24 train revolver/image_0081.jpg 24 train schooner/image_0062.jpg 25 train schooner/image_0037.jpg 25 train schooner/image_0049.jpg 25 train schooner/image_0051.jpg 25 train schooner/image_0006.jpg 25 train schooner/image_0016.jpg 25 train schooner/image_0030.jpg 25 train schooner/image_0012.jpg 25 train schooner/image_0025.jpg 25 train schooner/image_0060.jpg 25 train schooner/image_0052.jpg 25 train schooner/image_0007.jpg 25 train schooner/image_0063.jpg 25 train schooner/image_0019.jpg 25 train schooner/image_0031.jpg 25 train schooner/image_0011.jpg 25 train schooner/image_0061.jpg 25 train schooner/image_0050.jpg 25 train schooner/image_0045.jpg 25 train schooner/image_0017.jpg 25 train schooner/image_0018.jpg 25 train schooner/image_0013.jpg 25 train schooner/image_0054.jpg 25 train schooner/image_0055.jpg 25 train schooner/image_0024.jpg 25 train schooner/image_0003.jpg 25 train schooner/image_0009.jpg 25 train schooner/image_0042.jpg 25 train schooner/image_0040.jpg 25 train schooner/image_0047.jpg 25 train schooner/image_0043.jpg 25 train schooner/image_0044.jpg 25 train schooner/image_0033.jpg 25 train schooner/image_0001.jpg 25 train schooner/image_0021.jpg 25 train schooner/image_0057.jpg 25 train schooner/image_0008.jpg 25 train schooner/image_0035.jpg 25 train schooner/image_0005.jpg 25 train schooner/image_0059.jpg 25 train schooner/image_0034.jpg 25 train schooner/image_0002.jpg 25 train schooner/image_0048.jpg 25 train schooner/image_0053.jpg 25 train schooner/image_0020.jpg 25 train schooner/image_0046.jpg 25 train schooner/image_0041.jpg 25 train schooner/image_0027.jpg 25 train schooner/image_0014.jpg 25 train schooner/image_0058.jpg 25 train minaret/image_0062.jpg 26 train minaret/image_0037.jpg 26 train minaret/image_0049.jpg 26 train minaret/image_0051.jpg 26 train minaret/image_0006.jpg 26 train minaret/image_0016.jpg 26 train minaret/image_0030.jpg 26 train minaret/image_0012.jpg 26 train minaret/image_0066.jpg 26 train minaret/image_0025.jpg 26 train minaret/image_0060.jpg 26 train minaret/image_0052.jpg 26 train minaret/image_0007.jpg 26 train minaret/image_0063.jpg 26 train minaret/image_0019.jpg 26 train minaret/image_0031.jpg 26 train minaret/image_0011.jpg 26 train minaret/image_0061.jpg 26 train minaret/image_0050.jpg 26 train minaret/image_0045.jpg 26 train minaret/image_0017.jpg 26 train minaret/image_0018.jpg 26 train minaret/image_0013.jpg 26 train minaret/image_0054.jpg 26 train minaret/image_0055.jpg 26 train minaret/image_0071.jpg 26 train minaret/image_0024.jpg 26 train minaret/image_0074.jpg 26 train minaret/image_0003.jpg 26 train minaret/image_0009.jpg 26 train minaret/image_0042.jpg 26 train minaret/image_0040.jpg 26 train minaret/image_0067.jpg 26 train minaret/image_0047.jpg 26 train minaret/image_0064.jpg 26 train minaret/image_0070.jpg 26 train minaret/image_0068.jpg 26 train minaret/image_0043.jpg 26 train minaret/image_0044.jpg 26 train minaret/image_0033.jpg 26 train minaret/image_0073.jpg 26 train minaret/image_0001.jpg 26 train minaret/image_0021.jpg 26 train minaret/image_0057.jpg 26 train minaret/image_0008.jpg 26 train minaret/image_0035.jpg 26 train minaret/image_0005.jpg 26 train minaret/image_0059.jpg 26 train minaret/image_0069.jpg 26 train minaret/image_0034.jpg 26 train minaret/image_0002.jpg 26 train minaret/image_0048.jpg 26 train minaret/image_0053.jpg 26 train minaret/image_0020.jpg 26 train minaret/image_0046.jpg 26 train minaret/image_0041.jpg 26 train minaret/image_0027.jpg 26 train minaret/image_0014.jpg 26 train minaret/image_0058.jpg 26 train minaret/image_0026.jpg 26 train stop_sign/image_0062.jpg 27 train stop_sign/image_0037.jpg 27 train stop_sign/image_0049.jpg 27 train stop_sign/image_0051.jpg 27 train stop_sign/image_0006.jpg 27 train stop_sign/image_0016.jpg 27 train stop_sign/image_0030.jpg 27 train stop_sign/image_0012.jpg 27 train stop_sign/image_0025.jpg 27 train stop_sign/image_0060.jpg 27 train stop_sign/image_0052.jpg 27 train stop_sign/image_0007.jpg 27 train stop_sign/image_0063.jpg 27 train stop_sign/image_0019.jpg 27 train stop_sign/image_0031.jpg 27 train stop_sign/image_0011.jpg 27 train stop_sign/image_0061.jpg 27 train stop_sign/image_0050.jpg 27 train stop_sign/image_0045.jpg 27 train stop_sign/image_0017.jpg 27 train stop_sign/image_0018.jpg 27 train stop_sign/image_0013.jpg 27 train stop_sign/image_0054.jpg 27 train stop_sign/image_0055.jpg 27 train stop_sign/image_0024.jpg 27 train stop_sign/image_0003.jpg 27 train stop_sign/image_0009.jpg 27 train stop_sign/image_0042.jpg 27 train stop_sign/image_0040.jpg 27 train stop_sign/image_0047.jpg 27 train stop_sign/image_0064.jpg 27 train stop_sign/image_0043.jpg 27 train stop_sign/image_0044.jpg 27 train stop_sign/image_0033.jpg 27 train stop_sign/image_0001.jpg 27 train stop_sign/image_0021.jpg 27 train stop_sign/image_0057.jpg 27 train stop_sign/image_0008.jpg 27 train stop_sign/image_0035.jpg 27 train stop_sign/image_0005.jpg 27 train stop_sign/image_0059.jpg 27 train stop_sign/image_0034.jpg 27 train stop_sign/image_0002.jpg 27 train stop_sign/image_0048.jpg 27 train stop_sign/image_0053.jpg 27 train stop_sign/image_0020.jpg 27 train stop_sign/image_0046.jpg 27 train stop_sign/image_0041.jpg 27 train stop_sign/image_0027.jpg 27 train stop_sign/image_0014.jpg 27 train stop_sign/image_0058.jpg 27 train beaver/image_0037.jpg 28 train beaver/image_0006.jpg 28 train beaver/image_0016.jpg 28 train beaver/image_0030.jpg 28 train beaver/image_0012.jpg 28 train beaver/image_0025.jpg 28 train beaver/image_0007.jpg 28 train beaver/image_0019.jpg 28 train beaver/image_0031.jpg 28 train beaver/image_0011.jpg 28 train beaver/image_0045.jpg 28 train beaver/image_0017.jpg 28 train beaver/image_0018.jpg 28 train beaver/image_0013.jpg 28 train beaver/image_0024.jpg 28 train beaver/image_0003.jpg 28 train beaver/image_0009.jpg 28 train beaver/image_0042.jpg 28 train beaver/image_0040.jpg 28 train beaver/image_0043.jpg 28 train beaver/image_0044.jpg 28 train beaver/image_0033.jpg 28 train beaver/image_0001.jpg 28 train beaver/image_0021.jpg 28 train beaver/image_0008.jpg 28 train beaver/image_0035.jpg 28 train beaver/image_0005.jpg 28 train beaver/image_0034.jpg 28 train beaver/image_0002.jpg 28 train beaver/image_0020.jpg 28 train beaver/image_0046.jpg 28 train beaver/image_0041.jpg 28 train beaver/image_0027.jpg 28 train beaver/image_0014.jpg 28 train beaver/image_0026.jpg 28 train beaver/image_0004.jpg 28 train laptop/image_0062.jpg 29 train laptop/image_0037.jpg 29 train laptop/image_0049.jpg 29 train laptop/image_0051.jpg 29 train laptop/image_0006.jpg 29 train laptop/image_0016.jpg 29 train laptop/image_0030.jpg 29 train laptop/image_0012.jpg 29 train laptop/image_0066.jpg 29 train laptop/image_0077.jpg 29 train laptop/image_0025.jpg 29 train laptop/image_0060.jpg 29 train laptop/image_0052.jpg 29 train laptop/image_0007.jpg 29 train laptop/image_0063.jpg 29 train laptop/image_0019.jpg 29 train laptop/image_0031.jpg 29 train laptop/image_0011.jpg 29 train laptop/image_0061.jpg 29 train laptop/image_0050.jpg 29 train laptop/image_0045.jpg 29 train laptop/image_0017.jpg 29 train laptop/image_0018.jpg 29 train laptop/image_0013.jpg 29 train laptop/image_0054.jpg 29 train laptop/image_0079.jpg 29 train laptop/image_0055.jpg 29 train laptop/image_0071.jpg 29 train laptop/image_0024.jpg 29 train laptop/image_0074.jpg 29 train laptop/image_0003.jpg 29 train laptop/image_0009.jpg 29 train laptop/image_0042.jpg 29 train laptop/image_0040.jpg 29 train laptop/image_0067.jpg 29 train laptop/image_0047.jpg 29 train laptop/image_0064.jpg 29 train laptop/image_0070.jpg 29 train laptop/image_0078.jpg 29 train laptop/image_0068.jpg 29 train laptop/image_0043.jpg 29 train laptop/image_0044.jpg 29 train laptop/image_0033.jpg 29 train laptop/image_0073.jpg 29 train laptop/image_0001.jpg 29 train laptop/image_0080.jpg 29 train laptop/image_0021.jpg 29 train laptop/image_0057.jpg 29 train laptop/image_0008.jpg 29 train laptop/image_0035.jpg 29 train laptop/image_0005.jpg 29 train laptop/image_0059.jpg 29 train laptop/image_0069.jpg 29 train laptop/image_0034.jpg 29 train laptop/image_0002.jpg 29 train laptop/image_0048.jpg 29 train laptop/image_0053.jpg 29 train laptop/image_0020.jpg 29 train laptop/image_0046.jpg 29 train laptop/image_0041.jpg 29 train laptop/image_0027.jpg 29 train laptop/image_0014.jpg 29 train laptop/image_0058.jpg 29 train laptop/image_0081.jpg 29 train ketch/image_0062.jpg 30 train ketch/image_0037.jpg 30 train ketch/image_0049.jpg 30 train ketch/image_0083.jpg 30 train ketch/image_0101.jpg 30 train ketch/image_0051.jpg 30 train ketch/image_0006.jpg 30 train ketch/image_0016.jpg 30 train ketch/image_0030.jpg 30 train ketch/image_0091.jpg 30 train ketch/image_0012.jpg 30 train ketch/image_0066.jpg 30 train ketch/image_0077.jpg 30 train ketch/image_0025.jpg 30 train ketch/image_0060.jpg 30 train ketch/image_0052.jpg 30 train ketch/image_0108.jpg 30 train ketch/image_0007.jpg 30 train ketch/image_0063.jpg 30 train ketch/image_0019.jpg 30 train ketch/image_0113.jpg 30 train ketch/image_0031.jpg 30 train ketch/image_0011.jpg 30 train ketch/image_0061.jpg 30 train ketch/image_0090.jpg 30 train ketch/image_0050.jpg 30 train ketch/image_0045.jpg 30 train ketch/image_0017.jpg 30 train ketch/image_0018.jpg 30 train ketch/image_0013.jpg 30 train ketch/image_0054.jpg 30 train ketch/image_0094.jpg 30 train ketch/image_0082.jpg 30 train ketch/image_0097.jpg 30 train ketch/image_0109.jpg 30 train ketch/image_0079.jpg 30 train ketch/image_0055.jpg 30 train ketch/image_0071.jpg 30 train ketch/image_0103.jpg 30 train ketch/image_0024.jpg 30 train ketch/image_0105.jpg 30 train ketch/image_0074.jpg 30 train ketch/image_0003.jpg 30 train ketch/image_0009.jpg 30 train ketch/image_0042.jpg 30 train ketch/image_0093.jpg 30 train ketch/image_0040.jpg 30 train ketch/image_0067.jpg 30 train ketch/image_0047.jpg 30 train ketch/image_0064.jpg 30 train ketch/image_0070.jpg 30 train ketch/image_0112.jpg 30 train ketch/image_0078.jpg 30 train ketch/image_0068.jpg 30 train ketch/image_0085.jpg 30 train ketch/image_0110.jpg 30 train ketch/image_0043.jpg 30 train ketch/image_0044.jpg 30 train ketch/image_0033.jpg 30 train ketch/image_0073.jpg 30 train ketch/image_0001.jpg 30 train ketch/image_0080.jpg 30 train ketch/image_0021.jpg 30 train ketch/image_0095.jpg 30 train ketch/image_0057.jpg 30 train ketch/image_0114.jpg 30 train ketch/image_0098.jpg 30 train ketch/image_0008.jpg 30 train ketch/image_0035.jpg 30 train ketch/image_0102.jpg 30 train ketch/image_0005.jpg 30 train ketch/image_0087.jpg 30 train ketch/image_0089.jpg 30 train ketch/image_0059.jpg 30 train ketch/image_0069.jpg 30 train ketch/image_0099.jpg 30 train ketch/image_0034.jpg 30 train ketch/image_0002.jpg 30 train ketch/image_0048.jpg 30 train ketch/image_0053.jpg 30 train ketch/image_0084.jpg 30 train ketch/image_0020.jpg 30 train ketch/image_0046.jpg 30 train ketch/image_0041.jpg 30 train ketch/image_0027.jpg 30 train ketch/image_0014.jpg 30 train ketch/image_0058.jpg 30 train ketch/image_0104.jpg 30 train ketch/image_0081.jpg 30 train ketch/image_0088.jpg 30 train ketch/image_0026.jpg 30 train gramophone/image_0037.jpg 31 train gramophone/image_0049.jpg 31 train gramophone/image_0051.jpg 31 train gramophone/image_0006.jpg 31 train gramophone/image_0016.jpg 31 train gramophone/image_0030.jpg 31 train gramophone/image_0012.jpg 31 train gramophone/image_0025.jpg 31 train gramophone/image_0007.jpg 31 train gramophone/image_0019.jpg 31 train gramophone/image_0031.jpg 31 train gramophone/image_0011.jpg 31 train gramophone/image_0050.jpg 31 train gramophone/image_0045.jpg 31 train gramophone/image_0017.jpg 31 train gramophone/image_0018.jpg 31 train gramophone/image_0013.jpg 31 train gramophone/image_0024.jpg 31 train gramophone/image_0003.jpg 31 train gramophone/image_0009.jpg 31 train gramophone/image_0042.jpg 31 train gramophone/image_0040.jpg 31 train gramophone/image_0047.jpg 31 train gramophone/image_0043.jpg 31 train gramophone/image_0044.jpg 31 train gramophone/image_0033.jpg 31 train gramophone/image_0001.jpg 31 train gramophone/image_0021.jpg 31 train gramophone/image_0008.jpg 31 train gramophone/image_0035.jpg 31 train gramophone/image_0005.jpg 31 train gramophone/image_0034.jpg 31 train gramophone/image_0002.jpg 31 train gramophone/image_0048.jpg 31 train gramophone/image_0020.jpg 31 train gramophone/image_0046.jpg 31 train gramophone/image_0041.jpg 31 train gramophone/image_0027.jpg 31 train gramophone/image_0014.jpg 31 train gramophone/image_0026.jpg 31 train menorah/image_0062.jpg 32 train menorah/image_0037.jpg 32 train menorah/image_0049.jpg 32 train menorah/image_0083.jpg 32 train menorah/image_0051.jpg 32 train menorah/image_0006.jpg 32 train menorah/image_0016.jpg 32 train menorah/image_0030.jpg 32 train menorah/image_0012.jpg 32 train menorah/image_0066.jpg 32 train menorah/image_0077.jpg 32 train menorah/image_0025.jpg 32 train menorah/image_0060.jpg 32 train menorah/image_0052.jpg 32 train menorah/image_0007.jpg 32 train menorah/image_0063.jpg 32 train menorah/image_0019.jpg 32 train menorah/image_0031.jpg 32 train menorah/image_0011.jpg 32 train menorah/image_0061.jpg 32 train menorah/image_0050.jpg 32 train menorah/image_0045.jpg 32 train menorah/image_0017.jpg 32 train menorah/image_0018.jpg 32 train menorah/image_0013.jpg 32 train menorah/image_0054.jpg 32 train menorah/image_0082.jpg 32 train menorah/image_0079.jpg 32 train menorah/image_0055.jpg 32 train menorah/image_0071.jpg 32 train menorah/image_0024.jpg 32 train menorah/image_0074.jpg 32 train menorah/image_0003.jpg 32 train menorah/image_0009.jpg 32 train menorah/image_0042.jpg 32 train menorah/image_0040.jpg 32 train menorah/image_0067.jpg 32 train menorah/image_0047.jpg 32 train menorah/image_0064.jpg 32 train menorah/image_0070.jpg 32 train menorah/image_0078.jpg 32 train menorah/image_0068.jpg 32 train menorah/image_0085.jpg 32 train menorah/image_0043.jpg 32 train menorah/image_0044.jpg 32 train menorah/image_0033.jpg 32 train menorah/image_0073.jpg 32 train menorah/image_0001.jpg 32 train menorah/image_0080.jpg 32 train menorah/image_0021.jpg 32 train menorah/image_0057.jpg 32 train menorah/image_0008.jpg 32 train menorah/image_0035.jpg 32 train menorah/image_0005.jpg 32 train menorah/image_0087.jpg 32 train menorah/image_0059.jpg 32 train menorah/image_0069.jpg 32 train menorah/image_0034.jpg 32 train menorah/image_0002.jpg 32 train menorah/image_0048.jpg 32 train menorah/image_0053.jpg 32 train menorah/image_0084.jpg 32 train menorah/image_0020.jpg 32 train menorah/image_0046.jpg 32 train menorah/image_0041.jpg 32 train menorah/image_0027.jpg 32 train menorah/image_0014.jpg 32 train menorah/image_0058.jpg 32 train menorah/image_0081.jpg 32 train euphonium/image_0062.jpg 33 train euphonium/image_0037.jpg 33 train euphonium/image_0049.jpg 33 train euphonium/image_0051.jpg 33 train euphonium/image_0006.jpg 33 train euphonium/image_0016.jpg 33 train euphonium/image_0030.jpg 33 train euphonium/image_0012.jpg 33 train euphonium/image_0025.jpg 33 train euphonium/image_0060.jpg 33 train euphonium/image_0052.jpg 33 train euphonium/image_0007.jpg 33 train euphonium/image_0063.jpg 33 train euphonium/image_0019.jpg 33 train euphonium/image_0031.jpg 33 train euphonium/image_0011.jpg 33 train euphonium/image_0061.jpg 33 train euphonium/image_0050.jpg 33 train euphonium/image_0045.jpg 33 train euphonium/image_0017.jpg 33 train euphonium/image_0018.jpg 33 train euphonium/image_0013.jpg 33 train euphonium/image_0054.jpg 33 train euphonium/image_0055.jpg 33 train euphonium/image_0024.jpg 33 train euphonium/image_0003.jpg 33 train euphonium/image_0009.jpg 33 train euphonium/image_0042.jpg 33 train euphonium/image_0040.jpg 33 train euphonium/image_0047.jpg 33 train euphonium/image_0064.jpg 33 train euphonium/image_0043.jpg 33 train euphonium/image_0044.jpg 33 train euphonium/image_0033.jpg 33 train euphonium/image_0001.jpg 33 train euphonium/image_0021.jpg 33 train euphonium/image_0057.jpg 33 train euphonium/image_0008.jpg 33 train euphonium/image_0035.jpg 33 train euphonium/image_0005.jpg 33 train euphonium/image_0059.jpg 33 train euphonium/image_0034.jpg 33 train euphonium/image_0002.jpg 33 train euphonium/image_0048.jpg 33 train euphonium/image_0053.jpg 33 train euphonium/image_0020.jpg 33 train euphonium/image_0046.jpg 33 train euphonium/image_0041.jpg 33 train euphonium/image_0027.jpg 33 train euphonium/image_0014.jpg 33 train euphonium/image_0058.jpg 33 train rhino/image_0037.jpg 34 train rhino/image_0049.jpg 34 train rhino/image_0051.jpg 34 train rhino/image_0006.jpg 34 train rhino/image_0016.jpg 34 train rhino/image_0030.jpg 34 train rhino/image_0012.jpg 34 train rhino/image_0025.jpg 34 train rhino/image_0052.jpg 34 train rhino/image_0007.jpg 34 train rhino/image_0019.jpg 34 train rhino/image_0031.jpg 34 train rhino/image_0011.jpg 34 train rhino/image_0050.jpg 34 train rhino/image_0045.jpg 34 train rhino/image_0017.jpg 34 train rhino/image_0018.jpg 34 train rhino/image_0013.jpg 34 train rhino/image_0054.jpg 34 train rhino/image_0055.jpg 34 train rhino/image_0024.jpg 34 train rhino/image_0003.jpg 34 train rhino/image_0009.jpg 34 train rhino/image_0042.jpg 34 train rhino/image_0040.jpg 34 train rhino/image_0047.jpg 34 train rhino/image_0043.jpg 34 train rhino/image_0044.jpg 34 train rhino/image_0033.jpg 34 train rhino/image_0001.jpg 34 train rhino/image_0021.jpg 34 train rhino/image_0057.jpg 34 train rhino/image_0008.jpg 34 train rhino/image_0035.jpg 34 train rhino/image_0005.jpg 34 train rhino/image_0059.jpg 34 train rhino/image_0034.jpg 34 train rhino/image_0002.jpg 34 train rhino/image_0048.jpg 34 train rhino/image_0053.jpg 34 train rhino/image_0020.jpg 34 train rhino/image_0046.jpg 34 train rhino/image_0041.jpg 34 train rhino/image_0027.jpg 34 train rhino/image_0014.jpg 34 train rhino/image_0058.jpg 34 train rhino/image_0026.jpg 34 train bonsai/image_0062.jpg 35 train bonsai/image_0037.jpg 35 train bonsai/image_0049.jpg 35 train bonsai/image_0083.jpg 35 train bonsai/image_0101.jpg 35 train bonsai/image_0051.jpg 35 train bonsai/image_0006.jpg 35 train bonsai/image_0016.jpg 35 train bonsai/image_0030.jpg 35 train bonsai/image_0091.jpg 35 train bonsai/image_0012.jpg 35 train bonsai/image_0066.jpg 35 train bonsai/image_0077.jpg 35 train bonsai/image_0025.jpg 35 train bonsai/image_0115.jpg 35 train bonsai/image_0123.jpg 35 train bonsai/image_0119.jpg 35 train bonsai/image_0060.jpg 35 train bonsai/image_0052.jpg 35 train bonsai/image_0108.jpg 35 train bonsai/image_0007.jpg 35 train bonsai/image_0063.jpg 35 train bonsai/image_0019.jpg 35 train bonsai/image_0113.jpg 35 train bonsai/image_0127.jpg 35 train bonsai/image_0031.jpg 35 train bonsai/image_0011.jpg 35 train bonsai/image_0061.jpg 35 train bonsai/image_0090.jpg 35 train bonsai/image_0126.jpg 35 train bonsai/image_0050.jpg 35 train bonsai/image_0045.jpg 35 train bonsai/image_0017.jpg 35 train bonsai/image_0018.jpg 35 train bonsai/image_0013.jpg 35 train bonsai/image_0054.jpg 35 train bonsai/image_0094.jpg 35 train bonsai/image_0082.jpg 35 train bonsai/image_0097.jpg 35 train bonsai/image_0109.jpg 35 train bonsai/image_0079.jpg 35 train bonsai/image_0055.jpg 35 train bonsai/image_0071.jpg 35 train bonsai/image_0103.jpg 35 train bonsai/image_0024.jpg 35 train bonsai/image_0105.jpg 35 train bonsai/image_0074.jpg 35 train bonsai/image_0003.jpg 35 train bonsai/image_0009.jpg 35 train bonsai/image_0118.jpg 35 train bonsai/image_0042.jpg 35 train bonsai/image_0093.jpg 35 train bonsai/image_0040.jpg 35 train bonsai/image_0067.jpg 35 train bonsai/image_0122.jpg 35 train bonsai/image_0125.jpg 35 train bonsai/image_0047.jpg 35 train bonsai/image_0064.jpg 35 train bonsai/image_0070.jpg 35 train bonsai/image_0112.jpg 35 train bonsai/image_0078.jpg 35 train bonsai/image_0068.jpg 35 train bonsai/image_0085.jpg 35 train bonsai/image_0110.jpg 35 train bonsai/image_0043.jpg 35 train bonsai/image_0044.jpg 35 train bonsai/image_0033.jpg 35 train bonsai/image_0073.jpg 35 train bonsai/image_0001.jpg 35 train bonsai/image_0080.jpg 35 train bonsai/image_0021.jpg 35 train bonsai/image_0095.jpg 35 train bonsai/image_0057.jpg 35 train bonsai/image_0114.jpg 35 train bonsai/image_0098.jpg 35 train bonsai/image_0008.jpg 35 train bonsai/image_0035.jpg 35 train bonsai/image_0128.jpg 35 train bonsai/image_0102.jpg 35 train bonsai/image_0005.jpg 35 train bonsai/image_0087.jpg 35 train bonsai/image_0089.jpg 35 train bonsai/image_0059.jpg 35 train bonsai/image_0069.jpg 35 train bonsai/image_0099.jpg 35 train bonsai/image_0034.jpg 35 train bonsai/image_0121.jpg 35 train bonsai/image_0002.jpg 35 train bonsai/image_0048.jpg 35 train bonsai/image_0053.jpg 35 train bonsai/image_0084.jpg 35 train bonsai/image_0020.jpg 35 train bonsai/image_0046.jpg 35 train bonsai/image_0041.jpg 35 train bonsai/image_0027.jpg 35 train bonsai/image_0014.jpg 35 train bonsai/image_0058.jpg 35 train bonsai/image_0104.jpg 35 train bonsai/image_0081.jpg 35 train bonsai/image_0088.jpg 35 train bonsai/image_0026.jpg 35 train bonsai/image_0004.jpg 35 train chair/image_0062.jpg 36 train chair/image_0037.jpg 36 train chair/image_0049.jpg 36 train chair/image_0051.jpg 36 train chair/image_0006.jpg 36 train chair/image_0016.jpg 36 train chair/image_0030.jpg 36 train chair/image_0012.jpg 36 train chair/image_0025.jpg 36 train chair/image_0060.jpg 36 train chair/image_0052.jpg 36 train chair/image_0007.jpg 36 train chair/image_0019.jpg 36 train chair/image_0031.jpg 36 train chair/image_0011.jpg 36 train chair/image_0061.jpg 36 train chair/image_0050.jpg 36 train chair/image_0045.jpg 36 train chair/image_0017.jpg 36 train chair/image_0018.jpg 36 train chair/image_0013.jpg 36 train chair/image_0054.jpg 36 train chair/image_0055.jpg 36 train chair/image_0024.jpg 36 train chair/image_0003.jpg 36 train chair/image_0009.jpg 36 train chair/image_0042.jpg 36 train chair/image_0040.jpg 36 train chair/image_0047.jpg 36 train chair/image_0043.jpg 36 train chair/image_0044.jpg 36 train chair/image_0033.jpg 36 train chair/image_0001.jpg 36 train chair/image_0021.jpg 36 train chair/image_0057.jpg 36 train chair/image_0008.jpg 36 train chair/image_0035.jpg 36 train chair/image_0005.jpg 36 train chair/image_0059.jpg 36 train chair/image_0034.jpg 36 train chair/image_0002.jpg 36 train chair/image_0048.jpg 36 train chair/image_0053.jpg 36 train chair/image_0020.jpg 36 train chair/image_0046.jpg 36 train chair/image_0041.jpg 36 train chair/image_0027.jpg 36 train chair/image_0014.jpg 36 train chair/image_0058.jpg 36 train snoopy/image_0006.jpg 37 train snoopy/image_0016.jpg 37 train snoopy/image_0030.jpg 37 train snoopy/image_0012.jpg 37 train snoopy/image_0025.jpg 37 train snoopy/image_0007.jpg 37 train snoopy/image_0019.jpg 37 train snoopy/image_0031.jpg 37 train snoopy/image_0011.jpg 37 train snoopy/image_0017.jpg 37 train snoopy/image_0018.jpg 37 train snoopy/image_0013.jpg 37 train snoopy/image_0024.jpg 37 train snoopy/image_0003.jpg 37 train snoopy/image_0009.jpg 37 train snoopy/image_0033.jpg 37 train snoopy/image_0001.jpg 37 train snoopy/image_0021.jpg 37 train snoopy/image_0008.jpg 37 train snoopy/image_0035.jpg 37 train snoopy/image_0005.jpg 37 train snoopy/image_0034.jpg 37 train snoopy/image_0002.jpg 37 train snoopy/image_0020.jpg 37 train snoopy/image_0027.jpg 37 train snoopy/image_0014.jpg 37 train snoopy/image_0026.jpg 37 train snoopy/image_0004.jpg 37 train buddha/image_0062.jpg 38 train buddha/image_0037.jpg 38 train buddha/image_0049.jpg 38 train buddha/image_0083.jpg 38 train buddha/image_0051.jpg 38 train buddha/image_0006.jpg 38 train buddha/image_0016.jpg 38 train buddha/image_0030.jpg 38 train buddha/image_0012.jpg 38 train buddha/image_0066.jpg 38 train buddha/image_0077.jpg 38 train buddha/image_0025.jpg 38 train buddha/image_0060.jpg 38 train buddha/image_0052.jpg 38 train buddha/image_0007.jpg 38 train buddha/image_0063.jpg 38 train buddha/image_0019.jpg 38 train buddha/image_0031.jpg 38 train buddha/image_0011.jpg 38 train buddha/image_0061.jpg 38 train buddha/image_0050.jpg 38 train buddha/image_0045.jpg 38 train buddha/image_0017.jpg 38 train buddha/image_0018.jpg 38 train buddha/image_0013.jpg 38 train buddha/image_0054.jpg 38 train buddha/image_0082.jpg 38 train buddha/image_0079.jpg 38 train buddha/image_0055.jpg 38 train buddha/image_0071.jpg 38 train buddha/image_0024.jpg 38 train buddha/image_0074.jpg 38 train buddha/image_0003.jpg 38 train buddha/image_0009.jpg 38 train buddha/image_0042.jpg 38 train buddha/image_0040.jpg 38 train buddha/image_0067.jpg 38 train buddha/image_0047.jpg 38 train buddha/image_0064.jpg 38 train buddha/image_0070.jpg 38 train buddha/image_0078.jpg 38 train buddha/image_0068.jpg 38 train buddha/image_0085.jpg 38 train buddha/image_0043.jpg 38 train buddha/image_0044.jpg 38 train buddha/image_0033.jpg 38 train buddha/image_0073.jpg 38 train buddha/image_0001.jpg 38 train buddha/image_0080.jpg 38 train buddha/image_0021.jpg 38 train buddha/image_0057.jpg 38 train buddha/image_0008.jpg 38 train buddha/image_0035.jpg 38 train buddha/image_0005.jpg 38 train buddha/image_0059.jpg 38 train buddha/image_0069.jpg 38 train buddha/image_0034.jpg 38 train buddha/image_0002.jpg 38 train buddha/image_0048.jpg 38 train buddha/image_0053.jpg 38 train buddha/image_0084.jpg 38 train buddha/image_0020.jpg 38 train buddha/image_0046.jpg 38 train buddha/image_0041.jpg 38 train buddha/image_0027.jpg 38 train buddha/image_0014.jpg 38 train buddha/image_0058.jpg 38 train buddha/image_0081.jpg 38 train scissors/image_0037.jpg 39 train scissors/image_0006.jpg 39 train scissors/image_0016.jpg 39 train scissors/image_0030.jpg 39 train scissors/image_0012.jpg 39 train scissors/image_0025.jpg 39 train scissors/image_0007.jpg 39 train scissors/image_0019.jpg 39 train scissors/image_0031.jpg 39 train scissors/image_0011.jpg 39 train scissors/image_0017.jpg 39 train scissors/image_0018.jpg 39 train scissors/image_0013.jpg 39 train scissors/image_0024.jpg 39 train scissors/image_0003.jpg 39 train scissors/image_0009.jpg 39 train scissors/image_0033.jpg 39 train scissors/image_0001.jpg 39 train scissors/image_0021.jpg 39 train scissors/image_0008.jpg 39 train scissors/image_0035.jpg 39 train scissors/image_0005.jpg 39 train scissors/image_0034.jpg 39 train scissors/image_0002.jpg 39 train scissors/image_0020.jpg 39 train scissors/image_0027.jpg 39 train scissors/image_0014.jpg 39 train scissors/image_0026.jpg 39 train scissors/image_0004.jpg 39 train scissors/image_0015.jpg 39 train scissors/image_0029.jpg 39 train cougar_body/image_0037.jpg 40 train cougar_body/image_0006.jpg 40 train cougar_body/image_0016.jpg 40 train cougar_body/image_0030.jpg 40 train cougar_body/image_0012.jpg 40 train cougar_body/image_0025.jpg 40 train cougar_body/image_0007.jpg 40 train cougar_body/image_0019.jpg 40 train cougar_body/image_0031.jpg 40 train cougar_body/image_0011.jpg 40 train cougar_body/image_0045.jpg 40 train cougar_body/image_0017.jpg 40 train cougar_body/image_0018.jpg 40 train cougar_body/image_0013.jpg 40 train cougar_body/image_0024.jpg 40 train cougar_body/image_0003.jpg 40 train cougar_body/image_0009.jpg 40 train cougar_body/image_0042.jpg 40 train cougar_body/image_0040.jpg 40 train cougar_body/image_0047.jpg 40 train cougar_body/image_0043.jpg 40 train cougar_body/image_0044.jpg 40 train cougar_body/image_0033.jpg 40 train cougar_body/image_0001.jpg 40 train cougar_body/image_0021.jpg 40 train cougar_body/image_0008.jpg 40 train cougar_body/image_0035.jpg 40 train cougar_body/image_0005.jpg 40 train cougar_body/image_0034.jpg 40 train cougar_body/image_0002.jpg 40 train cougar_body/image_0020.jpg 40 train cougar_body/image_0046.jpg 40 train cougar_body/image_0041.jpg 40 train cougar_body/image_0027.jpg 40 train cougar_body/image_0014.jpg 40 train cougar_body/image_0026.jpg 40 train cougar_body/image_0004.jpg 40 train binocular/image_0006.jpg 41 train binocular/image_0016.jpg 41 train binocular/image_0030.jpg 41 train binocular/image_0012.jpg 41 train binocular/image_0025.jpg 41 train binocular/image_0007.jpg 41 train binocular/image_0019.jpg 41 train binocular/image_0031.jpg 41 train binocular/image_0011.jpg 41 train binocular/image_0017.jpg 41 train binocular/image_0018.jpg 41 train binocular/image_0013.jpg 41 train binocular/image_0024.jpg 41 train binocular/image_0003.jpg 41 train binocular/image_0009.jpg 41 train binocular/image_0033.jpg 41 train binocular/image_0001.jpg 41 train binocular/image_0021.jpg 41 train binocular/image_0008.jpg 41 train binocular/image_0005.jpg 41 train binocular/image_0002.jpg 41 train binocular/image_0020.jpg 41 train binocular/image_0027.jpg 41 train binocular/image_0014.jpg 41 train binocular/image_0026.jpg 41 train binocular/image_0004.jpg 41 train grand_piano/image_0062.jpg 42 train grand_piano/image_0037.jpg 42 train grand_piano/image_0049.jpg 42 train grand_piano/image_0083.jpg 42 train grand_piano/image_0051.jpg 42 train grand_piano/image_0006.jpg 42 train grand_piano/image_0016.jpg 42 train grand_piano/image_0030.jpg 42 train grand_piano/image_0091.jpg 42 train grand_piano/image_0012.jpg 42 train grand_piano/image_0066.jpg 42 train grand_piano/image_0077.jpg 42 train grand_piano/image_0025.jpg 42 train grand_piano/image_0060.jpg 42 train grand_piano/image_0052.jpg 42 train grand_piano/image_0007.jpg 42 train grand_piano/image_0063.jpg 42 train grand_piano/image_0019.jpg 42 train grand_piano/image_0031.jpg 42 train grand_piano/image_0011.jpg 42 train grand_piano/image_0061.jpg 42 train grand_piano/image_0090.jpg 42 train grand_piano/image_0050.jpg 42 train grand_piano/image_0045.jpg 42 train grand_piano/image_0017.jpg 42 train grand_piano/image_0018.jpg 42 train grand_piano/image_0013.jpg 42 train grand_piano/image_0054.jpg 42 train grand_piano/image_0094.jpg 42 train grand_piano/image_0082.jpg 42 train grand_piano/image_0097.jpg 42 train grand_piano/image_0079.jpg 42 train grand_piano/image_0055.jpg 42 train grand_piano/image_0071.jpg 42 train grand_piano/image_0024.jpg 42 train grand_piano/image_0074.jpg 42 train grand_piano/image_0003.jpg 42 train grand_piano/image_0009.jpg 42 train grand_piano/image_0042.jpg 42 train grand_piano/image_0093.jpg 42 train grand_piano/image_0040.jpg 42 train grand_piano/image_0067.jpg 42 train grand_piano/image_0047.jpg 42 train grand_piano/image_0064.jpg 42 train grand_piano/image_0070.jpg 42 train grand_piano/image_0078.jpg 42 train grand_piano/image_0068.jpg 42 train grand_piano/image_0085.jpg 42 train grand_piano/image_0043.jpg 42 train grand_piano/image_0044.jpg 42 train grand_piano/image_0033.jpg 42 train grand_piano/image_0073.jpg 42 train grand_piano/image_0001.jpg 42 train grand_piano/image_0080.jpg 42 train grand_piano/image_0021.jpg 42 train grand_piano/image_0095.jpg 42 train grand_piano/image_0057.jpg 42 train grand_piano/image_0098.jpg 42 train grand_piano/image_0008.jpg 42 train grand_piano/image_0035.jpg 42 train grand_piano/image_0005.jpg 42 train grand_piano/image_0087.jpg 42 train grand_piano/image_0089.jpg 42 train grand_piano/image_0059.jpg 42 train grand_piano/image_0069.jpg 42 train grand_piano/image_0099.jpg 42 train grand_piano/image_0034.jpg 42 train grand_piano/image_0002.jpg 42 train grand_piano/image_0048.jpg 42 train grand_piano/image_0053.jpg 42 train grand_piano/image_0084.jpg 42 train grand_piano/image_0020.jpg 42 train grand_piano/image_0046.jpg 42 train grand_piano/image_0041.jpg 42 train grand_piano/image_0027.jpg 42 train grand_piano/image_0014.jpg 42 train grand_piano/image_0058.jpg 42 train grand_piano/image_0081.jpg 42 train grand_piano/image_0088.jpg 42 train ibis/image_0062.jpg 43 train ibis/image_0037.jpg 43 train ibis/image_0049.jpg 43 train ibis/image_0051.jpg 43 train ibis/image_0006.jpg 43 train ibis/image_0016.jpg 43 train ibis/image_0030.jpg 43 train ibis/image_0012.jpg 43 train ibis/image_0066.jpg 43 train ibis/image_0077.jpg 43 train ibis/image_0025.jpg 43 train ibis/image_0060.jpg 43 train ibis/image_0052.jpg 43 train ibis/image_0007.jpg 43 train ibis/image_0063.jpg 43 train ibis/image_0019.jpg 43 train ibis/image_0031.jpg 43 train ibis/image_0011.jpg 43 train ibis/image_0061.jpg 43 train ibis/image_0050.jpg 43 train ibis/image_0045.jpg 43 train ibis/image_0017.jpg 43 train ibis/image_0018.jpg 43 train ibis/image_0013.jpg 43 train ibis/image_0054.jpg 43 train ibis/image_0079.jpg 43 train ibis/image_0055.jpg 43 train ibis/image_0071.jpg 43 train ibis/image_0024.jpg 43 train ibis/image_0074.jpg 43 train ibis/image_0003.jpg 43 train ibis/image_0009.jpg 43 train ibis/image_0042.jpg 43 train ibis/image_0040.jpg 43 train ibis/image_0067.jpg 43 train ibis/image_0047.jpg 43 train ibis/image_0064.jpg 43 train ibis/image_0070.jpg 43 train ibis/image_0078.jpg 43 train ibis/image_0068.jpg 43 train ibis/image_0043.jpg 43 train ibis/image_0044.jpg 43 train ibis/image_0033.jpg 43 train ibis/image_0073.jpg 43 train ibis/image_0001.jpg 43 train ibis/image_0080.jpg 43 train ibis/image_0021.jpg 43 train ibis/image_0057.jpg 43 train ibis/image_0008.jpg 43 train ibis/image_0035.jpg 43 train ibis/image_0005.jpg 43 train ibis/image_0059.jpg 43 train ibis/image_0069.jpg 43 train ibis/image_0034.jpg 43 train ibis/image_0002.jpg 43 train ibis/image_0048.jpg 43 train ibis/image_0053.jpg 43 train ibis/image_0020.jpg 43 train ibis/image_0046.jpg 43 train ibis/image_0041.jpg 43 train ibis/image_0027.jpg 43 train ibis/image_0014.jpg 43 train ibis/image_0058.jpg 43 train ibis/image_0026.jpg 43 train brontosaurus/image_0037.jpg 44 train brontosaurus/image_0006.jpg 44 train brontosaurus/image_0016.jpg 44 train brontosaurus/image_0030.jpg 44 train brontosaurus/image_0012.jpg 44 train brontosaurus/image_0025.jpg 44 train brontosaurus/image_0007.jpg 44 train brontosaurus/image_0019.jpg 44 train brontosaurus/image_0031.jpg 44 train brontosaurus/image_0011.jpg 44 train brontosaurus/image_0017.jpg 44 train brontosaurus/image_0018.jpg 44 train brontosaurus/image_0013.jpg 44 train brontosaurus/image_0024.jpg 44 train brontosaurus/image_0003.jpg 44 train brontosaurus/image_0009.jpg 44 train brontosaurus/image_0042.jpg 44 train brontosaurus/image_0040.jpg 44 train brontosaurus/image_0043.jpg 44 train brontosaurus/image_0033.jpg 44 train brontosaurus/image_0001.jpg 44 train brontosaurus/image_0021.jpg 44 train brontosaurus/image_0008.jpg 44 train brontosaurus/image_0035.jpg 44 train brontosaurus/image_0005.jpg 44 train brontosaurus/image_0034.jpg 44 train brontosaurus/image_0002.jpg 44 train brontosaurus/image_0020.jpg 44 train brontosaurus/image_0041.jpg 44 train brontosaurus/image_0027.jpg 44 train brontosaurus/image_0014.jpg 44 train brontosaurus/image_0026.jpg 44 train brontosaurus/image_0004.jpg 44 train brontosaurus/image_0015.jpg 44 train saxophone/image_0037.jpg 45 train saxophone/image_0006.jpg 45 train saxophone/image_0016.jpg 45 train saxophone/image_0030.jpg 45 train saxophone/image_0012.jpg 45 train saxophone/image_0025.jpg 45 train saxophone/image_0007.jpg 45 train saxophone/image_0019.jpg 45 train saxophone/image_0031.jpg 45 train saxophone/image_0011.jpg 45 train saxophone/image_0017.jpg 45 train saxophone/image_0018.jpg 45 train saxophone/image_0013.jpg 45 train saxophone/image_0024.jpg 45 train saxophone/image_0003.jpg 45 train saxophone/image_0009.jpg 45 train saxophone/image_0040.jpg 45 train saxophone/image_0033.jpg 45 train saxophone/image_0001.jpg 45 train saxophone/image_0021.jpg 45 train saxophone/image_0008.jpg 45 train saxophone/image_0035.jpg 45 train saxophone/image_0005.jpg 45 train saxophone/image_0034.jpg 45 train saxophone/image_0002.jpg 45 train saxophone/image_0020.jpg 45 train saxophone/image_0027.jpg 45 train saxophone/image_0014.jpg 45 train saxophone/image_0026.jpg 45 train saxophone/image_0004.jpg 45 train saxophone/image_0015.jpg 45 train saxophone/image_0029.jpg 45 train stapler/image_0037.jpg 46 train stapler/image_0006.jpg 46 train stapler/image_0016.jpg 46 train stapler/image_0030.jpg 46 train stapler/image_0012.jpg 46 train stapler/image_0025.jpg 46 train stapler/image_0007.jpg 46 train stapler/image_0019.jpg 46 train stapler/image_0031.jpg 46 train stapler/image_0011.jpg 46 train stapler/image_0045.jpg 46 train stapler/image_0017.jpg 46 train stapler/image_0018.jpg 46 train stapler/image_0013.jpg 46 train stapler/image_0024.jpg 46 train stapler/image_0003.jpg 46 train stapler/image_0009.jpg 46 train stapler/image_0042.jpg 46 train stapler/image_0040.jpg 46 train stapler/image_0043.jpg 46 train stapler/image_0044.jpg 46 train stapler/image_0033.jpg 46 train stapler/image_0001.jpg 46 train stapler/image_0021.jpg 46 train stapler/image_0008.jpg 46 train stapler/image_0035.jpg 46 train stapler/image_0005.jpg 46 train stapler/image_0034.jpg 46 train stapler/image_0002.jpg 46 train stapler/image_0020.jpg 46 train stapler/image_0041.jpg 46 train stapler/image_0027.jpg 46 train stapler/image_0014.jpg 46 train stapler/image_0026.jpg 46 train stapler/image_0004.jpg 46 train stapler/image_0015.jpg 46 train electric_guitar/image_0062.jpg 47 train electric_guitar/image_0037.jpg 47 train electric_guitar/image_0049.jpg 47 train electric_guitar/image_0051.jpg 47 train electric_guitar/image_0006.jpg 47 train electric_guitar/image_0016.jpg 47 train electric_guitar/image_0030.jpg 47 train electric_guitar/image_0012.jpg 47 train electric_guitar/image_0066.jpg 47 train electric_guitar/image_0025.jpg 47 train electric_guitar/image_0060.jpg 47 train electric_guitar/image_0052.jpg 47 train electric_guitar/image_0007.jpg 47 train electric_guitar/image_0063.jpg 47 train electric_guitar/image_0019.jpg 47 train electric_guitar/image_0031.jpg 47 train electric_guitar/image_0011.jpg 47 train electric_guitar/image_0061.jpg 47 train electric_guitar/image_0050.jpg 47 train electric_guitar/image_0045.jpg 47 train electric_guitar/image_0017.jpg 47 train electric_guitar/image_0018.jpg 47 train electric_guitar/image_0013.jpg 47 train electric_guitar/image_0054.jpg 47 train electric_guitar/image_0055.jpg 47 train electric_guitar/image_0071.jpg 47 train electric_guitar/image_0024.jpg 47 train electric_guitar/image_0074.jpg 47 train electric_guitar/image_0003.jpg 47 train electric_guitar/image_0009.jpg 47 train electric_guitar/image_0042.jpg 47 train electric_guitar/image_0040.jpg 47 train electric_guitar/image_0067.jpg 47 train electric_guitar/image_0047.jpg 47 train electric_guitar/image_0064.jpg 47 train electric_guitar/image_0070.jpg 47 train electric_guitar/image_0068.jpg 47 train electric_guitar/image_0043.jpg 47 train electric_guitar/image_0044.jpg 47 train electric_guitar/image_0033.jpg 47 train electric_guitar/image_0073.jpg 47 train electric_guitar/image_0001.jpg 47 train electric_guitar/image_0021.jpg 47 train electric_guitar/image_0057.jpg 47 train electric_guitar/image_0008.jpg 47 train electric_guitar/image_0035.jpg 47 train electric_guitar/image_0005.jpg 47 train electric_guitar/image_0059.jpg 47 train electric_guitar/image_0069.jpg 47 train electric_guitar/image_0034.jpg 47 train electric_guitar/image_0002.jpg 47 train electric_guitar/image_0048.jpg 47 train electric_guitar/image_0053.jpg 47 train electric_guitar/image_0020.jpg 47 train electric_guitar/image_0046.jpg 47 train electric_guitar/image_0041.jpg 47 train electric_guitar/image_0027.jpg 47 train electric_guitar/image_0014.jpg 47 train electric_guitar/image_0058.jpg 47 train electric_guitar/image_0026.jpg 47 train octopus/image_0006.jpg 48 train octopus/image_0016.jpg 48 train octopus/image_0030.jpg 48 train octopus/image_0012.jpg 48 train octopus/image_0025.jpg 48 train octopus/image_0007.jpg 48 train octopus/image_0019.jpg 48 train octopus/image_0031.jpg 48 train octopus/image_0011.jpg 48 train octopus/image_0017.jpg 48 train octopus/image_0018.jpg 48 train octopus/image_0013.jpg 48 train octopus/image_0024.jpg 48 train octopus/image_0003.jpg 48 train octopus/image_0009.jpg 48 train octopus/image_0033.jpg 48 train octopus/image_0001.jpg 48 train octopus/image_0021.jpg 48 train octopus/image_0008.jpg 48 train octopus/image_0035.jpg 48 train octopus/image_0005.jpg 48 train octopus/image_0034.jpg 48 train octopus/image_0002.jpg 48 train octopus/image_0020.jpg 48 train octopus/image_0027.jpg 48 train octopus/image_0014.jpg 48 train octopus/image_0026.jpg 48 train octopus/image_0004.jpg 48 train kangaroo/image_0062.jpg 49 train kangaroo/image_0037.jpg 49 train kangaroo/image_0049.jpg 49 train kangaroo/image_0083.jpg 49 train kangaroo/image_0051.jpg 49 train kangaroo/image_0006.jpg 49 train kangaroo/image_0016.jpg 49 train kangaroo/image_0030.jpg 49 train kangaroo/image_0012.jpg 49 train kangaroo/image_0066.jpg 49 train kangaroo/image_0077.jpg 49 train kangaroo/image_0025.jpg 49 train kangaroo/image_0060.jpg 49 train kangaroo/image_0052.jpg 49 train kangaroo/image_0007.jpg 49 train kangaroo/image_0063.jpg 49 train kangaroo/image_0019.jpg 49 train kangaroo/image_0031.jpg 49 train kangaroo/image_0011.jpg 49 train kangaroo/image_0061.jpg 49 train kangaroo/image_0050.jpg 49 train kangaroo/image_0045.jpg 49 train kangaroo/image_0017.jpg 49 train kangaroo/image_0018.jpg 49 train kangaroo/image_0013.jpg 49 train kangaroo/image_0054.jpg 49 train kangaroo/image_0082.jpg 49 train kangaroo/image_0079.jpg 49 train kangaroo/image_0055.jpg 49 train kangaroo/image_0071.jpg 49 train kangaroo/image_0024.jpg 49 train kangaroo/image_0074.jpg 49 train kangaroo/image_0003.jpg 49 train kangaroo/image_0009.jpg 49 train kangaroo/image_0042.jpg 49 train kangaroo/image_0040.jpg 49 train kangaroo/image_0067.jpg 49 train kangaroo/image_0047.jpg 49 train kangaroo/image_0064.jpg 49 train kangaroo/image_0070.jpg 49 train kangaroo/image_0078.jpg 49 train kangaroo/image_0068.jpg 49 train kangaroo/image_0085.jpg 49 train kangaroo/image_0043.jpg 49 train kangaroo/image_0044.jpg 49 train kangaroo/image_0033.jpg 49 train kangaroo/image_0073.jpg 49 train kangaroo/image_0001.jpg 49 train kangaroo/image_0080.jpg 49 train kangaroo/image_0021.jpg 49 train kangaroo/image_0057.jpg 49 train kangaroo/image_0008.jpg 49 train kangaroo/image_0035.jpg 49 train kangaroo/image_0005.jpg 49 train kangaroo/image_0059.jpg 49 train kangaroo/image_0069.jpg 49 train kangaroo/image_0034.jpg 49 train kangaroo/image_0002.jpg 49 train kangaroo/image_0048.jpg 49 train kangaroo/image_0053.jpg 49 train kangaroo/image_0084.jpg 49 train kangaroo/image_0020.jpg 49 train kangaroo/image_0046.jpg 49 train kangaroo/image_0041.jpg 49 train kangaroo/image_0027.jpg 49 train kangaroo/image_0014.jpg 49 train kangaroo/image_0058.jpg 49 train kangaroo/image_0081.jpg 49 train okapi/image_0037.jpg 50 train okapi/image_0006.jpg 50 train okapi/image_0016.jpg 50 train okapi/image_0030.jpg 50 train okapi/image_0012.jpg 50 train okapi/image_0025.jpg 50 train okapi/image_0007.jpg 50 train okapi/image_0019.jpg 50 train okapi/image_0031.jpg 50 train okapi/image_0011.jpg 50 train okapi/image_0017.jpg 50 train okapi/image_0018.jpg 50 train okapi/image_0013.jpg 50 train okapi/image_0024.jpg 50 train okapi/image_0003.jpg 50 train okapi/image_0009.jpg 50 train okapi/image_0033.jpg 50 train okapi/image_0001.jpg 50 train okapi/image_0021.jpg 50 train okapi/image_0008.jpg 50 train okapi/image_0035.jpg 50 train okapi/image_0005.jpg 50 train okapi/image_0034.jpg 50 train okapi/image_0002.jpg 50 train okapi/image_0020.jpg 50 train okapi/image_0027.jpg 50 train okapi/image_0014.jpg 50 train okapi/image_0026.jpg 50 train okapi/image_0004.jpg 50 train okapi/image_0015.jpg 50 train okapi/image_0029.jpg 50 train sunflower/image_0062.jpg 51 train sunflower/image_0037.jpg 51 train sunflower/image_0049.jpg 51 train sunflower/image_0083.jpg 51 train sunflower/image_0051.jpg 51 train sunflower/image_0006.jpg 51 train sunflower/image_0016.jpg 51 train sunflower/image_0030.jpg 51 train sunflower/image_0012.jpg 51 train sunflower/image_0066.jpg 51 train sunflower/image_0077.jpg 51 train sunflower/image_0025.jpg 51 train sunflower/image_0060.jpg 51 train sunflower/image_0052.jpg 51 train sunflower/image_0007.jpg 51 train sunflower/image_0063.jpg 51 train sunflower/image_0019.jpg 51 train sunflower/image_0031.jpg 51 train sunflower/image_0011.jpg 51 train sunflower/image_0061.jpg 51 train sunflower/image_0050.jpg 51 train sunflower/image_0045.jpg 51 train sunflower/image_0017.jpg 51 train sunflower/image_0018.jpg 51 train sunflower/image_0013.jpg 51 train sunflower/image_0054.jpg 51 train sunflower/image_0082.jpg 51 train sunflower/image_0079.jpg 51 train sunflower/image_0055.jpg 51 train sunflower/image_0071.jpg 51 train sunflower/image_0024.jpg 51 train sunflower/image_0074.jpg 51 train sunflower/image_0003.jpg 51 train sunflower/image_0009.jpg 51 train sunflower/image_0042.jpg 51 train sunflower/image_0040.jpg 51 train sunflower/image_0067.jpg 51 train sunflower/image_0047.jpg 51 train sunflower/image_0064.jpg 51 train sunflower/image_0070.jpg 51 train sunflower/image_0078.jpg 51 train sunflower/image_0068.jpg 51 train sunflower/image_0085.jpg 51 train sunflower/image_0043.jpg 51 train sunflower/image_0044.jpg 51 train sunflower/image_0033.jpg 51 train sunflower/image_0073.jpg 51 train sunflower/image_0001.jpg 51 train sunflower/image_0080.jpg 51 train sunflower/image_0021.jpg 51 train sunflower/image_0057.jpg 51 train sunflower/image_0008.jpg 51 train sunflower/image_0035.jpg 51 train sunflower/image_0005.jpg 51 train sunflower/image_0059.jpg 51 train sunflower/image_0069.jpg 51 train sunflower/image_0034.jpg 51 train sunflower/image_0002.jpg 51 train sunflower/image_0048.jpg 51 train sunflower/image_0053.jpg 51 train sunflower/image_0084.jpg 51 train sunflower/image_0020.jpg 51 train sunflower/image_0046.jpg 51 train sunflower/image_0041.jpg 51 train sunflower/image_0027.jpg 51 train sunflower/image_0014.jpg 51 train sunflower/image_0058.jpg 51 train sunflower/image_0081.jpg 51 train garfield/image_0006.jpg 52 train garfield/image_0016.jpg 52 train garfield/image_0030.jpg 52 train garfield/image_0012.jpg 52 train garfield/image_0025.jpg 52 train garfield/image_0007.jpg 52 train garfield/image_0019.jpg 52 train garfield/image_0031.jpg 52 train garfield/image_0011.jpg 52 train garfield/image_0017.jpg 52 train garfield/image_0018.jpg 52 train garfield/image_0013.jpg 52 train garfield/image_0024.jpg 52 train garfield/image_0003.jpg 52 train garfield/image_0009.jpg 52 train garfield/image_0033.jpg 52 train garfield/image_0001.jpg 52 train garfield/image_0021.jpg 52 train garfield/image_0008.jpg 52 train garfield/image_0005.jpg 52 train garfield/image_0034.jpg 52 train garfield/image_0002.jpg 52 train garfield/image_0020.jpg 52 train garfield/image_0027.jpg 52 train garfield/image_0014.jpg 52 train garfield/image_0026.jpg 52 train garfield/image_0004.jpg 52 train bass/image_0037.jpg 53 train bass/image_0049.jpg 53 train bass/image_0051.jpg 53 train bass/image_0006.jpg 53 train bass/image_0016.jpg 53 train bass/image_0030.jpg 53 train bass/image_0012.jpg 53 train bass/image_0025.jpg 53 train bass/image_0052.jpg 53 train bass/image_0007.jpg 53 train bass/image_0019.jpg 53 train bass/image_0031.jpg 53 train bass/image_0011.jpg 53 train bass/image_0050.jpg 53 train bass/image_0045.jpg 53 train bass/image_0017.jpg 53 train bass/image_0018.jpg 53 train bass/image_0013.jpg 53 train bass/image_0054.jpg 53 train bass/image_0024.jpg 53 train bass/image_0003.jpg 53 train bass/image_0009.jpg 53 train bass/image_0042.jpg 53 train bass/image_0040.jpg 53 train bass/image_0047.jpg 53 train bass/image_0043.jpg 53 train bass/image_0044.jpg 53 train bass/image_0033.jpg 53 train bass/image_0001.jpg 53 train bass/image_0021.jpg 53 train bass/image_0008.jpg 53 train bass/image_0035.jpg 53 train bass/image_0005.jpg 53 train bass/image_0034.jpg 53 train bass/image_0002.jpg 53 train bass/image_0048.jpg 53 train bass/image_0053.jpg 53 train bass/image_0020.jpg 53 train bass/image_0046.jpg 53 train bass/image_0041.jpg 53 train bass/image_0027.jpg 53 train bass/image_0014.jpg 53 train bass/image_0026.jpg 53 train cellphone/image_0037.jpg 54 train cellphone/image_0049.jpg 54 train cellphone/image_0051.jpg 54 train cellphone/image_0006.jpg 54 train cellphone/image_0016.jpg 54 train cellphone/image_0030.jpg 54 train cellphone/image_0012.jpg 54 train cellphone/image_0025.jpg 54 train cellphone/image_0052.jpg 54 train cellphone/image_0007.jpg 54 train cellphone/image_0019.jpg 54 train cellphone/image_0031.jpg 54 train cellphone/image_0011.jpg 54 train cellphone/image_0050.jpg 54 train cellphone/image_0045.jpg 54 train cellphone/image_0017.jpg 54 train cellphone/image_0018.jpg 54 train cellphone/image_0013.jpg 54 train cellphone/image_0054.jpg 54 train cellphone/image_0055.jpg 54 train cellphone/image_0024.jpg 54 train cellphone/image_0003.jpg 54 train cellphone/image_0009.jpg 54 train cellphone/image_0042.jpg 54 train cellphone/image_0040.jpg 54 train cellphone/image_0047.jpg 54 train cellphone/image_0043.jpg 54 train cellphone/image_0044.jpg 54 train cellphone/image_0033.jpg 54 train cellphone/image_0001.jpg 54 train cellphone/image_0021.jpg 54 train cellphone/image_0057.jpg 54 train cellphone/image_0008.jpg 54 train cellphone/image_0035.jpg 54 train cellphone/image_0005.jpg 54 train cellphone/image_0059.jpg 54 train cellphone/image_0034.jpg 54 train cellphone/image_0002.jpg 54 train cellphone/image_0048.jpg 54 train cellphone/image_0053.jpg 54 train cellphone/image_0020.jpg 54 train cellphone/image_0046.jpg 54 train cellphone/image_0041.jpg 54 train cellphone/image_0027.jpg 54 train cellphone/image_0014.jpg 54 train cellphone/image_0058.jpg 54 train cellphone/image_0026.jpg 54 train brain/image_0062.jpg 55 train brain/image_0037.jpg 55 train brain/image_0049.jpg 55 train brain/image_0083.jpg 55 train brain/image_0051.jpg 55 train brain/image_0006.jpg 55 train brain/image_0016.jpg 55 train brain/image_0030.jpg 55 train brain/image_0091.jpg 55 train brain/image_0012.jpg 55 train brain/image_0066.jpg 55 train brain/image_0077.jpg 55 train brain/image_0025.jpg 55 train brain/image_0060.jpg 55 train brain/image_0052.jpg 55 train brain/image_0007.jpg 55 train brain/image_0063.jpg 55 train brain/image_0019.jpg 55 train brain/image_0031.jpg 55 train brain/image_0011.jpg 55 train brain/image_0061.jpg 55 train brain/image_0090.jpg 55 train brain/image_0050.jpg 55 train brain/image_0045.jpg 55 train brain/image_0017.jpg 55 train brain/image_0018.jpg 55 train brain/image_0013.jpg 55 train brain/image_0054.jpg 55 train brain/image_0094.jpg 55 train brain/image_0082.jpg 55 train brain/image_0097.jpg 55 train brain/image_0079.jpg 55 train brain/image_0055.jpg 55 train brain/image_0071.jpg 55 train brain/image_0024.jpg 55 train brain/image_0074.jpg 55 train brain/image_0003.jpg 55 train brain/image_0009.jpg 55 train brain/image_0042.jpg 55 train brain/image_0093.jpg 55 train brain/image_0040.jpg 55 train brain/image_0067.jpg 55 train brain/image_0047.jpg 55 train brain/image_0064.jpg 55 train brain/image_0070.jpg 55 train brain/image_0078.jpg 55 train brain/image_0068.jpg 55 train brain/image_0085.jpg 55 train brain/image_0043.jpg 55 train brain/image_0044.jpg 55 train brain/image_0033.jpg 55 train brain/image_0073.jpg 55 train brain/image_0001.jpg 55 train brain/image_0080.jpg 55 train brain/image_0021.jpg 55 train brain/image_0095.jpg 55 train brain/image_0057.jpg 55 train brain/image_0098.jpg 55 train brain/image_0008.jpg 55 train brain/image_0035.jpg 55 train brain/image_0005.jpg 55 train brain/image_0087.jpg 55 train brain/image_0089.jpg 55 train brain/image_0059.jpg 55 train brain/image_0069.jpg 55 train brain/image_0034.jpg 55 train brain/image_0002.jpg 55 train brain/image_0048.jpg 55 train brain/image_0053.jpg 55 train brain/image_0084.jpg 55 train brain/image_0020.jpg 55 train brain/image_0046.jpg 55 train brain/image_0041.jpg 55 train brain/image_0027.jpg 55 train brain/image_0014.jpg 55 train brain/image_0058.jpg 55 train brain/image_0081.jpg 55 train brain/image_0088.jpg 55 train lobster/image_0037.jpg 56 train lobster/image_0006.jpg 56 train lobster/image_0016.jpg 56 train lobster/image_0030.jpg 56 train lobster/image_0012.jpg 56 train lobster/image_0025.jpg 56 train lobster/image_0007.jpg 56 train lobster/image_0019.jpg 56 train lobster/image_0031.jpg 56 train lobster/image_0011.jpg 56 train lobster/image_0017.jpg 56 train lobster/image_0018.jpg 56 train lobster/image_0013.jpg 56 train lobster/image_0024.jpg 56 train lobster/image_0003.jpg 56 train lobster/image_0009.jpg 56 train lobster/image_0040.jpg 56 train lobster/image_0033.jpg 56 train lobster/image_0001.jpg 56 train lobster/image_0021.jpg 56 train lobster/image_0008.jpg 56 train lobster/image_0035.jpg 56 train lobster/image_0005.jpg 56 train lobster/image_0034.jpg 56 train lobster/image_0002.jpg 56 train lobster/image_0020.jpg 56 train lobster/image_0041.jpg 56 train lobster/image_0027.jpg 56 train lobster/image_0014.jpg 56 train lobster/image_0026.jpg 56 train lobster/image_0004.jpg 56 train lobster/image_0015.jpg 56 train headphone/image_0037.jpg 57 train headphone/image_0006.jpg 57 train headphone/image_0016.jpg 57 train headphone/image_0030.jpg 57 train headphone/image_0012.jpg 57 train headphone/image_0025.jpg 57 train headphone/image_0007.jpg 57 train headphone/image_0019.jpg 57 train headphone/image_0031.jpg 57 train headphone/image_0011.jpg 57 train headphone/image_0017.jpg 57 train headphone/image_0018.jpg 57 train headphone/image_0013.jpg 57 train headphone/image_0024.jpg 57 train headphone/image_0003.jpg 57 train headphone/image_0009.jpg 57 train headphone/image_0042.jpg 57 train headphone/image_0040.jpg 57 train headphone/image_0033.jpg 57 train headphone/image_0001.jpg 57 train headphone/image_0021.jpg 57 train headphone/image_0008.jpg 57 train headphone/image_0035.jpg 57 train headphone/image_0005.jpg 57 train headphone/image_0034.jpg 57 train headphone/image_0002.jpg 57 train headphone/image_0020.jpg 57 train headphone/image_0041.jpg 57 train headphone/image_0027.jpg 57 train headphone/image_0014.jpg 57 train headphone/image_0026.jpg 57 train headphone/image_0004.jpg 57 train headphone/image_0015.jpg 57 train barrel/image_0037.jpg 58 train barrel/image_0006.jpg 58 train barrel/image_0016.jpg 58 train barrel/image_0030.jpg 58 train barrel/image_0012.jpg 58 train barrel/image_0025.jpg 58 train barrel/image_0007.jpg 58 train barrel/image_0019.jpg 58 train barrel/image_0031.jpg 58 train barrel/image_0011.jpg 58 train barrel/image_0045.jpg 58 train barrel/image_0017.jpg 58 train barrel/image_0018.jpg 58 train barrel/image_0013.jpg 58 train barrel/image_0024.jpg 58 train barrel/image_0003.jpg 58 train barrel/image_0009.jpg 58 train barrel/image_0042.jpg 58 train barrel/image_0040.jpg 58 train barrel/image_0047.jpg 58 train barrel/image_0043.jpg 58 train barrel/image_0044.jpg 58 train barrel/image_0033.jpg 58 train barrel/image_0001.jpg 58 train barrel/image_0021.jpg 58 train barrel/image_0008.jpg 58 train barrel/image_0035.jpg 58 train barrel/image_0005.jpg 58 train barrel/image_0034.jpg 58 train barrel/image_0002.jpg 58 train barrel/image_0020.jpg 58 train barrel/image_0046.jpg 58 train barrel/image_0041.jpg 58 train barrel/image_0027.jpg 58 train barrel/image_0014.jpg 58 train barrel/image_0026.jpg 58 train barrel/image_0004.jpg 58 train pigeon/image_0037.jpg 59 train pigeon/image_0006.jpg 59 train pigeon/image_0016.jpg 59 train pigeon/image_0030.jpg 59 train pigeon/image_0012.jpg 59 train pigeon/image_0025.jpg 59 train pigeon/image_0007.jpg 59 train pigeon/image_0019.jpg 59 train pigeon/image_0031.jpg 59 train pigeon/image_0011.jpg 59 train pigeon/image_0045.jpg 59 train pigeon/image_0017.jpg 59 train pigeon/image_0018.jpg 59 train pigeon/image_0013.jpg 59 train pigeon/image_0024.jpg 59 train pigeon/image_0003.jpg 59 train pigeon/image_0009.jpg 59 train pigeon/image_0042.jpg 59 train pigeon/image_0040.jpg 59 train pigeon/image_0043.jpg 59 train pigeon/image_0044.jpg 59 train pigeon/image_0033.jpg 59 train pigeon/image_0001.jpg 59 train pigeon/image_0021.jpg 59 train pigeon/image_0008.jpg 59 train pigeon/image_0035.jpg 59 train pigeon/image_0005.jpg 59 train pigeon/image_0034.jpg 59 train pigeon/image_0002.jpg 59 train pigeon/image_0020.jpg 59 train pigeon/image_0041.jpg 59 train pigeon/image_0027.jpg 59 train pigeon/image_0014.jpg 59 train pigeon/image_0026.jpg 59 train pigeon/image_0004.jpg 59 train pigeon/image_0015.jpg 59 train inline_skate/image_0006.jpg 60 train inline_skate/image_0016.jpg 60 train inline_skate/image_0030.jpg 60 train inline_skate/image_0012.jpg 60 train inline_skate/image_0025.jpg 60 train inline_skate/image_0007.jpg 60 train inline_skate/image_0019.jpg 60 train inline_skate/image_0031.jpg 60 train inline_skate/image_0011.jpg 60 train inline_skate/image_0017.jpg 60 train inline_skate/image_0018.jpg 60 train inline_skate/image_0013.jpg 60 train inline_skate/image_0024.jpg 60 train inline_skate/image_0003.jpg 60 train inline_skate/image_0009.jpg 60 train inline_skate/image_0001.jpg 60 train inline_skate/image_0021.jpg 60 train inline_skate/image_0008.jpg 60 train inline_skate/image_0005.jpg 60 train inline_skate/image_0002.jpg 60 train inline_skate/image_0020.jpg 60 train inline_skate/image_0027.jpg 60 train inline_skate/image_0014.jpg 60 train inline_skate/image_0026.jpg 60 train cannon/image_0037.jpg 61 train cannon/image_0006.jpg 61 train cannon/image_0016.jpg 61 train cannon/image_0030.jpg 61 train cannon/image_0012.jpg 61 train cannon/image_0025.jpg 61 train cannon/image_0007.jpg 61 train cannon/image_0019.jpg 61 train cannon/image_0031.jpg 61 train cannon/image_0011.jpg 61 train cannon/image_0017.jpg 61 train cannon/image_0018.jpg 61 train cannon/image_0013.jpg 61 train cannon/image_0024.jpg 61 train cannon/image_0003.jpg 61 train cannon/image_0009.jpg 61 train cannon/image_0042.jpg 61 train cannon/image_0040.jpg 61 train cannon/image_0043.jpg 61 train cannon/image_0033.jpg 61 train cannon/image_0001.jpg 61 train cannon/image_0021.jpg 61 train cannon/image_0008.jpg 61 train cannon/image_0035.jpg 61 train cannon/image_0005.jpg 61 train cannon/image_0034.jpg 61 train cannon/image_0002.jpg 61 train cannon/image_0020.jpg 61 train cannon/image_0041.jpg 61 train cannon/image_0027.jpg 61 train cannon/image_0014.jpg 61 train cannon/image_0026.jpg 61 train cannon/image_0004.jpg 61 train cannon/image_0015.jpg 61 train BACKGROUND_Google/image_0316.jpg 62 train BACKGROUND_Google/image_0258.jpg 62 train BACKGROUND_Google/image_0426.jpg 62 train BACKGROUND_Google/image_0302.jpg 62 train BACKGROUND_Google/image_0171.jpg 62 train BACKGROUND_Google/image_0062.jpg 62 train BACKGROUND_Google/image_0346.jpg 62 train BACKGROUND_Google/image_0435.jpg 62 train BACKGROUND_Google/image_0341.jpg 62 train BACKGROUND_Google/image_0340.jpg 62 train BACKGROUND_Google/image_0331.jpg 62 train BACKGROUND_Google/image_0037.jpg 62 train BACKGROUND_Google/image_0049.jpg 62 train BACKGROUND_Google/image_0337.jpg 62 train BACKGROUND_Google/image_0317.jpg 62 train BACKGROUND_Google/image_0357.jpg 62 train BACKGROUND_Google/image_0083.jpg 62 train BACKGROUND_Google/image_0101.jpg 62 train BACKGROUND_Google/image_0051.jpg 62 train BACKGROUND_Google/image_0006.jpg 62 train BACKGROUND_Google/image_0142.jpg 62 train BACKGROUND_Google/image_0250.jpg 62 train BACKGROUND_Google/image_0427.jpg 62 train BACKGROUND_Google/image_0396.jpg 62 train BACKGROUND_Google/image_0332.jpg 62 train BACKGROUND_Google/image_0448.jpg 62 train BACKGROUND_Google/image_0335.jpg 62 train BACKGROUND_Google/image_0201.jpg 62 train BACKGROUND_Google/image_0016.jpg 62 train BACKGROUND_Google/image_0373.jpg 62 train BACKGROUND_Google/image_0176.jpg 62 train BACKGROUND_Google/image_0411.jpg 62 train BACKGROUND_Google/image_0155.jpg 62 train BACKGROUND_Google/image_0030.jpg 62 train BACKGROUND_Google/image_0091.jpg 62 train BACKGROUND_Google/image_0397.jpg 62 train BACKGROUND_Google/image_0012.jpg 62 train BACKGROUND_Google/image_0066.jpg 62 train BACKGROUND_Google/image_0465.jpg 62 train BACKGROUND_Google/image_0464.jpg 62 train BACKGROUND_Google/image_0283.jpg 62 train BACKGROUND_Google/image_0318.jpg 62 train BACKGROUND_Google/image_0224.jpg 62 train BACKGROUND_Google/image_0148.jpg 62 train BACKGROUND_Google/image_0077.jpg 62 train BACKGROUND_Google/image_0129.jpg 62 train BACKGROUND_Google/image_0339.jpg 62 train BACKGROUND_Google/image_0025.jpg 62 train BACKGROUND_Google/image_0328.jpg 62 train BACKGROUND_Google/image_0139.jpg 62 train BACKGROUND_Google/image_0425.jpg 62 train BACKGROUND_Google/image_0292.jpg 62 train BACKGROUND_Google/image_0348.jpg 62 train BACKGROUND_Google/image_0115.jpg 62 train BACKGROUND_Google/image_0123.jpg 62 train BACKGROUND_Google/image_0119.jpg 62 train BACKGROUND_Google/image_0060.jpg 62 train BACKGROUND_Google/image_0269.jpg 62 train BACKGROUND_Google/image_0052.jpg 62 train BACKGROUND_Google/image_0108.jpg 62 train BACKGROUND_Google/image_0239.jpg 62 train BACKGROUND_Google/image_0442.jpg 62 train BACKGROUND_Google/image_0370.jpg 62 train BACKGROUND_Google/image_0326.jpg 62 train BACKGROUND_Google/image_0223.jpg 62 train BACKGROUND_Google/image_0172.jpg 62 train BACKGROUND_Google/image_0372.jpg 62 train BACKGROUND_Google/image_0344.jpg 62 train BACKGROUND_Google/image_0390.jpg 62 train BACKGROUND_Google/image_0007.jpg 62 train BACKGROUND_Google/image_0213.jpg 62 train BACKGROUND_Google/image_0312.jpg 62 train BACKGROUND_Google/image_0463.jpg 62 train BACKGROUND_Google/image_0063.jpg 62 train BACKGROUND_Google/image_0019.jpg 62 train BACKGROUND_Google/image_0407.jpg 62 train BACKGROUND_Google/image_0192.jpg 62 train BACKGROUND_Google/image_0384.jpg 62 train BACKGROUND_Google/image_0195.jpg 62 train BACKGROUND_Google/image_0113.jpg 62 train BACKGROUND_Google/image_0127.jpg 62 train BACKGROUND_Google/image_0215.jpg 62 train BACKGROUND_Google/image_0389.jpg 62 train BACKGROUND_Google/image_0031.jpg 62 train BACKGROUND_Google/image_0349.jpg 62 train BACKGROUND_Google/image_0228.jpg 62 train BACKGROUND_Google/image_0380.jpg 62 train BACKGROUND_Google/image_0219.jpg 62 train BACKGROUND_Google/image_0353.jpg 62 train BACKGROUND_Google/image_0011.jpg 62 train BACKGROUND_Google/image_0233.jpg 62 train BACKGROUND_Google/image_0447.jpg 62 train BACKGROUND_Google/image_0061.jpg 62 train BACKGROUND_Google/image_0444.jpg 62 train BACKGROUND_Google/image_0090.jpg 62 train BACKGROUND_Google/image_0351.jpg 62 train BACKGROUND_Google/image_0261.jpg 62 train BACKGROUND_Google/image_0229.jpg 62 train BACKGROUND_Google/image_0126.jpg 62 train BACKGROUND_Google/image_0240.jpg 62 train BACKGROUND_Google/image_0467.jpg 62 train BACKGROUND_Google/image_0455.jpg 62 train BACKGROUND_Google/image_0441.jpg 62 train BACKGROUND_Google/image_0305.jpg 62 train BACKGROUND_Google/image_0334.jpg 62 train BACKGROUND_Google/image_0354.jpg 62 train BACKGROUND_Google/image_0218.jpg 62 train BACKGROUND_Google/image_0366.jpg 62 train BACKGROUND_Google/image_0288.jpg 62 train BACKGROUND_Google/image_0216.jpg 62 train BACKGROUND_Google/image_0459.jpg 62 train BACKGROUND_Google/image_0429.jpg 62 train BACKGROUND_Google/image_0277.jpg 62 train BACKGROUND_Google/image_0358.jpg 62 train BACKGROUND_Google/image_0050.jpg 62 train BACKGROUND_Google/image_0246.jpg 62 train BACKGROUND_Google/image_0361.jpg 62 train BACKGROUND_Google/image_0177.jpg 62 train BACKGROUND_Google/image_0356.jpg 62 train BACKGROUND_Google/image_0196.jpg 62 train BACKGROUND_Google/image_0293.jpg 62 train BACKGROUND_Google/image_0406.jpg 62 train BACKGROUND_Google/image_0347.jpg 62 train BACKGROUND_Google/image_0443.jpg 62 train BACKGROUND_Google/image_0187.jpg 62 train BACKGROUND_Google/image_0234.jpg 62 train BACKGROUND_Google/image_0045.jpg 62 train BACKGROUND_Google/image_0303.jpg 62 train BACKGROUND_Google/image_0363.jpg 62 train BACKGROUND_Google/image_0017.jpg 62 train BACKGROUND_Google/image_0131.jpg 62 train BACKGROUND_Google/image_0296.jpg 62 train BACKGROUND_Google/image_0150.jpg 62 train BACKGROUND_Google/image_0018.jpg 62 train BACKGROUND_Google/image_0350.jpg 62 train BACKGROUND_Google/image_0209.jpg 62 train BACKGROUND_Google/image_0013.jpg 62 train BACKGROUND_Google/image_0054.jpg 62 train BACKGROUND_Google/image_0094.jpg 62 train BACKGROUND_Google/image_0082.jpg 62 train BACKGROUND_Google/image_0097.jpg 62 train BACKGROUND_Google/image_0130.jpg 62 train BACKGROUND_Google/image_0457.jpg 62 train BACKGROUND_Google/image_0382.jpg 62 train BACKGROUND_Google/image_0141.jpg 62 train BACKGROUND_Google/image_0144.jpg 62 train BACKGROUND_Google/image_0109.jpg 62 train BACKGROUND_Google/image_0079.jpg 62 train BACKGROUND_Google/image_0306.jpg 62 train BACKGROUND_Google/image_0398.jpg 62 train BACKGROUND_Google/image_0055.jpg 62 train BACKGROUND_Google/image_0367.jpg 62 train BACKGROUND_Google/image_0365.jpg 62 train BACKGROUND_Google/image_0217.jpg 62 train BACKGROUND_Google/image_0343.jpg 62 train BACKGROUND_Google/image_0432.jpg 62 train BACKGROUND_Google/image_0071.jpg 62 train BACKGROUND_Google/image_0282.jpg 62 train BACKGROUND_Google/image_0393.jpg 62 train BACKGROUND_Google/image_0287.jpg 62 train BACKGROUND_Google/image_0198.jpg 62 train BACKGROUND_Google/image_0262.jpg 62 train BACKGROUND_Google/image_0462.jpg 62 train BACKGROUND_Google/image_0168.jpg 62 train BACKGROUND_Google/image_0368.jpg 62 train BACKGROUND_Google/image_0420.jpg 62 train BACKGROUND_Google/image_0211.jpg 62 train BACKGROUND_Google/image_0270.jpg 62 train BACKGROUND_Google/image_0103.jpg 62 train BACKGROUND_Google/image_0024.jpg 62 train BACKGROUND_Google/image_0297.jpg 62 train BACKGROUND_Google/image_0188.jpg 62 train BACKGROUND_Google/image_0105.jpg 62 train BACKGROUND_Google/image_0074.jpg 62 train BACKGROUND_Google/image_0294.jpg 62 train BACKGROUND_Google/image_0440.jpg 62 train BACKGROUND_Google/image_0391.jpg 62 train BACKGROUND_Google/image_0189.jpg 62 train BACKGROUND_Google/image_0003.jpg 62 train BACKGROUND_Google/image_0009.jpg 62 train BACKGROUND_Google/image_0387.jpg 62 train BACKGROUND_Google/image_0275.jpg 62 train BACKGROUND_Google/image_0226.jpg 62 train BACKGROUND_Google/image_0185.jpg 62 train BACKGROUND_Google/image_0118.jpg 62 train BACKGROUND_Google/image_0304.jpg 62 train BACKGROUND_Google/image_0278.jpg 62 train BACKGROUND_Google/image_0230.jpg 62 train BACKGROUND_Google/image_0404.jpg 62 train BACKGROUND_Google/tmp 62 train BACKGROUND_Google/image_0460.jpg 62 train BACKGROUND_Google/image_0134.jpg 62 train BACKGROUND_Google/image_0249.jpg 62 train BACKGROUND_Google/image_0042.jpg 62 train BACKGROUND_Google/image_0352.jpg 62 train BACKGROUND_Google/image_0333.jpg 62 train BACKGROUND_Google/image_0221.jpg 62 train BACKGROUND_Google/image_0093.jpg 62 train BACKGROUND_Google/image_0266.jpg 62 train BACKGROUND_Google/image_0412.jpg 62 train BACKGROUND_Google/image_0301.jpg 62 train BACKGROUND_Google/image_0392.jpg 62 train BACKGROUND_Google/image_0254.jpg 62 train BACKGROUND_Google/image_0321.jpg 62 train BACKGROUND_Google/image_0386.jpg 62 train BACKGROUND_Google/image_0232.jpg 62 train BACKGROUND_Google/image_0324.jpg 62 train BACKGROUND_Google/image_0040.jpg 62 train BACKGROUND_Google/image_0067.jpg 62 train BACKGROUND_Google/image_0310.jpg 62 train BACKGROUND_Google/image_0330.jpg 62 train BACKGROUND_Google/image_0122.jpg 62 train BACKGROUND_Google/image_0205.jpg 62 train BACKGROUND_Google/image_0161.jpg 62 train BACKGROUND_Google/image_0146.jpg 62 train BACKGROUND_Google/image_0300.jpg 62 train BACKGROUND_Google/image_0369.jpg 62 train BACKGROUND_Google/image_0451.jpg 62 train BACKGROUND_Google/image_0461.jpg 62 train BACKGROUND_Google/image_0204.jpg 62 train BACKGROUND_Google/image_0125.jpg 62 train BACKGROUND_Google/image_0047.jpg 62 train BACKGROUND_Google/image_0285.jpg 62 train BACKGROUND_Google/image_0160.jpg 62 train BACKGROUND_Google/image_0422.jpg 62 train BACKGROUND_Google/image_0064.jpg 62 train BACKGROUND_Google/image_0070.jpg 62 train BACKGROUND_Google/image_0264.jpg 62 train BACKGROUND_Google/image_0112.jpg 62 train BACKGROUND_Google/image_0078.jpg 62 train BACKGROUND_Google/image_0068.jpg 62 train BACKGROUND_Google/image_0433.jpg 62 train BACKGROUND_Google/image_0085.jpg 62 train BACKGROUND_Google/image_0410.jpg 62 train BACKGROUND_Google/image_0110.jpg 62 train BACKGROUND_Google/image_0259.jpg 62 train BACKGROUND_Google/image_0143.jpg 62 train BACKGROUND_Google/image_0043.jpg 62 train BACKGROUND_Google/image_0162.jpg 62 train BACKGROUND_Google/image_0132.jpg 62 train BACKGROUND_Google/image_0157.jpg 62 train BACKGROUND_Google/image_0206.jpg 62 train BACKGROUND_Google/image_0376.jpg 62 train BACKGROUND_Google/image_0319.jpg 62 train BACKGROUND_Google/image_0371.jpg 62 train BACKGROUND_Google/image_0044.jpg 62 train BACKGROUND_Google/image_0199.jpg 62 train BACKGROUND_Google/image_0033.jpg 62 train BACKGROUND_Google/image_0136.jpg 62 train BACKGROUND_Google/image_0180.jpg 62 train BACKGROUND_Google/image_0260.jpg 62 train BACKGROUND_Google/image_0184.jpg 62 train BACKGROUND_Google/image_0073.jpg 62 train BACKGROUND_Google/image_0395.jpg 62 train BACKGROUND_Google/image_0257.jpg 62 train BACKGROUND_Google/image_0364.jpg 62 train BACKGROUND_Google/image_0001.jpg 62 train BACKGROUND_Google/image_0167.jpg 62 train BACKGROUND_Google/image_0284.jpg 62 train BACKGROUND_Google/image_0375.jpg 62 train BACKGROUND_Google/image_0080.jpg 62 train BACKGROUND_Google/image_0156.jpg 62 train BACKGROUND_Google/image_0021.jpg 62 train BACKGROUND_Google/image_0325.jpg 62 train BACKGROUND_Google/image_0267.jpg 62 train BACKGROUND_Google/image_0299.jpg 62 train BACKGROUND_Google/image_0421.jpg 62 train BACKGROUND_Google/image_0436.jpg 62 train BACKGROUND_Google/image_0360.jpg 62 train BACKGROUND_Google/image_0095.jpg 62 train BACKGROUND_Google/image_0057.jpg 62 train BACKGROUND_Google/image_0403.jpg 62 train BACKGROUND_Google/image_0114.jpg 62 train BACKGROUND_Google/image_0153.jpg 62 train BACKGROUND_Google/image_0416.jpg 62 train BACKGROUND_Google/image_0098.jpg 62 train BACKGROUND_Google/image_0256.jpg 62 train BACKGROUND_Google/image_0402.jpg 62 train BACKGROUND_Google/image_0203.jpg 62 train BACKGROUND_Google/image_0338.jpg 62 train BACKGROUND_Google/image_0265.jpg 62 train BACKGROUND_Google/image_0147.jpg 62 train BACKGROUND_Google/image_0008.jpg 62 train BACKGROUND_Google/image_0446.jpg 62 train BACKGROUND_Google/image_0207.jpg 62 train BACKGROUND_Google/image_0208.jpg 62 train BACKGROUND_Google/image_0222.jpg 62 train BACKGROUND_Google/image_0320.jpg 62 train BACKGROUND_Google/image_0227.jpg 62 train BACKGROUND_Google/image_0286.jpg 62 train BACKGROUND_Google/image_0035.jpg 62 train BACKGROUND_Google/image_0399.jpg 62 train BACKGROUND_Google/image_0166.jpg 62 train BACKGROUND_Google/image_0169.jpg 62 train BACKGROUND_Google/image_0191.jpg 62 train BACKGROUND_Google/image_0128.jpg 62 train BACKGROUND_Google/image_0414.jpg 62 train BACKGROUND_Google/image_0102.jpg 62 train BACKGROUND_Google/image_0434.jpg 62 train BACKGROUND_Google/image_0178.jpg 62 train BACKGROUND_Google/image_0431.jpg 62 train BACKGROUND_Google/image_0243.jpg 62 train BACKGROUND_Google/image_0005.jpg 62 train BACKGROUND_Google/image_0263.jpg 62 train BACKGROUND_Google/image_0173.jpg 62 train BACKGROUND_Google/image_0087.jpg 62 train BACKGROUND_Google/image_0417.jpg 62 train BACKGROUND_Google/image_0089.jpg 62 train BACKGROUND_Google/image_0059.jpg 62 train BACKGROUND_Google/image_0253.jpg 62 train BACKGROUND_Google/image_0159.jpg 62 train BACKGROUND_Google/image_0225.jpg 62 train BACKGROUND_Google/image_0445.jpg 62 train BACKGROUND_Google/image_0186.jpg 62 train BACKGROUND_Google/image_0388.jpg 62 train BACKGROUND_Google/image_0069.jpg 62 train BACKGROUND_Google/image_0315.jpg 62 train BACKGROUND_Google/image_0295.jpg 62 train BACKGROUND_Google/image_0200.jpg 62 train BACKGROUND_Google/image_0099.jpg 62 train BACKGROUND_Google/image_0336.jpg 62 train BACKGROUND_Google/image_0034.jpg 62 train BACKGROUND_Google/image_0409.jpg 62 train BACKGROUND_Google/image_0314.jpg 62 train BACKGROUND_Google/image_0276.jpg 62 train BACKGROUND_Google/image_0121.jpg 62 train BACKGROUND_Google/image_0438.jpg 62 train BACKGROUND_Google/image_0428.jpg 62 train BACKGROUND_Google/image_0181.jpg 62 train BACKGROUND_Google/image_0280.jpg 62 train BACKGROUND_Google/image_0183.jpg 62 train BACKGROUND_Google/image_0383.jpg 62 train BACKGROUND_Google/image_0175.jpg 62 train BACKGROUND_Google/image_0002.jpg 62 train BACKGROUND_Google/image_0291.jpg 62 train BACKGROUND_Google/image_0165.jpg 62 train BACKGROUND_Google/image_0154.jpg 62 train BACKGROUND_Google/image_0271.jpg 62 train BACKGROUND_Google/image_0193.jpg 62 train BACKGROUND_Google/image_0048.jpg 62 train BACKGROUND_Google/image_0437.jpg 62 train BACKGROUND_Google/image_0053.jpg 62 train BACKGROUND_Google/image_0255.jpg 62 train BACKGROUND_Google/image_0450.jpg 62 train BACKGROUND_Google/image_0084.jpg 62 train BACKGROUND_Google/image_0313.jpg 62 train BACKGROUND_Google/image_0020.jpg 62 train BACKGROUND_Google/image_0251.jpg 62 train BACKGROUND_Google/image_0046.jpg 62 train BACKGROUND_Google/image_0041.jpg 62 train BACKGROUND_Google/image_0027.jpg 62 train BACKGROUND_Google/image_0322.jpg 62 train BACKGROUND_Google/image_0458.jpg 62 train BACKGROUND_Google/image_0014.jpg 62 train BACKGROUND_Google/image_0058.jpg 62 train BACKGROUND_Google/image_0408.jpg 62 train BACKGROUND_Google/image_0329.jpg 62 train BACKGROUND_Google/image_0212.jpg 62 train BACKGROUND_Google/image_0163.jpg 62 train BACKGROUND_Google/image_0202.jpg 62 train BACKGROUND_Google/image_0104.jpg 62 train BACKGROUND_Google/image_0081.jpg 62 train BACKGROUND_Google/image_0272.jpg 62 train BACKGROUND_Google/image_0400.jpg 62 train BACKGROUND_Google/image_0237.jpg 62 train BACKGROUND_Google/image_0308.jpg 62 train BACKGROUND_Google/image_0149.jpg 62 train BACKGROUND_Google/image_0088.jpg 62 train BACKGROUND_Google/image_0309.jpg 62 train BACKGROUND_Google/image_0026.jpg 62 train BACKGROUND_Google/image_0231.jpg 62 train BACKGROUND_Google/image_0004.jpg 62 train BACKGROUND_Google/image_0015.jpg 62 train BACKGROUND_Google/image_0355.jpg 62 train mandolin/image_0037.jpg 63 train mandolin/image_0006.jpg 63 train mandolin/image_0016.jpg 63 train mandolin/image_0030.jpg 63 train mandolin/image_0012.jpg 63 train mandolin/image_0025.jpg 63 train mandolin/image_0007.jpg 63 train mandolin/image_0019.jpg 63 train mandolin/image_0031.jpg 63 train mandolin/image_0011.jpg 63 train mandolin/image_0017.jpg 63 train mandolin/image_0018.jpg 63 train mandolin/image_0013.jpg 63 train mandolin/image_0024.jpg 63 train mandolin/image_0003.jpg 63 train mandolin/image_0009.jpg 63 train mandolin/image_0042.jpg 63 train mandolin/image_0040.jpg 63 train mandolin/image_0043.jpg 63 train mandolin/image_0033.jpg 63 train mandolin/image_0001.jpg 63 train mandolin/image_0021.jpg 63 train mandolin/image_0008.jpg 63 train mandolin/image_0035.jpg 63 train mandolin/image_0005.jpg 63 train mandolin/image_0034.jpg 63 train mandolin/image_0002.jpg 63 train mandolin/image_0020.jpg 63 train mandolin/image_0041.jpg 63 train mandolin/image_0027.jpg 63 train mandolin/image_0014.jpg 63 train mandolin/image_0026.jpg 63 train mandolin/image_0004.jpg 63 train mandolin/image_0015.jpg 63 train Motorbikes/image_0316.jpg 64 train Motorbikes/image_0716.jpg 64 train Motorbikes/image_0258.jpg 64 train Motorbikes/image_0426.jpg 64 train Motorbikes/image_0511.jpg 64 train Motorbikes/image_0676.jpg 64 train Motorbikes/image_0528.jpg 64 train Motorbikes/image_0720.jpg 64 train Motorbikes/image_0556.jpg 64 train Motorbikes/image_0302.jpg 64 train Motorbikes/image_0171.jpg 64 train Motorbikes/image_0062.jpg 64 train Motorbikes/image_0504.jpg 64 train Motorbikes/image_0346.jpg 64 train Motorbikes/image_0546.jpg 64 train Motorbikes/image_0435.jpg 64 train Motorbikes/image_0341.jpg 64 train Motorbikes/image_0776.jpg 64 train Motorbikes/image_0340.jpg 64 train Motorbikes/image_0331.jpg 64 train Motorbikes/image_0037.jpg 64 train Motorbikes/image_0737.jpg 64 train Motorbikes/image_0786.jpg 64 train Motorbikes/image_0560.jpg 64 train Motorbikes/image_0472.jpg 64 train Motorbikes/image_0049.jpg 64 train Motorbikes/image_0337.jpg 64 train Motorbikes/image_0779.jpg 64 train Motorbikes/image_0317.jpg 64 train Motorbikes/image_0357.jpg 64 train Motorbikes/image_0083.jpg 64 train Motorbikes/image_0101.jpg 64 train Motorbikes/image_0051.jpg 64 train Motorbikes/image_0006.jpg 64 train Motorbikes/image_0142.jpg 64 train Motorbikes/image_0681.jpg 64 train Motorbikes/image_0250.jpg 64 train Motorbikes/image_0427.jpg 64 train Motorbikes/image_0486.jpg 64 train Motorbikes/image_0396.jpg 64 train Motorbikes/image_0733.jpg 64 train Motorbikes/image_0332.jpg 64 train Motorbikes/image_0448.jpg 64 train Motorbikes/image_0335.jpg 64 train Motorbikes/image_0752.jpg 64 train Motorbikes/image_0201.jpg 64 train Motorbikes/image_0505.jpg 64 train Motorbikes/image_0016.jpg 64 train Motorbikes/image_0373.jpg 64 train Motorbikes/image_0772.jpg 64 train Motorbikes/image_0565.jpg 64 train Motorbikes/image_0513.jpg 64 train Motorbikes/image_0709.jpg 64 train Motorbikes/image_0176.jpg 64 train Motorbikes/image_0411.jpg 64 train Motorbikes/image_0155.jpg 64 train Motorbikes/image_0030.jpg 64 train Motorbikes/image_0091.jpg 64 train Motorbikes/image_0664.jpg 64 train Motorbikes/image_0563.jpg 64 train Motorbikes/image_0397.jpg 64 train Motorbikes/image_0618.jpg 64 train Motorbikes/image_0012.jpg 64 train Motorbikes/image_0066.jpg 64 train Motorbikes/image_0657.jpg 64 train Motorbikes/image_0765.jpg 64 train Motorbikes/image_0585.jpg 64 train Motorbikes/image_0612.jpg 64 train Motorbikes/image_0544.jpg 64 train Motorbikes/image_0703.jpg 64 train Motorbikes/image_0465.jpg 64 train Motorbikes/image_0728.jpg 64 train Motorbikes/image_0464.jpg 64 train Motorbikes/image_0283.jpg 64 train Motorbikes/image_0551.jpg 64 train Motorbikes/image_0318.jpg 64 train Motorbikes/image_0224.jpg 64 train Motorbikes/image_0771.jpg 64 train Motorbikes/image_0148.jpg 64 train Motorbikes/image_0501.jpg 64 train Motorbikes/image_0496.jpg 64 train Motorbikes/image_0077.jpg 64 train Motorbikes/image_0129.jpg 64 train Motorbikes/image_0339.jpg 64 train Motorbikes/image_0554.jpg 64 train Motorbikes/image_0025.jpg 64 train Motorbikes/image_0328.jpg 64 train Motorbikes/image_0549.jpg 64 train Motorbikes/image_0659.jpg 64 train Motorbikes/image_0540.jpg 64 train Motorbikes/image_0518.jpg 64 train Motorbikes/image_0479.jpg 64 train Motorbikes/image_0539.jpg 64 train Motorbikes/image_0527.jpg 64 train Motorbikes/image_0783.jpg 64 train Motorbikes/image_0621.jpg 64 train Motorbikes/image_0476.jpg 64 train Motorbikes/image_0139.jpg 64 train Motorbikes/image_0519.jpg 64 train Motorbikes/image_0425.jpg 64 train Motorbikes/image_0292.jpg 64 train Motorbikes/image_0348.jpg 64 train Motorbikes/image_0499.jpg 64 train Motorbikes/image_0115.jpg 64 train Motorbikes/image_0123.jpg 64 train Motorbikes/image_0119.jpg 64 train Motorbikes/image_0670.jpg 64 train Motorbikes/image_0721.jpg 64 train Motorbikes/image_0639.jpg 64 train Motorbikes/image_0060.jpg 64 train Motorbikes/image_0269.jpg 64 train Motorbikes/image_0052.jpg 64 train Motorbikes/image_0108.jpg 64 train Motorbikes/image_0239.jpg 64 train Motorbikes/image_0442.jpg 64 train Motorbikes/image_0684.jpg 64 train Motorbikes/image_0370.jpg 64 train Motorbikes/image_0491.jpg 64 train Motorbikes/image_0469.jpg 64 train Motorbikes/image_0326.jpg 64 train Motorbikes/image_0223.jpg 64 train Motorbikes/image_0172.jpg 64 train Motorbikes/image_0372.jpg 64 train Motorbikes/image_0704.jpg 64 train Motorbikes/image_0344.jpg 64 train Motorbikes/image_0390.jpg 64 train Motorbikes/image_0007.jpg 64 train Motorbikes/image_0213.jpg 64 train Motorbikes/image_0739.jpg 64 train Motorbikes/image_0312.jpg 64 train Motorbikes/image_0463.jpg 64 train Motorbikes/image_0063.jpg 64 train Motorbikes/image_0019.jpg 64 train Motorbikes/image_0407.jpg 64 train Motorbikes/image_0192.jpg 64 train Motorbikes/image_0490.jpg 64 train Motorbikes/image_0533.jpg 64 train Motorbikes/image_0743.jpg 64 train Motorbikes/image_0384.jpg 64 train Motorbikes/image_0195.jpg 64 train Motorbikes/image_0113.jpg 64 train Motorbikes/image_0677.jpg 64 train Motorbikes/image_0127.jpg 64 train Motorbikes/image_0215.jpg 64 train Motorbikes/image_0389.jpg 64 train Motorbikes/image_0031.jpg 64 train Motorbikes/image_0349.jpg 64 train Motorbikes/image_0228.jpg 64 train Motorbikes/image_0380.jpg 64 train Motorbikes/image_0219.jpg 64 train Motorbikes/image_0796.jpg 64 train Motorbikes/image_0688.jpg 64 train Motorbikes/image_0597.jpg 64 train Motorbikes/image_0353.jpg 64 train Motorbikes/image_0011.jpg 64 train Motorbikes/image_0562.jpg 64 train Motorbikes/image_0233.jpg 64 train Motorbikes/image_0447.jpg 64 train Motorbikes/image_0747.jpg 64 train Motorbikes/image_0477.jpg 64 train Motorbikes/image_0680.jpg 64 train Motorbikes/image_0061.jpg 64 train Motorbikes/image_0444.jpg 64 train Motorbikes/image_0090.jpg 64 train Motorbikes/image_0745.jpg 64 train Motorbikes/image_0351.jpg 64 train Motorbikes/image_0261.jpg 64 train Motorbikes/image_0229.jpg 64 train Motorbikes/image_0126.jpg 64 train Motorbikes/image_0761.jpg 64 train Motorbikes/image_0775.jpg 64 train Motorbikes/image_0794.jpg 64 train Motorbikes/image_0240.jpg 64 train Motorbikes/image_0467.jpg 64 train Motorbikes/image_0768.jpg 64 train Motorbikes/image_0455.jpg 64 train Motorbikes/image_0441.jpg 64 train Motorbikes/image_0305.jpg 64 train Motorbikes/image_0572.jpg 64 train Motorbikes/image_0334.jpg 64 train Motorbikes/image_0602.jpg 64 train Motorbikes/image_0354.jpg 64 train Motorbikes/image_0542.jpg 64 train Motorbikes/image_0218.jpg 64 train Motorbikes/image_0573.jpg 64 train Motorbikes/image_0366.jpg 64 train Motorbikes/image_0288.jpg 64 train Motorbikes/image_0583.jpg 64 train Motorbikes/image_0216.jpg 64 train Motorbikes/image_0459.jpg 64 train Motorbikes/image_0429.jpg 64 train Motorbikes/image_0277.jpg 64 train Motorbikes/image_0623.jpg 64 train Motorbikes/image_0358.jpg 64 train Motorbikes/image_0588.jpg 64 train Motorbikes/image_0050.jpg 64 train Motorbikes/image_0246.jpg 64 train Motorbikes/image_0361.jpg 64 train Motorbikes/image_0177.jpg 64 train Motorbikes/image_0356.jpg 64 train Motorbikes/image_0648.jpg 64 train Motorbikes/image_0789.jpg 64 train Motorbikes/image_0196.jpg 64 train Motorbikes/image_0293.jpg 64 train Motorbikes/image_0406.jpg 64 train Motorbikes/image_0596.jpg 64 train Motorbikes/image_0640.jpg 64 train Motorbikes/image_0575.jpg 64 train Motorbikes/image_0625.jpg 64 train Motorbikes/image_0347.jpg 64 train Motorbikes/image_0521.jpg 64 train Motorbikes/image_0443.jpg 64 train Motorbikes/image_0187.jpg 64 train Motorbikes/image_0558.jpg 64 train Motorbikes/image_0234.jpg 64 train Motorbikes/image_0045.jpg 64 train Motorbikes/image_0303.jpg 64 train Motorbikes/image_0363.jpg 64 train Motorbikes/image_0679.jpg 64 train Motorbikes/image_0017.jpg 64 train Motorbikes/image_0131.jpg 64 train Motorbikes/image_0762.jpg 64 train Motorbikes/image_0492.jpg 64 train Motorbikes/image_0296.jpg 64 train Motorbikes/image_0555.jpg 64 train Motorbikes/image_0735.jpg 64 train Motorbikes/image_0150.jpg 64 train Motorbikes/image_0018.jpg 64 train Motorbikes/image_0350.jpg 64 train Motorbikes/image_0795.jpg 64 train Motorbikes/image_0209.jpg 64 train Motorbikes/image_0013.jpg 64 train Motorbikes/image_0570.jpg 64 train Motorbikes/image_0748.jpg 64 train Motorbikes/image_0569.jpg 64 train Motorbikes/image_0696.jpg 64 train Motorbikes/image_0614.jpg 64 train Motorbikes/image_0054.jpg 64 train Motorbikes/image_0629.jpg 64 train Motorbikes/image_0094.jpg 64 train Motorbikes/image_0613.jpg 64 train Motorbikes/image_0082.jpg 64 train Motorbikes/image_0097.jpg 64 train Motorbikes/image_0130.jpg 64 train Motorbikes/image_0457.jpg 64 train Motorbikes/image_0488.jpg 64 train Motorbikes/image_0382.jpg 64 train Motorbikes/image_0484.jpg 64 train Motorbikes/image_0141.jpg 64 train Motorbikes/image_0700.jpg 64 train Motorbikes/image_0144.jpg 64 train Motorbikes/image_0109.jpg 64 train Motorbikes/image_0645.jpg 64 train Motorbikes/image_0468.jpg 64 train Motorbikes/image_0515.jpg 64 train Motorbikes/image_0079.jpg 64 train Motorbikes/image_0306.jpg 64 train Motorbikes/image_0398.jpg 64 train Motorbikes/image_0055.jpg 64 train Motorbikes/image_0367.jpg 64 train Motorbikes/image_0365.jpg 64 train Motorbikes/image_0566.jpg 64 train Motorbikes/image_0217.jpg 64 train Motorbikes/image_0343.jpg 64 train Motorbikes/image_0432.jpg 64 train Motorbikes/image_0538.jpg 64 train Motorbikes/image_0600.jpg 64 train Motorbikes/image_0502.jpg 64 train Motorbikes/image_0661.jpg 64 train Motorbikes/image_0731.jpg 64 train Motorbikes/image_0071.jpg 64 train Motorbikes/image_0590.jpg 64 train Motorbikes/image_0282.jpg 64 train Motorbikes/image_0393.jpg 64 train Motorbikes/image_0287.jpg 64 train Motorbikes/image_0198.jpg 64 train Motorbikes/image_0262.jpg 64 train Motorbikes/image_0686.jpg 64 train Motorbikes/image_0462.jpg 64 train Motorbikes/image_0698.jpg 64 train Motorbikes/image_0168.jpg 64 train Motorbikes/image_0368.jpg 64 train Motorbikes/image_0420.jpg 64 train Motorbikes/image_0650.jpg 64 train Motorbikes/image_0763.jpg 64 train Motorbikes/image_0211.jpg 64 train Motorbikes/image_0537.jpg 64 train Motorbikes/image_0580.jpg 64 train Motorbikes/image_0627.jpg 64 train Motorbikes/image_0270.jpg 64 train Motorbikes/image_0594.jpg 64 train Motorbikes/image_0103.jpg 64 train Motorbikes/image_0643.jpg 64 train Motorbikes/image_0591.jpg 64 train Motorbikes/image_0024.jpg 64 train Motorbikes/image_0787.jpg 64 train Motorbikes/image_0297.jpg 64 train Motorbikes/image_0188.jpg 64 train Motorbikes/image_0792.jpg 64 train Motorbikes/image_0105.jpg 64 train Motorbikes/image_0074.jpg 64 train Motorbikes/image_0294.jpg 64 train Motorbikes/image_0440.jpg 64 train Motorbikes/image_0391.jpg 64 train Motorbikes/image_0189.jpg 64 train Motorbikes/image_0003.jpg 64 train Motorbikes/image_0009.jpg 64 train Motorbikes/image_0387.jpg 64 train Motorbikes/image_0275.jpg 64 train Motorbikes/image_0226.jpg 64 train Motorbikes/image_0628.jpg 64 train Motorbikes/image_0718.jpg 64 train Motorbikes/image_0714.jpg 64 train Motorbikes/image_0658.jpg 64 train Motorbikes/image_0601.jpg 64 train Motorbikes/image_0185.jpg 64 train Motorbikes/image_0790.jpg 64 train Motorbikes/image_0656.jpg 64 train Motorbikes/image_0567.jpg 64 train Motorbikes/image_0485.jpg 64 train Motorbikes/image_0541.jpg 64 train Motorbikes/image_0118.jpg 64 train Motorbikes/image_0652.jpg 64 train Motorbikes/image_0304.jpg 64 train Motorbikes/image_0278.jpg 64 train Motorbikes/image_0512.jpg 64 train Motorbikes/image_0230.jpg 64 train Motorbikes/image_0474.jpg 64 train Motorbikes/image_0553.jpg 64 train Motorbikes/image_0404.jpg 64 train Motorbikes/image_0487.jpg 64 train Motorbikes/image_0531.jpg 64 train Motorbikes/image_0638.jpg 64 train Motorbikes/image_0525.jpg 64 train Motorbikes/image_0482.jpg 64 train Motorbikes/image_0522.jpg 64 train Motorbikes/image_0460.jpg 64 train Motorbikes/image_0608.jpg 64 train Motorbikes/image_0134.jpg 64 train Motorbikes/image_0249.jpg 64 train Motorbikes/image_0042.jpg 64 train Motorbikes/image_0730.jpg 64 train Motorbikes/image_0352.jpg 64 train Motorbikes/image_0333.jpg 64 train Motorbikes/image_0221.jpg 64 train Motorbikes/image_0093.jpg 64 train Motorbikes/image_0266.jpg 64 train Motorbikes/image_0412.jpg 64 train Motorbikes/image_0759.jpg 64 train Motorbikes/image_0470.jpg 64 train Motorbikes/image_0497.jpg 64 train Motorbikes/image_0751.jpg 64 train Motorbikes/image_0301.jpg 64 train Motorbikes/image_0392.jpg 64 train Motorbikes/image_0473.jpg 64 train Motorbikes/image_0254.jpg 64 train Motorbikes/image_0561.jpg 64 train Motorbikes/image_0530.jpg 64 train Motorbikes/image_0321.jpg 64 train Motorbikes/image_0557.jpg 64 train Motorbikes/image_0386.jpg 64 train Motorbikes/image_0232.jpg 64 train Motorbikes/image_0324.jpg 64 train Motorbikes/image_0040.jpg 64 train Motorbikes/image_0067.jpg 64 train Motorbikes/image_0769.jpg 64 train Motorbikes/image_0310.jpg 64 train Motorbikes/image_0330.jpg 64 train Motorbikes/image_0548.jpg 64 train Motorbikes/image_0122.jpg 64 train Motorbikes/image_0671.jpg 64 train Motorbikes/image_0205.jpg 64 train Motorbikes/image_0161.jpg 64 train Motorbikes/image_0146.jpg 64 train Motorbikes/image_0300.jpg 64 train Motorbikes/image_0369.jpg 64 train Motorbikes/image_0451.jpg 64 train Motorbikes/image_0461.jpg 64 train Motorbikes/image_0204.jpg 64 train Motorbikes/image_0125.jpg 64 train Motorbikes/image_0047.jpg 64 train Motorbikes/image_0285.jpg 64 train Motorbikes/image_0160.jpg 64 train Motorbikes/image_0678.jpg 64 train Motorbikes/image_0422.jpg 64 train Motorbikes/image_0579.jpg 64 train Motorbikes/image_0064.jpg 64 train Motorbikes/image_0070.jpg 64 train Motorbikes/image_0552.jpg 64 train Motorbikes/image_0264.jpg 64 train Motorbikes/image_0112.jpg 64 train Motorbikes/image_0078.jpg 64 train Motorbikes/image_0637.jpg 64 train Motorbikes/image_0068.jpg 64 train Motorbikes/image_0506.jpg 64 train Motorbikes/image_0433.jpg 64 train Motorbikes/image_0547.jpg 64 train Motorbikes/image_0085.jpg 64 train Motorbikes/image_0410.jpg 64 train Motorbikes/image_0110.jpg 64 train Motorbikes/image_0722.jpg 64 train Motorbikes/image_0593.jpg 64 train Motorbikes/image_0259.jpg 64 train Motorbikes/image_0143.jpg 64 train Motorbikes/image_0043.jpg 64 train Motorbikes/image_0162.jpg 64 train Motorbikes/image_0132.jpg 64 train Motorbikes/image_0157.jpg 64 train Motorbikes/image_0475.jpg 64 train Motorbikes/image_0206.jpg 64 train Motorbikes/image_0376.jpg 64 train Motorbikes/image_0644.jpg 64 train Motorbikes/image_0319.jpg 64 train Motorbikes/image_0371.jpg 64 train Motorbikes/image_0044.jpg 64 train Motorbikes/image_0199.jpg 64 train Motorbikes/image_0729.jpg 64 train Motorbikes/image_0691.jpg 64 train Motorbikes/image_0033.jpg 64 train Motorbikes/image_0136.jpg 64 train Motorbikes/image_0180.jpg 64 train Motorbikes/image_0260.jpg 64 train Motorbikes/image_0604.jpg 64 train Motorbikes/image_0184.jpg 64 train Motorbikes/image_0719.jpg 64 train Motorbikes/image_0672.jpg 64 train Motorbikes/image_0784.jpg 64 train Motorbikes/image_0073.jpg 64 train Motorbikes/image_0395.jpg 64 train Motorbikes/image_0257.jpg 64 train Motorbikes/image_0364.jpg 64 train Motorbikes/image_0756.jpg 64 train Motorbikes/image_0001.jpg 64 train Motorbikes/image_0167.jpg 64 train Motorbikes/image_0284.jpg 64 train Motorbikes/image_0375.jpg 64 train Motorbikes/image_0080.jpg 64 train Motorbikes/image_0665.jpg 64 train Motorbikes/image_0156.jpg 64 train Motorbikes/image_0021.jpg 64 train Motorbikes/image_0571.jpg 64 train Motorbikes/image_0325.jpg 64 train Motorbikes/image_0517.jpg 64 train Motorbikes/image_0713.jpg 64 train Motorbikes/image_0516.jpg 64 train Motorbikes/image_0574.jpg 64 train Motorbikes/image_0471.jpg 64 train Motorbikes/image_0767.jpg 64 train Motorbikes/image_0581.jpg 64 train Motorbikes/image_0267.jpg 64 train Motorbikes/image_0299.jpg 64 train Motorbikes/image_0421.jpg 64 train Motorbikes/image_0495.jpg 64 train Motorbikes/image_0529.jpg 64 train Motorbikes/image_0436.jpg 64 train Motorbikes/image_0360.jpg 64 train Motorbikes/image_0649.jpg 64 train Motorbikes/image_0095.jpg 64 train Motorbikes/image_0057.jpg 64 train Motorbikes/image_0707.jpg 64 train Motorbikes/image_0797.jpg 64 train Motorbikes/image_0403.jpg 64 train Motorbikes/image_0114.jpg 64 train Motorbikes/image_0641.jpg 64 train Motorbikes/image_0595.jpg 64 train Motorbikes/image_0620.jpg 64 train Motorbikes/image_0153.jpg 64 train Motorbikes/image_0697.jpg 64 train Motorbikes/image_0654.jpg 64 train Motorbikes/image_0564.jpg 64 train Motorbikes/image_0416.jpg 64 train Motorbikes/image_0607.jpg 64 train Motorbikes/image_0098.jpg 64 train Motorbikes/image_0256.jpg 64 train Motorbikes/image_0402.jpg 64 train Motorbikes/image_0788.jpg 64 train Motorbikes/image_0203.jpg 64 train Motorbikes/image_0577.jpg 64 train Motorbikes/image_0764.jpg 64 train Motorbikes/image_0338.jpg 64 train Motorbikes/image_0582.jpg 64 train Motorbikes/image_0265.jpg 64 train Motorbikes/image_0147.jpg 64 train Motorbikes/image_0536.jpg 64 train Motorbikes/image_0008.jpg 64 train Motorbikes/image_0446.jpg 64 train Motorbikes/image_0514.jpg 64 train Motorbikes/image_0617.jpg 64 train Motorbikes/image_0619.jpg 64 train Motorbikes/image_0727.jpg 64 train Motorbikes/image_0589.jpg 64 train Motorbikes/image_0207.jpg 64 train Motorbikes/image_0208.jpg 64 train Motorbikes/image_0222.jpg 64 train Motorbikes/image_0320.jpg 64 train Motorbikes/image_0781.jpg 64 train Motorbikes/image_0532.jpg 64 train Motorbikes/image_0227.jpg 64 train Motorbikes/image_0651.jpg 64 train Motorbikes/image_0286.jpg 64 train Motorbikes/image_0035.jpg 64 train Motorbikes/image_0399.jpg 64 train Motorbikes/image_0166.jpg 64 train Motorbikes/image_0711.jpg 64 train Motorbikes/image_0669.jpg 64 train Motorbikes/image_0169.jpg 64 train Motorbikes/image_0191.jpg 64 train Motorbikes/image_0128.jpg 64 train Motorbikes/image_0493.jpg 64 train Motorbikes/image_0414.jpg 64 train Motorbikes/image_0606.jpg 64 train Motorbikes/image_0734.jpg 64 train Motorbikes/image_0683.jpg 64 train Motorbikes/image_0102.jpg 64 train Motorbikes/image_0523.jpg 64 train Motorbikes/image_0434.jpg 64 train Motorbikes/image_0699.jpg 64 train Motorbikes/image_0178.jpg 64 train Motorbikes/image_0431.jpg 64 train Motorbikes/image_0243.jpg 64 train Motorbikes/image_0005.jpg 64 train Motorbikes/image_0263.jpg 64 train Motorbikes/image_0780.jpg 64 train Motorbikes/image_0173.jpg 64 train Motorbikes/image_0622.jpg 64 train Motorbikes/image_0692.jpg 64 train Motorbikes/image_0087.jpg 64 train Motorbikes/image_0417.jpg 64 train Motorbikes/image_0089.jpg 64 train Motorbikes/image_0059.jpg 64 train Motorbikes/image_0673.jpg 64 train Motorbikes/image_0253.jpg 64 train Motorbikes/image_0159.jpg 64 train Motorbikes/image_0225.jpg 64 train Motorbikes/image_0445.jpg 64 train Motorbikes/image_0186.jpg 64 train Motorbikes/image_0706.jpg 64 train Motorbikes/image_0388.jpg 64 train Motorbikes/image_0694.jpg 64 train Motorbikes/image_0069.jpg 64 train Motorbikes/image_0758.jpg 64 train Motorbikes/image_0315.jpg 64 train Motorbikes/image_0295.jpg 64 train Motorbikes/image_0746.jpg 64 train Motorbikes/image_0200.jpg 64 train Motorbikes/image_0099.jpg 64 train Motorbikes/image_0738.jpg 64 train Motorbikes/image_0336.jpg 64 train Motorbikes/image_0034.jpg 64 train Motorbikes/image_0740.jpg 64 train Motorbikes/image_0712.jpg 64 train Motorbikes/image_0559.jpg 64 train Motorbikes/image_0409.jpg 64 train Motorbikes/image_0791.jpg 64 train Motorbikes/image_0635.jpg 64 train Motorbikes/image_0668.jpg 64 train Motorbikes/image_0314.jpg 64 train Motorbikes/image_0276.jpg 64 train Motorbikes/image_0121.jpg 64 train Motorbikes/image_0438.jpg 64 train Motorbikes/image_0766.jpg 64 train Motorbikes/image_0428.jpg 64 train Motorbikes/image_0568.jpg 64 train Motorbikes/image_0489.jpg 64 train Motorbikes/image_0181.jpg 64 train Motorbikes/image_0742.jpg 64 train Motorbikes/image_0280.jpg 64 train Motorbikes/image_0183.jpg 64 train Motorbikes/image_0605.jpg 64 train Motorbikes/image_0383.jpg 64 train Motorbikes/image_0175.jpg 64 train Motorbikes/image_0002.jpg 64 train Motorbikes/image_0291.jpg 64 train Motorbikes/image_0646.jpg 64 train Motorbikes/image_0165.jpg 64 train Motorbikes/image_0154.jpg 64 train Motorbikes/image_0271.jpg 64 train Motorbikes/image_0193.jpg 64 train Motorbikes/image_0048.jpg 64 train Motorbikes/image_0437.jpg 64 train Motorbikes/image_0480.jpg 64 train Motorbikes/image_0723.jpg 64 train Motorbikes/image_0053.jpg 64 train Motorbikes/image_0255.jpg 64 train Motorbikes/image_0450.jpg 64 train Motorbikes/image_0736.jpg 64 train Motorbikes/image_0616.jpg 64 train Motorbikes/image_0084.jpg 64 train Motorbikes/image_0313.jpg 64 train Motorbikes/image_0757.jpg 64 train Motorbikes/image_0020.jpg 64 train Motorbikes/image_0586.jpg 64 train Motorbikes/image_0494.jpg 64 train Motorbikes/image_0633.jpg 64 train Motorbikes/image_0251.jpg 64 train Motorbikes/image_0631.jpg 64 train Motorbikes/image_0046.jpg 64 train Motorbikes/image_0041.jpg 64 train Motorbikes/image_0027.jpg 64 train Motorbikes/image_0655.jpg 64 train Motorbikes/image_0322.jpg 64 train Motorbikes/image_0458.jpg 64 train Motorbikes/image_0014.jpg 64 train Motorbikes/image_0732.jpg 64 train Motorbikes/image_0058.jpg 64 train Motorbikes/image_0717.jpg 64 train Motorbikes/image_0408.jpg 64 train Motorbikes/image_0329.jpg 64 train Motorbikes/image_0212.jpg 64 train Motorbikes/image_0483.jpg 64 train Motorbikes/image_0611.jpg 64 train Motorbikes/image_0615.jpg 64 train Motorbikes/image_0741.jpg 64 train Motorbikes/image_0163.jpg 64 train Motorbikes/image_0202.jpg 64 train Motorbikes/image_0690.jpg 64 train Motorbikes/image_0104.jpg 64 train Motorbikes/image_0509.jpg 64 train Motorbikes/image_0081.jpg 64 train Motorbikes/image_0272.jpg 64 train Motorbikes/image_0689.jpg 64 train Motorbikes/image_0534.jpg 64 train Motorbikes/image_0400.jpg 64 train Motorbikes/image_0237.jpg 64 train Motorbikes/image_0543.jpg 64 train Motorbikes/image_0667.jpg 64 train Motorbikes/image_0308.jpg 64 train Motorbikes/image_0149.jpg 64 train Motorbikes/image_0500.jpg 64 train Motorbikes/image_0088.jpg 64 train Motorbikes/image_0309.jpg 64 train Motorbikes/image_0026.jpg 64 train Motorbikes/image_0231.jpg 64 train Motorbikes/image_0004.jpg 64 train Motorbikes/image_0015.jpg 64 train Motorbikes/image_0634.jpg 64 train Motorbikes/image_0675.jpg 64 train Motorbikes/image_0770.jpg 64 train dollar_bill/image_0037.jpg 65 train dollar_bill/image_0049.jpg 65 train dollar_bill/image_0051.jpg 65 train dollar_bill/image_0006.jpg 65 train dollar_bill/image_0016.jpg 65 train dollar_bill/image_0030.jpg 65 train dollar_bill/image_0012.jpg 65 train dollar_bill/image_0025.jpg 65 train dollar_bill/image_0052.jpg 65 train dollar_bill/image_0007.jpg 65 train dollar_bill/image_0019.jpg 65 train dollar_bill/image_0031.jpg 65 train dollar_bill/image_0011.jpg 65 train dollar_bill/image_0050.jpg 65 train dollar_bill/image_0045.jpg 65 train dollar_bill/image_0017.jpg 65 train dollar_bill/image_0018.jpg 65 train dollar_bill/image_0013.jpg 65 train dollar_bill/image_0024.jpg 65 train dollar_bill/image_0003.jpg 65 train dollar_bill/image_0009.jpg 65 train dollar_bill/image_0042.jpg 65 train dollar_bill/image_0040.jpg 65 train dollar_bill/image_0047.jpg 65 train dollar_bill/image_0043.jpg 65 train dollar_bill/image_0044.jpg 65 train dollar_bill/image_0033.jpg 65 train dollar_bill/image_0001.jpg 65 train dollar_bill/image_0021.jpg 65 train dollar_bill/image_0008.jpg 65 train dollar_bill/image_0035.jpg 65 train dollar_bill/image_0005.jpg 65 train dollar_bill/image_0034.jpg 65 train dollar_bill/image_0002.jpg 65 train dollar_bill/image_0048.jpg 65 train dollar_bill/image_0020.jpg 65 train dollar_bill/image_0046.jpg 65 train dollar_bill/image_0041.jpg 65 train dollar_bill/image_0027.jpg 65 train dollar_bill/image_0014.jpg 65 train dollar_bill/image_0026.jpg 65 train nautilus/image_0037.jpg 66 train nautilus/image_0049.jpg 66 train nautilus/image_0051.jpg 66 train nautilus/image_0006.jpg 66 train nautilus/image_0016.jpg 66 train nautilus/image_0030.jpg 66 train nautilus/image_0012.jpg 66 train nautilus/image_0025.jpg 66 train nautilus/image_0052.jpg 66 train nautilus/image_0007.jpg 66 train nautilus/image_0019.jpg 66 train nautilus/image_0031.jpg 66 train nautilus/image_0011.jpg 66 train nautilus/image_0050.jpg 66 train nautilus/image_0045.jpg 66 train nautilus/image_0017.jpg 66 train nautilus/image_0018.jpg 66 train nautilus/image_0013.jpg 66 train nautilus/image_0054.jpg 66 train nautilus/image_0055.jpg 66 train nautilus/image_0024.jpg 66 train nautilus/image_0003.jpg 66 train nautilus/image_0009.jpg 66 train nautilus/image_0042.jpg 66 train nautilus/image_0040.jpg 66 train nautilus/image_0047.jpg 66 train nautilus/image_0043.jpg 66 train nautilus/image_0044.jpg 66 train nautilus/image_0033.jpg 66 train nautilus/image_0001.jpg 66 train nautilus/image_0021.jpg 66 train nautilus/image_0008.jpg 66 train nautilus/image_0035.jpg 66 train nautilus/image_0005.jpg 66 train nautilus/image_0034.jpg 66 train nautilus/image_0002.jpg 66 train nautilus/image_0048.jpg 66 train nautilus/image_0053.jpg 66 train nautilus/image_0020.jpg 66 train nautilus/image_0046.jpg 66 train nautilus/image_0041.jpg 66 train nautilus/image_0027.jpg 66 train nautilus/image_0014.jpg 66 train nautilus/image_0026.jpg 66 train crab/image_0062.jpg 67 train crab/image_0037.jpg 67 train crab/image_0049.jpg 67 train crab/image_0051.jpg 67 train crab/image_0006.jpg 67 train crab/image_0016.jpg 67 train crab/image_0030.jpg 67 train crab/image_0012.jpg 67 train crab/image_0066.jpg 67 train crab/image_0025.jpg 67 train crab/image_0060.jpg 67 train crab/image_0052.jpg 67 train crab/image_0007.jpg 67 train crab/image_0063.jpg 67 train crab/image_0019.jpg 67 train crab/image_0031.jpg 67 train crab/image_0011.jpg 67 train crab/image_0061.jpg 67 train crab/image_0050.jpg 67 train crab/image_0045.jpg 67 train crab/image_0017.jpg 67 train crab/image_0018.jpg 67 train crab/image_0013.jpg 67 train crab/image_0054.jpg 67 train crab/image_0055.jpg 67 train crab/image_0071.jpg 67 train crab/image_0024.jpg 67 train crab/image_0003.jpg 67 train crab/image_0009.jpg 67 train crab/image_0042.jpg 67 train crab/image_0040.jpg 67 train crab/image_0067.jpg 67 train crab/image_0047.jpg 67 train crab/image_0064.jpg 67 train crab/image_0070.jpg 67 train crab/image_0068.jpg 67 train crab/image_0043.jpg 67 train crab/image_0044.jpg 67 train crab/image_0033.jpg 67 train crab/image_0073.jpg 67 train crab/image_0001.jpg 67 train crab/image_0021.jpg 67 train crab/image_0057.jpg 67 train crab/image_0008.jpg 67 train crab/image_0035.jpg 67 train crab/image_0005.jpg 67 train crab/image_0059.jpg 67 train crab/image_0069.jpg 67 train crab/image_0034.jpg 67 train crab/image_0002.jpg 67 train crab/image_0048.jpg 67 train crab/image_0053.jpg 67 train crab/image_0020.jpg 67 train crab/image_0046.jpg 67 train crab/image_0041.jpg 67 train crab/image_0027.jpg 67 train crab/image_0014.jpg 67 train crab/image_0058.jpg 67 train accordion/image_0037.jpg 68 train accordion/image_0049.jpg 68 train accordion/image_0051.jpg 68 train accordion/image_0006.jpg 68 train accordion/image_0016.jpg 68 train accordion/image_0030.jpg 68 train accordion/image_0012.jpg 68 train accordion/image_0025.jpg 68 train accordion/image_0052.jpg 68 train accordion/image_0007.jpg 68 train accordion/image_0019.jpg 68 train accordion/image_0031.jpg 68 train accordion/image_0011.jpg 68 train accordion/image_0050.jpg 68 train accordion/image_0045.jpg 68 train accordion/image_0017.jpg 68 train accordion/image_0018.jpg 68 train accordion/image_0013.jpg 68 train accordion/image_0054.jpg 68 train accordion/image_0055.jpg 68 train accordion/image_0024.jpg 68 train accordion/image_0003.jpg 68 train accordion/image_0009.jpg 68 train accordion/image_0042.jpg 68 train accordion/image_0040.jpg 68 train accordion/image_0047.jpg 68 train accordion/image_0043.jpg 68 train accordion/image_0044.jpg 68 train accordion/image_0033.jpg 68 train accordion/image_0001.jpg 68 train accordion/image_0021.jpg 68 train accordion/image_0008.jpg 68 train accordion/image_0035.jpg 68 train accordion/image_0005.jpg 68 train accordion/image_0034.jpg 68 train accordion/image_0002.jpg 68 train accordion/image_0048.jpg 68 train accordion/image_0053.jpg 68 train accordion/image_0020.jpg 68 train accordion/image_0046.jpg 68 train accordion/image_0041.jpg 68 train accordion/image_0027.jpg 68 train accordion/image_0014.jpg 68 train accordion/image_0026.jpg 68 train crayfish/image_0062.jpg 69 train crayfish/image_0037.jpg 69 train crayfish/image_0049.jpg 69 train crayfish/image_0051.jpg 69 train crayfish/image_0006.jpg 69 train crayfish/image_0016.jpg 69 train crayfish/image_0030.jpg 69 train crayfish/image_0012.jpg 69 train crayfish/image_0066.jpg 69 train crayfish/image_0025.jpg 69 train crayfish/image_0060.jpg 69 train crayfish/image_0052.jpg 69 train crayfish/image_0007.jpg 69 train crayfish/image_0063.jpg 69 train crayfish/image_0019.jpg 69 train crayfish/image_0031.jpg 69 train crayfish/image_0011.jpg 69 train crayfish/image_0061.jpg 69 train crayfish/image_0050.jpg 69 train crayfish/image_0045.jpg 69 train crayfish/image_0017.jpg 69 train crayfish/image_0018.jpg 69 train crayfish/image_0013.jpg 69 train crayfish/image_0054.jpg 69 train crayfish/image_0055.jpg 69 train crayfish/image_0024.jpg 69 train crayfish/image_0003.jpg 69 train crayfish/image_0009.jpg 69 train crayfish/image_0042.jpg 69 train crayfish/image_0040.jpg 69 train crayfish/image_0067.jpg 69 train crayfish/image_0047.jpg 69 train crayfish/image_0064.jpg 69 train crayfish/image_0070.jpg 69 train crayfish/image_0068.jpg 69 train crayfish/image_0043.jpg 69 train crayfish/image_0044.jpg 69 train crayfish/image_0033.jpg 69 train crayfish/image_0001.jpg 69 train crayfish/image_0021.jpg 69 train crayfish/image_0057.jpg 69 train crayfish/image_0008.jpg 69 train crayfish/image_0035.jpg 69 train crayfish/image_0005.jpg 69 train crayfish/image_0059.jpg 69 train crayfish/image_0069.jpg 69 train crayfish/image_0034.jpg 69 train crayfish/image_0002.jpg 69 train crayfish/image_0048.jpg 69 train crayfish/image_0053.jpg 69 train crayfish/image_0020.jpg 69 train crayfish/image_0046.jpg 69 train crayfish/image_0041.jpg 69 train crayfish/image_0027.jpg 69 train crayfish/image_0014.jpg 69 train crayfish/image_0058.jpg 69 train flamingo_head/image_0037.jpg 70 train flamingo_head/image_0006.jpg 70 train flamingo_head/image_0016.jpg 70 train flamingo_head/image_0030.jpg 70 train flamingo_head/image_0012.jpg 70 train flamingo_head/image_0025.jpg 70 train flamingo_head/image_0007.jpg 70 train flamingo_head/image_0019.jpg 70 train flamingo_head/image_0031.jpg 70 train flamingo_head/image_0011.jpg 70 train flamingo_head/image_0045.jpg 70 train flamingo_head/image_0017.jpg 70 train flamingo_head/image_0018.jpg 70 train flamingo_head/image_0013.jpg 70 train flamingo_head/image_0024.jpg 70 train flamingo_head/image_0003.jpg 70 train flamingo_head/image_0009.jpg 70 train flamingo_head/image_0042.jpg 70 train flamingo_head/image_0040.jpg 70 train flamingo_head/image_0043.jpg 70 train flamingo_head/image_0044.jpg 70 train flamingo_head/image_0033.jpg 70 train flamingo_head/image_0001.jpg 70 train flamingo_head/image_0021.jpg 70 train flamingo_head/image_0008.jpg 70 train flamingo_head/image_0035.jpg 70 train flamingo_head/image_0005.jpg 70 train flamingo_head/image_0034.jpg 70 train flamingo_head/image_0002.jpg 70 train flamingo_head/image_0020.jpg 70 train flamingo_head/image_0041.jpg 70 train flamingo_head/image_0027.jpg 70 train flamingo_head/image_0014.jpg 70 train flamingo_head/image_0026.jpg 70 train flamingo_head/image_0004.jpg 70 train flamingo_head/image_0015.jpg 70 train emu/image_0037.jpg 71 train emu/image_0049.jpg 71 train emu/image_0051.jpg 71 train emu/image_0006.jpg 71 train emu/image_0016.jpg 71 train emu/image_0030.jpg 71 train emu/image_0012.jpg 71 train emu/image_0025.jpg 71 train emu/image_0052.jpg 71 train emu/image_0007.jpg 71 train emu/image_0019.jpg 71 train emu/image_0031.jpg 71 train emu/image_0011.jpg 71 train emu/image_0050.jpg 71 train emu/image_0045.jpg 71 train emu/image_0017.jpg 71 train emu/image_0018.jpg 71 train emu/image_0013.jpg 71 train emu/image_0024.jpg 71 train emu/image_0003.jpg 71 train emu/image_0009.jpg 71 train emu/image_0042.jpg 71 train emu/image_0040.jpg 71 train emu/image_0047.jpg 71 train emu/image_0043.jpg 71 train emu/image_0044.jpg 71 train emu/image_0033.jpg 71 train emu/image_0001.jpg 71 train emu/image_0021.jpg 71 train emu/image_0008.jpg 71 train emu/image_0035.jpg 71 train emu/image_0005.jpg 71 train emu/image_0034.jpg 71 train emu/image_0002.jpg 71 train emu/image_0048.jpg 71 train emu/image_0053.jpg 71 train emu/image_0020.jpg 71 train emu/image_0046.jpg 71 train emu/image_0041.jpg 71 train emu/image_0027.jpg 71 train emu/image_0014.jpg 71 train emu/image_0026.jpg 71 train trilobite/image_0062.jpg 72 train trilobite/image_0037.jpg 72 train trilobite/image_0049.jpg 72 train trilobite/image_0083.jpg 72 train trilobite/image_0051.jpg 72 train trilobite/image_0006.jpg 72 train trilobite/image_0016.jpg 72 train trilobite/image_0030.jpg 72 train trilobite/image_0012.jpg 72 train trilobite/image_0066.jpg 72 train trilobite/image_0077.jpg 72 train trilobite/image_0025.jpg 72 train trilobite/image_0060.jpg 72 train trilobite/image_0052.jpg 72 train trilobite/image_0007.jpg 72 train trilobite/image_0063.jpg 72 train trilobite/image_0019.jpg 72 train trilobite/image_0031.jpg 72 train trilobite/image_0011.jpg 72 train trilobite/image_0061.jpg 72 train trilobite/image_0050.jpg 72 train trilobite/image_0045.jpg 72 train trilobite/image_0017.jpg 72 train trilobite/image_0018.jpg 72 train trilobite/image_0013.jpg 72 train trilobite/image_0054.jpg 72 train trilobite/image_0082.jpg 72 train trilobite/image_0079.jpg 72 train trilobite/image_0055.jpg 72 train trilobite/image_0071.jpg 72 train trilobite/image_0024.jpg 72 train trilobite/image_0074.jpg 72 train trilobite/image_0003.jpg 72 train trilobite/image_0009.jpg 72 train trilobite/image_0042.jpg 72 train trilobite/image_0040.jpg 72 train trilobite/image_0067.jpg 72 train trilobite/image_0047.jpg 72 train trilobite/image_0064.jpg 72 train trilobite/image_0070.jpg 72 train trilobite/image_0078.jpg 72 train trilobite/image_0068.jpg 72 train trilobite/image_0085.jpg 72 train trilobite/image_0043.jpg 72 train trilobite/image_0044.jpg 72 train trilobite/image_0033.jpg 72 train trilobite/image_0073.jpg 72 train trilobite/image_0001.jpg 72 train trilobite/image_0080.jpg 72 train trilobite/image_0021.jpg 72 train trilobite/image_0057.jpg 72 train trilobite/image_0008.jpg 72 train trilobite/image_0035.jpg 72 train trilobite/image_0005.jpg 72 train trilobite/image_0059.jpg 72 train trilobite/image_0069.jpg 72 train trilobite/image_0034.jpg 72 train trilobite/image_0002.jpg 72 train trilobite/image_0048.jpg 72 train trilobite/image_0053.jpg 72 train trilobite/image_0084.jpg 72 train trilobite/image_0020.jpg 72 train trilobite/image_0046.jpg 72 train trilobite/image_0041.jpg 72 train trilobite/image_0027.jpg 72 train trilobite/image_0014.jpg 72 train trilobite/image_0058.jpg 72 train trilobite/image_0081.jpg 72 train camera/image_0037.jpg 73 train camera/image_0049.jpg 73 train camera/image_0006.jpg 73 train camera/image_0016.jpg 73 train camera/image_0030.jpg 73 train camera/image_0012.jpg 73 train camera/image_0025.jpg 73 train camera/image_0007.jpg 73 train camera/image_0019.jpg 73 train camera/image_0031.jpg 73 train camera/image_0011.jpg 73 train camera/image_0050.jpg 73 train camera/image_0045.jpg 73 train camera/image_0017.jpg 73 train camera/image_0018.jpg 73 train camera/image_0013.jpg 73 train camera/image_0024.jpg 73 train camera/image_0003.jpg 73 train camera/image_0009.jpg 73 train camera/image_0042.jpg 73 train camera/image_0040.jpg 73 train camera/image_0047.jpg 73 train camera/image_0043.jpg 73 train camera/image_0044.jpg 73 train camera/image_0033.jpg 73 train camera/image_0001.jpg 73 train camera/image_0021.jpg 73 train camera/image_0008.jpg 73 train camera/image_0035.jpg 73 train camera/image_0005.jpg 73 train camera/image_0034.jpg 73 train camera/image_0002.jpg 73 train camera/image_0048.jpg 73 train camera/image_0020.jpg 73 train camera/image_0046.jpg 73 train camera/image_0041.jpg 73 train camera/image_0027.jpg 73 train camera/image_0014.jpg 73 train camera/image_0026.jpg 73 train camera/image_0004.jpg 73 train platypus/image_0006.jpg 74 train platypus/image_0016.jpg 74 train platypus/image_0030.jpg 74 train platypus/image_0012.jpg 74 train platypus/image_0025.jpg 74 train platypus/image_0007.jpg 74 train platypus/image_0019.jpg 74 train platypus/image_0031.jpg 74 train platypus/image_0011.jpg 74 train platypus/image_0017.jpg 74 train platypus/image_0018.jpg 74 train platypus/image_0013.jpg 74 train platypus/image_0024.jpg 74 train platypus/image_0003.jpg 74 train platypus/image_0009.jpg 74 train platypus/image_0033.jpg 74 train platypus/image_0001.jpg 74 train platypus/image_0021.jpg 74 train platypus/image_0008.jpg 74 train platypus/image_0005.jpg 74 train platypus/image_0034.jpg 74 train platypus/image_0002.jpg 74 train platypus/image_0020.jpg 74 train platypus/image_0027.jpg 74 train platypus/image_0014.jpg 74 train platypus/image_0026.jpg 74 train platypus/image_0004.jpg 74 train chandelier/image_0062.jpg 75 train chandelier/image_0037.jpg 75 train chandelier/image_0049.jpg 75 train chandelier/image_0083.jpg 75 train chandelier/image_0101.jpg 75 train chandelier/image_0051.jpg 75 train chandelier/image_0006.jpg 75 train chandelier/image_0016.jpg 75 train chandelier/image_0030.jpg 75 train chandelier/image_0091.jpg 75 train chandelier/image_0012.jpg 75 train chandelier/image_0066.jpg 75 train chandelier/image_0077.jpg 75 train chandelier/image_0025.jpg 75 train chandelier/image_0060.jpg 75 train chandelier/image_0052.jpg 75 train chandelier/image_0007.jpg 75 train chandelier/image_0063.jpg 75 train chandelier/image_0019.jpg 75 train chandelier/image_0031.jpg 75 train chandelier/image_0011.jpg 75 train chandelier/image_0061.jpg 75 train chandelier/image_0090.jpg 75 train chandelier/image_0050.jpg 75 train chandelier/image_0045.jpg 75 train chandelier/image_0017.jpg 75 train chandelier/image_0018.jpg 75 train chandelier/image_0013.jpg 75 train chandelier/image_0054.jpg 75 train chandelier/image_0094.jpg 75 train chandelier/image_0082.jpg 75 train chandelier/image_0097.jpg 75 train chandelier/image_0079.jpg 75 train chandelier/image_0055.jpg 75 train chandelier/image_0071.jpg 75 train chandelier/image_0103.jpg 75 train chandelier/image_0024.jpg 75 train chandelier/image_0105.jpg 75 train chandelier/image_0074.jpg 75 train chandelier/image_0003.jpg 75 train chandelier/image_0009.jpg 75 train chandelier/image_0042.jpg 75 train chandelier/image_0093.jpg 75 train chandelier/image_0040.jpg 75 train chandelier/image_0067.jpg 75 train chandelier/image_0047.jpg 75 train chandelier/image_0064.jpg 75 train chandelier/image_0070.jpg 75 train chandelier/image_0078.jpg 75 train chandelier/image_0068.jpg 75 train chandelier/image_0085.jpg 75 train chandelier/image_0043.jpg 75 train chandelier/image_0044.jpg 75 train chandelier/image_0033.jpg 75 train chandelier/image_0073.jpg 75 train chandelier/image_0001.jpg 75 train chandelier/image_0080.jpg 75 train chandelier/image_0021.jpg 75 train chandelier/image_0095.jpg 75 train chandelier/image_0057.jpg 75 train chandelier/image_0098.jpg 75 train chandelier/image_0008.jpg 75 train chandelier/image_0035.jpg 75 train chandelier/image_0102.jpg 75 train chandelier/image_0005.jpg 75 train chandelier/image_0087.jpg 75 train chandelier/image_0089.jpg 75 train chandelier/image_0059.jpg 75 train chandelier/image_0069.jpg 75 train chandelier/image_0099.jpg 75 train chandelier/image_0034.jpg 75 train chandelier/image_0002.jpg 75 train chandelier/image_0048.jpg 75 train chandelier/image_0053.jpg 75 train chandelier/image_0084.jpg 75 train chandelier/image_0020.jpg 75 train chandelier/image_0046.jpg 75 train chandelier/image_0041.jpg 75 train chandelier/image_0027.jpg 75 train chandelier/image_0014.jpg 75 train chandelier/image_0058.jpg 75 train chandelier/image_0104.jpg 75 train chandelier/image_0081.jpg 75 train chandelier/image_0088.jpg 75 train chandelier/image_0026.jpg 75 train crocodile/image_0037.jpg 76 train crocodile/image_0049.jpg 76 train crocodile/image_0006.jpg 76 train crocodile/image_0016.jpg 76 train crocodile/image_0030.jpg 76 train crocodile/image_0012.jpg 76 train crocodile/image_0025.jpg 76 train crocodile/image_0007.jpg 76 train crocodile/image_0019.jpg 76 train crocodile/image_0031.jpg 76 train crocodile/image_0011.jpg 76 train crocodile/image_0050.jpg 76 train crocodile/image_0045.jpg 76 train crocodile/image_0017.jpg 76 train crocodile/image_0018.jpg 76 train crocodile/image_0013.jpg 76 train crocodile/image_0024.jpg 76 train crocodile/image_0003.jpg 76 train crocodile/image_0009.jpg 76 train crocodile/image_0042.jpg 76 train crocodile/image_0040.jpg 76 train crocodile/image_0047.jpg 76 train crocodile/image_0043.jpg 76 train crocodile/image_0044.jpg 76 train crocodile/image_0033.jpg 76 train crocodile/image_0001.jpg 76 train crocodile/image_0021.jpg 76 train crocodile/image_0008.jpg 76 train crocodile/image_0035.jpg 76 train crocodile/image_0005.jpg 76 train crocodile/image_0034.jpg 76 train crocodile/image_0002.jpg 76 train crocodile/image_0048.jpg 76 train crocodile/image_0020.jpg 76 train crocodile/image_0046.jpg 76 train crocodile/image_0041.jpg 76 train crocodile/image_0027.jpg 76 train crocodile/image_0014.jpg 76 train crocodile/image_0026.jpg 76 train crocodile/image_0004.jpg 76 train car_side/image_0062.jpg 77 train car_side/image_0037.jpg 77 train car_side/image_0049.jpg 77 train car_side/image_0083.jpg 77 train car_side/image_0101.jpg 77 train car_side/image_0051.jpg 77 train car_side/image_0006.jpg 77 train car_side/image_0016.jpg 77 train car_side/image_0030.jpg 77 train car_side/image_0091.jpg 77 train car_side/image_0012.jpg 77 train car_side/image_0066.jpg 77 train car_side/image_0077.jpg 77 train car_side/image_0025.jpg 77 train car_side/image_0115.jpg 77 train car_side/image_0123.jpg 77 train car_side/image_0119.jpg 77 train car_side/image_0060.jpg 77 train car_side/image_0052.jpg 77 train car_side/image_0108.jpg 77 train car_side/image_0007.jpg 77 train car_side/image_0063.jpg 77 train car_side/image_0019.jpg 77 train car_side/image_0113.jpg 77 train car_side/image_0031.jpg 77 train car_side/image_0011.jpg 77 train car_side/image_0061.jpg 77 train car_side/image_0090.jpg 77 train car_side/image_0050.jpg 77 train car_side/image_0045.jpg 77 train car_side/image_0017.jpg 77 train car_side/image_0018.jpg 77 train car_side/image_0013.jpg 77 train car_side/image_0054.jpg 77 train car_side/image_0094.jpg 77 train car_side/image_0082.jpg 77 train car_side/image_0097.jpg 77 train car_side/image_0109.jpg 77 train car_side/image_0079.jpg 77 train car_side/image_0055.jpg 77 train car_side/image_0071.jpg 77 train car_side/image_0103.jpg 77 train car_side/image_0024.jpg 77 train car_side/image_0105.jpg 77 train car_side/image_0074.jpg 77 train car_side/image_0003.jpg 77 train car_side/image_0009.jpg 77 train car_side/image_0118.jpg 77 train car_side/image_0042.jpg 77 train car_side/image_0093.jpg 77 train car_side/image_0040.jpg 77 train car_side/image_0067.jpg 77 train car_side/image_0122.jpg 77 train car_side/image_0047.jpg 77 train car_side/image_0064.jpg 77 train car_side/image_0070.jpg 77 train car_side/image_0112.jpg 77 train car_side/image_0078.jpg 77 train car_side/image_0068.jpg 77 train car_side/image_0085.jpg 77 train car_side/image_0110.jpg 77 train car_side/image_0043.jpg 77 train car_side/image_0044.jpg 77 train car_side/image_0033.jpg 77 train car_side/image_0073.jpg 77 train car_side/image_0001.jpg 77 train car_side/image_0080.jpg 77 train car_side/image_0021.jpg 77 train car_side/image_0095.jpg 77 train car_side/image_0057.jpg 77 train car_side/image_0114.jpg 77 train car_side/image_0098.jpg 77 train car_side/image_0008.jpg 77 train car_side/image_0035.jpg 77 train car_side/image_0102.jpg 77 train car_side/image_0005.jpg 77 train car_side/image_0087.jpg 77 train car_side/image_0089.jpg 77 train car_side/image_0059.jpg 77 train car_side/image_0069.jpg 77 train car_side/image_0099.jpg 77 train car_side/image_0034.jpg 77 train car_side/image_0121.jpg 77 train car_side/image_0002.jpg 77 train car_side/image_0048.jpg 77 train car_side/image_0053.jpg 77 train car_side/image_0084.jpg 77 train car_side/image_0020.jpg 77 train car_side/image_0046.jpg 77 train car_side/image_0041.jpg 77 train car_side/image_0027.jpg 77 train car_side/image_0014.jpg 77 train car_side/image_0058.jpg 77 train car_side/image_0104.jpg 77 train car_side/image_0081.jpg 77 train car_side/image_0088.jpg 77 train car_side/image_0026.jpg 77 train car_side/image_0004.jpg 77 train joshua_tree/image_0062.jpg 78 train joshua_tree/image_0037.jpg 78 train joshua_tree/image_0049.jpg 78 train joshua_tree/image_0051.jpg 78 train joshua_tree/image_0006.jpg 78 train joshua_tree/image_0016.jpg 78 train joshua_tree/image_0030.jpg 78 train joshua_tree/image_0012.jpg 78 train joshua_tree/image_0025.jpg 78 train joshua_tree/image_0060.jpg 78 train joshua_tree/image_0052.jpg 78 train joshua_tree/image_0007.jpg 78 train joshua_tree/image_0063.jpg 78 train joshua_tree/image_0019.jpg 78 train joshua_tree/image_0031.jpg 78 train joshua_tree/image_0011.jpg 78 train joshua_tree/image_0061.jpg 78 train joshua_tree/image_0050.jpg 78 train joshua_tree/image_0045.jpg 78 train joshua_tree/image_0017.jpg 78 train joshua_tree/image_0018.jpg 78 train joshua_tree/image_0013.jpg 78 train joshua_tree/image_0054.jpg 78 train joshua_tree/image_0055.jpg 78 train joshua_tree/image_0024.jpg 78 train joshua_tree/image_0003.jpg 78 train joshua_tree/image_0009.jpg 78 train joshua_tree/image_0042.jpg 78 train joshua_tree/image_0040.jpg 78 train joshua_tree/image_0047.jpg 78 train joshua_tree/image_0064.jpg 78 train joshua_tree/image_0043.jpg 78 train joshua_tree/image_0044.jpg 78 train joshua_tree/image_0033.jpg 78 train joshua_tree/image_0001.jpg 78 train joshua_tree/image_0021.jpg 78 train joshua_tree/image_0057.jpg 78 train joshua_tree/image_0008.jpg 78 train joshua_tree/image_0035.jpg 78 train joshua_tree/image_0005.jpg 78 train joshua_tree/image_0059.jpg 78 train joshua_tree/image_0034.jpg 78 train joshua_tree/image_0002.jpg 78 train joshua_tree/image_0048.jpg 78 train joshua_tree/image_0053.jpg 78 train joshua_tree/image_0020.jpg 78 train joshua_tree/image_0046.jpg 78 train joshua_tree/image_0041.jpg 78 train joshua_tree/image_0027.jpg 78 train joshua_tree/image_0014.jpg 78 train joshua_tree/image_0058.jpg 78 train tick/image_0037.jpg 79 train tick/image_0049.jpg 79 train tick/image_0006.jpg 79 train tick/image_0016.jpg 79 train tick/image_0030.jpg 79 train tick/image_0012.jpg 79 train tick/image_0025.jpg 79 train tick/image_0007.jpg 79 train tick/image_0019.jpg 79 train tick/image_0031.jpg 79 train tick/image_0011.jpg 79 train tick/image_0045.jpg 79 train tick/image_0017.jpg 79 train tick/image_0018.jpg 79 train tick/image_0013.jpg 79 train tick/image_0024.jpg 79 train tick/image_0003.jpg 79 train tick/image_0009.jpg 79 train tick/image_0042.jpg 79 train tick/image_0040.jpg 79 train tick/image_0047.jpg 79 train tick/image_0043.jpg 79 train tick/image_0044.jpg 79 train tick/image_0033.jpg 79 train tick/image_0001.jpg 79 train tick/image_0021.jpg 79 train tick/image_0008.jpg 79 train tick/image_0035.jpg 79 train tick/image_0005.jpg 79 train tick/image_0034.jpg 79 train tick/image_0002.jpg 79 train tick/image_0048.jpg 79 train tick/image_0020.jpg 79 train tick/image_0046.jpg 79 train tick/image_0041.jpg 79 train tick/image_0027.jpg 79 train tick/image_0014.jpg 79 train tick/image_0026.jpg 79 train tick/image_0004.jpg 79 train hawksbill/image_0062.jpg 80 train hawksbill/image_0037.jpg 80 train hawksbill/image_0049.jpg 80 train hawksbill/image_0083.jpg 80 train hawksbill/image_0051.jpg 80 train hawksbill/image_0006.jpg 80 train hawksbill/image_0016.jpg 80 train hawksbill/image_0030.jpg 80 train hawksbill/image_0091.jpg 80 train hawksbill/image_0012.jpg 80 train hawksbill/image_0066.jpg 80 train hawksbill/image_0077.jpg 80 train hawksbill/image_0025.jpg 80 train hawksbill/image_0060.jpg 80 train hawksbill/image_0052.jpg 80 train hawksbill/image_0007.jpg 80 train hawksbill/image_0063.jpg 80 train hawksbill/image_0019.jpg 80 train hawksbill/image_0031.jpg 80 train hawksbill/image_0011.jpg 80 train hawksbill/image_0061.jpg 80 train hawksbill/image_0090.jpg 80 train hawksbill/image_0050.jpg 80 train hawksbill/image_0045.jpg 80 train hawksbill/image_0017.jpg 80 train hawksbill/image_0018.jpg 80 train hawksbill/image_0013.jpg 80 train hawksbill/image_0054.jpg 80 train hawksbill/image_0094.jpg 80 train hawksbill/image_0082.jpg 80 train hawksbill/image_0097.jpg 80 train hawksbill/image_0079.jpg 80 train hawksbill/image_0055.jpg 80 train hawksbill/image_0071.jpg 80 train hawksbill/image_0024.jpg 80 train hawksbill/image_0074.jpg 80 train hawksbill/image_0003.jpg 80 train hawksbill/image_0009.jpg 80 train hawksbill/image_0042.jpg 80 train hawksbill/image_0093.jpg 80 train hawksbill/image_0040.jpg 80 train hawksbill/image_0067.jpg 80 train hawksbill/image_0047.jpg 80 train hawksbill/image_0064.jpg 80 train hawksbill/image_0070.jpg 80 train hawksbill/image_0078.jpg 80 train hawksbill/image_0068.jpg 80 train hawksbill/image_0085.jpg 80 train hawksbill/image_0043.jpg 80 train hawksbill/image_0044.jpg 80 train hawksbill/image_0033.jpg 80 train hawksbill/image_0073.jpg 80 train hawksbill/image_0001.jpg 80 train hawksbill/image_0080.jpg 80 train hawksbill/image_0021.jpg 80 train hawksbill/image_0095.jpg 80 train hawksbill/image_0057.jpg 80 train hawksbill/image_0098.jpg 80 train hawksbill/image_0008.jpg 80 train hawksbill/image_0035.jpg 80 train hawksbill/image_0005.jpg 80 train hawksbill/image_0087.jpg 80 train hawksbill/image_0089.jpg 80 train hawksbill/image_0059.jpg 80 train hawksbill/image_0069.jpg 80 train hawksbill/image_0099.jpg 80 train hawksbill/image_0034.jpg 80 train hawksbill/image_0002.jpg 80 train hawksbill/image_0048.jpg 80 train hawksbill/image_0053.jpg 80 train hawksbill/image_0084.jpg 80 train hawksbill/image_0020.jpg 80 train hawksbill/image_0046.jpg 80 train hawksbill/image_0041.jpg 80 train hawksbill/image_0027.jpg 80 train hawksbill/image_0014.jpg 80 train hawksbill/image_0058.jpg 80 train hawksbill/image_0081.jpg 80 train hawksbill/image_0088.jpg 80 train hawksbill/image_0026.jpg 80 train helicopter/image_0062.jpg 81 train helicopter/image_0037.jpg 81 train helicopter/image_0049.jpg 81 train helicopter/image_0083.jpg 81 train helicopter/image_0051.jpg 81 train helicopter/image_0006.jpg 81 train helicopter/image_0016.jpg 81 train helicopter/image_0030.jpg 81 train helicopter/image_0012.jpg 81 train helicopter/image_0066.jpg 81 train helicopter/image_0077.jpg 81 train helicopter/image_0025.jpg 81 train helicopter/image_0060.jpg 81 train helicopter/image_0052.jpg 81 train helicopter/image_0007.jpg 81 train helicopter/image_0063.jpg 81 train helicopter/image_0019.jpg 81 train helicopter/image_0031.jpg 81 train helicopter/image_0011.jpg 81 train helicopter/image_0061.jpg 81 train helicopter/image_0050.jpg 81 train helicopter/image_0045.jpg 81 train helicopter/image_0017.jpg 81 train helicopter/image_0018.jpg 81 train helicopter/image_0013.jpg 81 train helicopter/image_0054.jpg 81 train helicopter/image_0082.jpg 81 train helicopter/image_0079.jpg 81 train helicopter/image_0055.jpg 81 train helicopter/image_0071.jpg 81 train helicopter/image_0024.jpg 81 train helicopter/image_0074.jpg 81 train helicopter/image_0003.jpg 81 train helicopter/image_0009.jpg 81 train helicopter/image_0042.jpg 81 train helicopter/image_0040.jpg 81 train helicopter/image_0067.jpg 81 train helicopter/image_0047.jpg 81 train helicopter/image_0064.jpg 81 train helicopter/image_0070.jpg 81 train helicopter/image_0078.jpg 81 train helicopter/image_0068.jpg 81 train helicopter/image_0085.jpg 81 train helicopter/image_0043.jpg 81 train helicopter/image_0044.jpg 81 train helicopter/image_0033.jpg 81 train helicopter/image_0073.jpg 81 train helicopter/image_0001.jpg 81 train helicopter/image_0080.jpg 81 train helicopter/image_0021.jpg 81 train helicopter/image_0057.jpg 81 train helicopter/image_0008.jpg 81 train helicopter/image_0035.jpg 81 train helicopter/image_0005.jpg 81 train helicopter/image_0087.jpg 81 train helicopter/image_0059.jpg 81 train helicopter/image_0069.jpg 81 train helicopter/image_0034.jpg 81 train helicopter/image_0002.jpg 81 train helicopter/image_0048.jpg 81 train helicopter/image_0053.jpg 81 train helicopter/image_0084.jpg 81 train helicopter/image_0020.jpg 81 train helicopter/image_0046.jpg 81 train helicopter/image_0041.jpg 81 train helicopter/image_0027.jpg 81 train helicopter/image_0014.jpg 81 train helicopter/image_0058.jpg 81 train helicopter/image_0081.jpg 81 train helicopter/image_0088.jpg 81 train pagoda/image_0037.jpg 82 train pagoda/image_0006.jpg 82 train pagoda/image_0016.jpg 82 train pagoda/image_0030.jpg 82 train pagoda/image_0012.jpg 82 train pagoda/image_0025.jpg 82 train pagoda/image_0007.jpg 82 train pagoda/image_0019.jpg 82 train pagoda/image_0031.jpg 82 train pagoda/image_0011.jpg 82 train pagoda/image_0045.jpg 82 train pagoda/image_0017.jpg 82 train pagoda/image_0018.jpg 82 train pagoda/image_0013.jpg 82 train pagoda/image_0024.jpg 82 train pagoda/image_0003.jpg 82 train pagoda/image_0009.jpg 82 train pagoda/image_0042.jpg 82 train pagoda/image_0040.jpg 82 train pagoda/image_0047.jpg 82 train pagoda/image_0043.jpg 82 train pagoda/image_0044.jpg 82 train pagoda/image_0033.jpg 82 train pagoda/image_0001.jpg 82 train pagoda/image_0021.jpg 82 train pagoda/image_0008.jpg 82 train pagoda/image_0035.jpg 82 train pagoda/image_0005.jpg 82 train pagoda/image_0034.jpg 82 train pagoda/image_0002.jpg 82 train pagoda/image_0020.jpg 82 train pagoda/image_0046.jpg 82 train pagoda/image_0041.jpg 82 train pagoda/image_0027.jpg 82 train pagoda/image_0014.jpg 82 train pagoda/image_0026.jpg 82 train pagoda/image_0004.jpg 82 train ewer/image_0062.jpg 83 train ewer/image_0037.jpg 83 train ewer/image_0049.jpg 83 train ewer/image_0083.jpg 83 train ewer/image_0051.jpg 83 train ewer/image_0006.jpg 83 train ewer/image_0016.jpg 83 train ewer/image_0030.jpg 83 train ewer/image_0012.jpg 83 train ewer/image_0066.jpg 83 train ewer/image_0077.jpg 83 train ewer/image_0025.jpg 83 train ewer/image_0060.jpg 83 train ewer/image_0052.jpg 83 train ewer/image_0007.jpg 83 train ewer/image_0063.jpg 83 train ewer/image_0019.jpg 83 train ewer/image_0031.jpg 83 train ewer/image_0011.jpg 83 train ewer/image_0061.jpg 83 train ewer/image_0050.jpg 83 train ewer/image_0045.jpg 83 train ewer/image_0017.jpg 83 train ewer/image_0018.jpg 83 train ewer/image_0013.jpg 83 train ewer/image_0054.jpg 83 train ewer/image_0082.jpg 83 train ewer/image_0079.jpg 83 train ewer/image_0055.jpg 83 train ewer/image_0071.jpg 83 train ewer/image_0024.jpg 83 train ewer/image_0074.jpg 83 train ewer/image_0003.jpg 83 train ewer/image_0009.jpg 83 train ewer/image_0042.jpg 83 train ewer/image_0040.jpg 83 train ewer/image_0067.jpg 83 train ewer/image_0047.jpg 83 train ewer/image_0064.jpg 83 train ewer/image_0070.jpg 83 train ewer/image_0078.jpg 83 train ewer/image_0068.jpg 83 train ewer/image_0085.jpg 83 train ewer/image_0043.jpg 83 train ewer/image_0044.jpg 83 train ewer/image_0033.jpg 83 train ewer/image_0073.jpg 83 train ewer/image_0001.jpg 83 train ewer/image_0080.jpg 83 train ewer/image_0021.jpg 83 train ewer/image_0057.jpg 83 train ewer/image_0008.jpg 83 train ewer/image_0035.jpg 83 train ewer/image_0005.jpg 83 train ewer/image_0059.jpg 83 train ewer/image_0069.jpg 83 train ewer/image_0034.jpg 83 train ewer/image_0002.jpg 83 train ewer/image_0048.jpg 83 train ewer/image_0053.jpg 83 train ewer/image_0084.jpg 83 train ewer/image_0020.jpg 83 train ewer/image_0046.jpg 83 train ewer/image_0041.jpg 83 train ewer/image_0027.jpg 83 train ewer/image_0014.jpg 83 train ewer/image_0058.jpg 83 train ewer/image_0081.jpg 83 train panda/image_0037.jpg 84 train panda/image_0006.jpg 84 train panda/image_0016.jpg 84 train panda/image_0030.jpg 84 train panda/image_0012.jpg 84 train panda/image_0025.jpg 84 train panda/image_0007.jpg 84 train panda/image_0019.jpg 84 train panda/image_0031.jpg 84 train panda/image_0011.jpg 84 train panda/image_0017.jpg 84 train panda/image_0018.jpg 84 train panda/image_0013.jpg 84 train panda/image_0024.jpg 84 train panda/image_0003.jpg 84 train panda/image_0009.jpg 84 train panda/image_0033.jpg 84 train panda/image_0001.jpg 84 train panda/image_0021.jpg 84 train panda/image_0008.jpg 84 train panda/image_0035.jpg 84 train panda/image_0005.jpg 84 train panda/image_0034.jpg 84 train panda/image_0002.jpg 84 train panda/image_0020.jpg 84 train panda/image_0027.jpg 84 train panda/image_0014.jpg 84 train panda/image_0026.jpg 84 train panda/image_0004.jpg 84 train panda/image_0015.jpg 84 train pizza/image_0037.jpg 85 train pizza/image_0049.jpg 85 train pizza/image_0051.jpg 85 train pizza/image_0006.jpg 85 train pizza/image_0016.jpg 85 train pizza/image_0030.jpg 85 train pizza/image_0012.jpg 85 train pizza/image_0025.jpg 85 train pizza/image_0052.jpg 85 train pizza/image_0007.jpg 85 train pizza/image_0019.jpg 85 train pizza/image_0031.jpg 85 train pizza/image_0011.jpg 85 train pizza/image_0050.jpg 85 train pizza/image_0045.jpg 85 train pizza/image_0017.jpg 85 train pizza/image_0018.jpg 85 train pizza/image_0013.jpg 85 train pizza/image_0024.jpg 85 train pizza/image_0003.jpg 85 train pizza/image_0009.jpg 85 train pizza/image_0042.jpg 85 train pizza/image_0040.jpg 85 train pizza/image_0047.jpg 85 train pizza/image_0043.jpg 85 train pizza/image_0044.jpg 85 train pizza/image_0033.jpg 85 train pizza/image_0001.jpg 85 train pizza/image_0021.jpg 85 train pizza/image_0008.jpg 85 train pizza/image_0035.jpg 85 train pizza/image_0005.jpg 85 train pizza/image_0034.jpg 85 train pizza/image_0002.jpg 85 train pizza/image_0048.jpg 85 train pizza/image_0053.jpg 85 train pizza/image_0020.jpg 85 train pizza/image_0046.jpg 85 train pizza/image_0041.jpg 85 train pizza/image_0027.jpg 85 train pizza/image_0014.jpg 85 train pizza/image_0026.jpg 85 train cup/image_0037.jpg 86 train cup/image_0049.jpg 86 train cup/image_0051.jpg 86 train cup/image_0006.jpg 86 train cup/image_0016.jpg 86 train cup/image_0030.jpg 86 train cup/image_0012.jpg 86 train cup/image_0025.jpg 86 train cup/image_0052.jpg 86 train cup/image_0007.jpg 86 train cup/image_0019.jpg 86 train cup/image_0031.jpg 86 train cup/image_0011.jpg 86 train cup/image_0050.jpg 86 train cup/image_0045.jpg 86 train cup/image_0017.jpg 86 train cup/image_0018.jpg 86 train cup/image_0013.jpg 86 train cup/image_0054.jpg 86 train cup/image_0055.jpg 86 train cup/image_0024.jpg 86 train cup/image_0003.jpg 86 train cup/image_0009.jpg 86 train cup/image_0042.jpg 86 train cup/image_0040.jpg 86 train cup/image_0047.jpg 86 train cup/image_0043.jpg 86 train cup/image_0044.jpg 86 train cup/image_0033.jpg 86 train cup/image_0001.jpg 86 train cup/image_0021.jpg 86 train cup/image_0057.jpg 86 train cup/image_0008.jpg 86 train cup/image_0035.jpg 86 train cup/image_0005.jpg 86 train cup/image_0034.jpg 86 train cup/image_0002.jpg 86 train cup/image_0048.jpg 86 train cup/image_0053.jpg 86 train cup/image_0020.jpg 86 train cup/image_0046.jpg 86 train cup/image_0041.jpg 86 train cup/image_0027.jpg 86 train cup/image_0014.jpg 86 train cup/image_0026.jpg 86 train anchor/image_0037.jpg 87 train anchor/image_0006.jpg 87 train anchor/image_0016.jpg 87 train anchor/image_0030.jpg 87 train anchor/image_0012.jpg 87 train anchor/image_0025.jpg 87 train anchor/image_0007.jpg 87 train anchor/image_0019.jpg 87 train anchor/image_0031.jpg 87 train anchor/image_0011.jpg 87 train anchor/image_0017.jpg 87 train anchor/image_0018.jpg 87 train anchor/image_0013.jpg 87 train anchor/image_0024.jpg 87 train anchor/image_0003.jpg 87 train anchor/image_0009.jpg 87 train anchor/image_0042.jpg 87 train anchor/image_0040.jpg 87 train anchor/image_0033.jpg 87 train anchor/image_0001.jpg 87 train anchor/image_0021.jpg 87 train anchor/image_0008.jpg 87 train anchor/image_0035.jpg 87 train anchor/image_0005.jpg 87 train anchor/image_0034.jpg 87 train anchor/image_0002.jpg 87 train anchor/image_0020.jpg 87 train anchor/image_0041.jpg 87 train anchor/image_0027.jpg 87 train anchor/image_0014.jpg 87 train anchor/image_0026.jpg 87 train anchor/image_0004.jpg 87 train anchor/image_0015.jpg 87 train hedgehog/image_0037.jpg 88 train hedgehog/image_0049.jpg 88 train hedgehog/image_0051.jpg 88 train hedgehog/image_0006.jpg 88 train hedgehog/image_0016.jpg 88 train hedgehog/image_0030.jpg 88 train hedgehog/image_0012.jpg 88 train hedgehog/image_0025.jpg 88 train hedgehog/image_0052.jpg 88 train hedgehog/image_0007.jpg 88 train hedgehog/image_0019.jpg 88 train hedgehog/image_0031.jpg 88 train hedgehog/image_0011.jpg 88 train hedgehog/image_0050.jpg 88 train hedgehog/image_0045.jpg 88 train hedgehog/image_0017.jpg 88 train hedgehog/image_0018.jpg 88 train hedgehog/image_0013.jpg 88 train hedgehog/image_0054.jpg 88 train hedgehog/image_0024.jpg 88 train hedgehog/image_0003.jpg 88 train hedgehog/image_0009.jpg 88 train hedgehog/image_0042.jpg 88 train hedgehog/image_0040.jpg 88 train hedgehog/image_0047.jpg 88 train hedgehog/image_0043.jpg 88 train hedgehog/image_0044.jpg 88 train hedgehog/image_0033.jpg 88 train hedgehog/image_0001.jpg 88 train hedgehog/image_0021.jpg 88 train hedgehog/image_0008.jpg 88 train hedgehog/image_0035.jpg 88 train hedgehog/image_0005.jpg 88 train hedgehog/image_0034.jpg 88 train hedgehog/image_0002.jpg 88 train hedgehog/image_0048.jpg 88 train hedgehog/image_0053.jpg 88 train hedgehog/image_0020.jpg 88 train hedgehog/image_0046.jpg 88 train hedgehog/image_0041.jpg 88 train hedgehog/image_0027.jpg 88 train hedgehog/image_0014.jpg 88 train hedgehog/image_0026.jpg 88 train flamingo/image_0062.jpg 89 train flamingo/image_0037.jpg 89 train flamingo/image_0049.jpg 89 train flamingo/image_0051.jpg 89 train flamingo/image_0006.jpg 89 train flamingo/image_0016.jpg 89 train flamingo/image_0030.jpg 89 train flamingo/image_0012.jpg 89 train flamingo/image_0066.jpg 89 train flamingo/image_0025.jpg 89 train flamingo/image_0060.jpg 89 train flamingo/image_0052.jpg 89 train flamingo/image_0007.jpg 89 train flamingo/image_0063.jpg 89 train flamingo/image_0019.jpg 89 train flamingo/image_0031.jpg 89 train flamingo/image_0011.jpg 89 train flamingo/image_0061.jpg 89 train flamingo/image_0050.jpg 89 train flamingo/image_0045.jpg 89 train flamingo/image_0017.jpg 89 train flamingo/image_0018.jpg 89 train flamingo/image_0013.jpg 89 train flamingo/image_0054.jpg 89 train flamingo/image_0055.jpg 89 train flamingo/image_0024.jpg 89 train flamingo/image_0003.jpg 89 train flamingo/image_0009.jpg 89 train flamingo/image_0042.jpg 89 train flamingo/image_0040.jpg 89 train flamingo/image_0067.jpg 89 train flamingo/image_0047.jpg 89 train flamingo/image_0064.jpg 89 train flamingo/image_0043.jpg 89 train flamingo/image_0044.jpg 89 train flamingo/image_0033.jpg 89 train flamingo/image_0001.jpg 89 train flamingo/image_0021.jpg 89 train flamingo/image_0057.jpg 89 train flamingo/image_0008.jpg 89 train flamingo/image_0035.jpg 89 train flamingo/image_0005.jpg 89 train flamingo/image_0059.jpg 89 train flamingo/image_0034.jpg 89 train flamingo/image_0002.jpg 89 train flamingo/image_0048.jpg 89 train flamingo/image_0053.jpg 89 train flamingo/image_0020.jpg 89 train flamingo/image_0046.jpg 89 train flamingo/image_0041.jpg 89 train flamingo/image_0027.jpg 89 train flamingo/image_0014.jpg 89 train flamingo/image_0058.jpg 89 train stegosaurus/image_0037.jpg 90 train stegosaurus/image_0049.jpg 90 train stegosaurus/image_0051.jpg 90 train stegosaurus/image_0006.jpg 90 train stegosaurus/image_0016.jpg 90 train stegosaurus/image_0030.jpg 90 train stegosaurus/image_0012.jpg 90 train stegosaurus/image_0025.jpg 90 train stegosaurus/image_0052.jpg 90 train stegosaurus/image_0007.jpg 90 train stegosaurus/image_0019.jpg 90 train stegosaurus/image_0031.jpg 90 train stegosaurus/image_0011.jpg 90 train stegosaurus/image_0050.jpg 90 train stegosaurus/image_0045.jpg 90 train stegosaurus/image_0017.jpg 90 train stegosaurus/image_0018.jpg 90 train stegosaurus/image_0013.jpg 90 train stegosaurus/image_0054.jpg 90 train stegosaurus/image_0055.jpg 90 train stegosaurus/image_0024.jpg 90 train stegosaurus/image_0003.jpg 90 train stegosaurus/image_0009.jpg 90 train stegosaurus/image_0042.jpg 90 train stegosaurus/image_0040.jpg 90 train stegosaurus/image_0047.jpg 90 train stegosaurus/image_0043.jpg 90 train stegosaurus/image_0044.jpg 90 train stegosaurus/image_0033.jpg 90 train stegosaurus/image_0001.jpg 90 train stegosaurus/image_0021.jpg 90 train stegosaurus/image_0057.jpg 90 train stegosaurus/image_0008.jpg 90 train stegosaurus/image_0035.jpg 90 train stegosaurus/image_0005.jpg 90 train stegosaurus/image_0059.jpg 90 train stegosaurus/image_0034.jpg 90 train stegosaurus/image_0002.jpg 90 train stegosaurus/image_0048.jpg 90 train stegosaurus/image_0053.jpg 90 train stegosaurus/image_0020.jpg 90 train stegosaurus/image_0046.jpg 90 train stegosaurus/image_0041.jpg 90 train stegosaurus/image_0027.jpg 90 train stegosaurus/image_0014.jpg 90 train stegosaurus/image_0058.jpg 90 train stegosaurus/image_0026.jpg 90 train ferry/image_0062.jpg 91 train ferry/image_0037.jpg 91 train ferry/image_0049.jpg 91 train ferry/image_0051.jpg 91 train ferry/image_0006.jpg 91 train ferry/image_0016.jpg 91 train ferry/image_0030.jpg 91 train ferry/image_0012.jpg 91 train ferry/image_0066.jpg 91 train ferry/image_0025.jpg 91 train ferry/image_0060.jpg 91 train ferry/image_0052.jpg 91 train ferry/image_0007.jpg 91 train ferry/image_0063.jpg 91 train ferry/image_0019.jpg 91 train ferry/image_0031.jpg 91 train ferry/image_0011.jpg 91 train ferry/image_0061.jpg 91 train ferry/image_0050.jpg 91 train ferry/image_0045.jpg 91 train ferry/image_0017.jpg 91 train ferry/image_0018.jpg 91 train ferry/image_0013.jpg 91 train ferry/image_0054.jpg 91 train ferry/image_0055.jpg 91 train ferry/image_0024.jpg 91 train ferry/image_0003.jpg 91 train ferry/image_0009.jpg 91 train ferry/image_0042.jpg 91 train ferry/image_0040.jpg 91 train ferry/image_0067.jpg 91 train ferry/image_0047.jpg 91 train ferry/image_0064.jpg 91 train ferry/image_0043.jpg 91 train ferry/image_0044.jpg 91 train ferry/image_0033.jpg 91 train ferry/image_0001.jpg 91 train ferry/image_0021.jpg 91 train ferry/image_0057.jpg 91 train ferry/image_0008.jpg 91 train ferry/image_0035.jpg 91 train ferry/image_0005.jpg 91 train ferry/image_0059.jpg 91 train ferry/image_0034.jpg 91 train ferry/image_0002.jpg 91 train ferry/image_0048.jpg 91 train ferry/image_0053.jpg 91 train ferry/image_0020.jpg 91 train ferry/image_0046.jpg 91 train ferry/image_0041.jpg 91 train ferry/image_0027.jpg 91 train ferry/image_0014.jpg 91 train ferry/image_0058.jpg 91 train dalmatian/image_0062.jpg 92 train dalmatian/image_0037.jpg 92 train dalmatian/image_0049.jpg 92 train dalmatian/image_0051.jpg 92 train dalmatian/image_0006.jpg 92 train dalmatian/image_0016.jpg 92 train dalmatian/image_0030.jpg 92 train dalmatian/image_0012.jpg 92 train dalmatian/image_0066.jpg 92 train dalmatian/image_0025.jpg 92 train dalmatian/image_0060.jpg 92 train dalmatian/image_0052.jpg 92 train dalmatian/image_0007.jpg 92 train dalmatian/image_0063.jpg 92 train dalmatian/image_0019.jpg 92 train dalmatian/image_0031.jpg 92 train dalmatian/image_0011.jpg 92 train dalmatian/image_0061.jpg 92 train dalmatian/image_0050.jpg 92 train dalmatian/image_0045.jpg 92 train dalmatian/image_0017.jpg 92 train dalmatian/image_0018.jpg 92 train dalmatian/image_0013.jpg 92 train dalmatian/image_0054.jpg 92 train dalmatian/image_0055.jpg 92 train dalmatian/image_0024.jpg 92 train dalmatian/image_0003.jpg 92 train dalmatian/image_0009.jpg 92 train dalmatian/image_0042.jpg 92 train dalmatian/image_0040.jpg 92 train dalmatian/image_0067.jpg 92 train dalmatian/image_0047.jpg 92 train dalmatian/image_0064.jpg 92 train dalmatian/image_0043.jpg 92 train dalmatian/image_0044.jpg 92 train dalmatian/image_0033.jpg 92 train dalmatian/image_0001.jpg 92 train dalmatian/image_0021.jpg 92 train dalmatian/image_0057.jpg 92 train dalmatian/image_0008.jpg 92 train dalmatian/image_0035.jpg 92 train dalmatian/image_0005.jpg 92 train dalmatian/image_0059.jpg 92 train dalmatian/image_0034.jpg 92 train dalmatian/image_0002.jpg 92 train dalmatian/image_0048.jpg 92 train dalmatian/image_0053.jpg 92 train dalmatian/image_0020.jpg 92 train dalmatian/image_0046.jpg 92 train dalmatian/image_0041.jpg 92 train dalmatian/image_0027.jpg 92 train dalmatian/image_0014.jpg 92 train dalmatian/image_0058.jpg 92 train wheelchair/image_0037.jpg 93 train wheelchair/image_0049.jpg 93 train wheelchair/image_0051.jpg 93 train wheelchair/image_0006.jpg 93 train wheelchair/image_0016.jpg 93 train wheelchair/image_0030.jpg 93 train wheelchair/image_0012.jpg 93 train wheelchair/image_0025.jpg 93 train wheelchair/image_0052.jpg 93 train wheelchair/image_0007.jpg 93 train wheelchair/image_0019.jpg 93 train wheelchair/image_0031.jpg 93 train wheelchair/image_0011.jpg 93 train wheelchair/image_0050.jpg 93 train wheelchair/image_0045.jpg 93 train wheelchair/image_0017.jpg 93 train wheelchair/image_0018.jpg 93 train wheelchair/image_0013.jpg 93 train wheelchair/image_0054.jpg 93 train wheelchair/image_0055.jpg 93 train wheelchair/image_0024.jpg 93 train wheelchair/image_0003.jpg 93 train wheelchair/image_0009.jpg 93 train wheelchair/image_0042.jpg 93 train wheelchair/image_0040.jpg 93 train wheelchair/image_0047.jpg 93 train wheelchair/image_0043.jpg 93 train wheelchair/image_0044.jpg 93 train wheelchair/image_0033.jpg 93 train wheelchair/image_0001.jpg 93 train wheelchair/image_0021.jpg 93 train wheelchair/image_0057.jpg 93 train wheelchair/image_0008.jpg 93 train wheelchair/image_0035.jpg 93 train wheelchair/image_0005.jpg 93 train wheelchair/image_0059.jpg 93 train wheelchair/image_0034.jpg 93 train wheelchair/image_0002.jpg 93 train wheelchair/image_0048.jpg 93 train wheelchair/image_0053.jpg 93 train wheelchair/image_0020.jpg 93 train wheelchair/image_0046.jpg 93 train wheelchair/image_0041.jpg 93 train wheelchair/image_0027.jpg 93 train wheelchair/image_0014.jpg 93 train wheelchair/image_0058.jpg 93 train wheelchair/image_0026.jpg 93 train watch/image_0171.jpg 94 train watch/image_0062.jpg 94 train watch/image_0037.jpg 94 train watch/image_0049.jpg 94 train watch/image_0083.jpg 94 train watch/image_0101.jpg 94 train watch/image_0051.jpg 94 train watch/image_0006.jpg 94 train watch/image_0142.jpg 94 train watch/image_0201.jpg 94 train watch/image_0016.jpg 94 train watch/image_0176.jpg 94 train watch/image_0155.jpg 94 train watch/image_0030.jpg 94 train watch/image_0091.jpg 94 train watch/image_0012.jpg 94 train watch/image_0066.jpg 94 train watch/image_0224.jpg 94 train watch/image_0148.jpg 94 train watch/image_0077.jpg 94 train watch/image_0129.jpg 94 train watch/image_0025.jpg 94 train watch/image_0139.jpg 94 train watch/image_0115.jpg 94 train watch/image_0123.jpg 94 train watch/image_0119.jpg 94 train watch/image_0060.jpg 94 train watch/image_0052.jpg 94 train watch/image_0108.jpg 94 train watch/image_0239.jpg 94 train watch/image_0223.jpg 94 train watch/image_0172.jpg 94 train watch/image_0007.jpg 94 train watch/image_0213.jpg 94 train watch/image_0063.jpg 94 train watch/image_0019.jpg 94 train watch/image_0192.jpg 94 train watch/image_0195.jpg 94 train watch/image_0113.jpg 94 train watch/image_0127.jpg 94 train watch/image_0215.jpg 94 train watch/image_0031.jpg 94 train watch/image_0228.jpg 94 train watch/image_0219.jpg 94 train watch/image_0011.jpg 94 train watch/image_0233.jpg 94 train watch/image_0061.jpg 94 train watch/image_0090.jpg 94 train watch/image_0229.jpg 94 train watch/image_0126.jpg 94 train watch/image_0218.jpg 94 train watch/image_0216.jpg 94 train watch/image_0050.jpg 94 train watch/image_0177.jpg 94 train watch/image_0196.jpg 94 train watch/image_0187.jpg 94 train watch/image_0234.jpg 94 train watch/image_0045.jpg 94 train watch/image_0017.jpg 94 train watch/image_0131.jpg 94 train watch/image_0150.jpg 94 train watch/image_0018.jpg 94 train watch/image_0209.jpg 94 train watch/image_0013.jpg 94 train watch/image_0054.jpg 94 train watch/image_0094.jpg 94 train watch/image_0082.jpg 94 train watch/image_0097.jpg 94 train watch/image_0130.jpg 94 train watch/image_0141.jpg 94 train watch/image_0144.jpg 94 train watch/image_0109.jpg 94 train watch/image_0079.jpg 94 train watch/image_0055.jpg 94 train watch/image_0217.jpg 94 train watch/image_0071.jpg 94 train watch/image_0198.jpg 94 train watch/image_0168.jpg 94 train watch/image_0211.jpg 94 train watch/image_0103.jpg 94 train watch/image_0024.jpg 94 train watch/image_0188.jpg 94 train watch/image_0105.jpg 94 train watch/image_0074.jpg 94 train watch/image_0189.jpg 94 train watch/image_0003.jpg 94 train watch/image_0009.jpg 94 train watch/image_0226.jpg 94 train watch/image_0185.jpg 94 train watch/image_0118.jpg 94 train watch/image_0230.jpg 94 train watch/image_0134.jpg 94 train watch/image_0042.jpg 94 train watch/image_0221.jpg 94 train watch/image_0093.jpg 94 train watch/image_0232.jpg 94 train watch/image_0040.jpg 94 train watch/image_0067.jpg 94 train watch/image_0122.jpg 94 train watch/image_0205.jpg 94 train watch/image_0161.jpg 94 train watch/image_0146.jpg 94 train watch/image_0204.jpg 94 train watch/image_0125.jpg 94 train watch/image_0047.jpg 94 train watch/image_0160.jpg 94 train watch/image_0064.jpg 94 train watch/image_0070.jpg 94 train watch/image_0112.jpg 94 train watch/image_0078.jpg 94 train watch/image_0068.jpg 94 train watch/image_0085.jpg 94 train watch/image_0110.jpg 94 train watch/image_0143.jpg 94 train watch/image_0043.jpg 94 train watch/image_0162.jpg 94 train watch/image_0132.jpg 94 train watch/image_0157.jpg 94 train watch/image_0206.jpg 94 train watch/image_0044.jpg 94 train watch/image_0199.jpg 94 train watch/image_0033.jpg 94 train watch/image_0136.jpg 94 train watch/image_0180.jpg 94 train watch/image_0184.jpg 94 train watch/image_0073.jpg 94 train watch/image_0001.jpg 94 train watch/image_0167.jpg 94 train watch/image_0080.jpg 94 train watch/image_0156.jpg 94 train watch/image_0021.jpg 94 train watch/image_0095.jpg 94 train watch/image_0057.jpg 94 train watch/image_0114.jpg 94 train watch/image_0153.jpg 94 train watch/image_0098.jpg 94 train watch/image_0203.jpg 94 train watch/image_0147.jpg 94 train watch/image_0008.jpg 94 train watch/image_0207.jpg 94 train watch/image_0208.jpg 94 train watch/image_0222.jpg 94 train watch/image_0227.jpg 94 train watch/image_0035.jpg 94 train watch/image_0166.jpg 94 train watch/image_0169.jpg 94 train watch/image_0191.jpg 94 train watch/image_0128.jpg 94 train watch/image_0102.jpg 94 train watch/image_0178.jpg 94 train watch/image_0005.jpg 94 train watch/image_0173.jpg 94 train watch/image_0087.jpg 94 train watch/image_0089.jpg 94 train watch/image_0059.jpg 94 train watch/image_0159.jpg 94 train watch/image_0225.jpg 94 train watch/image_0186.jpg 94 train watch/image_0069.jpg 94 train watch/image_0200.jpg 94 train watch/image_0099.jpg 94 train watch/image_0034.jpg 94 train watch/image_0121.jpg 94 train watch/image_0181.jpg 94 train watch/image_0183.jpg 94 train watch/image_0175.jpg 94 train watch/image_0002.jpg 94 train watch/image_0165.jpg 94 train watch/image_0154.jpg 94 train watch/image_0193.jpg 94 train watch/image_0048.jpg 94 train watch/image_0053.jpg 94 train watch/image_0084.jpg 94 train watch/image_0020.jpg 94 train watch/image_0046.jpg 94 train watch/image_0041.jpg 94 train watch/image_0027.jpg 94 train watch/image_0014.jpg 94 train watch/image_0058.jpg 94 train watch/image_0212.jpg 94 train watch/image_0163.jpg 94 train watch/image_0202.jpg 94 train watch/image_0104.jpg 94 train watch/image_0081.jpg 94 train watch/image_0237.jpg 94 train watch/image_0149.jpg 94 train watch/image_0088.jpg 94 train watch/image_0026.jpg 94 train watch/image_0231.jpg 94 train watch/image_0004.jpg 94 train watch/image_0015.jpg 94 train sea_horse/image_0037.jpg 95 train sea_horse/image_0049.jpg 95 train sea_horse/image_0051.jpg 95 train sea_horse/image_0006.jpg 95 train sea_horse/image_0016.jpg 95 train sea_horse/image_0030.jpg 95 train sea_horse/image_0012.jpg 95 train sea_horse/image_0025.jpg 95 train sea_horse/image_0052.jpg 95 train sea_horse/image_0007.jpg 95 train sea_horse/image_0019.jpg 95 train sea_horse/image_0031.jpg 95 train sea_horse/image_0011.jpg 95 train sea_horse/image_0050.jpg 95 train sea_horse/image_0045.jpg 95 train sea_horse/image_0017.jpg 95 train sea_horse/image_0018.jpg 95 train sea_horse/image_0013.jpg 95 train sea_horse/image_0054.jpg 95 train sea_horse/image_0055.jpg 95 train sea_horse/image_0024.jpg 95 train sea_horse/image_0003.jpg 95 train sea_horse/image_0009.jpg 95 train sea_horse/image_0042.jpg 95 train sea_horse/image_0040.jpg 95 train sea_horse/image_0047.jpg 95 train sea_horse/image_0043.jpg 95 train sea_horse/image_0044.jpg 95 train sea_horse/image_0033.jpg 95 train sea_horse/image_0001.jpg 95 train sea_horse/image_0021.jpg 95 train sea_horse/image_0057.jpg 95 train sea_horse/image_0008.jpg 95 train sea_horse/image_0035.jpg 95 train sea_horse/image_0005.jpg 95 train sea_horse/image_0034.jpg 95 train sea_horse/image_0002.jpg 95 train sea_horse/image_0048.jpg 95 train sea_horse/image_0053.jpg 95 train sea_horse/image_0020.jpg 95 train sea_horse/image_0046.jpg 95 train sea_horse/image_0041.jpg 95 train sea_horse/image_0027.jpg 95 train sea_horse/image_0014.jpg 95 train sea_horse/image_0026.jpg 95 train pyramid/image_0037.jpg 96 train pyramid/image_0049.jpg 96 train pyramid/image_0051.jpg 96 train pyramid/image_0006.jpg 96 train pyramid/image_0016.jpg 96 train pyramid/image_0030.jpg 96 train pyramid/image_0012.jpg 96 train pyramid/image_0025.jpg 96 train pyramid/image_0052.jpg 96 train pyramid/image_0007.jpg 96 train pyramid/image_0019.jpg 96 train pyramid/image_0031.jpg 96 train pyramid/image_0011.jpg 96 train pyramid/image_0050.jpg 96 train pyramid/image_0045.jpg 96 train pyramid/image_0017.jpg 96 train pyramid/image_0018.jpg 96 train pyramid/image_0013.jpg 96 train pyramid/image_0054.jpg 96 train pyramid/image_0055.jpg 96 train pyramid/image_0024.jpg 96 train pyramid/image_0003.jpg 96 train pyramid/image_0009.jpg 96 train pyramid/image_0042.jpg 96 train pyramid/image_0040.jpg 96 train pyramid/image_0047.jpg 96 train pyramid/image_0043.jpg 96 train pyramid/image_0044.jpg 96 train pyramid/image_0033.jpg 96 train pyramid/image_0001.jpg 96 train pyramid/image_0021.jpg 96 train pyramid/image_0057.jpg 96 train pyramid/image_0008.jpg 96 train pyramid/image_0035.jpg 96 train pyramid/image_0005.jpg 96 train pyramid/image_0034.jpg 96 train pyramid/image_0002.jpg 96 train pyramid/image_0048.jpg 96 train pyramid/image_0053.jpg 96 train pyramid/image_0020.jpg 96 train pyramid/image_0046.jpg 96 train pyramid/image_0041.jpg 96 train pyramid/image_0027.jpg 96 train pyramid/image_0014.jpg 96 train pyramid/image_0026.jpg 96 train strawberry/image_0006.jpg 97 train strawberry/image_0016.jpg 97 train strawberry/image_0030.jpg 97 train strawberry/image_0012.jpg 97 train strawberry/image_0025.jpg 97 train strawberry/image_0007.jpg 97 train strawberry/image_0019.jpg 97 train strawberry/image_0031.jpg 97 train strawberry/image_0011.jpg 97 train strawberry/image_0017.jpg 97 train strawberry/image_0018.jpg 97 train strawberry/image_0013.jpg 97 train strawberry/image_0024.jpg 97 train strawberry/image_0003.jpg 97 train strawberry/image_0009.jpg 97 train strawberry/image_0033.jpg 97 train strawberry/image_0001.jpg 97 train strawberry/image_0021.jpg 97 train strawberry/image_0008.jpg 97 train strawberry/image_0035.jpg 97 train strawberry/image_0005.jpg 97 train strawberry/image_0034.jpg 97 train strawberry/image_0002.jpg 97 train strawberry/image_0020.jpg 97 train strawberry/image_0027.jpg 97 train strawberry/image_0014.jpg 97 train strawberry/image_0026.jpg 97 train strawberry/image_0004.jpg 97 train Faces_easy/image_0316.jpg 98 train Faces_easy/image_0258.jpg 98 train Faces_easy/image_0426.jpg 98 train Faces_easy/image_0302.jpg 98 train Faces_easy/image_0171.jpg 98 train Faces_easy/image_0062.jpg 98 train Faces_easy/image_0346.jpg 98 train Faces_easy/image_0435.jpg 98 train Faces_easy/image_0341.jpg 98 train Faces_easy/image_0340.jpg 98 train Faces_easy/image_0331.jpg 98 train Faces_easy/image_0037.jpg 98 train Faces_easy/image_0049.jpg 98 train Faces_easy/image_0337.jpg 98 train Faces_easy/image_0317.jpg 98 train Faces_easy/image_0357.jpg 98 train Faces_easy/image_0083.jpg 98 train Faces_easy/image_0101.jpg 98 train Faces_easy/image_0051.jpg 98 train Faces_easy/image_0006.jpg 98 train Faces_easy/image_0142.jpg 98 train Faces_easy/image_0250.jpg 98 train Faces_easy/image_0427.jpg 98 train Faces_easy/image_0396.jpg 98 train Faces_easy/image_0332.jpg 98 train Faces_easy/image_0335.jpg 98 train Faces_easy/image_0201.jpg 98 train Faces_easy/image_0016.jpg 98 train Faces_easy/image_0373.jpg 98 train Faces_easy/image_0176.jpg 98 train Faces_easy/image_0411.jpg 98 train Faces_easy/image_0155.jpg 98 train Faces_easy/image_0030.jpg 98 train Faces_easy/image_0091.jpg 98 train Faces_easy/image_0397.jpg 98 train Faces_easy/image_0012.jpg 98 train Faces_easy/image_0066.jpg 98 train Faces_easy/image_0283.jpg 98 train Faces_easy/image_0318.jpg 98 train Faces_easy/image_0224.jpg 98 train Faces_easy/image_0148.jpg 98 train Faces_easy/image_0077.jpg 98 train Faces_easy/image_0129.jpg 98 train Faces_easy/image_0339.jpg 98 train Faces_easy/image_0025.jpg 98 train Faces_easy/image_0328.jpg 98 train Faces_easy/image_0139.jpg 98 train Faces_easy/image_0425.jpg 98 train Faces_easy/image_0292.jpg 98 train Faces_easy/image_0348.jpg 98 train Faces_easy/image_0115.jpg 98 train Faces_easy/image_0123.jpg 98 train Faces_easy/image_0119.jpg 98 train Faces_easy/image_0060.jpg 98 train Faces_easy/image_0269.jpg 98 train Faces_easy/image_0052.jpg 98 train Faces_easy/image_0108.jpg 98 train Faces_easy/image_0239.jpg 98 train Faces_easy/image_0370.jpg 98 train Faces_easy/image_0326.jpg 98 train Faces_easy/image_0223.jpg 98 train Faces_easy/image_0172.jpg 98 train Faces_easy/image_0372.jpg 98 train Faces_easy/image_0344.jpg 98 train Faces_easy/image_0390.jpg 98 train Faces_easy/image_0007.jpg 98 train Faces_easy/image_0213.jpg 98 train Faces_easy/image_0312.jpg 98 train Faces_easy/image_0063.jpg 98 train Faces_easy/image_0019.jpg 98 train Faces_easy/image_0407.jpg 98 train Faces_easy/image_0192.jpg 98 train Faces_easy/image_0384.jpg 98 train Faces_easy/image_0195.jpg 98 train Faces_easy/image_0113.jpg 98 train Faces_easy/image_0127.jpg 98 train Faces_easy/image_0215.jpg 98 train Faces_easy/image_0389.jpg 98 train Faces_easy/image_0031.jpg 98 train Faces_easy/image_0349.jpg 98 train Faces_easy/image_0228.jpg 98 train Faces_easy/image_0380.jpg 98 train Faces_easy/image_0219.jpg 98 train Faces_easy/image_0353.jpg 98 train Faces_easy/image_0011.jpg 98 train Faces_easy/image_0233.jpg 98 train Faces_easy/image_0061.jpg 98 train Faces_easy/image_0090.jpg 98 train Faces_easy/image_0351.jpg 98 train Faces_easy/image_0261.jpg 98 train Faces_easy/image_0229.jpg 98 train Faces_easy/image_0126.jpg 98 train Faces_easy/image_0240.jpg 98 train Faces_easy/image_0305.jpg 98 train Faces_easy/image_0334.jpg 98 train Faces_easy/image_0354.jpg 98 train Faces_easy/image_0218.jpg 98 train Faces_easy/image_0366.jpg 98 train Faces_easy/image_0288.jpg 98 train Faces_easy/image_0216.jpg 98 train Faces_easy/image_0429.jpg 98 train Faces_easy/image_0277.jpg 98 train Faces_easy/image_0358.jpg 98 train Faces_easy/image_0050.jpg 98 train Faces_easy/image_0246.jpg 98 train Faces_easy/image_0361.jpg 98 train Faces_easy/image_0177.jpg 98 train Faces_easy/image_0356.jpg 98 train Faces_easy/image_0196.jpg 98 train Faces_easy/image_0293.jpg 98 train Faces_easy/image_0406.jpg 98 train Faces_easy/image_0347.jpg 98 train Faces_easy/image_0187.jpg 98 train Faces_easy/image_0234.jpg 98 train Faces_easy/image_0045.jpg 98 train Faces_easy/image_0303.jpg 98 train Faces_easy/image_0363.jpg 98 train Faces_easy/image_0017.jpg 98 train Faces_easy/image_0131.jpg 98 train Faces_easy/image_0296.jpg 98 train Faces_easy/image_0150.jpg 98 train Faces_easy/image_0018.jpg 98 train Faces_easy/image_0350.jpg 98 train Faces_easy/image_0209.jpg 98 train Faces_easy/image_0013.jpg 98 train Faces_easy/image_0054.jpg 98 train Faces_easy/image_0094.jpg 98 train Faces_easy/image_0082.jpg 98 train Faces_easy/image_0097.jpg 98 train Faces_easy/image_0130.jpg 98 train Faces_easy/image_0382.jpg 98 train Faces_easy/image_0141.jpg 98 train Faces_easy/image_0144.jpg 98 train Faces_easy/image_0109.jpg 98 train Faces_easy/image_0079.jpg 98 train Faces_easy/image_0306.jpg 98 train Faces_easy/image_0398.jpg 98 train Faces_easy/image_0055.jpg 98 train Faces_easy/image_0367.jpg 98 train Faces_easy/image_0365.jpg 98 train Faces_easy/image_0217.jpg 98 train Faces_easy/image_0343.jpg 98 train Faces_easy/image_0432.jpg 98 train Faces_easy/image_0071.jpg 98 train Faces_easy/image_0282.jpg 98 train Faces_easy/image_0393.jpg 98 train Faces_easy/image_0287.jpg 98 train Faces_easy/image_0198.jpg 98 train Faces_easy/image_0262.jpg 98 train Faces_easy/image_0168.jpg 98 train Faces_easy/image_0368.jpg 98 train Faces_easy/image_0420.jpg 98 train Faces_easy/image_0211.jpg 98 train Faces_easy/image_0270.jpg 98 train Faces_easy/image_0103.jpg 98 train Faces_easy/image_0024.jpg 98 train Faces_easy/image_0297.jpg 98 train Faces_easy/image_0188.jpg 98 train Faces_easy/image_0105.jpg 98 train Faces_easy/image_0074.jpg 98 train Faces_easy/image_0294.jpg 98 train Faces_easy/image_0391.jpg 98 train Faces_easy/image_0189.jpg 98 train Faces_easy/image_0003.jpg 98 train Faces_easy/image_0009.jpg 98 train Faces_easy/image_0387.jpg 98 train Faces_easy/image_0275.jpg 98 train Faces_easy/image_0226.jpg 98 train Faces_easy/image_0185.jpg 98 train Faces_easy/image_0118.jpg 98 train Faces_easy/image_0304.jpg 98 train Faces_easy/image_0278.jpg 98 train Faces_easy/image_0230.jpg 98 train Faces_easy/image_0404.jpg 98 train Faces_easy/image_0134.jpg 98 train Faces_easy/image_0249.jpg 98 train Faces_easy/image_0042.jpg 98 train Faces_easy/image_0352.jpg 98 train Faces_easy/image_0333.jpg 98 train Faces_easy/image_0221.jpg 98 train Faces_easy/image_0093.jpg 98 train Faces_easy/image_0266.jpg 98 train Faces_easy/image_0412.jpg 98 train Faces_easy/image_0301.jpg 98 train Faces_easy/image_0392.jpg 98 train Faces_easy/image_0254.jpg 98 train Faces_easy/image_0321.jpg 98 train Faces_easy/image_0386.jpg 98 train Faces_easy/image_0232.jpg 98 train Faces_easy/image_0324.jpg 98 train Faces_easy/image_0040.jpg 98 train Faces_easy/image_0067.jpg 98 train Faces_easy/image_0310.jpg 98 train Faces_easy/image_0330.jpg 98 train Faces_easy/image_0122.jpg 98 train Faces_easy/image_0205.jpg 98 train Faces_easy/image_0161.jpg 98 train Faces_easy/image_0146.jpg 98 train Faces_easy/image_0300.jpg 98 train Faces_easy/image_0369.jpg 98 train Faces_easy/image_0204.jpg 98 train Faces_easy/image_0125.jpg 98 train Faces_easy/image_0047.jpg 98 train Faces_easy/image_0285.jpg 98 train Faces_easy/image_0160.jpg 98 train Faces_easy/image_0422.jpg 98 train Faces_easy/image_0064.jpg 98 train Faces_easy/image_0070.jpg 98 train Faces_easy/image_0264.jpg 98 train Faces_easy/image_0112.jpg 98 train Faces_easy/image_0078.jpg 98 train Faces_easy/image_0068.jpg 98 train Faces_easy/image_0433.jpg 98 train Faces_easy/image_0085.jpg 98 train Faces_easy/image_0410.jpg 98 train Faces_easy/image_0110.jpg 98 train Faces_easy/image_0259.jpg 98 train Faces_easy/image_0143.jpg 98 train Faces_easy/image_0043.jpg 98 train Faces_easy/image_0162.jpg 98 train Faces_easy/image_0132.jpg 98 train Faces_easy/image_0157.jpg 98 train Faces_easy/image_0206.jpg 98 train Faces_easy/image_0376.jpg 98 train Faces_easy/image_0319.jpg 98 train Faces_easy/image_0371.jpg 98 train Faces_easy/image_0044.jpg 98 train Faces_easy/image_0199.jpg 98 train Faces_easy/image_0033.jpg 98 train Faces_easy/image_0136.jpg 98 train Faces_easy/image_0180.jpg 98 train Faces_easy/image_0260.jpg 98 train Faces_easy/image_0184.jpg 98 train Faces_easy/image_0073.jpg 98 train Faces_easy/image_0395.jpg 98 train Faces_easy/image_0257.jpg 98 train Faces_easy/image_0364.jpg 98 train Faces_easy/image_0001.jpg 98 train Faces_easy/image_0167.jpg 98 train Faces_easy/image_0284.jpg 98 train Faces_easy/image_0375.jpg 98 train Faces_easy/image_0080.jpg 98 train Faces_easy/image_0156.jpg 98 train Faces_easy/image_0021.jpg 98 train Faces_easy/image_0325.jpg 98 train Faces_easy/image_0267.jpg 98 train Faces_easy/image_0299.jpg 98 train Faces_easy/image_0421.jpg 98 train Faces_easy/image_0360.jpg 98 train Faces_easy/image_0095.jpg 98 train Faces_easy/image_0057.jpg 98 train Faces_easy/image_0403.jpg 98 train Faces_easy/image_0114.jpg 98 train Faces_easy/image_0153.jpg 98 train Faces_easy/image_0416.jpg 98 train Faces_easy/image_0098.jpg 98 train Faces_easy/image_0256.jpg 98 train Faces_easy/image_0402.jpg 98 train Faces_easy/image_0203.jpg 98 train Faces_easy/image_0338.jpg 98 train Faces_easy/image_0265.jpg 98 train Faces_easy/image_0147.jpg 98 train Faces_easy/image_0008.jpg 98 train Faces_easy/image_0207.jpg 98 train Faces_easy/image_0208.jpg 98 train Faces_easy/image_0222.jpg 98 train Faces_easy/image_0320.jpg 98 train Faces_easy/image_0227.jpg 98 train Faces_easy/image_0286.jpg 98 train Faces_easy/image_0035.jpg 98 train Faces_easy/image_0399.jpg 98 train Faces_easy/image_0166.jpg 98 train Faces_easy/image_0169.jpg 98 train Faces_easy/image_0191.jpg 98 train Faces_easy/image_0128.jpg 98 train Faces_easy/image_0414.jpg 98 train Faces_easy/image_0102.jpg 98 train Faces_easy/image_0434.jpg 98 train Faces_easy/image_0178.jpg 98 train Faces_easy/image_0431.jpg 98 train Faces_easy/image_0243.jpg 98 train Faces_easy/image_0005.jpg 98 train Faces_easy/image_0263.jpg 98 train Faces_easy/image_0173.jpg 98 train Faces_easy/image_0087.jpg 98 train Faces_easy/image_0417.jpg 98 train Faces_easy/image_0089.jpg 98 train Faces_easy/image_0059.jpg 98 train Faces_easy/image_0253.jpg 98 train Faces_easy/image_0159.jpg 98 train Faces_easy/image_0225.jpg 98 train Faces_easy/image_0186.jpg 98 train Faces_easy/image_0388.jpg 98 train Faces_easy/image_0069.jpg 98 train Faces_easy/image_0315.jpg 98 train Faces_easy/image_0295.jpg 98 train Faces_easy/image_0200.jpg 98 train Faces_easy/image_0099.jpg 98 train Faces_easy/image_0336.jpg 98 train Faces_easy/image_0034.jpg 98 train Faces_easy/image_0409.jpg 98 train Faces_easy/image_0314.jpg 98 train Faces_easy/image_0276.jpg 98 train Faces_easy/image_0121.jpg 98 train Faces_easy/image_0428.jpg 98 train Faces_easy/image_0181.jpg 98 train Faces_easy/image_0280.jpg 98 train Faces_easy/image_0183.jpg 98 train Faces_easy/image_0383.jpg 98 train Faces_easy/image_0175.jpg 98 train Faces_easy/image_0002.jpg 98 train Faces_easy/image_0291.jpg 98 train Faces_easy/image_0165.jpg 98 train Faces_easy/image_0154.jpg 98 train Faces_easy/image_0271.jpg 98 train Faces_easy/image_0193.jpg 98 train Faces_easy/image_0048.jpg 98 train Faces_easy/image_0053.jpg 98 train Faces_easy/image_0255.jpg 98 train Faces_easy/image_0084.jpg 98 train Faces_easy/image_0313.jpg 98 train Faces_easy/image_0020.jpg 98 train Faces_easy/image_0251.jpg 98 train Faces_easy/image_0046.jpg 98 train Faces_easy/image_0041.jpg 98 train Faces_easy/image_0027.jpg 98 train Faces_easy/image_0322.jpg 98 train Faces_easy/image_0014.jpg 98 train Faces_easy/image_0058.jpg 98 train Faces_easy/image_0408.jpg 98 train Faces_easy/image_0329.jpg 98 train Faces_easy/image_0212.jpg 98 train Faces_easy/image_0163.jpg 98 train Faces_easy/image_0202.jpg 98 train Faces_easy/image_0104.jpg 98 train Faces_easy/image_0081.jpg 98 train Faces_easy/image_0272.jpg 98 train Faces_easy/image_0400.jpg 98 train Faces_easy/image_0237.jpg 98 train Faces_easy/image_0308.jpg 98 train Faces_easy/image_0149.jpg 98 train Faces_easy/image_0088.jpg 98 train Faces_easy/image_0309.jpg 98 train Faces_easy/image_0026.jpg 98 train Faces_easy/image_0231.jpg 98 train Faces_easy/image_0004.jpg 98 train Faces_easy/image_0015.jpg 98 train Faces_easy/image_0355.jpg 98 train cougar_face/image_0062.jpg 99 train cougar_face/image_0037.jpg 99 train cougar_face/image_0049.jpg 99 train cougar_face/image_0051.jpg 99 train cougar_face/image_0006.jpg 99 train cougar_face/image_0016.jpg 99 train cougar_face/image_0030.jpg 99 train cougar_face/image_0012.jpg 99 train cougar_face/image_0066.jpg 99 train cougar_face/image_0025.jpg 99 train cougar_face/image_0060.jpg 99 train cougar_face/image_0052.jpg 99 train cougar_face/image_0007.jpg 99 train cougar_face/image_0063.jpg 99 train cougar_face/image_0019.jpg 99 train cougar_face/image_0031.jpg 99 train cougar_face/image_0011.jpg 99 train cougar_face/image_0061.jpg 99 train cougar_face/image_0050.jpg 99 train cougar_face/image_0045.jpg 99 train cougar_face/image_0017.jpg 99 train cougar_face/image_0018.jpg 99 train cougar_face/image_0013.jpg 99 train cougar_face/image_0054.jpg 99 train cougar_face/image_0055.jpg 99 train cougar_face/image_0024.jpg 99 train cougar_face/image_0003.jpg 99 train cougar_face/image_0009.jpg 99 train cougar_face/image_0042.jpg 99 train cougar_face/image_0040.jpg 99 train cougar_face/image_0067.jpg 99 train cougar_face/image_0047.jpg 99 train cougar_face/image_0064.jpg 99 train cougar_face/image_0068.jpg 99 train cougar_face/image_0043.jpg 99 train cougar_face/image_0044.jpg 99 train cougar_face/image_0033.jpg 99 train cougar_face/image_0001.jpg 99 train cougar_face/image_0021.jpg 99 train cougar_face/image_0057.jpg 99 train cougar_face/image_0008.jpg 99 train cougar_face/image_0035.jpg 99 train cougar_face/image_0005.jpg 99 train cougar_face/image_0059.jpg 99 train cougar_face/image_0069.jpg 99 train cougar_face/image_0034.jpg 99 train cougar_face/image_0002.jpg 99 train cougar_face/image_0048.jpg 99 train cougar_face/image_0053.jpg 99 train cougar_face/image_0020.jpg 99 train cougar_face/image_0046.jpg 99 train cougar_face/image_0041.jpg 99 train cougar_face/image_0027.jpg 99 train cougar_face/image_0014.jpg 99 train cougar_face/image_0058.jpg 99 train mayfly/image_0037.jpg 100 train mayfly/image_0006.jpg 100 train mayfly/image_0016.jpg 100 train mayfly/image_0030.jpg 100 train mayfly/image_0012.jpg 100 train mayfly/image_0025.jpg 100 train mayfly/image_0007.jpg 100 train mayfly/image_0019.jpg 100 train mayfly/image_0031.jpg 100 train mayfly/image_0011.jpg 100 train mayfly/image_0017.jpg 100 train mayfly/image_0018.jpg 100 train mayfly/image_0013.jpg 100 train mayfly/image_0024.jpg 100 train mayfly/image_0003.jpg 100 train mayfly/image_0009.jpg 100 train mayfly/image_0040.jpg 100 train mayfly/image_0033.jpg 100 train mayfly/image_0001.jpg 100 train mayfly/image_0021.jpg 100 train mayfly/image_0008.jpg 100 train mayfly/image_0035.jpg 100 train mayfly/image_0005.jpg 100 train mayfly/image_0034.jpg 100 train mayfly/image_0002.jpg 100 train mayfly/image_0020.jpg 100 train mayfly/image_0027.jpg 100 train mayfly/image_0014.jpg 100 train mayfly/image_0026.jpg 100 train mayfly/image_0004.jpg 100 train mayfly/image_0015.jpg 100 train mayfly/image_0029.jpg 100 train wrench/image_0037.jpg 101 train wrench/image_0006.jpg 101 train wrench/image_0016.jpg 101 train wrench/image_0030.jpg 101 train wrench/image_0012.jpg 101 train wrench/image_0025.jpg 101 train wrench/image_0007.jpg 101 train wrench/image_0019.jpg 101 train wrench/image_0031.jpg 101 train wrench/image_0011.jpg 101 train wrench/image_0017.jpg 101 train wrench/image_0018.jpg 101 train wrench/image_0013.jpg 101 train wrench/image_0024.jpg 101 train wrench/image_0003.jpg 101 train wrench/image_0009.jpg 101 train wrench/image_0033.jpg 101 train wrench/image_0001.jpg 101 train wrench/image_0021.jpg 101 train wrench/image_0008.jpg 101 train wrench/image_0035.jpg 101 train wrench/image_0005.jpg 101 train wrench/image_0034.jpg 101 train wrench/image_0002.jpg 101 train wrench/image_0020.jpg 101 train wrench/image_0027.jpg 101 train wrench/image_0014.jpg 101 train wrench/image_0026.jpg 101 train wrench/image_0004.jpg 101 train wrench/image_0015.jpg 101 train wrench/image_0029.jpg 101 train airplanes/image_0029.jpg 0 test airplanes/image_0662.jpg 0 test airplanes/image_0647.jpg 0 test airplanes/image_0798.jpg 0 test airplanes/image_0327.jpg 0 test airplanes/image_0120.jpg 0 test airplanes/image_0281.jpg 0 test airplanes/image_0603.jpg 0 test airplanes/image_0653.jpg 0 test airplanes/image_0381.jpg 0 test airplanes/image_0220.jpg 0 test airplanes/image_0345.jpg 0 test airplanes/image_0520.jpg 0 test airplanes/image_0685.jpg 0 test airplanes/image_0503.jpg 0 test airplanes/image_0023.jpg 0 test airplanes/image_0135.jpg 0 test airplanes/image_0452.jpg 0 test airplanes/image_0584.jpg 0 test airplanes/image_0423.jpg 0 test airplanes/image_0782.jpg 0 test airplanes/image_0454.jpg 0 test airplanes/image_0466.jpg 0 test airplanes/image_0111.jpg 0 test airplanes/image_0124.jpg 0 test airplanes/image_0279.jpg 0 test airplanes/image_0478.jpg 0 test airplanes/image_0642.jpg 0 test airplanes/image_0666.jpg 0 test airplanes/image_0708.jpg 0 test airplanes/image_0424.jpg 0 test airplanes/image_0418.jpg 0 test airplanes/image_0379.jpg 0 test airplanes/image_0626.jpg 0 test airplanes/image_0610.jpg 0 test airplanes/image_0273.jpg 0 test airplanes/image_0587.jpg 0 test airplanes/image_0072.jpg 0 test airplanes/image_0598.jpg 0 test airplanes/image_0508.jpg 0 test airplanes/image_0145.jpg 0 test airplanes/image_0750.jpg 0 test airplanes/image_0214.jpg 0 test airplanes/image_0174.jpg 0 test airplanes/image_0754.jpg 0 test airplanes/image_0726.jpg 0 test airplanes/image_0133.jpg 0 test airplanes/image_0152.jpg 0 test airplanes/image_0289.jpg 0 test airplanes/image_0674.jpg 0 test airplanes/image_0609.jpg 0 test airplanes/image_0342.jpg 0 test airplanes/image_0298.jpg 0 test airplanes/image_0710.jpg 0 test airplanes/image_0010.jpg 0 test airplanes/image_0248.jpg 0 test airplanes/image_0245.jpg 0 test airplanes/image_0056.jpg 0 test airplanes/image_0022.jpg 0 test airplanes/image_0687.jpg 0 test airplanes/image_0307.jpg 0 test airplanes/image_0755.jpg 0 test airplanes/image_0164.jpg 0 test airplanes/image_0377.jpg 0 test airplanes/image_0715.jpg 0 test airplanes/image_0065.jpg 0 test airplanes/image_0244.jpg 0 test airplanes/image_0777.jpg 0 test airplanes/image_0100.jpg 0 test airplanes/image_0137.jpg 0 test airplanes/image_0274.jpg 0 test airplanes/image_0323.jpg 0 test airplanes/image_0576.jpg 0 test airplanes/image_0526.jpg 0 test airplanes/image_0036.jpg 0 test airplanes/image_0785.jpg 0 test airplanes/image_0599.jpg 0 test airplanes/image_0086.jpg 0 test airplanes/image_0032.jpg 0 test airplanes/image_0405.jpg 0 test airplanes/image_0394.jpg 0 test airplanes/image_0076.jpg 0 test airplanes/image_0682.jpg 0 test airplanes/image_0439.jpg 0 test airplanes/image_0705.jpg 0 test airplanes/image_0252.jpg 0 test airplanes/image_0235.jpg 0 test airplanes/image_0510.jpg 0 test airplanes/image_0190.jpg 0 test airplanes/image_0290.jpg 0 test airplanes/image_0453.jpg 0 test airplanes/image_0106.jpg 0 test airplanes/image_0140.jpg 0 test airplanes/image_0415.jpg 0 test airplanes/image_0117.jpg 0 test airplanes/image_0038.jpg 0 test airplanes/image_0773.jpg 0 test airplanes/image_0663.jpg 0 test airplanes/image_0624.jpg 0 test airplanes/image_0210.jpg 0 test airplanes/image_0456.jpg 0 test airplanes/image_0107.jpg 0 test airplanes/image_0247.jpg 0 test airplanes/image_0800.jpg 0 test airplanes/image_0362.jpg 0 test airplanes/image_0760.jpg 0 test airplanes/image_0535.jpg 0 test airplanes/image_0158.jpg 0 test airplanes/image_0498.jpg 0 test airplanes/image_0430.jpg 0 test airplanes/image_0268.jpg 0 test airplanes/image_0660.jpg 0 test airplanes/image_0630.jpg 0 test airplanes/image_0592.jpg 0 test airplanes/image_0242.jpg 0 test airplanes/image_0401.jpg 0 test airplanes/image_0179.jpg 0 test airplanes/image_0170.jpg 0 test airplanes/image_0524.jpg 0 test airplanes/image_0378.jpg 0 test airplanes/image_0753.jpg 0 test airplanes/image_0507.jpg 0 test airplanes/image_0749.jpg 0 test airplanes/image_0039.jpg 0 test airplanes/image_0075.jpg 0 test airplanes/image_0744.jpg 0 test airplanes/image_0194.jpg 0 test airplanes/image_0695.jpg 0 test airplanes/image_0693.jpg 0 test airplanes/image_0096.jpg 0 test airplanes/image_0481.jpg 0 test airplanes/image_0238.jpg 0 test airplanes/image_0701.jpg 0 test airplanes/image_0236.jpg 0 test airplanes/image_0311.jpg 0 test airplanes/image_0724.jpg 0 test airplanes/image_0116.jpg 0 test airplanes/image_0419.jpg 0 test airplanes/image_0702.jpg 0 test airplanes/image_0725.jpg 0 test airplanes/image_0138.jpg 0 test airplanes/image_0774.jpg 0 test airplanes/image_0793.jpg 0 test airplanes/image_0449.jpg 0 test airplanes/image_0778.jpg 0 test airplanes/image_0197.jpg 0 test airplanes/image_0578.jpg 0 test airplanes/image_0241.jpg 0 test airplanes/image_0028.jpg 0 test airplanes/image_0151.jpg 0 test airplanes/image_0385.jpg 0 test airplanes/image_0550.jpg 0 test airplanes/image_0636.jpg 0 test airplanes/image_0545.jpg 0 test airplanes/image_0092.jpg 0 test airplanes/image_0632.jpg 0 test airplanes/image_0182.jpg 0 test airplanes/image_0413.jpg 0 test airplanes/image_0359.jpg 0 test airplanes/image_0374.jpg 0 test ant/image_0029.jpg 1 test ant/image_0023.jpg 1 test ant/image_0010.jpg 1 test ant/image_0022.jpg 1 test ant/image_0036.jpg 1 test ant/image_0032.jpg 1 test ant/image_0038.jpg 1 test ant/image_0039.jpg 1 test ant/image_0028.jpg 1 test Faces/image_0029.jpg 2 test Faces/image_0327.jpg 2 test Faces/image_0120.jpg 2 test Faces/image_0281.jpg 2 test Faces/image_0381.jpg 2 test Faces/image_0220.jpg 2 test Faces/image_0345.jpg 2 test Faces/image_0023.jpg 2 test Faces/image_0135.jpg 2 test Faces/image_0423.jpg 2 test Faces/image_0111.jpg 2 test Faces/image_0124.jpg 2 test Faces/image_0279.jpg 2 test Faces/image_0424.jpg 2 test Faces/image_0418.jpg 2 test Faces/image_0379.jpg 2 test Faces/image_0273.jpg 2 test Faces/image_0072.jpg 2 test Faces/image_0145.jpg 2 test Faces/image_0214.jpg 2 test Faces/image_0174.jpg 2 test Faces/image_0133.jpg 2 test Faces/image_0152.jpg 2 test Faces/image_0289.jpg 2 test Faces/image_0342.jpg 2 test Faces/image_0298.jpg 2 test Faces/image_0010.jpg 2 test Faces/image_0248.jpg 2 test Faces/image_0245.jpg 2 test Faces/image_0056.jpg 2 test Faces/image_0022.jpg 2 test Faces/image_0307.jpg 2 test Faces/image_0164.jpg 2 test Faces/image_0377.jpg 2 test Faces/image_0065.jpg 2 test Faces/image_0244.jpg 2 test Faces/image_0100.jpg 2 test Faces/image_0137.jpg 2 test Faces/image_0274.jpg 2 test Faces/image_0323.jpg 2 test Faces/image_0036.jpg 2 test Faces/image_0086.jpg 2 test Faces/image_0032.jpg 2 test Faces/image_0405.jpg 2 test Faces/image_0394.jpg 2 test Faces/image_0076.jpg 2 test Faces/image_0252.jpg 2 test Faces/image_0235.jpg 2 test Faces/image_0190.jpg 2 test Faces/image_0290.jpg 2 test Faces/image_0106.jpg 2 test Faces/image_0140.jpg 2 test Faces/image_0415.jpg 2 test Faces/image_0117.jpg 2 test Faces/image_0038.jpg 2 test Faces/image_0210.jpg 2 test Faces/image_0107.jpg 2 test Faces/image_0247.jpg 2 test Faces/image_0362.jpg 2 test Faces/image_0158.jpg 2 test Faces/image_0430.jpg 2 test Faces/image_0268.jpg 2 test Faces/image_0242.jpg 2 test Faces/image_0401.jpg 2 test Faces/image_0179.jpg 2 test Faces/image_0170.jpg 2 test Faces/image_0378.jpg 2 test Faces/image_0039.jpg 2 test Faces/image_0075.jpg 2 test Faces/image_0194.jpg 2 test Faces/image_0096.jpg 2 test Faces/image_0238.jpg 2 test Faces/image_0236.jpg 2 test Faces/image_0311.jpg 2 test Faces/image_0116.jpg 2 test Faces/image_0419.jpg 2 test Faces/image_0138.jpg 2 test Faces/image_0197.jpg 2 test Faces/image_0241.jpg 2 test Faces/image_0028.jpg 2 test Faces/image_0151.jpg 2 test Faces/image_0385.jpg 2 test Faces/image_0092.jpg 2 test Faces/image_0182.jpg 2 test Faces/image_0413.jpg 2 test Faces/image_0359.jpg 2 test Faces/image_0374.jpg 2 test rooster/image_0015.jpg 3 test rooster/image_0029.jpg 3 test rooster/image_0023.jpg 3 test rooster/image_0010.jpg 3 test rooster/image_0022.jpg 3 test rooster/image_0036.jpg 3 test rooster/image_0032.jpg 3 test rooster/image_0038.jpg 3 test rooster/image_0039.jpg 3 test rooster/image_0028.jpg 3 test water_lilly/image_0015.jpg 4 test water_lilly/image_0029.jpg 4 test water_lilly/image_0023.jpg 4 test water_lilly/image_0010.jpg 4 test water_lilly/image_0022.jpg 4 test water_lilly/image_0036.jpg 4 test water_lilly/image_0032.jpg 4 test water_lilly/image_0028.jpg 4 test elephant/image_0026.jpg 5 test elephant/image_0004.jpg 5 test elephant/image_0015.jpg 5 test elephant/image_0029.jpg 5 test elephant/image_0023.jpg 5 test elephant/image_0010.jpg 5 test elephant/image_0056.jpg 5 test elephant/image_0022.jpg 5 test elephant/image_0036.jpg 5 test elephant/image_0032.jpg 5 test elephant/image_0038.jpg 5 test elephant/image_0039.jpg 5 test elephant/image_0028.jpg 5 test umbrella/image_0004.jpg 6 test umbrella/image_0015.jpg 6 test umbrella/image_0029.jpg 6 test umbrella/image_0023.jpg 6 test umbrella/image_0072.jpg 6 test umbrella/image_0010.jpg 6 test umbrella/image_0056.jpg 6 test umbrella/image_0022.jpg 6 test umbrella/image_0065.jpg 6 test umbrella/image_0036.jpg 6 test umbrella/image_0032.jpg 6 test umbrella/image_0038.jpg 6 test umbrella/image_0039.jpg 6 test umbrella/image_0075.jpg 6 test umbrella/image_0028.jpg 6 test dolphin/image_0004.jpg 7 test dolphin/image_0015.jpg 7 test dolphin/image_0029.jpg 7 test dolphin/image_0023.jpg 7 test dolphin/image_0010.jpg 7 test dolphin/image_0056.jpg 7 test dolphin/image_0022.jpg 7 test dolphin/image_0065.jpg 7 test dolphin/image_0036.jpg 7 test dolphin/image_0032.jpg 7 test dolphin/image_0038.jpg 7 test dolphin/image_0039.jpg 7 test dolphin/image_0028.jpg 7 test gerenuk/image_0015.jpg 8 test gerenuk/image_0029.jpg 8 test gerenuk/image_0023.jpg 8 test gerenuk/image_0010.jpg 8 test gerenuk/image_0022.jpg 8 test gerenuk/image_0032.jpg 8 test gerenuk/image_0028.jpg 8 test dragonfly/image_0026.jpg 9 test dragonfly/image_0004.jpg 9 test dragonfly/image_0015.jpg 9 test dragonfly/image_0029.jpg 9 test dragonfly/image_0023.jpg 9 test dragonfly/image_0010.jpg 9 test dragonfly/image_0056.jpg 9 test dragonfly/image_0022.jpg 9 test dragonfly/image_0065.jpg 9 test dragonfly/image_0036.jpg 9 test dragonfly/image_0032.jpg 9 test dragonfly/image_0038.jpg 9 test dragonfly/image_0039.jpg 9 test dragonfly/image_0028.jpg 9 test yin_yang/image_0004.jpg 10 test yin_yang/image_0015.jpg 10 test yin_yang/image_0029.jpg 10 test yin_yang/image_0023.jpg 10 test yin_yang/image_0010.jpg 10 test yin_yang/image_0056.jpg 10 test yin_yang/image_0022.jpg 10 test yin_yang/image_0036.jpg 10 test yin_yang/image_0032.jpg 10 test yin_yang/image_0038.jpg 10 test yin_yang/image_0039.jpg 10 test yin_yang/image_0028.jpg 10 test starfish/image_0026.jpg 11 test starfish/image_0004.jpg 11 test starfish/image_0015.jpg 11 test starfish/image_0029.jpg 11 test starfish/image_0023.jpg 11 test starfish/image_0072.jpg 11 test starfish/image_0010.jpg 11 test starfish/image_0056.jpg 11 test starfish/image_0022.jpg 11 test starfish/image_0065.jpg 11 test starfish/image_0036.jpg 11 test starfish/image_0086.jpg 11 test starfish/image_0032.jpg 11 test starfish/image_0076.jpg 11 test starfish/image_0038.jpg 11 test starfish/image_0039.jpg 11 test starfish/image_0075.jpg 11 test starfish/image_0028.jpg 11 test ceiling_fan/image_0015.jpg 12 test ceiling_fan/image_0029.jpg 12 test ceiling_fan/image_0023.jpg 12 test ceiling_fan/image_0010.jpg 12 test ceiling_fan/image_0022.jpg 12 test ceiling_fan/image_0036.jpg 12 test ceiling_fan/image_0032.jpg 12 test ceiling_fan/image_0038.jpg 12 test ceiling_fan/image_0039.jpg 12 test ceiling_fan/image_0028.jpg 12 test soccer_ball/image_0026.jpg 13 test soccer_ball/image_0004.jpg 13 test soccer_ball/image_0015.jpg 13 test soccer_ball/image_0029.jpg 13 test soccer_ball/image_0023.jpg 13 test soccer_ball/image_0010.jpg 13 test soccer_ball/image_0056.jpg 13 test soccer_ball/image_0022.jpg 13 test soccer_ball/image_0036.jpg 13 test soccer_ball/image_0032.jpg 13 test soccer_ball/image_0038.jpg 13 test soccer_ball/image_0039.jpg 13 test soccer_ball/image_0028.jpg 13 test Leopards/image_0023.jpg 14 test Leopards/image_0135.jpg 14 test Leopards/image_0111.jpg 14 test Leopards/image_0124.jpg 14 test Leopards/image_0072.jpg 14 test Leopards/image_0145.jpg 14 test Leopards/image_0174.jpg 14 test Leopards/image_0133.jpg 14 test Leopards/image_0152.jpg 14 test Leopards/image_0010.jpg 14 test Leopards/image_0056.jpg 14 test Leopards/image_0022.jpg 14 test Leopards/image_0164.jpg 14 test Leopards/image_0065.jpg 14 test Leopards/image_0100.jpg 14 test Leopards/image_0137.jpg 14 test Leopards/image_0036.jpg 14 test Leopards/image_0086.jpg 14 test Leopards/image_0032.jpg 14 test Leopards/image_0076.jpg 14 test Leopards/image_0190.jpg 14 test Leopards/image_0106.jpg 14 test Leopards/image_0140.jpg 14 test Leopards/image_0117.jpg 14 test Leopards/image_0038.jpg 14 test Leopards/image_0107.jpg 14 test Leopards/image_0158.jpg 14 test Leopards/image_0179.jpg 14 test Leopards/image_0170.jpg 14 test Leopards/image_0039.jpg 14 test Leopards/image_0075.jpg 14 test Leopards/image_0194.jpg 14 test Leopards/image_0096.jpg 14 test Leopards/image_0116.jpg 14 test Leopards/image_0138.jpg 14 test Leopards/image_0197.jpg 14 test Leopards/image_0028.jpg 14 test Leopards/image_0151.jpg 14 test Leopards/image_0092.jpg 14 test Leopards/image_0182.jpg 14 test scorpion/image_0026.jpg 15 test scorpion/image_0004.jpg 15 test scorpion/image_0015.jpg 15 test scorpion/image_0029.jpg 15 test scorpion/image_0023.jpg 15 test scorpion/image_0072.jpg 15 test scorpion/image_0010.jpg 15 test scorpion/image_0056.jpg 15 test scorpion/image_0022.jpg 15 test scorpion/image_0065.jpg 15 test scorpion/image_0036.jpg 15 test scorpion/image_0032.jpg 15 test scorpion/image_0076.jpg 15 test scorpion/image_0038.jpg 15 test scorpion/image_0039.jpg 15 test scorpion/image_0075.jpg 15 test scorpion/image_0028.jpg 15 test llama/image_0004.jpg 16 test llama/image_0015.jpg 16 test llama/image_0029.jpg 16 test llama/image_0023.jpg 16 test llama/image_0072.jpg 16 test llama/image_0010.jpg 16 test llama/image_0056.jpg 16 test llama/image_0022.jpg 16 test llama/image_0065.jpg 16 test llama/image_0036.jpg 16 test llama/image_0032.jpg 16 test llama/image_0076.jpg 16 test llama/image_0038.jpg 16 test llama/image_0039.jpg 16 test llama/image_0075.jpg 16 test llama/image_0028.jpg 16 test wild_cat/image_0015.jpg 17 test wild_cat/image_0029.jpg 17 test wild_cat/image_0023.jpg 17 test wild_cat/image_0010.jpg 17 test wild_cat/image_0022.jpg 17 test wild_cat/image_0032.jpg 17 test wild_cat/image_0028.jpg 17 test lamp/image_0026.jpg 18 test lamp/image_0004.jpg 18 test lamp/image_0015.jpg 18 test lamp/image_0029.jpg 18 test lamp/image_0023.jpg 18 test lamp/image_0010.jpg 18 test lamp/image_0056.jpg 18 test lamp/image_0022.jpg 18 test lamp/image_0036.jpg 18 test lamp/image_0032.jpg 18 test lamp/image_0038.jpg 18 test lamp/image_0039.jpg 18 test lamp/image_0028.jpg 18 test lotus/image_0026.jpg 19 test lotus/image_0004.jpg 19 test lotus/image_0015.jpg 19 test lotus/image_0029.jpg 19 test lotus/image_0023.jpg 19 test lotus/image_0010.jpg 19 test lotus/image_0056.jpg 19 test lotus/image_0022.jpg 19 test lotus/image_0065.jpg 19 test lotus/image_0036.jpg 19 test lotus/image_0032.jpg 19 test lotus/image_0038.jpg 19 test lotus/image_0039.jpg 19 test lotus/image_0028.jpg 19 test crocodile_head/image_0004.jpg 20 test crocodile_head/image_0015.jpg 20 test crocodile_head/image_0029.jpg 20 test crocodile_head/image_0023.jpg 20 test crocodile_head/image_0010.jpg 20 test crocodile_head/image_0022.jpg 20 test crocodile_head/image_0036.jpg 20 test crocodile_head/image_0032.jpg 20 test crocodile_head/image_0038.jpg 20 test crocodile_head/image_0039.jpg 20 test crocodile_head/image_0028.jpg 20 test butterfly/image_0088.jpg 21 test butterfly/image_0026.jpg 21 test butterfly/image_0004.jpg 21 test butterfly/image_0015.jpg 21 test butterfly/image_0029.jpg 21 test butterfly/image_0023.jpg 21 test butterfly/image_0072.jpg 21 test butterfly/image_0010.jpg 21 test butterfly/image_0056.jpg 21 test butterfly/image_0022.jpg 21 test butterfly/image_0065.jpg 21 test butterfly/image_0036.jpg 21 test butterfly/image_0086.jpg 21 test butterfly/image_0032.jpg 21 test butterfly/image_0076.jpg 21 test butterfly/image_0038.jpg 21 test butterfly/image_0039.jpg 21 test butterfly/image_0075.jpg 21 test butterfly/image_0028.jpg 21 test windsor_chair/image_0004.jpg 22 test windsor_chair/image_0015.jpg 22 test windsor_chair/image_0029.jpg 22 test windsor_chair/image_0023.jpg 22 test windsor_chair/image_0010.jpg 22 test windsor_chair/image_0056.jpg 22 test windsor_chair/image_0022.jpg 22 test windsor_chair/image_0036.jpg 22 test windsor_chair/image_0032.jpg 22 test windsor_chair/image_0038.jpg 22 test windsor_chair/image_0039.jpg 22 test windsor_chair/image_0028.jpg 22 test metronome/image_0015.jpg 23 test metronome/image_0029.jpg 23 test metronome/image_0023.jpg 23 test metronome/image_0010.jpg 23 test metronome/image_0022.jpg 23 test metronome/image_0032.jpg 23 test metronome/image_0028.jpg 23 test revolver/image_0026.jpg 24 test revolver/image_0004.jpg 24 test revolver/image_0015.jpg 24 test revolver/image_0029.jpg 24 test revolver/image_0023.jpg 24 test revolver/image_0072.jpg 24 test revolver/image_0010.jpg 24 test revolver/image_0056.jpg 24 test revolver/image_0022.jpg 24 test revolver/image_0065.jpg 24 test revolver/image_0036.jpg 24 test revolver/image_0032.jpg 24 test revolver/image_0076.jpg 24 test revolver/image_0038.jpg 24 test revolver/image_0039.jpg 24 test revolver/image_0075.jpg 24 test revolver/image_0028.jpg 24 test schooner/image_0026.jpg 25 test schooner/image_0004.jpg 25 test schooner/image_0015.jpg 25 test schooner/image_0029.jpg 25 test schooner/image_0023.jpg 25 test schooner/image_0010.jpg 25 test schooner/image_0056.jpg 25 test schooner/image_0022.jpg 25 test schooner/image_0036.jpg 25 test schooner/image_0032.jpg 25 test schooner/image_0038.jpg 25 test schooner/image_0039.jpg 25 test schooner/image_0028.jpg 25 test minaret/image_0004.jpg 26 test minaret/image_0015.jpg 26 test minaret/image_0029.jpg 26 test minaret/image_0023.jpg 26 test minaret/image_0072.jpg 26 test minaret/image_0010.jpg 26 test minaret/image_0056.jpg 26 test minaret/image_0022.jpg 26 test minaret/image_0065.jpg 26 test minaret/image_0036.jpg 26 test minaret/image_0032.jpg 26 test minaret/image_0076.jpg 26 test minaret/image_0038.jpg 26 test minaret/image_0039.jpg 26 test minaret/image_0075.jpg 26 test minaret/image_0028.jpg 26 test stop_sign/image_0026.jpg 27 test stop_sign/image_0004.jpg 27 test stop_sign/image_0015.jpg 27 test stop_sign/image_0029.jpg 27 test stop_sign/image_0023.jpg 27 test stop_sign/image_0010.jpg 27 test stop_sign/image_0056.jpg 27 test stop_sign/image_0022.jpg 27 test stop_sign/image_0036.jpg 27 test stop_sign/image_0032.jpg 27 test stop_sign/image_0038.jpg 27 test stop_sign/image_0039.jpg 27 test stop_sign/image_0028.jpg 27 test beaver/image_0015.jpg 28 test beaver/image_0029.jpg 28 test beaver/image_0023.jpg 28 test beaver/image_0010.jpg 28 test beaver/image_0022.jpg 28 test beaver/image_0036.jpg 28 test beaver/image_0032.jpg 28 test beaver/image_0038.jpg 28 test beaver/image_0039.jpg 28 test beaver/image_0028.jpg 28 test laptop/image_0026.jpg 29 test laptop/image_0004.jpg 29 test laptop/image_0015.jpg 29 test laptop/image_0029.jpg 29 test laptop/image_0023.jpg 29 test laptop/image_0072.jpg 29 test laptop/image_0010.jpg 29 test laptop/image_0056.jpg 29 test laptop/image_0022.jpg 29 test laptop/image_0065.jpg 29 test laptop/image_0036.jpg 29 test laptop/image_0032.jpg 29 test laptop/image_0076.jpg 29 test laptop/image_0038.jpg 29 test laptop/image_0039.jpg 29 test laptop/image_0075.jpg 29 test laptop/image_0028.jpg 29 test ketch/image_0004.jpg 30 test ketch/image_0015.jpg 30 test ketch/image_0029.jpg 30 test ketch/image_0023.jpg 30 test ketch/image_0111.jpg 30 test ketch/image_0072.jpg 30 test ketch/image_0010.jpg 30 test ketch/image_0056.jpg 30 test ketch/image_0022.jpg 30 test ketch/image_0065.jpg 30 test ketch/image_0100.jpg 30 test ketch/image_0036.jpg 30 test ketch/image_0086.jpg 30 test ketch/image_0032.jpg 30 test ketch/image_0076.jpg 30 test ketch/image_0106.jpg 30 test ketch/image_0038.jpg 30 test ketch/image_0107.jpg 30 test ketch/image_0039.jpg 30 test ketch/image_0075.jpg 30 test ketch/image_0096.jpg 30 test ketch/image_0028.jpg 30 test ketch/image_0092.jpg 30 test gramophone/image_0004.jpg 31 test gramophone/image_0015.jpg 31 test gramophone/image_0029.jpg 31 test gramophone/image_0023.jpg 31 test gramophone/image_0010.jpg 31 test gramophone/image_0022.jpg 31 test gramophone/image_0036.jpg 31 test gramophone/image_0032.jpg 31 test gramophone/image_0038.jpg 31 test gramophone/image_0039.jpg 31 test gramophone/image_0028.jpg 31 test menorah/image_0026.jpg 32 test menorah/image_0004.jpg 32 test menorah/image_0015.jpg 32 test menorah/image_0029.jpg 32 test menorah/image_0023.jpg 32 test menorah/image_0072.jpg 32 test menorah/image_0010.jpg 32 test menorah/image_0056.jpg 32 test menorah/image_0022.jpg 32 test menorah/image_0065.jpg 32 test menorah/image_0036.jpg 32 test menorah/image_0086.jpg 32 test menorah/image_0032.jpg 32 test menorah/image_0076.jpg 32 test menorah/image_0038.jpg 32 test menorah/image_0039.jpg 32 test menorah/image_0075.jpg 32 test menorah/image_0028.jpg 32 test euphonium/image_0026.jpg 33 test euphonium/image_0004.jpg 33 test euphonium/image_0015.jpg 33 test euphonium/image_0029.jpg 33 test euphonium/image_0023.jpg 33 test euphonium/image_0010.jpg 33 test euphonium/image_0056.jpg 33 test euphonium/image_0022.jpg 33 test euphonium/image_0036.jpg 33 test euphonium/image_0032.jpg 33 test euphonium/image_0038.jpg 33 test euphonium/image_0039.jpg 33 test euphonium/image_0028.jpg 33 test rhino/image_0004.jpg 34 test rhino/image_0015.jpg 34 test rhino/image_0029.jpg 34 test rhino/image_0023.jpg 34 test rhino/image_0010.jpg 34 test rhino/image_0056.jpg 34 test rhino/image_0022.jpg 34 test rhino/image_0036.jpg 34 test rhino/image_0032.jpg 34 test rhino/image_0038.jpg 34 test rhino/image_0039.jpg 34 test rhino/image_0028.jpg 34 test bonsai/image_0015.jpg 35 test bonsai/image_0029.jpg 35 test bonsai/image_0120.jpg 35 test bonsai/image_0023.jpg 35 test bonsai/image_0111.jpg 35 test bonsai/image_0124.jpg 35 test bonsai/image_0072.jpg 35 test bonsai/image_0010.jpg 35 test bonsai/image_0056.jpg 35 test bonsai/image_0022.jpg 35 test bonsai/image_0065.jpg 35 test bonsai/image_0100.jpg 35 test bonsai/image_0036.jpg 35 test bonsai/image_0086.jpg 35 test bonsai/image_0032.jpg 35 test bonsai/image_0076.jpg 35 test bonsai/image_0106.jpg 35 test bonsai/image_0117.jpg 35 test bonsai/image_0038.jpg 35 test bonsai/image_0107.jpg 35 test bonsai/image_0039.jpg 35 test bonsai/image_0075.jpg 35 test bonsai/image_0096.jpg 35 test bonsai/image_0116.jpg 35 test bonsai/image_0028.jpg 35 test bonsai/image_0092.jpg 35 test chair/image_0026.jpg 36 test chair/image_0004.jpg 36 test chair/image_0015.jpg 36 test chair/image_0029.jpg 36 test chair/image_0023.jpg 36 test chair/image_0010.jpg 36 test chair/image_0056.jpg 36 test chair/image_0022.jpg 36 test chair/image_0036.jpg 36 test chair/image_0032.jpg 36 test chair/image_0038.jpg 36 test chair/image_0039.jpg 36 test chair/image_0028.jpg 36 test snoopy/image_0015.jpg 37 test snoopy/image_0029.jpg 37 test snoopy/image_0023.jpg 37 test snoopy/image_0010.jpg 37 test snoopy/image_0022.jpg 37 test snoopy/image_0032.jpg 37 test snoopy/image_0028.jpg 37 test buddha/image_0026.jpg 38 test buddha/image_0004.jpg 38 test buddha/image_0015.jpg 38 test buddha/image_0029.jpg 38 test buddha/image_0023.jpg 38 test buddha/image_0072.jpg 38 test buddha/image_0010.jpg 38 test buddha/image_0056.jpg 38 test buddha/image_0022.jpg 38 test buddha/image_0065.jpg 38 test buddha/image_0036.jpg 38 test buddha/image_0032.jpg 38 test buddha/image_0076.jpg 38 test buddha/image_0038.jpg 38 test buddha/image_0039.jpg 38 test buddha/image_0075.jpg 38 test buddha/image_0028.jpg 38 test scissors/image_0023.jpg 39 test scissors/image_0010.jpg 39 test scissors/image_0022.jpg 39 test scissors/image_0036.jpg 39 test scissors/image_0032.jpg 39 test scissors/image_0038.jpg 39 test scissors/image_0039.jpg 39 test scissors/image_0028.jpg 39 test cougar_body/image_0015.jpg 40 test cougar_body/image_0029.jpg 40 test cougar_body/image_0023.jpg 40 test cougar_body/image_0010.jpg 40 test cougar_body/image_0022.jpg 40 test cougar_body/image_0036.jpg 40 test cougar_body/image_0032.jpg 40 test cougar_body/image_0038.jpg 40 test cougar_body/image_0039.jpg 40 test cougar_body/image_0028.jpg 40 test binocular/image_0015.jpg 41 test binocular/image_0029.jpg 41 test binocular/image_0023.jpg 41 test binocular/image_0010.jpg 41 test binocular/image_0022.jpg 41 test binocular/image_0032.jpg 41 test binocular/image_0028.jpg 41 test grand_piano/image_0026.jpg 42 test grand_piano/image_0004.jpg 42 test grand_piano/image_0015.jpg 42 test grand_piano/image_0029.jpg 42 test grand_piano/image_0023.jpg 42 test grand_piano/image_0072.jpg 42 test grand_piano/image_0010.jpg 42 test grand_piano/image_0056.jpg 42 test grand_piano/image_0022.jpg 42 test grand_piano/image_0065.jpg 42 test grand_piano/image_0036.jpg 42 test grand_piano/image_0086.jpg 42 test grand_piano/image_0032.jpg 42 test grand_piano/image_0076.jpg 42 test grand_piano/image_0038.jpg 42 test grand_piano/image_0039.jpg 42 test grand_piano/image_0075.jpg 42 test grand_piano/image_0096.jpg 42 test grand_piano/image_0028.jpg 42 test grand_piano/image_0092.jpg 42 test ibis/image_0004.jpg 43 test ibis/image_0015.jpg 43 test ibis/image_0029.jpg 43 test ibis/image_0023.jpg 43 test ibis/image_0072.jpg 43 test ibis/image_0010.jpg 43 test ibis/image_0056.jpg 43 test ibis/image_0022.jpg 43 test ibis/image_0065.jpg 43 test ibis/image_0036.jpg 43 test ibis/image_0032.jpg 43 test ibis/image_0076.jpg 43 test ibis/image_0038.jpg 43 test ibis/image_0039.jpg 43 test ibis/image_0075.jpg 43 test ibis/image_0028.jpg 43 test brontosaurus/image_0029.jpg 44 test brontosaurus/image_0023.jpg 44 test brontosaurus/image_0010.jpg 44 test brontosaurus/image_0022.jpg 44 test brontosaurus/image_0036.jpg 44 test brontosaurus/image_0032.jpg 44 test brontosaurus/image_0038.jpg 44 test brontosaurus/image_0039.jpg 44 test brontosaurus/image_0028.jpg 44 test saxophone/image_0023.jpg 45 test saxophone/image_0010.jpg 45 test saxophone/image_0022.jpg 45 test saxophone/image_0036.jpg 45 test saxophone/image_0032.jpg 45 test saxophone/image_0038.jpg 45 test saxophone/image_0039.jpg 45 test saxophone/image_0028.jpg 45 test stapler/image_0029.jpg 46 test stapler/image_0023.jpg 46 test stapler/image_0010.jpg 46 test stapler/image_0022.jpg 46 test stapler/image_0036.jpg 46 test stapler/image_0032.jpg 46 test stapler/image_0038.jpg 46 test stapler/image_0039.jpg 46 test stapler/image_0028.jpg 46 test electric_guitar/image_0004.jpg 47 test electric_guitar/image_0015.jpg 47 test electric_guitar/image_0029.jpg 47 test electric_guitar/image_0023.jpg 47 test electric_guitar/image_0072.jpg 47 test electric_guitar/image_0010.jpg 47 test electric_guitar/image_0056.jpg 47 test electric_guitar/image_0022.jpg 47 test electric_guitar/image_0065.jpg 47 test electric_guitar/image_0036.jpg 47 test electric_guitar/image_0032.jpg 47 test electric_guitar/image_0038.jpg 47 test electric_guitar/image_0039.jpg 47 test electric_guitar/image_0075.jpg 47 test electric_guitar/image_0028.jpg 47 test octopus/image_0015.jpg 48 test octopus/image_0029.jpg 48 test octopus/image_0023.jpg 48 test octopus/image_0010.jpg 48 test octopus/image_0022.jpg 48 test octopus/image_0032.jpg 48 test octopus/image_0028.jpg 48 test kangaroo/image_0026.jpg 49 test kangaroo/image_0004.jpg 49 test kangaroo/image_0015.jpg 49 test kangaroo/image_0029.jpg 49 test kangaroo/image_0023.jpg 49 test kangaroo/image_0072.jpg 49 test kangaroo/image_0010.jpg 49 test kangaroo/image_0056.jpg 49 test kangaroo/image_0022.jpg 49 test kangaroo/image_0065.jpg 49 test kangaroo/image_0036.jpg 49 test kangaroo/image_0086.jpg 49 test kangaroo/image_0032.jpg 49 test kangaroo/image_0076.jpg 49 test kangaroo/image_0038.jpg 49 test kangaroo/image_0039.jpg 49 test kangaroo/image_0075.jpg 49 test kangaroo/image_0028.jpg 49 test okapi/image_0023.jpg 50 test okapi/image_0010.jpg 50 test okapi/image_0022.jpg 50 test okapi/image_0036.jpg 50 test okapi/image_0032.jpg 50 test okapi/image_0038.jpg 50 test okapi/image_0039.jpg 50 test okapi/image_0028.jpg 50 test sunflower/image_0026.jpg 51 test sunflower/image_0004.jpg 51 test sunflower/image_0015.jpg 51 test sunflower/image_0029.jpg 51 test sunflower/image_0023.jpg 51 test sunflower/image_0072.jpg 51 test sunflower/image_0010.jpg 51 test sunflower/image_0056.jpg 51 test sunflower/image_0022.jpg 51 test sunflower/image_0065.jpg 51 test sunflower/image_0036.jpg 51 test sunflower/image_0032.jpg 51 test sunflower/image_0076.jpg 51 test sunflower/image_0038.jpg 51 test sunflower/image_0039.jpg 51 test sunflower/image_0075.jpg 51 test sunflower/image_0028.jpg 51 test garfield/image_0015.jpg 52 test garfield/image_0029.jpg 52 test garfield/image_0023.jpg 52 test garfield/image_0010.jpg 52 test garfield/image_0022.jpg 52 test garfield/image_0032.jpg 52 test garfield/image_0028.jpg 52 test bass/image_0004.jpg 53 test bass/image_0015.jpg 53 test bass/image_0029.jpg 53 test bass/image_0023.jpg 53 test bass/image_0010.jpg 53 test bass/image_0022.jpg 53 test bass/image_0036.jpg 53 test bass/image_0032.jpg 53 test bass/image_0038.jpg 53 test bass/image_0039.jpg 53 test bass/image_0028.jpg 53 test cellphone/image_0004.jpg 54 test cellphone/image_0015.jpg 54 test cellphone/image_0029.jpg 54 test cellphone/image_0023.jpg 54 test cellphone/image_0010.jpg 54 test cellphone/image_0056.jpg 54 test cellphone/image_0022.jpg 54 test cellphone/image_0036.jpg 54 test cellphone/image_0032.jpg 54 test cellphone/image_0038.jpg 54 test cellphone/image_0039.jpg 54 test cellphone/image_0028.jpg 54 test brain/image_0026.jpg 55 test brain/image_0004.jpg 55 test brain/image_0015.jpg 55 test brain/image_0029.jpg 55 test brain/image_0023.jpg 55 test brain/image_0072.jpg 55 test brain/image_0010.jpg 55 test brain/image_0056.jpg 55 test brain/image_0022.jpg 55 test brain/image_0065.jpg 55 test brain/image_0036.jpg 55 test brain/image_0086.jpg 55 test brain/image_0032.jpg 55 test brain/image_0076.jpg 55 test brain/image_0038.jpg 55 test brain/image_0039.jpg 55 test brain/image_0075.jpg 55 test brain/image_0096.jpg 55 test brain/image_0028.jpg 55 test brain/image_0092.jpg 55 test lobster/image_0029.jpg 56 test lobster/image_0023.jpg 56 test lobster/image_0010.jpg 56 test lobster/image_0022.jpg 56 test lobster/image_0036.jpg 56 test lobster/image_0032.jpg 56 test lobster/image_0038.jpg 56 test lobster/image_0039.jpg 56 test lobster/image_0028.jpg 56 test headphone/image_0029.jpg 57 test headphone/image_0023.jpg 57 test headphone/image_0010.jpg 57 test headphone/image_0022.jpg 57 test headphone/image_0036.jpg 57 test headphone/image_0032.jpg 57 test headphone/image_0038.jpg 57 test headphone/image_0039.jpg 57 test headphone/image_0028.jpg 57 test barrel/image_0015.jpg 58 test barrel/image_0029.jpg 58 test barrel/image_0023.jpg 58 test barrel/image_0010.jpg 58 test barrel/image_0022.jpg 58 test barrel/image_0036.jpg 58 test barrel/image_0032.jpg 58 test barrel/image_0038.jpg 58 test barrel/image_0039.jpg 58 test barrel/image_0028.jpg 58 test pigeon/image_0029.jpg 59 test pigeon/image_0023.jpg 59 test pigeon/image_0010.jpg 59 test pigeon/image_0022.jpg 59 test pigeon/image_0036.jpg 59 test pigeon/image_0032.jpg 59 test pigeon/image_0038.jpg 59 test pigeon/image_0039.jpg 59 test pigeon/image_0028.jpg 59 test inline_skate/image_0004.jpg 60 test inline_skate/image_0015.jpg 60 test inline_skate/image_0029.jpg 60 test inline_skate/image_0023.jpg 60 test inline_skate/image_0010.jpg 60 test inline_skate/image_0022.jpg 60 test inline_skate/image_0028.jpg 60 test cannon/image_0029.jpg 61 test cannon/image_0023.jpg 61 test cannon/image_0010.jpg 61 test cannon/image_0022.jpg 61 test cannon/image_0036.jpg 61 test cannon/image_0032.jpg 61 test cannon/image_0038.jpg 61 test cannon/image_0039.jpg 61 test cannon/image_0028.jpg 61 test BACKGROUND_Google/image_0029.jpg 62 test BACKGROUND_Google/image_0327.jpg 62 test BACKGROUND_Google/image_0120.jpg 62 test BACKGROUND_Google/image_0281.jpg 62 test BACKGROUND_Google/image_0381.jpg 62 test BACKGROUND_Google/image_0220.jpg 62 test BACKGROUND_Google/image_0345.jpg 62 test BACKGROUND_Google/image_0023.jpg 62 test BACKGROUND_Google/image_0135.jpg 62 test BACKGROUND_Google/image_0452.jpg 62 test BACKGROUND_Google/image_0423.jpg 62 test BACKGROUND_Google/image_0454.jpg 62 test BACKGROUND_Google/image_0466.jpg 62 test BACKGROUND_Google/image_0111.jpg 62 test BACKGROUND_Google/image_0124.jpg 62 test BACKGROUND_Google/image_0279.jpg 62 test BACKGROUND_Google/image_0424.jpg 62 test BACKGROUND_Google/image_0418.jpg 62 test BACKGROUND_Google/image_0379.jpg 62 test BACKGROUND_Google/image_0273.jpg 62 test BACKGROUND_Google/image_0072.jpg 62 test BACKGROUND_Google/image_0145.jpg 62 test BACKGROUND_Google/image_0214.jpg 62 test BACKGROUND_Google/image_0174.jpg 62 test BACKGROUND_Google/image_0133.jpg 62 test BACKGROUND_Google/image_0152.jpg 62 test BACKGROUND_Google/image_0289.jpg 62 test BACKGROUND_Google/image_0342.jpg 62 test BACKGROUND_Google/image_0298.jpg 62 test BACKGROUND_Google/image_0010.jpg 62 test BACKGROUND_Google/image_0248.jpg 62 test BACKGROUND_Google/image_0245.jpg 62 test BACKGROUND_Google/image_0056.jpg 62 test BACKGROUND_Google/image_0022.jpg 62 test BACKGROUND_Google/image_0307.jpg 62 test BACKGROUND_Google/image_0164.jpg 62 test BACKGROUND_Google/image_0377.jpg 62 test BACKGROUND_Google/image_0065.jpg 62 test BACKGROUND_Google/image_0244.jpg 62 test BACKGROUND_Google/image_0100.jpg 62 test BACKGROUND_Google/image_0137.jpg 62 test BACKGROUND_Google/image_0274.jpg 62 test BACKGROUND_Google/image_0323.jpg 62 test BACKGROUND_Google/image_0036.jpg 62 test BACKGROUND_Google/image_0086.jpg 62 test BACKGROUND_Google/image_0032.jpg 62 test BACKGROUND_Google/image_0405.jpg 62 test BACKGROUND_Google/image_0394.jpg 62 test BACKGROUND_Google/image_0076.jpg 62 test BACKGROUND_Google/image_0439.jpg 62 test BACKGROUND_Google/image_0252.jpg 62 test BACKGROUND_Google/image_0235.jpg 62 test BACKGROUND_Google/image_0190.jpg 62 test BACKGROUND_Google/image_0290.jpg 62 test BACKGROUND_Google/image_0453.jpg 62 test BACKGROUND_Google/image_0106.jpg 62 test BACKGROUND_Google/image_0140.jpg 62 test BACKGROUND_Google/image_0415.jpg 62 test BACKGROUND_Google/image_0117.jpg 62 test BACKGROUND_Google/image_0038.jpg 62 test BACKGROUND_Google/image_0210.jpg 62 test BACKGROUND_Google/image_0456.jpg 62 test BACKGROUND_Google/image_0107.jpg 62 test BACKGROUND_Google/image_0247.jpg 62 test BACKGROUND_Google/image_0362.jpg 62 test BACKGROUND_Google/image_0158.jpg 62 test BACKGROUND_Google/image_0430.jpg 62 test BACKGROUND_Google/image_0268.jpg 62 test BACKGROUND_Google/image_0242.jpg 62 test BACKGROUND_Google/image_0401.jpg 62 test BACKGROUND_Google/image_0179.jpg 62 test BACKGROUND_Google/image_0170.jpg 62 test BACKGROUND_Google/image_0378.jpg 62 test BACKGROUND_Google/image_0039.jpg 62 test BACKGROUND_Google/image_0075.jpg 62 test BACKGROUND_Google/image_0194.jpg 62 test BACKGROUND_Google/image_0096.jpg 62 test BACKGROUND_Google/image_0238.jpg 62 test BACKGROUND_Google/image_0236.jpg 62 test BACKGROUND_Google/image_0311.jpg 62 test BACKGROUND_Google/image_0116.jpg 62 test BACKGROUND_Google/image_0419.jpg 62 test BACKGROUND_Google/image_0138.jpg 62 test BACKGROUND_Google/image_0449.jpg 62 test BACKGROUND_Google/image_0197.jpg 62 test BACKGROUND_Google/image_0241.jpg 62 test BACKGROUND_Google/image_0028.jpg 62 test BACKGROUND_Google/image_0151.jpg 62 test BACKGROUND_Google/image_0385.jpg 62 test BACKGROUND_Google/image_0092.jpg 62 test BACKGROUND_Google/image_0182.jpg 62 test BACKGROUND_Google/image_0413.jpg 62 test BACKGROUND_Google/image_0359.jpg 62 test BACKGROUND_Google/image_0374.jpg 62 test mandolin/image_0029.jpg 63 test mandolin/image_0023.jpg 63 test mandolin/image_0010.jpg 63 test mandolin/image_0022.jpg 63 test mandolin/image_0036.jpg 63 test mandolin/image_0032.jpg 63 test mandolin/image_0038.jpg 63 test mandolin/image_0039.jpg 63 test mandolin/image_0028.jpg 63 test Motorbikes/image_0355.jpg 64 test Motorbikes/image_0029.jpg 64 test Motorbikes/image_0662.jpg 64 test Motorbikes/image_0647.jpg 64 test Motorbikes/image_0798.jpg 64 test Motorbikes/image_0327.jpg 64 test Motorbikes/image_0120.jpg 64 test Motorbikes/image_0281.jpg 64 test Motorbikes/image_0603.jpg 64 test Motorbikes/image_0653.jpg 64 test Motorbikes/image_0381.jpg 64 test Motorbikes/image_0220.jpg 64 test Motorbikes/image_0345.jpg 64 test Motorbikes/image_0520.jpg 64 test Motorbikes/image_0685.jpg 64 test Motorbikes/image_0503.jpg 64 test Motorbikes/image_0023.jpg 64 test Motorbikes/image_0135.jpg 64 test Motorbikes/image_0452.jpg 64 test Motorbikes/image_0584.jpg 64 test Motorbikes/image_0423.jpg 64 test Motorbikes/image_0782.jpg 64 test Motorbikes/image_0454.jpg 64 test Motorbikes/image_0466.jpg 64 test Motorbikes/image_0111.jpg 64 test Motorbikes/image_0124.jpg 64 test Motorbikes/image_0279.jpg 64 test Motorbikes/image_0478.jpg 64 test Motorbikes/image_0642.jpg 64 test Motorbikes/image_0666.jpg 64 test Motorbikes/image_0708.jpg 64 test Motorbikes/image_0424.jpg 64 test Motorbikes/image_0418.jpg 64 test Motorbikes/image_0379.jpg 64 test Motorbikes/image_0626.jpg 64 test Motorbikes/image_0610.jpg 64 test Motorbikes/image_0273.jpg 64 test Motorbikes/image_0587.jpg 64 test Motorbikes/image_0072.jpg 64 test Motorbikes/image_0598.jpg 64 test Motorbikes/image_0508.jpg 64 test Motorbikes/image_0145.jpg 64 test Motorbikes/image_0750.jpg 64 test Motorbikes/image_0214.jpg 64 test Motorbikes/image_0174.jpg 64 test Motorbikes/image_0754.jpg 64 test Motorbikes/image_0726.jpg 64 test Motorbikes/image_0133.jpg 64 test Motorbikes/image_0152.jpg 64 test Motorbikes/image_0289.jpg 64 test Motorbikes/image_0674.jpg 64 test Motorbikes/image_0609.jpg 64 test Motorbikes/image_0342.jpg 64 test Motorbikes/image_0298.jpg 64 test Motorbikes/image_0710.jpg 64 test Motorbikes/image_0010.jpg 64 test Motorbikes/image_0248.jpg 64 test Motorbikes/image_0245.jpg 64 test Motorbikes/image_0056.jpg 64 test Motorbikes/image_0022.jpg 64 test Motorbikes/image_0687.jpg 64 test Motorbikes/image_0307.jpg 64 test Motorbikes/image_0755.jpg 64 test Motorbikes/image_0164.jpg 64 test Motorbikes/image_0377.jpg 64 test Motorbikes/image_0715.jpg 64 test Motorbikes/image_0065.jpg 64 test Motorbikes/image_0244.jpg 64 test Motorbikes/image_0777.jpg 64 test Motorbikes/image_0100.jpg 64 test Motorbikes/image_0137.jpg 64 test Motorbikes/image_0274.jpg 64 test Motorbikes/image_0323.jpg 64 test Motorbikes/image_0576.jpg 64 test Motorbikes/image_0526.jpg 64 test Motorbikes/image_0036.jpg 64 test Motorbikes/image_0785.jpg 64 test Motorbikes/image_0599.jpg 64 test Motorbikes/image_0086.jpg 64 test Motorbikes/image_0032.jpg 64 test Motorbikes/image_0405.jpg 64 test Motorbikes/image_0394.jpg 64 test Motorbikes/image_0076.jpg 64 test Motorbikes/image_0682.jpg 64 test Motorbikes/image_0439.jpg 64 test Motorbikes/image_0705.jpg 64 test Motorbikes/image_0252.jpg 64 test Motorbikes/image_0235.jpg 64 test Motorbikes/image_0510.jpg 64 test Motorbikes/image_0190.jpg 64 test Motorbikes/image_0290.jpg 64 test Motorbikes/image_0453.jpg 64 test Motorbikes/image_0106.jpg 64 test Motorbikes/image_0140.jpg 64 test Motorbikes/image_0415.jpg 64 test Motorbikes/image_0117.jpg 64 test Motorbikes/image_0038.jpg 64 test Motorbikes/image_0773.jpg 64 test Motorbikes/image_0663.jpg 64 test Motorbikes/image_0624.jpg 64 test Motorbikes/image_0210.jpg 64 test Motorbikes/image_0456.jpg 64 test Motorbikes/image_0107.jpg 64 test Motorbikes/image_0247.jpg 64 test Motorbikes/image_0362.jpg 64 test Motorbikes/image_0760.jpg 64 test Motorbikes/image_0535.jpg 64 test Motorbikes/image_0158.jpg 64 test Motorbikes/image_0498.jpg 64 test Motorbikes/image_0430.jpg 64 test Motorbikes/image_0268.jpg 64 test Motorbikes/image_0660.jpg 64 test Motorbikes/image_0630.jpg 64 test Motorbikes/image_0592.jpg 64 test Motorbikes/image_0242.jpg 64 test Motorbikes/image_0401.jpg 64 test Motorbikes/image_0179.jpg 64 test Motorbikes/image_0170.jpg 64 test Motorbikes/image_0524.jpg 64 test Motorbikes/image_0378.jpg 64 test Motorbikes/image_0753.jpg 64 test Motorbikes/image_0507.jpg 64 test Motorbikes/image_0749.jpg 64 test Motorbikes/image_0039.jpg 64 test Motorbikes/image_0075.jpg 64 test Motorbikes/image_0744.jpg 64 test Motorbikes/image_0194.jpg 64 test Motorbikes/image_0695.jpg 64 test Motorbikes/image_0693.jpg 64 test Motorbikes/image_0096.jpg 64 test Motorbikes/image_0481.jpg 64 test Motorbikes/image_0238.jpg 64 test Motorbikes/image_0701.jpg 64 test Motorbikes/image_0236.jpg 64 test Motorbikes/image_0311.jpg 64 test Motorbikes/image_0724.jpg 64 test Motorbikes/image_0116.jpg 64 test Motorbikes/image_0419.jpg 64 test Motorbikes/image_0702.jpg 64 test Motorbikes/image_0725.jpg 64 test Motorbikes/image_0138.jpg 64 test Motorbikes/image_0774.jpg 64 test Motorbikes/image_0793.jpg 64 test Motorbikes/image_0449.jpg 64 test Motorbikes/image_0778.jpg 64 test Motorbikes/image_0197.jpg 64 test Motorbikes/image_0578.jpg 64 test Motorbikes/image_0241.jpg 64 test Motorbikes/image_0028.jpg 64 test Motorbikes/image_0151.jpg 64 test Motorbikes/image_0385.jpg 64 test Motorbikes/image_0550.jpg 64 test Motorbikes/image_0636.jpg 64 test Motorbikes/image_0545.jpg 64 test Motorbikes/image_0092.jpg 64 test Motorbikes/image_0632.jpg 64 test Motorbikes/image_0182.jpg 64 test Motorbikes/image_0413.jpg 64 test Motorbikes/image_0359.jpg 64 test Motorbikes/image_0374.jpg 64 test dollar_bill/image_0004.jpg 65 test dollar_bill/image_0015.jpg 65 test dollar_bill/image_0029.jpg 65 test dollar_bill/image_0023.jpg 65 test dollar_bill/image_0010.jpg 65 test dollar_bill/image_0022.jpg 65 test dollar_bill/image_0036.jpg 65 test dollar_bill/image_0032.jpg 65 test dollar_bill/image_0038.jpg 65 test dollar_bill/image_0039.jpg 65 test dollar_bill/image_0028.jpg 65 test nautilus/image_0004.jpg 66 test nautilus/image_0015.jpg 66 test nautilus/image_0029.jpg 66 test nautilus/image_0023.jpg 66 test nautilus/image_0010.jpg 66 test nautilus/image_0022.jpg 66 test nautilus/image_0036.jpg 66 test nautilus/image_0032.jpg 66 test nautilus/image_0038.jpg 66 test nautilus/image_0039.jpg 66 test nautilus/image_0028.jpg 66 test crab/image_0026.jpg 67 test crab/image_0004.jpg 67 test crab/image_0015.jpg 67 test crab/image_0029.jpg 67 test crab/image_0023.jpg 67 test crab/image_0072.jpg 67 test crab/image_0010.jpg 67 test crab/image_0056.jpg 67 test crab/image_0022.jpg 67 test crab/image_0065.jpg 67 test crab/image_0036.jpg 67 test crab/image_0032.jpg 67 test crab/image_0038.jpg 67 test crab/image_0039.jpg 67 test crab/image_0028.jpg 67 test accordion/image_0004.jpg 68 test accordion/image_0015.jpg 68 test accordion/image_0029.jpg 68 test accordion/image_0023.jpg 68 test accordion/image_0010.jpg 68 test accordion/image_0022.jpg 68 test accordion/image_0036.jpg 68 test accordion/image_0032.jpg 68 test accordion/image_0038.jpg 68 test accordion/image_0039.jpg 68 test accordion/image_0028.jpg 68 test crayfish/image_0026.jpg 69 test crayfish/image_0004.jpg 69 test crayfish/image_0015.jpg 69 test crayfish/image_0029.jpg 69 test crayfish/image_0023.jpg 69 test crayfish/image_0010.jpg 69 test crayfish/image_0056.jpg 69 test crayfish/image_0022.jpg 69 test crayfish/image_0065.jpg 69 test crayfish/image_0036.jpg 69 test crayfish/image_0032.jpg 69 test crayfish/image_0038.jpg 69 test crayfish/image_0039.jpg 69 test crayfish/image_0028.jpg 69 test flamingo_head/image_0029.jpg 70 test flamingo_head/image_0023.jpg 70 test flamingo_head/image_0010.jpg 70 test flamingo_head/image_0022.jpg 70 test flamingo_head/image_0036.jpg 70 test flamingo_head/image_0032.jpg 70 test flamingo_head/image_0038.jpg 70 test flamingo_head/image_0039.jpg 70 test flamingo_head/image_0028.jpg 70 test emu/image_0004.jpg 71 test emu/image_0015.jpg 71 test emu/image_0029.jpg 71 test emu/image_0023.jpg 71 test emu/image_0010.jpg 71 test emu/image_0022.jpg 71 test emu/image_0036.jpg 71 test emu/image_0032.jpg 71 test emu/image_0038.jpg 71 test emu/image_0039.jpg 71 test emu/image_0028.jpg 71 test trilobite/image_0026.jpg 72 test trilobite/image_0004.jpg 72 test trilobite/image_0015.jpg 72 test trilobite/image_0029.jpg 72 test trilobite/image_0023.jpg 72 test trilobite/image_0072.jpg 72 test trilobite/image_0010.jpg 72 test trilobite/image_0056.jpg 72 test trilobite/image_0022.jpg 72 test trilobite/image_0065.jpg 72 test trilobite/image_0036.jpg 72 test trilobite/image_0086.jpg 72 test trilobite/image_0032.jpg 72 test trilobite/image_0076.jpg 72 test trilobite/image_0038.jpg 72 test trilobite/image_0039.jpg 72 test trilobite/image_0075.jpg 72 test trilobite/image_0028.jpg 72 test camera/image_0015.jpg 73 test camera/image_0029.jpg 73 test camera/image_0023.jpg 73 test camera/image_0010.jpg 73 test camera/image_0022.jpg 73 test camera/image_0036.jpg 73 test camera/image_0032.jpg 73 test camera/image_0038.jpg 73 test camera/image_0039.jpg 73 test camera/image_0028.jpg 73 test platypus/image_0015.jpg 74 test platypus/image_0029.jpg 74 test platypus/image_0023.jpg 74 test platypus/image_0010.jpg 74 test platypus/image_0022.jpg 74 test platypus/image_0032.jpg 74 test platypus/image_0028.jpg 74 test chandelier/image_0004.jpg 75 test chandelier/image_0015.jpg 75 test chandelier/image_0029.jpg 75 test chandelier/image_0023.jpg 75 test chandelier/image_0072.jpg 75 test chandelier/image_0010.jpg 75 test chandelier/image_0056.jpg 75 test chandelier/image_0022.jpg 75 test chandelier/image_0065.jpg 75 test chandelier/image_0100.jpg 75 test chandelier/image_0036.jpg 75 test chandelier/image_0086.jpg 75 test chandelier/image_0032.jpg 75 test chandelier/image_0076.jpg 75 test chandelier/image_0106.jpg 75 test chandelier/image_0038.jpg 75 test chandelier/image_0107.jpg 75 test chandelier/image_0039.jpg 75 test chandelier/image_0075.jpg 75 test chandelier/image_0096.jpg 75 test chandelier/image_0028.jpg 75 test chandelier/image_0092.jpg 75 test crocodile/image_0015.jpg 76 test crocodile/image_0029.jpg 76 test crocodile/image_0023.jpg 76 test crocodile/image_0010.jpg 76 test crocodile/image_0022.jpg 76 test crocodile/image_0036.jpg 76 test crocodile/image_0032.jpg 76 test crocodile/image_0038.jpg 76 test crocodile/image_0039.jpg 76 test crocodile/image_0028.jpg 76 test car_side/image_0015.jpg 77 test car_side/image_0029.jpg 77 test car_side/image_0120.jpg 77 test car_side/image_0023.jpg 77 test car_side/image_0111.jpg 77 test car_side/image_0072.jpg 77 test car_side/image_0010.jpg 77 test car_side/image_0056.jpg 77 test car_side/image_0022.jpg 77 test car_side/image_0065.jpg 77 test car_side/image_0100.jpg 77 test car_side/image_0036.jpg 77 test car_side/image_0086.jpg 77 test car_side/image_0032.jpg 77 test car_side/image_0076.jpg 77 test car_side/image_0106.jpg 77 test car_side/image_0117.jpg 77 test car_side/image_0038.jpg 77 test car_side/image_0107.jpg 77 test car_side/image_0039.jpg 77 test car_side/image_0075.jpg 77 test car_side/image_0096.jpg 77 test car_side/image_0116.jpg 77 test car_side/image_0028.jpg 77 test car_side/image_0092.jpg 77 test joshua_tree/image_0026.jpg 78 test joshua_tree/image_0004.jpg 78 test joshua_tree/image_0015.jpg 78 test joshua_tree/image_0029.jpg 78 test joshua_tree/image_0023.jpg 78 test joshua_tree/image_0010.jpg 78 test joshua_tree/image_0056.jpg 78 test joshua_tree/image_0022.jpg 78 test joshua_tree/image_0036.jpg 78 test joshua_tree/image_0032.jpg 78 test joshua_tree/image_0038.jpg 78 test joshua_tree/image_0039.jpg 78 test joshua_tree/image_0028.jpg 78 test tick/image_0015.jpg 79 test tick/image_0029.jpg 79 test tick/image_0023.jpg 79 test tick/image_0010.jpg 79 test tick/image_0022.jpg 79 test tick/image_0036.jpg 79 test tick/image_0032.jpg 79 test tick/image_0038.jpg 79 test tick/image_0039.jpg 79 test tick/image_0028.jpg 79 test hawksbill/image_0004.jpg 80 test hawksbill/image_0015.jpg 80 test hawksbill/image_0029.jpg 80 test hawksbill/image_0023.jpg 80 test hawksbill/image_0072.jpg 80 test hawksbill/image_0010.jpg 80 test hawksbill/image_0056.jpg 80 test hawksbill/image_0022.jpg 80 test hawksbill/image_0065.jpg 80 test hawksbill/image_0100.jpg 80 test hawksbill/image_0036.jpg 80 test hawksbill/image_0086.jpg 80 test hawksbill/image_0032.jpg 80 test hawksbill/image_0076.jpg 80 test hawksbill/image_0038.jpg 80 test hawksbill/image_0039.jpg 80 test hawksbill/image_0075.jpg 80 test hawksbill/image_0096.jpg 80 test hawksbill/image_0028.jpg 80 test hawksbill/image_0092.jpg 80 test helicopter/image_0026.jpg 81 test helicopter/image_0004.jpg 81 test helicopter/image_0015.jpg 81 test helicopter/image_0029.jpg 81 test helicopter/image_0023.jpg 81 test helicopter/image_0072.jpg 81 test helicopter/image_0010.jpg 81 test helicopter/image_0056.jpg 81 test helicopter/image_0022.jpg 81 test helicopter/image_0065.jpg 81 test helicopter/image_0036.jpg 81 test helicopter/image_0086.jpg 81 test helicopter/image_0032.jpg 81 test helicopter/image_0076.jpg 81 test helicopter/image_0038.jpg 81 test helicopter/image_0039.jpg 81 test helicopter/image_0075.jpg 81 test helicopter/image_0028.jpg 81 test pagoda/image_0015.jpg 82 test pagoda/image_0029.jpg 82 test pagoda/image_0023.jpg 82 test pagoda/image_0010.jpg 82 test pagoda/image_0022.jpg 82 test pagoda/image_0036.jpg 82 test pagoda/image_0032.jpg 82 test pagoda/image_0038.jpg 82 test pagoda/image_0039.jpg 82 test pagoda/image_0028.jpg 82 test ewer/image_0026.jpg 83 test ewer/image_0004.jpg 83 test ewer/image_0015.jpg 83 test ewer/image_0029.jpg 83 test ewer/image_0023.jpg 83 test ewer/image_0072.jpg 83 test ewer/image_0010.jpg 83 test ewer/image_0056.jpg 83 test ewer/image_0022.jpg 83 test ewer/image_0065.jpg 83 test ewer/image_0036.jpg 83 test ewer/image_0032.jpg 83 test ewer/image_0076.jpg 83 test ewer/image_0038.jpg 83 test ewer/image_0039.jpg 83 test ewer/image_0075.jpg 83 test ewer/image_0028.jpg 83 test panda/image_0029.jpg 84 test panda/image_0023.jpg 84 test panda/image_0010.jpg 84 test panda/image_0022.jpg 84 test panda/image_0036.jpg 84 test panda/image_0032.jpg 84 test panda/image_0038.jpg 84 test panda/image_0028.jpg 84 test pizza/image_0004.jpg 85 test pizza/image_0015.jpg 85 test pizza/image_0029.jpg 85 test pizza/image_0023.jpg 85 test pizza/image_0010.jpg 85 test pizza/image_0022.jpg 85 test pizza/image_0036.jpg 85 test pizza/image_0032.jpg 85 test pizza/image_0038.jpg 85 test pizza/image_0039.jpg 85 test pizza/image_0028.jpg 85 test cup/image_0004.jpg 86 test cup/image_0015.jpg 86 test cup/image_0029.jpg 86 test cup/image_0023.jpg 86 test cup/image_0010.jpg 86 test cup/image_0056.jpg 86 test cup/image_0022.jpg 86 test cup/image_0036.jpg 86 test cup/image_0032.jpg 86 test cup/image_0038.jpg 86 test cup/image_0039.jpg 86 test cup/image_0028.jpg 86 test anchor/image_0029.jpg 87 test anchor/image_0023.jpg 87 test anchor/image_0010.jpg 87 test anchor/image_0022.jpg 87 test anchor/image_0036.jpg 87 test anchor/image_0032.jpg 87 test anchor/image_0038.jpg 87 test anchor/image_0039.jpg 87 test anchor/image_0028.jpg 87 test hedgehog/image_0004.jpg 88 test hedgehog/image_0015.jpg 88 test hedgehog/image_0029.jpg 88 test hedgehog/image_0023.jpg 88 test hedgehog/image_0010.jpg 88 test hedgehog/image_0022.jpg 88 test hedgehog/image_0036.jpg 88 test hedgehog/image_0032.jpg 88 test hedgehog/image_0038.jpg 88 test hedgehog/image_0039.jpg 88 test hedgehog/image_0028.jpg 88 test flamingo/image_0026.jpg 89 test flamingo/image_0004.jpg 89 test flamingo/image_0015.jpg 89 test flamingo/image_0029.jpg 89 test flamingo/image_0023.jpg 89 test flamingo/image_0010.jpg 89 test flamingo/image_0056.jpg 89 test flamingo/image_0022.jpg 89 test flamingo/image_0065.jpg 89 test flamingo/image_0036.jpg 89 test flamingo/image_0032.jpg 89 test flamingo/image_0038.jpg 89 test flamingo/image_0039.jpg 89 test flamingo/image_0028.jpg 89 test stegosaurus/image_0004.jpg 90 test stegosaurus/image_0015.jpg 90 test stegosaurus/image_0029.jpg 90 test stegosaurus/image_0023.jpg 90 test stegosaurus/image_0010.jpg 90 test stegosaurus/image_0056.jpg 90 test stegosaurus/image_0022.jpg 90 test stegosaurus/image_0036.jpg 90 test stegosaurus/image_0032.jpg 90 test stegosaurus/image_0038.jpg 90 test stegosaurus/image_0039.jpg 90 test stegosaurus/image_0028.jpg 90 test ferry/image_0026.jpg 91 test ferry/image_0004.jpg 91 test ferry/image_0015.jpg 91 test ferry/image_0029.jpg 91 test ferry/image_0023.jpg 91 test ferry/image_0010.jpg 91 test ferry/image_0056.jpg 91 test ferry/image_0022.jpg 91 test ferry/image_0065.jpg 91 test ferry/image_0036.jpg 91 test ferry/image_0032.jpg 91 test ferry/image_0038.jpg 91 test ferry/image_0039.jpg 91 test ferry/image_0028.jpg 91 test dalmatian/image_0026.jpg 92 test dalmatian/image_0004.jpg 92 test dalmatian/image_0015.jpg 92 test dalmatian/image_0029.jpg 92 test dalmatian/image_0023.jpg 92 test dalmatian/image_0010.jpg 92 test dalmatian/image_0056.jpg 92 test dalmatian/image_0022.jpg 92 test dalmatian/image_0065.jpg 92 test dalmatian/image_0036.jpg 92 test dalmatian/image_0032.jpg 92 test dalmatian/image_0038.jpg 92 test dalmatian/image_0039.jpg 92 test dalmatian/image_0028.jpg 92 test wheelchair/image_0004.jpg 93 test wheelchair/image_0015.jpg 93 test wheelchair/image_0029.jpg 93 test wheelchair/image_0023.jpg 93 test wheelchair/image_0010.jpg 93 test wheelchair/image_0056.jpg 93 test wheelchair/image_0022.jpg 93 test wheelchair/image_0036.jpg 93 test wheelchair/image_0032.jpg 93 test wheelchair/image_0038.jpg 93 test wheelchair/image_0039.jpg 93 test wheelchair/image_0028.jpg 93 test watch/image_0029.jpg 94 test watch/image_0120.jpg 94 test watch/image_0220.jpg 94 test watch/image_0023.jpg 94 test watch/image_0135.jpg 94 test watch/image_0111.jpg 94 test watch/image_0124.jpg 94 test watch/image_0072.jpg 94 test watch/image_0145.jpg 94 test watch/image_0214.jpg 94 test watch/image_0174.jpg 94 test watch/image_0133.jpg 94 test watch/image_0152.jpg 94 test watch/image_0010.jpg 94 test watch/image_0056.jpg 94 test watch/image_0022.jpg 94 test watch/image_0164.jpg 94 test watch/image_0065.jpg 94 test watch/image_0100.jpg 94 test watch/image_0137.jpg 94 test watch/image_0036.jpg 94 test watch/image_0086.jpg 94 test watch/image_0032.jpg 94 test watch/image_0076.jpg 94 test watch/image_0235.jpg 94 test watch/image_0190.jpg 94 test watch/image_0106.jpg 94 test watch/image_0140.jpg 94 test watch/image_0117.jpg 94 test watch/image_0038.jpg 94 test watch/image_0210.jpg 94 test watch/image_0107.jpg 94 test watch/image_0158.jpg 94 test watch/image_0179.jpg 94 test watch/image_0170.jpg 94 test watch/image_0039.jpg 94 test watch/image_0075.jpg 94 test watch/image_0194.jpg 94 test watch/image_0096.jpg 94 test watch/image_0238.jpg 94 test watch/image_0236.jpg 94 test watch/image_0116.jpg 94 test watch/image_0138.jpg 94 test watch/image_0197.jpg 94 test watch/image_0028.jpg 94 test watch/image_0151.jpg 94 test watch/image_0092.jpg 94 test watch/image_0182.jpg 94 test sea_horse/image_0004.jpg 95 test sea_horse/image_0015.jpg 95 test sea_horse/image_0029.jpg 95 test sea_horse/image_0023.jpg 95 test sea_horse/image_0010.jpg 95 test sea_horse/image_0056.jpg 95 test sea_horse/image_0022.jpg 95 test sea_horse/image_0036.jpg 95 test sea_horse/image_0032.jpg 95 test sea_horse/image_0038.jpg 95 test sea_horse/image_0039.jpg 95 test sea_horse/image_0028.jpg 95 test pyramid/image_0004.jpg 96 test pyramid/image_0015.jpg 96 test pyramid/image_0029.jpg 96 test pyramid/image_0023.jpg 96 test pyramid/image_0010.jpg 96 test pyramid/image_0056.jpg 96 test pyramid/image_0022.jpg 96 test pyramid/image_0036.jpg 96 test pyramid/image_0032.jpg 96 test pyramid/image_0038.jpg 96 test pyramid/image_0039.jpg 96 test pyramid/image_0028.jpg 96 test strawberry/image_0015.jpg 97 test strawberry/image_0029.jpg 97 test strawberry/image_0023.jpg 97 test strawberry/image_0010.jpg 97 test strawberry/image_0022.jpg 97 test strawberry/image_0032.jpg 97 test strawberry/image_0028.jpg 97 test Faces_easy/image_0029.jpg 98 test Faces_easy/image_0327.jpg 98 test Faces_easy/image_0120.jpg 98 test Faces_easy/image_0281.jpg 98 test Faces_easy/image_0381.jpg 98 test Faces_easy/image_0220.jpg 98 test Faces_easy/image_0345.jpg 98 test Faces_easy/image_0023.jpg 98 test Faces_easy/image_0135.jpg 98 test Faces_easy/image_0423.jpg 98 test Faces_easy/image_0111.jpg 98 test Faces_easy/image_0124.jpg 98 test Faces_easy/image_0279.jpg 98 test Faces_easy/image_0424.jpg 98 test Faces_easy/image_0418.jpg 98 test Faces_easy/image_0379.jpg 98 test Faces_easy/image_0273.jpg 98 test Faces_easy/image_0072.jpg 98 test Faces_easy/image_0145.jpg 98 test Faces_easy/image_0214.jpg 98 test Faces_easy/image_0174.jpg 98 test Faces_easy/image_0133.jpg 98 test Faces_easy/image_0152.jpg 98 test Faces_easy/image_0289.jpg 98 test Faces_easy/image_0342.jpg 98 test Faces_easy/image_0298.jpg 98 test Faces_easy/image_0010.jpg 98 test Faces_easy/image_0248.jpg 98 test Faces_easy/image_0245.jpg 98 test Faces_easy/image_0056.jpg 98 test Faces_easy/image_0022.jpg 98 test Faces_easy/image_0307.jpg 98 test Faces_easy/image_0164.jpg 98 test Faces_easy/image_0377.jpg 98 test Faces_easy/image_0065.jpg 98 test Faces_easy/image_0244.jpg 98 test Faces_easy/image_0100.jpg 98 test Faces_easy/image_0137.jpg 98 test Faces_easy/image_0274.jpg 98 test Faces_easy/image_0323.jpg 98 test Faces_easy/image_0036.jpg 98 test Faces_easy/image_0086.jpg 98 test Faces_easy/image_0032.jpg 98 test Faces_easy/image_0405.jpg 98 test Faces_easy/image_0394.jpg 98 test Faces_easy/image_0076.jpg 98 test Faces_easy/image_0252.jpg 98 test Faces_easy/image_0235.jpg 98 test Faces_easy/image_0190.jpg 98 test Faces_easy/image_0290.jpg 98 test Faces_easy/image_0106.jpg 98 test Faces_easy/image_0140.jpg 98 test Faces_easy/image_0415.jpg 98 test Faces_easy/image_0117.jpg 98 test Faces_easy/image_0038.jpg 98 test Faces_easy/image_0210.jpg 98 test Faces_easy/image_0107.jpg 98 test Faces_easy/image_0247.jpg 98 test Faces_easy/image_0362.jpg 98 test Faces_easy/image_0158.jpg 98 test Faces_easy/image_0430.jpg 98 test Faces_easy/image_0268.jpg 98 test Faces_easy/image_0242.jpg 98 test Faces_easy/image_0401.jpg 98 test Faces_easy/image_0179.jpg 98 test Faces_easy/image_0170.jpg 98 test Faces_easy/image_0378.jpg 98 test Faces_easy/image_0039.jpg 98 test Faces_easy/image_0075.jpg 98 test Faces_easy/image_0194.jpg 98 test Faces_easy/image_0096.jpg 98 test Faces_easy/image_0238.jpg 98 test Faces_easy/image_0236.jpg 98 test Faces_easy/image_0311.jpg 98 test Faces_easy/image_0116.jpg 98 test Faces_easy/image_0419.jpg 98 test Faces_easy/image_0138.jpg 98 test Faces_easy/image_0197.jpg 98 test Faces_easy/image_0241.jpg 98 test Faces_easy/image_0028.jpg 98 test Faces_easy/image_0151.jpg 98 test Faces_easy/image_0385.jpg 98 test Faces_easy/image_0092.jpg 98 test Faces_easy/image_0182.jpg 98 test Faces_easy/image_0413.jpg 98 test Faces_easy/image_0359.jpg 98 test Faces_easy/image_0374.jpg 98 test cougar_face/image_0026.jpg 99 test cougar_face/image_0004.jpg 99 test cougar_face/image_0015.jpg 99 test cougar_face/image_0029.jpg 99 test cougar_face/image_0023.jpg 99 test cougar_face/image_0010.jpg 99 test cougar_face/image_0056.jpg 99 test cougar_face/image_0022.jpg 99 test cougar_face/image_0065.jpg 99 test cougar_face/image_0036.jpg 99 test cougar_face/image_0032.jpg 99 test cougar_face/image_0038.jpg 99 test cougar_face/image_0039.jpg 99 test cougar_face/image_0028.jpg 99 test mayfly/image_0023.jpg 100 test mayfly/image_0010.jpg 100 test mayfly/image_0022.jpg 100 test mayfly/image_0036.jpg 100 test mayfly/image_0032.jpg 100 test mayfly/image_0038.jpg 100 test mayfly/image_0039.jpg 100 test mayfly/image_0028.jpg 100 test wrench/image_0023.jpg 101 test wrench/image_0010.jpg 101 test wrench/image_0022.jpg 101 test wrench/image_0036.jpg 101 test wrench/image_0032.jpg 101 test wrench/image_0038.jpg 101 test wrench/image_0039.jpg 101 test wrench/image_0028.jpg 101 test ================================================ FILE: cpp/dcgan/CMakeLists.txt ================================================ cmake_minimum_required(VERSION 3.5) project(dcgan) find_package(Torch REQUIRED) option(DOWNLOAD_MNIST "Download the MNIST dataset from the internet" ON) if (DOWNLOAD_MNIST) message(STATUS "Downloading MNIST dataset") execute_process( COMMAND python ${CMAKE_CURRENT_LIST_DIR}/../tools/download_mnist.py -d ${CMAKE_BINARY_DIR}/data ERROR_VARIABLE DOWNLOAD_ERROR) if (DOWNLOAD_ERROR) message(FATAL_ERROR "Error downloading MNIST dataset: ${DOWNLOAD_ERROR}") endif() endif() add_executable(dcgan dcgan.cpp) target_link_libraries(dcgan "${TORCH_LIBRARIES}") set_property(TARGET dcgan PROPERTY CXX_STANDARD 17) if (MSVC) file(GLOB TORCH_DLLS "${TORCH_INSTALL_PREFIX}/lib/*.dll") add_custom_command(TARGET dcgan POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different ${TORCH_DLLS} $) endif (MSVC) ================================================ FILE: cpp/dcgan/README.md ================================================ # DCGAN Example with the PyTorch C++ Frontend This folder contains an example of training a DCGAN to generate MNIST digits with the PyTorch C++ frontend. The entire training code is contained in `dcgan.cpp`. You can find the commands to install argparse [here](https://github.com/pytorch/examples/blob/main/.github/workflows/main_cpp.yml#L34). To build the code, run the following commands from your terminal: ```shell $ cd dcgan $ mkdir build $ cd build $ cmake -DCMAKE_PREFIX_PATH=/path/to/libtorch .. $ make ``` where `/path/to/libtorch` should be the path to the unzipped _LibTorch_ distribution, which you can get from the [PyTorch homepage](https://pytorch.org/get-started/locally/). Execute the compiled binary to train the model: ```shell $ ./dcgan [ 1/30][200/938] D_loss: 0.4953 | G_loss: 4.0195 -> checkpoint 1 [ 1/30][400/938] D_loss: 0.3610 | G_loss: 4.8148 -> checkpoint 2 [ 1/30][600/938] D_loss: 0.4072 | G_loss: 4.36760 -> checkpoint 3 [ 1/30][800/938] D_loss: 0.4444 | G_loss: 4.0250 -> checkpoint 4 [ 2/30][200/938] D_loss: 0.3761 | G_loss: 3.8790 -> checkpoint 5 [ 2/30][400/938] D_loss: 0.3977 | G_loss: 3.3315 -> checkpoint 6 [ 2/30][600/938] D_loss: 0.3815 | G_loss: 3.5696 -> checkpoint 7 [ 2/30][800/938] D_loss: 0.4039 | G_loss: 3.2759 -> checkpoint 8 [ 3/30][200/938] D_loss: 0.4236 | G_loss: 4.5132 -> checkpoint 9 [ 3/30][400/938] D_loss: 0.3645 | G_loss: 3.9759 -> checkpoint 10 ... ``` We can also specify the `--epochs` to change the number of epochs to train as follows: ```shell $ ./dcgan --epochs 10 ``` Without specifying the `--epochs` flag, the default number of epochs to train is 30. The training script periodically generates image samples. Use the `display_samples.py` script situated in this folder to generate a plot image. For example: ```shell $ python display_samples.py -i dcgan-sample-10.pt Saved out.png ``` ================================================ FILE: cpp/dcgan/dcgan.cpp ================================================ #include #include #include #include #include // The size of the noise vector fed to the generator. const int64_t kNoiseSize = 100; // The batch size for training. const int64_t kBatchSize = 64; // Where to find the MNIST dataset. const char* kDataFolder = "./data"; // After how many batches to create a new checkpoint periodically. const int64_t kCheckpointEvery = 200; // How many images to sample at every checkpoint. const int64_t kNumberOfSamplesPerCheckpoint = 10; // Set to `true` to restore models and optimizers from previously saved // checkpoints. const bool kRestoreFromCheckpoint = false; // After how many batches to log a new update with the loss value. const int64_t kLogInterval = 10; using namespace torch; struct DCGANGeneratorImpl : nn::Module { DCGANGeneratorImpl(int kNoiseSize) : conv1(nn::ConvTranspose2dOptions(kNoiseSize, 256, 4) .bias(false)), batch_norm1(256), conv2(nn::ConvTranspose2dOptions(256, 128, 3) .stride(2) .padding(1) .bias(false)), batch_norm2(128), conv3(nn::ConvTranspose2dOptions(128, 64, 4) .stride(2) .padding(1) .bias(false)), batch_norm3(64), conv4(nn::ConvTranspose2dOptions(64, 1, 4) .stride(2) .padding(1) .bias(false)) { // register_module() is needed if we want to use the parameters() method later on register_module("conv1", conv1); register_module("conv2", conv2); register_module("conv3", conv3); register_module("conv4", conv4); register_module("batch_norm1", batch_norm1); register_module("batch_norm2", batch_norm2); register_module("batch_norm3", batch_norm3); } torch::Tensor forward(torch::Tensor x) { x = torch::relu(batch_norm1(conv1(x))); x = torch::relu(batch_norm2(conv2(x))); x = torch::relu(batch_norm3(conv3(x))); x = torch::tanh(conv4(x)); return x; } nn::ConvTranspose2d conv1, conv2, conv3, conv4; nn::BatchNorm2d batch_norm1, batch_norm2, batch_norm3; }; TORCH_MODULE(DCGANGenerator); nn::Sequential create_discriminator() { return nn::Sequential( // Layer 1 nn::Conv2d(nn::Conv2dOptions(1, 64, 4).stride(2).padding(1).bias(false)), nn::LeakyReLU(nn::LeakyReLUOptions().negative_slope(0.2)), // Layer 2 nn::Conv2d(nn::Conv2dOptions(64, 128, 4).stride(2).padding(1).bias(false)), nn::BatchNorm2d(128), nn::LeakyReLU(nn::LeakyReLUOptions().negative_slope(0.2)), // Layer 3 nn::Conv2d( nn::Conv2dOptions(128, 256, 4).stride(2).padding(1).bias(false)), nn::BatchNorm2d(256), nn::LeakyReLU(nn::LeakyReLUOptions().negative_slope(0.2)), // Layer 4 nn::Conv2d(nn::Conv2dOptions(256, 1, 3).stride(1).padding(0).bias(false)), nn::Sigmoid()); } int main(int argc, const char* argv[]) { argparse::ArgumentParser parser("cpp/dcgan example"); parser.add_argument("--epochs") .help("Number of epochs to train") .default_value(std::int64_t{30}) .scan<'i', int64_t>(); try { parser.parse_args(argc, argv); } catch (const std::exception& err) { std::cout << err.what() << std::endl; std::cout << parser; std::exit(1); } // The number of epochs to train, default value is 30. const int64_t kNumberOfEpochs = parser.get("--epochs"); std::cout << "Traning with number of epochs: " << kNumberOfEpochs << std::endl; torch::manual_seed(1); // Create the device we pass around based on whether CUDA is available. torch::Device device(torch::kCPU); if (torch::cuda::is_available()) { std::cout << "CUDA is available! Training on GPU." << std::endl; device = torch::Device(torch::kCUDA); } DCGANGenerator generator(kNoiseSize); generator->to(device); nn::Sequential discriminator = create_discriminator(); discriminator->to(device); // Assume the MNIST dataset is available under `kDataFolder`; auto dataset = torch::data::datasets::MNIST(kDataFolder) .map(torch::data::transforms::Normalize<>(0.5, 0.5)) .map(torch::data::transforms::Stack<>()); const int64_t batches_per_epoch = static_cast( std::ceil(dataset.size().value() / static_cast(kBatchSize))); auto data_loader = torch::data::make_data_loader( std::move(dataset), torch::data::DataLoaderOptions().batch_size(kBatchSize).workers(2)); torch::optim::Adam generator_optimizer( generator->parameters(), torch::optim::AdamOptions(2e-4).betas(std::make_tuple (0.5, 0.5))); torch::optim::Adam discriminator_optimizer( discriminator->parameters(), torch::optim::AdamOptions(2e-4).betas(std::make_tuple (0.5, 0.5))); if (kRestoreFromCheckpoint) { torch::load(generator, "generator-checkpoint.pt"); torch::load(generator_optimizer, "generator-optimizer-checkpoint.pt"); torch::load(discriminator, "discriminator-checkpoint.pt"); torch::load( discriminator_optimizer, "discriminator-optimizer-checkpoint.pt"); } int64_t checkpoint_counter = 1; for (int64_t epoch = 1; epoch <= kNumberOfEpochs; ++epoch) { int64_t batch_index = 0; for (const torch::data::Example<>& batch : *data_loader) { // Train discriminator with real images. discriminator->zero_grad(); torch::Tensor real_images = batch.data.to(device); torch::Tensor real_labels = torch::empty(batch.data.size(0), device).uniform_(0.8, 1.0); torch::Tensor real_output = discriminator->forward(real_images).reshape(real_labels.sizes()); torch::Tensor d_loss_real = torch::binary_cross_entropy(real_output, real_labels); d_loss_real.backward(); // Train discriminator with fake images. torch::Tensor noise = torch::randn({batch.data.size(0), kNoiseSize, 1, 1}, device); torch::Tensor fake_images = generator->forward(noise); torch::Tensor fake_labels = torch::zeros(batch.data.size(0), device); torch::Tensor fake_output = discriminator->forward(fake_images.detach()).reshape(fake_labels.sizes()); torch::Tensor d_loss_fake = torch::binary_cross_entropy(fake_output, fake_labels); d_loss_fake.backward(); torch::Tensor d_loss = d_loss_real + d_loss_fake; discriminator_optimizer.step(); // Train generator. generator->zero_grad(); fake_labels.fill_(1); fake_output = discriminator->forward(fake_images).reshape(fake_labels.sizes()); torch::Tensor g_loss = torch::binary_cross_entropy(fake_output, fake_labels); g_loss.backward(); generator_optimizer.step(); batch_index++; if (batch_index % kLogInterval == 0) { std::printf( "\r[%2ld/%2ld][%3ld/%3ld] D_loss: %.4f | G_loss: %.4f\n", epoch, kNumberOfEpochs, batch_index, batches_per_epoch, d_loss.item(), g_loss.item()); } if (batch_index % kCheckpointEvery == 0) { // Checkpoint the model and optimizer state. torch::save(generator, "generator-checkpoint.pt"); torch::save(generator_optimizer, "generator-optimizer-checkpoint.pt"); torch::save(discriminator, "discriminator-checkpoint.pt"); torch::save( discriminator_optimizer, "discriminator-optimizer-checkpoint.pt"); // Sample the generator and save the images. torch::Tensor samples = generator->forward(torch::randn( {kNumberOfSamplesPerCheckpoint, kNoiseSize, 1, 1}, device)); torch::save( (samples + 1.0) / 2.0, torch::str("dcgan-sample-", checkpoint_counter, ".pt")); std::cout << "\n-> checkpoint " << ++checkpoint_counter << '\n'; } } } std::cout << "Training complete!" << std::endl; } ================================================ FILE: cpp/dcgan/display_samples.py ================================================ from __future__ import print_function from __future__ import unicode_literals import argparse import matplotlib.pyplot as plt import torch parser = argparse.ArgumentParser() parser.add_argument("-i", "--sample-file", required=True) parser.add_argument("-o", "--out-file", default="out.png") parser.add_argument("-d", "--dimension", type=int, default=3) options = parser.parse_args() module = torch.jit.load(options.sample_file) images = list(module.parameters())[0] for index in range(options.dimension * options.dimension): image = images[index].detach().cpu().reshape(28, 28).mul(255).to(torch.uint8) array = image.numpy() axis = plt.subplot(options.dimension, options.dimension, 1 + index) plt.imshow(array, cmap="gray") axis.get_xaxis().set_visible(False) axis.get_yaxis().set_visible(False) plt.savefig(options.out_file) print("Saved ", options.out_file) ================================================ FILE: cpp/distributed/CMakeLists.txt ================================================ cmake_minimum_required(VERSION 3.5) project(dist-mnist) find_package(Torch REQUIRED) find_package(MPI REQUIRED) include_directories(SYSTEM ${MPI_C_INCLUDE_PATH} ${MPI_CXX_INCLUDE_PATH}) add_executable(dist-mnist dist-mnist.cpp) target_link_libraries(dist-mnist ${TORCH_LIBRARIES}) target_link_libraries(dist-mnist ${MPI_LIBRARIES}) target_link_libraries(dist-mnist ${CMAKE_PREFIX_PATH}/lib/libc10d.a) if(MPI_COMPILE_FLAGS) set_target_properties(dist-mnist PROPERTIES COMPILE_FLAGS "${MPI_COMPILE_FLAGS}") endif() if(MPI_LINK_FLAGS) set_target_properties(dist-mnist PROPERTIES LINK_FLAGS "${MPI_LINK_FLAGS}") endif() ================================================ FILE: cpp/distributed/README.md ================================================ # Distributed Training on MNIST using PyTorch C++ Frontend (Libtorch) This folder contains an example of data-parallel training of a convolutional neural network on the MNIST dataset. For parallelization, Message Passing Interface (MPI) is used. The entire code is contained in dist-mnist.cpp You can find instructions on how to install MPI [here] (https://www.open-mpi.org/faq/?category=building). This code was tested on Open MPI but it should run on other MPI distributions as well such as MPICH, MVAPICH, etc. To build the code, run the following commands from the terminal: ```shell $ cd distributed $ mkdir build $ cd build $ cmake -DCMAKE_PREFIX_PATH=/path/to/libtorch .. $ make ``` where /path/to/libtorch should be the path to the unzipped LibTorch distribution. Note that the LibTorch from the [PyTorch homepage] ((https://pytorch.org/get-started/locally/) does not include MPI headers and cannot be used for this example. You have to compile LibTorch manually - a set of guidelines is provided [here] (https://gist.github.com/lasagnaphil/3e0099816837318e8e8bcab7edcfd5d9), however this may vary for different systems. To run the code, ```shell mpirun -np {NUM-PROCS} ./dist-mnist ``` ================================================ FILE: cpp/distributed/dist-mnist.cpp ================================================ #include #include #include // Define a Convolutional Module struct Model : torch::nn::Module { Model() : conv1(torch::nn::Conv2dOptions(1, 10, 5)), conv2(torch::nn::Conv2dOptions(10, 20, 5)), fc1(320, 50), fc2(50, 10) { register_module("conv1", conv1); register_module("conv2", conv2); register_module("conv2_drop", conv2_drop); register_module("fc1", fc1); register_module("fc2", fc2); } torch::Tensor forward(torch::Tensor x) { x = torch::relu(torch::max_pool2d(conv1->forward(x), 2)); x = torch::relu( torch::max_pool2d(conv2_drop->forward(conv2->forward(x)), 2)); x = x.view({-1, 320}); x = torch::relu(fc1->forward(x)); x = torch::dropout(x, 0.5, is_training()); x = fc2->forward(x); return torch::log_softmax(x, 1); } torch::nn::Conv2d conv1; torch::nn::Conv2d conv2; torch::nn::Dropout2d conv2_drop; torch::nn::Linear fc1; torch::nn::Linear fc2; }; void waitWork( std::shared_ptr pg, std::vector> works) { for (auto& work : works) { try { work->wait(); } catch (const std::exception& ex) { std::cerr << "Exception received: " << ex.what() << std::endl; pg->abort(); } } } int main(int argc, char* argv[]) { // Creating MPI Process Group auto pg = c10d::ProcessGroupMPI::createProcessGroupMPI(); // Retrieving MPI environment variables auto numranks = pg->getSize(); auto rank = pg->getRank(); // TRAINING // Read train dataset const char* kDataRoot = "../data"; auto train_dataset = torch::data::datasets::MNIST(kDataRoot) .map(torch::data::transforms::Normalize<>(0.1307, 0.3081)) .map(torch::data::transforms::Stack<>()); // Distributed Random Sampler auto data_sampler = torch::data::samplers::DistributedRandomSampler( train_dataset.size().value(), numranks, rank, false); auto num_train_samples_per_proc = train_dataset.size().value() / numranks; // Generate dataloader auto total_batch_size = 64; auto batch_size_per_proc = total_batch_size / numranks; // effective batch size in each processor auto data_loader = torch::data::make_data_loader( std::move(train_dataset), data_sampler, batch_size_per_proc); // setting manual seed torch::manual_seed(0); auto model = std::make_shared(); auto learning_rate = 1e-2; torch::optim::SGD optimizer(model->parameters(), learning_rate); // Number of epochs size_t num_epochs = 10; for (size_t epoch = 1; epoch <= num_epochs; ++epoch) { size_t num_correct = 0; for (auto& batch : *data_loader) { auto ip = batch.data; auto op = batch.target.squeeze(); // convert to required formats ip = ip.to(torch::kF32); op = op.to(torch::kLong); // Reset gradients model->zero_grad(); // Execute forward pass auto prediction = model->forward(ip); auto loss = torch::nll_loss(torch::log_softmax(prediction, 1), op); // Backpropagation loss.backward(); // Averaging the gradients of the parameters in all the processors // Note: This may lag behind DistributedDataParallel (DDP) in performance // since this synchronizes parameters after backward pass while DDP // overlaps synchronizing parameters and computing gradients in backward // pass std::vector> works; for (auto& param : model->named_parameters()) { std::vector tmp = {param.value().grad()}; auto work = pg->allreduce(tmp); works.push_back(std::move(work)); } waitWork(pg, works); for (auto& param : model->named_parameters()) { param.value().grad().data() = param.value().grad().data() / numranks; } // Update parameters optimizer.step(); auto guess = prediction.argmax(1); num_correct += torch::sum(guess.eq_(op)).item(); } // end batch loader auto accuracy = 100.0 * num_correct / num_train_samples_per_proc; std::cout << "Accuracy in rank " << rank << " in epoch " << epoch << " - " << accuracy << std::endl; } // end epoch // TESTING ONLY IN RANK 0 if (rank == 0) { auto test_dataset = torch::data::datasets::MNIST( kDataRoot, torch::data::datasets::MNIST::Mode::kTest) .map(torch::data::transforms::Normalize<>(0.1307, 0.3081)) .map(torch::data::transforms::Stack<>()); auto num_test_samples = test_dataset.size().value(); auto test_loader = torch::data::make_data_loader( std::move(test_dataset), num_test_samples); model->eval(); // enable eval mode to prevent backprop size_t num_correct = 0; for (auto& batch : *test_loader) { auto ip = batch.data; auto op = batch.target.squeeze(); // convert to required format ip = ip.to(torch::kF32); op = op.to(torch::kLong); auto prediction = model->forward(ip); auto loss = torch::nll_loss(torch::log_softmax(prediction, 1), op); std::cout << "Test loss - " << loss.item() << std::endl; auto guess = prediction.argmax(1); num_correct += torch::sum(guess.eq_(op)).item(); } // end test loader std::cout << "Num correct - " << num_correct << std::endl; std::cout << "Test Accuracy - " << 100.0 * num_correct / num_test_samples << std::endl; } // end rank 0 } ================================================ FILE: cpp/mnist/CMakeLists.txt ================================================ cmake_minimum_required(VERSION 3.5) project(mnist) set(CMAKE_CXX_STANDARD 17) find_package(Torch REQUIRED) option(DOWNLOAD_MNIST "Download the MNIST dataset from the internet" ON) if (DOWNLOAD_MNIST) message(STATUS "Downloading MNIST dataset") execute_process( COMMAND python ${CMAKE_CURRENT_LIST_DIR}/../tools/download_mnist.py -d ${CMAKE_BINARY_DIR}/data ERROR_VARIABLE DOWNLOAD_ERROR) if (DOWNLOAD_ERROR) message(FATAL_ERROR "Error downloading MNIST dataset: ${DOWNLOAD_ERROR}") endif() endif() add_executable(mnist mnist.cpp) target_compile_features(mnist PUBLIC cxx_range_for) target_link_libraries(mnist ${TORCH_LIBRARIES}) if (MSVC) file(GLOB TORCH_DLLS "${TORCH_INSTALL_PREFIX}/lib/*.dll") add_custom_command(TARGET mnist POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different ${TORCH_DLLS} $) endif (MSVC) ================================================ FILE: cpp/mnist/README.md ================================================ # MNIST Example with the PyTorch C++ Frontend This folder contains an example of training a computer vision model to recognize digits in images from the MNIST dataset, using the PyTorch C++ frontend. The entire training code is contained in `mnist.cpp`. To build the code, run the following commands from your terminal: ```shell $ cd mnist $ mkdir build $ cd build $ cmake -DCMAKE_PREFIX_PATH=/path/to/libtorch .. $ make ``` where `/path/to/libtorch` should be the path to the unzipped _LibTorch_ distribution, which you can get from the [PyTorch homepage](https://pytorch.org/get-started/locally/). Execute the compiled binary to train the model: ```shell $ ./mnist Train Epoch: 1 [59584/60000] Loss: 0.4232 Test set: Average loss: 0.1989 | Accuracy: 0.940 Train Epoch: 2 [59584/60000] Loss: 0.1926 Test set: Average loss: 0.1338 | Accuracy: 0.959 Train Epoch: 3 [59584/60000] Loss: 0.1390 Test set: Average loss: 0.0997 | Accuracy: 0.969 Train Epoch: 4 [59584/60000] Loss: 0.1239 Test set: Average loss: 0.0875 | Accuracy: 0.972 ... ``` ================================================ FILE: cpp/mnist/mnist.cpp ================================================ #include #include #include #include #include #include // Where to find the MNIST dataset. const char* kDataRoot = "./data"; // The batch size for training. const int64_t kTrainBatchSize = 64; // The batch size for testing. const int64_t kTestBatchSize = 1000; // The number of epochs to train. const int64_t kNumberOfEpochs = 10; // After how many batches to log a new update with the loss value. const int64_t kLogInterval = 10; struct Net : torch::nn::Module { Net() : conv1(torch::nn::Conv2dOptions(1, 10, /*kernel_size=*/5)), conv2(torch::nn::Conv2dOptions(10, 20, /*kernel_size=*/5)), fc1(320, 50), fc2(50, 10) { register_module("conv1", conv1); register_module("conv2", conv2); register_module("conv2_drop", conv2_drop); register_module("fc1", fc1); register_module("fc2", fc2); } torch::Tensor forward(torch::Tensor x) { x = torch::relu(torch::max_pool2d(conv1->forward(x), 2)); x = torch::relu( torch::max_pool2d(conv2_drop->forward(conv2->forward(x)), 2)); x = x.view({-1, 320}); x = torch::relu(fc1->forward(x)); x = torch::dropout(x, /*p=*/0.5, /*training=*/is_training()); x = fc2->forward(x); return torch::log_softmax(x, /*dim=*/1); } torch::nn::Conv2d conv1; torch::nn::Conv2d conv2; torch::nn::Dropout2d conv2_drop; torch::nn::Linear fc1; torch::nn::Linear fc2; }; template void train( size_t epoch, Net& model, torch::Device device, DataLoader& data_loader, torch::optim::Optimizer& optimizer, size_t dataset_size) { model.train(); size_t batch_idx = 0; for (auto& batch : data_loader) { auto data = batch.data.to(device), targets = batch.target.to(device); optimizer.zero_grad(); auto output = model.forward(data); auto loss = torch::nll_loss(output, targets); AT_ASSERT(!std::isnan(loss.template item())); loss.backward(); optimizer.step(); if (batch_idx++ % kLogInterval == 0) { std::printf( "\rTrain Epoch: %ld [%5ld/%5ld] Loss: %.4f", epoch, batch_idx * batch.data.size(0), dataset_size, loss.template item()); } } } template void test( Net& model, torch::Device device, DataLoader& data_loader, size_t dataset_size) { torch::NoGradGuard no_grad; model.eval(); double test_loss = 0; int32_t correct = 0; for (const auto& batch : data_loader) { auto data = batch.data.to(device), targets = batch.target.to(device); auto output = model.forward(data); test_loss += torch::nll_loss( output, targets, /*weight=*/{}, torch::Reduction::Sum) .template item(); auto pred = output.argmax(1); correct += pred.eq(targets).sum().template item(); } test_loss /= dataset_size; std::printf( "\nTest set: Average loss: %.4f | Accuracy: %.3f\n", test_loss, static_cast(correct) / dataset_size); } auto main() -> int { torch::manual_seed(1); torch::DeviceType device_type; if (torch::cuda::is_available()) { std::cout << "CUDA available! Training on GPU." << std::endl; device_type = torch::kCUDA; } else { std::cout << "Training on CPU." << std::endl; device_type = torch::kCPU; } torch::Device device(device_type); Net model; model.to(device); auto train_dataset = torch::data::datasets::MNIST(kDataRoot) .map(torch::data::transforms::Normalize<>(0.1307, 0.3081)) .map(torch::data::transforms::Stack<>()); const size_t train_dataset_size = train_dataset.size().value(); auto train_loader = torch::data::make_data_loader( std::move(train_dataset), kTrainBatchSize); auto test_dataset = torch::data::datasets::MNIST( kDataRoot, torch::data::datasets::MNIST::Mode::kTest) .map(torch::data::transforms::Normalize<>(0.1307, 0.3081)) .map(torch::data::transforms::Stack<>()); const size_t test_dataset_size = test_dataset.size().value(); auto test_loader = torch::data::make_data_loader(std::move(test_dataset), kTestBatchSize); torch::optim::SGD optimizer( model.parameters(), torch::optim::SGDOptions(0.01).momentum(0.5)); for (size_t epoch = 1; epoch <= kNumberOfEpochs; ++epoch) { train(epoch, model, device, *train_loader, optimizer, train_dataset_size); test(model, device, *test_loader, test_dataset_size); } } ================================================ FILE: cpp/regression/CMakeLists.txt ================================================ cmake_minimum_required(VERSION 3.5) project(regression) set(CMAKE_CXX_STANDARD 17) find_package(Torch REQUIRED) add_executable(${PROJECT_NAME} "regression.cpp") target_link_libraries(${PROJECT_NAME} "${TORCH_LIBRARIES}") # The following code block is suggested to be used on Windows. # According to https://github.com/pytorch/pytorch/issues/25457, # the DLLs need to be copied to avoid memory errors. if (MSVC) file(GLOB TORCH_DLLS "${TORCH_INSTALL_PREFIX}/lib/*.dll") add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different ${TORCH_DLLS} $) endif (MSVC) ================================================ FILE: cpp/regression/README.md ================================================ # Linear regression example Trains a single fully-connected layer to fit a 4th degree polynomial. To build the code, run the following commands from your terminal: ```shell $ cd regression $ mkdir build $ cd build $ cmake -DCMAKE_PREFIX_PATH=/path/to/libtorch .. $ make ``` where `/path/to/libtorch` should be the path to the unzipped _LibTorch_ distribution, which you can get from the [PyTorch homepage](https://pytorch.org/get-started/locally/). Execute the compiled binary to run: ```shell $ ./regression Loss: 0.000301158 after 584 batches ==> Learned function: y = 11.6441 x^4 -3.10164 x^3 2.19786 x^2 -3.83606 x^1 + 4.37066 ==> Actual function: y = 11.669 x^4 -3.16023 x^3 2.19182 x^2 -3.81505 x^1 + 4.38219 ... ``` ================================================ FILE: cpp/regression/regression.cpp ================================================ #include #include #include #include #include #define POLY_DEGREE 4 // Builds features i.e. a matrix with columns [x, x^2, x^3, x^4]. torch::Tensor make_features(torch::Tensor x) { x = x.unsqueeze(1); std::vector xs; for (int64_t i = 0; i < POLY_DEGREE; ++i) xs.push_back(x.pow(i + 1)); return torch::cat(xs, 1); } // Approximated function. torch::Tensor f( torch::Tensor x, torch::Tensor W_target, torch::Tensor b_target) { return x.mm(W_target) + b_target.item(); } // Creates a string description of a polynomial. std::string poly_desc(torch::Tensor W, torch::Tensor b) { auto size = W.size(0); std::ostringstream stream; stream << "y = "; for (int64_t i = 0; i < size; ++i) stream << W[i].item() << " x^" << size - i << " "; stream << "+ " << b[0].item(); return stream.str(); } // Builds a batch i.e. (x, f(x)) pair. std::pair get_batch( torch::Tensor W_target, torch::Tensor b_target, int64_t batch_size = 32) { auto random = torch::randn({batch_size}); auto x = make_features(random); auto y = f(x, W_target, b_target); return std::make_pair(x, y); } int main() { auto W_target = torch::randn({POLY_DEGREE, 1}) * 5; auto b_target = torch::randn({1}) * 5; // Define the model and optimizer auto fc = torch::nn::Linear(W_target.size(0), 1); torch::optim::SGD optim(fc->parameters(), .1); float loss = 0; int64_t batch_idx = 0; while (++batch_idx) { // Get data torch::Tensor batch_x, batch_y; std::tie(batch_x, batch_y) = get_batch(W_target, b_target); // Reset gradients optim.zero_grad(); // Forward pass auto output = torch::smooth_l1_loss(fc(batch_x), batch_y); loss = output.item(); // Backward pass output.backward(); // Apply gradients optim.step(); // Stop criterion if (loss < 1e-3f) break; } std::cout << "Loss: " << loss << " after " << batch_idx << " batches" << std::endl; std::cout << "==> Learned function:\t" << poly_desc(fc->weight.view({-1}), fc->bias) << std::endl; std::cout << "==> Actual function:\t" << poly_desc(W_target.view({-1}), b_target) << std::endl; return 0; } ================================================ FILE: cpp/tools/InstallingOpenCV.md ================================================ # Installing OpenCV ## Linux with Package Manager ### Arch Linux ```shell pacman -Syu base-devel opencv ``` ### Fedora ```shell sudo dnf install opencv opencv-dev ``` ## Linux From Source Required Packages: ```shell sudo apt-get install build-essential cmake git libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev ``` Optional Packages: ```shell sudo apt-get install python-dev python-numpy libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev libjasper-dev libdc1394-22-dev ``` Building from Source: ```shell git clone https://github.com/opencv/opencv.git git clone https://github.com/opencv/opencv_contrib.git cd opencv && mkdir build && cd build cmake -D CMAKE_BUILD_TYPE=Release -D CMAKE_INSTALL_PREFIX=/usr/local .. make -j8 # runs 8 jobs in parallel sudo make install ``` ## Windows You can download the pre-built libraries from [OpenCV releases](https://github.com/opencv/opencv/releases) and install them easily. ================================================ FILE: cpp/tools/download_mnist.py ================================================ from __future__ import division from __future__ import print_function import argparse import gzip import os import sys import urllib try: from urllib.error import URLError from urllib.request import urlretrieve except ImportError: from urllib2 import URLError from urllib import urlretrieve RESOURCES = [ 'train-images-idx3-ubyte.gz', 'train-labels-idx1-ubyte.gz', 't10k-images-idx3-ubyte.gz', 't10k-labels-idx1-ubyte.gz', ] def report_download_progress(chunk_number, chunk_size, file_size): if file_size != -1: percent = min(1, (chunk_number * chunk_size) / file_size) bar = '#' * int(64 * percent) sys.stdout.write('\r0% |{:<64}| {}%'.format(bar, int(percent * 100))) def download(destination_path, url, quiet): if os.path.exists(destination_path): if not quiet: print('{} already exists, skipping ...'.format(destination_path)) else: print('Downloading {} ...'.format(url)) try: hook = None if quiet else report_download_progress urlretrieve(url, destination_path, reporthook=hook) except URLError: raise RuntimeError('Error downloading resource!') finally: if not quiet: # Just a newline. print() def unzip(zipped_path, quiet): unzipped_path = os.path.splitext(zipped_path)[0] if os.path.exists(unzipped_path): if not quiet: print('{} already exists, skipping ... '.format(unzipped_path)) return with gzip.open(zipped_path, 'rb') as zipped_file: with open(unzipped_path, 'wb') as unzipped_file: unzipped_file.write(zipped_file.read()) if not quiet: print('Unzipped {} ...'.format(zipped_path)) def main(): parser = argparse.ArgumentParser( description='Download the MNIST dataset from the internet') parser.add_argument( '-d', '--destination', default='.', help='Destination directory') parser.add_argument( '-q', '--quiet', action='store_true', help="Don't report about progress") options = parser.parse_args() if not os.path.exists(options.destination): os.makedirs(options.destination) try: for resource in RESOURCES: path = os.path.join(options.destination, resource) # url = 'http://yann.lecun.com/exdb/mnist/{}'.format(resource) url = 'https://ossci-datasets.s3.amazonaws.com/mnist/{}'.format(resource) download(path, url, options.quiet) unzip(path, options.quiet) except KeyboardInterrupt: print('Interrupted') if __name__ == '__main__': main() ================================================ FILE: cpp/transfer-learning/CMakeLists.txt ================================================ cmake_minimum_required(VERSION 3.5) project(example) find_package(Torch REQUIRED) find_package(OpenCV 4.1.0 REQUIRED) include_directories(${OpenCV_INCLUDE_DIRS}) add_executable(example main.cpp main.h) add_executable(classify classify.cpp) target_link_libraries(example ${OpenCV_LIBS}) target_link_libraries(example "${TORCH_LIBRARIES}") target_link_libraries(classify ${OpenCV_LIBS}) target_link_libraries(classify "${TORCH_LIBRARIES}") set_property(TARGET classify PROPERTY CXX_STANDARD 17) set_property(TARGET example PROPERTY CXX_STANDARD 17) ================================================ FILE: cpp/transfer-learning/README.md ================================================ # Transfer Learning on Dogs vs Cats Dataset using Libtorch and OpenCV Transfer Learning on Dogs vs Cats dataset using PyTorch C++ API. ## Usage For **training**: 1. Remove final layer of `ResNet18` pre-trained model and convert to `torch.jit` module: `python3 convert.py`. 2. Create build directory: `mkdir build && cd build` 3. `cmake -DCMAKE_PREFIX_PATH=/absolute/path/to/libtorch ..` 4. `make` 5. Run training code: `./example ` For **prediction**: 1. `cd build` 2. `./classify ` : `./classify ../resnet18_without_last_layer.pt model_linear.pt` Detailed blog on applying Transfer Learning using Libtorch: https://krshrimali.github.io/Applying-Transfer-Learning-Dogs-Cats/. ================================================ FILE: cpp/transfer-learning/classify.cpp ================================================ // // classify.cpp // transfer-learning // // Created by Kushashwa Ravi Shrimali on 15/08/19. // #include #include #include #include #include // Utility function to load image from given folder // File type accepted: .jpg std::vector load_images(std::string folder_name) { std::vector list_images; std::string base_name = folder_name; DIR* dir; struct dirent *ent; if((dir = opendir(base_name.c_str())) != NULL) { while((ent = readdir(dir)) != NULL) { std::string filename = ent->d_name; if(filename.length() > 4 && filename.substr(filename.length() - 3) == "jpg") { std::string newf = base_name + filename; list_images.push_back(newf); } } } return list_images; } void print_probabilities(std::string loc, std::string model_path, std::string model_path_linear) { // Load image with OpenCV. cv::Mat img = cv::imread(loc); cv::resize(img, img, cv::Size(224, 224), cv::INTER_CUBIC); // Convert the image and label to a tensor. torch::Tensor img_tensor = torch::from_blob(img.data, {1, img.rows, img.cols, 3}, torch::kByte); img_tensor = img_tensor.permute({0, 3, 1, 2}); // convert to CxHxW img_tensor = img_tensor.to(torch::kF32); // Load the model. torch::jit::script::Module model; model = torch::jit::load(model_path); torch::nn::Linear model_linear(512, 2); torch::load(model_linear, model_path_linear); // Predict the probabilities for the classes. std::vector input; input.push_back(img_tensor); torch::Tensor prob = model.forward(input).toTensor(); prob = prob.view({prob.size(0), -1}); prob = model_linear(prob); std::cout << "Printing for location: " << loc << std::endl; std::cout << "Cat prob: " << *(prob.data())*100. << std::endl; std::cout << "Dog prob: " << *(prob.data()+1)*100. << std::endl; } int main(int arc, char** argv) { // argv[1] should is the test image std::string location = argv[1]; // argv[2] contains pre-trained model without last layer // argv[3] contains trained last FC layer std::string model_path = argv[2]; std::string model_path_linear = argv[3]; // Load the model. // You can also use: auto model = torch::jit::load(model_path); torch::jit::script::Module model = torch::jit::load(model_path); // Print probabilities for dog and cat classes print_probabilities(location, model_path, model_path_linear); return 0; } ================================================ FILE: cpp/transfer-learning/convert.py ================================================ """ This python script converts the network into Script Module """ import torch from torchvision import models # Download and load the pre-trained model model = models.resnet18(weights=models.ResNet18_Weights.IMAGENET1K_V1) # Set upgrading the gradients to False for param in model.parameters(): param.requires_grad = False # Save the model except the final FC Layer resnet18 = torch.nn.Sequential(*list(model.children())[:-1]) example_input = torch.rand(1, 3, 224, 224) script_module = torch.jit.trace(resnet18, example_input) script_module.save('resnet18_without_last_layer.pt') ================================================ FILE: cpp/transfer-learning/main.cpp ================================================ // // main.cpp // transfer-learning // // Created by Kushashwa Ravi Shrimali on 12/08/19. // #include "main.h" torch::Tensor read_data(std::string location) { /* Function to return image read at location given as type torch::Tensor Resizes image to (224, 224, 3) Parameters =========== 1. location (std::string type) - required to load image from the location Returns =========== torch::Tensor type - image read as tensor */ cv::Mat img = cv::imread(location, 1); cv::resize(img, img, cv::Size(224, 224), cv::INTER_CUBIC); torch::Tensor img_tensor = torch::from_blob(img.data, {img.rows, img.cols, 3}, torch::kByte); img_tensor = img_tensor.permute({2, 0, 1}); return img_tensor.clone(); } torch::Tensor read_label(int label) { /* Function to return label from int (0, 1 for binary and 0, 1, ..., n-1 for n-class classification) as type torch::Tensor Parameters =========== 1. label (int type) - required to convert int to tensor Returns =========== torch::Tensor type - label read as tensor */ torch::Tensor label_tensor = torch::full({1}, label); return label_tensor.clone(); } std::vector process_images(std::vector list_images) { /* Function returns vector of tensors (images) read from the list of images in a folder Parameters =========== 1. list_images (std::vector type) - list of image paths in a folder to be read Returns =========== std::vector type - Images read as tensors */ std::vector states; for(std::vector::iterator it = list_images.begin(); it != list_images.end(); ++it) { torch::Tensor img = read_data(*it); states.push_back(img); } return states; } std::vector process_labels(std::vector list_labels) { /* Function returns vector of tensors (labels) read from the list of labels Parameters =========== 1. list_labels (std::vector list_labels) - Returns =========== std::vector type - returns vector of tensors (labels) */ std::vector labels; for(std::vector::iterator it = list_labels.begin(); it != list_labels.end(); ++it) { torch::Tensor label = read_label(*it); labels.push_back(label); } return labels; } std::pair,std::vector> load_data_from_folder(std::vector folders_name) { /* Function to load data from given folder(s) name(s) (folders_name) Returns pair of vectors of string (image locations) and int (respective labels) Parameters =========== 1. folders_name (std::vector type) - name of folders as a vector to load data from Returns =========== std::pair, std::vector> type - returns pair of vector of strings (image paths) and respective labels' vector (int label) */ std::vector list_images; std::vector list_labels; int label = 0; for(auto const& value: folders_name) { std::string base_name = value + "/"; // cout << "Reading from: " << base_name << endl; DIR* dir; struct dirent *ent; if((dir = opendir(base_name.c_str())) != NULL) { while((ent = readdir(dir)) != NULL) { std::string filename = ent->d_name; if(filename.length() > 4 && filename.substr(filename.length() - 3) == "jpg") { // cout << base_name + ent->d_name << endl; // cv::Mat temp = cv::imread(base_name + "/" + ent->d_name, 1); list_images.push_back(base_name + ent->d_name); list_labels.push_back(label); } } closedir(dir); } else { std::cout << "Could not open directory" << std::endl; // return EXIT_FAILURE; } label += 1; } return std::make_pair(list_images, list_labels); } template void train(torch::jit::script::Module net, torch::nn::Linear lin, Dataloader& data_loader, torch::optim::Optimizer& optimizer, size_t dataset_size) { /* This function trains the network on our data loader using optimizer. Also saves the model as model.pt after every epoch. Parameters =========== 1. net (torch::jit::script::Module type) - Pre-trained model without last FC layer 2. lin (torch::nn::Linear type) - last FC layer with revised out_features depending on the number of classes 3. data_loader (DataLoader& type) - Training data loader 4. optimizer (torch::optim::Optimizer& type) - Optimizer like Adam, SGD etc. 5. size_t (dataset_size type) - Size of training dataset Returns =========== Nothing (void) */ float best_accuracy = 0.0; int batch_index = 0; for(int i=0; i<25; i++) { float mse = 0; float Acc = 0.0; for(auto& batch: *data_loader) { auto data = batch.data; auto target = batch.target.squeeze(); // Should be of length: batch_size data = data.to(torch::kF32); target = target.to(torch::kInt64); std::vector input; input.push_back(data); optimizer.zero_grad(); auto output = net.forward(input).toTensor(); // For transfer learning output = output.view({output.size(0), -1}); output = lin(output); auto loss = torch::nll_loss(torch::log_softmax(output, 1), target); loss.backward(); optimizer.step(); auto acc = output.argmax(1).eq(target).sum(); Acc += acc.template item(); mse += loss.template item(); batch_index += 1; } mse = mse/float(batch_index); // Take mean of loss std::cout << "Epoch: " << i << ", " << "Accuracy: " << Acc/dataset_size << ", " << "MSE: " << mse << std::endl; test(net, lin, data_loader, dataset_size); if(Acc/dataset_size > best_accuracy) { best_accuracy = Acc/dataset_size; std::cout << "Saving model" << std::endl; net.save("model.pt"); torch::save(lin, "model_linear.pt"); } } } template void test(torch::jit::script::Module network, torch::nn::Linear lin, Dataloader& loader, size_t data_size) { /* Function to test the network on test data Parameters =========== 1. network (torch::jit::script::Module type) - Pre-trained model without last FC layer 2. lin (torch::nn::Linear type) - last FC layer with revised out_features depending on the number of classes 3. loader (Dataloader& type) - test data loader 4. data_size (size_t type) - test data size Returns =========== Nothing (void) */ network.eval(); float Loss = 0, Acc = 0; for (const auto& batch : *loader) { auto data = batch.data; auto targets = batch.target.squeeze(); data = data.to(torch::kF32); targets = targets.to(torch::kInt64); std::vector input; input.push_back(data); auto output = network.forward(input).toTensor(); output = output.view({output.size(0), -1}); output = lin(output); auto loss = torch::nll_loss(torch::log_softmax(output, 1), targets); auto acc = output.argmax(1).eq(targets).sum(); Loss += loss.template item(); Acc += acc.template item(); } std::cout << "Test Loss: " << Loss/data_size << ", Acc:" << Acc/data_size << std::endl; } int main(int argc, const char * argv[]) { // Set folder names for cat and dog images std::string cats_name = "/Users/krshrimali/Documents/krshrimali-blogs/dataset/train/cat_test"; std::string dogs_name = "/Users/krshrimali/Documents/krshrimali-blogs/dataset/train/dog_test"; std::vector folders_name; folders_name.push_back(cats_name); folders_name.push_back(dogs_name); // Get paths of images and labels as int from the folder paths std::pair, std::vector> pair_images_labels = load_data_from_folder(folders_name); std::vector list_images = pair_images_labels.first; std::vector list_labels = pair_images_labels.second; // Initialize CustomDataset class and read data auto custom_dataset = CustomDataset(list_images, list_labels).map(torch::data::transforms::Stack<>()); // Load pre-trained model // You can also use: auto module = torch::jit::load(argv[1]); torch::jit::script::Module module = torch::jit::load(argv[1]); // Resource: https://discuss.pytorch.org/t/how-to-load-the-prebuilt-resnet-models-or-any-other-prebuilt-models/40269/8 // For VGG: 512 * 14 * 14, 2 torch::nn::Linear lin(512, 2); // the last layer of resnet, which we want to replace, has dimensions 512x1000 torch::optim::Adam opt(lin->parameters(), torch::optim::AdamOptions(1e-3 /*learning rate*/)); auto data_loader = torch::data::make_data_loader(std::move(custom_dataset), 4); train(module, lin, data_loader, opt, custom_dataset.size().value()); return 0; } ================================================ FILE: cpp/transfer-learning/main.h ================================================ // // main.h // transfer-learning // // Created by Kushashwa Ravi Shrimali on 15/08/19. // #ifndef main_h #define main_h #include #include #include #include #include // Function to return image read at location given as type torch::Tensor // Resizes image to (224, 224, 3) torch::Tensor read_data(std::string location); // Function to return label from int (0, 1 for binary and 0, 1, ..., n-1 for n-class classification) as type torch::Tensor torch::Tensor read_label(int label); // Function returns vector of tensors (images) read from the list of images in a folder std::vector process_images(std::vector list_images); // Function returns vector of tensors (labels) read from the list of labels std::vector process_labels(std::vector list_labels); // Function to load data from given folder(s) name(s) (folders_name) // Returns pair of vectors of string (image locations) and int (respective labels) std::pair, std::vector> load_data_from_folder(std::vector folders_name); // Function to train the network on train data template void train(torch::jit::script::Module net, torch::nn::Linear lin, Dataloader& data_loader, torch::optim::Optimizer& optimizer, size_t dataset_size); // Function to test the network on test data template void test(torch::jit::script::Module network, torch::nn::Linear lin, Dataloader& loader, size_t data_size); // Custom Dataset class class CustomDataset : public torch::data::Dataset { private: /* data */ // Should be 2 tensors std::vector states, labels; size_t ds_size; public: CustomDataset(std::vector list_images, std::vector list_labels) { states = process_images(list_images); labels = process_labels(list_labels); ds_size = states.size(); }; torch::data::Example<> get(size_t index) override { /* This should return {torch::Tensor, torch::Tensor} */ torch::Tensor sample_img = states.at(index); torch::Tensor sample_label = labels.at(index); return {sample_img.clone(), sample_label.clone()}; }; torch::optional size() const override { return ds_size; }; }; #endif /* main_h */ ================================================ FILE: dcgan/README.md ================================================ # Deep Convolution Generative Adversarial Networks This example implements the paper [Unsupervised Representation Learning with Deep Convolutional Generative Adversarial Networks](http://arxiv.org/abs/1511.06434) The implementation is very close to the Torch implementation [dcgan.torch](https://github.com/soumith/dcgan.torch) After every 100 training iterations, the files `real_samples.png` and `fake_samples.png` are written to disk with the samples from the generative model. After every epoch, models are saved to: `netG_epoch_%d.pth` and `netD_epoch_%d.pth` ## Downloading the dataset You can download the LSUN dataset by cloning [this repo](https://github.com/fyu/lsun) and running ``` python download.py -c bedroom ``` ## Usage ``` usage: main.py [-h] --dataset DATASET [--dataroot DATAROOT] [--workers WORKERS] [--batchSize BATCHSIZE] [--imageSize IMAGESIZE] [--nz NZ] [--ngf NGF] [--ndf NDF] [--niter NITER] [--lr LR] [--beta1 BETA1] [--dry-run] [--ngpu NGPU] [--netG NETG] [--netD NETD] [--outf OUTF] [--manualSeed MANUALSEED] [--classes CLASSES] [--accel] options: -h, --help show this help message and exit --dataset DATASET cifar10 | lsun | mnist |imagenet | folder | lfw | fake --dataroot DATAROOT path to dataset --workers WORKERS number of data loading workers --batchSize BATCHSIZE input batch size --imageSize IMAGESIZE the height / width of the input image to network --nz NZ size of the latent z vector --ngf NGF number of generator filters --ndf NDF number of discriminator filters --niter NITER number of epochs to train for --lr LR learning rate, default=0.0002 --beta1 BETA1 beta1 for adam. default=0.5 --dry-run check a single training cycle works --ngpu NGPU number of GPUs to use --netG NETG path to netG (to continue training) --netD NETD path to netD (to continue training) --outf OUTF folder to output images and model checkpoints --manualSeed MANUALSEED manual seed --classes CLASSES comma separated list of classes for the lsun data set --accel enables accelerator ``` ================================================ FILE: dcgan/main.py ================================================ from __future__ import print_function import argparse import os import random import torch import torch.nn as nn import torch.nn.parallel import torch.backends.cudnn as cudnn import torch.optim as optim import torch.utils.data import torchvision.datasets as dset import torchvision.transforms as transforms import torchvision.utils as vutils parser = argparse.ArgumentParser() parser.add_argument('--dataset', required=True, help='cifar10 | lsun | mnist |imagenet | folder | lfw | fake') parser.add_argument('--dataroot', required=False, help='path to dataset') parser.add_argument('--workers', type=int, help='number of data loading workers', default=2) parser.add_argument('--batchSize', type=int, default=64, help='input batch size') parser.add_argument('--imageSize', type=int, default=64, help='the height / width of the input image to network') parser.add_argument('--nz', type=int, default=100, help='size of the latent z vector') parser.add_argument('--ngf', type=int, default=64, help='number of generator filters') parser.add_argument('--ndf', type=int, default=64, help='number of discriminator filters') parser.add_argument('--niter', type=int, default=25, help='number of epochs to train for') parser.add_argument('--lr', type=float, default=0.0002, help='learning rate, default=0.0002') parser.add_argument('--beta1', type=float, default=0.5, help='beta1 for adam. default=0.5') parser.add_argument('--dry-run', action='store_true', help='check a single training cycle works') parser.add_argument('--ngpu', type=int, default=1, help='number of GPUs to use') parser.add_argument('--netG', default='', help="path to netG (to continue training)") parser.add_argument('--netD', default='', help="path to netD (to continue training)") parser.add_argument('--outf', default='.', help='folder to output images and model checkpoints') parser.add_argument('--manualSeed', type=int, help='manual seed') parser.add_argument('--classes', default='bedroom', help='comma separated list of classes for the lsun data set') parser.add_argument('--accel', action='store_true', default=False, help='enables accelerator') opt = parser.parse_args() print(opt) try: os.makedirs(opt.outf) except OSError: pass if opt.manualSeed is None: opt.manualSeed = random.randint(1, 10000) print("Random Seed: ", opt.manualSeed) random.seed(opt.manualSeed) torch.manual_seed(opt.manualSeed) cudnn.benchmark = True if opt.accel and torch.accelerator.is_available(): device = torch.accelerator.current_accelerator() else: device = torch.device("cpu") print(f"Using device: {device}") if opt.dataroot is None and str(opt.dataset).lower() != 'fake': raise ValueError("`dataroot` parameter is required for dataset \"%s\"" % opt.dataset) if opt.dataset in ['imagenet', 'folder', 'lfw']: # folder dataset dataset = dset.ImageFolder(root=opt.dataroot, transform=transforms.Compose([ transforms.Resize(opt.imageSize), transforms.CenterCrop(opt.imageSize), transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), ])) nc=3 elif opt.dataset == 'lsun': classes = [ c + '_train' for c in opt.classes.split(',')] dataset = dset.LSUN(root=opt.dataroot, classes=classes, transform=transforms.Compose([ transforms.Resize(opt.imageSize), transforms.CenterCrop(opt.imageSize), transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), ])) nc=3 elif opt.dataset == 'cifar10': dataset = dset.CIFAR10(root=opt.dataroot, download=True, transform=transforms.Compose([ transforms.Resize(opt.imageSize), transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), ])) nc=3 elif opt.dataset == 'mnist': dataset = dset.MNIST(root=opt.dataroot, download=True, transform=transforms.Compose([ transforms.Resize(opt.imageSize), transforms.ToTensor(), transforms.Normalize((0.5,), (0.5,)), ])) nc=1 elif opt.dataset == 'fake': dataset = dset.FakeData(image_size=(3, opt.imageSize, opt.imageSize), transform=transforms.ToTensor()) nc=3 assert dataset dataloader = torch.utils.data.DataLoader(dataset, batch_size=opt.batchSize, shuffle=True, num_workers=int(opt.workers)) ngpu = int(opt.ngpu) nz = int(opt.nz) ngf = int(opt.ngf) ndf = int(opt.ndf) # custom weights initialization called on netG and netD def weights_init(m): classname = m.__class__.__name__ if classname.find('Conv') != -1: torch.nn.init.normal_(m.weight, 0.0, 0.02) elif classname.find('BatchNorm') != -1: torch.nn.init.normal_(m.weight, 1.0, 0.02) torch.nn.init.zeros_(m.bias) class Generator(nn.Module): def __init__(self, ngpu): super(Generator, self).__init__() self.ngpu = ngpu self.main = nn.Sequential( # input is Z, going into a convolution nn.ConvTranspose2d( nz, ngf * 8, 4, 1, 0, bias=False), nn.BatchNorm2d(ngf * 8), nn.ReLU(True), # state size. (ngf*8) x 4 x 4 nn.ConvTranspose2d(ngf * 8, ngf * 4, 4, 2, 1, bias=False), nn.BatchNorm2d(ngf * 4), nn.ReLU(True), # state size. (ngf*4) x 8 x 8 nn.ConvTranspose2d(ngf * 4, ngf * 2, 4, 2, 1, bias=False), nn.BatchNorm2d(ngf * 2), nn.ReLU(True), # state size. (ngf*2) x 16 x 16 nn.ConvTranspose2d(ngf * 2, ngf, 4, 2, 1, bias=False), nn.BatchNorm2d(ngf), nn.ReLU(True), # state size. (ngf) x 32 x 32 nn.ConvTranspose2d( ngf, nc, 4, 2, 1, bias=False), nn.Tanh() # state size. (nc) x 64 x 64 ) def forward(self, input): if (input.is_cuda or input.is_xpu) and self.ngpu > 1: output = nn.parallel.data_parallel(self.main, input, range(self.ngpu)) else: output = self.main(input) return output netG = Generator(ngpu).to(device) netG.apply(weights_init) if opt.netG != '': netG.load_state_dict(torch.load(opt.netG)) print(netG) class Discriminator(nn.Module): def __init__(self, ngpu): super(Discriminator, self).__init__() self.ngpu = ngpu self.main = nn.Sequential( # input is (nc) x 64 x 64 nn.Conv2d(nc, ndf, 4, 2, 1, bias=False), nn.LeakyReLU(0.2, inplace=True), # state size. (ndf) x 32 x 32 nn.Conv2d(ndf, ndf * 2, 4, 2, 1, bias=False), nn.BatchNorm2d(ndf * 2), nn.LeakyReLU(0.2, inplace=True), # state size. (ndf*2) x 16 x 16 nn.Conv2d(ndf * 2, ndf * 4, 4, 2, 1, bias=False), nn.BatchNorm2d(ndf * 4), nn.LeakyReLU(0.2, inplace=True), # state size. (ndf*4) x 8 x 8 nn.Conv2d(ndf * 4, ndf * 8, 4, 2, 1, bias=False), nn.BatchNorm2d(ndf * 8), nn.LeakyReLU(0.2, inplace=True), # state size. (ndf*8) x 4 x 4 nn.Conv2d(ndf * 8, 1, 4, 1, 0, bias=False), nn.Sigmoid() ) def forward(self, input): if (input.is_cuda or input.is_xpu) and self.ngpu > 1: output = nn.parallel.data_parallel(self.main, input, range(self.ngpu)) else: output = self.main(input) return output.view(-1, 1).squeeze(1) netD = Discriminator(ngpu).to(device) netD.apply(weights_init) if opt.netD != '': netD.load_state_dict(torch.load(opt.netD)) print(netD) criterion = nn.BCELoss() fixed_noise = torch.randn(opt.batchSize, nz, 1, 1, device=device) real_label = 1 fake_label = 0 # setup optimizer optimizerD = optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) if opt.dry_run: opt.niter = 1 for epoch in range(opt.niter): for i, data in enumerate(dataloader, 0): ############################ # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z))) ########################### # train with real netD.zero_grad() real_cpu = data[0].to(device) batch_size = real_cpu.size(0) label = torch.full((batch_size,), real_label, dtype=real_cpu.dtype, device=device) output = netD(real_cpu) errD_real = criterion(output, label) errD_real.backward() D_x = output.mean().item() # train with fake noise = torch.randn(batch_size, nz, 1, 1, device=device) fake = netG(noise) label.fill_(fake_label) output = netD(fake.detach()) errD_fake = criterion(output, label) errD_fake.backward() D_G_z1 = output.mean().item() errD = errD_real + errD_fake optimizerD.step() ############################ # (2) Update G network: maximize log(D(G(z))) ########################### netG.zero_grad() label.fill_(real_label) # fake labels are real for generator cost output = netD(fake) errG = criterion(output, label) errG.backward() D_G_z2 = output.mean().item() optimizerG.step() print('[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)): %.4f / %.4f' % (epoch, opt.niter, i, len(dataloader), errD.item(), errG.item(), D_x, D_G_z1, D_G_z2)) if i % 100 == 0: vutils.save_image(real_cpu, '%s/real_samples.png' % opt.outf, normalize=True) fake = netG(fixed_noise) vutils.save_image(fake.detach(), '%s/fake_samples_epoch_%03d.png' % (opt.outf, epoch), normalize=True) if opt.dry_run: break # do checkpointing torch.save(netG.state_dict(), '%s/netG_epoch_%d.pth' % (opt.outf, epoch)) torch.save(netD.state_dict(), '%s/netD_epoch_%d.pth' % (opt.outf, epoch)) ================================================ FILE: dcgan/requirements.txt ================================================ torch torchvision lmdb ================================================ FILE: distributed/FSDP/.gitignore ================================================ __pycache__/ *.pt *.csv ================================================ FILE: distributed/FSDP/README.md ================================================ Note: FSDP1 is deprecated. Please follow [FSDP2 tutorial](https://docs.pytorch.org/tutorials/intermediate/FSDP_tutorial.html) and [code examples](https://github.com/pytorch/examples/tree/main/distributed/FSDP2). ## FSDP1 T5 To run the T5 example with FSDP1 for text summarization: ## Get the wikihow dataset ```bash sh download_dataset.sh ``` ## Install the requirements: ~~~ pip install -r requirements.txt ~~~ ## Ensure you are running a recent version of PyTorch: see https://pytorch.org to install at least 1.12 and ideally a current nightly build. Start the training with Torchrun (adjust nproc_per_node to your GPU count): ``` torchrun --nnodes 1 --nproc_per_node 4 T5_training.py ``` ================================================ FILE: distributed/FSDP/T5_training.py ================================================ import os import argparse import torch import torch.nn as nn import torch.nn.functional as F import torch.optim as optim from transformers import AutoTokenizer, GPT2TokenizerFast from transformers import T5Tokenizer, T5ForConditionalGeneration import functools from torch.optim.lr_scheduler import StepLR import torch.nn.functional as F import torch.distributed as dist import torch.multiprocessing as mp from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.data.distributed import DistributedSampler from transformers.models.t5.modeling_t5 import T5Block from nlp import load_dataset from torch.distributed.fsdp import ( FullyShardedDataParallel as FSDP, CPUOffload, MixedPrecision, BackwardPrefetch, ShardingStrategy, FullStateDictConfig, StateDictType, ) from functools import partial from torch.utils.data import DataLoader from pathlib import Path from summarization_dataset import * import policies import model_checkpointing from configs import fsdp_config, train_config from utils import (bfloat_support, setup, cleanup, get_date_of_run, format_metrics_to_gb, train,validation,setup_model) from transformers.models.t5.modeling_t5 import T5Block from typing import Type import time import tqdm from datetime import datetime def get_policies(cfg, rank): """establish current policies for mixed precision and fsdp wrapping""" mixed_precision_policy = None wrapping_policy = None # mixed precision ----- if cfg.mixed_precision: bfloat_available = bfloat_support() if bfloat_available and not cfg.use_fp16: mixed_precision_policy = policies.bfSixteen if rank == 0: print(f"bFloat16 enabled for mixed precision - using bfSixteen policy") elif cfg.use_fp16: mixed_precision_policy = policies.fpSixteen if rank == 0: print(f"FP16 enabled. ") else: # mixed_precision_policy = policies.fpSixteen print( f"bFloat16 support not present. Will use FP32, and not mixed precision" ) wrapping_policy = policies.get_t5_wrapper() return mixed_precision_policy, wrapping_policy def fsdp_main(args): model, tokenizer = setup_model(train_config.model_name) local_rank = int(os.environ['LOCAL_RANK']) rank = int(os.environ['RANK']) world_size = int(os.environ['WORLD_SIZE']) dataset = load_dataset('wikihow', 'all', data_dir='data/') print(dataset.keys()) print("Size of train dataset: ", dataset['train'].shape) print("Size of Validation dataset: ", dataset['validation'].shape) #wikihow(tokenizer, type_path, num_samples, input_length, output_length, print_text=False) train_dataset = wikihow(tokenizer, 'train', 1500, 512, 150, False) val_dataset = wikihow(tokenizer, 'validation', 300, 512, 150, False) sampler1 = DistributedSampler(train_dataset, rank=rank, num_replicas=world_size, shuffle=True) sampler2 = DistributedSampler(val_dataset, rank=rank, num_replicas=world_size) setup() train_kwargs = {'batch_size': args.batch_size, 'sampler': sampler1} test_kwargs = {'batch_size': args.test_batch_size, 'sampler': sampler2} cuda_kwargs = {'num_workers': 2, 'pin_memory': True, 'shuffle': False} train_kwargs.update(cuda_kwargs) test_kwargs.update(cuda_kwargs) train_loader = torch.utils.data.DataLoader(train_dataset,**train_kwargs) val_loader = torch.utils.data.DataLoader(val_dataset, **test_kwargs) torch.cuda.set_device(local_rank) # Set up FSDP parameters mixed_precision_policy, t5_auto_wrap_policy = get_policies(train_config, rank) # Apply FSDP wrapping to the model model = FSDP(model, auto_wrap_policy=t5_auto_wrap_policy, mixed_precision=mixed_precision_policy, sharding_strategy=fsdp_config.sharding_strategy, device_id=torch.cuda.current_device(), limit_all_gathers=fsdp_config.limit_all_gathers) # Enabling this causes https://github.com/pytorch/examples/issues/1210 if fsdp_config.fsdp_activation_checkpointing: policies.apply_fsdp_checkpointing(model) # Set up optimizer and scheduler optimizer = optim.AdamW(model.parameters(), lr=train_config.lr) scheduler = StepLR(optimizer, step_size=1, gamma=train_config.gamma) best_val_loss = float("inf") curr_val_loss = float("inf") file_save_name = "T5-model-" if rank == 0: time_of_run = get_date_of_run() dur = [] train_acc_tracking = [] val_acc_tracking = [] training_start_time = time.time() if rank == 0 and args.track_memory: mem_alloc_tracker = [] mem_reserved_tracker = [] for epoch in range(1, args.epochs + 1): t0 = time.time() train_accuracy = train(args, model, rank, world_size, train_loader, optimizer, epoch, sampler=sampler1) if args.run_validation: curr_val_loss = validation(model, rank, world_size, val_loader) scheduler.step() if rank == 0: print(f"--> epoch {epoch} completed...entering save and stats zone") dur.append(time.time() - t0) train_acc_tracking.append(train_accuracy.item()) if args.run_validation: val_acc_tracking.append(curr_val_loss.item()) if args.track_memory: mem_alloc_tracker.append( format_metrics_to_gb(torch.cuda.memory_allocated()) ) mem_reserved_tracker.append( format_metrics_to_gb(torch.cuda.memory_reserved()) ) if train_config.save_model and curr_val_loss < best_val_loss: if fsdp_config.checkpoint_type == StateDictType.FULL_STATE_DICT: model_checkpointing.save_model_checkpoint( model, optimizer, rank, fsdp_config, epoch=1 ) elif fsdp_config.checkpoint_type == StateDictType.SHARDED_STATE_DICT: model_checkpointing.save_model_and_optimizer_sharded(model, rank, fsdp_config) if fsdp_config.save_optimizer: model_checkpointing.save_model_and_optimizer_sharded(model, rank, fsdp_config, optim=optimizer) if fsdp_config.save_optimizer: model_checkpointing.save_optimizer_checkpoint( model, optimizer, rank, fsdp_config, epoch=1 ) if curr_val_loss < best_val_loss: best_val_loss = curr_val_loss if rank==0: print(f"-->>>> New Val Loss Record: {best_val_loss}") dist.barrier() cleanup() if __name__ == '__main__': # Training settings parser = argparse.ArgumentParser(description='PyTorch T5 FSDP Example') parser.add_argument('--batch-size', type=int, default=4, metavar='N', help='input batch size for training (default: 4)') parser.add_argument('--test-batch-size', type=int, default=4, metavar='N', help='input batch size for testing (default: 4)') parser.add_argument('--epochs', type=int, default=2, metavar='N', help='number of epochs to train (default: 2)') parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') parser.add_argument('--track_memory', action='store_false', default=True, help='track the gpu memory') parser.add_argument('--run_validation', action='store_false', default=True, help='running the validation') args = parser.parse_args() torch.manual_seed(args.seed) fsdp_main(args) ================================================ FILE: distributed/FSDP/configs/__init__.py ================================================ from .fsdp import fsdp_config from .training import train_config ================================================ FILE: distributed/FSDP/configs/fsdp.py ================================================ from dataclasses import dataclass, field from typing import ClassVar from torch.distributed.fsdp import ShardingStrategy from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType @dataclass class fsdp_config: mixed_precision: bool=True use_fp16: bool=False seed: int=42 fsdp_activation_checkpointing: bool=False limit_all_gathers: bool=True sharding_strategy: ShardingStrategy = ShardingStrategy.FULL_SHARD #HYBRID_SHARD, SHARD_GRAD_OP checkpoint_type: StateDictType = StateDictType.FULL_STATE_DICT # alternatively can use SHARDED_STATE_DICT to avoid OOMs save_optimizer: bool=False ================================================ FILE: distributed/FSDP/configs/training.py ================================================ from dataclasses import dataclass from typing import ClassVar @dataclass class train_config: model_name: str="t5-base" run_validation: bool=True batch_size_training: int=4 num_workers_dataloader: int=2 lr: float=0.002 weight_decay: float=0.0 gamma: float= 0.85 use_fp16: bool=False mixed_precision: bool=True save_model: bool=False ================================================ FILE: distributed/FSDP/download_dataset.sh ================================================ #!/bin/bash # Create the "data" folder if it doesn't exist mkdir -p data # Download the files into the "data" folder wget -P data https://public-nlp-datasets.s3.us-west-2.amazonaws.com/wikihowAll.csv wget -P data https://public-nlp-datasets.s3.us-west-2.amazonaws.com/wikihowSep.csv ================================================ FILE: distributed/FSDP/model_checkpointing/__init__.py ================================================ from .checkpoint_handler import ( load_model_checkpoint, save_model_checkpoint, save_distributed_model_checkpoint, load_distributed_model_checkpoint, load_optimizer_checkpoint, save_optimizer_checkpoint, save_model_and_optimizer_sharded, load_model_sharded, ) ================================================ FILE: distributed/FSDP/model_checkpointing/checkpoint_handler.py ================================================ from pathlib import Path from datetime import datetime import torch import time from torch.distributed.fsdp import ( FullyShardedDataParallel as FSDP, StateDictType, FullStateDictConfig, # general model non-sharded, non-flattened params LocalStateDictConfig, # flattened params, usable only by FSDP # ShardedStateDictConfig, # un-flattened param but shards, usable by other parallel schemes. ) from torch.distributed.checkpoint import ( FileSystemReader, FileSystemWriter, save_state_dict, load_state_dict, ) from torch.distributed.checkpoint.default_planner import ( DefaultSavePlanner, DefaultLoadPlanner, ) from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType import torch.distributed.checkpoint as dist_cp import torch.distributed as dist def get_date_of_run(): """create date and time for file save uniqueness example: 2022-05-07-08:31:12_PM' """ date_of_run = datetime.now().strftime("%Y-%m-%d-%I:%M:%S_%p") print(f"--> current date and time of run = {date_of_run}") return date_of_run # create singleton saving policies to avoid making over and over fullstate_save_policy = FullStateDictConfig(offload_to_cpu=True, rank0_only=True) def load_model_sharded(model, rank, cfg, verbose=True): # torch.manual_seed(103) folder_name = ( cfg.dist_checkpoint_root_folder + "/" + cfg.dist_checkpoint_folder + "-" + cfg.model_name ) load_dir = Path.cwd() / folder_name if not load_dir.exists(): if rank == 0: print(f"No sharded_state_dict checkpoint directory found...skipping") return reader = FileSystemReader(load_dir) with FSDP.state_dict_type(model, StateDictType.SHARDED_STATE_DICT): checkpoint = model.state_dict() if rank == 0: ck = checkpoint.keys() print(f" checkpoint key len = {len(ck)} and \n keys = {ck}") dist_cp.load_state_dict( state_dict=checkpoint, storage_reader=reader, ) if rank == 0: print(f"checkpoint after load_state_dict()") ck = checkpoint.keys() print(f" checkpoint key len = {len(ck)} and \n keys = {ck}") model.load_state_dict(checkpoint) if rank == 0: print(f"Sharded state checkpoint loaded from {load_dir}") def save_model_and_optimizer_sharded(model, rank, cfg,optim=None, verbose=True): """save model and optimizer via sharded_state_dict to save_dir""" folder_name = ( cfg.dist_checkpoint_root_folder + "/" + cfg.dist_checkpoint_folder + "-" + cfg.model_name ) save_dir = Path.cwd() / folder_name if rank == 0: print(f"Saving model to {save_dir}") distributed_writer = dist_cp.FileSystemWriter( save_dir, ) t0 = time.perf_counter() with FSDP.state_dict_type(model, StateDictType.SHARDED_STATE_DICT): state_dict = {"model": model.state_dict()} if optim is not None: state_dict["optim"] = FSDP.optim_state_dict(model, optim) dist_cp.save_state_dict( state_dict=state_dict, storage_writer=distributed_writer, planner=DefaultSavePlanner(), ) dist.barrier() t1 = time.perf_counter() if rank == 0: print(f"Sharded state checkpoint saved to {save_dir}") print( f"Checkpoint Time = {t1-t0:.4f}\n using {cfg.save_using_num_threads=} total threads" ) def save_model_checkpoint( model, optimizer, rank, cfg, epoch=1, ): """saving model via rank0 cpu streaming and full_state_dict""" # saving with rank0 cpu if not cfg.checkpoint_type == StateDictType.FULL_STATE_DICT: print(f" unable to handle checkpoint type {cfg.checkpoint_type}, aborting") with FSDP.state_dict_type( model, StateDictType.FULL_STATE_DICT, fullstate_save_policy ): cpu_state = model.state_dict() if cfg.verbose: print(f"saving process: rank {rank} done w model state_dict\n") if rank == 0: print(f"--> saving model ...") # create save path save_dir = Path.cwd() / cfg.checkpoint_folder save_dir.mkdir(parents=True, exist_ok=True) save_name = cfg.model_save_name + "-" + str(epoch) + ".pt" save_full_path = str(save_dir) + "/" + save_name # save model torch.save(cpu_state, save_full_path) if cfg.verbose: print(f"model checkpoint saved for epoch {epoch} at {save_full_path}\n") def load_model_checkpoint(model, rank, cfg, verbose=True): """load local checkpoint to rank0 cpu must be called * before * passing to FSDP""" if rank != 0: return # where is the checkpoint at... full_state_dict_model_path = ( Path.cwd() / cfg.checkpoint_folder / cfg.checkpoint_model_filename ) # is it present... if not full_state_dict_model_path.is_file(): print( f"model checkpoint {full_state_dict_model_path} not present. Returning..." ) return model_checkpoint = torch.load(full_state_dict_model_path) # integrate into loaded model model.load_state_dict(model_checkpoint) if cfg.verbose: print(f"model checkpoint loaded to rank0 cpu") def save_optimizer_checkpoint(model, optimizer, rank, cfg, epoch=1): """save optimizer state via full state dict""" if cfg.verbose: print(f"--> optim state call on rank {rank}\n") # pull all sharded optimizer states to rank0 cpu... optim_state = FSDP.full_optim_state_dict(model, optimizer) if cfg.verbose: print(f"optim state dict ready on {rank} and len of {len(optim_state)}\n") if rank == 0: save_dir = Path.cwd() / cfg.checkpoint_folder save_dir.mkdir(parents=True, exist_ok=True) opt_save_name = ( cfg.optimizer_name + "-" + cfg.model_save_name + "-" + str(epoch) + ".pt" ) opt_save_full_path = save_dir / opt_save_name print(f"--> saving optimizer state...") torch.save(optim_state, opt_save_full_path) print(f"--> saved {opt_save_full_path} to disk") def load_optimizer_checkpoint(model, optimizer, rank, cfg): """load an fdsp optimizer full_state checkpoint using scatter method this ensures only rank 0 loads the optimizer state dict and scatters to other ranks """ opt_file_path = Path.cwd() / cfg.checkpoint_folder / cfg.optimizer_checkpoint_file if not opt_file_path.is_file(): print( f"warning - optimizer checkpoint not present {opt_file_path}. Returning. " ) return full_osd = None if rank == 0: full_osd = torch.load(opt_file_path) if cfg.verbose: print(f"loaded full osd on rank 0") # called from all ranks, though only rank0 has a valid param for full_osd sharded_osd = FSDP.scatter_full_optim_state_dict(full_osd, model) if cfg.verbose: print(f"optimizer shard loaded on rank {rank}") def load_distributed_model_checkpoint(model, rank, cfg): if cfg.checkpoint_type == StateDictType.LOCAL_STATE_DICT: print(f"loading distributed checkpoint, rank {rank}...") folder_name = ( cfg.dist_checkpoint_root_folder + "/" + cfg.dist_checkpoint_folder + "-" + cfg.model_name ) checkdir = Path.cwd() / folder_name if not checkdir.exists(): if rank == 0: print(f"No checkpoint directory found...skipping") return reader = FileSystemReader(checkdir) with FSDP.state_dict_type( model, StateDictType.LOCAL_STATE_DICT, ): state_dict = model.state_dict() load_state_dict(state_dict, reader) model.load_state_dict(state_dict) print(f"--> local state loaded on rank {rank}") return def save_distributed_model_checkpoint(model, rank, cfg, epoch=1): # distributed checkpoint saving # confirm type of checkpoint and save if cfg.checkpoint_type == StateDictType.LOCAL_STATE_DICT: # create writer to current path folder_name = ( cfg.dist_checkpoint_root_folder + "/" + cfg.dist_checkpoint_folder + "-" + cfg.model_name ) save_dir = Path.cwd() / folder_name writer = FileSystemWriter( save_dir, ) with FSDP.state_dict_type( model, StateDictType.LOCAL_STATE_DICT, ): state_dict = model.state_dict() # write out distributed checkpoint save_state_dict(state_dict, writer) return ================================================ FILE: distributed/FSDP/policies/__init__.py ================================================ from .mixed_precision import * from .wrapping import * from .activation_checkpointing_functions import apply_fsdp_checkpointing ================================================ FILE: distributed/FSDP/policies/activation_checkpointing_functions.py ================================================ import torch import os import torch.distributed as dist from torch.distributed.algorithms._checkpoint.checkpoint_wrapper import ( checkpoint_wrapper, CheckpointImpl, apply_activation_checkpointing, ) from transformers.models.t5.modeling_t5 import T5Block from functools import partial non_reentrant_wrapper = partial( checkpoint_wrapper, offload_to_cpu=False, checkpoint_impl=CheckpointImpl.NO_REENTRANT, ) check_fn = lambda submodule: isinstance(submodule, T5Block) def apply_fsdp_checkpointing(model): """apply activation checkpointing to model returns None as model is updated directly """ print(f"--> applying fdsp activation checkpointing...") apply_activation_checkpointing( model, checkpoint_wrapper_fn=non_reentrant_wrapper, check_fn=check_fn ) ================================================ FILE: distributed/FSDP/policies/mixed_precision.py ================================================ import torch from torch.distributed.fsdp import ( # FullyShardedDataParallel as FSDP, # CPUOffload, MixedPrecision, # BackwardPrefetch, # ShardingStrategy, ) # requires grad scaler in main loop fpSixteen = MixedPrecision( param_dtype=torch.float16, # Gradient communication precision. reduce_dtype=torch.float16, # Buffer precision. buffer_dtype=torch.float16, ) bfSixteen = MixedPrecision( param_dtype=torch.bfloat16, # Gradient communication precision. reduce_dtype=torch.bfloat16, # Buffer precision. buffer_dtype=torch.bfloat16, ) bfSixteen_working = MixedPrecision( param_dtype=torch.float32, reduce_dtype=torch.bfloat16, buffer_dtype=torch.bfloat16, ) fp32_policy = MixedPrecision( param_dtype=torch.float32, reduce_dtype=torch.float32, buffer_dtype=torch.float32, ) ================================================ FILE: distributed/FSDP/policies/wrapping.py ================================================ # holds various wrapping policies for fsdp import torch.distributed as dist import torch.nn as nn import torch from transformers.models.t5.modeling_t5 import T5Block from torch.distributed.fsdp.fully_sharded_data_parallel import ( FullyShardedDataParallel as FSDP, CPUOffload, BackwardPrefetch, MixedPrecision, ) from torch.distributed.fsdp.wrap import ( transformer_auto_wrap_policy, size_based_auto_wrap_policy, enable_wrap, wrap, ) import functools from typing import Type def get_size_policy(min_params=1e8): num_wrap_policy = functools.partial( size_based_auto_wrap_policy, min_num_params=min_params ) return num_wrap_policy def get_t5_wrapper(): """we register our main layer class and use the fsdp transformer wrapping policy ensures embedding layers are in the root fsdp unit for shared access and that fsdp units map to transformer layers """ # ==== use new transformer wrapper t5_auto_wrap_policy = functools.partial( transformer_auto_wrap_policy, transformer_layer_cls={ T5Block, }, ) return t5_auto_wrap_policy ================================================ FILE: distributed/FSDP/requirements.txt ================================================ transformers datasets tqdm protobuf SentencePiece nlp ================================================ FILE: distributed/FSDP/summarization_dataset.py ================================================ import argparse import glob import os import json import time import logging import random import re from itertools import chain from string import punctuation import pandas as pd import numpy as np import torch from torch.utils.data import Dataset, DataLoader from nlp import load_dataset from transformers import ( AdamW, T5ForConditionalGeneration, T5Tokenizer, get_linear_schedule_with_warmup ) class wikihow(Dataset): def __init__(self, tokenizer, type_path, num_samples, input_length, output_length, print_text=False): self.dataset = load_dataset('wikihow', 'all', data_dir='data/', split=type_path) if num_samples: self.dataset = self.dataset.select(list(range(0, num_samples))) self.input_length = input_length self.tokenizer = tokenizer self.output_length = output_length self.print_text = print_text def __len__(self): return self.dataset.shape[0] def clean_text(self, text): text = text.replace('Example of text:', '') text = text.replace('Example of Summary:', '') text = text.replace('\n','') text = text.replace('``', '') text = text.replace('"', '') return text def convert_to_features(self, example_batch): # Tokenize contexts and questions (as pairs of inputs) if self.print_text: print("Input Text: ", self.clean_text(example_batch['text'])) # input_ = self.clean_text(example_batch['text']) + " " # target_ = self.clean_text(example_batch['headline']) + " " input_ = self.clean_text(example_batch['text']) target_ = self.clean_text(example_batch['headline']) source = self.tokenizer.batch_encode_plus([input_], max_length=self.input_length, padding='max_length', truncation=True, return_tensors="pt") targets = self.tokenizer.batch_encode_plus([target_], max_length=self.output_length, padding='max_length', truncation=True, return_tensors="pt") return source, targets def __getitem__(self, index): source, targets = self.convert_to_features(self.dataset[index]) source_ids = source["input_ids"].squeeze() target_ids = targets["input_ids"].squeeze() src_mask = source["attention_mask"].squeeze() target_mask = targets["attention_mask"].squeeze() return {"source_ids": source_ids, "source_mask": src_mask, "target_ids": target_ids, "target_mask": target_mask} def get_dataset(tokenizer, type_path, num_samples, args): return wikihow(tokenizer=tokenizer, type_path=type_path, num_samples=num_samples, input_length=max_input_length, output_length=max_output_length) ================================================ FILE: distributed/FSDP/utils/__init__.py ================================================ from .environment import bfloat_support from .train_utils import setup, cleanup, get_date_of_run, format_metrics_to_gb, train, validation,setup_model ================================================ FILE: distributed/FSDP/utils/environment.py ================================================ # Copyright (c) 2022 Meta Platforms, Inc. and its affiliates. # All rights reserved. # # This is a simple check to confirm that your current server has full bfloat support - # both GPU native support, and Network communication support. # Be warned that if you run on V100 without a check like this, you will be running without native Bfloat16 # support and will find significant performance degradation (but it will not complain via an error). # Hence the reason for a checker! from pkg_resources import packaging import torch import torch.cuda.nccl as nccl import torch.distributed as dist # global flag that confirms ampere architecture, cuda version and # nccl version to verify bfloat16 native support is ready def bfloat_support(): return ( torch.version.cuda and torch.cuda.is_bf16_supported() and packaging.version.parse(torch.version.cuda).release >= (11, 0) and dist.is_nccl_available() and nccl.version() >= (2, 10) ) ================================================ FILE: distributed/FSDP/utils/train_utils.py ================================================ import os import torch import torch.distributed as dist from datetime import datetime import tqdm from transformers import AutoTokenizer, GPT2TokenizerFast from transformers import T5Tokenizer, T5ForConditionalGeneration g_gigabyte = 1024**3 def setup(): # initialize the process group dist.init_process_group("nccl") def cleanup(): dist.destroy_process_group() def get_date_of_run(): """create date and time for file save uniqueness example: 2022-05-07-08:31:12_PM' """ date_of_run = datetime.now().strftime("%Y-%m-%d-%I:%M:%S_%p") print(f"--> current date and time of run = {date_of_run}") return date_of_run def format_metrics_to_gb(item): """quick function to format numbers to gigabyte and round to 4 digit precision""" metric_num = item / g_gigabyte metric_num = round(metric_num, ndigits=4) return metric_num def train(args, model, rank, world_size, train_loader, optimizer, epoch, sampler=None): model.train() local_rank = int(os.environ['LOCAL_RANK']) fsdp_loss = torch.zeros(2).to(local_rank) if sampler: sampler.set_epoch(epoch) if rank==0: inner_pbar = tqdm.tqdm( range(len(train_loader)), colour="blue", desc="r0 Training Epoch" ) for batch in train_loader: for key in batch.keys(): batch[key] = batch[key].to(local_rank) optimizer.zero_grad() output = model(input_ids=batch["source_ids"],attention_mask=batch["source_mask"],labels=batch["target_ids"] ) loss = output["loss"] loss.backward() optimizer.step() fsdp_loss[0] += loss.item() fsdp_loss[1] += len(batch) if rank==0: inner_pbar.update(1) dist.all_reduce(fsdp_loss, op=dist.ReduceOp.SUM) train_accuracy = fsdp_loss[0] / fsdp_loss[1] if rank == 0: inner_pbar.close() print( f"Train Epoch: \t{epoch}, Loss: \t{train_accuracy:.4f}" ) return train_accuracy def validation(model, rank, world_size, val_loader): model.eval() correct = 0 local_rank = int(os.environ['LOCAL_RANK']) fsdp_loss = torch.zeros(2).to(local_rank) if rank == 0: inner_pbar = tqdm.tqdm( range(len(val_loader)), colour="green", desc="Validation Epoch" ) with torch.no_grad(): for batch in val_loader: for key in batch.keys(): batch[key] = batch[key].to(local_rank) output = model(input_ids=batch["source_ids"],attention_mask=batch["source_mask"],labels=batch["target_ids"]) fsdp_loss[0] += output["loss"].item() # sum up batch loss fsdp_loss[1] += len(batch) if rank==0: inner_pbar.update(1) dist.all_reduce(fsdp_loss, op=dist.ReduceOp.SUM) val_loss = fsdp_loss[0] / fsdp_loss[1] if rank == 0: inner_pbar.close() print(f"Validation Loss: {val_loss:.4f}") return val_loss def setup_model(model_name): model = T5ForConditionalGeneration.from_pretrained(model_name) tokenizer = T5Tokenizer.from_pretrained(model_name, legacy=False) return model, tokenizer ================================================ FILE: distributed/FSDP2/README.md ================================================ ## FSDP2 To run FSDP2 on transformer model: ``` cd distributed/FSDP2 pip install -r requirements.txt torchrun --nproc_per_node 2 example.py ``` * For 1st time, it creates a "checkpoints" folder and saves state dicts there * For 2nd time, it loads from previous checkpoints To enable explicit prefetching ``` torchrun --nproc_per_node 2 example.py --explicit-prefetch ``` To enable mixed precision ``` torchrun --nproc_per_node 2 example.py --mixed-precision ``` To showcase DCP API ``` torchrun --nproc_per_node 2 example.py --dcp-api ``` ## Ensure you are running a recent version of PyTorch: see https://pytorch.org/get-started/locally/ to install at least 2.5 and ideally a current nightly build. ================================================ FILE: distributed/FSDP2/checkpoint.py ================================================ import os import time import torch import torch.nn as nn from torch.distributed.checkpoint.state_dict import ( _init_optim_state, get_model_state_dict, get_optimizer_state_dict, set_model_state_dict, set_optimizer_state_dict, StateDictOptions, ) from torch.distributed.fsdp import FSDPModule from torch.distributed.tensor import distribute_tensor, DTensor MODEL_CHECKPOINT = "model_state_dict.pt" OPTIM_CHECKPOINT = "optim_state_dict.pt" PARAMS = "params" def get_latest_checkpoint_folder(path): max_num = None if not os.path.exists(path): return max_num for name in os.listdir(path): folder_path = os.path.join(path, name) if os.path.isdir(folder_path): try: num = int(name) if max_num is None or num > max_num: max_num = num except ValueError: pass # Skip non-numeric folder names return max_num class Checkpointer: def __init__(self, folder: str, dcp_api: bool): self.folder = folder self.dcp_api = dcp_api self.last_training_time = get_latest_checkpoint_folder( f"{folder}/{'dcp_api' if dcp_api else 'dtensor_api'}" ) def is_empty(self): return self.last_training_time is None def load_model(self, model: FSDPModule): last_model_checkpoint = ( f"{self.folder}/{'dcp_api' if self.dcp_api else 'dtensor_api'}" f"/{self.last_training_time}/{MODEL_CHECKPOINT}" ) full_sd = torch.load( last_model_checkpoint, mmap=True, weights_only=True, map_location="cpu" ) if self.dcp_api: set_model_state_dict( model=model, model_state_dict=full_sd, options=StateDictOptions( full_state_dict=True, broadcast_from_rank0=True, ), ) return meta_sharded_sd = model.state_dict() sharded_sd = {} for param_name, full_tensor in full_sd.items(): sharded_meta_param = meta_sharded_sd.get(param_name) sharded_tensor = distribute_tensor( full_tensor, sharded_meta_param.device_mesh, sharded_meta_param.placements, ) sharded_sd[param_name] = nn.Parameter(sharded_tensor) # choose `assign=True` since we cannot call `copy_` on meta tensor model.load_state_dict(sharded_sd, strict=False, assign=True) def load_optim(self, model: FSDPModule, opt: torch.optim.Optimizer): last_optim_checkpoint = ( f"{self.folder}/{'dcp_api' if self.dcp_api else 'dtensor_api'}" f"/{self.last_training_time}/{OPTIM_CHECKPOINT}" ) full_sd = torch.load( last_optim_checkpoint, mmap=True, weights_only=True, map_location="cpu" ) if self.dcp_api: set_optimizer_state_dict( model=model, optimizers=opt, optim_state_dict=full_sd, options=StateDictOptions( full_state_dict=True, broadcast_from_rank0=True, ), ) return _init_optim_state(opt) param_groups = opt.state_dict()["param_groups"] state = opt.state_dict()["state"] full_param_groups = full_sd["param_groups"] full_state = full_sd["state"] for param_group, full_param_group in zip(param_groups, full_param_groups): for key, value in full_param_group.items(): if key == PARAMS: continue param_group[key] = value for pid, full_pid in zip(param_group[PARAMS], full_param_group[PARAMS]): if pid not in state: continue param_state = state[pid] full_param_state = full_state[full_pid] for attr, full_tensor in full_param_state.items(): sharded_tensor = param_state[attr] if isinstance(sharded_tensor, DTensor): # exp_avg is DTensor param_state[attr] = distribute_tensor( full_tensor, sharded_tensor.device_mesh, sharded_tensor.placements, ) else: # step is plain tensor param_state[attr] = full_tensor opt.load_state_dict( { "param_groups": param_groups, "state": state, } ) def _get_full_model_state_dict(self, model: FSDPModule): if self.dcp_api: return get_model_state_dict( model=model, options=StateDictOptions( full_state_dict=True, cpu_offload=True, ), ) sharded_sd = model.state_dict() cpu_state_dict = {} for param_name, sharded_param in sharded_sd.items(): full_param = sharded_param.full_tensor() if torch.distributed.get_rank() == 0: cpu_state_dict[param_name] = full_param.cpu() else: del full_param return cpu_state_dict def _get_full_optimizer_state_dict( self, model: FSDPModule, opt: torch.optim.Optimizer, ): if self.dcp_api: return get_optimizer_state_dict( model=model, optimizers=opt, options=StateDictOptions( full_state_dict=True, cpu_offload=True, ), ) is_rank_zero = torch.distributed.get_rank() == 0 sharded_sd = opt.state_dict() sharded_state = sharded_sd["state"] full_state = {} for group_id, sharded_group in sharded_state.items(): group_state = {} for attr, sharded_tensor in sharded_group.items(): if isinstance(sharded_tensor, DTensor): # "exp_avg" in AdamW is `DTensor` full_tensor = sharded_tensor.full_tensor() else: # "step" in AdamW is plain tensor full_tensor = sharded_tensor if is_rank_zero: group_state[attr] = full_tensor.cpu() else: del full_tensor if is_rank_zero: full_state[group_id] = group_state else: del group_state if is_rank_zero: return { "param_groups": sharded_sd["param_groups"], "state": full_state, } else: return {} def save(self, model: FSDPModule, optim: torch.optim.Optimizer): model_state_dict = self._get_full_model_state_dict(model) optim_state_dict = self._get_full_optimizer_state_dict(model, optim) if torch.distributed.get_rank() == 0: new_training_time = int(time.time() * 1000) new_checkpoint_folder = f"{self.folder}/{'dcp_api' if self.dcp_api else 'dtensor_api'}/{new_training_time}" new_model_checkpoint = f"{new_checkpoint_folder}/{MODEL_CHECKPOINT}" new_optim_checkpoint = f"{new_checkpoint_folder}/{OPTIM_CHECKPOINT}" os.makedirs(new_checkpoint_folder, exist_ok=True) torch.save(model_state_dict, new_model_checkpoint) torch.save(optim_state_dict, new_optim_checkpoint) ================================================ FILE: distributed/FSDP2/example.py ================================================ import argparse import os import torch from checkpoint import Checkpointer from model import ModelArgs, Transformer from torch.distributed.fsdp import fully_shard, MixedPrecisionPolicy from utils import inspect_mixed_precision, inspect_model def verify_min_gpu_count(min_gpus: int = 2) -> bool: """ verification that we have at least 2 gpus to run dist examples """ has_gpu = torch.accelerator.is_available() gpu_count = torch.accelerator.device_count() return has_gpu and gpu_count >= min_gpus def set_modules_to_forward_prefetch(model, num_to_forward_prefetch): for i, layer in enumerate(model.layers): if i >= len(model.layers) - num_to_forward_prefetch: break layers_to_prefetch = [ model.layers[i + j] for j in range(1, num_to_forward_prefetch + 1) ] layer.set_modules_to_forward_prefetch(layers_to_prefetch) def set_modules_to_backward_prefetch(model, num_to_backward_prefetch): for i, layer in enumerate(model.layers): if i < num_to_backward_prefetch: continue layers_to_prefetch = [ model.layers[i - j] for j in range(1, num_to_backward_prefetch + 1) ] layer.set_modules_to_backward_prefetch(layers_to_prefetch) def main(args): _min_gpu_count = 2 if not verify_min_gpu_count(min_gpus=_min_gpu_count): print(f"Unable to locate sufficient {_min_gpu_count} gpus to run this example. Exiting.") exit() rank = int(os.environ["LOCAL_RANK"]) if torch.accelerator.is_available(): device_type = torch.accelerator.current_accelerator() device = torch.device(f"{device_type}:{rank}") torch.accelerator.device_index(rank) print(f"Running on rank {rank} on device {device}") else: device = torch.device("cpu") print(f"Running on device {device}") backend = torch.distributed.get_default_backend_for_device(device) torch.distributed.init_process_group(backend=backend, device_id=device) torch.manual_seed(0) vocab_size = 1024 batch_size = 32 seq_len = 64 model_args = ModelArgs( n_layers=10, n_heads=4, vocab_size=vocab_size, max_seq_len=seq_len, dropout_p=0, ) with torch.device("meta"): model = Transformer(model_args) fsdp_kwargs = {} if args.mixed_precision: fsdp_kwargs["mp_policy"] = MixedPrecisionPolicy( param_dtype=torch.bfloat16, reduce_dtype=torch.float32, ) for layer in model.layers: fully_shard(layer, **fsdp_kwargs) fully_shard(model, **fsdp_kwargs) inspect_model(model) if args.explicit_prefetching: set_modules_to_forward_prefetch(model, num_to_forward_prefetch=2) set_modules_to_backward_prefetch(model, num_to_backward_prefetch=2) checkpointer = Checkpointer("checkpoints", dcp_api=args.dcp_api) if checkpointer.last_training_time is None: model.to_empty(device=device) model.reset_parameters() else: checkpointer.load_model(model) if args.mixed_precision: inspect_mixed_precision(model) optim = torch.optim.Adam(model.parameters(), lr=1e-2) if checkpointer.last_training_time is not None: checkpointer.load_optim(model, optim) for _ in range(10): if args.explicit_prefetching: model.unshard() x = torch.randint(0, vocab_size, (batch_size, seq_len), device=device) loss = model(x).sum() loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0) optim.step() optim.zero_grad() checkpointer.save(model, optim) torch.distributed.destroy_process_group() if __name__ == "__main__": parser = argparse.ArgumentParser(description="PyTorch FSDP2 example") parser.add_argument("--explicit-prefetching", action="store_true", default=False) parser.add_argument("--mixed-precision", action="store_true", default=False) parser.add_argument("--dcp-api", action="store_true", default=False) args = parser.parse_args() main(args) ================================================ FILE: distributed/FSDP2/model.py ================================================ from dataclasses import dataclass import torch import torch.nn as nn import torch.nn.functional as F @dataclass class ModelArgs: n_layers: int = 2 vocab_size: int = 8 max_seq_len: int = 16 dim: int = 16 n_heads: int = 4 dropout_p: float = 0.1 class Attention(nn.Module): def __init__(self, args: ModelArgs): super().__init__() assert args.dim % args.n_heads == 0 self.head_dim = args.dim // args.n_heads self.n_heads = args.n_heads self.dropout_p = args.dropout_p self.resid_dropout = nn.Dropout(args.dropout_p) self.wq = nn.Linear(args.dim, args.dim, bias=False) self.wk = nn.Linear(args.dim, args.dim, bias=False) self.wv = nn.Linear(args.dim, args.dim, bias=False) self.wo = nn.Linear(args.dim, args.dim, bias=False) def forward(self, x): bsz, seq_len, _ = x.size() queries, keys, values = self.wq(x), self.wk(x), self.wv(x) queries = queries.view(bsz, seq_len, self.n_heads, self.head_dim) keys = keys.view(bsz, seq_len, self.n_heads, self.head_dim) values = values.view(bsz, seq_len, self.n_heads, self.head_dim) queries = queries.transpose(1, 2) # (bsz, n_heads, seq_len, head_dim) keys = keys.transpose(1, 2) # (bsz, n_heads, seq_len, head_dim) values = values.transpose(1, 2) # (bsz, n_heads, seq_len, head_dim) output = F.scaled_dot_product_attention( queries, keys, values, None, self.dropout_p if self.training else 0, ) output = output.transpose(1, 2).contiguous().view(bsz, seq_len, -1) return self.resid_dropout(self.wo(output)) def reset_parameters(self): self.wq.reset_parameters() self.wk.reset_parameters() self.wv.reset_parameters() self.wo.reset_parameters() class FeedForward(nn.Module): def __init__(self, dim, hidden_dim, dropout_p): super().__init__() self.w1 = nn.Linear(dim, hidden_dim) self.gelu = nn.GELU() self.w2 = nn.Linear(hidden_dim, dim) self.resid_dropout = nn.Dropout(dropout_p) def forward(self, x): return self.resid_dropout(self.w2(self.gelu(self.w1(x)))) def reset_parameters(self): self.w1.reset_parameters() self.w2.reset_parameters() class TransformerBlock(nn.Module): def __init__(self, args: ModelArgs): super().__init__() self.attention_norm = nn.LayerNorm(args.dim) self.attention = Attention(args) self.ffn_norm = nn.LayerNorm(args.dim) self.feed_forward = FeedForward( args.dim, hidden_dim=4 * args.dim, dropout_p=args.dropout_p ) def forward(self, x): h = x + self.attention(self.attention_norm(x)) out = h + self.feed_forward(self.ffn_norm(h)) return out def reset_parameters(self): self.attention_norm.reset_parameters() self.attention.reset_parameters() self.ffn_norm.reset_parameters() self.feed_forward.reset_parameters() # A toy transformer model, partly inspired by the nanoGPT model: # https://github.com/karpathy/nanoGPT. class Transformer(nn.Module): def __init__(self, args: ModelArgs): super().__init__() assert args.vocab_size is not None assert args.max_seq_len is not None self.model_args = args self.max_seq_len = args.max_seq_len self.tok_embeddings = nn.Embedding(args.vocab_size, args.dim) self.pos_embeddings = nn.Embedding(args.max_seq_len, args.dim) self.dropout = nn.Dropout(args.dropout_p) self.layers = nn.ModuleList() for _ in range(args.n_layers): self.layers.append(TransformerBlock(args)) self.norm = nn.LayerNorm(args.dim) self.output = nn.Linear(args.dim, args.vocab_size, bias=False) def forward(self, tokens): _bsz, seq_len = tokens.size() assert seq_len <= self.max_seq_len h = self.tok_embeddings(tokens) pos = torch.arange(0, seq_len, device=tokens.device) p = self.pos_embeddings(pos) # positional embeddings of shape (seq_len, dim) h = h + p h = self.dropout(h) for layer in self.layers: h = layer(h) h = self.norm(h) output = self.output(h).float() return output def reset_parameters(self): self.tok_embeddings.reset_parameters() self.pos_embeddings.reset_parameters() self.norm.reset_parameters() self.output.reset_parameters() ================================================ FILE: distributed/FSDP2/requirements.txt ================================================ torch>=2.7 numpy ================================================ FILE: distributed/FSDP2/run_example.sh ================================================ # /bin/bash # bash run_example.sh {file_to_run.py} {num_gpus} # where file_to_run = example to run. Default = 'example.py' # num_gpus = num local gpus to use (must be at least 2). Default = 4 # samples to run include: # example.py echo "Launching ${1:-example.py} with ${2:-4} gpus" torchrun --nnodes=1 --nproc_per_node=${2:-4} ${1:-example.py} ================================================ FILE: distributed/FSDP2/utils.py ================================================ import torch from model import Transformer from torch.distributed.fsdp import FSDPModule from torch.distributed.tensor import Shard def inspect_model(model: FSDPModule): assert isinstance(model, Transformer) assert isinstance(model, FSDPModule) if torch.distributed.get_rank() == 0: print(model) for param in model.parameters(): assert param.placements == (Shard(0),) assert param.dtype == torch.float32 # print(param.get_local_tensor()) def inspect_mixed_precision(model: FSDPModule): model.unshard() for param in model.parameters(recurse=False): assert param.dtype == torch.bfloat16 model.reshard() ================================================ FILE: distributed/ddp/README.md ================================================ # Distributed Data Parallel (DDP) Applications with PyTorch This guide demonstrates how to structure a distributed model training application for convenient multi-node launches using `torchrun`. --- ## Prerequisites You should be familiar with: - [PyTorch basics](https://pytorch.org/tutorials/beginner/deep_learning_60min_blitz.html) - [Writing distributed applications](https://pytorch.org/tutorials/intermediate/dist_tuto.html) - [Distributed model training](https://pytorch.org/tutorials/intermediate/ddp_tutorial.html) This tutorial uses the [`torch.nn.parallel.DistributedDataParallel`](https://docs.pytorch.org/docs/stable/generated/torch.nn.parallel.DistributedDataParallel.html#torch.nn.parallel.DistributedDataParallel) (DDP) class for data parallel training: multiple workers train the same global model on different data shards, compute local gradients, and synchronize them using AllReduce. In High-Performance Computing (HCP), this is called _Single Program Multiple Data_ (SPMD). --- ## Application Process Topologies A Distributed Data Parallel (DDP) application can be executed on multiple nodes where each node can consist of multiple GPU devices. Each node in turn can run multiple copies of the DDP application, each of which processes its models on multiple GPUs. Let: - _N_ = number of nodes - _G_ = number of GPUs per node - _W_ = **World Size** = total number of processes - _L_ = **Local World Size** = processes per node Each process has: - **Local rank**: in `[0, L-1]` - **Global rank**: in `[0, W-1]` **Example:** If you launch a DDP app on 2 nodes, each with 4 GPUs, and want each process to span 2 GPUs, the mapping is as follows: ![ProcessMapping](https://user-images.githubusercontent.com/875518/77676984-4c81e400-6f4c-11ea-87d8-f2ff505a99da.png) While there are quite a few ways to map processes to nodes, a good rule of thumb is to have one process span a single GPU. This enables the DDP application to have as many parallel reader streams as there are GPUs and in practice provides a good balance between I/O and computational costs. In the rest of this tutorial, we assume that the application follows this heuristic. # Preparing and launching a DDP application Independent of how a DDP application is launched, each process needs a mechanism to know its global and local ranks. Once this is known, all processes create a `ProcessGroup` that enables them to participate in collective communication operations such as AllReduce. A convenient way to start multiple DDP processes and initialize all values needed to create a `ProcessGroup` is to use the [`torchrun`](https://docs.pytorch.org/docs/stable/elastic/run.html) script provided with PyTorch. --- ## Sample Application This example is based on the ["Hello, World" DDP tutorial](https://pytorch.org/tutorials/intermediate/ddp_tutorial.html). The application calls the `spmd_main` entrypoint: ```python if __name__ == "__main__": spmd_main() ``` In `spmd_main`, the process group is initialized using the Accelerator API. The rest of the rendezvous information comes from environment variables set by `torchrun`: ```python def spmd_main(): # These are the parameters used to initialize the process group env_dict = { key: os.environ[key] for key in ("MASTER_ADDR", "MASTER_PORT", "RANK", "WORLD_SIZE") } rank = int(env_dict['RANK']) local_rank = int(env_dict['LOCAL_RANK']) local_world_size = int(env_dict['LOCAL_WORLD_SIZE']) print(f"[{os.getpid()}] Initializing process group with: {env_dict}") acc = torch.accelerator.current_accelerator() vendor_backend = torch.distributed.get_default_backend_for_device(acc) torch.accelerator.set_device_index(rank) torch.distributed.init_process_group(backend=vendor_backend) demo_basic(rank) # Tear down the process group torch.distributed.destroy_process_group() ``` **Key points:** - Each process reads its rank and world size from environment variables. - The process group is initialized for distributed communication. The training function, `demo_basic`, initializes the DDP model on the appropriate GPU: ```python def demo_basic(rank): print( f"[{os.getpid()}] rank = {torch.distributed.get_rank()}, " + f"world_size = {torch.distributed.get_world_size()}" ) model = ToyModel().to(rank) ddp_model = DDP(model, device_ids=[rank]) loss_fn = nn.MSELoss() optimizer = optim.SGD(ddp_model.parameters(), lr=0.001) optimizer.zero_grad() outputs = ddp_model(torch.randn(20, 10)) labels = torch.randn(20, 5).to(rank) loss_fn(outputs, labels).backward() optimizer.step() ``` --- ## Launching the Application ```sh torchrun --nnodes=1 --nproc_per_node=8 example.py ``` --- ## Example Output Expected output: ```sh ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** [238627] Initializing process group with: {'MASTER_ADDR': '127.0.0.1', 'MASTER_PORT': '29500', 'RANK': '0', 'WORLD_SIZE': '8'} [238630] Initializing process group with: {'MASTER_ADDR': '127.0.0.1', 'MASTER_PORT': '29500', 'RANK': '3', 'WORLD_SIZE': '8'} [238628] Initializing process group with: {'MASTER_ADDR': '127.0.0.1', 'MASTER_PORT': '29500', 'RANK': '1', 'WORLD_SIZE': '8'} [238634] Initializing process group with: {'MASTER_ADDR': '127.0.0.1', 'MASTER_PORT': '29500', 'RANK': '7', 'WORLD_SIZE': '8'} [238631] Initializing process group with: {'MASTER_ADDR': '127.0.0.1', 'MASTER_PORT': '29500', 'RANK': '4', 'WORLD_SIZE': '8'} [238632] Initializing process group with: {'MASTER_ADDR': '127.0.0.1', 'MASTER_PORT': '29500', 'RANK': '5', 'WORLD_SIZE': '8'} [238629] Initializing process group with: {'MASTER_ADDR': '127.0.0.1', 'MASTER_PORT': '29500', 'RANK': '2', 'WORLD_SIZE': '8'} [238633] Initializing process group with: {'MASTER_ADDR': '127.0.0.1', 'MASTER_PORT': '29500', 'RANK': '6', 'WORLD_SIZE': '8'} [238633] world_size = 8, rank = 6, backend=nccl [238628] world_size = 8, rank = 1, backend=nccl [238629] world_size = 8, rank = 2, backend=nccl [238631] world_size = 8, rank = 4, backend=nccl [238630] world_size = 8, rank = 3, backend=nccl [238632] world_size = 8, rank = 5, backend=nccl [238634] world_size = 8, rank = 7, backend=nccl [238627] world_size = 8, rank = 0, backend=nccl [238633] rank = 6, world_size = 8 [238628] rank = 1, world_size = 8 [238632] rank = 5, world_size = 8 [238634] rank = 7, world_size = 8 [238629] rank = 2, world_size = 8 [238630] rank = 3, world_size = 8 [238631] rank = 4, world_size = 8 [238627] rank = 0, world_size = 8 ``` # Conclusions As the author of a distributed data parallel application, your code needs to be aware of two types of resources: compute nodes and the GPUs within each node. The process of setting up bookkeeping to track how the set of GPUs is mapped to the processes of your application can be tedious and error-prone. We hope that by structuring your application as shown in this example and using `torchrun`, the mechanics of setting up distributed training can be significantly simplified. ================================================ FILE: distributed/ddp/example.py ================================================ import argparse import os import sys import tempfile from urllib.parse import urlparse import torch import torch.distributed as dist import torch.nn as nn import torch.optim as optim from torch.nn.parallel import DistributedDataParallel as DDP def verify_min_gpu_count(min_gpus: int = 2) -> bool: """ verification that we have at least 2 gpus to run dist examples """ has_gpu = torch.accelerator.is_available() gpu_count = torch.accelerator.device_count() return has_gpu and gpu_count >= min_gpus class ToyModel(nn.Module): def __init__(self): super(ToyModel, self).__init__() self.net1 = nn.Linear(10, 10) self.relu = nn.ReLU() self.net2 = nn.Linear(10, 5) def forward(self, x): return self.net2(self.relu(self.net1(x))) def demo_basic(rank): print( f"[{os.getpid()}] rank = {dist.get_rank()}, " + f"world_size = {dist.get_world_size()}" ) model = ToyModel().to(rank) ddp_model = DDP(model, device_ids=[rank]) loss_fn = nn.MSELoss() optimizer = optim.SGD(ddp_model.parameters(), lr=0.001) optimizer.zero_grad() outputs = ddp_model(torch.randn(20, 10)) labels = torch.randn(20, 5).to(rank) loss_fn(outputs, labels).backward() optimizer.step() print(f"training completed in rank {rank}!") def main(): # These are the parameters used to initialize the process group env_dict = { key: os.environ[key] for key in ("MASTER_ADDR", "MASTER_PORT", "RANK", "LOCAL_RANK", "WORLD_SIZE", "LOCAL_WORLD_SIZE") } rank = int(env_dict['RANK']) local_rank = int(env_dict['LOCAL_RANK']) local_world_size = int(env_dict['LOCAL_WORLD_SIZE']) if sys.platform == "win32": # Distributed package only covers collective communications with Gloo # backend and FileStore on Windows platform. Set init_method parameter # in init_process_group to a local file. if "INIT_METHOD" in os.environ.keys(): print(f"init_method is {os.environ['INIT_METHOD']}") url_obj = urlparse(os.environ["INIT_METHOD"]) if url_obj.scheme.lower() != "file": raise ValueError("Windows only supports FileStore") else: init_method = os.environ["INIT_METHOD"] else: # It is a example application, For convience, we create a file in temp dir. temp_dir = tempfile.gettempdir() init_method = f"file:///{os.path.join(temp_dir, 'ddp_example')}" dist.init_process_group(backend="gloo", init_method=init_method, rank=int(env_dict["RANK"]), world_size=int(env_dict["WORLD_SIZE"])) else: print(f"[{os.getpid()}] Initializing process group with: {env_dict}") acc = torch.accelerator.current_accelerator() backend = torch.distributed.get_default_backend_for_device(acc) torch.accelerator.set_device_index(rank) dist.init_process_group(backend=backend) print( f"[{os.getpid()}]: world_size = {dist.get_world_size()}, " + f"rank = {dist.get_rank()}, backend={dist.get_backend()} \n", end='' ) demo_basic(rank) # Tear down the process group dist.destroy_process_group() if __name__ == "__main__": _min_gpu_count = 2 if not verify_min_gpu_count(min_gpus=_min_gpu_count): print(f"Unable to locate sufficient {_min_gpu_count} gpus to run this example. Exiting.") sys.exit() main() ================================================ FILE: distributed/ddp/requirements.txt ================================================ torch>=2.7 ================================================ FILE: distributed/ddp/run_example.sh ================================================ # /bin/bash # bash run_example.sh {file_to_run.py} {num_gpus} # where file_to_run = example to run. Default = 'example.py' # num_gpus = num local gpus to use (must be at least 2). Default = 2 # samples to run include: # example.py echo "Launching ${1:-example.py} with ${2:-2} gpus" torchrun --nnodes=1 --nproc_per_node=${2:-2} --rdzv_id=101 --rdzv_endpoint="localhost:5972" ${1:-example.py} ================================================ FILE: distributed/ddp-tutorial-series/README.md ================================================ # distributed-pytorch Code for the DDP tutorial series at https://pytorch.org/tutorials/beginner/ddp_series_intro.html Each code file extends upon the previous one. The series starts with a non-distributed script that runs on a single GPU and incrementally updates to end with multinode training on a Slurm cluster. ## Files * [single_gpu.py](single_gpu.py): Non-distributed training script * [multigpu.py](multigpu.py): DDP on a single node * [multigpu_torchrun.py](multigpu_torchrun.py): DDP on a single node using Torchrun * [multinode.py](multinode.py): DDP on multiple nodes using Torchrun (and optionally Slurm) * [slurm/setup_pcluster_slurm.md](slurm/setup_pcluster_slurm.md): instructions to set up an AWS cluster * [slurm/config.yaml.template](slurm/config.yaml.template): configuration to set up an AWS cluster * [slurm/sbatch_run.sh](slurm/sbatch_run.sh): slurm script to launch the training job ================================================ FILE: distributed/ddp-tutorial-series/datautils.py ================================================ import torch from torch.utils.data import Dataset class MyTrainDataset(Dataset): def __init__(self, size): self.size = size self.data = [(torch.rand(20), torch.rand(1)) for _ in range(size)] def __len__(self): return self.size def __getitem__(self, index): return self.data[index] ================================================ FILE: distributed/ddp-tutorial-series/multigpu.py ================================================ import torch import torch.nn.functional as F from torch.utils.data import Dataset, DataLoader from datautils import MyTrainDataset import torch.multiprocessing as mp from torch.utils.data.distributed import DistributedSampler from torch.nn.parallel import DistributedDataParallel as DDP from torch.distributed import init_process_group, destroy_process_group import os def ddp_setup(rank, world_size): """ Args: rank: Unique identifier of each process world_size: Total number of processes """ os.environ["MASTER_ADDR"] = "localhost" os.environ["MASTER_PORT"] = "12355" torch.cuda.set_device(rank) init_process_group(backend="nccl", rank=rank, world_size=world_size) class Trainer: def __init__( self, model: torch.nn.Module, train_data: DataLoader, optimizer: torch.optim.Optimizer, gpu_id: int, save_every: int, ) -> None: self.gpu_id = gpu_id self.model = model.to(gpu_id) self.train_data = train_data self.optimizer = optimizer self.save_every = save_every self.model = DDP(model, device_ids=[gpu_id]) def _run_batch(self, source, targets): self.optimizer.zero_grad() output = self.model(source) loss = F.cross_entropy(output, targets) loss.backward() self.optimizer.step() def _run_epoch(self, epoch): b_sz = len(next(iter(self.train_data))[0]) print(f"[GPU{self.gpu_id}] Epoch {epoch} | Batchsize: {b_sz} | Steps: {len(self.train_data)}") self.train_data.sampler.set_epoch(epoch) for source, targets in self.train_data: source = source.to(self.gpu_id) targets = targets.to(self.gpu_id) self._run_batch(source, targets) def _save_checkpoint(self, epoch): ckp = self.model.module.state_dict() PATH = "checkpoint.pt" torch.save(ckp, PATH) print(f"Epoch {epoch} | Training checkpoint saved at {PATH}") def train(self, max_epochs: int): for epoch in range(max_epochs): self._run_epoch(epoch) if self.gpu_id == 0 and epoch % self.save_every == 0: self._save_checkpoint(epoch) def load_train_objs(): train_set = MyTrainDataset(2048) # load your dataset model = torch.nn.Linear(20, 1) # load your model optimizer = torch.optim.SGD(model.parameters(), lr=1e-3) return train_set, model, optimizer def prepare_dataloader(dataset: Dataset, batch_size: int): return DataLoader( dataset, batch_size=batch_size, pin_memory=True, shuffle=False, sampler=DistributedSampler(dataset) ) def main(rank: int, world_size: int, save_every: int, total_epochs: int, batch_size: int): ddp_setup(rank, world_size) dataset, model, optimizer = load_train_objs() train_data = prepare_dataloader(dataset, batch_size) trainer = Trainer(model, train_data, optimizer, rank, save_every) trainer.train(total_epochs) destroy_process_group() if __name__ == "__main__": import argparse parser = argparse.ArgumentParser(description='simple distributed training job') parser.add_argument('total_epochs', type=int, help='Total epochs to train the model') parser.add_argument('save_every', type=int, help='How often to save a snapshot') parser.add_argument('--batch_size', default=32, type=int, help='Input batch size on each device (default: 32)') args = parser.parse_args() world_size = torch.cuda.device_count() mp.spawn(main, args=(world_size, args.save_every, args.total_epochs, args.batch_size), nprocs=world_size) ================================================ FILE: distributed/ddp-tutorial-series/multigpu_torchrun.py ================================================ import torch import torch.nn.functional as F from torch.utils.data import Dataset, DataLoader from datautils import MyTrainDataset import torch.multiprocessing as mp from torch.utils.data.distributed import DistributedSampler from torch.nn.parallel import DistributedDataParallel as DDP from torch.distributed import init_process_group, destroy_process_group import os def ddp_setup(): torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) init_process_group(backend="nccl") class Trainer: def __init__( self, model: torch.nn.Module, train_data: DataLoader, optimizer: torch.optim.Optimizer, save_every: int, snapshot_path: str, ) -> None: self.gpu_id = int(os.environ["LOCAL_RANK"]) self.model = model.to(self.gpu_id) self.train_data = train_data self.optimizer = optimizer self.save_every = save_every self.epochs_run = 0 self.snapshot_path = snapshot_path if os.path.exists(snapshot_path): print("Loading snapshot") self._load_snapshot(snapshot_path) self.model = DDP(self.model, device_ids=[self.gpu_id]) def _load_snapshot(self, snapshot_path): loc = f"cuda:{self.gpu_id}" snapshot = torch.load(snapshot_path, map_location=loc) self.model.load_state_dict(snapshot["MODEL_STATE"]) self.epochs_run = snapshot["EPOCHS_RUN"] print(f"Resuming training from snapshot at Epoch {self.epochs_run}") def _run_batch(self, source, targets): self.optimizer.zero_grad() output = self.model(source) loss = F.cross_entropy(output, targets) loss.backward() self.optimizer.step() def _run_epoch(self, epoch): b_sz = len(next(iter(self.train_data))[0]) print(f"[GPU{self.gpu_id}] Epoch {epoch} | Batchsize: {b_sz} | Steps: {len(self.train_data)}") self.train_data.sampler.set_epoch(epoch) for source, targets in self.train_data: source = source.to(self.gpu_id) targets = targets.to(self.gpu_id) self._run_batch(source, targets) def _save_snapshot(self, epoch): snapshot = { "MODEL_STATE": self.model.module.state_dict(), "EPOCHS_RUN": epoch, } torch.save(snapshot, self.snapshot_path) print(f"Epoch {epoch} | Training snapshot saved at {self.snapshot_path}") def train(self, max_epochs: int): for epoch in range(self.epochs_run, max_epochs): self._run_epoch(epoch) if self.gpu_id == 0 and epoch % self.save_every == 0: self._save_snapshot(epoch) def load_train_objs(): train_set = MyTrainDataset(2048) # load your dataset model = torch.nn.Linear(20, 1) # load your model optimizer = torch.optim.SGD(model.parameters(), lr=1e-3) return train_set, model, optimizer def prepare_dataloader(dataset: Dataset, batch_size: int): return DataLoader( dataset, batch_size=batch_size, pin_memory=True, shuffle=False, sampler=DistributedSampler(dataset) ) def main(save_every: int, total_epochs: int, batch_size: int, snapshot_path: str = "snapshot.pt"): ddp_setup() dataset, model, optimizer = load_train_objs() train_data = prepare_dataloader(dataset, batch_size) trainer = Trainer(model, train_data, optimizer, save_every, snapshot_path) trainer.train(total_epochs) destroy_process_group() if __name__ == "__main__": import argparse parser = argparse.ArgumentParser(description='simple distributed training job') parser.add_argument('total_epochs', type=int, help='Total epochs to train the model') parser.add_argument('save_every', type=int, help='How often to save a snapshot') parser.add_argument('--batch_size', default=32, type=int, help='Input batch size on each device (default: 32)') args = parser.parse_args() main(args.save_every, args.total_epochs, args.batch_size) ================================================ FILE: distributed/ddp-tutorial-series/multinode.py ================================================ import torch import torch.nn.functional as F from torch.utils.data import Dataset, DataLoader from datautils import MyTrainDataset import torch.multiprocessing as mp from torch.utils.data.distributed import DistributedSampler from torch.nn.parallel import DistributedDataParallel as DDP from torch.distributed import init_process_group, destroy_process_group import os def ddp_setup(): torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) init_process_group(backend="nccl") class Trainer: def __init__( self, model: torch.nn.Module, train_data: DataLoader, optimizer: torch.optim.Optimizer, save_every: int, snapshot_path: str, ) -> None: self.local_rank = int(os.environ["LOCAL_RANK"]) self.global_rank = int(os.environ["RANK"]) self.model = model.to(self.local_rank) self.train_data = train_data self.optimizer = optimizer self.save_every = save_every self.epochs_run = 0 self.snapshot_path = snapshot_path if os.path.exists(snapshot_path): print("Loading snapshot") self._load_snapshot(snapshot_path) self.model = DDP(self.model, device_ids=[self.local_rank]) def _load_snapshot(self, snapshot_path): loc = f"cuda:{self.local_rank}" snapshot = torch.load(snapshot_path, map_location=loc) self.model.load_state_dict(snapshot["MODEL_STATE"]) self.epochs_run = snapshot["EPOCHS_RUN"] print(f"Resuming training from snapshot at Epoch {self.epochs_run}") def _run_batch(self, source, targets): self.optimizer.zero_grad() output = self.model(source) loss = F.cross_entropy(output, targets) loss.backward() self.optimizer.step() def _run_epoch(self, epoch): b_sz = len(next(iter(self.train_data))[0]) print(f"[GPU{self.global_rank}] Epoch {epoch} | Batchsize: {b_sz} | Steps: {len(self.train_data)}") self.train_data.sampler.set_epoch(epoch) for source, targets in self.train_data: source = source.to(self.local_rank) targets = targets.to(self.local_rank) self._run_batch(source, targets) def _save_snapshot(self, epoch): snapshot = { "MODEL_STATE": self.model.module.state_dict(), "EPOCHS_RUN": epoch, } torch.save(snapshot, self.snapshot_path) print(f"Epoch {epoch} | Training snapshot saved at {self.snapshot_path}") def train(self, max_epochs: int): for epoch in range(self.epochs_run, max_epochs): self._run_epoch(epoch) if self.global_rank == 0 and epoch % self.save_every == 0: self._save_snapshot(epoch) def load_train_objs(): train_set = MyTrainDataset(2048) # load your dataset model = torch.nn.Linear(20, 1) # load your model optimizer = torch.optim.SGD(model.parameters(), lr=1e-3) return train_set, model, optimizer def prepare_dataloader(dataset: Dataset, batch_size: int): return DataLoader( dataset, batch_size=batch_size, pin_memory=True, shuffle=False, sampler=DistributedSampler(dataset) ) def main(save_every: int, total_epochs: int, batch_size: int, snapshot_path: str = "snapshot.pt"): ddp_setup() dataset, model, optimizer = load_train_objs() train_data = prepare_dataloader(dataset, batch_size) trainer = Trainer(model, train_data, optimizer, save_every, snapshot_path) trainer.train(total_epochs) destroy_process_group() if __name__ == "__main__": import argparse parser = argparse.ArgumentParser(description='simple distributed training job') parser.add_argument('total_epochs', type=int, help='Total epochs to train the model') parser.add_argument('save_every', type=int, help='How often to save a snapshot') parser.add_argument('--batch_size', default=32, type=int, help='Input batch size on each device (default: 32)') args = parser.parse_args() main(args.save_every, args.total_epochs, args.batch_size) ================================================ FILE: distributed/ddp-tutorial-series/requirements.txt ================================================ torch>=1.11.0 ================================================ FILE: distributed/ddp-tutorial-series/single_gpu.py ================================================ import torch import torch.nn.functional as F from torch.utils.data import Dataset, DataLoader from datautils import MyTrainDataset class Trainer: def __init__( self, model: torch.nn.Module, train_data: DataLoader, optimizer: torch.optim.Optimizer, gpu_id: int, save_every: int, ) -> None: self.gpu_id = gpu_id self.model = model.to(gpu_id) self.train_data = train_data self.optimizer = optimizer self.save_every = save_every def _run_batch(self, source, targets): self.optimizer.zero_grad() output = self.model(source) loss = F.cross_entropy(output, targets) loss.backward() self.optimizer.step() def _run_epoch(self, epoch): b_sz = len(next(iter(self.train_data))[0]) print(f"[GPU{self.gpu_id}] Epoch {epoch} | Batchsize: {b_sz} | Steps: {len(self.train_data)}") for source, targets in self.train_data: source = source.to(self.gpu_id) targets = targets.to(self.gpu_id) self._run_batch(source, targets) def _save_checkpoint(self, epoch): ckp = self.model.state_dict() PATH = "checkpoint.pt" torch.save(ckp, PATH) print(f"Epoch {epoch} | Training checkpoint saved at {PATH}") def train(self, max_epochs: int): for epoch in range(max_epochs): self._run_epoch(epoch) if epoch % self.save_every == 0: self._save_checkpoint(epoch) def load_train_objs(): train_set = MyTrainDataset(2048) # load your dataset model = torch.nn.Linear(20, 1) # load your model optimizer = torch.optim.SGD(model.parameters(), lr=1e-3) return train_set, model, optimizer def prepare_dataloader(dataset: Dataset, batch_size: int): return DataLoader( dataset, batch_size=batch_size, pin_memory=True, shuffle=True ) def main(device, total_epochs, save_every, batch_size): dataset, model, optimizer = load_train_objs() train_data = prepare_dataloader(dataset, batch_size) trainer = Trainer(model, train_data, optimizer, device, save_every) trainer.train(total_epochs) if __name__ == "__main__": import argparse parser = argparse.ArgumentParser(description='simple distributed training job') parser.add_argument('total_epochs', type=int, help='Total epochs to train the model') parser.add_argument('save_every', type=int, help='How often to save a snapshot') parser.add_argument('--batch_size', default=32, type=int, help='Input batch size on each device (default: 32)') args = parser.parse_args() device = 0 # shorthand for cuda:0 main(device, args.total_epochs, args.save_every, args.batch_size) ================================================ FILE: distributed/ddp-tutorial-series/slurm/config.yaml.template ================================================ Region: us-east-1 Image: Os: ubuntu1804 SharedStorage: - MountDir: /shared Name: shared-fs StorageType: FsxLustre FsxLustreSettings: StorageCapacity: 1200 DeploymentType: SCRATCH_1 StorageType: SSD HeadNode: InstanceType: c5.xlarge Networking: SubnetId: subnet-xxxxxxx Ssh: KeyName: your-keyname-file Scheduling: Scheduler: slurm SlurmQueues: - Name: train ComputeResources: - Name: p32xlarge InstanceType: p3.2xlarge MinCount: 0 MaxCount: 5 Networking: SubnetIds: - subnet-xxxxxxx ================================================ FILE: distributed/ddp-tutorial-series/slurm/sbatch_run.sh ================================================ #!/bin/bash #SBATCH --job-name=multinode-example #SBATCH --nodes=4 #SBATCH --ntasks=4 #SBATCH --gpus-per-task=1 #SBATCH --cpus-per-task=4 nodes=( $( scontrol show hostnames $SLURM_JOB_NODELIST ) ) nodes_array=($nodes) head_node=${nodes_array[0]} head_node_ip=$(srun --nodes=1 --ntasks=1 -w "$head_node" hostname --ip-address) echo Node IP: $head_node_ip export LOGLEVEL=INFO srun torchrun \ --nnodes 4 \ --nproc_per_node 1 \ --rdzv_id $RANDOM \ --rdzv_backend c10d \ --rdzv_endpoint $head_node_ip:29500 \ /shared/examples/multinode_torchrun.py 50 10 ================================================ FILE: distributed/ddp-tutorial-series/slurm/setup_pcluster_slurm.md ================================================ # Setup AWS cluster with pcluster ## 1. Sign in to an AWS instance ## 2. Install pcluster ``` pip3 install awscli -U --user pip3 install "aws-parallelcluster" --upgrade --user ``` ## 3. Create a cluster config file ``` pcluster configure --config config.yaml ``` See config.yaml.template for an example ## 4. Create the cluster ``` pcluster create-cluster --cluster-name dist-ml --cluster-configuration config.yaml ``` ### 4a. Track progress ``` pcluster list-clusters ``` ## 5. Login to cluster headnode ``` pcluster ssh --cluster-name dist-ml -i your-keyname-file ``` ## 6. Install dependencies ``` sudo apt-get update sudo apt-get install -y python3-venv python3 -m venv /shared/venv/ source /shared/venv/bin/activate pip install wheel echo 'source /shared/venv/bin/activate' >> ~/.bashrc ``` ## 7. Download training code and install requirements ``` cd /shared git clone --depth 1 https://github.com/pytorch/examples; cd /shared/examples git filter-branch --prune-empty --subdirectory-filter distributed/ddp-tutorial-series python3 -m pip install setuptools==59.5.0 pip install -r requirements.txt ``` ================================================ FILE: distributed/minGPT-ddp/README.md ================================================ # minGPT-DDP Code accompanying the tutorial at https://pytorch.org/tutorials/intermediate/ddp_series_minGPT.html for training a GPT-like model with Distributed Data Parallel (DDP) in PyTorch. Files marked with an asterisk (*) are adapted from the minGPT repo (https://github.com/karpathy/minGPT). - [trainer.py](mingpt/trainer.py) includes the Trainer class that runs the distributed training iterations on the model with the provided dataset. - [model.py *](mingpt/model.py) defines the model architecture. - [char_dataset.py *](mingpt/char_dataset.py) contains the `Dataset`class for a character-level dataset. - [gpt2_train_cfg.yaml](mingpt/gpt2_train_cfg.yaml) contains the configurations for data, model, optimizer and training run. - [main.py](mingpt/main.py) is the entry point to the trainig job. It sets up the DDP process group, reads all the configurations and runs the training job. - [slurm/](mingpt/slurm) contains files for setting up an AWS cluster and the slurm script to run multinode training. ================================================ FILE: distributed/minGPT-ddp/mingpt/char_dataset.py ================================================ import torch from torch.utils.data import Dataset import fsspec from dataclasses import dataclass """ Adapted from https://github.com/karpathy/minGPT/blob/master/projects/chargpt/chargpt.py """ @dataclass class DataConfig: path: str = None block_size: int = None train_split: float = None truncate: float = 1.0 class CharDataset(Dataset): def __init__(self, data_cfg: DataConfig): #data_path: str, block_size): data = fsspec.open(data_cfg.path).open().read().decode('utf-8') data = data[ : int(len(data) * data_cfg.truncate)] chars = sorted(list(set(data))) data_size, vocab_size = len(data), len(chars) print('Data has %d characters, %d unique.' % (data_size, vocab_size)) self.stoi = {ch: i for i, ch in enumerate(chars)} self.itos = {i: ch for i, ch in enumerate(chars)} self.block_size = data_cfg.block_size self.vocab_size = vocab_size self.data = data def __len__(self): return len(self.data) - self.block_size def __getitem__(self, idx): # grab a chunk of (block_size + 1) characters from the data chunk = self.data[idx:idx + self.block_size + 1] # encode every character to an integer dix = [self.stoi[s] for s in chunk] x = torch.tensor(dix[:-1], dtype=torch.long) y = torch.tensor(dix[1:], dtype=torch.long) return x, y ================================================ FILE: distributed/minGPT-ddp/mingpt/gpt2_train_cfg.yaml ================================================ data_config: path: https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt block_size: 128 train_split: 0.9 truncate: 0.05 gpt_config: n_layer: 8 n_head: 8 n_embd: 512 trainer_config: max_epochs: 10 batch_size: 216 data_loader_workers: 4 grad_norm_clip: 1.0 snapshot_path: gpt_snapshot.pt save_every: 3 use_amp: True optimizer_config: weight_decay: 0.1 learning_rate: 0.0003 hydra: run: dir: ./ ================================================ FILE: distributed/minGPT-ddp/mingpt/main.py ================================================ import os import sys import torch from torch.utils.data import random_split from torch.distributed import init_process_group, destroy_process_group from model import GPT, GPTConfig, OptimizerConfig, create_optimizer from trainer import Trainer, TrainerConfig from char_dataset import CharDataset, DataConfig from omegaconf import DictConfig import hydra def verify_min_gpu_count(min_gpus: int = 2) -> bool: has_gpu = torch.accelerator.is_available() gpu_count = torch.accelerator.device_count() return has_gpu and gpu_count >= min_gpus def ddp_setup(): acc = torch.accelerator.current_accelerator() rank = int(os.environ["LOCAL_RANK"]) device: torch.device = torch.device(f"{acc}:{rank}") backend = torch.distributed.get_default_backend_for_device(device) init_process_group(backend=backend) torch.accelerator.set_device_index(rank) def get_train_objs(gpt_cfg: GPTConfig, opt_cfg: OptimizerConfig, data_cfg: DataConfig): dataset = CharDataset(data_cfg) train_len = int(len(dataset) * data_cfg.train_split) train_set, test_set = random_split(dataset, [train_len, len(dataset) - train_len]) gpt_cfg.vocab_size = dataset.vocab_size gpt_cfg.block_size = dataset.block_size model = GPT(gpt_cfg) optimizer = create_optimizer(model, opt_cfg) return model, optimizer, train_set, test_set @hydra.main(version_base=None, config_path=".", config_name="gpt2_train_cfg") def main(cfg: DictConfig): ddp_setup() gpt_cfg = GPTConfig(**cfg['gpt_config']) opt_cfg = OptimizerConfig(**cfg['optimizer_config']) data_cfg = DataConfig(**cfg['data_config']) trainer_cfg = TrainerConfig(**cfg['trainer_config']) model, optimizer, train_data, test_data = get_train_objs(gpt_cfg, opt_cfg, data_cfg) trainer = Trainer(trainer_cfg, model, optimizer, train_data, test_data) trainer.train() destroy_process_group() if __name__ == "__main__": _min_gpu_count = 2 if not verify_min_gpu_count(min_gpus=_min_gpu_count): print(f"Unable to locate sufficient {_min_gpu_count} gpus to run this example. Exiting.") sys.exit() main() ================================================ FILE: distributed/minGPT-ddp/mingpt/model.py ================================================ """ Full definition of a GPT Language Model, all of it in this single file. Adapted from https://github.com/karpathy/minGPT/blob/master/mingpt/model.py """ from dataclasses import dataclass import math import torch import torch.nn as nn from torch.nn import functional as F @dataclass class GPTConfig: model_type: str = 'gpt2' # model configurations n_layer: int = None n_head: int = None n_embd: int = None # openai's values for gpt2 vocab_size: int = 50257 block_size: int = 1024 # dropout hyperparameters embd_pdrop: float = 0.1 resid_pdrop: float = 0.1 attn_pdrop: float = 0.1 @dataclass class OptimizerConfig: learning_rate: float = 3e-4 weight_decay: float = 0.1 class MultiheadAttentionLayer(nn.Module): """ A multi-head masked self-attention layer with a projection at the end. """ def __init__(self, config, device="cpu", dtype=torch.float32): super().__init__() assert config.n_embd % config.n_head == 0 self.resid_drop = nn.Dropout(config.resid_pdrop) self.c_proj = nn.Linear(config.n_embd, config.n_embd, device=device, dtype=dtype) self.register_buffer("mask", torch.tril(torch.ones(config.block_size, config.block_size)) .view(1, 1, config.block_size, config.block_size)) self.attn = torch.nn.MultiheadAttention( embed_dim=config.n_embd, num_heads=config.n_head, dropout=config.attn_pdrop, batch_first=True, device=device, dtype=dtype ) def forward(self, x): _, seq_size, _ = x.size() y = self.attn(x, x, x, attn_mask=self.mask[0, 0, :seq_size, :seq_size])[0] y = self.resid_drop(self.c_proj(y)) return y class Block(nn.Module): """ an unassuming Transformer block """ def __init__(self, config: GPTConfig): super().__init__() self.ln1 = nn.LayerNorm(config.n_embd) self.ln2 = nn.LayerNorm(config.n_embd) self.attn = MultiheadAttentionLayer(config) self.mlp = nn.Sequential( nn.Linear(config.n_embd, 4 * config.n_embd), nn.GELU(), nn.Linear(4 * config.n_embd, config.n_embd), nn.Dropout(config.resid_pdrop), ) def forward(self, x): x = x + self.attn(self.ln1(x)) x = x + self.mlp(self.ln2(x)) return x class EmbeddingStem(nn.Module): def __init__(self, config: GPTConfig, device="cpu", dtype=torch.float32): super().__init__() self.tok_emb = nn.Embedding(config.vocab_size, config.n_embd, device=device, dtype=dtype) self.pos_emb = nn.Parameter(torch.zeros(1, config.block_size, config.n_embd, device=device, dtype=dtype)) self.drop = nn.Dropout(config.embd_pdrop) self.block_size = config.block_size def reset_parameters(self): self.tok_emb.reset_parameters() def forward(self, idx): b, t = idx.size() assert t <= self.block_size, f"Cannot forward sequence of length {t}, block size is only {self.block_size}" token_embeddings = self.tok_emb(idx) # each index maps to a (learnable) embedding vector position_embeddings = self.pos_emb[:, :t, :] # each position maps to a (learnable) position vector return self.drop(token_embeddings + position_embeddings) class GPT(nn.Module): """ GPT Language Model """ def __init__(self, config: GPTConfig): super().__init__() self.block_size = config.block_size config = self._set_model_config(config) # input embedding stem self.emb_stem = EmbeddingStem(config) # transformer self.blocks = nn.Sequential(*[Block(config) for _ in range(config.n_layer)]) # decoder head self.ln_f = nn.LayerNorm(config.n_embd) self.head = nn.Linear(config.n_embd, config.vocab_size, bias=False) # init all weights, and apply a special scaled init to the residual projections, per GPT-2 paper self.apply(self._init_weights) for pn, p in self.named_parameters(): if pn.endswith('c_proj.weight'): p.data.normal_(mean=0.0, std=0.02/math.sqrt(2 * config.n_layer)) # report number of parameters (note we don't count the decoder parameters in lm_head) n_params = sum(p.numel() for p in self.blocks.parameters()) print("number of parameters: %.2fM" % (n_params/1e6,)) def _set_model_config(self, config): type_given = config.model_type is not None params_given = all([config.n_layer is not None, config.n_head is not None, config.n_embd is not None]) # assert type_given ^ params_given # exactly one of these (XOR) if type_given and not params_given: # translate from model_type to detailed configuration config.__dict__.update({ # names follow the huggingface naming conventions # GPT-1 'openai-gpt': dict(n_layer=12, n_head=12, n_embd=768), # 117M params # GPT-2 configs 'gpt2': dict(n_layer=12, n_head=12, n_embd=768), # 124M params 'gpt2-medium': dict(n_layer=24, n_head=16, n_embd=1024), # 350M params 'gpt2-large': dict(n_layer=36, n_head=20, n_embd=1280), # 774M params 'gpt2-xl': dict(n_layer=48, n_head=25, n_embd=1600), # 1558M params # Gophers 'gopher-44m': dict(n_layer=8, n_head=16, n_embd=512), # (there are a number more...) # I made these tiny models up 'gpt-mini': dict(n_layer=6, n_head=6, n_embd=192), 'gpt-micro': dict(n_layer=4, n_head=4, n_embd=128), 'gpt-nano': dict(n_layer=3, n_head=3, n_embd=48), }[config.model_type]) return config def _init_weights(self, module): if isinstance(module, (nn.Linear, nn.Embedding)): module.weight.data.normal_(mean=0.0, std=0.02) if isinstance(module, nn.Linear) and module.bias is not None: module.bias.data.zero_() elif isinstance(module, nn.LayerNorm): module.bias.data.zero_() module.weight.data.fill_(1.0) def forward(self, idx, targets=None): x = self.emb_stem(idx) x = self.blocks(x) x = self.ln_f(x) logits = self.head(x) # if we are given some desired targets also calculate the loss loss = None if targets is not None: loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1) return logits, loss @torch.no_grad() def generate(self, idx, max_new_tokens, temperature=1.0, do_sample=False, top_k=None): """ Take a conditioning sequence of indices idx (LongTensor of shape (b,t)) and complete the sequence max_new_tokens times, feeding the predictions back into the model each time. Most likely you'll want to make sure to be in model.eval() mode of operation for this. """ for _ in range(max_new_tokens): # if the sequence context is growing too long we must crop it at block_size idx_cond = idx if idx.size(1) <= self.block_size else idx[:, -self.block_size:] # forward the model to get the logits for the index in the sequence logits, _ = self(idx_cond) # pluck the logits at the final step and scale by desired temperature logits = logits[:, -1, :] / temperature # optionally crop the logits to only the top k options if top_k is not None: v, _ = torch.topk(logits, top_k) logits[logits < v[:, [-1]]] = -float('Inf') # apply softmax to convert logits to (normalized) probabilities probs = F.softmax(logits, dim=-1) # either sample from the distribution or take the most likely element if do_sample: idx_next = torch.multinomial(probs, num_samples=1) else: _, idx_next = torch.topk(probs, k=1, dim=-1) # append sampled index to the running sequence and continue idx = torch.cat((idx, idx_next), dim=1) return idx def create_optimizer(model: torch.nn.Module, opt_config: OptimizerConfig): """ This long function is unfortunately doing something very simple and is being very defensive: We are separating out all parameters of the model into two buckets: those that will experience weight decay for regularization and those that won't (biases, and layernorm/embedding weights). We are then returning the PyTorch optimizer object. """ # separate out all parameters to those that will and won't experience regularizing weight decay decay = set() no_decay = set() whitelist_weight_modules = (torch.nn.Linear, ) blacklist_weight_modules = (torch.nn.LayerNorm, torch.nn.Embedding) for mn, m in model.named_modules(): for pn, p in m.named_parameters(): fpn = '%s.%s' % (mn, pn) if mn else pn # full param name # random note: because named_modules and named_parameters are recursive # we will see the same tensors p many many times. but doing it this way # allows us to know which parent module any tensor p belongs to... if pn.endswith('bias'): # all biases will not be decayed no_decay.add(fpn) elif pn.endswith('weight') and isinstance(m, whitelist_weight_modules): # weights of whitelist modules will be weight decayed decay.add(fpn) elif pn.endswith('in_proj_weight'): # MHA projection layer decay.add(fpn) elif pn.endswith('weight') and isinstance(m, blacklist_weight_modules): # weights of blacklist modules will NOT be weight decayed no_decay.add(fpn) elif pn.endswith('pos_emb'): # positional embedding shouldn't be decayed no_decay.add(fpn) # validate that we considered every parameter param_dict = {pn: p for pn, p in model.named_parameters()} inter_params = decay & no_decay union_params = decay | no_decay assert len(inter_params) == 0, "parameters %s made it into both decay/no_decay sets!" % (str(inter_params), ) assert len(param_dict.keys() - union_params) == 0, "parameters %s were not separated into either decay/no_decay set!" \ % (str(param_dict.keys() - union_params), ) # create the pytorch optimizer object optim_groups = [ {"params": [param_dict[pn] for pn in sorted(list(decay))], "weight_decay": opt_config.weight_decay}, {"params": [param_dict[pn] for pn in sorted(list(no_decay))], "weight_decay": 0.0}, ] optimizer = torch.optim.AdamW(optim_groups, lr=opt_config.learning_rate, betas=(0.9, 0.95)) return optimizer ================================================ FILE: distributed/minGPT-ddp/mingpt/slurm/config.yaml.template ================================================ Region: us-east-1 Image: Os: ubuntu1804 SharedStorage: - MountDir: /shared Name: shared-fs StorageType: FsxLustre FsxLustreSettings: StorageCapacity: 1200 DeploymentType: SCRATCH_1 StorageType: SSD HeadNode: InstanceType: c5.xlarge Networking: SubnetId: subnet-xxxxxxx Ssh: KeyName: your-keyname-file Scheduling: Scheduler: slurm SlurmQueues: - Name: train ComputeResources: - Name: p32xlarge InstanceType: p3.2xlarge MinCount: 0 MaxCount: 5 Networking: SubnetIds: - subnet-xxxxxxx ================================================ FILE: distributed/minGPT-ddp/mingpt/slurm/sbatch_run.sh ================================================ #!/bin/bash #SBATCH --job-name=multinode-example #SBATCH --nodes=2 #SBATCH --ntasks=2 #SBATCH --gpus-per-task=1 #SBATCH --cpus-per-task=4 nodes=( $( scontrol show hostnames $SLURM_JOB_NODELIST ) ) nodes_array=($nodes) head_node=${nodes_array[0]} head_node_ip=$(srun --nodes=1 --ntasks=1 -w "$head_node" hostname --ip-address) echo Node IP: $head_node_ip export LOGLEVEL=INFO srun torchrun \ --nnodes 2 \ --nproc_per_node 1 \ --rdzv_id $RANDOM \ --rdzv_backend c10d \ --rdzv_endpoint $head_node_ip:29500 \ /shared/examples/mingpt/main.py ================================================ FILE: distributed/minGPT-ddp/mingpt/slurm/setup_pcluster_slurm.md ================================================ # Setup AWS cluster with pcluster Refer https://www.hpcworkshops.com/04-pcluster-cli.html ## 1. Sign in to an AWS instance ## 2. Install pcluster ``` pip3 install awscli -U --user pip3 install "aws-parallelcluster" --upgrade --user ``` ## 3. Create a cluster config file ``` pcluster configure --config config.yaml ``` See config.yaml.template for an example. Ensure you have a valid EC2 key-pair file ## 4. Create the cluster ``` pcluster create-cluster --cluster-name dist-ml --cluster-configuration config.yaml ``` ### 4a. Track progress ``` pcluster list-clusters ``` ## 5. Login to cluster headnode ``` pcluster ssh --cluster-name dist-ml -i your-keypair-file ``` ## 6. Install dependencies ``` sudo apt-get update sudo apt-get install -y python3-venv python3 -m venv /shared/venv/ source /shared/venv/bin/activate pip install wheel echo 'source /shared/venv/bin/activate' >> ~/.bashrc ``` ## 7. Download training code and install requirements ``` cd /shared git clone --depth 1 https://github.com/pytorch/examples; cd /shared/examples git filter-branch --prune-empty --subdirectory-filter distributed/minGPT-ddp python3 -m pip install setuptools==59.5.0 pip install -r requirements.txt ``` ================================================ FILE: distributed/minGPT-ddp/mingpt/trainer.py ================================================ """ Simple training loop; Boilerplate that could apply to any arbitrary neural network, so nothing in this file really has anything to do with GPT specifically. """ from dataclasses import dataclass, asdict from collections import OrderedDict from typing import Optional, Any, Dict import os import torch from torch.utils.data import Dataset, DataLoader from torch.nn.parallel import DistributedDataParallel as DDP from torch.utils.data.distributed import DistributedSampler import boto3 from urllib.parse import urlparse import fsspec import io @dataclass class TrainerConfig: max_epochs: int = None batch_size: int = None data_loader_workers: int = None grad_norm_clip: float = None snapshot_path: Optional[str] = None save_every: int = None use_amp: bool = None @dataclass class Snapshot: model_state: 'OrderedDict[str, torch.Tensor]' optimizer_state: Dict[str, Any] finished_epoch: int def upload_to_s3(obj, dst): buffer = io.BytesIO() torch.save(obj, buffer) buffer.seek(0) dst = urlparse(dst, allow_fragments=False) boto3.client('s3').upload_fileobj(buffer, dst.netloc, dst.path.lstrip('/')) class Trainer: def __init__(self, trainer_config: TrainerConfig, model, optimizer, train_dataset, test_dataset=None): self.config = trainer_config # set torchrun variables self.local_rank = int(os.environ["LOCAL_RANK"]) self.global_rank = int(os.environ["RANK"]) # set device self.acc = torch.accelerator.current_accelerator() self.device: torch.device = torch.device(f"{self.acc}:{self.local_rank}") self.device_type = self.device.type # data stuff self.train_dataset = train_dataset self.train_loader = self._prepare_dataloader(train_dataset) self.test_loader = self._prepare_dataloader(test_dataset) if test_dataset else None # initialize train states self.epochs_run = 0 self.model = model.to(self.local_rank) self.optimizer = optimizer self.save_every = self.config.save_every if self.config.use_amp: self.scaler = torch.amp.GradScaler(self.device_type) # load snapshot if available. only necessary on the first node. if self.config.snapshot_path is None: self.config.snapshot_path = "snapshot.pt" self._load_snapshot() # wrap with DDP. this step will synch model across all the processes. self.model = DDP(self.model, device_ids=[self.local_rank]) def _prepare_dataloader(self, dataset: Dataset): return DataLoader( dataset, batch_size=self.config.batch_size, pin_memory=True, shuffle=False, num_workers=self.config.data_loader_workers, sampler=DistributedSampler(dataset) ) def _load_snapshot(self): try: snapshot = fsspec.open(self.config.snapshot_path) with snapshot as f: snapshot_data = torch.load(f, map_location="cpu") except FileNotFoundError: print("Snapshot not found. Training model from scratch") return snapshot = Snapshot(**snapshot_data) self.model.load_state_dict(snapshot.model_state) self.optimizer.load_state_dict(snapshot.optimizer_state) self.epochs_run = snapshot.finished_epoch print(f"Resuming training from snapshot at Epoch {self.epochs_run}") def _run_batch(self, source, targets, train: bool = True) -> float: with torch.set_grad_enabled(train), torch.amp.autocast(device_type=self.device_type, dtype=torch.float16, enabled=(self.config.use_amp)): _, loss = self.model(source, targets) if train: self.optimizer.zero_grad(set_to_none=True) if self.config.use_amp: self.scaler.scale(loss).backward() torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.config.grad_norm_clip) self.scaler.step(self.optimizer) self.scaler.update() else: loss.backward() torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.config.grad_norm_clip) self.optimizer.step() return loss.item() def _run_epoch(self, epoch: int, dataloader: DataLoader, train: bool = True): if train: dataloader.sampler.set_epoch(epoch) for iter, (source, targets) in enumerate(dataloader): step_type = "Train" if train else "Eval" source = source.to(self.local_rank) targets = targets.to(self.local_rank) batch_loss = self._run_batch(source, targets, train) if iter % 100 == 0: print(f"[RANK{self.global_rank}] Epoch {epoch} | Iter {iter} | {step_type} Loss {batch_loss:.5f}") def _save_snapshot(self, epoch): # capture snapshot model = self.model raw_model = model.module if hasattr(model, "module") else model snapshot = Snapshot( model_state=raw_model.state_dict(), optimizer_state=self.optimizer.state_dict(), finished_epoch=epoch ) # save snapshot snapshot = asdict(snapshot) if self.config.snapshot_path.startswith("s3://"): upload_to_s3(snapshot, self.config.snapshot_path) else: torch.save(snapshot, self.config.snapshot_path) print(f"Snapshot saved at epoch {epoch}") def train(self): for epoch in range(self.epochs_run, self.config.max_epochs): epoch += 1 self._run_epoch(epoch, self.train_loader, train=True) if self.local_rank == 0 and epoch % self.save_every == 0: self._save_snapshot(epoch) # eval run if self.test_loader: self._run_epoch(epoch, self.test_loader, train=False) ================================================ FILE: distributed/minGPT-ddp/requirements.txt ================================================ torch>=2.7 boto3 hydra-core requests aiohttp ================================================ FILE: distributed/minGPT-ddp/run_example.sh ================================================ #!/bin/bash # Usage: bash run_example.sh {file_to_run.py} {num_gpus} # where file_to_run = example to run. Default = 'main.py' # num_gpus = num local gpus to use. Default = 16 # samples to run include: # main.py echo "Launching ${1:-main.py} with ${2:-16} gpus" torchrun --standalone --nproc_per_node=${2:-16} ${1:-main.py} ================================================ FILE: distributed/rpc/batch/README.md ================================================ # Examples For Asynchronous RPC User Functions This folder contains two examples for [`@rpc.functions.async_execution`](https://pytorch.org/docs/master/rpc.html#torch.distributed.rpc.functions.async_execution): 1. Synchronized Batch Update Parameter Server: uses `@rpc.functions.async_execution` for parameter update and retrieving. This serves as a simple starter example for batch RPC. ``` pip install -r requirements.txt python parameter_server.py ``` 2. Multi-Observer with Batch-Processing Agent: uses `@rpc.functions.async_execution` to run multiple observed states through the policy to get actions. ``` pip install -r requirements.txt python reinforce.py ``` ================================================ FILE: distributed/rpc/batch/parameter_server.py ================================================ import os import threading from datetime import datetime import warnings import torch import torch.distributed as dist import torch.distributed.rpc as rpc import torch.multiprocessing as mp import torch.nn as nn from torch import optim import torchvision # Suppress deprecated ProcessGroup warning warnings.filterwarnings("ignore", message="You are using a Backend.*ProcessGroup") batch_size = 20 image_w = 64 image_h = 64 num_classes = 30 batch_update_size = 5 num_batches = 6 def timed_log(text): print(f"{datetime.now().strftime('%H:%M:%S')} {text}") class BatchUpdateParameterServer(object): def __init__(self, batch_update_size=batch_update_size): self.model = torchvision.models.resnet50(num_classes=num_classes) self.lock = threading.Lock() self.future_model = torch.futures.Future() self.batch_update_size = batch_update_size self.curr_update_size = 0 self.optimizer = optim.SGD(self.model.parameters(), lr=0.001, momentum=0.9) for p in self.model.parameters(): p.grad = torch.zeros_like(p) def get_model(self): return self.model @staticmethod @rpc.functions.async_execution def update_and_fetch_model(ps_rref, grads): self = ps_rref.local_value() timed_log(f"PS got {self.curr_update_size}/{batch_update_size} updates") for p, g in zip(self.model.parameters(), grads): p.grad += g with self.lock: self.curr_update_size += 1 fut = self.future_model if self.curr_update_size >= self.batch_update_size: for p in self.model.parameters(): p.grad /= self.batch_update_size self.curr_update_size = 0 self.optimizer.step() self.optimizer.zero_grad(set_to_none=False) fut.set_result(self.model) timed_log("PS updated model") self.future_model = torch.futures.Future() return fut class Trainer(object): def __init__(self, ps_rref): self.ps_rref = ps_rref self.loss_fn = nn.MSELoss() self.one_hot_indices = torch.LongTensor(batch_size) \ .random_(0, num_classes) \ .view(batch_size, 1) def get_next_batch(self): for _ in range(num_batches): inputs = torch.randn(batch_size, 3, image_w, image_h) labels = torch.zeros(batch_size, num_classes) \ .scatter_(1, self.one_hot_indices, 1) yield inputs.cuda(), labels.cuda() def train(self): name = rpc.get_worker_info().name m = self.ps_rref.rpc_sync().get_model().cuda() for inputs, labels in self.get_next_batch(): timed_log(f"{name} processing one batch") self.loss_fn(m(inputs), labels).backward() timed_log(f"{name} reporting grads") m = rpc.rpc_sync( self.ps_rref.owner(), BatchUpdateParameterServer.update_and_fetch_model, args=(self.ps_rref, [p.grad for p in m.cpu().parameters()]), ).cuda() timed_log(f"{name} got updated model") def run_trainer(ps_rref): trainer = Trainer(ps_rref) trainer.train() def run_ps(trainers): timed_log("Start training") ps_rref = rpc.RRef(BatchUpdateParameterServer()) futs = [] for trainer in trainers: futs.append( rpc.rpc_async(trainer, run_trainer, args=(ps_rref,)) ) torch.futures.wait_all(futs) timed_log("Finish training") def run(rank, world_size): os.environ['MASTER_ADDR'] = 'localhost' os.environ['MASTER_PORT'] = '29500' # Initialize the process group first dist.init_process_group( backend="gloo", rank=rank, world_size=world_size ) options=rpc.TensorPipeRpcBackendOptions( num_worker_threads=16, rpc_timeout=60 ) if rank != 0: rpc.init_rpc( f"trainer{rank}", rank=rank, world_size=world_size, rpc_backend_options=options ) # trainer passively waiting for ps to kick off training iterations else: rpc.init_rpc( "ps", rank=rank, world_size=world_size, rpc_backend_options=options ) run_ps([f"trainer{r}" for r in range(1, world_size)]) # block until all rpcs finish rpc.shutdown() dist.destroy_process_group() if __name__=="__main__": world_size = batch_update_size + 1 mp.spawn(run, args=(world_size, ), nprocs=world_size, join=True) ================================================ FILE: distributed/rpc/batch/reinforce.py ================================================ import argparse import gymnasium as gym import os import threading import time import warnings # Suppress deprecated ProcessGroup warning warnings.filterwarnings("ignore", message="You are using a Backend.*ProcessGroup") import torch import torch.distributed.rpc as rpc import torch.multiprocessing as mp import torch.nn as nn import torch.nn.functional as F import torch.optim as optim from torch.distributed.rpc import RRef, rpc_sync, rpc_async, remote from torch.distributions import Categorical # demonstrating using rpc.functions.async_execution to speed up training NUM_STEPS = 500 AGENT_NAME = "agent" OBSERVER_NAME = "observer{}" parser = argparse.ArgumentParser(description='PyTorch RPC Batch RL example') parser.add_argument('--gamma', type=float, default=1.0, metavar='G', help='discount factor (default: 1.0)') parser.add_argument('--seed', type=int, default=543, metavar='N', help='random seed (default: 543)') parser.add_argument('--num-episode', type=int, default=10, metavar='E', help='number of episodes (default: 10)') parser.add_argument('--max-world-size', type=int, default=3, metavar='W', help='maximum world size to test (default: 3)') args = parser.parse_args() torch.manual_seed(args.seed) class Policy(nn.Module): r""" Borrowing the ``Policy`` class from the Reinforcement Learning example. Copying the code to make these two examples independent. See https://github.com/pytorch/examples/tree/main/reinforcement_learning """ def __init__(self, batch=True): super(Policy, self).__init__() self.affine1 = nn.Linear(4, 128) self.dropout = nn.Dropout(p=0.6) self.affine2 = nn.Linear(128, 2) self.dim = 2 if batch else 1 def forward(self, x): x = self.affine1(x) x = self.dropout(x) x = F.relu(x) action_scores = self.affine2(x) return F.softmax(action_scores, dim=self.dim) class Observer: r""" An observer has exclusive access to its own environment. Each observer captures the state from its environment, and send the state to the agent to select an action. Then, the observer applies the action to its environment and reports the reward to the agent. It is true that CartPole-v1 is a relatively inexpensive environment, and it might be an overkill to use RPC to connect observers and trainers in this specific use case. However, the main goal of this tutorial to how to build an application using the RPC API. Developers can extend the similar idea to other applications with much more expensive environment. """ def __init__(self, batch=True): self.id = rpc.get_worker_info().id - 1 self.env = gym.make('CartPole-v1') self.env.reset(seed=args.seed) self.select_action = Agent.select_action_batch if batch else Agent.select_action def run_episode(self, agent_rref, n_steps): r""" Run one episode of n_steps. Args: agent_rref (RRef): an RRef referencing the agent object. n_steps (int): number of steps in this episode """ state, _ = self.env.reset() ep_reward = NUM_STEPS rewards = torch.zeros(n_steps) start_step = 0 for step in range(n_steps): state = torch.from_numpy(state).float().unsqueeze(0) # send the state to the agent to get an action action = rpc.rpc_sync( agent_rref.owner(), self.select_action, args=(agent_rref, self.id, state) ) # apply the action to the environment, and get the reward state, reward, terminated, truncated, _ = self.env.step(action) rewards[step] = reward if terminated or truncated or step + 1 >= n_steps: curr_rewards = rewards[start_step:(step + 1)] R = 0 for i in range(curr_rewards.numel() -1, -1, -1): R = curr_rewards[i] + args.gamma * R curr_rewards[i] = R state, _ = self.env.reset() if start_step == 0: ep_reward = min(ep_reward, step - start_step + 1) start_step = step + 1 return [rewards, ep_reward] class Agent: def __init__(self, world_size, batch=True): self.ob_rrefs = [] self.agent_rref = RRef(self) self.rewards = {} self.policy = Policy(batch).cuda() self.optimizer = optim.Adam(self.policy.parameters(), lr=1e-2) self.running_reward = 0 for ob_rank in range(1, world_size): ob_info = rpc.get_worker_info(OBSERVER_NAME.format(ob_rank)) self.ob_rrefs.append(remote(ob_info, Observer, args=(batch,))) self.rewards[ob_info.id] = [] self.states = torch.zeros(len(self.ob_rrefs), 1, 4) self.batch = batch # With batching, saved_log_probs contains a list of tensors, where each # tensor contains probs from all observers in one step. # Without batching, saved_log_probs is a dictionary where the key is the # observer id and the value is a list of probs for that observer. self.saved_log_probs = [] if self.batch else {k:[] for k in range(len(self.ob_rrefs))} self.future_actions = torch.futures.Future() self.lock = threading.Lock() self.pending_states = len(self.ob_rrefs) @staticmethod @rpc.functions.async_execution def select_action_batch(agent_rref, ob_id, state): r""" Batching select_action: In each step, the agent waits for states from all observers, and process them together. This helps to reduce the number of CUDA kernels launched and hence speed up amortized inference speed. """ self = agent_rref.local_value() self.states[ob_id].copy_(state) future_action = self.future_actions.then( lambda future_actions: future_actions.wait()[ob_id].item() ) with self.lock: self.pending_states -= 1 if self.pending_states == 0: self.pending_states = len(self.ob_rrefs) probs = self.policy(self.states.cuda()) m = Categorical(probs) actions = m.sample() self.saved_log_probs.append(m.log_prob(actions).t()[0]) future_actions = self.future_actions self.future_actions = torch.futures.Future() future_actions.set_result(actions.cpu()) return future_action @staticmethod def select_action(agent_rref, ob_id, state): r""" Non-batching select_action, return the action right away. """ self = agent_rref.local_value() probs = self.policy(state.cuda()) m = Categorical(probs) action = m.sample() self.saved_log_probs[ob_id].append(m.log_prob(action)) return action.item() def run_episode(self, n_steps=0): r""" Run one episode. The agent will tell each oberser to run one episode with n_steps. Then it collects all actions and rewards, and use those to train the policy. """ futs = [] for ob_rref in self.ob_rrefs: # make async RPC to kick off an episode on all observers futs.append(ob_rref.rpc_async().run_episode(self.agent_rref, n_steps)) # wait until all obervers have finished this episode rets = torch.futures.wait_all(futs) rewards = torch.stack([ret[0] for ret in rets]).cuda().t() ep_rewards = sum([ret[1] for ret in rets]) / len(rets) if self.batch: probs = torch.stack(self.saved_log_probs) else: probs = [torch.stack(self.saved_log_probs[i]) for i in range(len(rets))] probs = torch.stack(probs) policy_loss = -probs * rewards / len(rets) policy_loss.sum().backward() self.optimizer.step() self.optimizer.zero_grad() # reset variables self.saved_log_probs = [] if self.batch else {k:[] for k in range(len(self.ob_rrefs))} self.states = torch.zeros(len(self.ob_rrefs), 1, 4) # calculate running rewards self.running_reward = 0.5 * ep_rewards + 0.5 * self.running_reward return ep_rewards, self.running_reward def run_worker(rank, world_size, n_episode, batch, print_log=True): r""" This is the entry point for all processes. The rank 0 is the agent. All other ranks are observers. """ os.environ['MASTER_ADDR'] = 'localhost' os.environ['MASTER_PORT'] = '29500' if rank == 0: # rank0 is the agent rpc.init_rpc(AGENT_NAME, rank=rank, world_size=world_size) agent = Agent(world_size, batch) for i_episode in range(n_episode): last_reward, running_reward = agent.run_episode(n_steps=NUM_STEPS) if print_log: print(f'Episode {i_episode}\tLast reward: {last_reward:.2f}\tAverage reward: {running_reward:.2f}') else: # other ranks are the observer rpc.init_rpc(OBSERVER_NAME.format(rank), rank=rank, world_size=world_size) # observers passively waiting for instructions from agents rpc.shutdown() def main(): for world_size in range(2, args.max_world_size): delays = [] for batch in [True, False]: tik = time.time() mp.spawn( run_worker, args=(world_size, args.num_episode, batch), nprocs=world_size, join=True ) tok = time.time() delays.append(tok - tik) print(f"{world_size}, {delays[0]}, {delays[1]}") if __name__ == '__main__': main() ================================================ FILE: distributed/rpc/batch/requirements.txt ================================================ torch torchvision numpy gymnasium ================================================ FILE: distributed/rpc/ddp_rpc/README.md ================================================ Distributed DataParallel + Distributed RPC Framework Example This example demonstrates how to combine Distributed DataParallel (DDP) with the Distributed RPC Framework. It requires two trainer nodes (each with a GPU), one master node, and one parameter server. The master node initializes an embedding table on the parameter server and orchestrates the training loop across the trainers. The model is composed of a dense component (`nn.Linear`), which is replicated on the trainers using DDP, and a sparse component (`nn.EmbeddingBag`), which resides on the parameter server. Each trainer performs embedding lookups on the parameter server via RPC, then processes the results through its local `nn.Linear` module. During the backward pass, DDP aggregates gradients for the dense part using allreduce, while the distributed backward pass updates the embedding table parameters on the parameter server. ``` pip install -r requirements.txt python main.py ``` ================================================ FILE: distributed/rpc/ddp_rpc/main.py ================================================ import random import torch import torch.distributed as dist import torch.distributed.autograd as dist_autograd import torch.distributed.rpc as rpc import torch.multiprocessing as mp import torch.optim as optim from torch.distributed.nn import RemoteModule from torch.distributed.optim import DistributedOptimizer from torch.distributed.rpc import RRef from torch.distributed.rpc import TensorPipeRpcBackendOptions from torch.nn.parallel import DistributedDataParallel as DDP NUM_EMBEDDINGS = 100 EMBEDDING_DIM = 16 def verify_min_gpu_count(min_gpus: int = 2) -> bool: """ verification that we have at least 2 gpus to run dist examples """ has_gpu = torch.accelerator.is_available() gpu_count = torch.accelerator.device_count() return has_gpu and gpu_count >= min_gpus class HybridModel(torch.nn.Module): r""" The model consists of a sparse part and a dense part. 1) The dense part is an nn.Linear module that is replicated across all trainers using DistributedDataParallel. 2) The sparse part is a Remote Module that holds an nn.EmbeddingBag on the parameter server. This remote model can get a Remote Reference to the embedding table on the parameter server. """ def __init__(self, remote_emb_module, rank): super(HybridModel, self).__init__() self.remote_emb_module = remote_emb_module self.fc = DDP(torch.nn.Linear(16, 8).to(rank)) self.rank = rank def forward(self, indices, offsets): emb_lookup = self.remote_emb_module.forward(indices, offsets) return self.fc(emb_lookup.to(self.rank)) def _run_trainer(remote_emb_module, rank): r""" Each trainer runs a forward pass which involves an embedding lookup on the parameter server and running nn.Linear locally. During the backward pass, DDP is responsible for aggregating the gradients for the dense part (nn.Linear) and distributed autograd ensures gradients updates are propagated to the parameter server. """ # Setup the model. model = HybridModel(remote_emb_module, rank) # Retrieve all model parameters as rrefs for DistributedOptimizer. # Retrieve parameters for embedding table. model_parameter_rrefs = model.remote_emb_module.remote_parameters() # model.fc.parameters() only includes local parameters. # NOTE: Cannot call model.parameters() here, # because this will call remote_emb_module.parameters(), # which supports remote_parameters() but not parameters(). for param in model.fc.parameters(): model_parameter_rrefs.append(RRef(param)) # Setup distributed optimizer opt = DistributedOptimizer( optim.SGD, model_parameter_rrefs, lr=0.05, ) criterion = torch.nn.CrossEntropyLoss() def get_next_batch(rank): for _ in range(10): num_indices = random.randint(20, 50) indices = torch.LongTensor(num_indices).random_(0, NUM_EMBEDDINGS) # Generate offsets. offsets = [] start = 0 batch_size = 0 while start < num_indices: offsets.append(start) start += random.randint(1, 10) batch_size += 1 offsets_tensor = torch.LongTensor(offsets) target = torch.LongTensor(batch_size).random_(8).to(rank) yield indices, offsets_tensor, target # Train for 100 epochs for epoch in range(100): # create distributed autograd context for indices, offsets, target in get_next_batch(rank): with dist_autograd.context() as context_id: output = model(indices, offsets) loss = criterion(output, target) # Run distributed backward pass dist_autograd.backward(context_id, [loss]) # Tun distributed optimizer opt.step(context_id) # Not necessary to zero grads as each iteration creates a different # distributed autograd context which hosts different grads print("Training done for epoch {}".format(epoch)) def run_worker(rank, world_size): r""" A wrapper function that initializes RPC, calls the function, and shuts down RPC. """ # We need to use different port numbers in TCP init_method for init_rpc and # init_process_group to avoid port conflicts. rpc_backend_options = TensorPipeRpcBackendOptions() rpc_backend_options.init_method = "tcp://localhost:29501" # Rank 2 is master, 3 is ps and 0 and 1 are trainers. if rank == 2: rpc.init_rpc( "master", rank=rank, world_size=world_size, rpc_backend_options=rpc_backend_options, ) remote_emb_module = RemoteModule( "ps", torch.nn.EmbeddingBag, args=(NUM_EMBEDDINGS, EMBEDDING_DIM), kwargs={"mode": "sum"}, ) # Run the training loop on trainers. futs = [] for trainer_rank in [0, 1]: trainer_name = "trainer{}".format(trainer_rank) fut = rpc.rpc_async( trainer_name, _run_trainer, args=(remote_emb_module, trainer_rank) ) futs.append(fut) # Wait for all training to finish. for fut in futs: fut.wait() elif rank <= 1: acc = torch.accelerator.current_accelerator() device = torch.device(acc) backend = torch.distributed.get_default_backend_for_device(device) torch.accelerator.device_index(rank) # Initialize process group for Distributed DataParallel on trainers. dist.init_process_group( backend=backend, rank=rank, world_size=2, init_method="tcp://localhost:29500" ) # Initialize RPC. trainer_name = "trainer{}".format(rank) rpc.init_rpc( trainer_name, rank=rank, world_size=world_size, rpc_backend_options=rpc_backend_options, ) # Trainer just waits for RPCs from master. else: rpc.init_rpc( "ps", rank=rank, world_size=world_size, rpc_backend_options=rpc_backend_options, ) # parameter server do nothing pass # block until all rpcs finish rpc.shutdown() # Clean up process group for trainers to avoid resource leaks if rank <= 1: dist.destroy_process_group() if __name__ == "__main__": # 2 trainers, 1 parameter server, 1 master. world_size = 4 _min_gpu_count = 2 if not verify_min_gpu_count(min_gpus=_min_gpu_count): print(f"Unable to locate sufficient {_min_gpu_count} gpus to run this example. Exiting.") exit() mp.spawn(run_worker, args=(world_size,), nprocs=world_size, join=True) print("Distributed RPC example completed successfully.") ================================================ FILE: distributed/rpc/ddp_rpc/requirements.txt ================================================ torch>=2.7.0 numpy ================================================ FILE: distributed/rpc/parameter_server/README.md ================================================ ### RPC-based distributed training This is a basic example of RPC-based training that uses several trainers remotely train a model hosted on a server. To run the example locally, run the following command worker for the server and each worker you wish to spawn, in separate terminal windows: `python rpc_parameter_server.py --world_size=WORLD_SIZE --rank=RANK`. For example, for a master node with world size of 2, the command would be `python rpc_parameter_server.py --world_size=2 --rank=0`. The trainer can then be launched with the command `python rpc_parameter_server.py --world_size=2 --rank=1` in a separate window, and this will begin training with one server and a single trainer. Note that for demonstration purposes, this example supports only between 0-2 GPUs, although the pattern can be extended to make use of additional GPUs. To configure the number of GPUs, pass in `--num_gpus=N` to your training command. You can pass in the command line arguments `--master_addr=
` and `master_port=PORT` to indicate the address:port that the master worker is listening on. All workers will contact the master for rendezvous during worker discovery. By default, `master_addr` will be `localhost` and `master_port` will be 29500. ================================================ FILE: distributed/rpc/parameter_server/requirements.txt ================================================ torch>=2.7.1 numpy ================================================ FILE: distributed/rpc/parameter_server/rpc_parameter_server.py ================================================ import argparse import os from threading import Lock import torch import torch.distributed.autograd as dist_autograd import torch.distributed.rpc as rpc import torch.multiprocessing as mp import torch.nn as nn import torch.nn.functional as F from torch import optim from torch.distributed.optim import DistributedOptimizer from torchvision import datasets, transforms # --------- MNIST Network to train, from pytorch/examples ----- class Net(nn.Module): def __init__(self, num_gpus=0): super(Net, self).__init__() print(f"Using {num_gpus} GPUs to train") self.num_gpus = num_gpus if torch.accelerator.is_available() and self.num_gpus > 0: acc = torch.accelerator.current_accelerator() device = torch.device(f'{acc}:0') else: device = torch.device("cpu") print(f"Putting first 2 convs on {str(device)}") # Put conv layers on the first accelerator device self.conv1 = nn.Conv2d(1, 32, 3, 1).to(device) self.conv2 = nn.Conv2d(32, 64, 3, 1).to(device) # Put rest of the network on the 2nd accelerator device, if there is one if torch.accelerator.is_available() and self.num_gpus > 0: acc = torch.accelerator.current_accelerator() device = torch.device(f'{acc}:1') print(f"Putting rest of layers on {str(device)}") self.dropout1 = nn.Dropout2d(0.25).to(device) self.dropout2 = nn.Dropout2d(0.5).to(device) self.fc1 = nn.Linear(9216, 128).to(device) self.fc2 = nn.Linear(128, 10).to(device) def forward(self, x): x = self.conv1(x) x = F.relu(x) x = self.conv2(x) x = F.max_pool2d(x, 2) x = self.dropout1(x) x = torch.flatten(x, 1) # Move tensor to next device if necessary next_device = next(self.fc1.parameters()).device x = x.to(next_device) x = self.fc1(x) x = F.relu(x) x = self.dropout2(x) x = self.fc2(x) output = F.log_softmax(x, dim=1) return output # --------- Helper Methods -------------------- # On the local node, call a method with first arg as the value held by the # RRef. Other args are passed in as arguments to the function called. # Useful for calling instance methods. def call_method(method, rref, *args, **kwargs): return method(rref.local_value(), *args, **kwargs) # Given an RRef, return the result of calling the passed in method on the value # held by the RRef. This call is done on the remote node that owns # the RRef. args and kwargs are passed into the method. # Example: If the value held by the RRef is of type Foo, then # remote_method(Foo.bar, rref, arg1, arg2) is equivalent to calling # .bar(arg1, arg2) on the remote node and getting the result # back. def remote_method(method, rref, *args, **kwargs): args = [method, rref] + list(args) return rpc.rpc_sync(rref.owner(), call_method, args=args, kwargs=kwargs) # --------- Parameter Server -------------------- class ParameterServer(nn.Module): def __init__(self, num_gpus=0): super().__init__() model = Net(num_gpus=num_gpus) self.model = model if torch.accelerator.is_available() and num_gpus > 0: acc = torch.accelerator.current_accelerator() self.input_device = torch.device(f'{acc}:0') else: self.input_device = torch.device("cpu") def forward(self, inp): inp = inp.to(self.input_device) out = self.model(inp) # This output is forwarded over RPC, which as of 1.5.0 only accepts CPU tensors. # Tensors must be moved in and out of GPU memory due to this. out = out.to("cpu") return out # Use dist autograd to retrieve gradients accumulated for this model. # Primarily used for verification. def get_dist_gradients(self, cid): grads = dist_autograd.get_gradients(cid) # This output is forwarded over RPC, which as of 1.5.0 only accepts CPU tensors. # Tensors must be moved in and out of GPU memory due to this. cpu_grads = {} for k, v in grads.items(): k_cpu, v_cpu = k.to("cpu"), v.to("cpu") cpu_grads[k_cpu] = v_cpu return cpu_grads # Wrap local parameters in a RRef. Needed for building the # DistributedOptimizer which optimizes parameters remotely. def get_param_rrefs(self): param_rrefs = [rpc.RRef(param) for param in self.model.parameters()] return param_rrefs param_server = None global_lock = Lock() def get_parameter_server(num_gpus=0): global param_server # Ensure that we get only one handle to the ParameterServer. with global_lock: if not param_server: # construct it once param_server = ParameterServer(num_gpus=num_gpus) return param_server def run_parameter_server(rank, world_size): # The parameter server just acts as a host for the model and responds to # requests from trainers, hence it does not need to run a loop. # rpc.shutdown() will wait for all workers to complete by default, which # in this case means that the parameter server will wait for all trainers # to complete, and then exit. print("PS master initializing RPC") rpc.init_rpc(name="parameter_server", rank=rank, world_size=world_size) print("RPC initialized! Running parameter server...") rpc.shutdown() print("RPC shutdown on parameter server.") # --------- Trainers -------------------- # nn.Module corresponding to the network trained by this trainer. The # forward() method simply invokes the network on the given parameter # server. class TrainerNet(nn.Module): def __init__(self, num_gpus=0): super().__init__() self.num_gpus = num_gpus self.param_server_rref = rpc.remote( "parameter_server", get_parameter_server, args=(num_gpus,)) def get_global_param_rrefs(self): remote_params = remote_method( ParameterServer.get_param_rrefs, self.param_server_rref) return remote_params def forward(self, x): model_output = remote_method( ParameterServer.forward, self.param_server_rref, x) return model_output def run_training_loop(rank, num_gpus, train_loader, test_loader): # Runs the typical neural network forward + backward + optimizer step, but # in a distributed fashion. net = TrainerNet(num_gpus=num_gpus) # Build DistributedOptimizer. param_rrefs = net.get_global_param_rrefs() opt = DistributedOptimizer(optim.SGD, param_rrefs, lr=0.03) for i, (data, target) in enumerate(train_loader): with dist_autograd.context() as cid: model_output = net(data) target = target.to(model_output.device) loss = F.nll_loss(model_output, target) if i % 5 == 0: print(f"Rank {rank} training batch {i} loss {loss.item()}") dist_autograd.backward(cid, [loss]) # Ensure that dist autograd ran successfully and gradients were # returned. assert remote_method( ParameterServer.get_dist_gradients, net.param_server_rref, cid) != {} opt.step(cid) print("Training complete!") print("Getting accuracy....") get_accuracy(test_loader, net) def get_accuracy(test_loader, model): model.eval() correct_sum = 0 # Use GPU to evaluate if possible if torch.accelerator.is_available() and model.num_gpus > 0: acc = torch.accelerator.current_accelerator() device = torch.device(f'{acc}:0') else: device = torch.device("cpu") with torch.no_grad(): for i, (data, target) in enumerate(test_loader): out = model(data) pred = out.argmax(dim=1, keepdim=True) pred, target = pred.to(device), target.to(device) correct = pred.eq(target.view_as(pred)).sum().item() correct_sum += correct print(f"Accuracy {correct_sum / len(test_loader.dataset)}") # Main loop for trainers. def run_worker(rank, world_size, num_gpus, train_loader, test_loader): print(f"Worker rank {rank} initializing RPC") rpc.init_rpc( name=f"trainer_{rank}", rank=rank, world_size=world_size) print(f"Worker {rank} done initializing RPC") run_training_loop(rank, num_gpus, train_loader, test_loader) rpc.shutdown() # --------- Launcher -------------------- if __name__ == '__main__': parser = argparse.ArgumentParser( description="Parameter-Server RPC based training") parser.add_argument( "--world_size", type=int, default=4, help="""Total number of participating processes. Should be the sum of master node and all training nodes.""") parser.add_argument( "--rank", type=int, default=None, help="Global rank of this process. Pass in 0 for master.") parser.add_argument( "--num_gpus", type=int, default=0, help="""Number of GPUs to use for training, currently supports between 0 and 2 GPUs. Note that this argument will be passed to the parameter servers.""") parser.add_argument( "--master_addr", type=str, default="localhost", help="""Address of master, will default to localhost if not provided. Master must be able to accept network traffic on the address + port.""") parser.add_argument( "--master_port", type=str, default="29500", help="""Port that master is listening on, will default to 29500 if not provided. Master must be able to accept network traffic on the host and port.""") args = parser.parse_args() assert args.rank is not None, "must provide rank argument." assert args.num_gpus <= 3, f"Only 0-2 GPUs currently supported (got {args.num_gpus})." os.environ['MASTER_ADDR'] = args.master_addr os.environ['MASTER_PORT'] = args.master_port processes = [] world_size = args.world_size # Note that Linux uses "fork" by default, which may cause deadlock. # Besides, cuda doesn't support "fork" and Windows only supports "spawn" mp.set_start_method("spawn") if args.rank == 0: p = mp.Process(target=run_parameter_server, args=(0, world_size)) p.start() processes.append(p) else: # Get data to train on train_loader = torch.utils.data.DataLoader( datasets.MNIST('../data', train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])), batch_size=32, shuffle=True) test_loader = torch.utils.data.DataLoader( datasets.MNIST('../data', train=False, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])), batch_size=32, shuffle=True) # start training worker on this node p = mp.Process( target=run_worker, args=( args.rank, world_size, args.num_gpus, train_loader, test_loader)) p.start() processes.append(p) for p in processes: p.join() ================================================ FILE: distributed/rpc/pipeline/README.md ================================================ Distributed Pipeline Parallel Example This example shows how to distribute a ResNet50 model on two RPC workers and then implement distributed pipeline parallelism using RPC. With pipeline parallelism, every input batch is divided into micro-batches and thse micro-batches are feed into the model in a pipelined fashion to increase the amortized device utilization. Note that this example only parallelizes the forward pass which can be viewed as the distributed counterpart of the [single machine pipeline parallel](https://pytorch.org/tutorials/intermediate/model_parallel_tutorial.html#speed-up-by-pipelining-inputs) example. ``` pip install -r requirements.txt python main.py ``` ================================================ FILE: distributed/rpc/pipeline/main.py ================================================ import os import threading import time import warnings from functools import wraps import torch import torch.nn as nn import torch.distributed.autograd as dist_autograd import torch.distributed.rpc as rpc import torch.multiprocessing as mp import torch.optim as optim from torch.distributed.rpc import RRef from torchvision.models.resnet import Bottleneck # Suppress warnings that can't be fixed from user code warnings.filterwarnings("ignore", message="You are using a Backend .* as a ProcessGroup. This usage is deprecated", category=UserWarning) warnings.filterwarnings("ignore", message="networkx backend defined more than once: nx-loopback", category=RuntimeWarning) ######################################################### # Define Model Parallel ResNet50 # ######################################################### # In order to split the ResNet50 and place it on two different workers, we # implement it in two model shards. The ResNetBase class defines common # attributes and methods shared by two shards. ResNetShard1 and ResNetShard2 # contain two partitions of the model layers respectively. num_classes = 1000 def conv1x1(in_planes, out_planes, stride=1): """1x1 convolution""" return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) class ResNetBase(nn.Module): def __init__(self, block, inplanes, num_classes=1000, groups=1, width_per_group=64, norm_layer=None): super(ResNetBase, self).__init__() self._lock = threading.Lock() self._block = block self._norm_layer = nn.BatchNorm2d self.inplanes = inplanes self.dilation = 1 self.groups = groups self.base_width = width_per_group def _make_layer(self, planes, blocks, stride=1): norm_layer = self._norm_layer downsample = None previous_dilation = self.dilation if stride != 1 or self.inplanes != planes * self._block.expansion: downsample = nn.Sequential( conv1x1(self.inplanes, planes * self._block.expansion, stride), norm_layer(planes * self._block.expansion), ) layers = [] layers.append(self._block(self.inplanes, planes, stride, downsample, self.groups, self.base_width, previous_dilation, norm_layer)) self.inplanes = planes * self._block.expansion for _ in range(1, blocks): layers.append(self._block(self.inplanes, planes, groups=self.groups, base_width=self.base_width, dilation=self.dilation, norm_layer=norm_layer)) return nn.Sequential(*layers) def parameter_rrefs(self): r""" Create one RRef for each parameter in the given local module, and return a list of RRefs. """ return [RRef(p) for p in self.parameters()] class ResNetShard1(ResNetBase): """ The first part of ResNet. """ def __init__(self, device, *args, **kwargs): super(ResNetShard1, self).__init__( Bottleneck, 64, num_classes=num_classes, *args, **kwargs) self.device = device self.seq = nn.Sequential( nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False), self._norm_layer(self.inplanes), nn.ReLU(inplace=True), nn.MaxPool2d(kernel_size=3, stride=2, padding=1), self._make_layer(64, 3), self._make_layer(128, 4, stride=2) ).to(self.device) for m in self.modules(): if isinstance(m, nn.Conv2d): nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') elif isinstance(m, nn.BatchNorm2d): nn.init.ones_(m.weight) nn.init.zeros_(m.bias) def forward(self, x_rref): x = x_rref.to_here().to(self.device) with self._lock: out = self.seq(x) return out.cpu() class ResNetShard2(ResNetBase): """ The second part of ResNet. """ def __init__(self, device, *args, **kwargs): super(ResNetShard2, self).__init__( Bottleneck, 512, num_classes=num_classes, *args, **kwargs) self.device = device self.seq = nn.Sequential( self._make_layer(256, 6, stride=2), self._make_layer(512, 3, stride=2), nn.AdaptiveAvgPool2d((1, 1)), ).to(self.device) self.fc = nn.Linear(512 * self._block.expansion, num_classes).to(self.device) def forward(self, x_rref): x = x_rref.to_here().to(self.device) with self._lock: out = self.fc(torch.flatten(self.seq(x), 1)) return out.cpu() class DistResNet50(nn.Module): """ Assemble two parts as an nn.Module and define pipelining logic """ def __init__(self, split_size, workers, *args, **kwargs): super(DistResNet50, self).__init__() self.split_size = split_size # Put the first part of the ResNet50 on workers[0] self.p1_rref = rpc.remote( workers[0], ResNetShard1, args = ("cuda:0",) + args, kwargs = kwargs ) # Put the second part of the ResNet50 on workers[1] self.p2_rref = rpc.remote( workers[1], ResNetShard2, args = ("cuda:1",) + args, kwargs = kwargs ) def forward(self, xs): # Split the input batch xs into micro-batches, and collect async RPC # futures into a list out_futures = [] for x in iter(xs.split(self.split_size, dim=0)): x_rref = RRef(x) y_rref = self.p1_rref.remote().forward(x_rref) z_fut = self.p2_rref.rpc_async().forward(y_rref) out_futures.append(z_fut) # collect and cat all output tensors into one tensor. return torch.cat(torch.futures.wait_all(out_futures)) def parameter_rrefs(self): remote_params = [] remote_params.extend(self.p1_rref.remote().parameter_rrefs().to_here()) remote_params.extend(self.p2_rref.remote().parameter_rrefs().to_here()) return remote_params ######################################################### # Run RPC Processes # ######################################################### num_batches = 3 batch_size = 120 image_w = 128 image_h = 128 def create_optimizer_for_remote_params(worker_name, param_rrefs, lr=0.05): """Create torch.compiled optimizers on each worker""" params = [p.to_here() for p in param_rrefs] opt = optim.SGD(params, lr=lr) opt.step = torch.compile(opt.step) return opt def run_master(split_size): # put the two model parts on worker1 and worker2 respectively model = DistResNet50(split_size, ["worker1", "worker2"]) loss_fn = nn.MSELoss() # Get parameter RRefs for each model shard p1_param_rrefs = model.p1_rref.remote().parameter_rrefs().to_here() p2_param_rrefs = model.p2_rref.remote().parameter_rrefs().to_here() # Create optimizers on remote workers opt1_rref = rpc.remote( "worker1", create_optimizer_for_remote_params, args=("worker1", p1_param_rrefs) ) opt2_rref = rpc.remote( "worker2", create_optimizer_for_remote_params, args=("worker2", p2_param_rrefs) ) one_hot_indices = torch.LongTensor(batch_size) \ .random_(0, num_classes) \ .view(batch_size, 1) for i in range(num_batches): print(f"Processing batch {i}") # generate random inputs and labels inputs = torch.randn(batch_size, 3, image_w, image_h) labels = torch.zeros(batch_size, num_classes) \ .scatter_(1, one_hot_indices, 1) # The distributed autograd context is the dedicated scope for the # distributed backward pass to store gradients, which can later be # retrieved using the context_id by the distributed optimizer. with dist_autograd.context() as context_id: outputs = model(inputs) dist_autograd.backward(context_id, [loss_fn(outputs, labels)]) opt1_rref.rpc_sync().step() opt2_rref.rpc_sync().step() opt1_rref.rpc_sync().zero_grad() opt2_rref.rpc_sync().zero_grad() def run_worker(rank, world_size, num_split): os.environ['MASTER_ADDR'] = 'localhost' os.environ['MASTER_PORT'] = '29500' # Higher timeout is added to accommodate for kernel compilation time in case of ROCm. options = rpc.TensorPipeRpcBackendOptions(num_worker_threads=256, rpc_timeout=300) if rank == 0: rpc.init_rpc( "master", rank=rank, world_size=world_size, rpc_backend_options=options ) run_master(num_split) else: rpc.init_rpc( f"worker{rank}", rank=rank, world_size=world_size, rpc_backend_options=options ) pass # block until all rpcs finish rpc.shutdown() if __name__=="__main__": # Suppress torch compile profiler warnings os.environ['TORCH_LOGS'] = '-dynamo' world_size = 3 for num_split in [1, 2, 4, 8]: tik = time.time() mp.spawn(run_worker, args=(world_size, num_split), nprocs=world_size, join=True) tok = time.time() print(f"number of splits = {num_split}, execution time = {tok - tik}") ================================================ FILE: distributed/rpc/pipeline/requirements.txt ================================================ torch torchvision ================================================ FILE: distributed/rpc/rl/README.md ================================================ Distributed Multi-Observer Single-Agent Reinforcement Learning Example This example demonstrates `torch.distributed.rpc` API using an CartPole reinforcement learning example. Please note that the goal is to present the RPC API instead of building the best CartPole solver. ``` pip install -r requirements.txt python main.py ``` ================================================ FILE: distributed/rpc/rl/main.py ================================================ import argparse import gymnasium as gym import numpy as np import os from itertools import count import torch import torch.distributed.rpc as rpc import torch.multiprocessing as mp import torch.nn as nn import torch.nn.functional as F import torch.optim as optim from torch.distributed.rpc import RRef, rpc_sync, rpc_async, remote from torch.distributions import Categorical TOTAL_EPISODE_STEP = 5000 AGENT_NAME = "agent" OBSERVER_NAME = "observer{}" parser = argparse.ArgumentParser(description='PyTorch RPC RL example') parser.add_argument('--world-size', type=int, default=2, metavar='W', help='world size for RPC, rank 0 is the agent, others are observers') parser.add_argument('--gamma', type=float, default=0.99, metavar='G', help='discount factor (default: 0.99)') parser.add_argument('--seed', type=int, default=543, metavar='N', help='random seed (default: 543)') parser.add_argument('--log-interval', type=int, default=10, metavar='N', help='interval between training status logs (default: 10)') args = parser.parse_args() torch.manual_seed(args.seed) def _call_method(method, rref, *args, **kwargs): r""" a helper function to call a method on the given RRef """ return method(rref.local_value(), *args, **kwargs) def _remote_method(method, rref, *args, **kwargs): r""" a helper function to run method on the owner of rref and fetch back the result using RPC """ args = [method, rref] + list(args) return rpc_sync(rref.owner(), _call_method, args=args, kwargs=kwargs) class Policy(nn.Module): r""" Borrowing the ``Policy`` class from the Reinforcement Learning example. Copying the code to make these two examples independent. See https://github.com/pytorch/examples/tree/main/reinforcement_learning """ def __init__(self): super(Policy, self).__init__() self.affine1 = nn.Linear(4, 128) self.dropout = nn.Dropout(p=0.6) self.affine2 = nn.Linear(128, 2) self.saved_log_probs = [] self.rewards = [] def forward(self, x): x = self.affine1(x) x = self.dropout(x) x = F.relu(x) action_scores = self.affine2(x) return F.softmax(action_scores, dim=1) class Observer: r""" An observer has exclusive access to its own environment. Each observer captures the state from its environment, and send the state to the agent to select an action. Then, the observer applies the action to its environment and reports the reward to the agent. It is true that CartPole-v1 is a relatively inexpensive environment, and it might be an overkill to use RPC to connect observers and trainers in this specific use case. However, the main goal of this tutorial to how to build an application using the RPC API. Developers can extend the similar idea to other applications with much more expensive environment. """ def __init__(self): self.id = rpc.get_worker_info().id self.env = gym.make('CartPole-v1') self.env.reset(seed=args.seed) def run_episode(self, agent_rref, n_steps): r""" Run one episode of n_steps. Args: agent_rref (RRef): an RRef referencing the agent object. n_steps (int): number of steps in this episode """ state, ep_reward = self.env.reset()[0], 0 for step in range(n_steps): # send the state to the agent to get an action action = _remote_method(Agent.select_action, agent_rref, self.id, state) # apply the action to the environment, and get the reward state, reward, terminated, truncated, _ = self.env.step(action) # report the reward to the agent for training purpose _remote_method(Agent.report_reward, agent_rref, self.id, reward) if terminated or truncated: break class Agent: def __init__(self, world_size): self.ob_rrefs = [] self.agent_rref = RRef(self) self.rewards = {} self.saved_log_probs = {} self.policy = Policy() self.optimizer = optim.Adam(self.policy.parameters(), lr=1e-2) self.eps = np.finfo(np.float32).eps.item() self.running_reward = 0 self.reward_threshold = gym.make('CartPole-v1').spec.reward_threshold for ob_rank in range(1, world_size): ob_info = rpc.get_worker_info(OBSERVER_NAME.format(ob_rank)) self.ob_rrefs.append(remote(ob_info, Observer)) self.rewards[ob_info.id] = [] self.saved_log_probs[ob_info.id] = [] def select_action(self, ob_id, state): r""" This function is mostly borrowed from the Reinforcement Learning example. See https://github.com/pytorch/examples/tree/main/reinforcement_learning The main difference is that instead of keeping all probs in one list, the agent keeps probs in a dictionary, one key per observer. NB: no need to enforce thread-safety here as GIL will serialize executions. """ state = torch.from_numpy(state).float().unsqueeze(0) probs = self.policy(state) m = Categorical(probs) action = m.sample() self.saved_log_probs[ob_id].append(m.log_prob(action)) return action.item() def report_reward(self, ob_id, reward): r""" Observers call this function to report rewards. """ self.rewards[ob_id].append(reward) def run_episode(self, n_steps=0): r""" Run one episode. The agent will tell each oberser to run n_steps. """ futs = [] for ob_rref in self.ob_rrefs: # make async RPC to kick off an episode on all observers futs.append( rpc_async( ob_rref.owner(), _call_method, args=(Observer.run_episode, ob_rref, self.agent_rref, n_steps) ) ) # wait until all obervers have finished this episode for fut in futs: fut.wait() def finish_episode(self): r""" This function is mostly borrowed from the Reinforcement Learning example. See https://github.com/pytorch/examples/tree/main/reinforcement_learning The main difference is that it joins all probs and rewards from different observers into one list, and uses the minimum observer rewards as the reward of the current episode. """ # joins probs and rewards from different observers into lists R, probs, rewards = 0, [], [] for ob_id in self.rewards: probs.extend(self.saved_log_probs[ob_id]) rewards.extend(self.rewards[ob_id]) # use the minimum observer reward to calculate the running reward min_reward = min([sum(self.rewards[ob_id]) for ob_id in self.rewards]) self.running_reward = 0.05 * min_reward + (1 - 0.05) * self.running_reward # clear saved probs and rewards for ob_id in self.rewards: self.rewards[ob_id] = [] self.saved_log_probs[ob_id] = [] policy_loss, returns = [], [] for r in rewards[::-1]: R = r + args.gamma * R returns.insert(0, R) returns = torch.tensor(returns) returns = (returns - returns.mean()) / (returns.std() + self.eps) for log_prob, R in zip(probs, returns): policy_loss.append(-log_prob * R) self.optimizer.zero_grad() policy_loss = torch.cat(policy_loss).sum() policy_loss.backward() self.optimizer.step() return min_reward def run_worker(rank, world_size): r""" This is the entry point for all processes. The rank 0 is the agent. All other ranks are observers. """ os.environ['MASTER_ADDR'] = 'localhost' os.environ['MASTER_PORT'] = '29500' if rank == 0: # rank0 is the agent rpc.init_rpc(AGENT_NAME, rank=rank, world_size=world_size) agent = Agent(world_size) for i_episode in count(1): n_steps = int(TOTAL_EPISODE_STEP / (args.world_size - 1)) agent.run_episode(n_steps=n_steps) last_reward = agent.finish_episode() if i_episode % args.log_interval == 0: print('Episode {}\tLast reward: {:.2f}\tAverage reward: {:.2f}'.format( i_episode, last_reward, agent.running_reward)) if agent.running_reward > agent.reward_threshold: print("Solved! Running reward is now {}!".format(agent.running_reward)) break else: # other ranks are the observer rpc.init_rpc(OBSERVER_NAME.format(rank), rank=rank, world_size=world_size) # observers passively waiting for instructions from agents rpc.shutdown() def main(): mp.spawn( run_worker, args=(args.world_size, ), nprocs=args.world_size, join=True ) if __name__ == '__main__': main() ================================================ FILE: distributed/rpc/rl/requirements.txt ================================================ torch numpy gymnasium ================================================ FILE: distributed/rpc/rnn/README.md ================================================ Distributed RNN Model Parallel Example This example shows how to build an RNN model using RPC where different components of the RNN model can be placed on different workers. ``` pip install -r requirements.txt python main.py ``` ================================================ FILE: distributed/rpc/rnn/main.py ================================================ import os import torch import torch.distributed.autograd as dist_autograd import torch.distributed.rpc as rpc import torch.multiprocessing as mp import torch.optim as optim from torch.distributed.optim import DistributedOptimizer import rnn def _run_trainer(): r""" The trainer creates a distributed RNNModel and a DistributedOptimizer. Then, it performs training using random input data. """ batch = 5 ntoken = 7 ninp = 2 nhid = 3 nindices = 6 nlayers = 4 hidden = ( torch.randn(nlayers, nindices, nhid), torch.randn(nlayers, nindices, nhid) ) model = rnn.RNNModel('ps', ntoken, ninp, nhid, nlayers) # setup distributed optimizer opt = DistributedOptimizer( optim.SGD, model.parameter_rrefs(), lr=0.05, ) criterion = torch.nn.CrossEntropyLoss() def get_next_batch(): for _ in range(5): data = torch.LongTensor(batch, nindices) % ntoken target = torch.LongTensor(batch, ntoken) % nindices yield data, target # train for 10 iterations for epoch in range(10): # create distributed autograd context for data, target in get_next_batch(): with dist_autograd.context() as context_id: hidden[0].detach_() hidden[1].detach_() output, hidden = model(data, hidden) loss = criterion(output, target) # run distributed backward pass dist_autograd.backward(context_id, [loss]) # run distributed optimizer opt.step(context_id) # not necessary to zero grads as each iteration creates a different # distributed autograd context which hosts different grads print("Training epoch {}".format(epoch)) def run_worker(rank, world_size): r""" A wrapper function that initializes RPC, calls the function, and shuts down RPC. """ os.environ['MASTER_ADDR'] = 'localhost' os.environ['MASTER_PORT'] = '29500' if rank == 1: rpc.init_rpc("trainer", rank=rank, world_size=world_size) _run_trainer() else: rpc.init_rpc("ps", rank=rank, world_size=world_size) # parameter server does nothing pass # block until all rpcs finish rpc.shutdown() if __name__ == "__main__": world_size = 2 mp.spawn(run_worker, args=(world_size, ), nprocs=world_size, join=True) ================================================ FILE: distributed/rpc/rnn/requirements.txt ================================================ torch>=2.7.1 numpy ================================================ FILE: distributed/rpc/rnn/rnn.py ================================================ import torch import torch.nn as nn import torch.distributed.rpc as rpc from torch.distributed.rpc import RRef def _call_method(method, rref, *args, **kwargs): r""" a helper function to call a method on the given RRef """ return method(rref.local_value(), *args, **kwargs) def _remote_method(method, rref, *args, **kwargs): r""" a helper function to run method on the owner of rref and fetch back the result using RPC """ return rpc.rpc_sync( rref.owner(), _call_method, args=[method, rref] + list(args), kwargs=kwargs ) def _parameter_rrefs(module): r""" Create one RRef for each parameter in the given local module, and return a list of RRefs. """ param_rrefs = [] for param in module.parameters(): param_rrefs.append(RRef(param)) return param_rrefs class EmbeddingTable(nn.Module): r""" Encoding layers of the RNNModel """ def __init__(self, ntoken, ninp, dropout): super(EmbeddingTable, self).__init__() self.drop = nn.Dropout(dropout) self.encoder = nn.Embedding(ntoken, ninp) if torch.accelerator.is_available(): device = torch.accelerator.current_accelerator() self.encoder = self.encoder.to(device) nn.init.uniform_(self.encoder.weight, -0.1, 0.1) def forward(self, input): if torch.accelerator.is_available(): device = torch.accelerator.current_accelerator() input = input.to(device) return self.drop(self.encoder(input)).cpu() class Decoder(nn.Module): r""" Decoding layers of the RNNModel """ def __init__(self, ntoken, nhid, dropout): super(Decoder, self).__init__() self.drop = nn.Dropout(dropout) self.decoder = nn.Linear(nhid, ntoken) nn.init.zeros_(self.decoder.bias) nn.init.uniform_(self.decoder.weight, -0.1, 0.1) def forward(self, output): return self.decoder(self.drop(output)) class RNNModel(nn.Module): r""" A distributed RNN model which puts embedding table and decoder parameters on a remote parameter server, and locally holds parameters for the LSTM module. The structure of the RNN model is borrowed from the word language model example. See https://github.com/pytorch/examples/blob/main/word_language_model/model.py """ def __init__(self, ps, ntoken, ninp, nhid, nlayers, dropout=0.5): super(RNNModel, self).__init__() # setup embedding table remotely self.emb_table_rref = rpc.remote(ps, EmbeddingTable, args=(ntoken, ninp, dropout)) # setup LSTM locally self.rnn = nn.LSTM(ninp, nhid, nlayers, dropout=dropout) # setup decoder remotely self.decoder_rref = rpc.remote(ps, Decoder, args=(ntoken, nhid, dropout)) def forward(self, input, hidden): # pass input to the remote embedding table and fetch emb tensor back emb = _remote_method(EmbeddingTable.forward, self.emb_table_rref, input) output, hidden = self.rnn(emb, hidden) # pass output to the remote decoder and get the decoded output back decoded = _remote_method(Decoder.forward, self.decoder_rref, output) return decoded, hidden def parameter_rrefs(self): remote_params = [] # get RRefs of embedding table remote_params.extend(_remote_method(_parameter_rrefs, self.emb_table_rref)) # create RRefs for local parameters remote_params.extend(_parameter_rrefs(self.rnn)) # get RRefs of decoder remote_params.extend(_remote_method(_parameter_rrefs, self.decoder_rref)) return remote_params ================================================ FILE: distributed/tensor_parallelism/README.md ================================================ # PyTorch native Tensor Parallel for distributed training This example demonstrates SPMD Megatron-LM style Tensor Parallel by using PyTorch native Tensor Parallel APIs, which include: 1. Simple module-level Tensor Parallelism on a dummy MLP model. 2. Simple module-level Tensor Parallelism with Sequence Parallel inputs/outputs on a dummy MLP model. 3. A E2E demo of Fully Sharded Data Parallel + Tensor Parallel (with Sequence Parallel) on a example Llama2 model. More details about the PyTorch native Tensor Parallel APIs, please see PyTorch docs: https://pytorch.org/docs/stable/distributed.tensor.parallel.html ## Installation ```bash pip install -r requirements.txt ``` ## Running Examples You can run the examples using `torchrun` to launch distributed training: ```bash # Simple Tensor Parallel example torchrun --nnodes=1 --nproc_per_node=4 tensor_parallel_example.py # Tensor Parallel with Sequence Parallel torchrun --nnodes=1 --nproc_per_node=4 sequence_parallel_example.py # FSDP + Tensor Parallel with Llama2 model torchrun --nnodes=1 --nproc_per_node=4 fsdp_tp_example.py ``` For more details, check the `run_examples.sh` script. ================================================ FILE: distributed/tensor_parallelism/fsdp_tp_example.py ================================================ """ This is the script to test 2D Parallel which combines Tensor/Sequence parallel with Fully Sharded Data Parallel (TP/SP + FSDP) on a example Llama2 model. We show an E2E working flow from forward, backward and optimization. We enabled Fully Sharded Data Parallel + Tensor Parallel in separate parallel dimensions: Data Parallel ("dp") across hosts Tensor Parallel ("tp") within each host We use a simple diagram to illustrate below: ====================================================================== ------------ ------------ ------------ ------------ | Host 1 | | Host 2 | | | | Host N | | 8 GPUs | | 8 GPUs | | | | 8 GPUs | | | | | | ... | | | | (TP) | | (TP) | | | | (TP) | |[0,1,..,7]| |[8,9..,15]| | | |[8N-8,8N-7| | | | | | | | .., 8N-1]| | | | | | | | | ------------ ------------ ------------ ------------ FSDP: [0, 8, ..., 8N-8], [1, 9, ..., 8N-7], ..., [7, 15, ..., 8N-1] ====================================================================== More details can be seen in the PyTorch tutorials: https://pytorch.org/tutorials/intermediate/TP_tutorial.html """ import sys import os import torch import torch.distributed as dist import torch.nn as nn import torch.nn.functional as F from log_utils import rank_log, get_logger, verify_min_gpu_count # ---- GPU check ------------ _min_gpu_count = 4 if not verify_min_gpu_count(min_gpus=_min_gpu_count): print(f"Unable to locate sufficient {_min_gpu_count} gpus to run this example. Exiting.") sys.exit() # --------------------------- from llama2_model import Transformer, ModelArgs from torch.distributed.device_mesh import init_device_mesh from torch.distributed.fsdp import fully_shard from torch.distributed._tensor import Shard, Replicate from torch.distributed.tensor.parallel import ( parallelize_module, ColwiseParallel, RowwiseParallel, PrepareModuleInput, SequenceParallel ) tp_size = 2 logger = get_logger() # understand world topology _rank = int(os.environ["RANK"]) _world_size = int(os.environ["WORLD_SIZE"]) print(f"Starting PyTorch 2D (FSDP + TP) example on rank {_rank}.") assert ( _world_size % tp_size == 0 ), f"World size {_world_size} needs to be divisible by TP size {tp_size}" # create a sharding plan based on the given world_size. dp_size = _world_size // tp_size device_type = torch.accelerator.current_accelerator().type # Create a device mesh with 2 dimensions. # First dim is the data parallel dimension # Second dim is the tensor parallel dimension. device_mesh = init_device_mesh(device_type, (dp_size, tp_size), mesh_dim_names=("dp", "tp")) rank_log(_rank, logger, f"Device Mesh created: {device_mesh=}") tp_mesh = device_mesh["tp"] dp_mesh = device_mesh["dp"] # For TP, input needs to be same across all TP ranks. # while for SP, input can be different across all ranks. # We will use dp_rank for setting the random seed # to mimic the behavior of the dataloader. dp_rank = dp_mesh.get_local_rank() # create model and move it to GPU - initdevice_type_mesh has already mapped GPU ids. simple_llama2_config = ModelArgs(dim=256, n_layers=2, n_heads=16, vocab_size=32000) model = Transformer.from_model_args(simple_llama2_config).to(device_type) # init model weights model.init_weights() # parallelize the first embedding and the last linear out projection model = parallelize_module( model, tp_mesh, { "tok_embeddings": RowwiseParallel( input_layouts=Replicate(), output_layouts=Shard(1), ), "norm": SequenceParallel(), "output": ColwiseParallel( input_layouts=Shard(1), output_layouts=Replicate() ), } ) for layer_id, transformer_block in enumerate(model.layers): layer_tp_plan = { "attention_norm": SequenceParallel(), "attention": PrepareModuleInput( input_layouts=(Shard(1), Replicate()), desired_input_layouts=(Replicate(), Replicate()), ), "attention.wq": ColwiseParallel(use_local_output=False), "attention.wk": ColwiseParallel(use_local_output=False), "attention.wv": ColwiseParallel(use_local_output=False), "attention.wo": RowwiseParallel(output_layouts=Shard(1)), "ffn_norm": SequenceParallel(), "feed_forward": PrepareModuleInput( input_layouts=(Shard(1),), desired_input_layouts=(Replicate(),), ), "feed_forward.w1": ColwiseParallel(), "feed_forward.w2": RowwiseParallel(output_layouts=Shard(1)), "feed_forward.w3": ColwiseParallel(), } # Custom parallelization plan for the model parallelize_module( module=transformer_block, device_mesh=tp_mesh, parallelize_plan=layer_tp_plan ) # Init FSDP using the dp device mesh sharded_model = fully_shard(model, mesh=dp_mesh) rank_log(_rank, logger, f"Model after parallelization {sharded_model=}\n") # Create an optimizer for the parallelized and sharded model. lr = 3e-3 rank_log(_rank, logger, f"Creating AdamW optimizer with learning rate {lr}") optimizer = torch.optim.AdamW(sharded_model.parameters(), lr=lr, foreach=True) # Training loop: # Perform a num of iterations of forward/backward # and optimizations for the sharded module. rank_log(_rank, logger, "\nStarting 2D training...") num_iterations = 10 batch_size = 2 for i in range(num_iterations): # seeding with dp_rank to ensure identical inputs for TP groups torch.manual_seed(i + dp_rank) inp = torch.randint(32000, (8, 256), device=device_type) output = sharded_model(inp) output.sum().backward() optimizer.step() rank_log(_rank, logger, f"2D iter {i} complete") rank_log(_rank, logger, "2D training successfully completed!") if dist.is_initialized(): dist.destroy_process_group() ================================================ FILE: distributed/tensor_parallelism/llama2_model.py ================================================ # Copyright (c) Meta Platforms, Inc. and affiliates. # This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. from dataclasses import dataclass from typing import Optional, Tuple import torch import torch.nn.functional as F from torch import nn @dataclass class ModelArgs: dim: int = 4096 n_layers: int = 32 n_heads: int = 32 n_kv_heads: Optional[int] = None vocab_size: int = -1 # defined later by tokenizer multiple_of: int = 256 # make SwiGLU hidden layer size multiple of large power of 2 ffn_dim_multiplier: Optional[float] = None norm_eps: float = 1e-5 max_batch_size: int = 32 max_seq_len: int = 32768 # If `True`, then each transformer block init uses its layer ID, and if # `False`, each uses the total number of transformer blocks depth_init: bool = True def precompute_freqs_cis(dim: int, end: int, theta: float = 10000.0): """ Precompute the frequency tensor for complex exponentials (cis) with given dimensions. This function calculates a frequency tensor with complex exponentials using the given dimension 'dim' and the end index 'end'. The 'theta' parameter scales the frequencies. The returned tensor contains complex values in complex64 data type. Args: dim (int): Dimension of the frequency tensor. end (int): End index for precomputing frequencies. theta (float, optional): Scaling factor for frequency computation. Defaults to 10000.0. Returns: torch.Tensor: Precomputed frequency tensor with complex exponentials. """ freqs = 1.0 / (theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim)) t = torch.arange(end, device=freqs.device) # type: ignore freqs = torch.outer(t, freqs).float() # type: ignore freqs_cis = torch.polar(torch.ones_like(freqs), freqs) # complex64 return freqs_cis def reshape_for_broadcast(freqs_cis: torch.Tensor, x: torch.Tensor): """ Reshape frequency tensor for broadcasting it with another tensor. This function reshapes the frequency tensor to have the same shape as the target tensor 'x' for the purpose of broadcasting the frequency tensor during element-wise operations. Args: freqs_cis (torch.Tensor): Frequency tensor to be reshaped. x (torch.Tensor): Target tensor for broadcasting compatibility. Returns: torch.Tensor: Reshaped frequency tensor. """ ndim = x.ndim assert 0 <= 1 < ndim assert freqs_cis.shape == (x.shape[1], x.shape[-1]) shape = [d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)] return freqs_cis.view(*shape) def apply_rotary_emb( xq: torch.Tensor, xk: torch.Tensor, freqs_cis: torch.Tensor, ) -> Tuple[torch.Tensor, torch.Tensor]: """ Apply rotary embeddings to input tensors using the given frequency tensor. This function applies rotary embeddings to the given query 'xq' and key 'xk' tensors using the provided frequency tensor 'freqs_cis'. The input tensors are reshaped as complex numbers, and the frequency tensor is reshaped for broadcasting compatibility. The resulting tensors contain rotary embeddings and are returned as real tensors. Args: xq (torch.Tensor): Query tensor to apply rotary embeddings. xk (torch.Tensor): Key tensor to apply rotary embeddings. freqs_cis (torch.Tensor): Precomputed frequency tensor for complex exponentials. Returns: Tuple[torch.Tensor, torch.Tensor]: Tuple of modified query tensor and key tensor with rotary embeddings. """ xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2)) xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2)) freqs_cis = reshape_for_broadcast(freqs_cis, xq_) xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3) xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3) return xq_out.type_as(xq), xk_out.type_as(xk) def repeat_kv(x: torch.Tensor, n_rep: int) -> torch.Tensor: """torch.repeat_interleave(x, dim=2, repeats=n_rep)""" bs, slen, n_kv_heads, head_dim = x.shape if n_rep == 1: return x return ( x[:, :, :, None, :] .expand(bs, slen, n_kv_heads, n_rep, head_dim) .reshape(bs, slen, n_kv_heads * n_rep, head_dim) ) class RMSNorm(nn.Module): """ Initialize the RMSNorm normalization layer. Args: dim (int): The dimension of the input tensor. eps (float, optional): A small value added to the denominator for numerical stability. Default is 1e-6. Attributes: eps (float): A small value added to the denominator for numerical stability. weight (nn.Parameter): Learnable scaling parameter. """ def __init__(self, dim: int, eps: float = 1e-6): super().__init__() self.eps = eps self.weight = nn.Parameter(torch.ones(dim)) def _norm(self, x: torch.Tensor): return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) def forward(self, x: torch.Tensor): output = self._norm(x.float()).type_as(x) return output * self.weight def reset_parameters(self): torch.nn.init.ones_(self.weight) # type: ignore class Attention(nn.Module): """ Multi-head attention module. Args: model_args (ModelArgs): Model configuration arguments. Attributes: n_kv_heads (int): Number of key and value heads. n_heads (int): Number of query heads. n_local_kv_heads (int): Number of local key and value heads. n_rep (int): Number of repetitions for local heads. head_dim (int): Dimension size of each attention head. wq (Linear): Linear transformation for queries. wk (Linear): Linear transformation for keys. wv (Linear): Linear transformation for values. wo (Linear): Linear transformation for output. """ def __init__(self, model_args: ModelArgs): super().__init__() self.n_heads = model_args.n_heads self.n_kv_heads = ( model_args.n_heads if model_args.n_kv_heads is None else model_args.n_kv_heads ) self.n_rep = self.n_heads // self.n_kv_heads self.head_dim = model_args.dim // model_args.n_heads self.wq = nn.Linear( model_args.dim, model_args.n_heads * self.head_dim, bias=False ) self.wk = nn.Linear(model_args.dim, self.n_kv_heads * self.head_dim, bias=False) self.wv = nn.Linear(model_args.dim, self.n_kv_heads * self.head_dim, bias=False) self.wo = nn.Linear( model_args.n_heads * self.head_dim, model_args.dim, bias=False ) def init_weights(self, init_std: float): for linear in (self.wq, self.wk, self.wv): nn.init.trunc_normal_(linear.weight, mean=0.0, std=0.02) nn.init.trunc_normal_(self.wo.weight, mean=0.0, std=init_std) def forward( self, x: torch.Tensor, freqs_cis: torch.Tensor, ): """ Forward pass of the attention module. Args: x (torch.Tensor): Input tensor. freqs_cis (torch.Tensor): Precomputed frequency tensor. Returns: torch.Tensor: Output tensor after attention. """ bsz, seqlen, _ = x.shape xq, xk, xv = self.wq(x), self.wk(x), self.wv(x) xq = xq.view(bsz, seqlen, self.n_heads, self.head_dim) xk = xk.view(bsz, seqlen, self.n_kv_heads, self.head_dim) xv = xv.view(bsz, seqlen, self.n_kv_heads, self.head_dim) xq, xk = apply_rotary_emb(xq, xk, freqs_cis=freqs_cis) keys = repeat_kv(xk, self.n_rep) # (bs, seqlen, n_local_heads, head_dim) values = repeat_kv(xv, self.n_rep) # (bs, seqlen, n_local_heads, head_dim) xq = xq.transpose(1, 2) # (bs, n_local_heads, seqlen, head_dim) xk = keys.transpose(1, 2) # (bs, n_local_heads, seqlen, head_dim) xv = values.transpose(1, 2) # (bs, n_local_heads, seqlen, head_dim) # we use casual mask for training output = F.scaled_dot_product_attention(xq, xk, xv, is_causal=True) output = output.transpose( 1, 2 ).contiguous() # (bs, seqlen, n_local_heads, head_dim) output = output.view(bsz, seqlen, -1) return self.wo(output) class FeedForward(nn.Module): """ FeedForward module Args: dim (int): Input dimension. hidden_dim (int): Hidden dimension of the feedforward layer. multiple_of (int): Value to ensure hidden dimension is a multiple of this value. ffn_dim_multiplier (Optional[float]): Custom multiplier for hidden dimension. Defaults to None. Attributes: w1 (Linear): Linear transformation for the first layer. w2 (Linear): Linear transformation for the second layer. w3 (Linear): Linear transformation for the third layer. """ def __init__( self, dim: int, hidden_dim: int, multiple_of: int, ffn_dim_multiplier: Optional[float], ): super().__init__() hidden_dim = int(2 * hidden_dim / 3) # custom dim factor multiplier if ffn_dim_multiplier is not None: hidden_dim = int(ffn_dim_multiplier * hidden_dim) hidden_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of) self.w1 = nn.Linear(dim, hidden_dim, bias=False) self.w2 = nn.Linear(hidden_dim, dim, bias=False) self.w3 = nn.Linear(dim, hidden_dim, bias=False) def forward(self, x): return self.w2(F.silu(self.w1(x)) * self.w3(x)) def init_weights(self, init_std: float): nn.init.trunc_normal_(self.w1.weight, mean=0.0, std=0.02) for linear in (self.w2, self.w3): nn.init.trunc_normal_(linear.weight, mean=0.0, std=init_std) class TransformerBlock(nn.Module): """ TransformerBlock Module Args: layer_id (int): Identifier for the layer. model_args (ModelArgs): Model configuration arguments. Attributes: n_heads (int): Number of attention heads. dim (int): Dimension size of the model. head_dim (int): Dimension size of each attention head. attention (Attention): Attention module. feed_forward (FeedForward): FeedForward module. layer_id (int): Identifier for the layer. attention_norm (RMSNorm): Layer normalization for attention output. ffn_norm (RMSNorm): Layer normalization for feedforward output. """ def __init__(self, layer_id: int, model_args: ModelArgs): super().__init__() self.n_heads = model_args.n_heads self.dim = model_args.dim self.attention = Attention(model_args) self.feed_forward = FeedForward( dim=model_args.dim, hidden_dim=4 * model_args.dim, multiple_of=model_args.multiple_of, ffn_dim_multiplier=model_args.ffn_dim_multiplier, ) self.layer_id = layer_id self.num_layers = model_args.n_layers self.attention_norm = RMSNorm( dim=model_args.dim, eps=model_args.norm_eps ) self.ffn_norm = RMSNorm( dim=model_args.dim, eps=model_args.norm_eps ) if model_args.depth_init: self.weight_init_std = 0.02 / (2 * (self.layer_id + 1)) ** 0.5 else: self.weight_init_std = 0.02 / (2 * self.num_layers) ** 0.5 def forward( self, x: torch.Tensor, freqs_cis: torch.Tensor, ): """ Perform a forward pass through the TransformerBlock. Args: x (torch.Tensor): Input tensor. freqs_cis (torch.Tensor): Precomputed cosine and sine frequencies. Returns: torch.Tensor: Output tensor after applying attention and feedforward layers. """ h = x + self.attention(self.attention_norm(x), freqs_cis) out = h + self.feed_forward(self.ffn_norm(h)) return out def init_weights(self): for norm in (self.attention_norm, self.ffn_norm): norm.reset_parameters() self.attention.init_weights(self.weight_init_std) self.feed_forward.init_weights(self.weight_init_std) class Transformer(nn.Module): """ Transformer Module Args: model_args (ModelArgs): Model configuration arguments. Attributes: model_args (ModelArgs): Model configuration arguments. vocab_size (int): Vocabulary size. n_layers (int): Number of layers in the model. tok_embeddings (ParallelEmbedding): Token embeddings. layers (torch.nn.ModuleList): List of Transformer blocks. norm (RMSNorm): Layer normalization for the model output. output (ColumnParallelLinear): Linear layer for final output. freqs_cis (torch.Tensor): Precomputed cosine and sine frequencies. """ def __init__(self, model_args: ModelArgs): super().__init__() self.model_args = model_args self.vocab_size = model_args.vocab_size self.n_layers = model_args.n_layers self.model_dim = model_args.dim self.tok_embeddings = nn.Embedding(model_args.vocab_size, model_args.dim) self.register_buffer( "freqs_cis", precompute_freqs_cis( model_args.dim // model_args.n_heads, # Need to compute until at least the max token limit for generation # (use 2x max sequence length to be safe) model_args.max_seq_len * 2, ), ) self.layers = torch.nn.ModuleList() for layer_id in range(model_args.n_layers): self.layers.append(TransformerBlock(layer_id, model_args)) self.norm = RMSNorm( dim=model_args.dim, eps=model_args.norm_eps ) self.output = nn.Linear(model_args.dim, model_args.vocab_size, bias=False) self.init_weights() def init_weights(self): """ [Note: On ``init_weights`` vs. ``reset_parameters``] Modules may define ``reset_parameters`` to initialize parameter values. ``reset_parameters`` is meant to only initialize directly owned parameters/buffers, not those of their child modules, and it can be used to give the initial values for these tensors. Separately, users may want custom initialization for their modules, different from that in ``reset_parameters``. For this, we define ``init_weights``. We only call it in the constructor of this ``Transformer`` root module to avoid reinitializing tensors. """ with torch.device(self.freqs_cis.device): self.freqs_cis = precompute_freqs_cis( self.model_args.dim // self.model_args.n_heads, # Need to compute until at least the max token limit for generation # (use 2x max sequence length to be safe) self.model_args.max_seq_len * 2, ) nn.init.normal_(self.tok_embeddings.weight) for layer in self.layers: layer.init_weights() self.norm.reset_parameters() final_out_std = self.model_args.dim**-0.5 cutoff_factor = 3 nn.init.trunc_normal_( self.output.weight, mean=0.0, std=final_out_std, a=-cutoff_factor * final_out_std, b=cutoff_factor * final_out_std, ) def forward(self, tokens: torch.Tensor): """ Perform a forward pass through the Transformer model. Args: tokens (torch.Tensor): Input token indices. Returns: torch.Tensor: Output logits after applying the Transformer model. """ _bsz, seqlen = tokens.shape h = self.tok_embeddings(tokens) self.freqs_cis = self.freqs_cis.to(h.device) freqs_cis = self.freqs_cis[0:seqlen] for layer in self.layers: h = layer(h, freqs_cis) h = self.norm(h) output = self.output(h).float() return output @classmethod def from_model_args(cls, model_args: ModelArgs) -> "Transformer": """ Initialize a Transformer model from a ModelArgs object. Args: model_args (ModelArgs): Model configuration arguments. Returns: Transformer: Transformer model. """ return cls(model_args) ================================================ FILE: distributed/tensor_parallelism/log_utils.py ================================================ import logging import torch logging.basicConfig( format="%(asctime)s %(message)s", datefmt="%m/%d/%Y %I:%M:%S %p", level=logging.INFO ) def get_logger(): return logging.getLogger(__name__) def rank_log(_rank, logger, msg): """helper function to log only on global rank 0""" if _rank == 0: logger.info(f" {msg}") def verify_min_gpu_count(min_gpus: int = 2) -> bool: """ verification that we have at least 2 gpus to run dist examples """ has_gpu = torch.accelerator.is_available() gpu_count = torch.accelerator.device_count() return has_gpu and gpu_count >= min_gpus ================================================ FILE: distributed/tensor_parallelism/requirements.txt ================================================ # Python dependencies required for running the example torch >= 2.7.1; sys_platform == "linux" ================================================ FILE: distributed/tensor_parallelism/run_example.sh ================================================ # To run samples: # bash run_example.sh {file_to_run.py} {num_gpus} # where file_to_run = example to launch. Default = 'fsdp_tp_example.py' # num_gpus = num local gpus to use (must be at least 2). Default = 4 # samples to run include: # sequence_parallel_example.py # tensor_parallel_example.py # fsdp_tp_example.py echo "Launching ${1:-fsdp_tp_example.py} with ${2:-4} gpus" torchrun --nnodes=1 --nproc_per_node=${2:-4} --rdzv_id=101 --rdzv_endpoint="localhost:5972" ${1:-fsdp_tp_example.py} ================================================ FILE: distributed/tensor_parallelism/sequence_parallel_example.py ================================================ """ This is the script to test Sequence Parallel(SP) on a toy model in a Megetron-LM SPMD style. We show an E2E working flow from forward, backward and optimization. We use the example of two `nn.Linear` layers with an element-wise `nn.RELU` in between to show an example of sequence parallel, which was proposed in paper: https://arxiv.org/pdf/2205.05198.pdf. Like tensor parallel, we parallelize the first linear layer by column and also parallelize the second linear layer by row. But the input in each rank now is different so that we need one all-gather for input and one reduce-scatter in the end of the second linear layer. The following is an example command to run this code torchrun --nnodes 1 --nproc-per-node 4 sequence_parallel_example.py """ import os import sys import torch import torch.nn as nn import torch.distributed as dist from torch.distributed._tensor import Shard from torch.distributed.tensor.parallel import ( parallelize_module, ColwiseParallel, RowwiseParallel, ) from log_utils import rank_log, get_logger, verify_min_gpu_count # ---- GPU check ------------ _min_gpu_count = 2 if not verify_min_gpu_count(min_gpus=_min_gpu_count): print(f"Unable to locate sufficient {_min_gpu_count} gpus to run this example. Exiting.") sys.exit() # --------------------------- from torch.distributed._tensor.device_mesh import init_device_mesh class ToyModel(nn.Module): """MLP based model""" def __init__(self): super().__init__() self.in_proj = nn.Linear(10, 32) self.relu = nn.ReLU() self.out_proj = nn.Linear(32, 5) def forward(self, x): return self.out_proj(self.relu(self.in_proj(x))) """ Main body of the demo of a basic version of sequence parallel by using PyTorch native APIs. """ logger = get_logger() device_type = torch.accelerator.current_accelerator().type # create a device mesh based on the given world_size. device_mesh = init_device_mesh( device_type=device_type, mesh_shape=(int(os.environ["WORLD_SIZE"]),) ) _rank = device_mesh.get_rank() print(f"Starting PyTorch Sequence Parallel example on rank {_rank}.") rank_log(_rank, logger, f"Device Mesh created: {device_mesh=}") # create model and move it to GPU. Init_device_mesh has already assigned gpu ids... model = ToyModel().to(device_type) # Custom parallelization plan for the model sp_model = parallelize_module( module=model, device_mesh=device_mesh, parallelize_plan={ "in_proj": ColwiseParallel(input_layouts=Shard(0)), "out_proj": RowwiseParallel(output_layouts=Shard(0)), }, ) # Create a optimizer for the parallelized module. lr = 0.25 optimizer = torch.optim.AdamW(sp_model.parameters(), lr=lr, foreach=True) # Perform a num of iterations of forward/backward # and optimizations for the sharded module. num_iters = 10 rank_log(_rank, logger, "Sequence Parallel training starting...") for i in range(num_iters): # For SP, input can be different across all ranks. inp = torch.rand(20, 10, device=device_type) output = sp_model(inp) output.sum().backward() optimizer.step() rank_log(_rank, logger, f"Sequence Parallel iter {i} completed") rank_log(_rank, logger, "Sequence Parallel training completed!") if dist.is_initialized(): dist.destroy_process_group() ================================================ FILE: distributed/tensor_parallelism/tensor_parallel_example.py ================================================ """ This is the script to test Tensor Parallel(TP) on a toy model in a Megetron-LM SPMD style. We show an E2E working flow from forward, backward and optimization. More context about API designs can be found in the design: https://github.com/pytorch/pytorch/issues/89884. And it is built on top of Distributed Tensor which is proposed in: https://github.com/pytorch/pytorch/issues/88838. We use the example of two `nn.Linear` layers with an element-wise `nn.RELU` in between to show an example of Megatron-LM, which was proposed in paper: https://arxiv.org/abs/1909.08053. The basic idea is that we parallelize the first linear layer by column and also parallelize the second linear layer by row so that we only need one all reduce in the end of the second linear layer. We can speed up the model training by avoiding communications between two layers. To parallelize a nn module, we need to specify what parallel style we want to use and our `parallelize_module` API will parse and parallelize the modules based on the given `ParallelStyle`. We are using this PyTorch native Tensor Parallelism APIs in this example to show users how to use them. The following is an example command to run this code torchrun --nnodes 1 --nproc-per-node 4 tensor_parallel_example.py """ import os import sys import torch import torch.nn as nn import torch.distributed as dist from torch.distributed.tensor.parallel import ( parallelize_module, ColwiseParallel, RowwiseParallel, ) from log_utils import rank_log, get_logger, verify_min_gpu_count # ---- GPU check ------------ _min_gpu_count = 2 if not verify_min_gpu_count(min_gpus=_min_gpu_count): print(f"Unable to locate sufficient {_min_gpu_count} gpus to run this example. Exiting.") sys.exit() # --------------------------- from torch.distributed._tensor.device_mesh import init_device_mesh class ToyModel(nn.Module): """MLP based model""" def __init__(self): super(ToyModel, self).__init__() self.in_proj = nn.Linear(10, 32) self.relu = nn.ReLU() self.out_proj = nn.Linear(32, 5) def forward(self, x): return self.out_proj(self.relu(self.in_proj(x))) """ Main body of the demo of a basic version of tensor parallel by using PyTorch native APIs. """ logger = get_logger() # create a device mesh based on the given world_size. _world_size = int(os.environ["WORLD_SIZE"]) device_type = torch.accelerator.current_accelerator().type device_mesh = init_device_mesh(device_type=device_type, mesh_shape=(_world_size,)) _rank = device_mesh.get_rank() print(f"Starting PyTorch TP example on rank {_rank}.") assert ( _world_size % 2 == 0 ), f"TP examples require even number of GPUs, but got {_world_size} gpus" rank_log(_rank, logger, f"Device Mesh created: {device_mesh=}") # create model and move it to GPU - initdevice_type_mesh has already mapped GPU ids. tp_model = ToyModel().to(device_type) # Custom parallelization plan for the model tp_model = parallelize_module( module=tp_model, device_mesh=device_mesh, parallelize_plan={ "in_proj": ColwiseParallel(), "out_proj": RowwiseParallel(), }, ) # Create an optimizer for the parallelized module. lr = 0.25 optimizer = torch.optim.AdamW(tp_model.parameters(), lr=lr, foreach=True) # Perform a num of iterations of forward/backward # and optimizations for the sharded module. num_iters = 10 rank_log(_rank, logger, "Tensor Parallel training starting...") for i in range(num_iters): # For TP, input needs to be same across all TP ranks. # Setting the random seed is to mimic the behavior of dataloader. torch.manual_seed(i) inp = torch.rand(20, 10, device=device_type) output = tp_model(inp) output.sum().backward() optimizer.step() rank_log(_rank, logger, f"Tensor Parallel iter {i} completed") rank_log(_rank, logger, "Tensor Parallel training completed!") if dist.is_initialized(): dist.destroy_process_group() ================================================ FILE: docs/Makefile ================================================ # Minimal makefile for Sphinx documentation # # You can set these variables from the command line, and also # from the environment for the first two. SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = source BUILDDIR = build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) ================================================ FILE: docs/make.bat ================================================ @ECHO OFF pushd %~dp0 REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set SOURCEDIR=source set BUILDDIR=build if "%1" == "" goto help %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( echo. echo.The 'sphinx-build' command was not found. Make sure you have Sphinx echo.installed, then set the SPHINXBUILD environment variable to point echo.to the full path of the 'sphinx-build' executable. Alternatively you echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from echo.https://www.sphinx-doc.org/ exit /b 1 ) %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% goto end :help %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% :end popd ================================================ FILE: docs/requirements.txt ================================================ sphinx==5.3.0 #Pinned versions: 5.3.0 -e git+https://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme sphinx-panels ================================================ FILE: docs/source/_static/.gitkeep ================================================ ================================================ FILE: docs/source/conf.py ================================================ #!/usr/bin/env python3 # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. # Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html # -- Path setup -------------------------------------------------------------- # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # import os import sys import pytorch_sphinx_theme current_dir = os.path.dirname(__file__) target_dir = os.path.abspath(os.path.join(current_dir, "../..")) sys.path.insert(0, target_dir) print(target_dir) # -- Project information ----------------------------------------------------- project = "PyTorchExamples" copyright = "2022, Meta" author = "Meta" # The full version, including alpha/beta/rc tags release = "1.11" # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = ["sphinx.ext.napoleon", "sphinx.ext.autodoc", "sphinx_panels"] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. exclude_patterns = [] # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # # html_theme = 'alabaster' html_theme = "pytorch_sphinx_theme" html_theme_path = [pytorch_sphinx_theme.get_html_theme_path()] # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] panels_add_fontawesome_latex = True html_theme_options = { 'pytorch_project': 'examples', 'collapse_navigation': False, 'display_version': True, 'logo_only': False, 'analytics_id': 'UA-117752657-2', } ================================================ FILE: docs/source/index.rst ================================================ PyTorch Examples ================ This pages lists various PyTorch examples that you can use to learn and experiment with PyTorch. .. panels:: Image Classification Using ConvNets ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This example demonstrates how to run image classification with `Convolutional Neural Networks ConvNets `__ on the `MNIST `__ database. `GO TO EXAMPLE `__ :opticon:`link-external` --- Measuring Similarity using Siamese Network ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This example demonstrates how to measure similarity between two images using `Siamese network `__ on the `MNIST `__ database. `GO TO EXAMPLE `__ :opticon:`link-external` --- Word-level Language Modeling using RNN and Transformer ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This example demonstrates how to train a multi-layer `recurrent neural network (RNN) `__, such as Elman, GRU, or LSTM, or Transformer on a language modeling task by using the Wikitext-2 dataset. `GO TO EXAMPLE `__ :opticon:`link-external` --- Training ImageNet Classifiers ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This example demonstrates how you can train some of the most popular model architectures, including `ResNet `__, `AlexNet `__, and `VGG `__ on the `ImageNet `__ dataset. `GO TO EXAMPLE `__ :opticon:`link-external` --- Generative Adversarial Networks (DCGAN) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This example implements the `Unsupervised Representation Learning with Deep Convolutional Generative Adversarial Networks `__ paper. `GO TO EXAMPLE `__ :opticon:`link-external` --- Variational Auto-Encoders ^^^^^^^^^^^^^^^^^^^^^^^^^ This example implements the `Auto-Encoding Variational Bayes `__ paper with `ReLUs `__ and the Adam optimizer. `GO TO EXAMPLE `__ :opticon:`link-external` --- Super-resolution Using an Efficient Sub-Pixel CNN ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This example demonstrates how to use the sub-pixel convolution layer described in `Real-Time Single Image and Video Super-Resolution Using an Efficient Sub-Pixel Convolutional Neural Network `__ paper. This example trains a super-resolution network on the `BSD300 dataset `__. `GO TO EXAMPLE `__ :opticon:`link-external` --- HOGWILD! Training of Shared ConvNets ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `HOGWILD! `__ is a scheme that allows Stochastic Gradient Descent (SGD) parallelization without memory locking. This example demonstrates how to perform HOGWILD! training of shared ConvNets on MNIST. `GO TO EXAMPLE `__ :opticon:`link-external` --- Training a CartPole to balance with actor-critic ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This reinforcement learning tutorial demonstrates how to train a CartPole to balance in the `Gymnasium `__ toolkit by using the `Actor-Critic `__ method. `GO TO EXAMPLE `__ :opticon:`link-external` --- Time Sequence Prediction ^^^^^^^^^^^^^^^^^^^^^^^^ This beginner example demonstrates how to use LSTMCell to learn sine wave signals to predict the signal values in the future. `GO TO EXAMPLE `__ :opticon:`link-external` --- Implement the Neural Style Transfer algorithm on images ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This tutorial demonstrates how you can use PyTorch's implementation of the `Neural Style Transfer (NST) `__ algorithm on images. `GO TO EXAMPLE `__ :opticon:`link-external` --- PyTorch Module Transformations using fx ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This set of examples demonstrates the torch.fx toolkit. For more information about `torch.fx`, see `torch.fx Overview `__. `GO TO EXAMPLE `__ :opticon:`link-external` --- Distributed PyTorch ^^^^^^^^^^^^^^^^^^^ This set of examples demonstrates `Distributed Data Parallel (DDP) `__ and `Distributed RPC framework `__. Includes the code used in the `DDP tutorial series `__. `GO TO EXAMPLES `__ :opticon:`link-external` --- C++ Frontend ^^^^^^^^^^^^ The PyTorch C++ frontend is a C++14 library for CPU and GPU tensor computation. This set of examples includes a linear regression, autograd, image recognition (MNIST), and other useful examples using PyTorch C++ frontend. `GO TO EXAMPLES `__ :opticon:`link-external` --- Image Classification Using Forward-Forward Algorithm ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This example implements the paper `The Forward-Forward Algorithm: Some Preliminary Investigations `__ by Geoffrey Hinton. on the `MNIST `__ database. It is an introductory example to the Forward-Forward algorithm. `GO TO EXAMPLE `__ :opticon:`link-external` --- Graph Convolutional Network ^^^^^^^^^^^^^^^^^^^^^^^^^^^ This example implements the `Semi-Supervised Classification with Graph Convolutional Networks `__ paper on the CORA database. `GO TO EXAMPLE `__ :opticon:`link-external` ================================================ FILE: fast_neural_style/README.md ================================================ # fast-neural-style :city_sunrise: :rocket: This repository contains a pytorch implementation of an algorithm for artistic style transfer. The algorithm can be used to mix the content of an image with the style of another image. For example, here is a photograph of a door arch rendered in the style of a stained glass painting. The model uses the method described in [Perceptual Losses for Real-Time Style Transfer and Super-Resolution](https://arxiv.org/abs/1603.08155) along with [Instance Normalization](https://arxiv.org/pdf/1607.08022.pdf). The saved-models for examples shown in the README can be downloaded from [here](https://www.dropbox.com/s/lrvwfehqdcxoza8/saved_models.zip?dl=0).

## Requirements The program is written in Python, and uses [pytorch](http://pytorch.org/), [scipy](https://www.scipy.org). A GPU is not necessary, but can provide a significant speed up especially for training a new model. Regular sized images can be styled on a laptop or desktop using saved models. ## Usage Stylize image ``` python neural_style/neural_style.py eval --content-image --model --output-image --accel ``` - `--content-image`: path to content image you want to stylize. - `--model`: saved model to be used for stylizing the image (eg: `mosaic.pth`) - `--output-image`: path for saving the output image. - `--content-scale`: factor for scaling down the content image if memory is an issue (eg: value of 2 will halve the height and width of content-image) - `--accel`: use accelerator Train model ```bash python neural_style/neural_style.py train --dataset --style-image --save-model-dir --epochs 2 --accel ``` There are several command line arguments, the important ones are listed below - `--dataset`: path to training dataset, the path should point to a folder containing another folder with all the training images. I used COCO 2014 Training images dataset [80K/13GB] [(download)](https://cocodataset.org/#download). - `--style-image`: path to style-image. - `--save-model-dir`: path to folder where trained model will be saved. - `--accel`: use accelerator. If `--accel` argument is given, pytorch will search for available hardware acceleration device and attempt to use it. This example is known to work on CUDA, MPS and XPU devices. Refer to `neural_style/neural_style.py` for other command line arguments. For training new models you might have to tune the values of `--content-weight` and `--style-weight`. The mosaic style model shown above was trained with `--content-weight 1e5` and `--style-weight 1e10`. The remaining 3 models were also trained with similar order of weight parameters with slight variation in the `--style-weight` (`5e10` or `1e11`). ## Models Models for the examples shown below can be downloaded from [here](https://www.dropbox.com/s/lrvwfehqdcxoza8/saved_models.zip?dl=0) or by running the script `download_saved_models.py`.

================================================ FILE: fast_neural_style/download_saved_models.py ================================================ import os import zipfile # PyTorch 1.1 moves _download_url_to_file # from torch.utils.model_zoo to torch.hub # PyTorch 1.0 exists another _download_url_to_file # 2 argument # TODO: If you remove support PyTorch 1.0 or older, # You should remove torch.utils.model_zoo # Ref. PyTorch #18758 # https://github.com/pytorch/pytorch/pull/18758/commits try: from torch.utils.model_zoo import _download_url_to_file except ImportError: try: from torch.hub import download_url_to_file as _download_url_to_file except ImportError: from torch.hub import _download_url_to_file def unzip(source_filename, dest_dir): with zipfile.ZipFile(source_filename) as zf: zf.extractall(path=dest_dir) if __name__ == '__main__': _download_url_to_file('https://www.dropbox.com/s/lrvwfehqdcxoza8/saved_models.zip?dl=1', 'saved_models.zip', None, True) unzip('saved_models.zip', '.') ================================================ FILE: fast_neural_style/neural_style/__init__.py ================================================ ================================================ FILE: fast_neural_style/neural_style/neural_style.py ================================================ import argparse import os import sys import time import re import numpy as np import torch from torch.optim import Adam from torch.utils.data import DataLoader from torchvision import datasets from torchvision import transforms import torch.onnx import utils from transformer_net import TransformerNet from vgg import Vgg16 def check_paths(args): try: if not os.path.exists(args.save_model_dir): os.makedirs(args.save_model_dir) if args.checkpoint_model_dir is not None and not (os.path.exists(args.checkpoint_model_dir)): os.makedirs(args.checkpoint_model_dir) except OSError as e: print(e) sys.exit(1) def train(args): if args.accel: device = torch.accelerator.current_accelerator() else: device = torch.device("cpu") print(f"Using device: {device}") np.random.seed(args.seed) torch.manual_seed(args.seed) transform = transforms.Compose([ transforms.Resize(args.image_size), transforms.CenterCrop(args.image_size), transforms.ToTensor(), transforms.Lambda(lambda x: x.mul(255)) ]) train_dataset = datasets.ImageFolder(args.dataset, transform) train_loader = DataLoader(train_dataset, batch_size=args.batch_size) transformer = TransformerNet().to(device) optimizer = Adam(transformer.parameters(), args.lr) mse_loss = torch.nn.MSELoss() vgg = Vgg16(requires_grad=False).to(device) style_transform = transforms.Compose([ transforms.ToTensor(), transforms.Lambda(lambda x: x.mul(255)) ]) style = utils.load_image(args.style_image, size=args.style_size) style = style_transform(style) style = style.repeat(args.batch_size, 1, 1, 1).to(device) features_style = vgg(utils.normalize_batch(style)) gram_style = [utils.gram_matrix(y) for y in features_style] for e in range(args.epochs): transformer.train() agg_content_loss = 0. agg_style_loss = 0. count = 0 for batch_id, (x, _) in enumerate(train_loader): n_batch = len(x) count += n_batch optimizer.zero_grad() x = x.to(device) y = transformer(x) y = utils.normalize_batch(y) x = utils.normalize_batch(x) features_y = vgg(y) features_x = vgg(x) content_loss = args.content_weight * mse_loss(features_y.relu2_2, features_x.relu2_2) style_loss = 0. for ft_y, gm_s in zip(features_y, gram_style): gm_y = utils.gram_matrix(ft_y) style_loss += mse_loss(gm_y, gm_s[:n_batch, :, :]) style_loss *= args.style_weight total_loss = content_loss + style_loss total_loss.backward() optimizer.step() agg_content_loss += content_loss.item() agg_style_loss += style_loss.item() if (batch_id + 1) % args.log_interval == 0: mesg = "{}\tEpoch {}:\t[{}/{}]\tcontent: {:.6f}\tstyle: {:.6f}\ttotal: {:.6f}".format( time.ctime(), e + 1, count, len(train_dataset), agg_content_loss / (batch_id + 1), agg_style_loss / (batch_id + 1), (agg_content_loss + agg_style_loss) / (batch_id + 1) ) print(mesg) if args.checkpoint_model_dir is not None and (batch_id + 1) % args.checkpoint_interval == 0: transformer.eval().cpu() ckpt_model_filename = "ckpt_epoch_" + str(e) + "_batch_id_" + str(batch_id + 1) + ".pth" ckpt_model_path = os.path.join(args.checkpoint_model_dir, ckpt_model_filename) torch.save(transformer.state_dict(), ckpt_model_path) transformer.to(device).train() # save model transformer.eval().cpu() timestamp = time.strftime("%Y-%m-%d_%H-%M-%S") save_model_filename = f"epoch_{args.epochs}_{timestamp}_{args.content_weight}_{args.style_weight}.model" save_model_path = os.path.join(args.save_model_dir, save_model_filename) torch.save(transformer.state_dict(), save_model_path) print("\nDone, trained model saved at", save_model_path) def stylize(args): if args.accel: device = torch.accelerator.current_accelerator() else: device = torch.device("cpu") print(f"Using device: {device}") content_image = utils.load_image(args.content_image, scale=args.content_scale) content_transform = transforms.Compose([ transforms.ToTensor(), transforms.Lambda(lambda x: x.mul(255)) ]) content_image = content_transform(content_image) content_image = content_image.unsqueeze(0).to(device) if args.model.endswith(".onnx"): output = stylize_onnx(content_image, args) else: with torch.no_grad(): style_model = TransformerNet() state_dict = torch.load(args.model) # remove saved deprecated running_* keys in InstanceNorm from the checkpoint for k in list(state_dict.keys()): if re.search(r'in\d+\.running_(mean|var)$', k): del state_dict[k] style_model.load_state_dict(state_dict) style_model.to(device) style_model.eval() if args.export_onnx: assert args.export_onnx.endswith(".onnx"), "Export model file should end with .onnx" output = torch.onnx._export( style_model, content_image, args.export_onnx, opset_version=11, ).cpu() else: output = style_model(content_image).cpu() utils.save_image(args.output_image, output[0]) def stylize_onnx(content_image, args): """ Read ONNX model and run it using onnxruntime """ assert not args.export_onnx import onnxruntime ort_session = onnxruntime.InferenceSession(args.model) def to_numpy(tensor): return ( tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy() ) ort_inputs = {ort_session.get_inputs()[0].name: to_numpy(content_image)} ort_outs = ort_session.run(None, ort_inputs) img_out_y = ort_outs[0] return torch.from_numpy(img_out_y) def main(): main_arg_parser = argparse.ArgumentParser(description="parser for fast-neural-style") subparsers = main_arg_parser.add_subparsers(title="subcommands", dest="subcommand") train_arg_parser = subparsers.add_parser("train", help="parser for training arguments") train_arg_parser.add_argument("--epochs", type=int, default=2, help="number of training epochs, default is 2") train_arg_parser.add_argument("--batch-size", type=int, default=4, help="batch size for training, default is 4") train_arg_parser.add_argument("--dataset", type=str, required=True, help="path to training dataset, the path should point to a folder " "containing another folder with all the training images") train_arg_parser.add_argument("--style-image", type=str, default="images/style-images/mosaic.jpg", help="path to style-image") train_arg_parser.add_argument("--save-model-dir", type=str, required=True, help="path to folder where trained model will be saved.") train_arg_parser.add_argument("--checkpoint-model-dir", type=str, default=None, help="path to folder where checkpoints of trained models will be saved") train_arg_parser.add_argument("--image-size", type=int, default=256, help="size of training images, default is 256 X 256") train_arg_parser.add_argument("--style-size", type=int, default=None, help="size of style-image, default is the original size of style image") train_arg_parser.add_argument('--accel', action='store_true', help='use accelerator') train_arg_parser.add_argument("--seed", type=int, default=42, help="random seed for training") train_arg_parser.add_argument("--content-weight", type=float, default=1e5, help="weight for content-loss, default is 1e5") train_arg_parser.add_argument("--style-weight", type=float, default=1e10, help="weight for style-loss, default is 1e10") train_arg_parser.add_argument("--lr", type=float, default=1e-3, help="learning rate, default is 1e-3") train_arg_parser.add_argument("--log-interval", type=int, default=500, help="number of images after which the training loss is logged, default is 500") train_arg_parser.add_argument("--checkpoint-interval", type=int, default=2000, help="number of batches after which a checkpoint of the trained model will be created") eval_arg_parser = subparsers.add_parser("eval", help="parser for evaluation/stylizing arguments") eval_arg_parser.add_argument("--content-image", type=str, required=True, help="path to content image you want to stylize") eval_arg_parser.add_argument("--content-scale", type=float, default=None, help="factor for scaling down the content image") eval_arg_parser.add_argument("--output-image", type=str, required=True, help="path for saving the output image") eval_arg_parser.add_argument("--model", type=str, required=True, help="saved model to be used for stylizing the image. If file ends in .pth - PyTorch path is used, if in .onnx - Caffe2 path") eval_arg_parser.add_argument("--export_onnx", type=str, help="export ONNX model to a given file") eval_arg_parser.add_argument('--accel', action='store_true', help='use accelerator') args = main_arg_parser.parse_args() if args.subcommand is None: print("ERROR: specify either train or eval") sys.exit(1) if args.accel and not torch.accelerator.is_available(): print("ERROR: accelerator is not available, try running on CPU") sys.exit(1) if not args.accel and torch.accelerator.is_available(): print("WARNING: accelerator is available, run with --accel to enable it") if args.subcommand == "train": check_paths(args) train(args) else: stylize(args) if __name__ == "__main__": main() ================================================ FILE: fast_neural_style/neural_style/transformer_net.py ================================================ import torch class TransformerNet(torch.nn.Module): def __init__(self): super(TransformerNet, self).__init__() # Initial convolution layers self.conv1 = ConvLayer(3, 32, kernel_size=9, stride=1) self.in1 = torch.nn.InstanceNorm2d(32, affine=True) self.conv2 = ConvLayer(32, 64, kernel_size=3, stride=2) self.in2 = torch.nn.InstanceNorm2d(64, affine=True) self.conv3 = ConvLayer(64, 128, kernel_size=3, stride=2) self.in3 = torch.nn.InstanceNorm2d(128, affine=True) # Residual layers self.res1 = ResidualBlock(128) self.res2 = ResidualBlock(128) self.res3 = ResidualBlock(128) self.res4 = ResidualBlock(128) self.res5 = ResidualBlock(128) # Upsampling Layers self.deconv1 = UpsampleConvLayer(128, 64, kernel_size=3, stride=1, upsample=2) self.in4 = torch.nn.InstanceNorm2d(64, affine=True) self.deconv2 = UpsampleConvLayer(64, 32, kernel_size=3, stride=1, upsample=2) self.in5 = torch.nn.InstanceNorm2d(32, affine=True) self.deconv3 = ConvLayer(32, 3, kernel_size=9, stride=1) # Non-linearities self.relu = torch.nn.ReLU() def forward(self, X): y = self.relu(self.in1(self.conv1(X))) y = self.relu(self.in2(self.conv2(y))) y = self.relu(self.in3(self.conv3(y))) y = self.res1(y) y = self.res2(y) y = self.res3(y) y = self.res4(y) y = self.res5(y) y = self.relu(self.in4(self.deconv1(y))) y = self.relu(self.in5(self.deconv2(y))) y = self.deconv3(y) return y class ConvLayer(torch.nn.Module): def __init__(self, in_channels, out_channels, kernel_size, stride): super(ConvLayer, self).__init__() reflection_padding = kernel_size // 2 self.reflection_pad = torch.nn.ReflectionPad2d(reflection_padding) self.conv2d = torch.nn.Conv2d(in_channels, out_channels, kernel_size, stride) def forward(self, x): out = self.reflection_pad(x) out = self.conv2d(out) return out class ResidualBlock(torch.nn.Module): """ResidualBlock introduced in: https://arxiv.org/abs/1512.03385 recommended architecture: http://torch.ch/blog/2016/02/04/resnets.html """ def __init__(self, channels): super(ResidualBlock, self).__init__() self.conv1 = ConvLayer(channels, channels, kernel_size=3, stride=1) self.in1 = torch.nn.InstanceNorm2d(channels, affine=True) self.conv2 = ConvLayer(channels, channels, kernel_size=3, stride=1) self.in2 = torch.nn.InstanceNorm2d(channels, affine=True) self.relu = torch.nn.ReLU() def forward(self, x): residual = x out = self.relu(self.in1(self.conv1(x))) out = self.in2(self.conv2(out)) out = out + residual return out class UpsampleConvLayer(torch.nn.Module): """UpsampleConvLayer Upsamples the input and then does a convolution. This method gives better results compared to ConvTranspose2d. ref: http://distill.pub/2016/deconv-checkerboard/ """ def __init__(self, in_channels, out_channels, kernel_size, stride, upsample=None): super(UpsampleConvLayer, self).__init__() self.upsample = upsample reflection_padding = kernel_size // 2 self.reflection_pad = torch.nn.ReflectionPad2d(reflection_padding) self.conv2d = torch.nn.Conv2d(in_channels, out_channels, kernel_size, stride) def forward(self, x): x_in = x if self.upsample: x_in = torch.nn.functional.interpolate(x_in, mode='nearest', scale_factor=self.upsample) out = self.reflection_pad(x_in) out = self.conv2d(out) return out ================================================ FILE: fast_neural_style/neural_style/utils.py ================================================ import torch from PIL import Image def load_image(filename, size=None, scale=None): img = Image.open(filename).convert('RGB') if size is not None: img = img.resize((size, size), Image.ANTIALIAS) elif scale is not None: img = img.resize((int(img.size[0] / scale), int(img.size[1] / scale)), Image.ANTIALIAS) return img def save_image(filename, data): img = data.clone().clamp(0, 255).numpy() img = img.transpose(1, 2, 0).astype("uint8") img = Image.fromarray(img) img.save(filename) def gram_matrix(y): (b, ch, h, w) = y.size() features = y.view(b, ch, w * h) features_t = features.transpose(1, 2) gram = features.bmm(features_t) / (ch * h * w) return gram def normalize_batch(batch): # normalize using imagenet mean and std mean = batch.new_tensor([0.485, 0.456, 0.406]).view(-1, 1, 1) std = batch.new_tensor([0.229, 0.224, 0.225]).view(-1, 1, 1) batch = batch.div_(255.0) return (batch - mean) / std ================================================ FILE: fast_neural_style/neural_style/vgg.py ================================================ from collections import namedtuple import torch from torchvision import models class Vgg16(torch.nn.Module): def __init__(self, requires_grad=False): super(Vgg16, self).__init__() vgg_pretrained_features = models.vgg16(weights=models.VGG16_Weights.IMAGENET1K_V1).features self.slice1 = torch.nn.Sequential() self.slice2 = torch.nn.Sequential() self.slice3 = torch.nn.Sequential() self.slice4 = torch.nn.Sequential() for x in range(4): self.slice1.add_module(str(x), vgg_pretrained_features[x]) for x in range(4, 9): self.slice2.add_module(str(x), vgg_pretrained_features[x]) for x in range(9, 16): self.slice3.add_module(str(x), vgg_pretrained_features[x]) for x in range(16, 23): self.slice4.add_module(str(x), vgg_pretrained_features[x]) if not requires_grad: for param in self.parameters(): param.requires_grad = False def forward(self, X): h = self.slice1(X) h_relu1_2 = h h = self.slice2(h) h_relu2_2 = h h = self.slice3(h) h_relu3_3 = h h = self.slice4(h) h_relu4_3 = h vgg_outputs = namedtuple("VggOutputs", ['relu1_2', 'relu2_2', 'relu3_3', 'relu4_3']) out = vgg_outputs(h_relu1_2, h_relu2_2, h_relu3_3, h_relu4_3) return out ================================================ FILE: fast_neural_style/requirements.txt ================================================ numpy torch>=2.6 torchvision ================================================ FILE: fx/README.md ================================================ # FX Examples This folder contains several examples of program transformations implemented using `torch.fx`. More information about FX can be found in the [documentation](https://pytorch.org/docs/master/fx.html). Note that all examples should be runnable as standalone Python files. In the case of an exception, the example will appear in a subfolder with a `README.md` file explaining how to run the example. As FX is currently in a Beta release, the API or these examples are subject to change. ================================================ FILE: fx/custom_tracer.py ================================================ import torch from torch.fx import symbolic_trace, Tracer, Graph, GraphModule, Node from typing import Any, Callable, Dict, Optional, Tuple, Union """ How to Create and Use Custom Tracers `Tracer`--the class that implements the symbolic tracing functionality of `torch.fx.symbolic_trace`--can be subclassed to override various behaviors of the tracing process. In this tutorial, we'll demonstrate how to customize the symbolic tracing process using some handwritten Tracers. Each example will show that, by simply overriding a few methods in the `Tracer` class, you can alter the Graph produced by symbolic tracing. For a complete description of the methods that can be changed, refer to the docstrings of the methods in the Tracer class. Information can be found at: https://pytorch.org/docs/master/fx.html#torch.fx.Tracer If you want a real-world example of a custom tracer, check out FX's AST Rewriter in `rewriter.py`. `RewritingTracer` inherits from Tracer but overrides the `trace` function so that we can rewrite all calls to `assert` to the more FX-friendly `torch.assert`. Note that a call to `symbolic_trace(m)` is equivalent to `GraphModule(m, Tracer().trace(m))`. (`Tracer` is the default implementation of Tracer as defined in `symbolic_trace.py`.) """ """ Custom Tracer #1: Trace Through All `torch.nn.ReLU` Submodules During symbolic tracing, some submodules are traced through and their constituent ops are recorded; other submodules appear as an atomic "call_module" Node in the IR. A module in this latter category is called a "leaf module". By default, all modules in the PyTorch standard library (`torch.nn`) are leaf modules. We can change this by creating a custom Tracer and overriding `is_leaf_module`. In this case, we'll keep the default behavior for all `torch.nn` Modules except for `ReLU`. """ class M1(torch.nn.Module): def __init__(self): super().__init__() self.relu = torch.nn.ReLU() def forward(self, x): return self.relu(x) default_traced: GraphModule = symbolic_trace(M1()) """ Tracing with the default tracer and calling `print_tabular` produces: opcode name target args kwargs ----------- ------ -------- --------- -------- placeholder x x () {} call_module relu_1 relu (x,) {} output output output (relu_1,) {} """ default_traced.graph.print_tabular() class LowerReluTracer(Tracer): def is_leaf_module(self, m : torch.nn.Module, qualname : str): if isinstance(m, torch.nn.ReLU): return False return super().is_leaf_module(m, qualname) """ Tracing with our custom tracer and calling `print_tabular` produces: opcode name target args kwargs ------------- ------ --------------------------------- --------- ------------------ placeholder x x () {} call_function relu_1 (x,) {'inplace': False} output output output (relu_1,) {} """ lower_relu_tracer = LowerReluTracer() custom_traced_graph: Graph = lower_relu_tracer.trace(M1()) custom_traced_graph.print_tabular() """ Custom Tracer #2: Add an Extra Attribute to Each Node Here, we'll override `create_node` so that we can add a new attribute to each Node during its creation """ class M2(torch.nn.Module): def forward(self, a, b): return a + b class TaggingTracer(Tracer): def create_node(self, kind : str, target : Union[str, Callable], args : Tuple[Any], kwargs : Dict[str, Any], name : Optional[str] = None, type_expr : Optional[Any] = None) -> Node: n = super().create_node(kind, target, args, kwargs, name) n.tag = "foo" return n custom_traced_graph: Graph = TaggingTracer().trace(M2()) def assert_all_nodes_have_tags(g: Graph) -> bool: for n in g.nodes: if not hasattr(n, "tag") or not n.tag == "foo": return False return True # Prints "True" print(assert_all_nodes_have_tags(custom_traced_graph)) ================================================ FILE: fx/inline_function.py ================================================ import torch from torch.fx import Proxy, symbolic_trace from torch.fx.node import map_arg ''' How to Inline a Function Into an Existing Graph One reason you might want to inline a function is to get around FX's default tracing behavior. For example, unless you've defined a custom Tracer, the out-of-the-box implementation of ``symbolic_trace`` causes references to ``torch.nn`` module instances to appear as ``call_module`` calls rather than being traced through. Let's say this behavior is almost what you need; the only problem is that there's a single module call that you want to replace with an inlined trace of the function. Creating a custom Tracer would be too much. Instead, you can accomplish this using Proxies. The following code demonstrates how to trace a module and inline it into an existing Graph using Proxy. We'll trace our Graph, then iterate through its Nodes until we find the right place to swap out the ``call_module`` Node with an inlined trace. At that point, we'll create Proxies from the Node's args and kwargs. Finally, we'll call the function we want to replace with those Proxies--which will, in essence, "trace" that function. Finally, we'll insert the result of that call into our Graph. (This last step will automatically inline the function.) ''' # Sample module class M(torch.nn.Module): def __init__(self): super().__init__() self.relu = torch.nn.ReLU() def forward(self, x): return self.relu(x) + 1.0 # Symbolically trace an instance of `M`. After tracing, `self.relu` is # represented as a `call_module` Node. The full operation in the # generated `forward` function's code will appear as `self.relu(x)` m = symbolic_trace(M()) # Insert nodes from the ReLU graph in place of the original call to # `self.relu` # create a graph-appending tracer pointing to the original graph tracer = torch.fx.proxy.GraphAppendingTracer(m.graph) for node in m.graph.nodes: # Find `call_module` Node in `m` that corresponds to `self.relu`. # This is the Node we want to swap out for an inlined version of the # same call if (node.op, node.target) == ("call_module", "relu"): with m.graph.inserting_before(node): # Create a Proxy from each Node in the current Node's # args/kwargs proxy_args = map_arg(node.args, lambda n: Proxy(n, tracer)) proxy_kwargs = map_arg(node.kwargs, lambda n: Proxy(n, tracer)) # Call `m.relu` with the newly-created Proxy arguments. # `m.relu` is the generic version of the function; by # calling it with Proxies created from Nodes in `m`, we're # emitting Nodes that reference exiting values in the IR. # The result of this call is another Proxy, which we can # hook into our existing Graph to complete the function # inlining. proxy_output = m.relu(*proxy_args, **proxy_kwargs) # Replace the relu `call_module` node with the inlined # version of the function node.replace_all_uses_with(proxy_output.node) # Make sure that the old relu Node is erased m.graph.erase_node(node) ================================================ FILE: fx/invert.py ================================================ import torch import torch.fx as fx # An inverse mapping is one that takes a function f(x) and returns a function g # such that f(g(x)) == x. For example,since log(exp(x)) == x, exp and log are # inverses. invert_mapping = {} def add_inverse(a, b): invert_mapping[a] = b invert_mapping[b] = a inverses = [ (torch.sin, torch.arcsin), (torch.cos, torch.arccos), (torch.tan, torch.arctan), (torch.exp, torch.log), ] for a, b in inverses: add_inverse(a, b) # The general strategy is that we walk the graph backwards, transforming each # node into its inverse. To do so, we swap the outputs and inputs of the # functions, and then we look up its inverse in `invert_mapping`. Note that # this transform assumes that all operations take in only one input and return # one output. def invert(model: torch.nn.Module) -> torch.nn.Module: fx_model = fx.symbolic_trace(model) new_graph = fx.Graph() # As we're building up a new graph env = {} for node in reversed(fx_model.graph.nodes): if node.op == 'call_function': # This creates a node in the new graph with the inverse function, # and passes `env[node.name]` (i.e. the previous output node) as # input. new_node = new_graph.call_function(invert_mapping[node.target], (env[node.name],)) env[node.args[0].name] = new_node elif node.op == 'output': # We turn the output into an input placeholder new_node = new_graph.placeholder(node.name) env[node.args[0].name] = new_node elif node.op == 'placeholder': # We turn the input placeholder into an output new_graph.output(env[node.name]) else: raise RuntimeError("Not implemented") new_graph.lint() return fx.GraphModule(fx_model, new_graph) def f(x): return torch.exp(torch.tan(x)) res = invert(f) print(res.code) """ def forward(self, output): log_1 = torch.log(output); output = None arctan_1 = torch.arctan(log_1); log_1 = None return arctan_1 """ print(f(res((torch.arange(5) + 1)))) # [1., 2., 3., 4, 5.] ================================================ FILE: fx/module_tracer.py ================================================ """ Recording Module Hierarchy With a Custom Tracer In this example, we are going to define a custom `fx.Tracer` instance that-- for each recorded operation--also notes down the qualified name of the module from which that operation originated. The _qualified name_ is the path to the Module from the root module. More information about this concept can be found in the documentation for `Module.get_submodule`: https://github.com/pytorch/pytorch/blob/9f2aea7b88f69fc74ad90b1418663802f80c1863/torch/nn/modules/module.py#L385 """ import torch import torch.fx from typing import Any, Callable, Dict, Optional, Tuple class ModulePathTracer(torch.fx.Tracer): """ ModulePathTracer is an FX tracer that--for each operation--also records the qualified name of the Module from which the operation originated. """ # The current qualified name of the Module being traced. The top-level # module is signified by empty string. This is updated when entering # call_module and restored when exiting call_module current_module_qualified_name : str = '' # A map from FX Node to the qualname of the Module from which it # originated. This is recorded by `create_proxy` when recording an # operation node_to_originating_module : Dict[torch.fx.Node, str] = {} def call_module(self, m: torch.nn.Module, forward: Callable[..., Any], args : Tuple[Any, ...], kwargs : Dict[str, Any]) -> Any: """ Override of Tracer.call_module (see https://pytorch.org/docs/stable/fx.html#torch.fx.Tracer.call_module). This override: 1) Stores away the qualified name of the caller for restoration later 2) Installs the qualified name of the caller in `current_module_qualified_name` for retrieval by `create_proxy` 3) Delegates into the normal Tracer.call_module method 4) Restores the caller's qualified name into current_module_qualified_name """ old_qualname = self.current_module_qualified_name try: self.current_module_qualified_name = self.path_of_module(m) return super().call_module(m, forward, args, kwargs) finally: self.current_module_qualified_name = old_qualname def create_proxy(self, kind: str, target: torch.fx.node.Target, args: Tuple[Any, ...], kwargs: Dict[str, Any], name: Optional[str] = None, type_expr: Optional[Any] = None): """ Override of `Tracer.create_proxy`. This override intercepts the recording of every operation and stores away the current traced module's qualified name in `node_to_originating_module` """ proxy = super().create_proxy(kind, target, args, kwargs, name, type_expr) self.node_to_originating_module[proxy.node] = self.current_module_qualified_name return proxy # Testing: let's see how this works on a torchvision ResNet18 model import torchvision.models as models # Model under test rn18 = models.resnet18() # Instantiate our ModulePathTracer and use that to trace our ResNet18 tracer = ModulePathTracer() traced_rn18 = tracer.trace(rn18) # Print (node, module qualified name) for every node in the Graph for node in traced_rn18.nodes: module_qualname = tracer.node_to_originating_module.get(node) print('Node', node, 'is from module', module_qualname) """ Node x is from module Node conv1 is from module conv1 Node bn1 is from module bn1 Node relu is from module relu Node maxpool is from module maxpool Node layer1_0_conv1 is from module layer1.0.conv1 Node layer1_0_bn1 is from module layer1.0.bn1 Node layer1_0_relu is from module layer1.0.relu Node layer1_0_conv2 is from module layer1.0.conv2 Node layer1_0_bn2 is from module layer1.0.bn2 Node add is from module layer1.0 Node layer1_0_relu_1 is from module layer1.0.relu Node layer1_1_conv1 is from module layer1.1.conv1 Node layer1_1_bn1 is from module layer1.1.bn1 Node layer1_1_relu is from module layer1.1.relu Node layer1_1_conv2 is from module layer1.1.conv2 Node layer1_1_bn2 is from module layer1.1.bn2 Node add_1 is from module layer1.1 Node layer1_1_relu_1 is from module layer1.1.relu Node layer2_0_conv1 is from module layer2.0.conv1 Node layer2_0_bn1 is from module layer2.0.bn1 Node layer2_0_relu is from module layer2.0.relu Node layer2_0_conv2 is from module layer2.0.conv2 Node layer2_0_bn2 is from module layer2.0.bn2 Node layer2_0_downsample_0 is from module layer2.0.downsample.0 Node layer2_0_downsample_1 is from module layer2.0.downsample.1 Node add_2 is from module layer2.0 Node layer2_0_relu_1 is from module layer2.0.relu Node layer2_1_conv1 is from module layer2.1.conv1 Node layer2_1_bn1 is from module layer2.1.bn1 Node layer2_1_relu is from module layer2.1.relu Node layer2_1_conv2 is from module layer2.1.conv2 Node layer2_1_bn2 is from module layer2.1.bn2 Node add_3 is from module layer2.1 Node layer2_1_relu_1 is from module layer2.1.relu Node layer3_0_conv1 is from module layer3.0.conv1 Node layer3_0_bn1 is from module layer3.0.bn1 Node layer3_0_relu is from module layer3.0.relu Node layer3_0_conv2 is from module layer3.0.conv2 Node layer3_0_bn2 is from module layer3.0.bn2 Node layer3_0_downsample_0 is from module layer3.0.downsample.0 Node layer3_0_downsample_1 is from module layer3.0.downsample.1 Node add_4 is from module layer3.0 Node layer3_0_relu_1 is from module layer3.0.relu Node layer3_1_conv1 is from module layer3.1.conv1 Node layer3_1_bn1 is from module layer3.1.bn1 Node layer3_1_relu is from module layer3.1.relu Node layer3_1_conv2 is from module layer3.1.conv2 Node layer3_1_bn2 is from module layer3.1.bn2 Node add_5 is from module layer3.1 Node layer3_1_relu_1 is from module layer3.1.relu Node layer4_0_conv1 is from module layer4.0.conv1 Node layer4_0_bn1 is from module layer4.0.bn1 Node layer4_0_relu is from module layer4.0.relu Node layer4_0_conv2 is from module layer4.0.conv2 Node layer4_0_bn2 is from module layer4.0.bn2 Node layer4_0_downsample_0 is from module layer4.0.downsample.0 Node layer4_0_downsample_1 is from module layer4.0.downsample.1 Node add_6 is from module layer4.0 Node layer4_0_relu_1 is from module layer4.0.relu Node layer4_1_conv1 is from module layer4.1.conv1 Node layer4_1_bn1 is from module layer4.1.bn1 Node layer4_1_relu is from module layer4.1.relu Node layer4_1_conv2 is from module layer4.1.conv2 Node layer4_1_bn2 is from module layer4.1.bn2 Node add_7 is from module layer4.1 Node layer4_1_relu_1 is from module layer4.1.relu Node avgpool is from module avgpool Node flatten is from module Node fc is from module fc Node output is from module None """ ================================================ FILE: fx/native_interpreter/CMakeLists.txt ================================================ cmake_minimum_required(VERSION 3.1 FATAL_ERROR) project(interpreter) find_package(Torch REQUIRED) # Define our library target add_library(interpreter SHARED interpreter.cpp) set(CMAKE_CXX_STANDARD 17) # Link against LibTorch target_link_libraries(interpreter "${TORCH_LIBRARIES}") ================================================ FILE: fx/native_interpreter/README.md ================================================ # Converting PyTorch Code to a Native Runtime With FX and TorchScript Custom Classes In this example, we are going to build a pipeline that does the following things: 1. Converts (or “lowers”) code in a PyTorch module into another representation (we will define the representation within the example) 2. Registers an interpreter for that code representation that can be used in TorchScript or Python 3. Wrap the converted code into a format that can still be used in TorchScript compilation. We are going to build up a trivial interpreter for this example, but you can imagine extending the same process to work with more sophisticated backends, ones which may do code optimization or offloading to an accelerator. We will be using [TorchScript custom classes](https://pytorch.org/tutorials/advanced/torch_script_custom_classes.html) to expose this Interpreter to Python and TorchScript. You may want to review that tutorial and documentation before reading this example project. ### Defining the Interpreter We define the interpreter in `interpreter.cpp`. This interpreter is very limited: it only supports two element-wise operations (`add` and `mul`) and it only supports `Tensor` values. When this interpreter runs code, it iterates through the list of instructions and simply calls the appropriate PyTorch operator from C++. To build the interpreter into a shared-object file to be loaded in for use, use the following commands from this example’s root: ``` $ mkdir build $ cd build $ cmake -DCMAKE_PREFIX_PATH="$(python -c 'import torch.utils; print(torch.utils.cmake_prefix_path)')" .. $ make -j ``` After the build finishes, you should see `build/libinterpreter.so` (or with a different extension depending on your OS). We will use this dynamic library next when we load it up into a process to be used in execution. ### Defining the Transformation We define the code that transforms a `PyTorch` module to the format the interpreter understands in `use_interpreter.py`. Note that that file loads in the shared object we built in the previous step via a `torch.classes.load_library` call. `use_interpreter.py` contains driver code and the end that can be directly run to test the lowering transformation. ### Questions, Comments, Feedback Please direct questions and discussion to the [PyTorch forums](https://discuss.pytorch.org/). To report any issues with PyTorch (including FX and custom classes), please use the [issue tracker](https://github.com/pytorch/pytorch/issues). ================================================ FILE: fx/native_interpreter/interpreter.cpp ================================================ #include #include // ElementwiseInterpreter is a class that takes in a list of Instructions // (represented as triples (op, inputs, outputs)) and executes them in a // C++-based interpreter loop. For brevity, this interpreter only supports // two operations: element-wise add and element-wise mul. struct ElementwiseInterpreter : torch::CustomClassHolder { using InstructionType = std::tuple /*inputs*/, std::string /*output*/>; ElementwiseInterpreter() {} // Load a list of instructions into the interpreter. As specified above, // instructions specify the operation (currently support "add" and "mul"), // the names of the input values, and the name of the single output value // from this instruction void setInstructions(std::vector instructions) { instructions_ = std::move(instructions); } // Add a constant. The interpreter maintains a set of constants across // calls. They are keyed by name, and constants can be referenced in // Instructions by the name specified void addConstant(const std::string &name, at::Tensor value) { constants_.insert_or_assign(name, std::move(value)); } // Set the string names for the positional inputs to the function this // interpreter represents. When invoked, the interpreter will assign // the positional inputs to the names in the corresponding position in // input_names. void setInputNames(std::vector input_names) { input_names_ = std::move(input_names); } // Specify the output name for the function this interpreter represents. This // should match the "output" field of one of the instructions in the // instruction list, typically the last instruction. void setOutputName(std::string output_name) { output_name_ = std::move(output_name); } // Invoke this interpreter. This takes a list of positional inputs and returns // a single output. Currently, inputs and outputs must all be Tensors. at::Tensor __call__(std::vector inputs) { // Environment to hold local variables std::unordered_map environment; // Load inputs according to the specified names if (inputs.size() != input_names_.size()) { std::stringstream err; err << "Expected " << input_names_.size() << " inputs, but got " << inputs.size() << "!"; throw std::runtime_error(err.str()); } for (size_t i = 0; i < inputs.size(); ++i) { environment[input_names_[i]] = inputs[i]; } if (!output_name_) { throw std::runtime_error("Output name not specified!"); } for (InstructionType &instr : instructions_) { // Retrieve all input values for this op std::vector inputs; for (const auto &input_name : std::get<1>(instr)) { // Operator output values shadow constants. // Imagine all constants are defined in statements at the beginning // of a function (a la K&R C). Any definition of an output value must // necessarily come after constant definition in textual order. Thus, // We look up values in the environment first then the constant table // second to implement this shadowing behavior if (environment.find(input_name) != environment.end()) { inputs.push_back(environment.at(input_name)); } else if (constants_.find(input_name) != constants_.end()) { inputs.push_back(constants_.at(input_name)); } else { std::stringstream err; err << "Instruction referenced unknown value " << input_name << "!"; throw std::runtime_error(err.str()); } } // Run the specified operation at::Tensor result; const auto &op = std::get<0>(instr); if (op == "add") { if (inputs.size() != 2) { throw std::runtime_error("Unexpected number of inputs for add op!"); } result = inputs[0] + inputs[1]; } else if (op == "mul") { if (inputs.size() != 2) { throw std::runtime_error("Unexpected number of inputs for mul op!"); } result = inputs[0] * inputs[1]; } else { std::stringstream err; err << "Unknown operator " << op << "!"; throw std::runtime_error(err.str()); } // Write back result into environment const auto &output_name = std::get<2>(instr); environment[output_name] = std::move(result); } if (!environment.count(*output_name_)) { std::stringstream err; err << "Execution expected an output value with name "; err << *output_name_; err << " but no instruction produced a value with that name!"; throw std::runtime_error(err.str()); } return environment.at(*output_name_); } // Ser/De infrastructure. See // https://pytorch.org/tutorials/advanced/torch_script_custom_classes.html#defining-serialization-deserialization-methods-for-custom-c-classes // for more info. // This is the type we will use to marshall information on disk during // ser/de. It is a simple tuple composed of primitive types and simple // collection types like vector, optional, and dict. using SerializationType = std::tuple /*input_names_*/, c10::optional /*output_name_*/, c10::Dict /*constants_*/, std::vector /*instructions_*/ >; // This function yields the SerializationType instance for `this`. SerializationType __getstate__() const { return SerializationType{input_names_, output_name_, constants_, instructions_}; } // This function will create an instance of `ElementwiseInterpreter` given // an instance of `SerializationType`. static c10::intrusive_ptr __setstate__(SerializationType state) { auto instance = c10::make_intrusive(); std::tie(instance->input_names_, instance->output_name_, instance->constants_, instance->instructions_) = std::move(state); return instance; } // Class members std::vector input_names_; c10::optional output_name_; c10::Dict constants_; std::vector instructions_; }; // Register ElementwiseInterpreter as callable from Python // and TorchScript. TORCH_LIBRARY(NativeInterpretation, m) { m.class_("ElementwiseInterpreter") .def(torch::init<>()) .def("set_instructions", &ElementwiseInterpreter::setInstructions) .def("add_constant", &ElementwiseInterpreter::addConstant) .def("set_input_names", &ElementwiseInterpreter::setInputNames) .def("set_output_name", &ElementwiseInterpreter::setOutputName) .def("__call__", &ElementwiseInterpreter::__call__) .def_pickle( /* __getstate__ */ [](const c10::intrusive_ptr &self) { return self->__getstate__(); }, /* __setstate__ */ [](ElementwiseInterpreter::SerializationType state) { return ElementwiseInterpreter::__setstate__(std::move(state)); }); } ================================================ FILE: fx/native_interpreter/use_interpreter.py ================================================ import torch import torch.fx import operator # Does this path not exist? Check that you've done the following: # 1) Read README.md and follow the instructions to build libinterpreter. # 2) If this file still does not exist after you've followed those instructions, # check if it is under a different extension (e.g. `dylib` on mac or `dll` on # windows). torch.classes.load_library('build/libinterpreter.so') # This is what a lowering pass should look like: a function that takes # a valid nn.Module, symbolically traces it, lowers the Module to some # representation, and wraps that representation up into another # nn.Module instance that handles dispatch to the compiled/lowered code. # This will ensure that this lowering transformation still fits into the # PyTorch programming model and enables features like composing with other # transformations and TorchScript compilation. def lower_to_elementwise_interpreter(orig_mod : torch.nn.Module) -> torch.nn.Module: # ===== Stage 1: Symbolic trace the module ===== mod = torch.fx.symbolic_trace(orig_mod) # ===== Stage 2: Lower GraphModule representation to the C++ # interpreter's instruction format ====== instructions = [] constant_idx = 0 constants = {} fn_input_names = [] target_to_name = { operator.add : "add", operator.mul : "mul" } output_node : Optional[torch.fx.Node] = None # For each instruction, create a triple # (instruction_name : str, inputs : List[str], output : str) # to feed into the C++ interpreter for n in mod.graph.nodes: target, args, out_name = n.target, n.args, n.name assert len(n.kwargs) == 0, "kwargs currently not supported" if n.op == 'placeholder': # Placeholders specify function argument names. Save these # for later when we generate the wrapper GraphModule fn_input_names.append(target) elif n.op == 'call_function': assert target in target_to_name, "Unsupported call target " + target arg_names = [] for arg in args: if not isinstance(arg, torch.fx.Node): # Pull out constants. These constants will later be # fed to the interpreter C++ object via add_constant() arg_name = f'constant_{constant_idx}' constants[arg_name] = torch.Tensor( [arg] if isinstance(arg, numbers.Number) else arg) arg_names.append(arg_name) constant_idx += 1 else: arg_names.append(arg.name) instructions.append((target_to_name[target], arg_names, out_name)) elif n.op == 'output': if output_node is not None: raise RuntimeError('Multiple output nodes!') output_node = n else: raise RuntimeError('Unsupported opcode ' + n.op) interpreter = torch.classes.NativeInterpretation.ElementwiseInterpreter() # Load constants for k, v in constants.items(): interpreter.add_constant(k, v) # Specify names for positional input arguments interpreter.set_input_names(fn_input_names) # Load instructions interpreter.set_instructions(instructions) # Specify name for single output assert isinstance(output_node.args[0], torch.fx.Node) interpreter.set_output_name(output_node.args[0].name) # ===== Stage 3: Create a wrapper GraphModule around the interpreter ===== class WrapperModule(torch.nn.Module): def __init__(self, interpreter): super().__init__() self.interpreter = interpreter wrapper = WrapperModule(interpreter) # Create a forward() function that is compatible with TorchScript compilation. # Create a graph that: 1) Takes function arguments 2) Invokes the interpreter # 3) Returns the specified return value graph = torch.fx.Graph() # Add placeholders for fn inputs placeholder_nodes = [] for name in fn_input_names: placeholder_nodes.append(graph.create_node('placeholder', name)) # Get the interpreter object interpreter_node = graph.create_node('get_attr', 'interpreter') # Add a node to call the interpreter instance output_node = graph.create_node( op='call_method', target='__call__', args=(interpreter_node, placeholder_nodes)) # Register output graph.output(output_node) graph.lint(wrapper) # Return final GraphModule!!! return torch.fx.GraphModule(wrapper, graph) class MyElementwiseModule(torch.nn.Module): def forward(self, x, y): return x * y + y mem = MyElementwiseModule() lowered = lower_to_elementwise_interpreter(mem) print(lowered.code) # The lowered module can also be compiled into TorchScript scripted = torch.jit.script(lowered) print(scripted.graph) # Stress test correctness for _ in range(50): x, y = torch.randn(10, 20, 30), torch.randn(10, 20, 30) torch.testing.assert_allclose(lowered(x, y), mem(x, y)) torch.testing.assert_allclose(scripted(x, y), mem(x, y)) ================================================ FILE: fx/primitive_library.py ================================================ import torch import torch.fx """ In this example we are going do define a library of "composite" operations. Composite operations are those that are defined as callable functions that are composed of several other operations in their implementation. Composite operations allow you to choose at what level of abstraction you want to interpret/manipulate the code. We show that we can provide a function to inline these functions as well as use a custom Tracer to auto- matically inline such functions. Composite operations can be useful for exposing higher- level context to a backend/transform while still maintaining the ability to examine things at a more fine-grained level. """ def sigmoid_lowp(x : torch.Tensor): x = x.float() x = x.sigmoid() return x.half() # wrap() indicates that the passed-in function should always # be recorded as a call_function node rather than being traced # through. Later, we will see how we can: # a. Inline the implementation of such a function and # b. Define a tracer that automatically traces through such a function torch.fx.wrap(sigmoid_lowp) def add_lowp(a : torch.Tensor, b : torch.Tensor): a, b = a.float(), b.float() c = a + b return c.half() torch.fx.wrap(add_lowp) # Let's see what happens when we symbolically trace through some code # that uses these functions class Foo(torch.nn.Module): def forward(self, x, y): x = sigmoid_lowp(x) y = sigmoid_lowp(y) return add_lowp(x, y) traced = torch.fx.symbolic_trace(Foo()) print(traced.code) """ def forward(self, x, y): sigmoid_lowp = __main___sigmoid_lowp(x); x = None sigmoid_lowp_1 = __main___sigmoid_lowp(y); y = None add_lowp = __main___add_lowp(sigmoid_lowp, sigmoid_lowp_1); sigmoid_lowp = sigmoid_lowp_1 = None return add_lowp """ # Notice that the calls to `sigmoid_lowp` and `add_lowp` # appear literally in the trace; they are not traced through # ***** Inlining calls ***** # Now let's define a function that allows for inlining these calls # during graph manipulation. def inline_lowp_func(n : torch.fx.Node): # If we find a call to a function in our "lowp" module, inline it if n.op == 'call_function' and n.target.__module__ == inline_lowp_func.__module__: # We want to insert the operations comprising the implementation of the # function before the function itself. Then, we can swap the output value # of the function call with the output value for its implementation nodes tracer = torch.fx.proxy.GraphAppendingTracer(n.graph) with n.graph.inserting_before(n): # We can inline code by using `fx.Proxy` instances. # map_arg traverses all aggregate types and applies the given function # to Node instances in the data structure. In this case, we are applying # the fx.Proxy constructor. proxy_args = torch.fx.node.map_arg(n.args, lambda x: torch.fx.Proxy(x, tracer)) proxy_kwargs = torch.fx.node.map_arg(n.kwargs, lambda x: torch.fx.Proxy(x, tracer)) # Call the function itself with proxy arguments. This will emit # nodes in the graph corresponding to the operations in the im- # plementation of the function output_proxy = n.target(*proxy_args, **proxy_kwargs) # Now replace the original node's uses with the output node of # the implementation. node.replace_all_uses_with(output_proxy.node) # Delete the old node node.graph.erase_node(node) for node in traced.graph.nodes: if node.op == 'call_function' and node.target is sigmoid_lowp: inline_lowp_func(node) # Don't forget to recompile after graph manipulation traced.recompile() print(traced.code) """ def forward(self, x, y): float_1 = x.float(); x = None sigmoid = float_1.sigmoid(); float_1 = None half = sigmoid.half(); sigmoid = None float_2 = y.float(); y = None sigmoid_1 = float_2.sigmoid(); float_2 = None half_1 = sigmoid_1.half(); sigmoid_1 = None add_lowp = __main___add_lowp(half, half_1); half = half_1 = None return add_lowp """ # At this point, the implementation of `sigmoid_lowp` has been substituted # in for all of the calls to that function. # ***** Inlining calls during tracing ***** # Now we are going to define a custom tracer that can selectively inline # calls to certain composite operations on-the-fly. # New instance of our module f = Foo() class InliningTracer(torch.fx.Tracer): FNS_TO_INLINE = [add_lowp] def create_node(self, kind, target, args, kwargs, name=None, type_expr=None): if kind == 'call_function' and target in self.FNS_TO_INLINE: tracer = torch.fx.proxy.GraphAppendingTracer(self.graph) # Trace through the implementation of the function rather than # create a node proxy_args = torch.fx.node.map_arg(args, lambda x: torch.fx.Proxy(x, tracer)) proxy_kwargs = torch.fx.node.map_arg(kwargs, lambda x: torch.fx.Proxy(x, tracer)) return target(*proxy_args, **proxy_kwargs).node else: return super().create_node(kind, target, args, kwargs, name, type_expr) tracer = InliningTracer() graph = tracer.trace(f) module = torch.fx.GraphModule(f, graph) print(module.code) """ def forward(self, x, y): sigmoid_lowp = __main___sigmoid_lowp(x); x = None sigmoid_lowp_1 = __main___sigmoid_lowp(y); y = None float_1 = sigmoid_lowp.float(); sigmoid_lowp = None float_2 = sigmoid_lowp_1.float(); sigmoid_lowp_1 = None add = float_1 + float_2; float_1 = float_2 = None half = add.half(); add = None return half """ # As you can see, the implementation for `add_lowp` has been # inlined in the course of tracing with our InliningTracer. # Such functionality can be used to, for example, implement # a backend that wants to see the lowered form of some operations # but the high-level form of another. # ***** Future direction ***** # # We may define an API, such as `Tracer.is_leaf_function`, that # Tracer implementers can use to more easily specify the inlining # behavior implemented in InliningTracer. Such a method would return # True by default, but a Tracer can override it and return `False` for # functions the Tracer wants to be traced through. ================================================ FILE: fx/profiling_tracer.py ================================================ """ This file demonstrates using a custom FX Tracer to override the behavior of `torch.autograd.profiler.record_function` and make profiler ranges appear in FX-traced code. This is done with Python dynamic patching magic, allowing us to explicitly emit calls to `torch.ops.profiler._record_function_enter/_record_function_exit`. Please note that before https://github.com/pytorch/pytorch/pull/65180 lands, these ranges may be elimineated by `Graph.eliminate_dead_code` """ import torch import torch.fx # Setup: a module with `record_function` class Foo(torch.nn.Module): def forward(self, x): with torch.profiler.record_function('foo'): return torch.relu(x) f = Foo() x = torch.randn(5, 3, 2) with torch.autograd.profiler.profile() as prof: f(x) print(prof) # "foo" range is correctly recorded with normal execution """ ------------------- ------------ ------------ ------------ ------------ ------------ ------------ Name Self CPU % Self CPU CPU total % CPU total CPU time avg # of Calls ------------------- ------------ ------------ ------------ ------------ ------------ ------------ aten::zeros 6.10% 10.298us 10.04% 16.943us 16.943us 1 aten::empty 2.88% 4.857us 2.88% 4.857us 4.857us 1 aten::zero_ 1.06% 1.788us 1.06% 1.788us 1.788us 1 foo 21.28% 35.925us 89.96% 151.888us 151.888us 1 aten::empty 11.59% 19.572us 11.59% 19.572us 19.572us 1 aten::relu 23.81% 40.203us 57.09% 96.391us 96.391us 1 aten::clamp_min 3.87% 6.539us 33.28% 56.188us 56.188us 1 aten::empty 1.09% 1.847us 1.09% 1.847us 1.847us 1 aten::clamp_min 28.31% 47.802us 28.31% 47.802us 47.802us 1 ------------------- ------------ ------------ ------------ ------------ ------------ ------------ Self CPU time total: 168.831us """ traced = torch.fx.symbolic_trace(f) with torch.autograd.profiler.profile() as prof: traced(x) print(prof) # "foo" range is not recorded with FX tracing """ ------------------- ------------ ------------ ------------ ------------ ------------ ------------ Name Self CPU % Self CPU CPU total % CPU total CPU time avg # of Calls ------------------- ------------ ------------ ------------ ------------ ------------ ------------ aten::relu 23.50% 10.618us 100.00% 45.186us 45.186us 1 aten::clamp_min 18.05% 8.154us 76.50% 34.568us 34.568us 1 aten::empty 11.77% 5.317us 11.77% 5.317us 5.317us 1 aten::clamp_min 46.69% 21.097us 46.69% 21.097us 21.097us 1 ------------------- ------------ ------------ ------------ ------------ ------------ ------------ Self CPU time total: 45.186us """ class ProfilerTracer(torch.fx.Tracer): def trace(self, root, concrete_args=None): orig_record_function_enter = torch.autograd.profiler.record_function.__enter__ orig_record_function_exit = torch.autograd.profiler.record_function.__exit__ def fake_profiler_enter(_self): nonlocal self handle_proxy = self.create_proxy( kind='call_function', target=torch.ops.profiler._record_function_enter, args=(_self.name,), kwargs={}) assert getattr(_self, '_fx_profiler_ctx', None) is None setattr(_self, '_fx_profiler_ctx', handle_proxy) return handle_proxy def fake_profiler_exit(_self, exc_type, exc_value, traceback): assert hasattr(_self, '_fx_profiler_ctx') handle_proxy = _self._fx_profiler_ctx torch.ops.profiler._record_function_exit(handle_proxy) setattr(_self, '_fx_profiler_ctx', None) torch.autograd.profiler.record_function.__enter__ = fake_profiler_enter torch.autograd.profiler.record_function.__exit__ = fake_profiler_exit try: return super().trace(root, concrete_args) finally: torch.autograd.profiler.record_function.__enter__ = orig_record_function_enter torch.autograd.profiler.record_function.__exit__ = orig_record_function_exit pt = ProfilerTracer() graph_with_profiler = pt.trace(f) traced_with_profiler = torch.fx.GraphModule(pt.root, graph_with_profiler) with torch.autograd.profiler.profile() as prof: traced_with_profiler(x) print(prof) # "foo" range is recorded with special tracer behavior """ ------------------- ------------ ------------ ------------ ------------ ------------ ------------ Name Self CPU % Self CPU CPU total % CPU total CPU time avg # of Calls ------------------- ------------ ------------ ------------ ------------ ------------ ------------ foo 19.76% 39.928us 100.00% 202.055us 202.055us 1 aten::empty 3.93% 7.950us 3.93% 7.950us 7.950us 1 aten::relu 33.79% 68.282us 76.30% 154.177us 154.177us 1 aten::clamp_min 27.32% 55.198us 42.51% 85.895us 85.895us 1 aten::empty 1.28% 2.585us 1.28% 2.585us 2.585us 1 aten::clamp_min 13.91% 28.112us 13.91% 28.112us 28.112us 1 ------------------- ------------ ------------ ------------ ------------ ------------ ------------ Self CPU time total: 202.055us """ ================================================ FILE: fx/proxy_based_graph_creation.py ================================================ import torch from torch.fx import Proxy, Graph, GraphModule ''' How to Create a Graph Using Proxy Objects Instead of Tracing It's possible to directly create a Proxy object around a raw Node. This can be used to create a Graph independently of symbolic tracing. The following code demonstrates how to use Proxy with a raw Node to append operations to a fresh Graph. We'll create two parameters (``x`` and ``y``), perform some operations on those parameters, then add everything we created to the new Graph. We'll then wrap that Graph in a GraphModule. Doing so creates a runnable instance of ``nn.Module`` where previously-created operations are represented in the Module's ``forward`` function. By the end of the tutorial, we'll have added the following method to an empty ``nn.Module`` class. .. code-block:: python def forward(self, x, y): cat_1 = torch.cat([x, y]); x = y = None tanh_1 = torch.tanh(cat_1); cat_1 = None neg_1 = torch.neg(tanh_1); tanh_1 = None return neg_1 ''' # Create a graph independently of symbolic tracing graph = Graph() tracer = torch.fx.proxy.GraphAppendingTracer(graph) # Create raw Nodes raw1 = graph.placeholder('x') raw2 = graph.placeholder('y') # Initialize Proxies using the raw Nodes and graph's default tracer y = Proxy(raw1, tracer) z = Proxy(raw2, tracer) # y = Proxy(raw1) # z = Proxy(raw2) # Create other operations using the Proxies `y` and `z` a = torch.cat([y, z]) b = torch.tanh(a) c = torch.neg(b) # By using the graph's own appending tracer to create Proxies, # notice we can now use n-ary operators on operations without # multiple tracers being created at run-time (line 52) which leads # to errors # To try this out for yourself, replace lines 42, 43 # with 44, 45 z = torch.add(b, c) # Create a new output Node and add it to the Graph. By doing this, the # Graph will contain all the Nodes we just created (since they're all # linked to the output Node) graph.output(c.node) # Wrap our created Graph in a GraphModule to get a final, runnable # `nn.Module` instance mod = GraphModule(torch.nn.Module(), graph) ================================================ FILE: fx/replace_op.py ================================================ import torch from torch.fx import symbolic_trace import operator """ How to Replace One Op With Another 1. Iterate through all Nodes in your GraphModule's Graph. 2. Determine if the current Node should be replaced. (Suggested: match on the Node's ``target`` attribute). 3. Create a replacement Node and add it to the Graph. 4. Use the FX built-in ``replace_all_uses_with`` to replace all uses of the current Node with the replacement. 5. Delete the old Node from the graph. 6. Call ``recompile`` on the GraphModule. This updates the generated Python code to reflect the new Graph state. Currently, FX does not provide any way to guarantee that replaced operators are syntactically valid. It's up to the user to confirm that any new operators will work with the existing operands. The following code demonstrates an example of replacing any instance of addition with a bitwise AND. To examine how the Graph evolves during op replacement, add the statement `print(traced.graph)` after the line you want to inspect. Alternatively, call `traced.graph.print_tabular()` to see the IR in a tabular format. """ # Sample module class M(torch.nn.Module): def forward(self, x, y): return x + y, torch.add(x, y), x.add(y) # Symbolically trace an instance of the module traced = symbolic_trace(M()) # As demonstrated in the above example, there are several different ways # to denote addition. The possible cases are: # 1. `x + y` - A `call_function` Node with target `operator.add`. # We can match for equality on that `operator.add` directly. # 2. `torch.add(x, y)` - A `call_function` Node with target # `torch.add`. Similarly, we can match this function directly. # 3. `x.add(y)` - The Tensor method call, whose target we can match # as a string. patterns = set([operator.add, torch.add, "add"]) # Go through all the nodes in the Graph for n in traced.graph.nodes: # If the target matches one of the patterns if any(n.target == pattern for pattern in patterns): # Set the insert point, add the new node, and replace all uses # of `n` with the new node with traced.graph.inserting_after(n): new_node = traced.graph.call_function(torch.bitwise_and, n.args, n.kwargs) n.replace_all_uses_with(new_node) # Remove the old node from the graph traced.graph.erase_node(n) # Don't forget to recompile! traced.recompile() ================================================ FILE: fx/requirements.txt ================================================ torch torchvision ================================================ FILE: fx/subgraph_rewriter_basic_use.py ================================================ import torch from torch.fx import symbolic_trace, replace_pattern ''' How to Use the FX Subgraph Rewriter For easy subgraph rewriting, FX exposes the utility function: replace_pattern(gm : GraphModule, pattern : Callable, replacement : Callable) -> None `replace_pattern` matches all possible non-overlapping sets of operators and their data dependencies (`pattern`) in the Graph of a GraphModule (`gm`), then replaces each of these matched subgraphs with another subgraph (`replacement). The docstring for `replace_pattern` (located in `subgraph_rewriter.py`) gives an in-depth explanation as to how `pattern` and `replacement` should be specified, what happens during pattern matching, and other important technical details. This tutorial, therefore, is only meant to give an overview as to the FX Subgraph Rewriter's basic functionality. Let's go rewrite a Graph! ''' # Sample module class M(torch.nn.Module): def __init__(self): super().__init__() def forward(self, x, w1, w2): val1 = torch.neg(w1) m1 = torch.cat([val1, w2]).sum() val2 = torch.neg(w1) m2 = torch.cat([val2, w2]).sum() return x + torch.max(m1) + torch.max(m2) # Symbolically trace an instance of `M` traced = symbolic_trace(M()) # Define the pattern. The FX Subgraph Rewriter will match all # non-overlapping instances of the pattern in the larger graph. # Note that Pattern-matching is done based on data dependencies, # not Node names. Even though we're operating on Nodes named `a1` and # `a2` instead of `w1` and `w2`, the pattern is still a valid match # for the two instances of `torch.cat([w1, w2]).sum()` above. Only # operations that contribute to the single output value of the pattern # are considered def pattern(a1, a2): val1 = torch.neg(a1) return torch.cat([val1, a2]).sum() # Define the replacement (same rules as the pattern) def replacement(w1, w2): return torch.stack([w1, w2]) # Replace `pattern` with `replacement` in `traced` replace_pattern(traced, pattern, replacement) # After calling `replace_pattern`, the generated code is: ''' def forward(self, x, w1, w2): stack = torch.stack([w1, w2]) max_1 = torch.max(stack); stack = None add = x + max_1; x = max_1 = None stack_1 = torch.stack([w1, w2]); w1 = w2 = None max_2 = torch.max(stack_1); stack_1 = None add_1 = add + max_2; add = max_2 = None return add_1 ''' ================================================ FILE: fx/wrap_output_dynamically.py ================================================ from enum import Enum, auto import torch from torch.fx import GraphModule, Node, Proxy, symbolic_trace ''' Wrap Graph Output Dynamically The following code demonstrates how change an existing Graph based on parameters specified at runtime. We'll let the user specify an activation function from a predefined Enum list, then we'll symbolically trace it. Next, we'll create a Proxy from the last operation in the Graph. We'll call our traced activation function with this Proxy and insert the ``output`` Node from that call into our Graph. (This final step will automatically inline the entire traced function.) ''' # Sample module class M(torch.nn.Module): def __init__(self): super().__init__() def forward(self, x, y): y = torch.cat([x, y]) return y # Symbolically trace an instance of `M` traced = symbolic_trace(M()) # Selected activation functions class ActivationFunction(Enum): RELU = auto() LEAKY_RELU = auto() PRELU = auto() # Map activation function names to their implementation activation_functions = { ActivationFunction.RELU: torch.nn.ReLU(), ActivationFunction.LEAKY_RELU: torch.nn.LeakyReLU(), ActivationFunction.PRELU: torch.nn.PReLU(), } def wrap_in_activation_function(m: GraphModule, fn: ActivationFunction) -> GraphModule: # Get output node output_node: Optional[Node] = None for n in reversed(m.graph.nodes): if n.op == "output": output_node = n break assert output_node # Get the actual output (the "input" of the output node). This is # the Node we want to wrap in a user-specified activation function assert len(output_node.all_input_nodes) == 1 wrap_node = output_node.all_input_nodes[0] # Wrap the actual output in a Proxy wrap_proxy = Proxy(wrap_node) # Get the implementation of the specified activation function and # symbolically trace it fn_impl = activation_functions[fn] fn_impl_traced = symbolic_trace(fn_impl) # Call the specified activation function using the Proxy wrapper for # `output_op`. The result of this call is another Proxy, which we # can hook into our existing Graph. with traced.graph.inserting_after(wrap_node): fn_impl_output_node = fn_impl_traced(wrap_proxy) new_args = (fn_impl_output_node.node,) output_node.args = new_args m.recompile() # Example call x, y = torch.randn(5, 3), torch.randn(5, 3) orig_output = traced(x, y) wrap_in_activation_function(traced, ActivationFunction.LEAKY_RELU) new_output = traced(x, y) torch.testing.assert_close(new_output, torch.nn.LeakyReLU()(orig_output)) ================================================ FILE: gat/README.md ================================================ # Graph Attention Network This repository contains a PyTorch implementation of the **Graph Attention Networks (GAT)** based on the paper ["Graph Attention Network" by Velickovic et al](https://arxiv.org/abs/1710.10903v3). The Graph Attention Network is a powerful graph neural network model for learning represtations on graph-structured data, which has shown excellent performance in various tasks such as node classification, link prediction, and graph classification. ## Overview The Graph Attention Network (GAT) is a graph neural network architecture designed specifically for handling graph-structured data. It leverages multi-head attention mechanism to capture the information of neighboring nodes in an attentive manner to learn represtations for each node. This attention mechanism allows the model to focus on relevant nodes and adaptively weight their contributions during message passing. Check out the following resources for more ino on GATs: - [Blog post by the main auther, Petar Velickovic](https://petar-v.com/GAT/) - [Main paper](https://doi.org/10.48550/arXiv.1710.10903) This repository provides a clean and short implementation of the official GAT model using PyTorch. The code is well-documented and easy to understand, making it a valuable resource for researchers and practitioners interested in graph deep learning. ## Key Features - **GAT Model**: Implementation of the Graph Attention Network model with multi-head attention based on the paper "Graph Attention Network" by Velickovic et al. - **Graph Attention Layers**: Implementation of graph convolutional layers that aggregate information from neighboring nodes using a self-attention mechanisms to learn node importance weights. - **Training and Evaluation**: Code for training GAT models on graph-structured data and evaluating their performance on node classification tasks on the *Cora* benchmark dataset. --- # Requirements - Python 3.7 or higher - PyTorch 2.0 or higher - Requests 2.31 or higher - NumPy 1.24 or higher # Dataset The implementation includes support for the Cora dataset, a standard benchmark dataset for graph-based machine learning tasks. The Cora dataset consists of scientific publications, where nodes represent papers and edges represent citation relationships. Each paper is associated with a binary label indicating one of seven classes. The dataset is downloaded, preprocessed and ready to use. # Model Architecture The official architecture (used in this project) proposed in the paper "Graph Attention Network" by Velickovic et al. consists of two graph attention layers which incorporates the multi-head attention mechanisms during its message trasformation and aggregation. Each graph attention layer applies a shared self-attention mechanism to every node in the graph, allowing them to learn different representations based on the importance of their neighbors. In terms of activation functions, the GAT model employs both the **Exponential Linear Unit (ELU)** and the **Leaky Rectified Linear Unit (LeakyReLU)** activations, which introduce non-linearity to the model. ELU is used as the activation function for the **hidden layers**, while LeakyReLU is applied to the **attention coefficients** to ensure non-zero gradients for negative values. Following the official implementation, the first GAT layer consists of **K = 8 attention heads** computing **F' = 8 features** each (for a **total of 64 features**) followed by an exponential linear unit (ELU) activation on the layer outputs. The second GAT layer is used for classification: a **single attention head** that computes C features (where C is the number of classes), followed by a softmax activation for probablisitic outputs. (we use log-softmax instead for computational convenience with using NLLLoss) *Note that due to being an educational example, this implementation uses the full dense form of the adjacency matrix of the graph, and not the sparse form of the matrix. Thus all the operations in the model implemeation is done in a non-sparse from. This will not affect the model's performance accuracy-wise. However an sparse-friendly implementation will help with the efficiency in the use of resources, storage, and speed.* # Usage Training and evaluating the GAT model on the Cora dataset can be done through running the `main.py` script as follows: 1. Clone the PyTorch examples repository: ``` git clone https://github.com/pytorch/examples.git cd examples/gat ``` 2. Install the required dependencies: ``` pip install -r requirements.txt ``` 3. Train the GAT model by running the `main.py` script as follows:: (Example using the default parameters) ```bash python main.py --epochs 300 --lr 0.005 --l2 5e-4 --dropout-p 0.6 --num-heads 8 --hidden-dim 64 --val-every 20 ``` In more detail, the `main.py` script recieves following arguments: ``` usage: main.py [-h] [--epochs EPOCHS] [--lr LR] [--l2 L2] [--dropout-p DROPOUT_P] [--hidden-dim HIDDEN_DIM] [--num-heads NUM_HEADS] [--concat-heads] [--val-every VAL_EVERY] [--no-cuda] [--no-mps] [--dry-run] [--seed S] PyTorch Graph Attention Network options: -h, --help show this help message and exit --epochs EPOCHS number of epochs to train (default: 300) --lr LR learning rate (default: 0.005) --l2 L2 weight decay (default: 6e-4) --dropout-p DROPOUT_P dropout probability (default: 0.6) --hidden-dim HIDDEN_DIM dimension of the hidden representation (default: 64) --num-heads NUM_HEADS number of the attention heads (default: 4) --concat-heads wether to concatinate attention heads, or average over them (default: False) --val-every VAL_EVERY epochs to wait for print training and validation evaluation (default: 20) --no-accel disables accelerator --dry-run quickly check a single pass --seed S random seed (default: 13) ``` # Results After training for **300 epochs** with default hyperparameters on random train/val/test data splits, the GAT model achieves around **%81.25** classification accuracy on the test split. This result is comparable to the performance reported in the original paper. However, the results can vary due to the randomness of the train/val/test split. # Reference ``` @article{ velickovic2018graph, title="{Graph Attention Networks}", author={Veli{\v{c}}kovi{\'{c}}, Petar and Cucurull, Guillem and Casanova, Arantxa and Romero, Adriana and Li{\`{o}}, Pietro and Bengio, Yoshua}, journal={International Conference on Learning Representations}, year={2018}, url={https://openreview.net/forum?id=rJXMpikCZ}, } ``` - Paper on arxiv: [arXiv:1710.10903v3](https://doi.org/10.48550/arXiv.1710.10903) - Original paper repository: [https://github.com/PetarV-/GAT](https://github.com/PetarV-/GAT) ================================================ FILE: gat/main.py ================================================ import os import time import requests import tarfile import numpy as np import argparse import torch from torch import nn import torch.nn.functional as F from torch.optim import Adam ################################ ### GAT LAYER DEFINITION ### ################################ class GraphAttentionLayer(nn.Module): """ Graph Attention Layer (GAT) as described in the paper `"Graph Attention Networks" `. This operation can be mathematically described as: e_ij = a(W h_i, W h_j) α_ij = softmax_j(e_ij) = exp(e_ij) / Σ_k(exp(e_ik)) h_i' = σ(Σ_j(α_ij W h_j)) where h_i and h_j are the feature vectors of nodes i and j respectively, W is a learnable weight matrix, a is an attention mechanism that computes the attention coefficients e_ij, and σ is an activation function. """ def __init__(self, in_features: int, out_features: int, n_heads: int, concat: bool = False, dropout: float = 0.4, leaky_relu_slope: float = 0.2): super(GraphAttentionLayer, self).__init__() self.n_heads = n_heads # Number of attention heads self.concat = concat # wether to concatenate the final attention heads self.dropout = dropout # Dropout rate if concat: # concatenating the attention heads self.out_features = out_features # Number of output features per node assert out_features % n_heads == 0 # Ensure that out_features is a multiple of n_heads self.n_hidden = out_features // n_heads else: # averaging output over the attention heads (Used in the main paper) self.n_hidden = out_features # A shared linear transformation, parametrized by a weight matrix W is applied to every node # Initialize the weight matrix W self.W = nn.Parameter(torch.empty(size=(in_features, self.n_hidden * n_heads))) # Initialize the attention weights a self.a = nn.Parameter(torch.empty(size=(n_heads, 2 * self.n_hidden, 1))) self.leakyrelu = nn.LeakyReLU(leaky_relu_slope) # LeakyReLU activation function self.softmax = nn.Softmax(dim=1) # softmax activation function to the attention coefficients self.reset_parameters() # Reset the parameters def reset_parameters(self): """ Reinitialize learnable parameters. """ nn.init.xavier_normal_(self.W) nn.init.xavier_normal_(self.a) def _get_attention_scores(self, h_transformed: torch.Tensor): """calculates the attention scores e_ij for all pairs of nodes (i, j) in the graph in vectorized parallel form. for each pair of source and target nodes (i, j), the attention score e_ij is computed as follows: e_ij = LeakyReLU(a^T [Wh_i || Wh_j]) where || denotes the concatenation operation, and a and W are the learnable parameters. Args: h_transformed (torch.Tensor): Transformed feature matrix with shape (n_nodes, n_heads, n_hidden), where n_nodes is the number of nodes and out_features is the number of output features per node. Returns: torch.Tensor: Attention score matrix with shape (n_heads, n_nodes, n_nodes), where n_nodes is the number of nodes. """ source_scores = torch.matmul(h_transformed, self.a[:, :self.n_hidden, :]) target_scores = torch.matmul(h_transformed, self.a[:, self.n_hidden:, :]) # broadcast add # (n_heads, n_nodes, 1) + (n_heads, 1, n_nodes) = (n_heads, n_nodes, n_nodes) e = source_scores + target_scores.mT return self.leakyrelu(e) def forward(self, h: torch.Tensor, adj_mat: torch.Tensor): """ Performs a graph attention layer operation. Args: h (torch.Tensor): Input tensor representing node features. adj_mat (torch.Tensor): Adjacency matrix representing graph structure. Returns: torch.Tensor: Output tensor after the graph convolution operation. """ n_nodes = h.shape[0] # Apply linear transformation to node feature -> W h # output shape (n_nodes, n_hidden * n_heads) h_transformed = torch.mm(h, self.W) h_transformed = F.dropout(h_transformed, self.dropout, training=self.training) # splitting the heads by reshaping the tensor and putting heads dim first # output shape (n_heads, n_nodes, n_hidden) h_transformed = h_transformed.view(n_nodes, self.n_heads, self.n_hidden).permute(1, 0, 2) # getting the attention scores # output shape (n_heads, n_nodes, n_nodes) e = self._get_attention_scores(h_transformed) # Set the attention score for non-existent edges to -9e15 (MASKING NON-EXISTENT EDGES) connectivity_mask = -9e16 * torch.ones_like(e) e = torch.where(adj_mat > 0, e, connectivity_mask) # masked attention scores # attention coefficients are computed as a softmax over the rows # for each column j in the attention score matrix e attention = F.softmax(e, dim=-1) attention = F.dropout(attention, self.dropout, training=self.training) # final node embeddings are computed as a weighted average of the features of its neighbors h_prime = torch.matmul(attention, h_transformed) # concatenating/averaging the attention heads # output shape (n_nodes, out_features) if self.concat: h_prime = h_prime.permute(1, 0, 2).contiguous().view(n_nodes, self.out_features) else: h_prime = h_prime.mean(dim=0) return h_prime ################################ ### MAIN GAT NETWORK MODULE ### ################################ class GAT(nn.Module): """ Graph Attention Network (GAT) as described in the paper `"Graph Attention Networks" `. Consists of a 2-layer stack of Graph Attention Layers (GATs). The fist GAT Layer is followed by an ELU activation. And the second (final) layer is a GAT layer with a single attention head and softmax activation function. """ def __init__(self, in_features, n_hidden, n_heads, num_classes, concat=False, dropout=0.4, leaky_relu_slope=0.2): """ Initializes the GAT model. Args: in_features (int): number of input features per node. n_hidden (int): output size of the first Graph Attention Layer. n_heads (int): number of attention heads in the first Graph Attention Layer. num_classes (int): number of classes to predict for each node. concat (bool, optional): Wether to concatinate attention heads or take an average over them for the output of the first Graph Attention Layer. Defaults to False. dropout (float, optional): dropout rate. Defaults to 0.4. leaky_relu_slope (float, optional): alpha (slope) of the leaky relu activation. Defaults to 0.2. """ super(GAT, self).__init__() # Define the Graph Attention layers self.gat1 = GraphAttentionLayer( in_features=in_features, out_features=n_hidden, n_heads=n_heads, concat=concat, dropout=dropout, leaky_relu_slope=leaky_relu_slope ) self.gat2 = GraphAttentionLayer( in_features=n_hidden, out_features=num_classes, n_heads=1, concat=False, dropout=dropout, leaky_relu_slope=leaky_relu_slope ) def forward(self, input_tensor: torch.Tensor , adj_mat: torch.Tensor): """ Performs a forward pass through the network. Args: input_tensor (torch.Tensor): Input tensor representing node features. adj_mat (torch.Tensor): Adjacency matrix representing graph structure. Returns: torch.Tensor: Output tensor after the forward pass. """ # Apply the first Graph Attention layer x = self.gat1(input_tensor, adj_mat) x = F.elu(x) # Apply ELU activation function to the output of the first layer # Apply the second Graph Attention layer x = self.gat2(x, adj_mat) return F.log_softmax(x, dim=1) # Apply log softmax activation function ################################ ### LOADING THE CORA DATASET ### ################################ def load_cora(path='./cora', device='cpu'): """ Loads the Cora dataset. The dataset is downloaded from https://linqs-data.soe.ucsc.edu/public/lbc/cora.tgz. """ # Set the paths to the data files content_path = os.path.join(path, 'cora.content') cites_path = os.path.join(path, 'cora.cites') # Load data from files content_tensor = np.genfromtxt(content_path, dtype=np.dtype(str)) cites_tensor = np.genfromtxt(cites_path, dtype=np.int32) # Process features features = torch.FloatTensor(content_tensor[:, 1:-1].astype(np.int32)) # Extract feature values scale_vector = torch.sum(features, dim=1) # Compute sum of features for each node scale_vector = 1 / scale_vector # Compute reciprocal of the sums scale_vector[scale_vector == float('inf')] = 0 # Handle division by zero cases scale_vector = torch.diag(scale_vector).to_sparse() # Convert the scale vector to a sparse diagonal matrix features = scale_vector @ features # Scale the features using the scale vector # Process labels classes, labels = np.unique(content_tensor[:, -1], return_inverse=True) # Extract unique classes and map labels to indices labels = torch.LongTensor(labels) # Convert labels to a tensor # Process adjacency matrix idx = content_tensor[:, 0].astype(np.int32) # Extract node indices idx_map = {id: pos for pos, id in enumerate(idx)} # Create a dictionary to map indices to positions # Map node indices to positions in the adjacency matrix edges = np.array( list(map(lambda edge: [idx_map[edge[0]], idx_map[edge[1]]], cites_tensor)), dtype=np.int32) V = len(idx) # Number of nodes E = edges.shape[0] # Number of edges adj_mat = torch.sparse_coo_tensor(edges.T, torch.ones(E), (V, V), dtype=torch.int64) # Create the initial adjacency matrix as a sparse tensor adj_mat = torch.eye(V) + adj_mat # Add self-loops to the adjacency matrix # return features.to_sparse().to(device), labels.to(device), adj_mat.to_sparse().to(device) return features.to(device), labels.to(device), adj_mat.to(device) ################################# ### TRAIN AND TEST FUNCTIONS ### ################################# def train_iter(epoch, model, optimizer, criterion, input, target, mask_train, mask_val, print_every=10): start_t = time.time() model.train() optimizer.zero_grad() # Forward pass output = model(*input) loss = criterion(output[mask_train], target[mask_train]) # Compute the loss using the training mask loss.backward() optimizer.step() # Evaluate the model performance on training and validation sets loss_train, acc_train = test(model, criterion, input, target, mask_train) loss_val, acc_val = test(model, criterion, input, target, mask_val) if epoch % print_every == 0: # Print the training progress at specified intervals print(f'Epoch: {epoch:04d} ({(time.time() - start_t):.4f}s) loss_train: {loss_train:.4f} acc_train: {acc_train:.4f} loss_val: {loss_val:.4f} acc_val: {acc_val:.4f}') def test(model, criterion, input, target, mask): model.eval() with torch.no_grad(): output = model(*input) output, target = output[mask], target[mask] loss = criterion(output, target) acc = (output.argmax(dim=1) == target).float().sum() / len(target) return loss.item(), acc.item() if __name__ == '__main__': # Training settings # All defalut values are the same as in the config used in the main paper parser = argparse.ArgumentParser(description='PyTorch Graph Attention Network') parser.add_argument('--epochs', type=int, default=300, help='number of epochs to train (default: 300)') parser.add_argument('--lr', type=float, default=0.005, help='learning rate (default: 0.005)') parser.add_argument('--l2', type=float, default=5e-4, help='weight decay (default: 6e-4)') parser.add_argument('--dropout-p', type=float, default=0.6, help='dropout probability (default: 0.6)') parser.add_argument('--hidden-dim', type=int, default=64, help='dimension of the hidden representation (default: 64)') parser.add_argument('--num-heads', type=int, default=8, help='number of the attention heads (default: 4)') parser.add_argument('--concat-heads', action='store_true', help='wether to concatinate attention heads, or average over them (default: False)') parser.add_argument('--val-every', type=int, default=20, help='epochs to wait for print training and validation evaluation (default: 20)') parser.add_argument('--no-accel', action='store_true', help='disables CUDA training') parser.add_argument('--dry-run', action='store_true', help='quickly check a single pass') parser.add_argument('--seed', type=int, default=13, metavar='S', help='random seed (default: 13)') args = parser.parse_args() torch.manual_seed(args.seed) use_accel = not args.no_accel and torch.accelerator.is_available() # Set the device to run on if use_accel: device = torch.accelerator.current_accelerator() else: device = torch.device('cpu') print(f'Using {device} device') # Load the dataset cora_url = 'https://linqs-data.soe.ucsc.edu/public/lbc/cora.tgz' path = './cora' if os.path.isfile(os.path.join(path, 'cora.content')) and os.path.isfile(os.path.join(path, 'cora.cites')): print('Dataset already downloaded...') else: print('Downloading dataset...') with requests.get(cora_url, stream=True) as tgz_file: with tarfile.open(fileobj=tgz_file.raw, mode='r:gz') as tgz_object: tgz_object.extractall() print('Loading dataset...') # Load the dataset features, labels, adj_mat = load_cora(device=device) # Split the dataset into training, validation, and test sets idx = torch.randperm(len(labels)).to(device) idx_test, idx_val, idx_train = idx[:1200], idx[1200:1600], idx[1600:] # Create the model # The model consists of a 2-layer stack of Graph Attention Layers (GATs). gat_net = GAT( in_features=features.shape[1], # Number of input features per node n_hidden=args.hidden_dim, # Output size of the first Graph Attention Layer n_heads=args.num_heads, # Number of attention heads in the first Graph Attention Layer num_classes=labels.max().item() + 1, # Number of classes to predict for each node concat=args.concat_heads, # Wether to concatinate attention heads dropout=args.dropout_p, # Dropout rate leaky_relu_slope=0.2 # Alpha (slope) of the leaky relu activation ).to(device) # configure the optimizer and loss function optimizer = Adam(gat_net.parameters(), lr=args.lr, weight_decay=args.l2) criterion = nn.NLLLoss() # Train and evaluate the model for epoch in range(args.epochs): train_iter(epoch + 1, gat_net, optimizer, criterion, (features, adj_mat), labels, idx_train, idx_val, args.val_every) if args.dry_run: break loss_test, acc_test = test(gat_net, criterion, (features, adj_mat), labels, idx_test) print(f'Test set results: loss {loss_test:.4f} accuracy {acc_test:.4f}') ================================================ FILE: gat/requirements.txt ================================================ torch requests numpy ================================================ FILE: gcn/README.md ================================================ # Graph Convolutional Network This repository contains an implementation of Graph Convolutional Networks (GCN) based on the paper "Semi-Supervised Classification with Graph Convolutional Networks" by Thomas N. Kipf and Max Welling. ## Overview This project implements the GCN model proposed in the paper for semi-supervised node classification on graph-structured data. GCN leverages graph convolutions to aggregate information from neighboring nodes and learn node representations for downstream tasks. The implementation provides a flexible and efficient GCN model for graph-based machine learning tasks. ## Requirements ```bash pip install -r requirements.txt ``` # Usage ```bash python main.py --epochs 200 --lr 0.01 --l2 5e-4 --dropout-p 0.5 --hidden-dim 16 --val-every 20 --include-bias ``` # Dataset The implementation includes support for the Cora dataset, a standard benchmark dataset for graph-based machine learning tasks. The Cora dataset consists of scientific publications, where nodes represent papers and edges represent citation relationships. Each paper is associated with a binary label indicating one of seven classes. The dataset is downloaded, preprocessed and ready to use. ## Model Architecture The GCN model architecture follows the details provided in the paper. It consists of multiple graph convolutional layers with ReLU activation, followed by a final softmax layer for classification. The implementation supports customizable hyperparameters such as the number of hidden units, the number of layers, and dropout rate. # Results The model achieves a classification accuracy of 82.5% on the test set of the Cora dataset after 200 epochs of training. This result is comparable to the performance reported in the original paper. However, the results can vary due to the randomness of the train/val/test split. References Thomas N. Kipf and Max Welling. "Semi-Supervised Classification with Graph Convolutional Networks." Link to the paper Original paper repository: [https://github.com/tkipf/gcn](https://github.com/tkipf/gcn) ================================================ FILE: gcn/main.py ================================================ import os import time import requests import tarfile import numpy as np import argparse import torch from torch import nn import torch.nn.functional as F from torch.optim import Adam class GraphConv(nn.Module): """ Graph Convolutional Layer described in "Semi-Supervised Classification with Graph Convolutional Networks". Given an input feature representation for each node in a graph, the Graph Convolutional Layer aims to aggregate information from the node's neighborhood to update its own representation. This is achieved by applying a graph convolutional operation that combines the features of a node with the features of its neighboring nodes. Mathematically, the Graph Convolutional Layer can be described as follows: H' = f(D^(-1/2) * A * D^(-1/2) * H * W) where: H: Input feature matrix with shape (N, F_in), where N is the number of nodes and F_in is the number of input features per node. A: Adjacency matrix of the graph with shape (N, N), representing the relationships between nodes. W: Learnable weight matrix with shape (F_in, F_out), where F_out is the number of output features per node. D: The degree matrix. """ def __init__(self, input_dim, output_dim, use_bias=False): super(GraphConv, self).__init__() # Initialize the weight matrix W (in this case called `kernel`) self.kernel = nn.Parameter(torch.Tensor(input_dim, output_dim)) nn.init.xavier_normal_(self.kernel) # Initialize the weights using Xavier initialization # Initialize the bias (if use_bias is True) self.bias = None if use_bias: self.bias = nn.Parameter(torch.Tensor(output_dim)) nn.init.zeros_(self.bias) # Initialize the bias to zeros def forward(self, input_tensor, adj_mat): """ Performs a graph convolution operation. Args: input_tensor (torch.Tensor): Input tensor representing node features. adj_mat (torch.Tensor): Normalized adjacency matrix representing graph structure. Returns: torch.Tensor: Output tensor after the graph convolution operation. """ support = torch.mm(input_tensor, self.kernel) # Matrix multiplication between input and weight matrix output = torch.spmm(adj_mat, support) # Sparse matrix multiplication between adjacency matrix and support # Add the bias (if bias is not None) if self.bias is not None: output = output + self.bias return output class GCN(nn.Module): """ Graph Convolutional Network (GCN) as described in the paper `"Semi-Supervised Classification with Graph Convolutional Networks" `. The Graph Convolutional Network is a deep learning architecture designed for semi-supervised node classification tasks on graph-structured data. It leverages the graph structure to learn node representations by propagating information through the graph using graph convolutional layers. The original implementation consists of two stacked graph convolutional layers. The ReLU activation function is applied to the hidden representations, and the Softmax activation function is applied to the output representations. """ def __init__(self, input_dim, hidden_dim, output_dim, use_bias=True, dropout_p=0.1): super(GCN, self).__init__() # Define the Graph Convolution layers self.gc1 = GraphConv(input_dim, hidden_dim, use_bias=use_bias) self.gc2 = GraphConv(hidden_dim, output_dim, use_bias=use_bias) # Define the dropout layer self.dropout = nn.Dropout(dropout_p) def forward(self, input_tensor, adj_mat): """ Performs forward pass of the Graph Convolutional Network (GCN). Args: input_tensor (torch.Tensor): Input node feature matrix with shape (N, input_dim), where N is the number of nodes and input_dim is the number of input features per node. adj_mat (torch.Tensor): Normalized adjacency matrix of the graph with shape (N, N), representing the relationships between nodes. Returns: torch.Tensor: Output tensor with shape (N, output_dim), representing the predicted class probabilities for each node. """ # Perform the first graph convolutional layer x = self.gc1(input_tensor, adj_mat) x = F.relu(x) # Apply ReLU activation function x = self.dropout(x) # Apply dropout regularization # Perform the second graph convolutional layer x = self.gc2(x, adj_mat) # Apply log-softmax activation function for classification return F.log_softmax(x, dim=1) def load_cora(path='./cora', device='cpu'): """ The graph convolutional operation rquires the normalized adjacency matrix: D^(-1/2) * A * D^(-1/2). This step scales the adjacency matrix such that the features of neighboring nodes are weighted appropriately during aggregation. The steps involved in the renormalization trick are as follows: - Compute the degree matrix. - Compute the inverse square root of the degree matrix. - Multiply the inverse square root of the degree matrix with the adjacency matrix. """ # Set the paths to the data files content_path = os.path.join(path, 'cora.content') cites_path = os.path.join(path, 'cora.cites') # Load data from files content_tensor = np.genfromtxt(content_path, dtype=np.dtype(str)) cites_tensor = np.genfromtxt(cites_path, dtype=np.int32) # Process features features = torch.FloatTensor(content_tensor[:, 1:-1].astype(np.int32)) # Extract feature values scale_vector = torch.sum(features, dim=1) # Compute sum of features for each node scale_vector = 1 / scale_vector # Compute reciprocal of the sums scale_vector[scale_vector == float('inf')] = 0 # Handle division by zero cases scale_vector = torch.diag(scale_vector).to_sparse() # Convert the scale vector to a sparse diagonal matrix features = scale_vector @ features # Scale the features using the scale vector # Process labels classes, labels = np.unique(content_tensor[:, -1], return_inverse=True) # Extract unique classes and map labels to indices labels = torch.LongTensor(labels) # Convert labels to a tensor # Process adjacency matrix idx = content_tensor[:, 0].astype(np.int32) # Extract node indices idx_map = {id: pos for pos, id in enumerate(idx)} # Create a dictionary to map indices to positions # Map node indices to positions in the adjacency matrix edges = np.array( list(map(lambda edge: [idx_map[edge[0]], idx_map[edge[1]]], cites_tensor)), dtype=np.int32) V = len(idx) # Number of nodes E = edges.shape[0] # Number of edges adj_mat = torch.sparse_coo_tensor(edges.T, torch.ones(E), (V, V), dtype=torch.int64) # Create the initial adjacency matrix as a sparse tensor adj_mat = torch.eye(V) + adj_mat # Add self-loops to the adjacency matrix degree_mat = torch.sum(adj_mat, dim=1) # Compute the sum of each row in the adjacency matrix (degree matrix) degree_mat = torch.sqrt(1 / degree_mat) # Compute the reciprocal square root of the degrees degree_mat[degree_mat == float('inf')] = 0 # Handle division by zero cases degree_mat = torch.diag(degree_mat).to_sparse() # Convert the degree matrix to a sparse diagonal matrix adj_mat = degree_mat @ adj_mat @ degree_mat # Apply the renormalization trick return features.to_sparse().to(device), labels.to(device), adj_mat.to_sparse().to(device) def train_iter(epoch, model, optimizer, criterion, input, target, mask_train, mask_val, print_every=10): start_t = time.time() model.train() optimizer.zero_grad() # Forward pass output = model(*input) loss = criterion(output[mask_train], target[mask_train]) # Compute the loss using the training mask loss.backward() optimizer.step() # Evaluate the model performance on training and validation sets loss_train, acc_train = test(model, criterion, input, target, mask_train) loss_val, acc_val = test(model, criterion, input, target, mask_val) if epoch % print_every == 0: # Print the training progress at specified intervals print(f'Epoch: {epoch:04d} ({(time.time() - start_t):.4f}s) loss_train: {loss_train:.4f} acc_train: {acc_train:.4f} loss_val: {loss_val:.4f} acc_val: {acc_val:.4f}') def test(model, criterion, input, target, mask): model.eval() with torch.no_grad(): output = model(*input) output, target = output[mask], target[mask] loss = criterion(output, target) acc = (output.argmax(dim=1) == target).float().sum() / len(target) return loss.item(), acc.item() if __name__ == '__main__': parser = argparse.ArgumentParser(description='PyTorch Graph Convolutional Network') parser.add_argument('--epochs', type=int, default=200, help='number of epochs to train (default: 200)') parser.add_argument('--lr', type=float, default=0.01, help='learning rate (default: 0.01)') parser.add_argument('--l2', type=float, default=5e-4, help='weight decay (default: 5e-4)') parser.add_argument('--dropout-p', type=float, default=0.5, help='dropout probability (default: 0.5)') parser.add_argument('--hidden-dim', type=int, default=16, help='dimension of the hidden representation (default: 16)') parser.add_argument('--val-every', type=int, default=20, help='epochs to wait for print training and validation evaluation (default: 20)') parser.add_argument('--include-bias', action='store_true', help='use bias term in convolutions (default: False)') parser.add_argument('--no-accel', action='store_true', help='disables accelerator') parser.add_argument('--dry-run', action='store_true', help='quickly check a single pass') parser.add_argument('--seed', type=int, default=42, metavar='S', help='random seed (default: 42)') args = parser.parse_args() use_accel = not args.no_accel and torch.accelerator.is_available() torch.manual_seed(args.seed) if use_accel: device = torch.accelerator.current_accelerator() else: device = torch.device('cpu') print(f'Using {device} device') cora_url = 'https://linqs-data.soe.ucsc.edu/public/lbc/cora.tgz' print('Downloading dataset...') with requests.get(cora_url, stream=True) as tgz_file: with tarfile.open(fileobj=tgz_file.raw, mode='r:gz') as tgz_object: tgz_object.extractall() print('Loading dataset...') features, labels, adj_mat = load_cora(device=device) idx = torch.randperm(len(labels)).to(device) idx_test, idx_val, idx_train = idx[:1000], idx[1000:1500], idx[1500:] gcn = GCN(features.shape[1], args.hidden_dim, labels.max().item() + 1, args.include_bias, args.dropout_p).to(device) optimizer = Adam(gcn.parameters(), lr=args.lr, weight_decay=args.l2) criterion = nn.NLLLoss() for epoch in range(args.epochs): train_iter(epoch + 1, gcn, optimizer, criterion, (features, adj_mat), labels, idx_train, idx_val, args.val_every) if args.dry_run: break loss_test, acc_test = test(gcn, criterion, (features, adj_mat), labels, idx_test) print(f'Test set results: loss {loss_test:.4f} accuracy {acc_test:.4f}') ================================================ FILE: gcn/requirements.txt ================================================ torch>=2.6 torchvision requests numpy ================================================ FILE: imagenet/README.md ================================================ # ImageNet training in PyTorch This implements training of popular model architectures, such as ResNet, AlexNet, and VGG on the ImageNet dataset. ## Requirements - Install PyTorch ([pytorch.org](http://pytorch.org)) - `pip install -r requirements.txt` - Download the ImageNet dataset from http://www.image-net.org/ - Then, move and extract the training and validation images to labeled subfolders, using [the following shell script](extract_ILSVRC.sh) ## Training To train a model, run `main.py` with the desired model architecture and the path to the ImageNet dataset: ```bash python main.py -a resnet18 [imagenet-folder with train and val folders] ``` The default learning rate schedule starts at 0.1 and decays by a factor of 10 every 30 epochs. This is appropriate for ResNet and models with batch normalization, but too high for AlexNet and VGG. Use 0.01 as the initial learning rate for AlexNet or VGG: ```bash python main.py -a alexnet --lr 0.01 [imagenet-folder with train and val folders] ``` ## Use Dummy Data ImageNet dataset is large and time-consuming to download. To get started quickly, run `main.py` using dummy data by "--dummy". It's also useful for training speed benchmark. Note that the loss or accuracy is useless in this case. ```bash python main.py -a resnet18 --dummy ``` ## Multi-processing Distributed Data Parallel Training If running on CUDA, you should always use the NCCL backend for multi-processing distributed training since it currently provides the best distributed training performance. For XPU multiprocessing is not supported as of PyTorch 2.6. ### Single node, multiple GPUs: ```bash python main.py -a resnet50 --dist-url 'tcp://127.0.0.1:FREEPORT' --dist-backend 'nccl' --multiprocessing-distributed --world-size 1 --rank 0 [imagenet-folder with train and val folders] ``` ### Multiple nodes: Node 0: ```bash python main.py -a resnet50 --dist-url 'tcp://IP_OF_NODE0:FREEPORT' --dist-backend 'nccl' --multiprocessing-distributed --world-size 2 --rank 0 [imagenet-folder with train and val folders] ``` Node 1: ```bash python main.py -a resnet50 --dist-url 'tcp://IP_OF_NODE0:FREEPORT' --dist-backend 'nccl' --multiprocessing-distributed --world-size 2 --rank 1 [imagenet-folder with train and val folders] ``` ## Usage ```bash usage: main.py [-h] [-a ARCH] [-j N] [--epochs N] [--start-epoch N] [-b N] [--lr LR] [--momentum M] [--wd W] [-p N] [--resume PATH] [-e] [--pretrained] [--world-size WORLD_SIZE] [--rank RANK] [--dist-url DIST_URL] [--dist-backend DIST_BACKEND] [--seed SEED] [--gpu GPU] [--no-accel][--multiprocessing-distributed] [--dummy] [DIR] PyTorch ImageNet Training positional arguments: DIR path to dataset (default: imagenet) optional arguments: -h, --help show this help message and exit -a ARCH, --arch ARCH model architecture: alexnet | convnext_base | convnext_large | convnext_small | convnext_tiny | densenet121 | densenet161 | densenet169 | densenet201 | efficientnet_b0 | efficientnet_b1 | efficientnet_b2 | efficientnet_b3 | efficientnet_b4 | efficientnet_b5 | efficientnet_b6 | efficientnet_b7 | googlenet | inception_v3 | mnasnet0_5 | mnasnet0_75 | mnasnet1_0 | mnasnet1_3 | mobilenet_v2 | mobilenet_v3_large | mobilenet_v3_small | regnet_x_16gf | regnet_x_1_6gf | regnet_x_32gf | regnet_x_3_2gf | regnet_x_400mf | regnet_x_800mf | regnet_x_8gf | regnet_y_128gf | regnet_y_16gf | regnet_y_1_6gf | regnet_y_32gf | regnet_y_3_2gf | regnet_y_400mf | regnet_y_800mf | regnet_y_8gf | resnet101 | resnet152 | resnet18 | resnet34 | resnet50 | resnext101_32x8d | resnext50_32x4d | shufflenet_v2_x0_5 | shufflenet_v2_x1_0 | shufflenet_v2_x1_5 | shufflenet_v2_x2_0 | squeezenet1_0 | squeezenet1_1 | vgg11 | vgg11_bn | vgg13 | vgg13_bn | vgg16 | vgg16_bn | vgg19 | vgg19_bn | vit_b_16 | vit_b_32 | vit_l_16 | vit_l_32 | wide_resnet101_2 | wide_resnet50_2 (default: resnet18) -j N, --workers N number of data loading workers (default: 4) --epochs N number of total epochs to run --start-epoch N manual epoch number (useful on restarts) -b N, --batch-size N mini-batch size (default: 256), this is the total batch size of all GPUs on the current node when using Data Parallel or Distributed Data Parallel --lr LR, --learning-rate LR initial learning rate --momentum M momentum --wd W, --weight-decay W weight decay (default: 1e-4) -p N, --print-freq N print frequency (default: 10) --resume PATH path to latest checkpoint (default: none) -e, --evaluate evaluate model on validation set --pretrained use pre-trained model --world-size WORLD_SIZE number of nodes for distributed training --rank RANK node rank for distributed training --dist-url DIST_URL url used to set up distributed training --dist-backend DIST_BACKEND distributed backend --seed SEED seed for initializing training. --gpu GPU GPU id to use. --no-accel disables accelerator --multiprocessing-distributed Use multi-processing distributed training to launch N processes per node, which has N GPUs. This is the fastest way to use PyTorch for either single node or multi node data parallel training --dummy use fake data to benchmark ``` ================================================ FILE: imagenet/extract_ILSVRC.sh ================================================ #!/bin/bash # # script to extract ImageNet dataset # ILSVRC2012_img_train.tar (about 138 GB) # ILSVRC2012_img_val.tar (about 6.3 GB) # make sure ILSVRC2012_img_train.tar & ILSVRC2012_img_val.tar in your current directory # # Adapted from: # https://github.com/facebook/fb.resnet.torch/blob/master/INSTALL.md # https://gist.github.com/BIGBALLON/8a71d225eff18d88e469e6ea9b39cef4 # # imagenet/train/ # ├── n01440764 # │ ├── n01440764_10026.JPEG # │ ├── n01440764_10027.JPEG # │ ├── ...... # ├── ...... # imagenet/val/ # ├── n01440764 # │ ├── ILSVRC2012_val_00000293.JPEG # │ ├── ILSVRC2012_val_00002138.JPEG # │ ├── ...... # ├── ...... # # # Make imagnet directory # mkdir imagenet # # Extract the training data: # # Create train directory; move .tar file; change directory mkdir imagenet/train && mv ILSVRC2012_img_train.tar imagenet/train/ && cd imagenet/train # Extract training set; remove compressed file tar -xvf ILSVRC2012_img_train.tar && rm -f ILSVRC2012_img_train.tar # # At this stage imagenet/train will contain 1000 compressed .tar files, one for each category # # For each .tar file: # 1. create directory with same name as .tar file # 2. extract and copy contents of .tar file into directory # 3. remove .tar file find . -name "*.tar" | while read NAME ; do mkdir -p "${NAME%.tar}"; tar -xvf "${NAME}" -C "${NAME%.tar}"; rm -f "${NAME}"; done # # This results in a training directory like so: # # imagenet/train/ # ├── n01440764 # │ ├── n01440764_10026.JPEG # │ ├── n01440764_10027.JPEG # │ ├── ...... # ├── ...... # # Change back to original directory cd ../.. # # Extract the validation data and move images to subfolders: # # Create validation directory; move .tar file; change directory; extract validation .tar; remove compressed file mkdir imagenet/val && mv ILSVRC2012_img_val.tar imagenet/val/ && cd imagenet/val && tar -xvf ILSVRC2012_img_val.tar && rm -f ILSVRC2012_img_val.tar # get script from soumith and run; this script creates all class directories and moves images into corresponding directories wget -qO- https://raw.githubusercontent.com/soumith/imagenetloader.torch/master/valprep.sh | bash # # This results in a validation directory like so: # # imagenet/val/ # ├── n01440764 # │ ├── ILSVRC2012_val_00000293.JPEG # │ ├── ILSVRC2012_val_00002138.JPEG # │ ├── ...... # ├── ...... # # # Check total files after extract # # $ find train/ -name "*.JPEG" | wc -l # 1281167 # $ find val/ -name "*.JPEG" | wc -l # 50000 # ================================================ FILE: imagenet/main.py ================================================ import argparse import os import random import shutil import time import warnings from enum import Enum import torch import torch.backends.cudnn as cudnn import torch.distributed as dist import torch.multiprocessing as mp import torch.nn as nn import torch.nn.parallel import torch.optim import torch.utils.data import torch.utils.data.distributed import torchvision.datasets as datasets import torchvision.models as models import torchvision.transforms as transforms from torch.optim.lr_scheduler import StepLR from torch.utils.data import Subset model_names = sorted(name for name in models.__dict__ if name.islower() and not name.startswith("__") and callable(models.__dict__[name])) parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') parser.add_argument('data', metavar='DIR', nargs='?', default='imagenet', help='path to dataset (default: imagenet)') parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet18', choices=model_names, help='model architecture: ' + ' | '.join(model_names) + ' (default: resnet18)') parser.add_argument('-j', '--workers', default=4, type=int, metavar='N', help='number of data loading workers (default: 4)') parser.add_argument('--epochs', default=90, type=int, metavar='N', help='number of total epochs to run') parser.add_argument('--start-epoch', default=0, type=int, metavar='N', help='manual epoch number (useful on restarts)') parser.add_argument('-b', '--batch-size', default=256, type=int, metavar='N', help='mini-batch size (default: 256), this is the total ' 'batch size of all GPUs on the current node when ' 'using Data Parallel or Distributed Data Parallel') parser.add_argument('--lr', '--learning-rate', default=0.1, type=float, metavar='LR', help='initial learning rate', dest='lr') parser.add_argument('--momentum', default=0.9, type=float, metavar='M', help='momentum') parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, metavar='W', help='weight decay (default: 1e-4)', dest='weight_decay') parser.add_argument('-p', '--print-freq', default=10, type=int, metavar='N', help='print frequency (default: 10)') parser.add_argument('--resume', default='', type=str, metavar='PATH', help='path to latest checkpoint (default: none)') parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', help='evaluate model on validation set') parser.add_argument('--pretrained', dest='pretrained', action='store_true', help='use pre-trained model') parser.add_argument('--world-size', default=-1, type=int, help='number of nodes for distributed training') parser.add_argument('--rank', default=-1, type=int, help='node rank for distributed training') parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str, help='url used to set up distributed training') parser.add_argument('--dist-backend', default='nccl', type=str, help='distributed backend') parser.add_argument('--seed', default=None, type=int, help='seed for initializing training. ') parser.add_argument('--gpu', default=None, type=int, help='GPU id to use.') parser.add_argument('--no-accel', action='store_true', help='disables accelerator') parser.add_argument('--multiprocessing-distributed', action='store_true', help='Use multi-processing distributed training to launch ' 'N processes per node, which has N GPUs. This is the ' 'fastest way to use PyTorch for either single node or ' 'multi node data parallel training') parser.add_argument('--dummy', action='store_true', help="use fake data to benchmark") best_acc1 = 0 def main(): args = parser.parse_args() if args.seed is not None: random.seed(args.seed) torch.manual_seed(args.seed) cudnn.deterministic = True cudnn.benchmark = False warnings.warn('You have chosen to seed training. ' 'This will turn on the CUDNN deterministic setting, ' 'which can slow down your training considerably! ' 'You may see unexpected behavior when restarting ' 'from checkpoints.') if args.gpu is not None: warnings.warn('You have chosen a specific GPU. This will completely ' 'disable data parallelism.') if args.dist_url == "env://" and args.world_size == -1: args.world_size = int(os.environ["WORLD_SIZE"]) args.distributed = args.world_size > 1 or args.multiprocessing_distributed use_accel = not args.no_accel and torch.accelerator.is_available() if use_accel: device = torch.accelerator.current_accelerator() else: device = torch.device("cpu") print(f"Using device: {device}") if device.type =='cuda': ngpus_per_node = torch.accelerator.device_count() if ngpus_per_node == 1 and args.dist_backend == "nccl": warnings.warn("nccl backend >=2.5 requires GPU count>1, see https://github.com/NVIDIA/nccl/issues/103 perhaps use 'gloo'") else: ngpus_per_node = 1 if args.multiprocessing_distributed: # Since we have ngpus_per_node processes per node, the total world_size # needs to be adjusted accordingly args.world_size = ngpus_per_node * args.world_size # Use torch.multiprocessing.spawn to launch distributed processes: the # main_worker process function mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) else: # Simply call main_worker function main_worker(args.gpu, ngpus_per_node, args) def main_worker(gpu, ngpus_per_node, args): global best_acc1 args.gpu = gpu use_accel = not args.no_accel and torch.accelerator.is_available() if use_accel: if args.gpu is not None: torch.accelerator.set_device_index(args.gpu) device = torch.accelerator.current_accelerator() else: device = torch.device("cpu") if args.distributed: if args.dist_url == "env://" and args.rank == -1: args.rank = int(os.environ["RANK"]) if args.multiprocessing_distributed: # For multiprocessing distributed training, rank needs to be the # global rank among all the processes args.rank = args.rank * ngpus_per_node + gpu dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=args.rank) # create model if args.pretrained: print("=> using pre-trained model '{}'".format(args.arch)) model = models.__dict__[args.arch](pretrained=True) else: print("=> creating model '{}'".format(args.arch)) model = models.__dict__[args.arch]() if not use_accel: print('using CPU, this will be slow') elif args.distributed: # For multiprocessing distributed, DistributedDataParallel constructor # should always set the single device scope, otherwise, # DistributedDataParallel will use all available devices. if device.type == 'cuda': if args.gpu is not None: torch.cuda.set_device(args.gpu) model.cuda(device) # When using a single GPU per process and per # DistributedDataParallel, we need to divide the batch size # ourselves based on the total number of GPUs of the current node. args.batch_size = int(args.batch_size / ngpus_per_node) args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) else: model.cuda() # DistributedDataParallel will divide and allocate batch_size to all # available GPUs if device_ids are not set model = torch.nn.parallel.DistributedDataParallel(model) elif device.type == 'cuda': # DataParallel will divide and allocate batch_size to all available GPUs if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): model.features = torch.nn.DataParallel(model.features) model.cuda() else: model = torch.nn.DataParallel(model).cuda() else: model.to(device) # define loss function (criterion), optimizer, and learning rate scheduler criterion = nn.CrossEntropyLoss().to(device) optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" scheduler = StepLR(optimizer, step_size=30, gamma=0.1) # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) if args.gpu is None: checkpoint = torch.load(args.resume) else: # Map model to be loaded to specified single gpu. loc = f'{device.type}:{args.gpu}' checkpoint = torch.load(args.resume, map_location=loc) args.start_epoch = checkpoint['epoch'] best_acc1 = checkpoint['best_acc1'] if args.gpu is not None: # best_acc1 may be from a checkpoint from a different GPU best_acc1 = best_acc1.to(args.gpu) model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) scheduler.load_state_dict(checkpoint['scheduler']) print("=> loaded checkpoint '{}' (epoch {})" .format(args.resume, checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(args.resume)) # Data loading code if args.dummy: print("=> Dummy data is used!") train_dataset = datasets.FakeData(1281167, (3, 224, 224), 1000, transforms.ToTensor()) val_dataset = datasets.FakeData(50000, (3, 224, 224), 1000, transforms.ToTensor()) else: traindir = os.path.join(args.data, 'train') valdir = os.path.join(args.data, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_dataset = datasets.ImageFolder( traindir, transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])) val_dataset = datasets.ImageFolder( valdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])) if args.distributed: train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) val_sampler = torch.utils.data.distributed.DistributedSampler(val_dataset, shuffle=False, drop_last=True) else: train_sampler = None val_sampler = None train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), num_workers=args.workers, pin_memory=True, sampler=train_sampler) val_loader = torch.utils.data.DataLoader( val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True, sampler=val_sampler) if args.evaluate: validate(val_loader, model, criterion, args) return for epoch in range(args.start_epoch, args.epochs): if args.distributed: train_sampler.set_epoch(epoch) # train for one epoch train(train_loader, model, criterion, optimizer, epoch, device, args) # evaluate on validation set acc1 = validate(val_loader, model, criterion, args) scheduler.step() # remember best acc@1 and save checkpoint is_best = acc1 > best_acc1 best_acc1 = max(acc1, best_acc1) if not args.multiprocessing_distributed or (args.multiprocessing_distributed and args.rank % ngpus_per_node == 0): save_checkpoint({ 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_acc1': best_acc1, 'optimizer' : optimizer.state_dict(), 'scheduler' : scheduler.state_dict() }, is_best) def train(train_loader, model, criterion, optimizer, epoch, device, args): use_accel = not args.no_accel and torch.accelerator.is_available() batch_time = AverageMeter('Time', use_accel, ':6.3f', Summary.NONE) data_time = AverageMeter('Data', use_accel, ':6.3f', Summary.NONE) losses = AverageMeter('Loss', use_accel, ':.4e', Summary.NONE) top1 = AverageMeter('Acc@1', use_accel, ':6.2f', Summary.NONE) top5 = AverageMeter('Acc@5', use_accel, ':6.2f', Summary.NONE) progress = ProgressMeter( len(train_loader), [batch_time, data_time, losses, top1, top5], prefix="Epoch: [{}]".format(epoch)) # switch to train mode model.train() end = time.time() for i, (images, target) in enumerate(train_loader): # measure data loading time data_time.update(time.time() - end) # move data to the same device as model images = images.to(device, non_blocking=True) target = target.to(device, non_blocking=True) # compute output output = model(images) loss = criterion(output, target) # measure accuracy and record loss acc1, acc5 = accuracy(output, target, topk=(1, 5)) losses.update(loss.item(), images.size(0)) top1.update(acc1[0], images.size(0)) top5.update(acc5[0], images.size(0)) # compute gradient and do SGD step optimizer.zero_grad() loss.backward() optimizer.step() # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: progress.display(i + 1) def validate(val_loader, model, criterion, args): use_accel = not args.no_accel and torch.accelerator.is_available() def run_validate(loader, base_progress=0): if use_accel: device = torch.accelerator.current_accelerator() else: device = torch.device("cpu") with torch.no_grad(): end = time.time() for i, (images, target) in enumerate(loader): i = base_progress + i if use_accel: if args.gpu is not None and device.type=='cuda': torch.accelerator.set_device_index(args.gpu) images = images.cuda(args.gpu, non_blocking=True) target = target.cuda(args.gpu, non_blocking=True) else: images = images.to(device) target = target.to(device) # compute output output = model(images) loss = criterion(output, target) # measure accuracy and record loss acc1, acc5 = accuracy(output, target, topk=(1, 5)) losses.update(loss.item(), images.size(0)) top1.update(acc1[0], images.size(0)) top5.update(acc5[0], images.size(0)) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: progress.display(i + 1) batch_time = AverageMeter('Time', use_accel, ':6.3f', Summary.NONE) losses = AverageMeter('Loss', use_accel, ':.4e', Summary.NONE) top1 = AverageMeter('Acc@1', use_accel, ':6.2f', Summary.AVERAGE) top5 = AverageMeter('Acc@5', use_accel, ':6.2f', Summary.AVERAGE) progress = ProgressMeter( len(val_loader) + (args.distributed and (len(val_loader.sampler) * args.world_size < len(val_loader.dataset))), [batch_time, losses, top1, top5], prefix='Test: ') # switch to evaluate mode model.eval() run_validate(val_loader) if args.distributed: top1.all_reduce() top5.all_reduce() if args.distributed and (len(val_loader.sampler) * args.world_size < len(val_loader.dataset)): aux_val_dataset = Subset(val_loader.dataset, range(len(val_loader.sampler) * args.world_size, len(val_loader.dataset))) aux_val_loader = torch.utils.data.DataLoader( aux_val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) run_validate(aux_val_loader, len(val_loader)) progress.display_summary() return top1.avg def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): torch.save(state, filename) if is_best: shutil.copyfile(filename, 'model_best.pth.tar') class Summary(Enum): NONE = 0 AVERAGE = 1 SUM = 2 COUNT = 3 class AverageMeter(object): """Computes and stores the average and current value""" def __init__(self, name, use_accel, fmt=':f', summary_type=Summary.AVERAGE): self.name = name self.use_accel = use_accel self.fmt = fmt self.summary_type = summary_type self.reset() def reset(self): self.val = 0 self.avg = 0 self.sum = 0 self.count = 0 def update(self, val, n=1): self.val = val self.sum += val * n self.count += n self.avg = self.sum / self.count def all_reduce(self): if self.use_accel: device = torch.accelerator.current_accelerator() else: device = torch.device("cpu") total = torch.tensor([self.sum, self.count], dtype=torch.float32, device=device) dist.all_reduce(total, dist.ReduceOp.SUM, async_op=False) self.sum, self.count = total.tolist() self.avg = self.sum / self.count def __str__(self): fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' return fmtstr.format(**self.__dict__) def summary(self): fmtstr = '' if self.summary_type is Summary.NONE: fmtstr = '' elif self.summary_type is Summary.AVERAGE: fmtstr = '{name} {avg:.3f}' elif self.summary_type is Summary.SUM: fmtstr = '{name} {sum:.3f}' elif self.summary_type is Summary.COUNT: fmtstr = '{name} {count:.3f}' else: raise ValueError('invalid summary type %r' % self.summary_type) return fmtstr.format(**self.__dict__) class ProgressMeter(object): def __init__(self, num_batches, meters, prefix=""): self.batch_fmtstr = self._get_batch_fmtstr(num_batches) self.meters = meters self.prefix = prefix def display(self, batch): entries = [self.prefix + self.batch_fmtstr.format(batch)] entries += [str(meter) for meter in self.meters] print('\t'.join(entries)) def display_summary(self): entries = [" *"] entries += [meter.summary() for meter in self.meters] print(' '.join(entries)) def _get_batch_fmtstr(self, num_batches): num_digits = len(str(num_batches // 1)) fmt = '{:' + str(num_digits) + 'd}' return '[' + fmt + '/' + fmt.format(num_batches) + ']' def accuracy(output, target, topk=(1,)): """Computes the accuracy over the k top predictions for the specified values of k""" with torch.no_grad(): maxk = max(topk) batch_size = target.size(0) _, pred = output.topk(maxk, 1, True, True) pred = pred.t() correct = pred.eq(target.view(1, -1).expand_as(pred)) res = [] for k in topk: correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True) res.append(correct_k.mul_(100.0 / batch_size)) return res if __name__ == '__main__': main() ================================================ FILE: imagenet/requirements.txt ================================================ torch>=2.6 torchvision ================================================ FILE: language_translation/README.md ================================================ # Language Translation This example shows how one might use transformers for language translation. In particular, this implementation is loosely based on the [Attention is All You Need paper](https://arxiv.org/abs/1706.03762). ## Requirements We will need a tokenizer for our languages. Torchtext does include a tokenizer for English, but unfortunately, we will need more languages then that. We can get these tokenizers via ```spacy``` ```bash python3 -m spacy download python3 -m spacy download en python3 -m spacy download de ``` Spacy supports many languages. For a full accounting of supported languages, please look [here](https://spacy.io/usage/models). This example will default from German to English. Torchtext is also required: ```bash pip install torchtext ``` Just running these commands will get you started: ```bash pip install -r requirements.txt python3 -m spacy download ``` ## Usage This example contains a lot of flags that you can set to change the behavior / training of the module. You can see all of them by running: ```bash python3 main.py -h ``` But in general, all of the settings have "sensible" defaults; however, the default translation is to translate from German to English. To *train* the model, you only need to run the following command, but there is also an example for how to use any language you want: ```bash python3 main.py python3 main.py --src en --tgt fr # For english to french translation ``` For model inference, you can use this command: ```bash python3 main.py --inference --model_path ``` After some loading time, this will open an interactive interface where you can type in whatever sentence you are interested in translating. ================================================ FILE: language_translation/main.py ================================================ from time import time # Track how long an epoch takes import os # Creating and finding files/directories import logging # Logging tools from datetime import date # Logging the date for model versioning import torch # For ML from tqdm import tqdm # For fancy progress bars from src.model import Translator # Our model from src.data import get_data, create_mask, generate_square_subsequent_mask # Loading data and data preprocessing from argparse import ArgumentParser # For args # Train on the GPU if possible DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") # Function to generate output sequence using greedy algorithm def greedy_decode(model, src, src_mask, max_len, start_symbol, end_symbol): # Move to device src = src.to(DEVICE) src_mask = src_mask.to(DEVICE) # Encode input memory = model.encode(src, src_mask) # Output will be stored here ys = torch.ones(1, 1).fill_(start_symbol).type(torch.long).to(DEVICE) # For each element in our translation (which could range up to the maximum translation length) for _ in range(max_len-1): # Decode the encoded representation of the input memory = memory.to(DEVICE) tgt_mask = (generate_square_subsequent_mask(ys.size(0), DEVICE).type(torch.bool)).to(DEVICE) out = model.decode(ys, memory, tgt_mask) # Reshape out = out.transpose(0, 1) # Covert to probabilities and take the max of these probabilities prob = model.ff(out[:, -1]) _, next_word = torch.max(prob, dim=1) next_word = next_word.item() # Now we have an output which is the vector representation of the translation ys = torch.cat([ys, torch.ones(1, 1).type_as(src.data).fill_(next_word)], dim=0) if next_word == end_symbol: break return ys # Opens an user interface where users can translate an arbitrary sentence def inference(opts): # Get training data, tokenizer and vocab # objects as well as any special symbols we added to our dataset _, _, src_vocab, tgt_vocab, src_transform, _, special_symbols = get_data(opts) src_vocab_size = len(src_vocab) tgt_vocab_size = len(tgt_vocab) # Create model model = Translator( num_encoder_layers=opts.enc_layers, num_decoder_layers=opts.dec_layers, embed_size=opts.embed_size, num_heads=opts.attn_heads, src_vocab_size=src_vocab_size, tgt_vocab_size=tgt_vocab_size, dim_feedforward=opts.dim_feedforward, dropout=opts.dropout ).to(DEVICE) # Load in weights model.load_state_dict(torch.load(opts.model_path)) # Set to inference model.eval() # Accept input and keep translating until they quit while True: print("> ", end="") sentence = input() # Convert to tokens src = src_transform(sentence).view(-1, 1) num_tokens = src.shape[0] src_mask = (torch.zeros(num_tokens, num_tokens)).type(torch.bool) # Decode tgt_tokens = greedy_decode( model, src, src_mask, max_len=num_tokens+5, start_symbol=special_symbols[""], end_symbol=special_symbols[""] ).flatten() # Convert to list of tokens output_as_list = list(tgt_tokens.cpu().numpy()) # Convert tokens to words output_list_words = tgt_vocab.lookup_tokens(output_as_list) # Remove special tokens and convert to string translation = " ".join(output_list_words).replace("", "").replace("", "") print(translation) # Train the model for 1 epoch def train(model, train_dl, loss_fn, optim, special_symbols, opts): # Object for accumulating losses losses = 0 # Put model into training mode model.train() for src, tgt in tqdm(train_dl, ascii=True): src = src.to(DEVICE) tgt = tgt.to(DEVICE) # We need to reshape the input slightly to fit into the transformer tgt_input = tgt[:-1, :] # Create masks src_mask, tgt_mask, src_padding_mask, tgt_padding_mask = create_mask(src, tgt_input, special_symbols[""], DEVICE) # Pass into model, get probability over the vocab out logits = model(src, tgt_input, src_mask, tgt_mask,src_padding_mask, tgt_padding_mask, src_padding_mask) # Reset gradients before we try to compute the gradients over the loss optim.zero_grad() # Get original shape back tgt_out = tgt[1:, :] # Compute loss and gradient over that loss loss = loss_fn(logits.reshape(-1, logits.shape[-1]), tgt_out.reshape(-1)) loss.backward() # Step weights optim.step() # Accumulate a running loss for reporting losses += loss.item() if opts.dry_run: break # Return the average loss return losses / len(list(train_dl)) # Check the model accuracy on the validation dataset def validate(model, valid_dl, loss_fn, special_symbols): # Object for accumulating losses losses = 0 # Turn off gradients a moment model.eval() for src, tgt in tqdm(valid_dl): src = src.to(DEVICE) tgt = tgt.to(DEVICE) # We need to reshape the input slightly to fit into the transformer tgt_input = tgt[:-1, :] # Create masks src_mask, tgt_mask, src_padding_mask, tgt_padding_mask = create_mask(src, tgt_input, special_symbols[""], DEVICE) # Pass into model, get probability over the vocab out logits = model(src, tgt_input, src_mask, tgt_mask,src_padding_mask, tgt_padding_mask, src_padding_mask) # Get original shape back, compute loss, accumulate that loss tgt_out = tgt[1:, :] loss = loss_fn(logits.reshape(-1, logits.shape[-1]), tgt_out.reshape(-1)) losses += loss.item() # Return the average loss return losses / len(list(valid_dl)) # Train the model def main(opts): # Set up logging os.makedirs(opts.logging_dir, exist_ok=True) logger = logging.getLogger(__name__) logging.basicConfig(filename=opts.logging_dir + "log.txt", level=logging.INFO) # This prints it to the screen as well console = logging.StreamHandler() console.setLevel(logging.INFO) logging.getLogger().addHandler(console) logging.info(f"Translation task: {opts.src} -> {opts.tgt}") logging.info(f"Using device: {DEVICE}") # Get training data, tokenizer and vocab # objects as well as any special symbols we added to our dataset train_dl, valid_dl, src_vocab, tgt_vocab, _, _, special_symbols = get_data(opts) logging.info("Loaded data") src_vocab_size = len(src_vocab) tgt_vocab_size = len(tgt_vocab) logging.info(f"{opts.src} vocab size: {src_vocab_size}") logging.info(f"{opts.tgt} vocab size: {tgt_vocab_size}") # Create model model = Translator( num_encoder_layers=opts.enc_layers, num_decoder_layers=opts.dec_layers, embed_size=opts.embed_size, num_heads=opts.attn_heads, src_vocab_size=src_vocab_size, tgt_vocab_size=tgt_vocab_size, dim_feedforward=opts.dim_feedforward, dropout=opts.dropout ).to(DEVICE) logging.info("Model created... starting training!") # Set up our learning tools loss_fn = torch.nn.CrossEntropyLoss(ignore_index=special_symbols[""]) # These special values are from the "Attention is all you need" paper optim = torch.optim.Adam(model.parameters(), lr=opts.lr, betas=(0.9, 0.98), eps=1e-9) best_val_loss = 1e6 for idx, epoch in enumerate(range(1, opts.epochs+1)): start_time = time() train_loss = train(model, train_dl, loss_fn, optim, special_symbols, opts) epoch_time = time() - start_time val_loss = validate(model, valid_dl, loss_fn, special_symbols) # Once training is done, we want to save out the model if val_loss < best_val_loss: best_val_loss = val_loss logging.info("New best model, saving...") torch.save(model.state_dict(), opts.logging_dir + "best.pt") torch.save(model.state_dict(), opts.logging_dir + "last.pt") logger.info(f"Epoch: {epoch}\n\tTrain loss: {train_loss:.3f}\n\tVal loss: {val_loss:.3f}\n\tEpoch time = {epoch_time:.1f} seconds\n\tETA = {epoch_time*(opts.epochs-idx-1):.1f} seconds") if __name__ == "__main__": parser = ArgumentParser( prog="Machine Translator training and inference", ) # Inference mode parser.add_argument("--inference", action="store_true", help="Set true to run inference") parser.add_argument("--model_path", type=str, help="Path to the model to run inference on") # Translation settings parser.add_argument("--src", type=str, default="de", help="Source language (translating FROM this language)") parser.add_argument("--tgt", type=str, default="en", help="Target language (translating TO this language)") # Training settings parser.add_argument("-e", "--epochs", type=int, default=30, help="Epochs") parser.add_argument("--lr", type=float, default=1e-4, help="Default learning rate") parser.add_argument("--batch", type=int, default=128, help="Batch size") parser.add_argument("--backend", type=str, default="cpu", help="Batch size") # Transformer settings parser.add_argument("--attn_heads", type=int, default=8, help="Number of attention heads") parser.add_argument("--enc_layers", type=int, default=5, help="Number of encoder layers") parser.add_argument("--dec_layers", type=int, default=5, help="Number of decoder layers") parser.add_argument("--embed_size", type=int, default=512, help="Size of the language embedding") parser.add_argument("--dim_feedforward", type=int, default=512, help="Feedforward dimensionality") parser.add_argument("--dropout", type=float, default=0.1, help="Transformer dropout") # Logging settings parser.add_argument("--logging_dir", type=str, default="./" + str(date.today()) + "/", help="Where the output of this program should be placed") # Just for continuous integration parser.add_argument("--dry_run", action="store_true") args = parser.parse_args() DEVICE = torch.device("cuda" if args.backend == "gpu" and torch.cuda.is_available() else "cpu") if args.inference: inference(args) else: main(args) ================================================ FILE: language_translation/requirements.txt ================================================ torch torchtext torchdata==0.9.0 spacy portalocker ================================================ FILE: language_translation/src/data.py ================================================ import torch from torch.nn.utils.rnn import pad_sequence from torch.utils.data import DataLoader from torchtext.data.utils import get_tokenizer from torchtext.vocab import build_vocab_from_iterator from torchtext.datasets import Multi30k, multi30k # Turns an iterable into a generator def _yield_tokens(iterable_data, tokenizer, src): # Iterable data stores the samples as (src, tgt) so this will help us select just one language or the other index = 0 if src else 1 for data in iterable_data: yield tokenizer(data[index]) # Get data, tokenizer, text transform, vocab objs, etc. Everything we # need to start training the model def get_data(opts): src_lang = opts.src tgt_lang = opts.tgt multi30k.URL["train"] = "https://raw.githubusercontent.com/neychev/small_DL_repo/master/datasets/Multi30k/training.tar.gz" multi30k.URL["valid"] = "https://raw.githubusercontent.com/neychev/small_DL_repo/master/datasets/Multi30k/validation.tar.gz" # Define a token "unkown", "padding", "beginning of sentence", and "end of sentence" special_symbols = { "":0, "":1, "":2, "":3 } # Get training examples from torchtext (the multi30k dataset) train_iterator = Multi30k(split="train", language_pair=(src_lang, tgt_lang)) valid_iterator = Multi30k(split="valid", language_pair=(src_lang, tgt_lang)) # Grab a tokenizer for these languages src_tokenizer = get_tokenizer("spacy", src_lang) tgt_tokenizer = get_tokenizer("spacy", tgt_lang) # Build a vocabulary object for these languages src_vocab = build_vocab_from_iterator( _yield_tokens(train_iterator, src_tokenizer, True), min_freq=1, specials=list(special_symbols.keys()), special_first=True ) tgt_vocab = build_vocab_from_iterator( _yield_tokens(train_iterator, tgt_tokenizer, False), min_freq=1, specials=list(special_symbols.keys()), special_first=True ) src_vocab.set_default_index(special_symbols[""]) tgt_vocab.set_default_index(special_symbols[""]) # Helper function to sequentially apply transformations def _seq_transform(*transforms): def func(txt_input): for transform in transforms: txt_input = transform(txt_input) return txt_input return func # Function to add BOS/EOS and create tensor for input sequence indices def _tensor_transform(token_ids): return torch.cat( (torch.tensor([special_symbols[""]]), torch.tensor(token_ids), torch.tensor([special_symbols[""]])) ) src_lang_transform = _seq_transform(src_tokenizer, src_vocab, _tensor_transform) tgt_lang_transform = _seq_transform(tgt_tokenizer, tgt_vocab, _tensor_transform) # Now we want to convert the torchtext data pipeline to a dataloader. We # will need to collate batches def _collate_fn(batch): src_batch, tgt_batch = [], [] for src_sample, tgt_sample in batch: src_batch.append(src_lang_transform(src_sample.rstrip("\n"))) tgt_batch.append(tgt_lang_transform(tgt_sample.rstrip("\n"))) src_batch = pad_sequence(src_batch, padding_value=special_symbols[""]) tgt_batch = pad_sequence(tgt_batch, padding_value=special_symbols[""]) return src_batch, tgt_batch # Create the dataloader train_dataloader = DataLoader(train_iterator, batch_size=opts.batch, collate_fn=_collate_fn) valid_dataloader = DataLoader(valid_iterator, batch_size=opts.batch, collate_fn=_collate_fn) return train_dataloader, valid_dataloader, src_vocab, tgt_vocab, src_lang_transform, tgt_lang_transform, special_symbols def generate_square_subsequent_mask(size, device): mask = (torch.triu(torch.ones((size, size), device=device)) == 1).transpose(0, 1) mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0)) return mask # Create masks for input into model def create_mask(src, tgt, pad_idx, device): # Get sequence length src_seq_len = src.shape[0] tgt_seq_len = tgt.shape[0] # Generate the mask tgt_mask = generate_square_subsequent_mask(tgt_seq_len, device) src_mask = torch.zeros((src_seq_len, src_seq_len),device=device).type(torch.bool) # Overlay the mask over the original input src_padding_mask = (src == pad_idx).transpose(0, 1) tgt_padding_mask = (tgt == pad_idx).transpose(0, 1) return src_mask, tgt_mask, src_padding_mask, tgt_padding_mask # A small test to make sure our data loasd in correctly if __name__=="__main__": class Opts: def __init__(self): self.src = "en", self.tgt = "de" self.batch = 128 opts = Opts() train_dl, valid_dl, src_vocab, tgt_vocab, src_lang_transform, tgt_lang_transform, special_symbols = get_data(opts) print(f"{opts.src} vocab size: {len(src_vocab)}") print(f"{opts.src} vocab size: {len(tgt_vocab)}") ================================================ FILE: language_translation/src/model.py ================================================ import math import torch from torch.nn import functional as F from torch import nn class PositionalEncoding(nn.Module): def __init__( self, emb_size, dropout, maxlen=5000 ): super(PositionalEncoding, self).__init__() den = torch.exp(- torch.arange(0, emb_size, 2)* math.log(10000) / emb_size) pos = torch.arange(0, maxlen).reshape(maxlen, 1) pos_embedding = torch.zeros((maxlen, emb_size)) pos_embedding[:, 0::2] = torch.sin(pos * den) pos_embedding[:, 1::2] = torch.cos(pos * den) pos_embedding = pos_embedding.unsqueeze(-2) self.dropout = nn.Dropout(dropout) self.register_buffer('pos_embedding', pos_embedding) def forward(self, token_embedding): return self.dropout(token_embedding + self.pos_embedding[:token_embedding.size(0), :]) class Translator(nn.Module): def __init__( self, num_encoder_layers, num_decoder_layers, embed_size, num_heads, src_vocab_size, tgt_vocab_size, dim_feedforward, dropout ): super(Translator, self).__init__() # Output of embedding must be equal (embed_size) self.src_embedding = nn.Embedding(src_vocab_size, embed_size) self.tgt_embedding = nn.Embedding(tgt_vocab_size, embed_size) self.pos_enc = PositionalEncoding(embed_size, dropout) self.transformer = nn.Transformer( d_model=embed_size, nhead=num_heads, num_encoder_layers=num_encoder_layers, num_decoder_layers=num_decoder_layers, dim_feedforward=dim_feedforward, dropout=dropout ) self.ff = nn.Linear(embed_size, tgt_vocab_size) self._init_weights() def _init_weights(self): for p in self.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p) def forward(self, src, trg, src_mask, tgt_mask, src_padding_mask, tgt_padding_mask, memory_key_padding_mask): src_emb = self.pos_enc(self.src_embedding(src)) tgt_emb = self.pos_enc(self.tgt_embedding(trg)) outs = self.transformer( src_emb, tgt_emb, src_mask, tgt_mask, None, src_padding_mask, tgt_padding_mask, memory_key_padding_mask ) return self.ff(outs) def encode(self, src, src_mask): embed = self.src_embedding(src) pos_enc = self.pos_enc(embed) return self.transformer.encoder(pos_enc, src_mask) def decode(self, tgt, memory, tgt_mask): embed = self.tgt_embedding(tgt) pos_enc = self.pos_enc(embed) return self.transformer.decoder(pos_enc, memory, tgt_mask) ================================================ FILE: legacy/snli/README.md ================================================ # PyTorch-based NLI Training with SNLI ## 📝 Overview This repository contains Python scripts to train a Natural Language Inference (NLI) model, specifically the `SNLIClassifier`, using the Stanford Natural Language Inference (SNLI) corpus. The trained model predicts textual entailment, identifying if a statement is entailed, contradicted, or neither by another statement. ## ⚙️ Dependencies Install the necessary Python libraries with: ```bash pip install -r requirements.txt ``` The `requirements.txt` file includes: ``` torch torchtext spacy ``` ## 💻 Usage Start the training process with: ```bash python train.py --lower --word-vectors [PATH_TO_WORD_VECTORS] --vector-cache [PATH_TO_VECTOR_CACHE] --epochs [NUMBER_OF_EPOCHS] --batch-size [BATCH_SIZE] --save-path [PATH_TO_SAVE_MODEL] --gpu [GPU_NUMBER] ``` ## 🏋️‍♀️ Training The script trains the model on mini-batches of data across a specified number of epochs. It saves the best-performing model on the validation set as a `.pt` file in the specified directory. ## 📚 Scripts - `model.py`: Defines the `SNLIClassifier` model and auxiliary classes. - `util.py`: Contains utility functions for directory creation and command-line argument parsing. ## 📣 Note Ensure the `model.py` and `util.py` scripts are available in your working directory. ================================================ FILE: legacy/snli/model.py ================================================ import torch import torch.nn as nn class Bottle(nn.Module): def forward(self, input): if len(input.size()) <= 2: return super(Bottle, self).forward(input) size = input.size()[:2] out = super(Bottle, self).forward(input.view(size[0]*size[1], -1)) return out.view(size[0], size[1], -1) class Linear(Bottle, nn.Linear): pass class Encoder(nn.Module): def __init__(self, config): super(Encoder, self).__init__() self.config = config input_size = config.d_proj if config.projection else config.d_embed dropout = 0 if config.n_layers == 1 else config.dp_ratio self.rnn = nn.LSTM(input_size=input_size, hidden_size=config.d_hidden, num_layers=config.n_layers, dropout=dropout, bidirectional=config.birnn) def forward(self, inputs): batch_size = inputs.size()[1] state_shape = self.config.n_cells, batch_size, self.config.d_hidden h0 = c0 = inputs.new_zeros(state_shape) outputs, (ht, ct) = self.rnn(inputs, (h0, c0)) return ht[-1] if not self.config.birnn else ht[-2:].transpose(0, 1).contiguous().view(batch_size, -1) class SNLIClassifier(nn.Module): def __init__(self, config): super(SNLIClassifier, self).__init__() self.config = config self.embed = nn.Embedding(config.n_embed, config.d_embed) self.projection = Linear(config.d_embed, config.d_proj) self.encoder = Encoder(config) self.dropout = nn.Dropout(p=config.dp_ratio) self.relu = nn.ReLU() seq_in_size = 2*config.d_hidden if self.config.birnn: seq_in_size *= 2 lin_config = [seq_in_size]*2 self.out = nn.Sequential( Linear(*lin_config), self.relu, self.dropout, Linear(*lin_config), self.relu, self.dropout, Linear(*lin_config), self.relu, self.dropout, Linear(seq_in_size, config.d_out)) def forward(self, batch): prem_embed = self.embed(batch.premise) hypo_embed = self.embed(batch.hypothesis) if self.config.fix_emb: prem_embed = prem_embed.detach() hypo_embed = hypo_embed.detach() if self.config.projection: prem_embed = self.relu(self.projection(prem_embed)) hypo_embed = self.relu(self.projection(hypo_embed)) premise = self.encoder(prem_embed) hypothesis = self.encoder(hypo_embed) scores = self.out(torch.cat([premise, hypothesis], 1)) return scores ================================================ FILE: legacy/snli/requirements.txt ================================================ torch torchtext spacy ================================================ FILE: legacy/snli/train.py ================================================ import os import time import glob import torch import torch.optim as O import torch.nn as nn from torchtext.legacy import data from torchtext.legacy import datasets from model import SNLIClassifier from util import get_args, makedirs args = get_args() if torch.cuda.is_available(): torch.cuda.set_device(args.gpu) device = torch.device('cuda:{}'.format(args.gpu)) elif torch.backends.mps.is_available(): device = torch.device('mps') else: device = torch.device('cpu') inputs = data.Field(lower=args.lower, tokenize='spacy') answers = data.Field(sequential=False) train, dev, test = datasets.SNLI.splits(inputs, answers) inputs.build_vocab(train, dev, test) if args.word_vectors: if os.path.isfile(args.vector_cache): inputs.vocab.vectors = torch.load(args.vector_cache) else: inputs.vocab.load_vectors(args.word_vectors) makedirs(os.path.dirname(args.vector_cache)) torch.save(inputs.vocab.vectors, args.vector_cache) answers.build_vocab(train) train_iter, dev_iter, test_iter = data.BucketIterator.splits( (train, dev, test), batch_size=args.batch_size, device=device) config = args config.n_embed = len(inputs.vocab) config.d_out = len(answers.vocab) config.n_cells = config.n_layers # double the number of cells for bidirectional networks if config.birnn: config.n_cells *= 2 if args.resume_snapshot: model = torch.load(args.resume_snapshot, map_location=device) else: model = SNLIClassifier(config) if args.word_vectors: model.embed.weight.data.copy_(inputs.vocab.vectors) model.to(device) criterion = nn.CrossEntropyLoss() opt = O.Adam(model.parameters(), lr=args.lr) iterations = 0 start = time.time() best_dev_acc = -1 header = ' Time Epoch Iteration Progress (%Epoch) Loss Dev/Loss Accuracy Dev/Accuracy' dev_log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>8.6f},{:8.6f},{:12.4f},{:12.4f}'.split(',')) log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>8.6f},{},{:12.4f},{}'.split(',')) makedirs(args.save_path) print(header) for epoch in range(args.epochs): train_iter.init_epoch() n_correct, n_total = 0, 0 for batch_idx, batch in enumerate(train_iter): # switch model to training mode, clear gradient accumulators model.train(); opt.zero_grad() iterations += 1 # forward pass answer = model(batch) # calculate accuracy of predictions in the current batch n_correct += (torch.max(answer, 1)[1].view(batch.label.size()) == batch.label).sum().item() n_total += batch.batch_size train_acc = 100. * n_correct/n_total # calculate loss of the network output with respect to training labels loss = criterion(answer, batch.label) # backpropagate and update optimizer learning rate loss.backward(); opt.step() # checkpoint model periodically if iterations % args.save_every == 0: snapshot_prefix = os.path.join(args.save_path, 'snapshot') snapshot_path = snapshot_prefix + '_acc_{:.4f}_loss_{:.6f}_iter_{}_model.pt'.format(train_acc, loss.item(), iterations) torch.save(model, snapshot_path) for f in glob.glob(snapshot_prefix + '*'): if f != snapshot_path: os.remove(f) # evaluate performance on validation set periodically if iterations % args.dev_every == 0: # switch model to evaluation mode model.eval(); dev_iter.init_epoch() # calculate accuracy on validation set n_dev_correct, dev_loss = 0, 0 with torch.no_grad(): for dev_batch_idx, dev_batch in enumerate(dev_iter): answer = model(dev_batch) n_dev_correct += (torch.max(answer, 1)[1].view(dev_batch.label.size()) == dev_batch.label).sum().item() dev_loss = criterion(answer, dev_batch.label) dev_acc = 100. * n_dev_correct / len(dev) print(dev_log_template.format(time.time()-start, epoch, iterations, 1+batch_idx, len(train_iter), 100. * (1+batch_idx) / len(train_iter), loss.item(), dev_loss.item(), train_acc, dev_acc)) # update best validation set accuracy if dev_acc > best_dev_acc: # found a model with better validation set accuracy best_dev_acc = dev_acc snapshot_prefix = os.path.join(args.save_path, 'best_snapshot') snapshot_path = snapshot_prefix + '_devacc_{}_devloss_{}__iter_{}_model.pt'.format(dev_acc, dev_loss.item(), iterations) # save model, delete previous 'best_snapshot' files torch.save(model, snapshot_path) for f in glob.glob(snapshot_prefix + '*'): if f != snapshot_path: os.remove(f) elif iterations % args.log_every == 0: # print progress message print(log_template.format(time.time()-start, epoch, iterations, 1+batch_idx, len(train_iter), 100. * (1+batch_idx) / len(train_iter), loss.item(), ' '*8, n_correct/n_total*100, ' '*12)) if args.dry_run: break ================================================ FILE: legacy/snli/util.py ================================================ import os from argparse import ArgumentParser def makedirs(name): """helper function for python 2 and 3 to call os.makedirs() avoiding an error if the directory to be created already exists""" import os, errno try: os.makedirs(name) except OSError as ex: if ex.errno == errno.EEXIST and os.path.isdir(name): # ignore existing directory pass else: # a different error happened raise def get_args(): parser = ArgumentParser(description='PyTorch/torchtext SNLI example') parser.add_argument('--epochs', type=int, default=50, help='the number of total epochs to run.') parser.add_argument('--batch_size', type=int, default=128, help='batch size. (default: 128)') parser.add_argument('--d_embed', type=int, default=100, help='the size of each embedding vector.') parser.add_argument('--d_proj', type=int, default=300, help='the size of each projection layer.') parser.add_argument('--d_hidden', type=int, default=300, help='the number of features in the hidden state.') parser.add_argument('--n_layers', type=int, default=1, help='the number of recurrent layers. (default: 50)') parser.add_argument('--log_every', type=int, default=50, help='iteration period to output log.') parser.add_argument('--lr',type=float, default=.001, help='initial learning rate.') parser.add_argument('--dev_every', type=int, default=1000, help='log period of validation results.') parser.add_argument('--save_every', type=int, default=1000, help='model checkpoint period.') parser.add_argument('--dp_ratio', type=int, default=0.2, help='probability of an element to be zeroed.') parser.add_argument('--no-bidirectional', action='store_false', dest='birnn', help='disable bidirectional LSTM.') parser.add_argument('--preserve-case', action='store_false', dest='lower', help='case-sensitivity.') parser.add_argument('--no-projection', action='store_false', dest='projection', help='disable projection layer.') parser.add_argument('--train_embed', action='store_false', dest='fix_emb', help='enable embedding word training.') parser.add_argument('--gpu', type=int, default=0, help='gpu id to use. (default: 0)') parser.add_argument('--save_path', type=str, default='results', help='save path of results.') parser.add_argument('--vector_cache', type=str, default=os.path.join(os.getcwd(), '.vector_cache/input_vectors.pt'), help='name of vector cache directory, which saved input word-vectors.') parser.add_argument('--word_vectors', type=str, default='glove.6B.100d', help='one of or a list containing instantiations of the GloVe, CharNGram, or Vectors classes.' 'Alternatively, one of or a list of available pretrained vectors: ' 'charngram.100d fasttext.en.300d fasttext.simple.300d' 'glove.42B.300d glove.840B.300d glove.twitter.27B.25d' 'glove.twitter.27B.50d glove.twitter.27B.100d glove.twitter.27B.200d' 'glove.6B.50d glove.6B.100d glove.6B.200d glove.6B.300d') parser.add_argument('--resume_snapshot', type=str, default='', help='model snapshot to resume.') parser.add_argument('--dry-run', action='store_true', help='run only a few iterations') args = parser.parse_args() return args ================================================ FILE: mnist/README.md ================================================ # Basic MNIST Example ```bash pip install -r requirements.txt python main.py # CUDA_VISIBLE_DEVICES=2 python main.py # to specify GPU id to ex. 2 ``` ================================================ FILE: mnist/main.py ================================================ import argparse import torch import torch.nn as nn import torch.nn.functional as F import torch.optim as optim from torchvision import datasets, transforms from torch.optim.lr_scheduler import StepLR class Net(nn.Module): def __init__(self): super(Net, self).__init__() self.conv1 = nn.Conv2d(1, 32, 3, 1) self.conv2 = nn.Conv2d(32, 64, 3, 1) self.dropout1 = nn.Dropout(0.25) self.dropout2 = nn.Dropout(0.5) self.fc1 = nn.Linear(9216, 128) self.fc2 = nn.Linear(128, 10) def forward(self, x): x = self.conv1(x) x = F.relu(x) x = self.conv2(x) x = F.relu(x) x = F.max_pool2d(x, 2) x = self.dropout1(x) x = torch.flatten(x, 1) x = self.fc1(x) x = F.relu(x) x = self.dropout2(x) x = self.fc2(x) output = F.log_softmax(x, dim=1) return output def train(args, model, device, train_loader, optimizer, epoch): model.train() for batch_idx, (data, target) in enumerate(train_loader): data, target = data.to(device), target.to(device) optimizer.zero_grad() output = model(data) loss = F.nll_loss(output, target) loss.backward() optimizer.step() if batch_idx % args.log_interval == 0: print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( epoch, batch_idx * len(data), len(train_loader.dataset), 100. * batch_idx / len(train_loader), loss.item())) if args.dry_run: break def test(model, device, test_loader): model.eval() test_loss = 0 correct = 0 with torch.no_grad(): for data, target in test_loader: data, target = data.to(device), target.to(device) output = model(data) test_loss += F.nll_loss(output, target, reduction='sum').item() # sum up batch loss pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability correct += pred.eq(target.view_as(pred)).sum().item() test_loss /= len(test_loader.dataset) print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( test_loss, correct, len(test_loader.dataset), 100. * correct / len(test_loader.dataset))) def main(): # Training settings parser = argparse.ArgumentParser(description='PyTorch MNIST Example') parser.add_argument('--batch-size', type=int, default=64, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=14, metavar='N', help='number of epochs to train (default: 14)') parser.add_argument('--lr', type=float, default=1.0, metavar='LR', help='learning rate (default: 1.0)') parser.add_argument('--gamma', type=float, default=0.7, metavar='M', help='Learning rate step gamma (default: 0.7)') parser.add_argument('--no-accel', action='store_true', help='disables accelerator') parser.add_argument('--dry-run', action='store_true', help='quickly check a single pass') parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') parser.add_argument('--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--save-model', action='store_true', help='For Saving the current Model') args = parser.parse_args() use_accel = not args.no_accel and torch.accelerator.is_available() torch.manual_seed(args.seed) if use_accel: device = torch.accelerator.current_accelerator() else: device = torch.device("cpu") train_kwargs = {'batch_size': args.batch_size} test_kwargs = {'batch_size': args.test_batch_size} if use_accel: accel_kwargs = {'num_workers': 1, 'persistent_workers': True, 'pin_memory': True, 'shuffle': True} train_kwargs.update(accel_kwargs) test_kwargs.update(accel_kwargs) transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ]) dataset1 = datasets.MNIST('../data', train=True, download=True, transform=transform) dataset2 = datasets.MNIST('../data', train=False, transform=transform) train_loader = torch.utils.data.DataLoader(dataset1,**train_kwargs) test_loader = torch.utils.data.DataLoader(dataset2, **test_kwargs) model = Net().to(device) optimizer = optim.Adadelta(model.parameters(), lr=args.lr) scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma) for epoch in range(1, args.epochs + 1): train(args, model, device, train_loader, optimizer, epoch) test(model, device, test_loader) scheduler.step() if args.save_model: torch.save(model.state_dict(), "mnist_cnn.pt") if __name__ == '__main__': main() ================================================ FILE: mnist/requirements.txt ================================================ torch torchvision ================================================ FILE: mnist_forward_forward/README.md ================================================ # Basic Forward-Forward Example This example implements the paper [The Forward-Forward Algorithm: Some Preliminary Investigations](https://arxiv.org/abs/2212.13345) by Geoffrey Hinton. the aim of this paper is to introduce a new learning procedure for neural networks. the forward and backward passes of backpropagation by two forward passes. ```bash pip install -r requirements.txt python main.py ``` The main.py script accepts the following arguments: ```bash optional arguments: -h, --help show this help message and exit --epochs EPOCHS number of epochs to train (default: 1000) --lr LR learning rate (default: 0.03) --no_accel disables accelerator --seed SEED random seed (default: 1) --save_model For saving the current Model --train_size TRAIN_SIZE size of training set --threshold THRESHOLD threshold for training --test_size TEST_SIZE size of test set --save-model For Saving the current Model --log-interval LOG_INTERVAL logging training status interval ``` ================================================ FILE: mnist_forward_forward/main.py ================================================ # This code is based on the implementation of Mohammad Pezeshki available at # https://github.com/mohammadpz/pytorch_forward_forward and licensed under the MIT License. # Modifications/Improvements to the original code have been made by Vivek V Patel. import argparse import torch import torch.nn as nn from torchvision.datasets import MNIST from torchvision.transforms import Compose, ToTensor, Normalize, Lambda from torch.utils.data import DataLoader from torch.optim import Adam def get_y_neg(y): y_neg = y.clone() for idx, y_samp in enumerate(y): allowed_indices = list(range(10)) allowed_indices.remove(y_samp.item()) y_neg[idx] = torch.tensor(allowed_indices)[ torch.randint(len(allowed_indices), size=(1,)) ].item() return y_neg.to(device) def overlay_y_on_x(x, y, classes=10): x_ = x.clone() x_[:, :classes] *= 0.0 x_[range(x.shape[0]), y] = x.max() return x_ class Net(torch.nn.Module): def __init__(self, dims): super().__init__() self.layers = [] for d in range(len(dims) - 1): self.layers = self.layers + [Layer(dims[d], dims[d + 1]).to(device)] def predict(self, x): goodness_per_label = [] for label in range(10): h = overlay_y_on_x(x, label) goodness = [] for layer in self.layers: h = layer(h) goodness = goodness + [h.pow(2).mean(1)] goodness_per_label += [sum(goodness).unsqueeze(1)] goodness_per_label = torch.cat(goodness_per_label, 1) return goodness_per_label.argmax(1) def train(self, x_pos, x_neg): h_pos, h_neg = x_pos, x_neg for i, layer in enumerate(self.layers): print("training layer: ", i) h_pos, h_neg = layer.train(h_pos, h_neg) class Layer(nn.Linear): def __init__(self, in_features, out_features, bias=True, device=None, dtype=None): super().__init__(in_features, out_features, bias, device, dtype) self.relu = torch.nn.ReLU() self.opt = Adam(self.parameters(), lr=args.lr) self.threshold = args.threshold self.num_epochs = args.epochs def forward(self, x): x_direction = x / (x.norm(2, 1, keepdim=True) + 1e-4) return self.relu(torch.mm(x_direction, self.weight.T) + self.bias.unsqueeze(0)) def train(self, x_pos, x_neg): for i in range(self.num_epochs): g_pos = self.forward(x_pos).pow(2).mean(1) g_neg = self.forward(x_neg).pow(2).mean(1) loss = torch.log1p( torch.exp( torch.cat([-g_pos + self.threshold, g_neg - self.threshold]) ) ).mean() self.opt.zero_grad() loss.backward() self.opt.step() if i % args.log_interval == 0: print("Loss: ", loss.item()) return self.forward(x_pos).detach(), self.forward(x_neg).detach() if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "--epochs", type=int, default=1000, metavar="N", help="number of epochs to train (default: 1000)", ) parser.add_argument( "--lr", type=float, default=0.03, metavar="LR", help="learning rate (default: 0.03)", ) parser.add_argument( "--no_accel", action="store_true", help="disables accelerator" ) parser.add_argument( "--seed", type=int, default=1, metavar="S", help="random seed (default: 1)" ) parser.add_argument( "--save_model", action="store_true", help="For saving the current Model", ) parser.add_argument( "--train_size", type=int, default=50000, help="size of training set" ) parser.add_argument( "--threshold", type=float, default=2, help="threshold for training" ) parser.add_argument("--test_size", type=int, default=10000, help="size of test set") parser.add_argument( "--save-model", action="store_true", help="For Saving the current Model", ) parser.add_argument( "--log-interval", type=int, default=10, metavar="N", help="how many batches to wait before logging training status", ) args = parser.parse_args() use_accel = not args.no_accel and torch.accelerator.is_available() if use_accel: device = torch.accelerator.current_accelerator() else: device = torch.device("cpu") train_kwargs = {"batch_size": args.train_size} test_kwargs = {"batch_size": args.test_size} if use_accel: accel_kwargs = {"num_workers": 1, "pin_memory": True, "shuffle": True} train_kwargs.update(accel_kwargs) test_kwargs.update(accel_kwargs) transform = Compose( [ ToTensor(), Normalize((0.1307,), (0.3081,)), Lambda(lambda x: torch.flatten(x)), ] ) train_loader = DataLoader( MNIST("./data/", train=True, download=True, transform=transform), **train_kwargs ) test_loader = DataLoader( MNIST("./data/", train=False, download=True, transform=transform), **test_kwargs ) net = Net([784, 500, 500]) x, y = next(iter(train_loader)) x, y = x.to(device), y.to(device) x_pos = overlay_y_on_x(x, y) y_neg = get_y_neg(y) x_neg = overlay_y_on_x(x, y_neg) net.train(x_pos, x_neg) print("train error:", 1.0 - net.predict(x).eq(y).float().mean().item()) x_te, y_te = next(iter(test_loader)) x_te, y_te = x_te.to(device), y_te.to(device) if args.save_model: torch.save(net.state_dict(), "mnist_ff.pt") print("test error:", 1.0 - net.predict(x_te).eq(y_te).float().mean().item()) ================================================ FILE: mnist_forward_forward/requirements.txt ================================================ torch torchvision ================================================ FILE: mnist_hogwild/README.md ================================================ # MNIST Hogwild Example ```bash pip install -r requirements.txt python main.py ``` The main.py script accepts the following arguments: ```bash optional arguments: -h, --help show this help message and exit --batch-size input batch_size for training (default: 64) --test-batch-size input batch size for testing (default: 1000) --epochs EPOCHS number of epochs to train (default: 10) --lr LR learning rate (default: 0.01) --momentum SGD momentum (default: 0.5) --seed SEED random seed (default: 1) --save_model save the trained model to state_dict --log-interval how many batches to wait before logging training status (default: 10) --num-processes how many training processes to use (default: 2) --cuda enables CUDA training --mps enables macos GPU training --dry-run quickly check a single pass ``` ================================================ FILE: mnist_hogwild/main.py ================================================ from __future__ import print_function import argparse import torch import torch.nn as nn import torch.nn.functional as F import torch.multiprocessing as mp from torch.utils.data.sampler import Sampler from torchvision import datasets, transforms from train import train, test # Training settings parser = argparse.ArgumentParser(description='PyTorch MNIST Example') parser.add_argument('--batch-size', type=int, default=64, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=10, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=0.01, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--momentum', type=float, default=0.5, metavar='M', help='SGD momentum (default: 0.5)') parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') parser.add_argument('--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--num-processes', type=int, default=2, metavar='N', help='how many training processes to use (default: 2)') parser.add_argument('--cuda', action='store_true', default=False, help='enables CUDA training') parser.add_argument('--mps', action='store_true', default=False, help='enables macOS GPU training') parser.add_argument('--save_model', action='store_true', default=False, help='save the trained model to state_dict') parser.add_argument('--dry-run', action='store_true', default=False, help='quickly check a single pass') class Net(nn.Module): def __init__(self): super(Net, self).__init__() self.conv1 = nn.Conv2d(1, 10, kernel_size=5) self.conv2 = nn.Conv2d(10, 20, kernel_size=5) self.conv2_drop = nn.Dropout2d() self.fc1 = nn.Linear(320, 50) self.fc2 = nn.Linear(50, 10) def forward(self, x): x = F.relu(F.max_pool2d(self.conv1(x), 2)) x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2)) x = x.view(-1, 320) x = F.relu(self.fc1(x)) x = F.dropout(x, training=self.training) x = self.fc2(x) return F.log_softmax(x, dim=1) if __name__ == '__main__': args = parser.parse_args() use_cuda = args.cuda and torch.cuda.is_available() use_mps = args.mps and torch.backends.mps.is_available() if use_cuda: device = torch.device("cuda") elif use_mps: device = torch.device("mps") else: device = torch.device("cpu") transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ]) dataset1 = datasets.MNIST('../data', train=True, download=True, transform=transform) dataset2 = datasets.MNIST('../data', train=False, transform=transform) kwargs = {'batch_size': args.batch_size, 'shuffle': True} if use_cuda: kwargs.update({'num_workers': 1, 'pin_memory': True, }) torch.manual_seed(args.seed) mp.set_start_method('spawn', force=True) model = Net().to(device) model.share_memory() # gradients are allocated lazily, so they are not shared here processes = [] for rank in range(args.num_processes): p = mp.Process(target=train, args=(rank, args, model, device, dataset1, kwargs)) # We first train the model across `num_processes` processes p.start() processes.append(p) for p in processes: p.join() if args.save_model: torch.save(model.state_dict(), "MNIST_hogwild.pt") # Once training is complete, we can test the model test(args, model, device, dataset2, kwargs) ================================================ FILE: mnist_hogwild/requirements.txt ================================================ torch torchvision==0.20.0 ================================================ FILE: mnist_hogwild/train.py ================================================ import os import torch import torch.optim as optim import torch.nn.functional as F def train(rank, args, model, device, dataset, dataloader_kwargs): torch.manual_seed(args.seed + rank) train_loader = torch.utils.data.DataLoader(dataset, **dataloader_kwargs) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) for epoch in range(1, args.epochs + 1): train_epoch(epoch, args, model, device, train_loader, optimizer) def test(args, model, device, dataset, dataloader_kwargs): torch.manual_seed(args.seed) test_loader = torch.utils.data.DataLoader(dataset, **dataloader_kwargs) test_epoch(model, device, test_loader) def train_epoch(epoch, args, model, device, data_loader, optimizer): model.train() pid = os.getpid() for batch_idx, (data, target) in enumerate(data_loader): optimizer.zero_grad() output = model(data.to(device)) loss = F.nll_loss(output, target.to(device)) loss.backward() optimizer.step() if batch_idx % args.log_interval == 0: print('{}\tTrain Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( pid, epoch, batch_idx * len(data), len(data_loader.dataset), 100. * batch_idx / len(data_loader), loss.item())) if args.dry_run: break def test_epoch(model, device, data_loader): model.eval() test_loss = 0 correct = 0 with torch.no_grad(): for data, target in data_loader: output = model(data.to(device)) test_loss += F.nll_loss(output, target.to(device), reduction='sum').item() # sum up batch loss pred = output.max(1)[1] # get the index of the max log-probability correct += pred.eq(target.to(device)).sum().item() test_loss /= len(data_loader.dataset) print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( test_loss, correct, len(data_loader.dataset), 100. * correct / len(data_loader.dataset))) ================================================ FILE: mnist_rnn/README.md ================================================ # Example of MNIST using RNN ## Motivation Create pytorch example similar to Official Tensorflow Keras RNN example using MNIST [here](https://www.tensorflow.org/guide/keras/rnn) ```bash pip install -r requirements.txt python main.py # CUDA_VISIBLE_DEVICES=2 python main.py # to specify GPU id to ex. 2 ``` ```bash optional arguments: -h, --help show this help message and exit --batch_size input batch_size for training (default:64) --testing_batch_size input batch size for testing (default: 1000) --epochs EPOCHS number of epochs to train (default: 14) --lr LR learning rate (default: 0.1) --gamma learning rate step gamma (default: 0.7) --accel enables accelerator --seed SEED random seed (default: 1) --save_model For saving the current Model --log_interval how many batches to wait before logging training status --dry-run quickly check a single pass ``` ================================================ FILE: mnist_rnn/main.py ================================================ from __future__ import print_function import argparse import torch import torch.nn as nn import torch.nn.functional as F import torch.optim as optim from torch.optim.lr_scheduler import StepLR from torchvision import datasets, transforms class Net(nn.Module): def __init__(self): super(Net, self).__init__() self.rnn = nn.LSTM(input_size=28, hidden_size=64, batch_first=True) self.batchnorm = nn.BatchNorm1d(64) self.dropout1 = nn.Dropout2d(0.25) self.dropout2 = nn.Dropout2d(0.5) self.fc1 = nn.Linear(64, 32) self.fc2 = nn.Linear(32, 10) def forward(self, input): # Shape of input is (batch_size,1, 28, 28) # converting shape of input to (batch_size, 28, 28) # as required by RNN when batch_first is set True input = input.reshape(-1, 28, 28) output, hidden = self.rnn(input) # RNN output shape is (seq_len, batch, input_size) # Get last output of RNN output = output[:, -1, :] output = self.batchnorm(output) output = self.dropout1(output) output = self.fc1(output) output = F.relu(output) output = self.dropout2(output) output = self.fc2(output) output = F.log_softmax(output, dim=1) return output def train(args, model, device, train_loader, optimizer, epoch): model.train() for batch_idx, (data, target) in enumerate(train_loader): data, target = data.to(device), target.to(device) optimizer.zero_grad() output = model(data) loss = F.nll_loss(output, target) loss.backward() optimizer.step() if batch_idx % args.log_interval == 0: print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( epoch, batch_idx * len(data), len(train_loader.dataset), 100. * batch_idx / len(train_loader), loss.item())) if args.dry_run: break def test(args, model, device, test_loader): model.eval() test_loss = 0 correct = 0 with torch.no_grad(): for data, target in test_loader: data, target = data.to(device), target.to(device) output = model(data) test_loss += F.nll_loss(output, target, reduction='sum').item() # sum up batch loss pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability correct += pred.eq(target.view_as(pred)).sum().item() if args.dry_run: break test_loss /= len(test_loader.dataset) print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( test_loss, correct, len(test_loader.dataset), 100. * correct / len(test_loader.dataset))) def main(): # Training settings parser = argparse.ArgumentParser(description='PyTorch MNIST Example using RNN') parser.add_argument('--batch-size', type=int, default=64, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=14, metavar='N', help='number of epochs to train (default: 14)') parser.add_argument('--lr', type=float, default=0.1, metavar='LR', help='learning rate (default: 0.1)') parser.add_argument('--gamma', type=float, default=0.7, metavar='M', help='learning rate step gamma (default: 0.7)') parser.add_argument('--accel', action='store_true', help='enables accelerator') parser.add_argument('--dry-run', action='store_true', help='quickly check a single pass') parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') parser.add_argument('--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--save-model', action='store_true', help='for Saving the current Model') args = parser.parse_args() if args.accel: device = torch.accelerator.current_accelerator() else: device = torch.device("cpu") torch.manual_seed(args.seed) kwargs = {'num_workers': 1, 'pin_memory': True} if args.accel else {} train_loader = torch.utils.data.DataLoader( datasets.MNIST('../data', train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])), batch_size=args.batch_size, shuffle=True, **kwargs) test_loader = torch.utils.data.DataLoader( datasets.MNIST('../data', train=False, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])), batch_size=args.test_batch_size, shuffle=True, **kwargs) model = Net().to(device) optimizer = optim.Adadelta(model.parameters(), lr=args.lr) scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma) for epoch in range(1, args.epochs + 1): train(args, model, device, train_loader, optimizer, epoch) test(args, model, device, test_loader) scheduler.step() if args.save_model: torch.save(model.state_dict(), "mnist_rnn.pt") if __name__ == '__main__': main() ================================================ FILE: mnist_rnn/requirements.txt ================================================ torch torchvision ================================================ FILE: regression/README.md ================================================ # Linear regression example Trains a single fully-connected layer to fit a 4th degree polynomial. ================================================ FILE: regression/main.py ================================================ #!/usr/bin/env python from __future__ import print_function from itertools import count import torch import torch.nn.functional as F POLY_DEGREE = 4 W_target = torch.randn(POLY_DEGREE, 1) * 5 b_target = torch.randn(1) * 5 def make_features(x): """Builds features i.e. a matrix with columns [x, x^2, x^3, x^4].""" x = x.unsqueeze(1) return torch.cat([x ** i for i in range(1, POLY_DEGREE+1)], 1) def f(x): """Approximated function.""" return x.mm(W_target) + b_target.item() def poly_desc(W, b): """Creates a string description of a polynomial.""" result = 'y = ' for i, w in enumerate(W): result += '{:+.2f} x^{} '.format(w, i + 1) result += '{:+.2f}'.format(b[0]) return result def get_batch(batch_size=32): """Builds a batch i.e. (x, f(x)) pair.""" random = torch.randn(batch_size) x = make_features(random) y = f(x) return x, y # Define model fc = torch.nn.Linear(W_target.size(0), 1) for batch_idx in count(1): # Get data batch_x, batch_y = get_batch() # Reset gradients fc.zero_grad() # Forward pass output = F.smooth_l1_loss(fc(batch_x), batch_y) loss = output.item() # Backward pass output.backward() # Apply gradients for param in fc.parameters(): param.data.add_(-0.1 * param.grad) # Stop criterion if loss < 1e-3: break print('Loss: {:.6f} after {} batches'.format(loss, batch_idx)) print('==> Learned function:\t' + poly_desc(fc.weight.view(-1), fc.bias)) print('==> Actual function:\t' + poly_desc(W_target.view(-1), b_target)) ================================================ FILE: regression/requirements.txt ================================================ torch ================================================ FILE: reinforcement_learning/README.md ================================================ # Reinforcement learning training example ```bash pip install -r requirements.txt # For REINFORCE: python reinforce.py # For actor critic: python actor_critic.py ``` ================================================ FILE: reinforcement_learning/actor_critic.py ================================================ import argparse import gymnasium as gym import numpy as np from itertools import count from collections import namedtuple import torch import torch.nn as nn import torch.nn.functional as F import torch.optim as optim from torch.distributions import Categorical # Cart Pole parser = argparse.ArgumentParser(description='PyTorch actor-critic example') parser.add_argument('--gamma', type=float, default=0.99, metavar='G', help='discount factor (default: 0.99)') parser.add_argument('--seed', type=int, default=543, metavar='N', help='random seed (default: 543)') parser.add_argument('--render', action='store_true', help='render the environment') parser.add_argument('--log-interval', type=int, default=10, metavar='N', help='interval between training status logs (default: 10)') args = parser.parse_args() render_mode = "human" if args.render else None env = gym.make('CartPole-v1', render_mode=render_mode) env.reset(seed=args.seed) torch.manual_seed(args.seed) SavedAction = namedtuple('SavedAction', ['log_prob', 'value']) class Policy(nn.Module): """ implements both actor and critic in one model """ def __init__(self): super(Policy, self).__init__() self.affine1 = nn.Linear(4, 128) # actor's layer self.action_head = nn.Linear(128, 2) # critic's layer self.value_head = nn.Linear(128, 1) # action & reward buffer self.saved_actions = [] self.rewards = [] def forward(self, x): """ forward of both actor and critic """ x = F.relu(self.affine1(x)) # actor: choses action to take from state s_t # by returning probability of each action action_prob = F.softmax(self.action_head(x), dim=-1) # critic: evaluates being in the state s_t state_values = self.value_head(x) # return values for both actor and critic as a tuple of 2 values: # 1. a list with the probability of each action over the action space # 2. the value from state s_t return action_prob, state_values model = Policy() optimizer = optim.Adam(model.parameters(), lr=3e-2) eps = np.finfo(np.float32).eps.item() def select_action(state): state = torch.from_numpy(state).float() probs, state_value = model(state) # create a categorical distribution over the list of probabilities of actions m = Categorical(probs) # and sample an action using the distribution action = m.sample() # save to action buffer model.saved_actions.append(SavedAction(m.log_prob(action), state_value)) # the action to take (left or right) return action.item() def finish_episode(): """ Training code. Calculates actor and critic loss and performs backprop. """ R = 0 saved_actions = model.saved_actions policy_losses = [] # list to save actor (policy) loss value_losses = [] # list to save critic (value) loss returns = [] # list to save the true values # calculate the true value using rewards returned from the environment for r in model.rewards[::-1]: # calculate the discounted value R = r + args.gamma * R returns.insert(0, R) returns = torch.tensor(returns) returns = (returns - returns.mean()) / (returns.std() + eps) for (log_prob, value), R in zip(saved_actions, returns): advantage = R - value.item() # calculate actor (policy) loss policy_losses.append(-log_prob * advantage) # calculate critic (value) loss using L1 smooth loss value_losses.append(F.smooth_l1_loss(value, torch.tensor([R]))) # reset gradients optimizer.zero_grad() # sum up all the values of policy_losses and value_losses loss = torch.stack(policy_losses).sum() + torch.stack(value_losses).sum() # perform backprop loss.backward() optimizer.step() # reset rewards and action buffer del model.rewards[:] del model.saved_actions[:] def main(): running_reward = 10 # run infinitely many episodes for i_episode in count(1): # reset environment and episode reward state, _ = env.reset() ep_reward = 0 # for each episode, only run 9999 steps so that we don't # infinite loop while learning for t in range(1, 10000): # select action from policy action = select_action(state) # take the action state, reward, terminated, truncated, _ = env.step(action) model.rewards.append(reward) ep_reward += reward if terminated or truncated: break # update cumulative reward running_reward = 0.05 * ep_reward + (1 - 0.05) * running_reward # perform backprop finish_episode() # log results if i_episode % args.log_interval == 0: print(f'Episode {i_episode}\tLast reward: {ep_reward:.2f}\tAverage reward: {running_reward:.2f}') # check if we have "solved" the cart pole problem if running_reward > env.spec.reward_threshold: print(f"Solved! Running reward is now {running_reward} and " f"the last episode runs to {t} time steps!") break if __name__ == '__main__': main() ================================================ FILE: reinforcement_learning/reinforce.py ================================================ import argparse import gymnasium as gym import numpy as np from itertools import count from collections import deque import torch import torch.nn as nn import torch.nn.functional as F import torch.optim as optim from torch.distributions import Categorical parser = argparse.ArgumentParser(description='PyTorch REINFORCE example') parser.add_argument('--gamma', type=float, default=0.99, metavar='G', help='discount factor (default: 0.99)') parser.add_argument('--seed', type=int, default=543, metavar='N', help='random seed (default: 543)') parser.add_argument('--render', action='store_true', help='render the environment') parser.add_argument('--log-interval', type=int, default=10, metavar='N', help='interval between training status logs (default: 10)') args = parser.parse_args() render_mode = "human" if args.render else None env = gym.make('CartPole-v1', render_mode=render_mode) env.reset(seed=args.seed) torch.manual_seed(args.seed) class Policy(nn.Module): def __init__(self): super(Policy, self).__init__() self.affine1 = nn.Linear(4, 128) self.dropout = nn.Dropout(p=0.6) self.affine2 = nn.Linear(128, 2) self.saved_log_probs = [] self.rewards = [] def forward(self, x): x = self.affine1(x) x = self.dropout(x) x = F.relu(x) action_scores = self.affine2(x) return F.softmax(action_scores, dim=1) policy = Policy() optimizer = optim.Adam(policy.parameters(), lr=1e-2) eps = np.finfo(np.float32).eps.item() def select_action(state): state = torch.from_numpy(state).float().unsqueeze(0) probs = policy(state) m = Categorical(probs) action = m.sample() policy.saved_log_probs.append(m.log_prob(action)) return action.item() def finish_episode(): R = 0 policy_loss = [] returns = deque() for r in policy.rewards[::-1]: R = r + args.gamma * R returns.appendleft(R) returns = torch.tensor(returns) returns = (returns - returns.mean()) / (returns.std() + eps) for log_prob, R in zip(policy.saved_log_probs, returns): policy_loss.append(-log_prob * R) optimizer.zero_grad() policy_loss = torch.cat(policy_loss).sum() policy_loss.backward() optimizer.step() del policy.rewards[:] del policy.saved_log_probs[:] def main(): running_reward = 10 for i_episode in count(1): state, _ = env.reset() ep_reward = 0 for t in range(1, 10000): # Don't infinite loop while learning action = select_action(state) state, reward, terminated, truncated, _ = env.step(action) if args.render: env.render() policy.rewards.append(reward) ep_reward += reward if terminated or truncated: break running_reward = 0.05 * ep_reward + (1 - 0.05) * running_reward finish_episode() if i_episode % args.log_interval == 0: print(f'Episode {i_episode}\tLast reward: {ep_reward:.2f}\tAverage reward: {running_reward:.2f}') if running_reward > env.spec.reward_threshold: print(f"Solved! Running reward is now {running_reward} and the last episode runs to {t} time steps!") break if __name__ == '__main__': main() ================================================ FILE: reinforcement_learning/requirements.txt ================================================ torch numpy gymnasium[classic-control] ================================================ FILE: run_cpp_examples.sh ================================================ #!/usr/bin/env bash # This script runs through the code in each of the cpp examples. # The purpose is just as an integration test, not to actually train models in any meaningful way. # Optionally specify a comma separated list of examples to run. # can be run as: # ./run_cpp_examples.sh "get_libtorch,run_all,clean" # To get libtorch, run all examples, and remove temporary/changed data files. BASE_DIR=`pwd`"/"`dirname $0` echo "BASE_DIR: $BASE_DIR" EXAMPLES=`echo $1 | sed -e 's/ //g'` HOME_DIR=$HOME ERRORS="" function error() { ERR=$1 ERRORS="$ERRORS\n$ERR" echo $ERR } function get_libtorch() { echo "Getting libtorch" cd $HOME_DIR if [ ! -d "libtorch" ]; then wget https://download.pytorch.org/libtorch/nightly/cpu/libtorch-cxx11-abi-shared-with-deps-latest.zip unzip libtorch-cxx11-abi-shared-with-deps-latest.zip fi if [ $? -eq 0 ]; then echo "Successfully downloaded and extracted libtorch" LIBTORCH_PATH="$HOME_DIR/libtorch" # Store the LibTorch path in a variable. echo "LibTorch path: $LIBTORCH_PATH" # Print the LibTorch path else error "Failed to download or extract LibTorch" fi } function start() { EXAMPLE=${FUNCNAME[1]} cd $BASE_DIR/cpp/$EXAMPLE echo "Running example: $EXAMPLE" } function check_run_success() { if [ $? -eq 0 ]; then echo "Successfully ran $1" else echo "Failed to run $1" error "Failed to run $1" exit 1 fi } function autograd() { start mkdir build cd build cmake -DCMAKE_PREFIX_PATH=$LIBTORCH_PATH .. make if [ $? -eq 0 ]; then echo "Successfully built $EXAMPLE" ./$EXAMPLE # Run the executable check_run_success $EXAMPLE else error "Failed to build $EXAMPLE" exit 1 fi } function custom-dataset() { start # Download the dataset and unzip it if [ ! -d "$BASE_DIR/cpp/$EXAMPLE/dataset" ]; then wget https://data.caltech.edu/records/mzrjq-6wc02/files/caltech-101.zip unzip caltech-101.zip cd caltech-101 tar -xzf 101_ObjectCategories.tar.gz mv 101_ObjectCategories $BASE_DIR/cpp/$EXAMPLE/dataset fi # build the executable and run it cd $BASE_DIR/cpp/$EXAMPLE mkdir build cd build cmake -DCMAKE_PREFIX_PATH=$LIBTORCH_PATH .. make if [ $? -eq 0 ]; then echo "Successfully built $EXAMPLE" cd $BASE_DIR/cpp/$EXAMPLE ./build/$EXAMPLE # Run the executable check_run_success $EXAMPLE else error "Failed to build $EXAMPLE" exit 1 fi } function dcgan() { start mkdir build cd build cmake -DCMAKE_PREFIX_PATH=$LIBTORCH_PATH .. make if [ $? -eq 0 ]; then echo "Successfully built $EXAMPLE" ./$EXAMPLE --epochs 5 # Run the executable with kNumberOfEpochs = 5 check_run_success $EXAMPLE else error "Failed to build $EXAMPLE" exit 1 fi } function mnist() { start mkdir build cd build cmake -DCMAKE_PREFIX_PATH=$LIBTORCH_PATH .. make if [ $? -eq 0 ]; then echo "Successfully built $EXAMPLE" ./$EXAMPLE # Run the executable check_run_success $EXAMPLE else error "Failed to build $EXAMPLE" exit 1 fi } function regression() { start mkdir build cd build cmake -DCMAKE_PREFIX_PATH=$LIBTORCH_PATH .. make if [ $? -eq 0 ]; then echo "Successfully built $EXAMPLE" ./$EXAMPLE # Run the executable check_run_success $EXAMPLE else error "Failed to build $EXAMPLE" exit 1 fi } function clean() { cd $BASE_DIR echo "Running clean to remove cruft" # Remove the build directories find . -type d -name 'build' -exec rm -rf {} + # Remove the libtorch directory rm -rf $HOME_DIR/libtorch rm -f $HOME_DIR/libtorch-shared-with-deps-latest.zip echo "Clean completed" } function run_all() { autograd custom-dataset regression dcgan mnist } # by default, run all examples if [ "" == "$EXAMPLES" ]; then run_all else for i in $(echo $EXAMPLES | sed "s/,/ /g") do echo "Starting $i" $i echo "Finished $i, status $?" done fi if [ "" == "$ERRORS" ]; then echo "Completed successfully with status $?" else echo "Some examples failed:" printf "$ERRORS" exit 1 fi ================================================ FILE: run_distributed_examples.sh ================================================ #!/usr/bin/env bash # # This script runs through the code in each of the python examples. # The purpose is just as an integration test, not to actually train models in any meaningful way. # For that reason, most of these set epochs = 1 and --dry-run. # # Optionally specify a comma separated list of examples to run. Can be run as: # * To run all examples: # ./run_distributed_examples.sh # * To run specific example: # ./run_distributed_examples.sh "distributed/tensor_parallelism,distributed/ddp" # # To test examples on CUDA accelerator, run as: # USE_CUDA=True ./run_distributed_examples.sh # # Script requires uv to be installed. When executed, script will install prerequisites from # `requirements.txt` for each example. If ran within activated virtual environment (uv venv, # python -m venv, conda) this might reinstall some of the packages. To change pip installation # index or to pass additional pip install options, run as: # PIP_INSTALL_ARGS="--pre -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html" \ # ./run_python_examples.sh # # To force script to create virtual environment for each example, run as: # VIRTUAL_ENV=".venv" ./run_distributed_examples.sh # Script will remove environments it creates in a teardown step after execution of each example. BASE_DIR="$(pwd)/$(dirname $0)" source $BASE_DIR/utils.sh USE_CUDA=${USE_CUDA:-False} case $USE_CUDA in "True") echo "using cuda" CUDA=1 CUDA_FLAG="--cuda" ;; "False") echo "not using cuda" CUDA=0 CUDA_FLAG="" ;; "") exit 1; ;; esac function distributed_tensor_parallelism() { uv run bash run_example.sh tensor_parallel_example.py || error "tensor parallel example failed" uv run bash run_example.sh sequence_parallel_example.py || error "sequence parallel example failed" uv run bash run_example.sh fsdp_tp_example.py || error "2D parallel example failed" } function distributed_FSDP2() { uv run bash run_example.sh example.py || error "FSDP2 example failed" } function distributed_ddp() { uv run bash run_example.sh example.py || error "ddp example failed" } function distributed_minGPT-ddp() { uv run bash run_example.sh mingpt/main.py || error "minGPT example failed" } function distributed_rpc_ddp_rpc() { uv run main.py || error "ddp_rpc example failed" } function distributed_rpc_rnn() { uv run main.py || error "rpc_rnn example failed" } function run_all() { run distributed/tensor_parallelism run distributed/ddp run distributed/minGPT-ddp run distributed/rpc/ddp_rpc run distributed/rpc/rnn } # by default, run all examples if [ "" == "$EXAMPLES" ]; then run_all else for i in $(echo $EXAMPLES | sed "s/,/ /g") do echo "Starting $i" run $i echo "Finished $i, status $?" done fi if [ "" == "$ERRORS" ]; then echo "Completed successfully with status $?" else echo "Some distributed examples failed:" printf "$ERRORS\n" #Exit with error (0-255) in case of failure in one of the tests. exit 1 fi ================================================ FILE: run_python_examples.sh ================================================ #!/bin/bash # # This script runs through the code in each of the python examples. # The purpose is just as an integration test, not to actually train models in any meaningful way. # For that reason, most of these set epochs = 1 and --dry-run. # # Optionally specify a comma separated list of examples to run. Can be run as: # * To run all examples: # ./run_python_examples.sh # * To run few specific examples: # ./run_python_examples.sh "dcgan,fast_neural_style" # # To test examples on CUDA accelerator, run as: # USE_CUDA=True ./run_python_examples.sh # # To test examples on hardware accelerator (CUDA, MPS, XPU, etc.), run as: # USE_ACCEL=True ./run_python_examples.sh # NOTE: USE_ACCEL relies on torch.accelerator API and not all examples are converted # to use it at the moment. Thus, expect failures using this flag on non-CUDA accelerators # and consider to run examples one by one. # # Script requires uv to be installed. When executed, script will install prerequisites from # `requirements.txt` for each example. If ran within activated virtual environment (uv venv, # python -m venv, conda) this might reinstall some of the packages. To change pip installation # index or to pass additional pip install options, run as: # PIP_INSTALL_ARGS="--pre -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html" \ # ./run_python_examples.sh # # To force script to create virtual environment for each example, run as: # VIRTUAL_ENV=".venv" ./run_python_examples.sh # Script will remove environments it creates in a teardown step after execution of each example. BASE_DIR="$(pwd)/$(dirname $0)" source $BASE_DIR/utils.sh # TODO: Leave only USE_ACCEL and drop USE_CUDA once all examples will be converted # to torch.accelerator API. For now, just add USE_ACCEL as an alias for USE_CUDA. if [ -n "$USE_ACCEL" ]; then USE_CUDA=$USE_ACCEL fi USE_CUDA=${USE_CUDA:-False} case $USE_CUDA in "True") echo "using cuda" CUDA=1 CUDA_FLAG="--cuda" ACCEL_FLAG="--accel" ;; "False") echo "not using cuda" CUDA=0 CUDA_FLAG="" ACCEL_FLAG="" ;; "") exit 1; ;; esac function dcgan() { uv run main.py --dataset fake $ACCEL_FLAG --dry-run || error "dcgan failed" } function fast_neural_style() { if [ ! -d "saved_models" ]; then echo "downloading saved models for fast neural style" uv run download_saved_models.py fi test -d "saved_models" || { error "saved models not found"; return; } echo "running fast neural style model" uv run neural_style/neural_style.py eval --content-image images/content-images/amber.jpg --model saved_models/candy.pth --output-image images/output-images/amber-candy.jpg $ACCEL_FLAG || error "neural_style.py failed" } function imagenet() { if [[ ! -d "sample/val" || ! -d "sample/train" ]]; then mkdir -p sample/val/n mkdir -p sample/train/n curl -O "https://upload.wikimedia.org/wikipedia/commons/5/5a/Socks-clinton.jpg" || { error "couldn't download sample image for imagenet"; return; } mv Socks-clinton.jpg sample/train/n cp sample/train/n/* sample/val/n/ fi uv run main.py --epochs 1 sample/ || error "imagenet example failed" uv run main.py --epochs 1 --gpu 0 sample/ || error "imagenet example failed" } function language_translation() { uv run -m spacy download en || error "couldn't download en package from spacy" uv run -m spacy download de || error "couldn't download de package from spacy" uv run main.py -e 1 --enc_layers 1 --dec_layers 1 --backend cpu --logging_dir output/ --dry_run || error "language translation example failed" } function mnist() { uv run main.py --epochs 1 --dry-run || error "mnist example failed" } function mnist_forward_forward() { uv run main.py --epochs 1 --no_accel || error "mnist forward forward failed" } function mnist_hogwild() { uv run main.py --epochs 1 --dry-run $CUDA_FLAG || error "mnist hogwild failed" } function mnist_rnn() { uv run main.py --epochs 1 --dry-run || error "mnist rnn example failed" } function regression() { uv run main.py --epochs 1 $CUDA_FLAG || error "regression failed" } function siamese_network() { uv run main.py --epochs 1 --dry-run || error "siamese network example failed" } function reinforcement_learning() { uv run reinforce.py || error "reinforcement learning reinforce failed" uv run actor_critic.py || error "reinforcement learning actor_critic failed" } function snli() { echo "installing 'en' model if not installed" uv run -m spacy download en || { error "couldn't download 'en' model needed for snli"; return; } echo "training..." uv run train.py --epochs 1 --dev_every 1 --no-bidirectional --dry-run || error "couldn't train snli" } function fx() { # uv run custom_tracer.py || error "fx custom tracer has failed" UnboundLocalError: local variable 'tabulate' referenced before assignment uv run invert.py || error "fx invert has failed" uv run module_tracer.py || error "fx module tracer has failed" uv run primitive_library.py || error "fx primitive library has failed" uv run profiling_tracer.py || error "fx profiling tracer has failed" uv run replace_op.py || error "fx replace op has failed" uv run subgraph_rewriter_basic_use.py || error "fx subgraph has failed" uv run wrap_output_dynamically.py || error "vmap output dynamically has failed" } function super_resolution() { uv run main.py --upscale_factor 3 --batchSize 4 --testBatchSize 100 --nEpochs 1 --lr 0.001 $ACCEL_FLAG || error "super resolution failed" uv run super_resolve.py --input_image dataset/BSDS300/images/test/16077.jpg --model model_epoch_1.pth --output_filename out.png $ACCEL_FLAG || error "super resolution upscaling failed" } function time_sequence_prediction() { uv run generate_sine_wave.py || { error "generate sine wave failed"; return; } uv run train.py --steps 2 || error "time sequence prediction training failed" } function vae() { uv run main.py --epochs 1 || error "vae failed" } function vision_transformer() { uv run main.py --epochs 1 --dry-run || error "vision transformer example failed" } function word_language_model() { uv run main.py --epochs 1 --dry-run $ACCEL_FLAG || error "word_language_model failed" uv run generate.py $ACCEL_FLAG || error "word_language_model generate failed" for model in "RNN_TANH" "RNN_RELU" "LSTM" "GRU" "Transformer"; do uv run main.py --model $model --epochs 1 --dry-run $ACCEL_FLAG || error "word_language_model failed" uv run generate.py $ACCEL_FLAG || error "word_language_model generate failed" done } function gcn() { uv run main.py --epochs 1 --dry-run || error "graph convolutional network failed" } function gat() { uv run main.py --epochs 1 --dry-run || error "graph attention network failed" } eval "base_$(declare -f stop)" function stop() { cd $BASE_DIR rm -rf dcgan/fake_samples_epoch_000.png \ dcgan/netD_epoch_0.pth \ dcgan/netG_epoch_0.pth \ dcgan/real_samples.png \ fast_neural_style/saved_models.zip \ fast_neural_style/saved_models/ \ imagenet/checkpoint.pth.tar \ imagenet/lsun/ \ imagenet/model_best.pth.tar \ imagenet/sample/ \ language_translation/output/ \ snli/.data/ \ snli/.vector_cache/ \ snli/results/ \ time_sequence_prediction/predict*.pdf \ time_sequence_prediction/traindata.pt \ word_language_model/model.pt \ gcn/cora/ \ gat/cora/ || error "couldn't clean up some files" git checkout fast_neural_style/images/output-images/amber-candy.jpg || error "couldn't clean up fast neural style image" base_stop "$1" } function run_all() { # cpp moved to `run_cpp_examples.sh``` run dcgan # distributed moved to `run_distributed_examples.sh` run fast_neural_style run imagenet # language_translation run mnist run mnist_forward_forward run mnist_hogwild run mnist_rnn run regression run reinforcement_learning run siamese_network # run super_resolution - flaky run time_sequence_prediction run vae # vision_transformer - example broken see https://github.com/pytorch/examples/issues/1184 and https://github.com/pytorch/examples/pull/1258 for more details run word_language_model run fx run gcn run gat } # by default, run all examples if [ "" == "$EXAMPLES" ]; then run_all else for i in $(echo $EXAMPLES | sed "s/,/ /g") do echo "Starting $i" run $i echo "Finished $i, status $?" done fi if [ "" == "$ERRORS" ]; then echo "Completed successfully with status $?" else echo "Some python examples failed:" printf "$ERRORS\n" #Exit with error (0-255) in case of failure in one of the tests. exit 1 fi ================================================ FILE: runtime.txt ================================================ 3.9 ================================================ FILE: siamese_network/README.md ================================================ # Siamese Network Example Siamese network for image similarity estimation. The network is composed of two identical networks, one for each input. The output of each network is concatenated and passed to a linear layer. The output of the linear layer passed through a sigmoid function. [FaceNet](https://arxiv.org/pdf/1503.03832.pdf) is a variant of the Siamese network. This implementation varies from FaceNet as we use the `ResNet-18` model from [Deep Residual Learning for Image Recognition](https://arxiv.org/pdf/1512.03385.pdf) as our feature extractor. In addition, we aren't using `TripletLoss` as the MNIST dataset is simple, so `BCELoss` can do the trick. ### Usage Install the required dependencies: ```bash pip install -r requirements.txt ``` To run the example, execute: ```bahs python main.py # CUDA_VISIBLE_DEVICES=2 python main.py # to specify GPU id to ex. 2 ``` If a hardware accelerator device is detected, the example will execute on the accelerator; otherwise, it will run on the CPU. To force execution on the CPU, use `--no-accel` command line argument: ```bash python main.py --no-accel ``` Optionally, you can add the following arguments to customize your execution. ```bash --batch-size input batch size for training (default: 64) --test-batch-size input batch size for testing (default: 1000) --epochs number of epochs to train (default: 14) --lr learning rate (default: 1.0) --gamma learning rate step gamma (default: 0.7) --no-accel disables accelerator --dry-run quickly check a single pass --seed random seed (default: 1) --log-interval how many batches to wait before logging training status --save-model Saving the current Model ``` ================================================ FILE: siamese_network/main.py ================================================ from __future__ import print_function import argparse, random, copy import numpy as np import torch import torch.nn as nn import torch.nn.functional as F import torch.optim as optim import torchvision from torch.utils.data import Dataset from torchvision import datasets from torchvision import transforms as T from torch.optim.lr_scheduler import StepLR class SiameseNetwork(nn.Module): """ Siamese network for image similarity estimation. The network is composed of two identical networks, one for each input. The output of each network is concatenated and passed to a linear layer. The output of the linear layer passed through a sigmoid function. `"FaceNet" `_ is a variant of the Siamese network. This implementation varies from FaceNet as we use the `ResNet-18` model from `"Deep Residual Learning for Image Recognition" `_ as our feature extractor. In addition, we aren't using `TripletLoss` as the MNIST dataset is simple, so `BCELoss` can do the trick. """ def __init__(self): super(SiameseNetwork, self).__init__() # get resnet model self.resnet = torchvision.models.resnet18(weights=None) # over-write the first conv layer to be able to read MNIST images # as resnet18 reads (3,x,x) where 3 is RGB channels # whereas MNIST has (1,x,x) where 1 is a gray-scale channel self.resnet.conv1 = nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False) self.fc_in_features = self.resnet.fc.in_features # remove the last layer of resnet18 (linear layer which is before avgpool layer) self.resnet = torch.nn.Sequential(*(list(self.resnet.children())[:-1])) # add linear layers to compare between the features of the two images self.fc = nn.Sequential( nn.Linear(self.fc_in_features * 2, 256), nn.ReLU(inplace=True), nn.Linear(256, 1), ) self.sigmoid = nn.Sigmoid() # initialize the weights self.resnet.apply(self.init_weights) self.fc.apply(self.init_weights) def init_weights(self, m): if isinstance(m, nn.Linear): torch.nn.init.xavier_uniform_(m.weight) m.bias.data.fill_(0.01) def forward_once(self, x): output = self.resnet(x) output = output.view(output.size()[0], -1) return output def forward(self, input1, input2): # get two images' features output1 = self.forward_once(input1) output2 = self.forward_once(input2) # concatenate both images' features output = torch.cat((output1, output2), 1) # pass the concatenation to the linear layers output = self.fc(output) # pass the out of the linear layers to sigmoid layer output = self.sigmoid(output) return output class APP_MATCHER(Dataset): def __init__(self, root, train, download=False): super(APP_MATCHER, self).__init__() # get MNIST dataset self.dataset = datasets.MNIST(root, train=train, download=download) # as `self.dataset.data`'s shape is (Nx28x28), where N is the number of # examples in MNIST dataset, a single example has the dimensions of # (28x28) for (WxH), where W and H are the width and the height of the image. # However, every example should have (CxWxH) dimensions where C is the number # of channels to be passed to the network. As MNIST contains gray-scale images, # we add an additional dimension to corresponds to the number of channels. self.data = self.dataset.data.unsqueeze(1).clone() self.group_examples() def group_examples(self): """ To ease the accessibility of data based on the class, we will use `group_examples` to group examples based on class. Every key in `grouped_examples` corresponds to a class in MNIST dataset. For every key in `grouped_examples`, every value will conform to all of the indices for the MNIST dataset examples that correspond to that key. """ # get the targets from MNIST dataset np_arr = np.array(self.dataset.targets.clone(), dtype=None, copy=None) # group examples based on class self.grouped_examples = {} for i in range(0,10): self.grouped_examples[i] = np.where((np_arr==i))[0] def __len__(self): return self.data.shape[0] def __getitem__(self, index): """ For every example, we will select two images. There are two cases, positive and negative examples. For positive examples, we will have two images from the same class. For negative examples, we will have two images from different classes. Given an index, if the index is even, we will pick the second image from the same class, but it won't be the same image we chose for the first class. This is used to ensure the positive example isn't trivial as the network would easily distinguish the similarity between same images. However, if the network were given two different images from the same class, the network will need to learn the similarity between two different images representing the same class. If the index is odd, we will pick the second image from a different class than the first image. """ # pick some random class for the first image selected_class = random.randint(0, 9) # pick a random index for the first image in the grouped indices based of the label # of the class random_index_1 = random.randint(0, self.grouped_examples[selected_class].shape[0]-1) # pick the index to get the first image index_1 = self.grouped_examples[selected_class][random_index_1] # get the first image image_1 = self.data[index_1].clone().float() # same class if index % 2 == 0: # pick a random index for the second image random_index_2 = random.randint(0, self.grouped_examples[selected_class].shape[0]-1) # ensure that the index of the second image isn't the same as the first image while random_index_2 == random_index_1: random_index_2 = random.randint(0, self.grouped_examples[selected_class].shape[0]-1) # pick the index to get the second image index_2 = self.grouped_examples[selected_class][random_index_2] # get the second image image_2 = self.data[index_2].clone().float() # set the label for this example to be positive (1) target = torch.tensor(1, dtype=torch.float) # different class else: # pick a random class other_selected_class = random.randint(0, 9) # ensure that the class of the second image isn't the same as the first image while other_selected_class == selected_class: other_selected_class = random.randint(0, 9) # pick a random index for the second image in the grouped indices based of the label # of the class random_index_2 = random.randint(0, self.grouped_examples[other_selected_class].shape[0]-1) # pick the index to get the second image index_2 = self.grouped_examples[other_selected_class][random_index_2] # get the second image image_2 = self.data[index_2].clone().float() # set the label for this example to be negative (0) target = torch.tensor(0, dtype=torch.float) return image_1, image_2, target def train(args, model, device, train_loader, optimizer, epoch): model.train() # we aren't using `TripletLoss` as the MNIST dataset is simple, so `BCELoss` can do the trick. criterion = nn.BCELoss() for batch_idx, (images_1, images_2, targets) in enumerate(train_loader): images_1, images_2, targets = images_1.to(device), images_2.to(device), targets.to(device) optimizer.zero_grad() outputs = model(images_1, images_2).squeeze() loss = criterion(outputs, targets) loss.backward() optimizer.step() if batch_idx % args.log_interval == 0: print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( epoch, batch_idx * len(images_1), len(train_loader.dataset), 100. * batch_idx / len(train_loader), loss.item())) if args.dry_run: break def test(model, device, test_loader): model.eval() test_loss = 0 correct = 0 # we aren't using `TripletLoss` as the MNIST dataset is simple, so `BCELoss` can do the trick. criterion = nn.BCELoss() with torch.no_grad(): for (images_1, images_2, targets) in test_loader: images_1, images_2, targets = images_1.to(device), images_2.to(device), targets.to(device) outputs = model(images_1, images_2).squeeze() test_loss += criterion(outputs, targets).sum().item() # sum up batch loss pred = torch.where(outputs > 0.5, 1, 0) # get the index of the max log-probability correct += pred.eq(targets.view_as(pred)).sum().item() test_loss /= len(test_loader.dataset) # for the 1st epoch, the average loss is 0.0001 and the accuracy 97-98% # using default settings. After completing the 10th epoch, the average # loss is 0.0000 and the accuracy 99.5-100% using default settings. print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( test_loss, correct, len(test_loader.dataset), 100. * correct / len(test_loader.dataset))) def main(): # Training settings parser = argparse.ArgumentParser(description='PyTorch Siamese network Example') parser.add_argument('--batch-size', type=int, default=64, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=14, metavar='N', help='number of epochs to train (default: 14)') parser.add_argument('--lr', type=float, default=1.0, metavar='LR', help='learning rate (default: 1.0)') parser.add_argument('--gamma', type=float, default=0.7, metavar='M', help='Learning rate step gamma (default: 0.7)') parser.add_argument('--no-accel', action='store_true', help='disables accelerator') parser.add_argument('--dry-run', action='store_true', default=False, help='quickly check a single pass') parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') parser.add_argument('--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--save-model', action='store_true', default=False, help='For Saving the current Model') args = parser.parse_args() use_accel = not args.no_accel and torch.accelerator.is_available() torch.manual_seed(args.seed) if use_accel: device = torch.accelerator.current_accelerator() else: device = torch.device("cpu") print(f"Using device: {device}") train_kwargs = {'batch_size': args.batch_size} test_kwargs = {'batch_size': args.test_batch_size} if use_accel: accel_kwargs = {'num_workers': 1, 'pin_memory': True, 'shuffle': True} train_kwargs.update(accel_kwargs) test_kwargs.update(accel_kwargs) train_dataset = APP_MATCHER('../data', train=True, download=True) test_dataset = APP_MATCHER('../data', train=False) train_loader = torch.utils.data.DataLoader(train_dataset,**train_kwargs) test_loader = torch.utils.data.DataLoader(test_dataset, **test_kwargs) model = SiameseNetwork().to(device) optimizer = optim.Adadelta(model.parameters(), lr=args.lr) scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma) for epoch in range(1, args.epochs + 1): train(args, model, device, train_loader, optimizer, epoch) test(model, device, test_loader) scheduler.step() if args.save_model: torch.save(model.state_dict(), "siamese_network.pt") if __name__ == '__main__': main() ================================================ FILE: siamese_network/requirements.txt ================================================ torch torchvision ================================================ FILE: super_resolution/README.md ================================================ # Superresolution using an efficient sub-pixel convolutional neural network This example illustrates how to use the efficient sub-pixel convolution layer described in ["Real-Time Single Image and Video Super-Resolution Using an Efficient Sub-Pixel Convolutional Neural Network" - Shi et al.](https://arxiv.org/abs/1609.05158) for increasing spatial resolution within your network for tasks such as superresolution. ``` usage: main.py [-h] --upscale_factor UPSCALE_FACTOR [--batchSize BATCHSIZE] [--testBatchSize TESTBATCHSIZE] [--nEpochs NEPOCHS] [--lr LR] [--accel] [--threads THREADS] [--seed SEED] PyTorch Super Res Example optional arguments: -h, --help show this help message and exit --upscale_factor super resolution upscale factor --batchSize training batch size --testBatchSize testing batch size --nEpochs number of epochs to train for --lr Learning Rate. Default=0.01 --accel use accelerator --threads number of threads for data loader to use Default=4 --seed random seed to use. Default=123 ``` This example trains a super-resolution network on the [BSD300 dataset](https://www2.eecs.berkeley.edu/Research/Projects/CS/vision/bsds/), using crops from the 200 training images, and evaluating on crops of the 100 test images. A snapshot of the model after every epoch with filename `model_epoch_.pth`. ## Example Usage: ### Train ```bash python main.py --upscale_factor 3 --batchSize 4 --testBatchSize 100 --nEpochs 30 --lr 0.001 --accel ``` ### Super Resolve ```bash python super_resolve.py --input_image dataset/BSDS300/images/test/16077.jpg --model model_epoch_30.pth --output_filename out.png --accel ``` ================================================ FILE: super_resolution/data.py ================================================ from os.path import exists, join, basename from os import makedirs, remove from six.moves import urllib import tarfile from torchvision.transforms import Compose, CenterCrop, ToTensor, Resize from dataset import DatasetFromFolder def download_bsd300(dest="dataset"): output_image_dir = join(dest, "BSDS300/images") if not exists(output_image_dir): makedirs(dest) url = "http://www2.eecs.berkeley.edu/Research/Projects/CS/vision/bsds/BSDS300-images.tgz" print("downloading url ", url) data = urllib.request.urlopen(url) file_path = join(dest, basename(url)) with open(file_path, 'wb') as f: f.write(data.read()) print("Extracting data") with tarfile.open(file_path) as tar: for item in tar: tar.extract(item, dest) remove(file_path) return output_image_dir def calculate_valid_crop_size(crop_size, upscale_factor): return crop_size - (crop_size % upscale_factor) def input_transform(crop_size, upscale_factor): return Compose([ CenterCrop(crop_size), Resize(crop_size // upscale_factor), ToTensor(), ]) def target_transform(crop_size): return Compose([ CenterCrop(crop_size), ToTensor(), ]) def get_training_set(upscale_factor): root_dir = download_bsd300() train_dir = join(root_dir, "train") crop_size = calculate_valid_crop_size(256, upscale_factor) return DatasetFromFolder(train_dir, input_transform=input_transform(crop_size, upscale_factor), target_transform=target_transform(crop_size)) def get_test_set(upscale_factor): root_dir = download_bsd300() test_dir = join(root_dir, "test") crop_size = calculate_valid_crop_size(256, upscale_factor) return DatasetFromFolder(test_dir, input_transform=input_transform(crop_size, upscale_factor), target_transform=target_transform(crop_size)) ================================================ FILE: super_resolution/dataset.py ================================================ import torch.utils.data as data from os import listdir from os.path import join from PIL import Image def is_image_file(filename): return any(filename.endswith(extension) for extension in [".png", ".jpg", ".jpeg"]) def load_img(filepath): img = Image.open(filepath).convert('YCbCr') y, _, _ = img.split() return y class DatasetFromFolder(data.Dataset): def __init__(self, image_dir, input_transform=None, target_transform=None): super(DatasetFromFolder, self).__init__() self.image_filenames = [join(image_dir, x) for x in listdir(image_dir) if is_image_file(x)] self.input_transform = input_transform self.target_transform = target_transform def __getitem__(self, index): input = load_img(self.image_filenames[index]) target = input.copy() if self.input_transform: input = self.input_transform(input) if self.target_transform: target = self.target_transform(target) return input, target def __len__(self): return len(self.image_filenames) ================================================ FILE: super_resolution/main.py ================================================ from __future__ import print_function import argparse from math import log10 import torch import torch.nn as nn import torch.optim as optim from torch.utils.data import DataLoader from model import Net from data import get_training_set, get_test_set # Training settings parser = argparse.ArgumentParser(description='PyTorch Super Res Example') parser.add_argument('--upscale_factor', type=int, required=True, help="super resolution upscale factor") parser.add_argument('--batchSize', type=int, default=64, help='training batch size') parser.add_argument('--testBatchSize', type=int, default=10, help='testing batch size') parser.add_argument('--nEpochs', type=int, default=2, help='number of epochs to train for') parser.add_argument('--lr', type=float, default=0.01, help='Learning Rate. Default=0.01') parser.add_argument('--accel', action='store_true', help='Enables acceleration for training, if available') parser.add_argument('--threads', type=int, default=4, help='number of threads for data loader to use') parser.add_argument('--seed', type=int, default=123, help='random seed to use. Default=123') opt = parser.parse_args() print(opt) torch.manual_seed(opt.seed) if opt.accel and torch.accelerator.is_available(): device = torch.accelerator.current_accelerator() else: device = torch.device("cpu") print('===> Loading datasets') train_set = get_training_set(opt.upscale_factor) test_set = get_test_set(opt.upscale_factor) training_data_loader = DataLoader(dataset=train_set, num_workers=opt.threads, batch_size=opt.batchSize, shuffle=True) testing_data_loader = DataLoader(dataset=test_set, num_workers=opt.threads, batch_size=opt.testBatchSize, shuffle=False) print('===> Building model') model = Net(upscale_factor=opt.upscale_factor).to(device) criterion = nn.MSELoss() optimizer = optim.Adam(model.parameters(), lr=opt.lr) def train(epoch): epoch_loss = 0 for iteration, batch in enumerate(training_data_loader, 1): input, target = batch[0].to(device), batch[1].to(device) optimizer.zero_grad() loss = criterion(model(input), target) epoch_loss += loss.item() loss.backward() optimizer.step() print("===> Epoch[{}]({}/{}): Loss: {:.4f}".format(epoch, iteration, len(training_data_loader), loss.item())) print("===> Epoch {} Complete: Avg. Loss: {:.4f}".format(epoch, epoch_loss / len(training_data_loader))) def test(): avg_psnr = 0 with torch.no_grad(): for batch in testing_data_loader: input, target = batch[0].to(device), batch[1].to(device) prediction = model(input) mse = criterion(prediction, target) psnr = 10 * log10(1 / mse.item()) avg_psnr += psnr print("===> Avg. PSNR: {:.4f} dB".format(avg_psnr / len(testing_data_loader))) def checkpoint(epoch): model_out_path = "model_epoch_{}.pth".format(epoch) torch.save(model, model_out_path) print("Checkpoint saved to {}".format(model_out_path)) for epoch in range(1, opt.nEpochs + 1): train(epoch) test() checkpoint(epoch) ================================================ FILE: super_resolution/model.py ================================================ import torch import torch.nn as nn import torch.nn.init as init class Net(nn.Module): def __init__(self, upscale_factor): super(Net, self).__init__() self.relu = nn.ReLU() self.conv1 = nn.Conv2d(1, 64, (5, 5), (1, 1), (2, 2)) self.conv2 = nn.Conv2d(64, 64, (3, 3), (1, 1), (1, 1)) self.conv3 = nn.Conv2d(64, 32, (3, 3), (1, 1), (1, 1)) self.conv4 = nn.Conv2d(32, upscale_factor ** 2, (3, 3), (1, 1), (1, 1)) self.pixel_shuffle = nn.PixelShuffle(upscale_factor) self._initialize_weights() def forward(self, x): x = self.relu(self.conv1(x)) x = self.relu(self.conv2(x)) x = self.relu(self.conv3(x)) x = self.pixel_shuffle(self.conv4(x)) return x def _initialize_weights(self): init.orthogonal_(self.conv1.weight, init.calculate_gain('relu')) init.orthogonal_(self.conv2.weight, init.calculate_gain('relu')) init.orthogonal_(self.conv3.weight, init.calculate_gain('relu')) init.orthogonal_(self.conv4.weight) ================================================ FILE: super_resolution/requirements.txt ================================================ six torch torchvision ================================================ FILE: super_resolution/super_resolve.py ================================================ from __future__ import print_function import argparse import torch from PIL import Image from torchvision.transforms import ToTensor from model import Net import numpy as np # Training settings parser = argparse.ArgumentParser(description='PyTorch Super Res Example') parser.add_argument('--input_image', type=str, required=True, help='input image to use') parser.add_argument('--model', type=str, required=True, help='model file to use') parser.add_argument('--output_filename', type=str, help='where to save the output image') parser.add_argument('--accel', action='store_true', help='Enables acceleration device, if available') opt = parser.parse_args() print(opt) img = Image.open(opt.input_image).convert('YCbCr') y, cb, cr = img.split() with open(opt.model, 'rb') as f: safe_globals = [ Net, torch.nn.modules.activation.ReLU, torch.nn.modules.conv.Conv2d, torch.nn.modules.pixelshuffle.PixelShuffle, ] with torch.serialization.safe_globals(safe_globals): model = torch.load(f) img_to_tensor = ToTensor() input = img_to_tensor(y).view(1, -1, y.size[1], y.size[0]) if opt.accel: device = torch.accelerator.current_accelerator() model = model.to(device) input = input.to(device) out = model(input) out = out.cpu() out_img_y = out[0].detach().numpy() out_img_y *= 255.0 out_img_y = out_img_y.clip(0, 255) out_img_y = Image.fromarray(np.uint8(out_img_y[0]), mode='L') out_img_cb = cb.resize(out_img_y.size, Image.BICUBIC) out_img_cr = cr.resize(out_img_y.size, Image.BICUBIC) out_img = Image.merge('YCbCr', [out_img_y, out_img_cb, out_img_cr]).convert('RGB') out_img.save(opt.output_filename) print('output image saved to ', opt.output_filename) ================================================ FILE: time_sequence_prediction/README.md ================================================ # Time Sequence Prediction This is a toy example for beginners to start with. It helps learn both PyTorch and time sequence prediction. Two LSTMCell units are used in this example to learn some sine wave signals starting at different phases. After learning the sine waves, the network tries to predict the signal values in the future. The results are shown in the picture below. ## Usage ``` python generate_sine_wave.py python train.py ``` ## Result The initial signal and the predicted results are shown in the image. We first give some initial signals (full line). The network will subsequently give some predicted results (dash line). It can be concluded that the network can generate new sine waves. ![image](https://cloud.githubusercontent.com/assets/1419566/24184438/e24f5280-0f08-11e7-8f8b-4d972b527a81.png) ================================================ FILE: time_sequence_prediction/generate_sine_wave.py ================================================ import numpy as np import torch np.random.seed(2) T = 20 L = 1000 N = 100 x = np.empty((N, L), 'int64') x[:] = np.array(range(L)) + np.random.randint(-4 * T, 4 * T, N).reshape(N, 1) data = np.sin(x / 1.0 / T).astype('float64') torch.save(data, open('traindata.pt', 'wb')) ================================================ FILE: time_sequence_prediction/requirements.txt ================================================ torch<2.6 matplotlib ================================================ FILE: time_sequence_prediction/train.py ================================================ from __future__ import print_function import argparse import torch import torch.nn as nn import torch.optim as optim import numpy as np import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt class Sequence(nn.Module): def __init__(self): super(Sequence, self).__init__() self.lstm1 = nn.LSTMCell(1, 51) self.lstm2 = nn.LSTMCell(51, 51) self.linear = nn.Linear(51, 1) def forward(self, input, future = 0): outputs = [] h_t = torch.zeros(input.size(0), 51, dtype=torch.double) c_t = torch.zeros(input.size(0), 51, dtype=torch.double) h_t2 = torch.zeros(input.size(0), 51, dtype=torch.double) c_t2 = torch.zeros(input.size(0), 51, dtype=torch.double) for input_t in input.split(1, dim=1): h_t, c_t = self.lstm1(input_t, (h_t, c_t)) h_t2, c_t2 = self.lstm2(h_t, (h_t2, c_t2)) output = self.linear(h_t2) outputs += [output] for i in range(future):# if we should predict the future h_t, c_t = self.lstm1(output, (h_t, c_t)) h_t2, c_t2 = self.lstm2(h_t, (h_t2, c_t2)) output = self.linear(h_t2) outputs += [output] outputs = torch.cat(outputs, dim=1) return outputs if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--steps', type=int, default=15, help='steps to run') opt = parser.parse_args() # set random seed to 0 np.random.seed(0) torch.manual_seed(0) # load data and make training set data = torch.load('traindata.pt') input = torch.from_numpy(data[3:, :-1]) target = torch.from_numpy(data[3:, 1:]) test_input = torch.from_numpy(data[:3, :-1]) test_target = torch.from_numpy(data[:3, 1:]) # build the model seq = Sequence() seq.double() criterion = nn.MSELoss() # use LBFGS as optimizer since we can load the whole data to train optimizer = optim.LBFGS(seq.parameters(), lr=0.8) #begin to train for i in range(opt.steps): print('STEP: ', i) def closure(): optimizer.zero_grad() out = seq(input) loss = criterion(out, target) print('loss:', loss.item()) loss.backward() return loss optimizer.step(closure) # begin to predict, no need to track gradient here with torch.no_grad(): future = 1000 pred = seq(test_input, future=future) loss = criterion(pred[:, :-future], test_target) print('test loss:', loss.item()) y = pred.detach().numpy() # draw the result plt.figure(figsize=(30,10)) plt.title('Predict future values for time sequences\n(Dashlines are predicted values)', fontsize=30) plt.xlabel('x', fontsize=20) plt.ylabel('y', fontsize=20) plt.xticks(fontsize=20) plt.yticks(fontsize=20) def draw(yi, color): plt.plot(np.arange(input.size(1)), yi[:input.size(1)], color, linewidth = 2.0) plt.plot(np.arange(input.size(1), input.size(1) + future), yi[input.size(1):], color + ':', linewidth = 2.0) draw(y[0], 'r') draw(y[1], 'g') draw(y[2], 'b') plt.savefig('predict%d.pdf'%i) plt.close() ================================================ FILE: utils.sh ================================================ #!/usr/bin/env bash # This script contains utility functions and initialize exmaple scripts. # Eg: run_python_examples.sh, run_distributed_examples.sh BASE_DIR="$(pwd)/$(dirname $0)" EXAMPLES=$(echo $1 | sed -e 's/ //g') # Redirect 'python' calls to 'python3' python() { command python3 "$@" } ERRORS=${ERRORS-""} function error() { ERR=$1 if [ "" == "$ERRORS" ]; then ERRORS="$ERR" else ERRORS="$ERRORS\n$ERR" fi } function start() { EXAMPLE=$1 cd $BASE_DIR/$EXAMPLE || { error "$EXAMPLE: no such example"; return 1; } echo "Install dependencies for $EXAMPLE" # Setting VIRTUAL_ENV=.venv externally will create uv virtual environment # for each sample in start() and remove it in stop(). Note that this environment # variable also forces other uv commands such as `uv pip...` and `uv run...` to # use the specified environment. if [ "$VIRTUAL_ENV" = ".venv" ]; then uv venv || { error "$EXAMPLE: failed to create virtual environment"; return 1; } fi uv pip install -r requirements.txt $PIP_INSTALL_ARGS || { error "$EXAMPLE: failed to install requirements"; return 1; } echo "Running example: $EXAMPLE" } function stop() { EXAMPLE=$1 if [ "$VIRTUAL_ENV" = ".venv" ]; then cd $BASE_DIR/$EXAMPLE && rm -rf .venv fi } function run() { EXAMPLE=$1 if start $EXAMPLE; then # drop trailing slash (occurs due to auto completion in bash interactive mode) # replace slashes with underscores: this allows to call nested examples EXAMPLE_FN=$(echo $EXAMPLE | sed "s@/\$@@" | sed 's@/@_@g') $EXAMPLE_FN fi stop $EXAMPLE } ================================================ FILE: vae/README.md ================================================ # Basic VAE Example This is an improved implementation of the paper [Auto-Encoding Variational Bayes](http://arxiv.org/abs/1312.6114) by Kingma and Welling. It uses ReLUs and the adam optimizer, instead of sigmoids and adagrad. These changes make the network converge much faster. ### Usage Install the required dependencies: ```bash pip install -r requirements.txt ``` To run the example, execute: ```bash python main.py ``` If a hardware accelerator device is detected, the example will execute on the accelerator; otherwise, it will run on the CPU. To force execution on the CPU, use `--no-accel` command line argument: ```bash python main.py --no-accel ``` The main.py script accepts the following optional arguments: ```bash --batch-size input batch size for training (default: 128) --epochs number of epochs to train (default: 10) --no-accel disables accelerator --seed random seed (default: 1) --log-interval how many batches to wait before logging training status ``` ================================================ FILE: vae/main.py ================================================ from __future__ import print_function import argparse import torch import torch.utils.data from torch import nn, optim from torch.nn import functional as F from torchvision import datasets, transforms from torchvision.utils import save_image parser = argparse.ArgumentParser(description='VAE MNIST Example') parser.add_argument('--batch-size', type=int, default=128, metavar='N', help='input batch size for training (default: 128)') parser.add_argument('--epochs', type=int, default=10, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--no-accel', action='store_true', help='disables accelerator') parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') parser.add_argument('--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') args = parser.parse_args() use_accel = not args.no_accel and torch.accelerator.is_available() torch.manual_seed(args.seed) if use_accel: device = torch.accelerator.current_accelerator() else: device = torch.device("cpu") print(f"Using device: {device}") kwargs = {'num_workers': 1, 'pin_memory': True} if use_accel else {} train_loader = torch.utils.data.DataLoader( datasets.MNIST('../data', train=True, download=True, transform=transforms.ToTensor()), batch_size=args.batch_size, shuffle=True, **kwargs) test_loader = torch.utils.data.DataLoader( datasets.MNIST('../data', train=False, transform=transforms.ToTensor()), batch_size=args.batch_size, shuffle=False, **kwargs) class VAE(nn.Module): def __init__(self): super(VAE, self).__init__() self.fc1 = nn.Linear(784, 400) self.fc21 = nn.Linear(400, 20) self.fc22 = nn.Linear(400, 20) self.fc3 = nn.Linear(20, 400) self.fc4 = nn.Linear(400, 784) def encode(self, x): h1 = F.relu(self.fc1(x)) return self.fc21(h1), self.fc22(h1) def reparameterize(self, mu, logvar): std = torch.exp(0.5*logvar) eps = torch.randn_like(std) return mu + eps*std def decode(self, z): h3 = F.relu(self.fc3(z)) return torch.sigmoid(self.fc4(h3)) def forward(self, x): mu, logvar = self.encode(x.view(-1, 784)) z = self.reparameterize(mu, logvar) return self.decode(z), mu, logvar model = VAE().to(device) optimizer = optim.Adam(model.parameters(), lr=1e-3) # Reconstruction + KL divergence losses summed over all elements and batch def loss_function(recon_x, x, mu, logvar): BCE = F.binary_cross_entropy(recon_x, x.view(-1, 784), reduction='sum') # see Appendix B from VAE paper: # Kingma and Welling. Auto-Encoding Variational Bayes. ICLR, 2014 # https://arxiv.org/abs/1312.6114 # 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2) KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp()) return BCE + KLD def train(epoch): model.train() train_loss = 0 for batch_idx, (data, _) in enumerate(train_loader): data = data.to(device) optimizer.zero_grad() recon_batch, mu, logvar = model(data) loss = loss_function(recon_batch, data, mu, logvar) loss.backward() train_loss += loss.item() optimizer.step() if batch_idx % args.log_interval == 0: print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( epoch, batch_idx * len(data), len(train_loader.dataset), 100. * batch_idx / len(train_loader), loss.item() / len(data))) print('====> Epoch: {} Average loss: {:.4f}'.format( epoch, train_loss / len(train_loader.dataset))) def test(epoch): model.eval() test_loss = 0 with torch.no_grad(): for i, (data, _) in enumerate(test_loader): data = data.to(device) recon_batch, mu, logvar = model(data) test_loss += loss_function(recon_batch, data, mu, logvar).item() if i == 0: n = min(data.size(0), 8) comparison = torch.cat([data[:n], recon_batch.view(args.batch_size, 1, 28, 28)[:n]]) save_image(comparison.cpu(), 'results/reconstruction_' + str(epoch) + '.png', nrow=n) test_loss /= len(test_loader.dataset) print('====> Test set loss: {:.4f}'.format(test_loss)) if __name__ == "__main__": for epoch in range(1, args.epochs + 1): train(epoch) test(epoch) with torch.no_grad(): sample = torch.randn(64, 20).to(device) sample = model.decode(sample).cpu() save_image(sample.view(64, 1, 28, 28), 'results/sample_' + str(epoch) + '.png') ================================================ FILE: vae/requirements.txt ================================================ torch torchvision tqdm six ================================================ FILE: vae/results/.gitignore ================================================ *.png ================================================ FILE: word_language_model/README.md ================================================ # Word-level Language Modeling using RNN and Transformer This example trains a multi-layer RNN (Elman, GRU, or LSTM) or Transformer on a language modeling task. By default, the training script uses the Wikitext-2 dataset, provided. The trained model can then be used by the generate script to generate new text. ```bash python main.py --accel --epochs 6 # Train a LSTM on Wikitext-2. python main.py --accel --epochs 6 --tied # Train a tied LSTM on Wikitext-2. python main.py --accel --tied # Train a tied LSTM on Wikitext-2for 40 epochs. python main.py --accel --epochs 6 --model Transformer --lr 5 # Train a Transformer model on Wikitext-2. python main.py --accel --epochs 6 --model Transformer --use-optimizer --lr 0.001 # Train a Transformer model with AdamW optimizer on Wikitext-2. python generate.py --accel # Generate samples from the default model checkpoint. ``` > [!NOTE] > Example supports running on acceleration devices (CUDA, MPS, XPU) The model uses the `nn.RNN` module (and its sister modules `nn.GRU` and `nn.LSTM`) or Transformer module (`nn.TransformerEncoder` and `nn.TransformerEncoderLayer`) which will automatically use the cuDNN backend if run on CUDA with cuDNN installed. During training, if a keyboard interrupt (Ctrl-C) is received, training is stopped and the current model is evaluated against the test dataset. The `main.py` script accepts the following arguments: ```bash optional arguments: -h, --help show this help message and exit --data DATA location of the data corpus --model MODEL type of network (RNN_TANH, RNN_RELU, LSTM, GRU, Transformer) --emsize EMSIZE size of word embeddings --nhid NHID number of hidden units per layer --nlayers NLAYERS number of layers --lr LR initial learning rate --clip CLIP gradient clipping --epochs EPOCHS upper epoch limit --batch_size N batch size --bptt BPTT sequence length --dropout DROPOUT dropout applied to layers (0 = no dropout) --tied tie the word embedding and softmax weights --seed SEED random seed --accel use accelerator --log-interval N report interval --save SAVE path to save the final model --onnx-export ONNX_EXPORT path to export the final model in onnx format --nhead NHEAD the number of heads in the encoder/decoder of the transformer model --dry-run verify the code and the model --use-optimizer specify whether to use an AdamW optimizer ``` With these arguments, a variety of models can be tested. As an example, the following arguments produce slower but better models: ```bash python main.py --accel --emsize 650 --nhid 650 --dropout 0.5 --epochs 40 python main.py --accel --emsize 650 --nhid 650 --dropout 0.5 --epochs 40 --tied python main.py --accel --emsize 1500 --nhid 1500 --dropout 0.65 --epochs 40 python main.py --accel --emsize 1500 --nhid 1500 --dropout 0.65 --epochs 40 --tied ``` ================================================ FILE: word_language_model/data.py ================================================ import os from io import open import torch class Dictionary(object): def __init__(self): self.word2idx = {} self.idx2word = [] def add_word(self, word): if word not in self.word2idx: self.idx2word.append(word) self.word2idx[word] = len(self.idx2word) - 1 return self.word2idx[word] def __len__(self): return len(self.idx2word) class Corpus(object): def __init__(self, path): self.dictionary = Dictionary() self.train = self.tokenize(os.path.join(path, 'train.txt')) self.valid = self.tokenize(os.path.join(path, 'valid.txt')) self.test = self.tokenize(os.path.join(path, 'test.txt')) def tokenize(self, path): """Tokenizes a text file.""" assert os.path.exists(path) # Add words to the dictionary with open(path, 'r', encoding="utf8") as f: for line in f: words = line.split() + [''] for word in words: self.dictionary.add_word(word) # Tokenize file content with open(path, 'r', encoding="utf8") as f: idss = [] for line in f: words = line.split() + [''] ids = [] for word in words: ids.append(self.dictionary.word2idx[word]) idss.append(torch.tensor(ids).type(torch.int64)) ids = torch.cat(idss) return ids ================================================ FILE: word_language_model/generate.py ================================================ ############################################################################### # Language Modeling on Wikitext-2 # # This file generates new sentences sampled from the language model. # ############################################################################### import argparse import torch import data from model import PositionalEncoding, RNNModel, TransformerModel parser = argparse.ArgumentParser(description='PyTorch Wikitext-2 Language Model') # Model parameters. parser.add_argument('--data', type=str, default='./data/wikitext-2', help='location of the data corpus') parser.add_argument('--checkpoint', type=str, default='./model.pt', help='model checkpoint to use') parser.add_argument('--outf', type=str, default='generated.txt', help='output file for generated text') parser.add_argument('--words', type=int, default='1000', help='number of words to generate') parser.add_argument('--seed', type=int, default=1111, help='random seed') parser.add_argument('--temperature', type=float, default=1.0, help='temperature - higher will increase diversity') parser.add_argument('--log-interval', type=int, default=100, help='reporting interval') parser.add_argument('--accel', action='store_true', default=False, help='use accelerator') args = parser.parse_args() # Set the random seed manually for reproducibility. torch.manual_seed(args.seed) if args.accel and torch.accelerator.is_available(): device = torch.accelerator.current_accelerator() else: device = torch.device("cpu") if args.temperature < 1e-3: parser.error("--temperature has to be greater or equal 1e-3.") with open(args.checkpoint, 'rb') as f: safe_globals = [ PositionalEncoding, RNNModel, TransformerModel, torch.nn.functional.relu, torch.nn.modules.activation.MultiheadAttention, torch.nn.modules.container.ModuleList, torch.nn.modules.dropout.Dropout, torch.nn.modules.linear.Linear, torch.nn.modules.linear.NonDynamicallyQuantizableLinear, torch.nn.modules.normalization.LayerNorm, torch.nn.modules.sparse.Embedding, torch.nn.modules.rnn.GRU, torch.nn.modules.rnn.LSTM, torch.nn.modules.rnn.RNN, torch.nn.modules.transformer.TransformerEncoder, torch.nn.modules.transformer.TransformerEncoderLayer, ] with torch.serialization.safe_globals(safe_globals): model = torch.load(f, map_location=device) model.eval() corpus = data.Corpus(args.data) ntokens = len(corpus.dictionary) is_transformer_model = hasattr(model, 'model_type') and model.model_type == 'Transformer' if not is_transformer_model: hidden = model.init_hidden(1) input = torch.randint(ntokens, (1, 1), dtype=torch.long).to(device) with open(args.outf, 'w') as outf: with torch.no_grad(): # no tracking history for i in range(args.words): if is_transformer_model: output = model(input, False) word_weights = output[-1].squeeze().div(args.temperature).exp().cpu() word_idx = torch.multinomial(word_weights, 1)[0] word_tensor = torch.Tensor([[word_idx]]).long().to(device) input = torch.cat([input, word_tensor], 0) else: output, hidden = model(input, hidden) word_weights = output.squeeze().div(args.temperature).exp().cpu() word_idx = torch.multinomial(word_weights, 1)[0] input.fill_(word_idx) word = corpus.dictionary.idx2word[word_idx] outf.write(word + ('\n' if i % 20 == 19 else ' ')) if i % args.log_interval == 0: print('| Generated {}/{} words'.format(i, args.words)) ================================================ FILE: word_language_model/main.py ================================================ # coding: utf-8 import argparse import time import math import os import torch import torch.nn as nn import torch.onnx import data from model import PositionalEncoding, RNNModel, TransformerModel parser = argparse.ArgumentParser(description='PyTorch Wikitext-2 RNN/LSTM/GRU/Transformer Language Model') parser.add_argument('--data', type=str, default='./data/wikitext-2', help='location of the data corpus') parser.add_argument('--model', type=str, default='LSTM', help='type of network (RNN_TANH, RNN_RELU, LSTM, GRU, Transformer)') parser.add_argument('--emsize', type=int, default=200, help='size of word embeddings') parser.add_argument('--nhid', type=int, default=200, help='number of hidden units per layer') parser.add_argument('--nlayers', type=int, default=2, help='number of layers') parser.add_argument('--lr', type=float, default=20, help='initial learning rate') parser.add_argument('--clip', type=float, default=0.25, help='gradient clipping') parser.add_argument('--epochs', type=int, default=40, help='upper epoch limit') parser.add_argument('--batch_size', type=int, default=20, metavar='N', help='batch size') parser.add_argument('--bptt', type=int, default=35, help='sequence length') parser.add_argument('--dropout', type=float, default=0.2, help='dropout applied to layers (0 = no dropout)') parser.add_argument('--tied', action='store_true', help='tie the word embedding and softmax weights') parser.add_argument('--seed', type=int, default=1111, help='random seed') parser.add_argument('--log-interval', type=int, default=200, metavar='N', help='report interval') parser.add_argument('--save', type=str, default='model.pt', help='path to save the final model') parser.add_argument('--onnx-export', type=str, default='', help='path to export the final model in onnx format') parser.add_argument('--nhead', type=int, default=2, help='the number of heads in the encoder/decoder of the transformer model') parser.add_argument('--dry-run', action='store_true', help='verify the code and the model') parser.add_argument('--accel', action='store_true', help='Enables accelerated training') parser.add_argument('--use-optimizer', action='store_true', help='Uses AdamW optimizer for gradient updating') args = parser.parse_args() # Set the random seed manually for reproducibility. torch.manual_seed(args.seed) if args.accel and torch.accelerator.is_available(): device = torch.accelerator.current_accelerator() else: device = torch.device("cpu") print("Using device:", device) ############################################################################### # Load data ############################################################################### corpus = data.Corpus(args.data) # Starting from sequential data, batchify arranges the dataset into columns. # For instance, with the alphabet as the sequence and batch size 4, we'd get # ┌ a g m s ┐ # │ b h n t │ # │ c i o u │ # │ d j p v │ # │ e k q w │ # └ f l r x ┘. # These columns are treated as independent by the model, which means that the # dependence of e. g. 'g' on 'f' can not be learned, but allows more efficient # batch processing. def batchify(data, bsz): # Work out how cleanly we can divide the dataset into bsz parts. nbatch = data.size(0) // bsz # Trim off any extra elements that wouldn't cleanly fit (remainders). data = data.narrow(0, 0, nbatch * bsz) # Evenly divide the data across the bsz batches. data = data.view(bsz, -1).t().contiguous() return data.to(device) eval_batch_size = 10 train_data = batchify(corpus.train, args.batch_size) val_data = batchify(corpus.valid, eval_batch_size) test_data = batchify(corpus.test, eval_batch_size) ############################################################################### # Build the model ############################################################################### ntokens = len(corpus.dictionary) if args.model == 'Transformer': model = TransformerModel(ntokens, args.emsize, args.nhead, args.nhid, args.nlayers, args.dropout).to(device) else: model = RNNModel(args.model, ntokens, args.emsize, args.nhid, args.nlayers, args.dropout, args.tied).to(device) criterion = nn.NLLLoss() if args.use_optimizer: optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr) ############################################################################### # Training code ############################################################################### def repackage_hidden(h): """Wraps hidden states in new Tensors, to detach them from their history.""" if isinstance(h, torch.Tensor): return h.detach() else: return tuple(repackage_hidden(v) for v in h) # get_batch subdivides the source data into chunks of length args.bptt. # If source is equal to the example output of the batchify function, with # a bptt-limit of 2, we'd get the following two Variables for i = 0: # ┌ a g m s ┐ ┌ b h n t ┐ # └ b h n t ┘ └ c i o u ┘ # Note that despite the name of the function, the subdivison of data is not # done along the batch dimension (i.e. dimension 1), since that was handled # by the batchify function. The chunks are along dimension 0, corresponding # to the seq_len dimension in the LSTM. def get_batch(source, i): seq_len = min(args.bptt, len(source) - 1 - i) data = source[i:i+seq_len] target = source[i+1:i+1+seq_len].view(-1) return data, target def evaluate(data_source): # Turn on evaluation mode which disables dropout. model.eval() total_loss = 0. ntokens = len(corpus.dictionary) if args.model != 'Transformer': hidden = model.init_hidden(eval_batch_size) with torch.no_grad(): for i in range(0, data_source.size(0) - 1, args.bptt): data, targets = get_batch(data_source, i) if args.model == 'Transformer': output = model(data) output = output.view(-1, ntokens) else: output, hidden = model(data, hidden) hidden = repackage_hidden(hidden) total_loss += len(data) * criterion(output, targets).item() return total_loss / (len(data_source) - 1) def train(): # Turn on training mode which enables dropout. model.train() total_loss = 0. start_time = time.time() ntokens = len(corpus.dictionary) if args.model != 'Transformer': hidden = model.init_hidden(args.batch_size) for batch, i in enumerate(range(0, train_data.size(0) - 1, args.bptt)): data, targets = get_batch(train_data, i) # Starting each batch, we detach the hidden state from how it was previously produced. # If we didn't, the model would try backpropagating all the way to start of the dataset. if args.use_optimizer: optimizer.zero_grad() else: model.zero_grad() if args.model == 'Transformer': output = model(data) output = output.view(-1, ntokens) else: hidden = repackage_hidden(hidden) output, hidden = model(data, hidden) loss = criterion(output, targets) loss.backward() # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs. torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip) if args.use_optimizer: optimizer.step() else: for p in model.parameters(): p.data.add_(p.grad, alpha=-lr) total_loss += loss.item() if batch % args.log_interval == 0 and batch > 0: cur_loss = total_loss / args.log_interval elapsed = time.time() - start_time print('| epoch {:3d} | {:5d}/{:5d} batches | lr {:02.2f} | ms/batch {:5.2f} | ' 'loss {:5.2f} | ppl {:8.2f}'.format( epoch, batch, len(train_data) // args.bptt, lr, elapsed * 1000 / args.log_interval, cur_loss, math.exp(cur_loss))) total_loss = 0 start_time = time.time() if args.dry_run: break def export_onnx(path, batch_size, seq_len): print('The model is also exported in ONNX format at {}.'.format(os.path.realpath(args.onnx_export))) model.eval() dummy_input = torch.LongTensor(seq_len * batch_size).zero_().view(-1, batch_size).to(device) hidden = model.init_hidden(batch_size) torch.onnx.export(model, (dummy_input, hidden), path) # Loop over epochs. lr = args.lr best_val_loss = None # At any point you can hit Ctrl + C to break out of training early. try: for epoch in range(1, args.epochs+1): epoch_start_time = time.time() train() val_loss = evaluate(val_data) print('-' * 89) print('| end of epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} | ' 'valid ppl {:8.2f}'.format(epoch, (time.time() - epoch_start_time), val_loss, math.exp(val_loss))) print('-' * 89) # Save the model if the validation loss is the best we've seen so far. if not best_val_loss or val_loss < best_val_loss: with open(args.save, 'wb') as f: torch.save(model, f) best_val_loss = val_loss else: # Anneal the learning rate if no improvement has been seen in the validation dataset. lr /= 4.0 except KeyboardInterrupt: print('-' * 89) print('Exiting from training early') # Load the best saved model. with open(args.save, 'rb') as f: if args.model == 'Transformer': safe_globals = [ PositionalEncoding, TransformerModel, torch.nn.functional.relu, torch.nn.modules.activation.MultiheadAttention, torch.nn.modules.container.ModuleList, torch.nn.modules.dropout.Dropout, torch.nn.modules.linear.Linear, torch.nn.modules.linear.NonDynamicallyQuantizableLinear, torch.nn.modules.normalization.LayerNorm, torch.nn.modules.sparse.Embedding, torch.nn.modules.transformer.TransformerEncoder, torch.nn.modules.transformer.TransformerEncoderLayer, ] else: safe_globals = [ RNNModel, torch.nn.modules.dropout.Dropout, torch.nn.modules.linear.Linear, torch.nn.modules.rnn.GRU, torch.nn.modules.rnn.LSTM, torch.nn.modules.rnn.RNN, torch.nn.modules.sparse.Embedding, ] with torch.serialization.safe_globals(safe_globals): model = torch.load(f) # after load the rnn params are not a continuous chunk of memory # this makes them a continuous chunk, and will speed up forward pass # Currently, only rnn model supports flatten_parameters function. if args.model in ['RNN_TANH', 'RNN_RELU', 'LSTM', 'GRU']: model.rnn.flatten_parameters() # Run on test data. test_loss = evaluate(test_data) print('=' * 89) print('| End of training | test loss {:5.2f} | test ppl {:8.2f}'.format( test_loss, math.exp(test_loss))) print('=' * 89) if len(args.onnx_export) > 0: # Export the model in ONNX format. export_onnx(args.onnx_export, batch_size=1, seq_len=args.bptt) ================================================ FILE: word_language_model/model.py ================================================ import math import torch import torch.nn as nn import torch.nn.functional as F class RNNModel(nn.Module): """Container module with an encoder, a recurrent module, and a decoder.""" def __init__(self, rnn_type, ntoken, ninp, nhid, nlayers, dropout=0.5, tie_weights=False): super(RNNModel, self).__init__() self.ntoken = ntoken self.drop = nn.Dropout(dropout) self.encoder = nn.Embedding(ntoken, ninp) if rnn_type in ['LSTM', 'GRU']: self.rnn = getattr(nn, rnn_type)(ninp, nhid, nlayers, dropout=dropout) else: try: nonlinearity = {'RNN_TANH': 'tanh', 'RNN_RELU': 'relu'}[rnn_type] except KeyError as e: raise ValueError( """An invalid option for `--model` was supplied, options are ['LSTM', 'GRU', 'RNN_TANH' or 'RNN_RELU']""") from e self.rnn = nn.RNN(ninp, nhid, nlayers, nonlinearity=nonlinearity, dropout=dropout) self.decoder = nn.Linear(nhid, ntoken) # Optionally tie weights as in: # "Using the Output Embedding to Improve Language Models" (Press & Wolf 2016) # https://arxiv.org/abs/1608.05859 # and # "Tying Word Vectors and Word Classifiers: A Loss Framework for Language Modeling" (Inan et al. 2016) # https://arxiv.org/abs/1611.01462 if tie_weights: if nhid != ninp: raise ValueError('When using the tied flag, nhid must be equal to emsize') self.decoder.weight = self.encoder.weight self.init_weights() self.rnn_type = rnn_type self.nhid = nhid self.nlayers = nlayers def init_weights(self): initrange = 0.1 nn.init.uniform_(self.encoder.weight, -initrange, initrange) nn.init.zeros_(self.decoder.bias) nn.init.uniform_(self.decoder.weight, -initrange, initrange) def forward(self, input, hidden): emb = self.drop(self.encoder(input)) output, hidden = self.rnn(emb, hidden) output = self.drop(output) decoded = self.decoder(output) decoded = decoded.view(-1, self.ntoken) return F.log_softmax(decoded, dim=1), hidden def init_hidden(self, bsz): weight = next(self.parameters()) if self.rnn_type == 'LSTM': return (weight.new_zeros(self.nlayers, bsz, self.nhid), weight.new_zeros(self.nlayers, bsz, self.nhid)) else: return weight.new_zeros(self.nlayers, bsz, self.nhid) # Temporarily leave PositionalEncoding module here. Will be moved somewhere else. class PositionalEncoding(nn.Module): r"""Inject some information about the relative or absolute position of the tokens in the sequence. The positional encodings have the same dimension as the embeddings, so that the two can be summed. Here, we use sine and cosine functions of different frequencies. .. math: \text{PosEncoder}(pos, 2i) = sin(pos/10000^(2i/d_model)) \text{PosEncoder}(pos, 2i+1) = cos(pos/10000^(2i/d_model)) \text{where pos is the word position and i is the embed idx) Args: d_model: the embed dim (required). dropout: the dropout value (default=0.1). max_len: the max. length of the incoming sequence (default=5000). Examples: >>> pos_encoder = PositionalEncoding(d_model) """ def __init__(self, d_model, dropout=0.1, max_len=5000): super(PositionalEncoding, self).__init__() self.dropout = nn.Dropout(p=dropout) pe = torch.zeros(max_len, d_model) position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)) pe[:, 0::2] = torch.sin(position * div_term) pe[:, 1::2] = torch.cos(position * div_term) pe = pe.unsqueeze(0).transpose(0, 1) self.register_buffer('pe', pe) def forward(self, x): r"""Inputs of forward function Args: x: the sequence fed to the positional encoder model (required). Shape: x: [sequence length, batch size, embed dim] output: [sequence length, batch size, embed dim] Examples: >>> output = pos_encoder(x) """ x = x + self.pe[:x.size(0), :] return self.dropout(x) class TransformerModel(nn.Transformer): """Container module with an encoder, a recurrent or transformer module, and a decoder.""" def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0.5): super(TransformerModel, self).__init__(d_model=ninp, nhead=nhead, dim_feedforward=nhid, num_encoder_layers=nlayers) self.model_type = 'Transformer' self.src_mask = None self.pos_encoder = PositionalEncoding(ninp, dropout) self.input_emb = nn.Embedding(ntoken, ninp) self.ninp = ninp self.decoder = nn.Linear(ninp, ntoken) self.init_weights() def _generate_square_subsequent_mask(self, sz): return torch.log(torch.tril(torch.ones(sz,sz))) def init_weights(self): initrange = 0.1 nn.init.uniform_(self.input_emb.weight, -initrange, initrange) nn.init.zeros_(self.decoder.bias) nn.init.uniform_(self.decoder.weight, -initrange, initrange) def forward(self, src, has_mask=True): if has_mask: device = src.device if self.src_mask is None or self.src_mask.size(0) != len(src): mask = self._generate_square_subsequent_mask(len(src)).to(device) self.src_mask = mask else: self.src_mask = None src = self.input_emb(src) * math.sqrt(self.ninp) src = self.pos_encoder(src) output = self.encoder(src, mask=self.src_mask) output = self.decoder(output) return F.log_softmax(output, dim=-1) ================================================ FILE: word_language_model/requirements.txt ================================================ torch>=2.6