Showing preview only (1,792K chars total). Download the full file or copy to clipboard to get everything.
Repository: google-deepmind/sonnet
Branch: v2
Commit: e501ca17bf9a
Files: 299
Total size: 32.5 MB
Directory structure:
gitextract_ho0gv3se/
├── .github/
│ └── workflows/
│ └── ci.yml
├── CONTRIBUTING.md
├── LICENSE
├── MANIFEST.in
├── README.md
├── WORKSPACE
├── docs/
│ ├── .gitignore
│ ├── Makefile
│ ├── api.rst
│ ├── conf.py
│ ├── ext/
│ │ ├── BUILD
│ │ ├── link_tf_api.py
│ │ └── link_tf_api_test.py
│ ├── index.rst
│ ├── modules.rst
│ ├── references.bib
│ └── requirements.txt
├── examples/
│ ├── BUILD
│ ├── README.md
│ ├── distributed_cifar10.ipynb
│ ├── functional_mlp_mnist.py
│ ├── little_gan_on_mnist.ipynb
│ ├── mlp_on_mnist.ipynb
│ ├── simple_mnist.py
│ ├── simple_mnist_test.py
│ └── vqvae_example.ipynb
├── readthedocs.yml
├── requirements-test.txt
├── requirements-tf.txt
├── requirements.txt
├── setup.py
├── sonnet/
│ ├── BUILD
│ ├── __init__.py
│ ├── distribute.py
│ ├── functional.py
│ ├── initializers.py
│ ├── mixed_precision.py
│ ├── nets/
│ │ ├── BUILD
│ │ ├── __init__.py
│ │ └── resnet.py
│ ├── optimizers.py
│ ├── pad.py
│ ├── regularizers.py
│ └── src/
│ ├── BUILD
│ ├── __init__.py
│ ├── axis_norm.py
│ ├── axis_norm_test.py
│ ├── base.py
│ ├── base_test.py
│ ├── batch_apply.py
│ ├── batch_apply_test.py
│ ├── batch_norm.py
│ ├── batch_norm_test.py
│ ├── bias.py
│ ├── bias_test.py
│ ├── build.py
│ ├── build_defs.bzl
│ ├── build_test.py
│ ├── conformance/
│ │ ├── BUILD
│ │ ├── __init__.py
│ │ ├── api_test.py
│ │ ├── build_test.py
│ │ ├── checkpoint_test.py
│ │ ├── checkpoints/
│ │ │ ├── BUILD
│ │ │ ├── README.md
│ │ │ ├── base_batch_norm_1x2x2x3/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── base_batch_norm_scale_offset_1x2x2x3/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── batch_norm_1x2x2x3/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── batch_norm_scale_offset_1x2x2x3/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── batch_norm_training_1x2x2x3/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── bias_3x3x3/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── cifar10_convnet_2x3_2x2_1x3x3x2/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── conv1d_3x3_2x2/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── conv1d_lstm_3x3_2x2/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── conv1d_transpose_3x3_2x2/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── conv2d_3x3_2x2/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── conv2d_lstm_3x3_2x2/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── conv2d_transpose_3x3_2x2/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── conv3d_3x3_2x2/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── conv3d_lstm_3x3_2x2/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── conv3d_transpose_3x3_2x2/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── cross_replica_batch_norm_1x2x2x3/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── depthwise_conv2d_3x3_2x2/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── dropout/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── ema_2/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── embed_100_100/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── generate.py
│ │ │ ├── group_norm_2_1x3x4/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── gru_1/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── instance_norm_1_1x3_2/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── layer_norm_1_1x3_2/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── linear_1x1/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── linear_nobias_1x1/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── lstm_1/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── lstm_8_projected_1/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── mean_2x2/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00002
│ │ │ │ ├── checkpoint-1.data-00001-of-00002
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── mlp_3x4x5_1x3/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── mlp_nobias_3x4x5_1x3/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── resnet50/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── sum_2x2/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00002
│ │ │ │ ├── checkpoint-1.data-00001-of-00002
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── trainable_state/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── unrolled_lstm_1/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── vanilla_rnn_8/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── vqvae/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ ├── vqvae_ema_eval/
│ │ │ │ ├── checkpoint
│ │ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ │ └── checkpoint-1.index
│ │ │ └── vqvae_ema_train/
│ │ │ ├── checkpoint
│ │ │ ├── checkpoint-1.data-00000-of-00001
│ │ │ └── checkpoint-1.index
│ │ ├── copy_test.py
│ │ ├── descriptors.py
│ │ ├── descriptors_test.py
│ │ ├── distribute_test.py
│ │ ├── doctest_test.py
│ │ ├── function_test.py
│ │ ├── goldens.py
│ │ ├── goldens_test.py
│ │ ├── keras_test.py
│ │ ├── optimizer_test.py
│ │ ├── pickle_test.py
│ │ ├── saved_model_test.py
│ │ ├── tensorflow1_test.py
│ │ └── xla_test.py
│ ├── conv.py
│ ├── conv_test.py
│ ├── conv_transpose.py
│ ├── conv_transpose_test.py
│ ├── custom_getter.py
│ ├── custom_getter_test.py
│ ├── deferred.py
│ ├── deferred_test.py
│ ├── depthwise_conv.py
│ ├── depthwise_conv_test.py
│ ├── distribute/
│ │ ├── BUILD
│ │ ├── __init__.py
│ │ ├── distributed_batch_norm.py
│ │ ├── distributed_batch_norm_test.py
│ │ ├── replicator.py
│ │ ├── replicator_test.py
│ │ └── replicator_test_utils.py
│ ├── dropout.py
│ ├── dropout_test.py
│ ├── embed.py
│ ├── embed_test.py
│ ├── functional/
│ │ ├── BUILD
│ │ ├── __init__.py
│ │ ├── haiku.py
│ │ ├── haiku_test.py
│ │ ├── jax.py
│ │ ├── jax_test.py
│ │ ├── optimizers.py
│ │ ├── optimizers_test.py
│ │ └── utils.py
│ ├── group_norm.py
│ ├── group_norm_test.py
│ ├── initializers.py
│ ├── initializers_test.py
│ ├── leaky_clip_by_value.py
│ ├── leaky_clip_by_value_test.py
│ ├── linear.py
│ ├── linear_test.py
│ ├── metrics.py
│ ├── metrics_test.py
│ ├── mixed_precision.py
│ ├── mixed_precision_test.py
│ ├── moving_averages.py
│ ├── moving_averages_test.py
│ ├── nets/
│ │ ├── BUILD
│ │ ├── __init__.py
│ │ ├── cifar10_convnet.py
│ │ ├── cifar10_convnet_test.py
│ │ ├── dnc/
│ │ │ ├── BUILD
│ │ │ ├── __init__.py
│ │ │ ├── control.py
│ │ │ ├── control_test.py
│ │ │ ├── read.py
│ │ │ ├── read_test.py
│ │ │ ├── util.py
│ │ │ ├── util_test.py
│ │ │ ├── write.py
│ │ │ └── write_test.py
│ │ ├── mlp.py
│ │ ├── mlp_test.py
│ │ ├── resnet.py
│ │ ├── resnet_test.py
│ │ ├── vqvae.py
│ │ └── vqvae_test.py
│ ├── once.py
│ ├── once_test.py
│ ├── optimizers/
│ │ ├── BUILD
│ │ ├── __init__.py
│ │ ├── adam.py
│ │ ├── adam_test.py
│ │ ├── momentum.py
│ │ ├── momentum_test.py
│ │ ├── optimizer_tests.py
│ │ ├── optimizer_utils.py
│ │ ├── rmsprop.py
│ │ ├── rmsprop_test.py
│ │ ├── sgd.py
│ │ └── sgd_test.py
│ ├── pad.py
│ ├── pad_test.py
│ ├── parallel_linear.py
│ ├── parallel_linear_test.py
│ ├── recurrent.py
│ ├── recurrent_test.py
│ ├── regularizers.py
│ ├── regularizers_test.py
│ ├── reshape.py
│ ├── reshape_test.py
│ ├── scale_gradient.py
│ ├── scale_gradient_test.py
│ ├── sequential.py
│ ├── sequential_test.py
│ ├── test_utils.py
│ ├── types.py
│ ├── utils.py
│ └── utils_test.py
└── test.sh
================================================
FILE CONTENTS
================================================
================================================
FILE: .github/workflows/ci.yml
================================================
name: pytest
on:
pull_request:
branches:
- v2
push:
branches:
- v2
jobs:
test-ubuntu:
name: "pytest on ${{ matrix.python-version }} on ${{ matrix.os }}"
runs-on: "${{ matrix.os }}"
strategy:
matrix:
python-version: [3.9, '3.10', '3.11']
os: [ubuntu-latest]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
pip install -r requirements-tf.txt
pip install -r requirements-test.txt
pip install .
- name: Test with pytest
run: |
pip install pytest pytest-xdist
pytest -n auto sonnet --ignore=sonnet/src/conformance/
================================================
FILE: CONTRIBUTING.md
================================================
# Contributing guidelines
## How to become a contributor and submit your own code
### Contributor License Agreements
We'd love to accept your patches! Before we can take them, we have to jump a
couple of legal hurdles.
Please fill out either the individual or corporate Contributor License Agreement
(CLA).
* If you are an individual writing original source code and you're sure you
own the intellectual property, then you'll need to sign an [individual
CLA](http://code.google.com/legal/individual-cla-v1.0.html).
* If you work for a company that wants to allow you to contribute your work,
then you'll need to sign a [corporate
CLA](http://code.google.com/legal/corporate-cla-v1.0.html).
Follow either of the two links above to access the appropriate CLA and
instructions for how to sign and return it. Once we receive it, we'll be able to
accept your pull requests.
***NOTE***: Only original source code from you and other people that have signed
the CLA can be accepted into the main repository.
### Contributing code
If you have improvements to Sonnet, send us your pull requests! For those just
getting started, Github has a
[howto](https://help.github.com/articles/using-pull-requests/).
If you want to contribute but you're not sure where to start, take a look at the
[issues with the "contributions welcome"
label](https://github.com/deepmind/sonnet/labels/stat%3Acontributions%20welcome).
These are issues that we believe are particularly well suited for outside
contributions, often because we probably won't get to them right now. If you
decide to start on an issue, leave a comment so that other people know that
you're working on it. If you want to help out, but not alone, use the issue
comment thread to coordinate.
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: MANIFEST.in
================================================
include README.md
include LICENSE
================================================
FILE: README.md
================================================
# Sonnet
[**Documentation**](https://sonnet.readthedocs.io/) | [**Examples**](#examples)
Sonnet is a library built on top of [TensorFlow 2](https://www.tensorflow.org/)
designed to provide simple, composable abstractions for machine learning
research.
# Introduction
Sonnet has been designed and built by researchers at DeepMind. It can be used to
construct neural networks for many different purposes (un/supervised learning,
reinforcement learning, ...). We find it is a successful abstraction for our
organization, you might too!
More specifically, Sonnet provides a simple but powerful programming model
centered around a single concept: `snt.Module`. Modules can hold references to
parameters, other modules and methods that apply some function on the user
input. Sonnet ships with many predefined modules (e.g. `snt.Linear`,
`snt.Conv2D`, `snt.BatchNorm`) and some predefined networks of modules (e.g.
`snt.nets.MLP`) but users are also encouraged to build their own modules.
Unlike many frameworks Sonnet is extremely unopinionated about **how** you will
use your modules. Modules are designed to be self contained and entirely
decoupled from one another. Sonnet does not ship with a training framework and
users are encouraged to build their own or adopt those built by others.
Sonnet is also designed to be simple to understand, our code is (hopefully!)
clear and focussed. Where we have picked defaults (e.g. defaults for initial
parameter values) we try to point out why.
# Getting Started
## Examples
The easiest way to try Sonnet is to use Google Colab which offers a free Python
notebook attached to a GPU or TPU.
- [Predicting MNIST with an MLP](https://colab.research.google.com/github/deepmind/sonnet/blob/v2/examples/mlp_on_mnist.ipynb)
- [Training a Little GAN on MNIST](https://colab.research.google.com/github/deepmind/sonnet/blob/v2/examples/little_gan_on_mnist.ipynb)
- [Distributed training with `snt.distribute`](https://colab.research.google.com/github/deepmind/sonnet/blob/v2/examples/distributed_cifar10.ipynb)
## Installation
To get started install TensorFlow 2.0 and Sonnet 2:
```shell
$ pip install tensorflow tensorflow-probability
$ pip install dm-sonnet
```
You can run the following to verify things installed correctly:
```python
import tensorflow as tf
import sonnet as snt
print("TensorFlow version {}".format(tf.__version__))
print("Sonnet version {}".format(snt.__version__))
```
### Using existing modules
Sonnet ships with a number of built in modules that you can trivially use. For
example to define an MLP we can use the `snt.Sequential` module to call a
sequence of modules, passing the output of a given module as the input for the
next module. We can use `snt.Linear` and `tf.nn.relu` to actually define our
computation:
```python
mlp = snt.Sequential([
snt.Linear(1024),
tf.nn.relu,
snt.Linear(10),
])
```
To use our module we need to "call" it. The `Sequential` module (and most
modules) define a `__call__` method that means you can call them by name:
```python
logits = mlp(tf.random.normal([batch_size, input_size]))
```
It is also very common to request all the parameters for your module. Most
modules in Sonnet create their parameters the first time they are called with
some input (since in most cases the shape of the parameters is a function of
the input). Sonnet modules provide two properties for accessing parameters.
The `variables` property returns **all** `tf.Variable`s that are referenced by
the given module:
```python
all_variables = mlp.variables
```
It is worth noting that `tf.Variable`s are not just used for parameters of your
model. For example they are used to hold state in metrics used in
`snt.BatchNorm`. In most cases users retrieve the module variables to pass them
to an optimizer to be updated. In this case non-trainable variables should
typically not be in that list as they are updated via a different mechanism.
TensorFlow has a built in mechanism to mark variables as "trainable" (parameters
of your model) vs. non-trainable (other variables). Sonnet provides a mechanism
to gather all trainable variables from your module which is probably what you
want to pass to an optimizer:
```python
model_parameters = mlp.trainable_variables
```
### Building your own module
Sonnet strongly encourages users to subclass `snt.Module` to define their own
modules. Let's start by creating a simple `Linear` layer called `MyLinear`:
```python
class MyLinear(snt.Module):
def __init__(self, output_size, name=None):
super(MyLinear, self).__init__(name=name)
self.output_size = output_size
@snt.once
def _initialize(self, x):
initial_w = tf.random.normal([x.shape[1], self.output_size])
self.w = tf.Variable(initial_w, name="w")
self.b = tf.Variable(tf.zeros([self.output_size]), name="b")
def __call__(self, x):
self._initialize(x)
return tf.matmul(x, self.w) + self.b
```
Using this module is trivial:
```python
mod = MyLinear(32)
mod(tf.ones([batch_size, input_size]))
```
By subclassing `snt.Module` you get many nice properties for free. For example
a default implementation of `__repr__` which shows constructor arguments (very
useful for debugging and introspection):
```python
>>> print(repr(mod))
MyLinear(output_size=10)
```
You also get the `variables` and `trainable_variables` properties:
```python
>>> mod.variables
(<tf.Variable 'my_linear/b:0' shape=(10,) ...)>,
<tf.Variable 'my_linear/w:0' shape=(1, 10) ...)>)
```
You may notice the `my_linear` prefix on the variables above. This is because
Sonnet modules also enter the modules name scope whenever methods are called.
By entering the module name scope we provide a much more useful graph for tools
like TensorBoard to consume (e.g. all operations that occur inside my_linear
will be in a group called my_linear).
Additionally your module will now support TensorFlow checkpointing and saved
model which are advanced features covered later.
# Serialization
Sonnet supports multiple serialization formats. The simplest format we support
is Python's `pickle`, and all built in modules are tested to make sure they can
be saved/loaded via pickle in the same Python process. In general we discourage
the use of pickle, it is not well supported by many parts of TensorFlow and in
our experience can be quite brittle.
## TensorFlow Checkpointing
**Reference:** https://www.tensorflow.org/alpha/guide/checkpoints
TensorFlow checkpointing can be used to save the value of parameters
periodically during training. This can be useful to save the progress of
training in case your program crashes or is stopped. Sonnet is designed to work
cleanly with TensorFlow checkpointing:
```python
checkpoint_root = "/tmp/checkpoints"
checkpoint_name = "example"
save_prefix = os.path.join(checkpoint_root, checkpoint_name)
my_module = create_my_sonnet_module() # Can be anything extending snt.Module.
# A `Checkpoint` object manages checkpointing of the TensorFlow state associated
# with the objects passed to it's constructor. Note that Checkpoint supports
# restore on create, meaning that the variables of `my_module` do **not** need
# to be created before you restore from a checkpoint (their value will be
# restored when they are created).
checkpoint = tf.train.Checkpoint(module=my_module)
# Most training scripts will want to restore from a checkpoint if one exists. This
# would be the case if you interrupted your training (e.g. to use your GPU for
# something else, or in a cloud environment if your instance is preempted).
latest = tf.train.latest_checkpoint(checkpoint_root)
if latest is not None:
checkpoint.restore(latest)
for step_num in range(num_steps):
train(my_module)
# During training we will occasionally save the values of weights. Note that
# this is a blocking call and can be slow (typically we are writing to the
# slowest storage on the machine). If you have a more reliable setup it might be
# appropriate to save less frequently.
if step_num and not step_num % 1000:
checkpoint.save(save_prefix)
# Make sure to save your final values!!
checkpoint.save(save_prefix)
```
## TensorFlow Saved Model
**Reference:** https://www.tensorflow.org/alpha/guide/saved_model
TensorFlow saved models can be used to save a copy of your network that is
decoupled from the Python source for it. This is enabled by saving a TensorFlow
graph describing the computation and a checkpoint containing the value of
weights.
The first thing to do in order to create a saved model is to create a
`snt.Module` that you want to save:
```python
my_module = snt.nets.MLP([1024, 1024, 10])
my_module(tf.ones([1, input_size]))
```
Next, we need to create another module describing the specific parts of our
model that we want to export. We advise doing this (rather than modifying the
original model in-place) so you have fine grained control over what is actually
exported. This is typically important to avoid creating very large saved models,
and such that you only share the parts of your model you want to (e.g. you only
want to share the generator for a GAN but keep the discriminator private).
```python
@tf.function(input_signature=[tf.TensorSpec([None, input_size])])
def inference(x):
return my_module(x)
to_save = snt.Module()
to_save.inference = inference
to_save.all_variables = list(my_module.variables)
tf.saved_model.save(to_save, "/tmp/example_saved_model")
```
We now have a saved model in the `/tmp/example_saved_model` folder:
```shell
$ ls -lh /tmp/example_saved_model
total 24K
drwxrwsr-t 2 tomhennigan 154432098 4.0K Apr 28 00:14 assets
-rw-rw-r-- 1 tomhennigan 154432098 14K Apr 28 00:15 saved_model.pb
drwxrwsr-t 2 tomhennigan 154432098 4.0K Apr 28 00:15 variables
```
Loading this model is simple and can be done on a different machine without any
of the Python code that built the saved model:
```python
loaded = tf.saved_model.load("/tmp/example_saved_model")
# Use the inference method. Note this doesn't run the Python code from `to_save`
# but instead uses the TensorFlow Graph that is part of the saved model.
loaded.inference(tf.ones([1, input_size]))
# The all_variables property can be used to retrieve the restored variables.
assert len(loaded.all_variables) > 0
```
Note that the loaded object is not a Sonnet module, it is a container object
that has the specific methods (e.g. `inference`) and properties (e.g.
`all_variables`) that we added in the previous block.
## Distributed training
**Example:** https://github.com/deepmind/sonnet/blob/v2/examples/distributed_cifar10.ipynb
Sonnet has support for distributed training using
[custom TensorFlow distribution strategies](https://sonnet.readthedocs.io/en/latest/api.html#module-sonnet.distribute).
A key difference between Sonnet and distributed training using `tf.keras` is
that Sonnet modules and optimizers do not behave differently when run under
distribution strategies (e.g. we do not average your gradients or sync your
batch norm stats). We believe that users should be in full control of these
aspects of their training and they should not be baked into the library. The
trade off here is that you need to implement these features in your training
script (typically this is just 2 lines of code to all reduce your gradients
before applying your optimizer) or swap in modules that are explicitly
distribution aware (e.g. `snt.distribute.CrossReplicaBatchNorm`).
Our [distributed Cifar-10](https://github.com/deepmind/sonnet/blob/v2/examples/distributed_cifar10.ipynb)
example walks through doing multi-GPU training with Sonnet.
================================================
FILE: WORKSPACE
================================================
workspace(name = "sonnet")
================================================
FILE: docs/.gitignore
================================================
_build
================================================
FILE: docs/Makefile
================================================
# Minimal makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
SOURCEDIR = .
BUILDDIR = _build
# Put it first so that "make" without argument is like "make help".
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
================================================
FILE: docs/api.rst
================================================
.. TODO(slebedev): add a title, e.g. "API docs"?
Base
----
.. currentmodule:: sonnet
Module
~~~~~~
.. autoclass:: Module
:members:
once
~~~~
.. autofunction:: once
no_name_scope
~~~~~~~~~~~~~
.. autofunction:: no_name_scope
Deferred
~~~~~~~~
.. autoclass:: Deferred
:members:
:exclude-members: __getattr__, __setattr__
Linear modules
--------------
Linear
~~~~~~
.. autoclass:: Linear
:members:
Bias
~~~~
.. autoclass:: Bias
:members:
Convolutional modules
---------------------
.. currentmodule:: sonnet
Conv1D
~~~~~~
.. autoclass:: Conv1D
:members:
Conv2D
~~~~~~
.. autoclass:: Conv2D
:members:
Conv3D
~~~~~~
.. autoclass:: Conv3D
:members:
Conv1DTranspose
~~~~~~~~~~~~~~~
.. autoclass:: Conv1DTranspose
:members:
Conv2DTranspose
~~~~~~~~~~~~~~~
.. autoclass:: Conv2DTranspose
:members:
Conv3DTranspose
~~~~~~~~~~~~~~~
.. autoclass:: Conv3DTranspose
:members:
DepthwiseConv2D
~~~~~~~~~~~~~~~
.. autoclass:: DepthwiseConv2D
:members:
Normalization modules
---------------------
.. currentmodule:: sonnet
LayerNorm
~~~~~~~~~
.. autoclass:: LayerNorm
:members:
InstanceNorm
~~~~~~~~~~~~
.. autoclass:: InstanceNorm
:members:
BaseBatchNorm
~~~~~~~~~~~~~
.. autoclass:: BaseBatchNorm
:members:
BatchNorm
~~~~~~~~~
.. autoclass:: BatchNorm
:members:
CrossReplicaBatchNorm
~~~~~~~~~~~~~~~~~~~~~
.. currentmodule:: sonnet.distribute
.. autoclass:: CrossReplicaBatchNorm
:members:
GroupNorm
~~~~~~~~~
.. currentmodule:: sonnet
.. autoclass:: GroupNorm
:members:
Recurrent modules
-----------------
.. currentmodule:: sonnet
RNNCore
~~~~~~~
.. autoclass:: RNNCore
:members:
UnrolledRNN
~~~~~~~~~~~
.. autoclass:: UnrolledRNN
:members:
TrainableState
~~~~~~~~~~~~~~
.. autoclass:: TrainableState
:members:
dynamic_unroll
~~~~~~~~~~~~~~
.. autofunction:: dynamic_unroll
static_unroll
~~~~~~~~~~~~~
.. autofunction:: static_unroll
VanillaRNN
~~~~~~~~~~
.. autoclass:: VanillaRNN
:members:
:special-members:
DeepRNN
~~~~~~~
.. autoclass:: DeepRNN
:members:
.. autofunction:: deep_rnn_with_skip_connections
.. autofunction:: deep_rnn_with_residual_connections
LSTM
~~~~
.. autoclass:: LSTM
:members:
.. autoclass:: LSTMState
lstm_with_recurrent_dropout
~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. autofunction:: lstm_with_recurrent_dropout
UnrolledLSTM
~~~~~~~~~~~~
.. autoclass:: UnrolledLSTM
:members:
Conv1DLSTM
~~~~~~~~~~
.. autoclass:: Conv1DLSTM
:members:
Conv2DLSTM
~~~~~~~~~~
.. autoclass:: Conv2DLSTM
:members:
Conv3DLSTM
~~~~~~~~~~
.. autoclass:: Conv3DLSTM
:members:
GRU
~~~
.. autoclass:: GRU
:members:
Batch
-----
.. currentmodule:: sonnet
reshape
~~~~~~~
.. autofunction:: reshape
Reshape
~~~~~~~
.. autoclass:: Reshape
:members:
flatten
~~~~~~~
.. autofunction:: flatten
Flatten
~~~~~~~
.. autoclass:: Flatten
:members:
BatchApply
~~~~~~~~~~
.. autoclass:: BatchApply
:members:
Embedding modules
-----------------
.. currentmodule:: sonnet
Embed
~~~~~
.. autoclass:: Embed
:members:
Optimizers
----------
.. automodule:: sonnet.optimizers
Adam
~~~~
.. autoclass:: Adam
:members:
Momentum
~~~~~~~~
.. autoclass:: Momentum
:members:
RMSProp
~~~~~~~
.. autoclass:: RMSProp
:members:
SGD
~~~
.. autoclass:: SGD
:members:
Initializers
------------
.. automodule:: sonnet.initializers
Initializer
~~~~~~~~~~~
.. autoclass:: Initializer
:members:
Constant
~~~~~~~~
.. autoclass:: Constant
:members:
Identity
~~~~~~~~
.. autoclass:: Identity
:members:
Ones
~~~~
.. autoclass:: Ones
:members:
Orthogonal
~~~~~~~~~~
.. autoclass:: Orthogonal
:members:
RandomNormal
~~~~~~~~~~~~
.. autoclass:: RandomNormal
:members:
RandomUniform
~~~~~~~~~~~~~
.. autoclass:: RandomUniform
:members:
TruncatedNormal
~~~~~~~~~~~~~~~
.. autoclass:: TruncatedNormal
:members:
VarianceScaling
~~~~~~~~~~~~~~~
.. autoclass:: VarianceScaling
:members:
Zeros
~~~~~
.. autoclass:: Zeros
:members:
Regularizers
------------
.. automodule:: sonnet.regularizers
Regularizer
~~~~~~~~~~~
.. autoclass:: Regularizer
:members:
L1
~~
.. autoclass:: L1
:members:
L2
~~
.. autoclass:: L2
:members:
OffDiagonalOrthogonal
~~~~~~~~~~~~~~~~~~~~~
.. autoclass:: OffDiagonalOrthogonal
:members:
Paddings
--------
.. automodule:: sonnet.pad
causal
~~~~~~
.. autofunction:: causal
create
~~~~~~
.. autofunction:: create
full
~~~~
.. autofunction:: full
reverse_causal
~~~~~~~~~~~~~~
.. autofunction:: reverse_causal
same
~~~~
.. autofunction:: same
valid
~~~~~
.. autofunction:: valid
.. TODO(petebu): better title?
Distribution
------------
.. automodule:: sonnet.distribute
Replicator
~~~~~~~~~~
.. autoclass:: Replicator
:members:
TpuReplicator
~~~~~~~~~~~~~
.. autoclass:: TpuReplicator
:members:
Metrics
-------
.. currentmodule:: sonnet
Metric
~~~~~~
.. autoclass:: Metric
:members:
Mean
~~~~~~
.. autoclass:: Mean
:members:
Sum
~~~~~~
.. autoclass:: Sum
:members:
.. TODO(tomhennigan): rename to something more appropriate.
Nets
----
.. automodule:: sonnet.nets
MLP
~~~
.. autoclass:: MLP
:members:
Cifar10ConvNet
~~~~~~~~~~~~~~
.. autoclass:: Cifar10ConvNet
:members:
ResNet
~~~~~~~~~~~~~~
.. autoclass:: ResNet
:members:
ResNet50
~~~~~~~~~~~~~~
.. autoclass:: ResNet50
:members:
VectorQuantizer
~~~~~~~~~~~~~~~
.. autoclass:: VectorQuantizer
:members:
VectorQuantizerEMA
~~~~~~~~~~~~~~~~~~
.. autoclass:: VectorQuantizerEMA
:members:
Mixed Precision
---------------
.. automodule:: sonnet.mixed_precision
modes
~~~~~
.. autofunction:: modes
enable
~~~~~~
.. autofunction:: enable
disable
~~~~~~~
.. autofunction:: disable
scope
~~~~~
.. autofunction:: scope
References
----------
.. bibliography:: references.bib
:style: unsrt
================================================
FILE: docs/conf.py
================================================
# Copyright 2019 The Sonnet Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Configuration file for the Sphinx documentation builder."""
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# http://www.sphinx-doc.org/en/master/config
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
# pylint: disable=g-bad-import-order
# pylint: disable=g-import-not-at-top
import doctest
import inspect
import os
import sys
sys.path.insert(0, os.path.abspath('../'))
sys.path.append(os.path.abspath('ext'))
import sonnet as snt
import sphinxcontrib.katex as katex
# -- Project information -----------------------------------------------------
project = 'Sonnet'
copyright = '2019, DeepMind' # pylint: disable=redefined-builtin
author = 'Sonnet Contributors'
# -- General configuration ---------------------------------------------------
master_doc = 'index'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'link_tf_api',
'sphinx.ext.autodoc',
'sphinx.ext.autosummary',
'sphinx.ext.doctest',
'sphinx.ext.inheritance_diagram',
'sphinx.ext.linkcode',
'sphinx.ext.napoleon',
'sphinxcontrib.bibtex',
'sphinxcontrib.katex',
'sphinx_autodoc_typehints',
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
# -- Options for autodoc -----------------------------------------------------
autodoc_default_options = {
'member-order': 'bysource',
'special-members': True,
'exclude-members': '__repr__, __str__, __weakref__',
}
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'sphinx_rtd_theme'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
html_favicon = '_static/favicon.ico'
# -- Options for doctest -----------------------------------------------------
doctest_test_doctest_blocks = 'true'
doctest_global_setup = """
import tensorflow as tf
import sonnet as snt
import tree
# `TpuReplicator` cannot be constructed without a TPU, however it has exactly
# the same API as `Replicator` so we can run doctests using that instead.
snt.distribute.TpuReplicator = snt.distribute.Replicator
"""
doctest_default_flags = (
doctest.ELLIPSIS
| doctest.IGNORE_EXCEPTION_DETAIL
| doctest.DONT_ACCEPT_TRUE_FOR_1
| doctest.NORMALIZE_WHITESPACE)
# -- Options for katex ------------------------------------------------------
# See: https://sphinxcontrib-katex.readthedocs.io/en/0.4.1/macros.html
latex_macros = r"""
\def \d #1{\operatorname{#1}}
"""
# Translate LaTeX macros to KaTeX and add to options for HTML builder
katex_macros = katex.latex_defs_to_katex_macros(latex_macros)
katex_options = 'macros: {' + katex_macros + '}'
# Add LaTeX macros for LATEX builder
latex_elements = {'preamble': latex_macros}
# -- Source code links -------------------------------------------------------
def linkcode_resolve(domain, info):
"""Resolve a GitHub URL corresponding to Python object."""
if domain != 'py':
return None
try:
mod = sys.modules[info['module']]
except ImportError:
return None
obj = mod
try:
for attr in info['fullname'].split('.'):
obj = getattr(obj, attr)
except AttributeError:
return None
else:
obj = inspect.unwrap(obj)
try:
filename = inspect.getsourcefile(obj)
except TypeError:
return None
try:
source, lineno = inspect.getsourcelines(obj)
except OSError:
return None
# TODO(slebedev): support tags after we release an initial version.
return 'https://github.com/deepmind/sonnet/blob/v2/sonnet/%s#L%d#L%d' % (
os.path.relpath(filename, start=os.path.dirname(
snt.__file__)), lineno, lineno + len(source) - 1)
================================================
FILE: docs/ext/BUILD
================================================
load("//sonnet/src:build_defs.bzl", "snt_py_library", "snt_py_test")
licenses(["notice"])
snt_py_library(
name = "link_tf_api",
srcs = ["link_tf_api.py"],
deps = [
# pip: docutils
# pip: tensorflow
],
)
snt_py_test(
name = "link_tf_api_test",
srcs = ["link_tf_api_test.py"],
gpu = False,
tpu = False,
deps = [
":link_tf_api",
# pip: absl/testing:absltest
],
)
================================================
FILE: docs/ext/link_tf_api.py
================================================
# Copyright 2019 The Sonnet Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Reference TensorFlow API symbols.
This extension allows to reference TensorFlow API symbols using the
``:tf:`` role. For example, the following::
Sonnet :py:`~base.Module` is based on :tf:`Module`.
generates a link to ``tf.Module``.
"""
import functools
from typing import Any, List, Tuple
from urllib import parse as urlparse
from docutils import nodes
from docutils.parsers.rst import states
import tensorflow as tf
from tensorflow.python.util import tf_export # pylint: disable=g-direct-tensorflow-import
__version__ = "0.1"
# TODO(slebedev): make the version configurable or infer from ``tf``?
TF_VERSION = "2.0"
TF_API_BASE_URL = (
"https://www.tensorflow.org/versions/r%s/api_docs/python/tf/" % TF_VERSION)
def tf_role_fn(
typ: str,
rawtext: str,
text: str,
lineno: int,
inliner: states.Inliner,
options: Any = None,
content: Any = None) -> Tuple[List[nodes.Node], List[nodes.system_message]]:
"""Generates a reference to a given TensorFlow API symbol.
Only exported API symbols can be referenced. For example, non-exported
:tf:`float32` will not produce a reference and will be rendered as
plain-text.
Args:
typ: Type of the role. Fixed to ``"tf"``.
rawtext: Raw contents of the role, e.g. ``":tf:`Module``"`.
text: The `contents` of the role e.g. ``"Module"``.
lineno: Line number of the parsed role.
inliner: Inline reST markup parser. Used for error reporting.
options: Unused.
content: Unused.
Returns:
Generated reST nodes and system messages.
"""
del options, content # Unused.
canonical_url = tf_doc_url(text)
xref = nodes.literal(rawtext, typ + "." + text, classes=["xref"])
if not canonical_url:
warning = (
"unable to expand :%s:`%s`; symbol is not exported by TensorFlow." %
(typ, text))
inliner.reporter.warning(warning, line=lineno)
return [xref], []
else:
node = nodes.reference(
rawtext, "", xref, internal=False, refuri=canonical_url)
return [node], []
def tf_doc_url(text):
"""Retrieves the TensorFlow doc URL for the given symbol.
Args:
text: A string for a symbol inside TF (e.g. ``"optimizers.Adam"``).
Returns:
A string URL linking to the TensorFlow doc site or ``None`` if a URL could
not be resolved.
"""
get_tf_name = functools.partial(
tf_export.get_canonical_name_for_symbol, add_prefix_to_v1_names=True)
try:
prev_symbol = None
symbol = tf
for chunk in text.split("."):
prev_symbol = symbol
symbol = getattr(prev_symbol, chunk)
except AttributeError:
return None
canonical_name = get_tf_name(symbol)
# Check if we're looking at a method reference (e.g. "TensorArray.read").
if prev_symbol and not canonical_name:
prev_canonical_name = get_tf_name(prev_symbol)
if prev_canonical_name:
canonical_name = prev_canonical_name + "#" + text.split(".")[-1]
if not canonical_name:
return None
return urlparse.urljoin(TF_API_BASE_URL, canonical_name.replace(".", "/"))
def setup(app):
app.add_role("tf", tf_role_fn)
return {
"version": __version__,
"parallel_read_safe": True,
"parallel_write_safe": True,
}
================================================
FILE: docs/ext/link_tf_api_test.py
================================================
# Copyright 2019 The Sonnet Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Tests for ``:tf:`` Sphinx role."""
from absl.testing import absltest
from docs.ext import link_tf_api
DOC_BASE_URL = "https://www.tensorflow.org/versions/r2.0/api_docs/python/tf"
class LinkTfApiTest(absltest.TestCase):
def test_non_existent(self):
self.assertIsNone(link_tf_api.tf_doc_url("tomhennigan"))
self.assertIsNone(link_tf_api.tf_doc_url("autograph.1"))
def test_link_to_top_level(self):
self.assertEqual(
link_tf_api.tf_doc_url("function"), DOC_BASE_URL + "/function")
self.assertEqual(link_tf_api.tf_doc_url("Module"), DOC_BASE_URL + "/Module")
def test_link_to_nested_package(self):
self.assertEqual(
link_tf_api.tf_doc_url("autograph.to_code"),
DOC_BASE_URL + "/autograph/to_code")
def test_link_to_method_of_exported_class(self):
self.assertEqual(
link_tf_api.tf_doc_url("TensorArray.read"),
DOC_BASE_URL + "/TensorArray#read")
def test_link_to_non_existent_method_of_exported_class(self):
self.assertIsNone(link_tf_api.tf_doc_url("TensorArray.tomhennigan"))
if __name__ == "__main__":
absltest.main()
================================================
FILE: docs/index.rst
================================================
:github_url: https://github.com/deepmind/sonnet/tree/v2/docs
Sonnet Documentation
====================
Sonnet is a library built on top of TensorFlow designed to provide simple,
composable abstractions for machine learning research.
.. code-block:: python
import sonnet as snt
import tensorflow as tf
mlp = snt.nets.MLP([1024, 1024, 10])
logits = mlp(tf.ones([8, 28 * 28]))
Installation
------------
Install Sonnet by running::
$ pip install tensorflow
$ pip install dm-sonnet
.. toctree::
:caption: Guides
:maxdepth: 1
modules
.. toctree::
:caption: Package Reference
:maxdepth: 1
api
Contribute
----------
- Issue tracker: https://github.com/deepmind/sonnet/issues
- Source code: https://github.com/deepmind/sonnet/tree/v2
Support
-------
If you are having issues, please let us know by filing an issue on our
`issue tracker <https://github.com/deepmind/sonnet/issues>`_.
License
-------
Sonnet is licensed under the Apache 2.0 License.
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
================================================
FILE: docs/modules.rst
================================================
.. currentmodule:: sonnet
Modules
=======
:class:`Module` is the core abstraction provided by Sonnet.
By organising your code into :class:`Module` subclasses, it is easy to keep
track of variables and deal with common tasks such as locating model parameters
and checkpointing state. Module also helps with debugging, adding a
:tf:`name_scope` around each method, making tools like TensorBoard even more
useful.
Sonnet ships with many predefined modules (e.g. :class:`Linear`,
:class:`Conv2D`, :class:`BatchNorm`) and some predefined networks of modules
(e.g. :class:`nets.MLP`). If you can't find what you're looking for then we
encourage you to subclass :class:`Module` and implement your ideas.
Using built-in modules
----------------------
Using :doc:`built in modules <api>` is as easy as using any other Python object:
>>> linear = snt.Linear(output_size=10)
>>> linear(tf.ones([8, 28 * 28]))
<tf.Tensor: shape=(8, 10), dtype=float32, ... dtype=float32)>
You can get access to the modules parameters using the ``trainable_variables``
property, note that most modules only create parameters the first time they
are called with an input:
>>> linear.trainable_variables
(<tf.Variable 'linear/b:0' shape=(10,) ...>,
<tf.Variable 'linear/w:0' shape=(784, 10) ...>)
Some modules contain references to other modules, Sonnet provides a convenient
way to find these referenced modules:
>>> mlp = snt.nets.MLP([1000, 10])
>>> mlp(tf.ones([1, 1]))
<tf.Tensor: ...>
>>> [s.name for s in mlp.submodules]
['linear_0', 'linear_1']
Writing your own modules
------------------------
To create your own module simply subclass :class:`Module` and implement
your logic. For example we can build our own simple multi-layer perceptron
module by reusing the built in :class:`Linear` modules and :tf:`nn.relu`
to add a non-linearity:
>>> class MyMLP(snt.Module):
... def __init__(self, name=None):
... super(MyMLP, self).__init__(name=name)
... self.hidden1 = snt.Linear(1024, name="hidden1")
... self.output = snt.Linear(10, name="output")
...
... def __call__(self, x):
... x = self.hidden1(x)
... x = tf.nn.relu(x)
... x = self.output(x)
... return x
You can use your module like you would any other Python object:
>>> mlp = MyMLP()
>>> mlp(tf.random.normal([8, 28 * 28]))
<tf.Tensor: shape=(8, 10), ...>
Additionally, the variable and submodule tracking features of :class:`Module`
will work without any additional code:
>>> mlp.trainable_variables
(<tf.Variable 'my_mlp/hidden1/b:0' shape=(1024,) ...>,
<tf.Variable 'my_mlp/hidden1/w:0' shape=(784, 1024) ...>,
<tf.Variable 'my_mlp/output/b:0' shape=(10,) ...>,
<tf.Variable 'my_mlp/output/w:0' shape=(1024, 10) ...>)
>>> mlp.submodules
(Linear(output_size=1024, name='hidden1'),
Linear(output_size=10, name='output'))
It is often useful to defer some one-time initialization until your module is
first used. For example in a linear layer the shape of the weights matrix
depends on the input shape and the desired output shape.
Sonnet provides the :func:`once` dectorator that means a given method is
evaluated once and only once per instance, regardless of other arguments. For
example we can build a simple linear layer like so:
.. code-block:: python
:emphasize-lines: 6-10,13
class MyLinear(snt.Module):
def __init__(self, output_size):
super(MyLinear, self).__init__()
self.output_size = output_size
@snt.once
def _initialize(self, inputs):
input_size = inputs.shape[1]
self.w = tf.Variable(tf.random.normal([input_size, self.output_size]))
self.b = tf.Variable(tf.zeros([self.output_size]))
def __call__(self, inputs):
self._initialize(inputs)
return tf.matmul(inputs, self.w) + self.b
================================================
FILE: docs/references.bib
================================================
@article{chung2014empirical,
title={Empirical evaluation of gated recurrent neural networks on sequence modeling},
author={Chung, Junyoung and Gulcehre, Caglar and Cho, KyungHyun and Bengio, Yoshua},
journal={arXiv preprint arXiv:1412.3555},
year={2014},
url={https://arxiv.org/abs/1412.3555}
}
@article{zaremba2014recurrent,
title={Recurrent neural network regularization},
author={Zaremba, Wojciech and Sutskever, Ilya and Vinyals, Oriol},
journal={arXiv preprint arXiv:1409.2329},
year={2014},
url={https://arxiv.org/abs/1409.2329}
}
@inproceedings{jozefowicz2015empirical,
title={An empirical exploration of recurrent network architectures},
author={Jozefowicz, Rafal and Zaremba, Wojciech and Sutskever, Ilya},
booktitle={International Conference on Machine Learning},
pages={2342--2350},
year={2015}
}
@article{sak2014long,
title={Long short-term memory based recurrent neural network architectures for large vocabulary speech recognition},
author={Sak, Ha{\c{s}}im and Senior, Andrew and Beaufays, Fran{\c{c}}oise},
journal={arXiv preprint arXiv:1402.1128},
year={2014},
url={https://arxiv.org/abs/1402.1128}
}
@inproceedings{gal2016theoretically,
title={A theoretically grounded application of dropout in recurrent neural networks},
author={Gal, Yarin and Ghahramani, Zoubin},
booktitle={Advances in neural information processing systems},
pages={1019--1027},
year={2016}
}
@inproceedings{xingjian2015convolutional,
title={Convolutional LSTM network: A machine learning approach for precipitation nowcasting},
author={Xingjian, SHI and Chen, Zhourong and Wang, Hao and Yeung, Dit-Yan and Wong, Wai-Kin and Woo, Wang-chun},
booktitle={Advances in neural information processing systems},
pages={802--810},
year={2015}
}
@article{buchlovsky2019tf,
title={{TF}-{R}eplicator: {D}istributed Machine Learning for Researchers},
author={Buchlovsky, Peter and Budden, David and Grewe, Dominik and Jones, Chris and Aslanides, John and Besse, Frederic and Brock, Andy and Clark, Aidan and Colmenarejo, Sergio G{\'o}mez and Pope, Aedan and others},
journal={arXiv preprint arXiv:1902.00465},
year={2019},
url={https://arxiv.org/abs/1902.00465}
}
@article{buchlovsky2019distribution,
author={Buchlovsky, Peter and Grewe, Dominik and Gupta, Priya and Hennigan, Tom and Hseu, Jonathan and Jones, Chris and Levenberg, Josh},
title={Distribution {S}trategy - {R}evised {API}},
journal={TensorFlow Community RFCs, Google / DeepMind},
year={2018},
url={https://github.com/tensorflow/community/pull/25}
}
@article{agarwal2019stateful,
author={Agarwal, Ashish and Berthelot, David and Hennigan, Tom and Passos, Alex and Reynolds, Malcolm},
title={Stateful Containers with tf.{M}odule},
journal={TensorFlow Community RFCs, Google / DeepMind},
year={2019},
url={https://github.com/tensorflow/community/pull/56}
}
@article{saxe2013exact,
title={Exact solutions to the nonlinear dynamics of learning in deep linear neural networks},
author={Saxe, Andrew M and McClelland, James L and Ganguli, Surya},
journal={arXiv preprint arXiv:1312.6120},
year={2013},
url={https://arxiv.org/abs/1312.6120}
}
@article{blundell2015weight,
title={Weight uncertainty in neural networks},
author={Blundell, Charles and Cornebise, Julien and Kavukcuoglu, Koray and Wierstra, Daan},
journal={arXiv preprint arXiv:1505.05424},
year={2015},
url={https://arxiv.org/abs/1505.05424}
}
@article{fortunato2017bayesian,
title={Bayesian recurrent neural networks},
author={Fortunato, Meire and Blundell, Charles and Vinyals, Oriol},
journal={arXiv preprint arXiv:1704.02798},
year={2017},
url={https://arxiv.org/abs/1704.02798}
}
@misc{kingma2014adam,
title={Adam: A Method for Stochastic Optimization},
author={Diederik P. Kingma and Jimmy Ba},
year={2014},
eprint={1412.6980},
archivePrefix={arXiv},
primaryClass={cs.LG}
}
================================================
FILE: docs/requirements.txt
================================================
sphinx>=2.0.1
sphinx_rtd_theme>=0.4.3
sphinxcontrib-katex>=0.4.1
sphinxcontrib-bibtex>=0.4.2,<2
sphinx-autodoc-typehints>=1.10.3
================================================
FILE: examples/BUILD
================================================
# buildifier: disable=out-of-order-load - Breaks copybara otherwise
load("//third_party/bazel_rules/rules_python/python:py_binary.bzl", "py_binary")
load("//sonnet/src:build_defs.bzl", "snt_py_library", "snt_py_test")
package(default_visibility = ["//visibility:private"])
licenses(["notice"])
py_binary(
name = "simple_mnist",
srcs = ["simple_mnist.py"],
strict_deps = False,
deps = [
# pip: absl:app
"//sonnet",
# pip: tensorflow
# pip: tensorflow_datasets
],
)
snt_py_library(
name = "simple_mnist_library",
srcs = ["simple_mnist.py"],
deps = [
# pip: absl:app
"//sonnet",
# pip: tensorflow
# pip: tensorflow_datasets
],
)
snt_py_test(
name = "simple_mnist_test",
srcs = ["simple_mnist_test.py"],
deps = [
":simple_mnist_library",
"//sonnet",
"//sonnet/src:test_utils",
# pip: tensorflow
],
)
py_binary(
name = "functional_mlp_mnist",
srcs = ["functional_mlp_mnist.py"],
strict_deps = False,
deps = [
# pip: absl:app
# pip: absl/logging
"//sonnet",
# pip: tensorflow
# pip: tensorflow_datasets
],
)
================================================
FILE: examples/README.md
================================================
Examples
========
Google Colab notebooks:
- [Predicting MNIST with an MLP](https://colab.research.google.com/github/deepmind/sonnet/blob/v2/examples/mlp_on_mnist.ipynb)
- [Training a Little GAN on MNIST](https://colab.research.google.com/github/deepmind/sonnet/blob/v2/examples/little_gan_on_mnist.ipynb)
Training scripts:
- [Simple ConvNet on MNIST](simple_mnist.py)
================================================
FILE: examples/distributed_cifar10.ipynb
================================================
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "8KcfZ0oRSJ-I"
},
"source": [
"**Copyright 2019 The Sonnet Authors. All Rights Reserved.**\n",
"\n",
"Licensed under the Apache License, Version 2.0 (the \"License\");\n",
"you may not use this file except in compliance with the License.\n",
"You may obtain a copy of the License at\n",
"\n",
" http://www.apache.org/licenses/LICENSE-2.0\n",
"\n",
"Unless required by applicable law or agreed to in writing, software\n",
"distributed under the License is distributed on an \"AS IS\" BASIS,\n",
"WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
"See the License for the specific language governing permissions and\n",
"limitations under the License.\n",
"\n",
"---"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "PUWojJ3iTgKw"
},
"source": [
"# Introduction\n",
"\n",
"This tutorial assumes you have already completed (and understood!) the Sonnet 2 \"Hello, world!\" example (MLP on MNIST).\n",
"\n",
"In this tutorial, we're going to scale things up with a bigger model and bigger dataset, and we're going to distribute the computation across multiple devices."
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "y4TOXSlnTcSB"
},
"source": [
"# Preamble"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "9f811iaTTbmI"
},
"outputs": [],
"source": [
"import sys\n",
"assert sys.version_info >= (3, 6), \"Sonnet 2 requires Python >=3.6\""
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "5R3-sAFoTiyB"
},
"outputs": [],
"source": [
"!pip install dm-sonnet tqdm"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "RdSVvLnkTmWY"
},
"outputs": [],
"source": [
"import sonnet as snt\n",
"import tensorflow as tf\n",
"import tensorflow_datasets as tfds"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "uPtNKJwhTnze"
},
"outputs": [],
"source": [
"print(\"TensorFlow version: {}\".format(tf.__version__))\n",
"print(\" Sonnet version: {}\".format(snt.__version__))"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "Ab-nCz85sIkh"
},
"source": [
"Finally lets take a quick look at the GPUs we have available:"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "elmPgnWJsIUI"
},
"outputs": [],
"source": [
"!grep Model: /proc/driver/nvidia/gpus/*/information | awk '{$1=\"\";print$0}'"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "-T4gfAHytWrk"
},
"source": [
"# Distribution strategy\n",
"\n",
"We need a strategy to distribute our computation across several devices. Since Google Colab only provides a single GPU we'll split it into four virtual GPUs:"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 34
},
"colab_type": "code",
"id": "14VQpLastTlW",
"outputId": "f8d1bcf2-ca45-43bb-8485-5e6589f0b2e3"
},
"outputs": [
{
"data": {
"text/plain": [
"[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]"
]
},
"execution_count": 8,
"metadata": {
"tags": []
},
"output_type": "execute_result"
}
],
"source": [
"physical_gpus = tf.config.experimental.list_physical_devices(\"GPU\")\n",
"physical_gpus"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "aoEY-15BtYaf"
},
"outputs": [],
"source": [
"tf.config.experimental.set_virtual_device_configuration(\n",
" physical_gpus[0],\n",
" [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=2000)] * 4\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 85
},
"colab_type": "code",
"id": "risJlPoDtx6-",
"outputId": "522e0953-de6f-47f8-f06d-5dc14ad32465"
},
"outputs": [
{
"data": {
"text/plain": [
"[LogicalDevice(name='/job:localhost/replica:0/task:0/device:GPU:0', device_type='GPU'),\n",
" LogicalDevice(name='/job:localhost/replica:0/task:0/device:GPU:1', device_type='GPU'),\n",
" LogicalDevice(name='/job:localhost/replica:0/task:0/device:GPU:2', device_type='GPU'),\n",
" LogicalDevice(name='/job:localhost/replica:0/task:0/device:GPU:3', device_type='GPU')]"
]
},
"execution_count": 12,
"metadata": {
"tags": []
},
"output_type": "execute_result"
}
],
"source": [
"gpus = tf.config.experimental.list_logical_devices(\"GPU\")\n",
"gpus"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "WhdyolCktTGU"
},
"source": [
"When using Sonnet optimizers, we must use either `Replicator` or `TpuReplicator` from `snt.distribute`, or we can use `tf.distribute.OneDeviceStrategy`. `Replicator` is equivalent to `MirroredStrategy` and `TpuReplicator` is equivalent to `TPUStrategy`."
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"cellView": "both",
"colab": {},
"colab_type": "code",
"id": "J82G9zqxtb1c"
},
"outputs": [],
"source": [
"strategy = snt.distribute.Replicator(\n",
" [\"/device:GPU:{}\".format(i) for i in range(4)],\n",
" tf.distribute.ReductionToOneDevice(\"GPU:0\"))"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "smiRXbgmT9SD"
},
"source": [
"# Dataset\n",
"\n",
"Basically the same as the MNIST example, but this time we're using CIFAR-10. CIFAR-10 contains 32x32 pixel color images in 10 different classes (airplanes, cars, birds, cats, deer, dogs, frogs, horses, ships, and trucks)."
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "1xOwe9y_T_A4"
},
"outputs": [],
"source": [
"# NOTE: This is the batch size across all GPUs.\n",
"batch_size = 100 * 4\n",
"\n",
"def process_batch(images, labels):\n",
" images = tf.cast(images, dtype=tf.float32)\n",
" images = ((images / 255.) - .5) * 2.\n",
" return images, labels\n",
"\n",
"def cifar10(split):\n",
" dataset = tfds.load(\"cifar10\", split=split, as_supervised=True)\n",
" dataset = dataset.map(process_batch)\n",
" dataset = dataset.batch(batch_size)\n",
" dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)\n",
" dataset = dataset.cache()\n",
" return dataset\n",
"\n",
"cifar10_train = cifar10(\"train\").shuffle(10)\n",
"cifar10_test = cifar10(\"test\")"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "BT2fP7jjpUGe"
},
"source": [
"# Model & Optimizer\n",
"\n",
"Conveniently, there is a pre-built model in `snt.nets` designed specifically for this dataset.\n",
"\n",
"We must build our model and optimizer within the strategy scope, to ensure that any variables created are distributed correctly. Alternatively, we could enter the scope for the entire program using `tf.distribute.experimental_set_strategy`."
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "vEk6eJUPpWB-"
},
"outputs": [],
"source": [
"learning_rate = 0.1\n",
"\n",
"with strategy.scope():\n",
" model = snt.nets.Cifar10ConvNet()\n",
" optimizer = snt.optimizers.Momentum(learning_rate, 0.9)"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "YMgcImzms2V0"
},
"source": [
"# Training the model\n",
"\n",
"The Sonnet optimizers are designed to be as clean and simple as possible. They do not contain any code to deal with distributed execution. It therefore requires a few additional lines of code.\n",
"\n",
"We must aggregate the gradients calculated on the different devices. This can be done using `ReplicaContext.all_reduce`.\n",
"\n",
"Note that when using `Replicator` / `TpuReplicator` it is the user's responsibility to ensure that the values remain identical in all replicas."
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "RUkuAJsxsjvt"
},
"outputs": [],
"source": [
"def step(images, labels):\n",
" \"\"\"Performs a single training step, returning the cross-entropy loss.\"\"\"\n",
" with tf.GradientTape() as tape:\n",
" logits = model(images, is_training=True)[\"logits\"]\n",
" loss = tf.reduce_mean(\n",
" tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels,\n",
" logits=logits))\n",
"\n",
" grads = tape.gradient(loss, model.trainable_variables)\n",
"\n",
" # Aggregate the gradients from the full batch.\n",
" replica_ctx = tf.distribute.get_replica_context()\n",
" grads = replica_ctx.all_reduce(\"mean\", grads)\n",
"\n",
" optimizer.apply(grads, model.trainable_variables)\n",
" return loss\n",
"\n",
"@tf.function\n",
"def train_step(images, labels):\n",
" per_replica_loss = strategy.run(step, args=(images, labels))\n",
" return strategy.reduce(\"sum\", per_replica_loss, axis=None)\n",
"\n",
"def train_epoch(dataset):\n",
" \"\"\"Performs one epoch of training, returning the mean cross-entropy loss.\"\"\"\n",
" total_loss = 0.0\n",
" num_batches = 0\n",
"\n",
" # Loop over the entire training set.\n",
" for images, labels in dataset:\n",
" total_loss += train_step(images, labels).numpy()\n",
" num_batches += 1\n",
"\n",
" return total_loss / num_batches\n",
"\n",
"cifar10_train_dist = strategy.experimental_distribute_dataset(cifar10_train)\n",
"\n",
"for epoch in range(20):\n",
" print(\"Training epoch\", epoch, \"...\", end=\" \")\n",
" print(\"loss :=\", train_epoch(cifar10_train_dist))"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "cHbiCxJ81wZo"
},
"source": [
"# Evaluating the model\n",
"\n",
"Note the use of the `axis` parameter with `strategy.reduce` to reduce across the batch dimension."
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"cellView": "both",
"colab": {},
"colab_type": "code",
"id": "hxL8-GOB1yAq"
},
"outputs": [],
"source": [
"num_cifar10_test_examples = 10000\n",
"\n",
"def is_predicted(images, labels):\n",
" logits = model(images, is_training=False)[\"logits\"]\n",
" # The reduction over the batch happens in `strategy.reduce`, below.\n",
" return tf.cast(tf.equal(labels, tf.argmax(logits, axis=1)), dtype=tf.int32)\n",
"\n",
"cifar10_test_dist = strategy.experimental_distribute_dataset(cifar10_test)\n",
"\n",
"@tf.function\n",
"def evaluate():\n",
" \"\"\"Returns the top-1 accuracy over the entire test set.\"\"\"\n",
" total_correct = 0\n",
"\n",
" for images, labels in cifar10_test_dist:\n",
" per_replica_correct = strategy.run(is_predicted, args=(images, labels))\n",
" total_correct += strategy.reduce(\"sum\", per_replica_correct, axis=0)\n",
"\n",
" return tf.cast(total_correct, tf.float32) / num_cifar10_test_examples\n",
"\n",
"print(\"Testing...\", end=\" \")\n",
"print(\"top-1 accuracy =\", evaluate().numpy())"
]
}
],
"metadata": {
"accelerator": "GPU",
"colab": {
"collapsed_sections": [],
"name": "Multi-GPU training with Sonnet 2",
"provenance": [],
"toc_visible": true
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
================================================
FILE: examples/functional_mlp_mnist.py
================================================
# Copyright 2020 The Sonnet Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Toy MLP on MNIST example of using TF2 JAX/HK shims."""
from absl import app
from absl import logging
import sonnet as snt
import tensorflow as tf
import tensorflow_datasets as tfds
fn = snt.functional
def main(unused_argv):
del unused_argv
with fn.variables():
net = snt.nets.MLP([1000, 100, 10])
def loss_fn(images, labels):
images = snt.flatten(images)
logits = net(images)
loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels,
logits=logits))
return loss
loss_fn = fn.transform(loss_fn)
def preprocess(images, labels):
images = tf.image.convert_image_dtype(images, tf.float32)
return images, labels
# _ _
# | |_ _ __ __ _(_)_ __
# | __| '__/ _` | | '_ \
# | |_| | | (_| | | | | |
# \__|_| \__,_|_|_| |_|
#
batch_size = 100
dataset = tfds.load("mnist", split="train", as_supervised=True)
dataset = dataset.map(preprocess)
dataset = dataset.cache()
dataset = dataset.shuffle(batch_size * 8)
dataset = dataset.batch(batch_size, drop_remainder=True)
dataset = dataset.prefetch()
# As before we want to unzip our loss_fn into init and apply.
optimizer = fn.adam(0.01)
# To get our initial state we need to pull a record from our dataset and pass
# it to our init function. We'll also be sure to use `device_put` such that
# the parameters are on the accelerator.
images, labels = next(iter(dataset))
params = fn.device_put(loss_fn.init(images, labels))
opt_state = fn.device_put(optimizer.init(params))
# Our training loop is to iterate through 10 epochs of the train dataset, and
# use sgd after each minibatch to update our parameters according to the
# gradient from our loss function.
grad_apply_fn = fn.jit(fn.value_and_grad(loss_fn.apply))
apply_opt_fn = fn.jit(optimizer.apply)
for epoch in range(10):
for images, labels in dataset:
loss, grads = grad_apply_fn(params, images, labels)
params, opt_state = apply_opt_fn(opt_state, grads, params)
logging.info("[Epoch %s] loss=%s", epoch, loss.numpy())
# _ _
# | |_ ___ ___| |_
# | __/ _ \/ __| __|
# | || __/\__ \ |_
# \__\___||___/\__|
#
def accuracy_fn(images, labels):
images = snt.flatten(images)
predictions = tf.argmax(net(images), axis=1)
correct = tf.math.count_nonzero(tf.equal(predictions, labels))
total = tf.shape(labels)[0]
return correct, total
accuracy_fn = fn.transform(accuracy_fn)
batch_size = 10000
dataset = tfds.load("mnist", split="test", as_supervised=True)
dataset = dataset.batch(batch_size, drop_remainder=True)
dataset = dataset.map(preprocess)
# Note that while we still unzip our accuracy function, we can ignore the
# init_fn here since we already have all the state we need from our training
# function.
apply_fn = fn.jit(accuracy_fn.apply)
# Compute top-1 accuracy.
num_correct = num_total = 0
for images, labels in dataset:
correct, total = apply_fn(params, images, labels)
num_correct += correct
num_total += total
accuracy = (int(num_correct) / int(num_total)) * 100
logging.info("Accuracy %.5f%%", accuracy)
if __name__ == "__main__":
app.run(main)
================================================
FILE: examples/little_gan_on_mnist.ipynb
================================================
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "Yu7u1mNfnxVP"
},
"source": [
"**Copyright 2019 The Sonnet Authors. All Rights Reserved.**\n",
"\n",
"Licensed under the Apache License, Version 2.0 (the \"License\");\n",
"you may not use this file except in compliance with the License.\n",
"You may obtain a copy of the License at\n",
"\n",
" http://www.apache.org/licenses/LICENSE-2.0\n",
"\n",
"Unless required by applicable law or agreed to in writing, software\n",
"distributed under the License is distributed on an \"AS IS\" BASIS,\n",
"WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
"See the License for the specific language governing permissions and\n",
"limitations under the License.\n",
"\n",
"---"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "YlYRfLOzyuW9"
},
"source": [
"## Generative Adversarial Networks (GANs)\n",
"\n",
"In this notebook we'll use Sonnet 2 and TensorFlow 2 to train a small image generator using the Generative Adversarial Nets (GAN) [1] framework. GANs consist of two modules:\n",
"\n",
"1. A **generator**, which takes randomly sampled noise or latents as inputs and produces data (in this case, images) as output.\n",
"2. A **discriminator**, which provides the learning signal for the generator. Its inputs are real images and generated images, and it's trained to predict whether each input is real or generated. The generator is trained to \"fool\" the discriminator into believing its outputs are real.\n",
"\n",
"Typically both the generator and discriminator are deep neural networks.\n",
"\n",
"For an extended tutorial on GANs, see Ian Goodfellow's [GAN Tutorial](https://arxiv.org/abs/1701.00160) [2].\n",
"\n",
"[1] I. Goodfellow, J. Pouget-Abadie, M. Mirza, B. Xu, D. Warde-Farley, S. Ozair, A. Courville, and Y. Bengio. [Generative Adversarial Nets](https://papers.nips.cc/paper/5423-generative-adversarial-nets.pdf). *NeurIPS*, 2014.\n",
"\n",
"[2] I. Goodfellow. [NIPS 2016 Tutorial: Generative Adversarial Networks](https://arxiv.org/abs/1701.00160). *arXiv:1701.00160*, 2017."
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "WAfR3cvnoGMB"
},
"source": [
"# Preamble"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "4FqOAJb_jJR9"
},
"outputs": [],
"source": [
"import sys\n",
"assert sys.version_info >= (3, 6), \"Sonnet 2 requires Python >=3.6\""
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "FZP3Wut53PTb"
},
"outputs": [],
"source": [
"!pip install dm-sonnet tqdm"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "ermIoeaTel6V"
},
"outputs": [],
"source": [
"import functools\n",
"import time\n",
"import numpy as np\n",
"import sonnet as snt\n",
"import tensorflow as tf\n",
"import tensorflow_datasets as tfds\n",
"import tqdm"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 51
},
"colab_type": "code",
"id": "Rpp_houJEHr9",
"outputId": "4edf64da-e887-4f06-90d7-e1e7e51e31fc"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"TensorFlow version: 2.0.0-rc1\n",
" Sonnet version: 2.0.0b0\n"
]
}
],
"source": [
"print(\"TensorFlow version: {}\".format(tf.__version__))\n",
"print(\" Sonnet version: {}\".format(snt.__version__))"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "ZAC0cJhzqE-a"
},
"source": [
"Finally lets take a quick look at the GPUs we have available:"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 34
},
"colab_type": "code",
"id": "6fOFkKYtqH8s",
"outputId": "0f1bb7bb-0df2-492d-fd9e-67733a20cb28"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" Tesla T4\n"
]
}
],
"source": [
"!grep Model: /proc/driver/nvidia/gpus/*/information | awk '{$1=\"\";print$0}'"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "UYYmqvOKfNbk"
},
"source": [
"# Dataset\n",
"\n",
"We need to get our dataset in a state where we can iterate over it easily. The TensorFlow Datasets package provides a simple API for this. It will download the dataset and prepare it for us to speedily process on a GPU. We can also add our own pre-processing functions to mutate the dataset before our model sees it:"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "UkBRriaQEr4z"
},
"outputs": [],
"source": [
"def process_batch(images, labels):\n",
" images = tf.squeeze(images, axis=[-1])\n",
" images = tf.cast(images, dtype=tf.float32)\n",
" images /= 255.\n",
" images = tf.clip_by_value(images, 0., 1.)\n",
" return images, labels\n",
"\n",
"batch_size = 100\n",
"def mnist(split, batch_size=batch_size):\n",
" dataset, ds_info = tfds.load('mnist:3.*.*', split=split, as_supervised=True,\n",
" with_info=True)\n",
" dataset = dataset.map(process_batch)\n",
" dataset = dataset.batch(batch_size)\n",
" dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)\n",
" dataset = dataset.cache()\n",
" return dataset, ds_info\n",
"\n",
"mnist_train, mnist_train_info = mnist('train')\n",
"mnist_test, mnist_test_info = mnist('test')\n",
"\n",
"mnist_shuffled = mnist_train.shuffle(10000).repeat()"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "JfOCWVGEfgcq"
},
"source": [
"MNIST contains `28x28` greyscale handwritten digits. Let's take a look at one:"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 286
},
"colab_type": "code",
"id": "I_yM0TVjFCZq",
"outputId": "24c80259-d3e5-476d-e238-296506c02d62"
},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.image.AxesImage at 0x7fb4d006c438>"
]
},
"execution_count": 8,
"metadata": {
"tags": []
},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADfxJREFUeJzt3X2MXXWdx/HPt53pVAqYVpY6W0Yo\nWMxWVlsy1oftEk2FAIsp/kOolBQlDmskSkJUUo1i1ii7i3UJGMIglcLyoBFIm1gfsJhFQCrDUwvO\nagu2sXXoAKPyoJRO+/WPOdUR5vzu7T3n3nNnvu9XcjP3nu95+ObCp+fe87v3/szdBSCeaVU3AKAa\nhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFAdrTzYDOvymZrVykMCobyil/Wq77V61i0UfjM7\nXdJVkqZL+pa7X5Faf6Zm6d22rMghASRs9k11r9vwy34zmy7pm5LOkLRQ0gozW9jo/gC0VpH3/Esk\nbXf3p939VUm3S1peTlsAmq1I+OdJ+u24x7uyZX/HzPrMbMDMBvZpb4HDAShT06/2u3u/u/e6e2+n\nupp9OAB1KhL+3ZJ6xj0+JlsGYBIoEv6HJC0ws/lmNkPSuZI2lNMWgGZreKjP3UfN7GJJP9LYUN9a\nd3+ytM4ANFWhcX533yhpY0m9AGghPt4LBEX4gaAIPxAU4QeCIvxAUIQfCKql3+fH5DNtUfqLml1X\nPZ+s/3m0M7+4bFcjLaEknPmBoAg/EBThB4Ii/EBQhB8IivADQTHUF9z0I49M1uddtzNZv77n/mT9\n+O9dlFtbIIb6qsSZHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCYpw/uO3XzU/WN/asS9b7//iPyfr8\n9aOH3BNagzM/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRVaJzfzHZIelHSfkmj7t5bRlMoz+8+875k\nffCUa2rsIX1+uO5/lifrR236eY39oyplfMjnA+7+XAn7AdBCvOwHgioafpf0YzN72Mz6ymgIQGsU\nfdm/1N13m9nRku42s/9393vHr5D9o9AnSTN1WMHDAShLoTO/u+/O/g5LukvSkgnW6Xf3Xnfv7VRX\nkcMBKFHD4TezWWZ2xMH7kk6T9ERZjQForiIv++dKusvMDu7nVnf/YSldAWi6hsPv7k9LemeJvaBB\n02bOzK1d8rE7k9tOt/SLvy89+/Zk/eibH0/WDySrqBJDfUBQhB8IivADQRF+ICjCDwRF+IGg+Onu\nKWDPBYtzaxe+8cFC+/7BmlOS9dl/4iu7kxVnfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IinH+SaBj\nXnoa7O+v/u9E9fDktieu+0SyPv+mYp8TQPvizA8ERfiBoAg/EBThB4Ii/EBQhB8IivADQTHOPwkM\nfq4nWe/uyB/Lf27/y8lt569P1+WermPS4swPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0HVHOc3s7WS\nzpI07O4nZcvmSPqOpOMk7ZB0jrv/vnltTm0db56brN/+oWtq7KEzt3L1yJL0pg9uqbFvTFX1nPlv\nlHT6a5ZdJmmTuy+QtCl7DGASqRl+d79X0shrFi+XtC67v07S2SX3BaDJGn3PP9fdh7L7z0hKv24F\n0HYKX/Bzd5eU+wFwM+szswEzG9invUUPB6AkjYZ/j5l1S1L2dzhvRXfvd/ded+/tVFeDhwNQtkbD\nv0HSquz+Kknry2kHQKvUDL+Z3Sbp55LeZma7zOxCSVdIOtXMtkn6YPYYwCRSc5zf3VfklJaV3Etc\nh70hWV7SlT+OX8sDn0qP80/Tow3vu9k6eo5J1g/MOSJdf3ywzHamHD7hBwRF+IGgCD8QFOEHgiL8\nQFCEHwiKn+5uA785Lz0Fdy17fV9ubdqfRwvtuyjryv9U587/PTG57VWLb0/WF3Smv0V+/qWX5tZm\nfW9zctsIOPMDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCM87dAR/ebk/WrL7iu0P6/OPyu/OIvthba\ndy2pcXxJev6OY3NrgyffXPDo+VOTS9KX//NbubU195+a3HZ06JmGOppMOPMDQRF+ICjCDwRF+IGg\nCD8QFOEHgiL8QFCM87fAy4t7kvVlb9hfaP9PvXRUovpcoX3XPPaXT07Wt518bcP7Hnz1T8n6P804\nLFlPPa//sSj936SLcX4AUxXhB4Ii/EBQhB8IivADQRF+ICjCDwRVc5zfzNZKOkvSsLuflC27XNLH\nJT2brbba3Tc2q0mkDf4g//fvjyk4zv+br703Wb//vCtr7GFWbuWbf0iPtd945VnJ+kNfSX+GYL8f\nyK3ZAU9uG0E9Z/4bJZ0+wfJvuPui7EbwgUmmZvjd/V5JIy3oBUALFXnPf7GZbTGztWY2u7SOALRE\no+G/VtIJkhZJGpL09bwVzazPzAbMbGCf9jZ4OABlayj87r7H3fe7+wFJ10takli339173b23U+kf\newTQOg2F38y6xz38sKQnymkHQKvUM9R3m6T3SzrKzHZJ+pKk95vZIkkuaYeki5rYI4AmqBl+d18x\nweIbmtDLlNU1kr7WMTT6UrLe3ZH+ffojlg4fck8HdczP/119SbpvZXoc/+jp+eP4kvSF4X/OrT36\nb+lx/pErXknWazl/x7Lc2owfDRTa91TAJ/yAoAg/EBThB4Ii/EBQhB8IivADQfHT3a3w4JZk+ern\n35esf3Vuevt73nFrbu2D534que2cf9+ZrNcayqvl1sdyP/ypzjX7kts+9a/fLnTsP/Qdnaj+vtC+\npwLO/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QlLm37ieMj7Q5/m7L/5plVAeWLkrW7/7uja1pZJJZ\n+MDKZP3YldtzawdeKfZ14Xa12TfpBR+xetblzA8ERfiBoAg/EBThB4Ii/EBQhB8IivADQfF9/jbQ\n8fhTyfpbb/lEsj74kWtya502vaGeWmFXjZ8sP63/s8l6z1ceSNbzJ+iGxJkfCIvwA0ERfiAowg8E\nRfiBoAg/EBThB4Kq+X1+M+uRdJOkuZJcUr+7X2VmcyR9R9JxknZIOsfdkz+Gzvf5m+P5j783t/ah\ni/8vuW3f7F8k67WmBy/ihHs+mqy/deWjTTv2VFX29/lHJV3q7gslvUfSJ81soaTLJG1y9wWSNmWP\nAUwSNcPv7kPu/kh2/0VJg5LmSVouaV222jpJZzerSQDlO6T3/GZ2nKTFkjZLmuvuQ1npGY29LQAw\nSdQdfjM7XNIdki5x9xfG13zswsGEFw/MrM/MBsxsYJ/2FmoWQHnqCr+ZdWos+Le4+53Z4j1m1p3V\nuyUNT7Stu/e7e6+793aqq4yeAZSgZvjNzCTdIGnQ3deMK22QtCq7v0rS+vLbA9As9Qz1LZX0M0lb\n9bdvSa7W2Pv+70p6i6SdGhvqG0nti6G+9vPHle9J1s9f/f1kfdWR25L1d9xxSW7txM+mh/J8L28T\nD9WhDPXV/D6/u98nKW9nJBmYpPiEHxAU4QeCIvxAUIQfCIrwA0ERfiAopugGphCm6AZQE+EHgiL8\nQFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii\n/EBQhB8IivADQRF+ICjCDwRVM/xm1mNmPzWzX5rZk2b26Wz55Wa228wey25nNr9dAGXpqGOdUUmX\nuvsjZnaEpIfN7O6s9g13v7J57QFolprhd/chSUPZ/RfNbFDSvGY3BqC5Duk9v5kdJ2mxpM3ZoovN\nbIuZrTWz2Tnb9JnZgJkN7NPeQs0CKE/d4TezwyXdIekSd39B0rWSTpC0SGOvDL4+0Xbu3u/uve7e\n26muEloGUIa6wm9mnRoL/i3ufqckufsed9/v7gckXS9pSfPaBFC2eq72m6QbJA26+5pxy7vHrfZh\nSU+U3x6AZqnnav+/SDpf0lYzeyxbtlrSCjNbJMkl7ZB0UVM6BNAU9Vztv0/SRPN9byy/HQCtwif8\ngKAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQZm7t+5gZs9K\n2jlu0VGSnmtZA4emXXtr174kemtUmb0d6+7/UM+KLQ3/6w5uNuDuvZU1kNCuvbVrXxK9Naqq3njZ\nDwRF+IGgqg5/f8XHT2nX3tq1L4neGlVJb5W+5wdQnarP/AAqUkn4zex0M/uVmW03s8uq6CGPme0w\ns63ZzMMDFfey1syGzeyJccvmmNndZrYt+zvhNGkV9dYWMzcnZpau9LlrtxmvW/6y38ymS/q1pFMl\n7ZL0kKQV7v7LljaSw8x2SOp198rHhM3sFEkvSbrJ3U/Klv2XpBF3vyL7h3O2u3+uTXq7XNJLVc/c\nnE0o0z1+ZmlJZ0u6QBU+d4m+zlEFz1sVZ/4lkra7+9Pu/qqk2yUtr6CPtufu90oaec3i5ZLWZffX\naex/npbL6a0tuPuQuz+S3X9R0sGZpSt97hJ9VaKK8M+T9Ntxj3epvab8dkk/NrOHzayv6mYmMDeb\nNl2SnpE0t8pmJlBz5uZWes3M0m3z3DUy43XZuOD3ekvd/WRJZ0j6ZPbyti352Hu2dhquqWvm5laZ\nYGbpv6ryuWt0xuuyVRH+3ZJ6xj0+JlvWFtx9d/Z3WNJdar/Zh/ccnCQ1+ztccT9/1U4zN080s7Ta\n4Llrpxmvqwj/Q5IWmNl8M5sh6VxJGyro43XMbFZ2IUZmNkvSaWq/2Yc3SFqV3V8laX2Fvfyddpm5\nOW9maVX83LXdjNfu3vKbpDM1dsX/KUmfr6KHnL6Ol/R4dnuy6t4k3aaxl4H7NHZt5EJJb5K0SdI2\nST+RNKeNertZ0lZJWzQWtO6KeluqsZf0WyQ9lt3OrPq5S/RVyfPGJ/yAoLjgBwRF+IGgCD8QFOEH\ngiL8QFCEHwiK8ANBEX4gqL8A74xLCC0psmEAAAAASUVORK5CYII=\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"tags": []
},
"output_type": "display_data"
}
],
"source": [
"import matplotlib.pyplot as plt\n",
"\n",
"images, _ = next(iter(mnist_test))\n",
"plt.imshow(images[0])"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "d7bsizs5gK3K"
},
"source": [
"# Sonnet\n",
"\n",
"The next step is to define a model. In Sonnet everything that contains TensorFlow variables (`tf.Variable`) extends `snt.Module`, this includes low level neural network components (e.g. `snt.Linear`, `snt.Conv2D`), larger nets containing subcomponents (e.g. `snt.nets.MLP`), optimizers (e.g. `snt.optimizers.Adam`) and whatever else you can think of.\n",
"\n",
"Modules provide a simple abstraction for storing parameters (and `Variable`s used for other purposes, like for storing moving averages in `BatchNorm`).\n",
"\n",
"To find all the parameters for a given module, simply do: `module.variables`. This will return a `tuple` of all the parameters that exist for this module, or any module it references:"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "GrN37pi1o4HT"
},
"source": [
"## Building the model"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "c6XoN56S2lSW"
},
"source": [
"In Sonnet you build neural networks out of `snt.Module`s. In this simple example we'll build multi-layer perceptron (MLP) based generators and discriminators.\n",
"\n",
"We'll make use of \"Spectral Normalization\" [1] in both modules, which serves to regularize them.\n",
"\n",
"[1] T. Miyato, T. Kataoka, M. Koyama, and Y. Yoshida. [Spectral Normalization for Generative Adversarial Networks](https://arxiv.org/abs/1802.05957). *arXiv:1802.05957*, 2018."
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "hgjyB9yhFclD"
},
"outputs": [],
"source": [
"class SpectralNormalizer(snt.Module):\n",
"\n",
" def __init__(self, epsilon=1e-12, name=None):\n",
" super().__init__(name=name)\n",
" self.l2_normalize = functools.partial(tf.math.l2_normalize, epsilon=epsilon)\n",
"\n",
" @snt.once\n",
" def _initialize(self, weights):\n",
" init = self.l2_normalize(snt.initializers.TruncatedNormal()(\n",
" shape=[1, weights.shape[-1]], dtype=weights.dtype))\n",
" # 'u' tracks our estimate of the first spectral vector for the given weight.\n",
" self.u = tf.Variable(init, name='u', trainable=False)\n",
"\n",
" def __call__(self, weights, is_training=True):\n",
" self._initialize(weights)\n",
" if is_training:\n",
" # Do a power iteration and update u and weights.\n",
" weights_matrix = tf.reshape(weights, [-1, weights.shape[-1]])\n",
" v = self.l2_normalize(self.u @ tf.transpose(weights_matrix))\n",
" v_w = v @ weights_matrix\n",
" u = self.l2_normalize(v_w)\n",
" sigma = tf.stop_gradient(tf.reshape(v_w @ tf.transpose(u), []))\n",
" self.u.assign(u)\n",
" weights.assign(weights / sigma)\n",
" return weights\n",
"\n",
"\n",
"class SpectrallyNormedLinear(snt.Linear):\n",
"\n",
" def __init__(self, *args, **kwargs):\n",
" super().__init__(*args, **kwargs)\n",
" self.spectral_normalizer = SpectralNormalizer()\n",
"\n",
" def __call__(self, inputs, is_training=True):\n",
" self._initialize(inputs)\n",
"\n",
" normed_w = self.spectral_normalizer(self.w, is_training=is_training)\n",
" outputs = tf.matmul(inputs, normed_w)\n",
" if self.with_bias:\n",
" outputs = tf.add(outputs, self.b)\n",
" return outputs\n",
"\n",
"\n",
"class SimpleBlock(snt.Module):\n",
"\n",
" def __init__(self, embed_dim, with_batch_norm=False, name=None):\n",
" super().__init__(name=name)\n",
" self.embed_dim = embed_dim\n",
" self.hidden = SpectrallyNormedLinear(self.embed_dim)\n",
" if with_batch_norm:\n",
" self.bn = snt.BatchNorm(create_scale=True, create_offset=True)\n",
" else:\n",
" self.bn = None\n",
"\n",
" def __call__(self, inputs, is_training=True):\n",
" output = self.hidden(inputs, is_training=is_training)\n",
" if self.bn:\n",
" output = self.bn(output, is_training=is_training)\n",
" output = tf.nn.relu(output)\n",
" return output\n",
"\n",
"\n",
"class Generator(snt.Module):\n",
"\n",
" def __init__(self, output_shape, num_layers=1, embed_dim=1024, name=None):\n",
" super().__init__(name=name)\n",
" self.layers = [\n",
" SimpleBlock(embed_dim, with_batch_norm=True, name='block_'+str(index))\n",
" for index in range(num_layers)\n",
" ]\n",
" self.output_shape = tuple(output_shape)\n",
" output_size = np.prod(self.output_shape, dtype=int)\n",
" self.outputs = snt.Linear(output_size, name='outputs')\n",
"\n",
" def __call__(self, inputs, is_training=True):\n",
" inputs = tf.convert_to_tensor(inputs)\n",
" output = snt.Flatten()(inputs)\n",
" for layer in self.layers:\n",
" output = layer(output, is_training=is_training)\n",
" output = self.outputs(output)\n",
" output = tf.reshape(output, [-1] + list(self.output_shape))\n",
" output = tf.sigmoid(output)\n",
" return output\n",
"\n",
"\n",
"class Discriminator(snt.Module):\n",
"\n",
" def __init__(self, num_layers=1, embed_dim=1024, name=None):\n",
" super().__init__(name=name)\n",
" self.layers = [\n",
" SimpleBlock(embed_dim, with_batch_norm=False, name='block_'+str(index))\n",
" for index in range(num_layers)\n",
" ]\n",
" self.outputs = SpectrallyNormedLinear(1, name='outputs')\n",
"\n",
" def __call__(self, inputs, is_training=True):\n",
" inputs = tf.convert_to_tensor(inputs)\n",
" output = snt.Flatten()(inputs)\n",
" for layer in self.layers:\n",
" output = layer(output, is_training=is_training)\n",
" output = self.outputs(output)\n",
" return tf.reshape(output, [-1])\n",
"\n",
"\n",
"class LittleGAN(snt.Module):\n",
"\n",
" def __init__(self, num_layers=2, embed_dim=1024, name=None):\n",
" super().__init__(name=name)\n",
" self.generator = Generator(\n",
" [28, 28], num_layers=num_layers, embed_dim=embed_dim)\n",
" self.discriminator = Discriminator(\n",
" num_layers=num_layers, embed_dim=embed_dim)\n",
"\n",
" def generate(self, noise, is_training=True):\n",
" return self.generator(noise, is_training=is_training)\n",
"\n",
" def discriminate(self, images):\n",
" return self.discriminator(images)"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "0i03px8y8gf7"
},
"source": [
"Now we'll create an instance of our class whose weights will be randomly initialized. We'll train this MLP such that it learns to recognize digits in the MNIST dataset."
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 34
},
"colab_type": "code",
"id": "XqL8oIMqGAnU",
"outputId": "6471fe0d-0db3-4da0-c4ce-cdf672da67d9"
},
"outputs": [
{
"data": {
"text/plain": [
"LittleGAN(num_layers=2)"
]
},
"execution_count": 10,
"metadata": {
"tags": []
},
"output_type": "execute_result"
}
],
"source": [
"gan = LittleGAN(num_layers=2)\n",
"gan"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "snzkUUh9oXPy"
},
"source": [
"## Using the model"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "On8wI6VwpDPm"
},
"source": [
"Let's feed some random noise through the generator and see what it generates. Since the model is randomly initialized and not trained yet, the images it produces should look like noise.\n",
"\n",
"Below, the top row of images are real MNIST digits; the bottom row are the outputs of our randomly initialized generator."
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 160
},
"colab_type": "code",
"id": "4T-qmIc0GHfP",
"outputId": "35babc71-6172-4756-f3fb-bb2f6f5cf363"
},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAABFQAAACPCAYAAADUS4+vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzsnWd4VNXWgN8zJb0nkIQUeiiKCAhi\nBUQUCypWsCsWrGCv31Wv3nvtimIDRbGjiIKCYgOkShGQ3lsghBTS68yc78c6E4gIEkibyXqfJ8/M\nnDnnzM5eZ+2y1tprG6ZpoiiKoiiKoiiKoiiKohw+toYugKIoiqIoiqIoiqIoiq+hBhVFURRFURRF\nURRFUZQaogYVRVEURVEURVEURVGUGqIGFUVRFEVRFEVRFEVRlBqiBhVFURRFURRFURRFUZQaogYV\nRVEURVEURVEURVGUGqIGFUVRFEVRFEVRFEVRlBpyVAYVwzAGGoaxzjCMjYZhPFxbhVLqF5Wj76My\n9A9Ujr6PytA/UDn6PipD/0Dl6PuoDP0DlePBMUzTPLILDcMOrAcGAOnAImCoaZqra694Sl2jcvR9\nVIb+gcrR91EZ+gcqR99HZegfqBx9H5Whf6ByPDSOo7i2F7DRNM3NAIZhfA5cCBy0YgOMQDOI0KP4\nyYanjGIqzHKjoctRi9RIjv4gQ4BC9mabptmsoctRS6gu+geqi76P6qJ/oLro+6gu+geqi76P6qJ/\noLp4CI7GoJIE7Njvczpw4l9PMgzjFuAWgCBCONHofxQ/2fD8bv7S0EWobf5Rjv4mQ4CfzYnbGroM\ntYjqon+guuj7qC76B6qLvo/qon+guuj7qC76B6qLh6DOk9KapjnGNM0TTNM8wUlgXf+cUgeoDP0D\nlaPvozL0D1SOvo/K0D9QOfo+KkP/QOXo+zRlGR5NhMpOIGW/z8nWsQbFdnxnAAJH5QBQ6nJC//SG\nLFJjp1HKUakRKkP/QOXo+6gM/YMGl+MFq2UMc0fUjmrH234xHIAOT67BnZdfn0XyNRpchkqtoHL0\nfVSG/oHK8RAcjUFlEdDeMIzWSIUOAa6slVIdAfaICACS3pHInLEpcwFoM/FW2qMGlUPQqOSoHBEq\nQ/9A5ej7qAz9gwaX4xufDQJg+PDR1Y5vuvxtANIvLmLIvfcBEDrx9/osmq/Q4DJUagWVo++jMvQP\nfEKO5ef0BODh1z8E4Ovc7gAsGXM8se/Or7PfPWKDimmaLsMw7gSmA3ZgnGmaq2qtZEq9oHL0fVSG\n/oHK0fdRGfoHKkffR2XoH6gcfR+VoX+gcjw0RxOhgmma04BptVSWo2LjO60BmJYyHoAx+S0AaD3Z\n1WBl8hUajRx7dQGgxWtbAXg/dTYAnxdG887dlwAQMH1xgxStsVPXMiy9qBcA2V2kySjvUArA2n7v\nVp3jNOwAVJpuADr+ehMAgeuDAYhb4SL4m4V1VUS/oNHoonLEqAz9g4aW48NXfXHI7xPtIYx/6SUA\nbs4bAYDz5yV1Xi5foqFlqNQOKkffR2XoHzR2OaY/cjITb30RgHZOyeEywynzlWaL8/HU4W/XeVJa\nRVEURVEURVEURVEUf+OoIlQaA7seOBmANad71xmLjeidVy8EIO6XulsvpdQO3vVur731OgDHOAMA\ncJvy/WVhORS/+h0AX10g8nZv2FzPpWx6OFKSASgZZ2d0+9cA6OC0Vztnf2tvpek9JkdXnzFGDpwh\nLxsrXdxx+1AAQodJ5Jhrh+Y3UhRF+SvPfH0ZAE+3FO9a5AyJ9Lvs7p8BuD9mHa0cIQDs7i2euJSf\n67uUyv5M37UM2BeluT+dZw0DIGRRSLXjzf8oxTZrad0XTjksvPkYPaVlAJiVFf94jS0khE3vtQdg\n1enjADj14TsBiPpI5yCKUtdsefYkAOZd9QK51sSk2+t3AZD65koAPAWr67QMGqGiKIqiKIqiKIqi\nKIpSQ3w6QsUWFMTIGycBYDfENvRE1jEANP9oOUCdrpdSjh57XCz/fmMsAM1sErVw/MIbAUh4Wbxu\nWy4MZN3QNwEom7wIgKmntAPAvXdvvZa3KVHYQ/IQvZ/2MjvdYfI+PwmAd96UCLDg7H0aZhryaliR\nKgE37gZgSLLIrEvQDqYf8yUAj0+WnCyrrFxH7n676urfUJQmQeEVvQEIn7CAjPskks8VJN+lPie5\ni0yX5hTzFVo/8vee7VkTEgHImBHJSwki129vfh6AIbseACBmnHrF6xOvd7TSXGK9Hhihsvx0idh0\n9qmea+zr4kS+z+lS7dx5izsA0OGRlXiKi+um0MrfsmXksQAk95Htym39dxzqdACyhnZlxekSxesd\nET32L8nn+MZHabVfSEVp4hjWSob1Y6Xt3DDgDQCey+nOjNulPU6aMw+AA1vjusGnDSqZ13djWOSC\nase+f/l0AKJLdEDhC6x9qh2nBP4EwAn/fRCAFm/Mq3ZO2zkGXVpdC8DSk94HYPRdsqVk6r+rn6vU\nHt4EskMjHyByk4Sd2+ZISHNzDqPeP5eXr2kGwFenncWT498D4Jl4KzltvLyccvPdxL5r6bJp1kLp\nFcX/MBwOjI5iTC5LEiPnMc+sAODqWBlQfHz/yXyeIEnZwmxilL5vsBhbCl3hAOy5LFKX2/ko7oIC\nADacn8BPs2UZ0AB54fhb/wRg+7gGKVqTJWrtkV87ODSDwaEZ1Y45W/4CQP9fbiN4siZyrw9yhskk\nbPKwFwBo6ZAJ2wX0PKL7JTnyALCntcW9flMtlFBRFC+ZE9sAsO4EMVTPLnMCMPfCjti2LGuQMumS\nH0VRFEVRFEVRFEVRlBrikxEqjiRZJjD10RcA8dKljb8NgNYfLjjYZUojpOOTm+m1/A4AEj4Wq+IB\ny7RMk5RLJanQA4sllP1/13wIwJh3TsaduadeytpUiR5fO9FettlL+Xeb7gDYZ4gOf502GYD5T46m\nR4QkkEp8SaOOjpa914u3LXy7JNRz/LpvO9Wy82W5VWVodXt65Jo8PH8ehatVqXPscbF8M/2TQ57T\no8U8ILDasZcSq/eLj0/pwZTJ0pamPqn65ou4MnZz9xeyPHbNdRKd9H+J0wG4se/d2Gf+0WBla2pE\nfyB95IULhx7wXV6XGABS714PwClREq0wLHLDP94397pikibXVimVv8MeK/IZdPcsYF9kSk1oviCX\nWaWSbLhPcAkAOe5QAMwMHZ/WNbk3nkRO33IAnj5RFObZ1WcDULYu8oDzk36Tpa87Tz9wCpw0y/qu\nj3x3Up9VAOy+vw3G3IaJfFD2UXClRNvO7vEqAOnWMuYX+lwAgCt9W8MUDI1QURRFURRFURRFURRF\nqTE+GaGy5qEUABIdYWS7JWFX68lW4i7Nv+BTuLOyiBuTBRxeAuEZn8t61lfu+R2AdxJiQSNUfAbv\nVsydI3ce8F3syn/enlA5NFetlbwYV4RLgjy31R6muyurzikzJVrBjnyXZiX3KjErKDdFC72W9vfy\njgfgq+3yWv5TM5J+ypF7r1pXV/+G8hcGr5Y20mbsa+v2emRbz9dyxGPjsbJC2wyz2nuATsGS9Pny\nMLn+meZLePQm8ar3ct4LQKvHNO+YrxG8x6j2eVl5cwACsorrLRGfsg/36vUHHAu3durcO0Fe373n\nPACG3ffqP94vZnxorZVN+Xt2XdkRgEfjfrKOSO/X8QuJnG7HP0e9u1et4zurr+wXLOdXIsmHPYWF\ntVlc5W/I7uVmff+x1Y5d3usjedPrwPOLrpJoFm+Osf3Jv0r61UhbULXjX4xbxYcdUmqhtMqRYju2\nIw899TEATkP0a+hjIwGITG/41SkaoaIoiqIoiqIoiqIoilJDfCpCxZEgW4J8Pmi0dcTJ67mW+XHB\nnw1TKKVeidimfjdfwRYkFn6zU1sAdvaPZOCV4gWv2uXH4vT77yB6vrjyVMJHzoe3yTrS0anieQnO\nldoMzigBt0QrmEtlTTA2sfCXndej6vodA8TjbYbLutT7T/wRgOnHyRaQYccHknmv7Ph0wf9ki9Zm\nb2lkQ10zPEoiuipNN7eny052C77sCkCLF/85B8ribgMBeOryCABWXTuaEEMikypiVeN8leSvtgPw\n0fAEAK4Jl63qH7osltRVDVYsZT8crVsC0OLzbACmp7wJQKVpP+g1XedfB0DqzhI05rpuuf32bwDw\nWDHS9+w6DYC0f62yjh8eHtNW7T5uU/3V9cX9p39fo/Oj7ZLv5u+2N/9rZIqXKLtuX97QrL09gvNC\n8gE45rebAGj9ccNHpnhRjVcURVEURVEURVEURakhPhWhQkgwAL0CnVWH5t0tESo2lv7j5d78DZ6Y\ncHldvqa2S6jUMZm9xAa4tlLWQBrFZQ1ZnKZL7+MAWH/LfhnxrXwNWPkbQiIlkuGP3h8AYMNW5b35\nvVx0ePi42wFI+WyeRqbUAvYZsrNH9F+O/62X0yM1HvTtvmih9t9WP2UKsQC89sx9AKy8YTTxdmmH\ni2WjJpodVYmVw2HIljMA2P56GlE/y+4gLbIPf3ceb1RSmxXS5XdqOYw1fd6r5VIq9Y1rh+RMenuz\nRC1d0/ULAFr32artaQNiWHmptj12ApcPlt1jHoqVMao3MsXrHV9TCWsrEgF48uvLAWjzsET9aXRK\n3XNDxA5gXyTKosxUAGIKD8yHozQucm+Q3QyvCH8RqB5Zcl+G5Bb7I/vAvCdh/ye5iTJOlblg+Nm7\nDzjnubSJAPQKVC1saAqGWjv7nP8iIONP1x55NXp2AcCeXSDHtzTcLj8+ZVDZclWLap/LzUpspa5D\nXmMEBrLt4zQARnX7HID2zr0AXHPffYRO/L0OSqrUFS27Sej7uJxTAHBv3NKQxWk6GGIkKZjWBoDf\njnv/gFO8SaIODKMUI9gLOZ355LP+ACT/TyaDKeiWrb7A/tHpXqNY86WHGwytHC17T8kFIJwFRzVR\nNoJlEKLGFD/BWrYX5JBxkN2QtrbU5aTmm78qtYUtrTUAS246MPHsM9nijPjkZ1la0mK2h+BvxKjd\nBl0+WZ/sevBkYElDF0M5QioiZFy6/zKdAasuASD0WnHohe7efMB1XhNJwiLrzSsH3nvUnAEAfNL6\nx9oprHLEzHlRlkl6LGMKwLpL35A3l8pLvpWk/8P8Lry3TgxtKUPE+WRW1s+GF7rkR1EURVEURVEU\nRVEUpYb4RISKI1ESrr1+/TvVjv9rT09YuOJvrzECJSljzlctWdP9o798GwbAU8+9y8tzxQrpyjgw\n5EtpPJgnSQLGyR3lGej223AA2rCswcrUFCn+WRJDp3cW638Lx75t5yots7/nIGncfsrsSMLC8rot\noFInuBL3WfjfzZcopZBJGt2nKA3JjkdOBGDFMZKo/8MC2TY5+DabLvlppMx69GQA2k5tPMkUmyqV\nYeZ+kbVyLGx0ZI3vU3hFb15pYXnMLT+13dAIzrrmhKEHbkayd5qsZAjcfWTRz+5+3QF4NOkt64hP\nTJP9ksIhstTHbljzPNPDoPXnA5D3jizNy5RTeP+CtwEYEb2REb03ApD2vKQUaHdP/bS1GqGiKIqi\nKIqiKIqiKIpSQ3zC9FbcTZIK9Q+u7nPZVBQHZP/tNZueEivjhu5vVR1bU1ECQKeAkKr7PX283DtQ\nI1QaL4ZB5kPiIZ9WIhES7Z+QBETqhasnTHHfJL4kVv/rNt4LgCt4n0122BOy/aA36d6A0HUAJFtR\nLD92nsT8sfLdbUuvAiDlGev2S3WPz8aIPUK22X2w1w9Vx9aVJFjvKhugRIriPzjatAIgv5v0azlD\nZIzSfHwwwTurb9NpLl8r18RbaaAdDqJPqz5uCbVZkWT2g2/JqzQs1780GYCnBl0EQNrwhYc6XalL\nTKMq55s3snbbOaI77X846FUH4Lo254DIXN02ue6ptOr47bw2fJMhUeyRWw+dV/OfKI2TDROOCag+\nPb5n8eW05sCIGKXuiPxWVqC4XxLdKvCU4XpcojDD50rUSbikRuW516Q9XXdHAsuukNxVyy+T10s/\nHgaAuaRu5xmq8YqiKIqiKIqiKIqiKDXEJyJUDsaa79NI/kuEypb/SXbfuVe9aB0J5Y08iUL54EVZ\ne7XoGYlacZseDI9uidXQ2KNkzWpB/44A7DlB7HxnninbDC7IaMkfPT4BoNP4OwBovUGz4TckwZMP\n9Kp98XlCtc/jrrsAgLRbZXvy91v+womBEtXg3UqZ7+TlgqSedVNQ5Yiwhcq2gltGHgvAsMgZVd8l\nBEh02A+f9T7o9d4t7dqP0DwBjYm8QcdY72Y2ZDGaNEbPLuQ9JTmonkyTfcoHBJdWP+nkA6/rt1J2\nr7goaTkAcY4CrgrfU+2cH/bKFpLudRtrs8hKDTHyiwB4Ied4Ho2rnufv+giR2VWDrOjpQdDlvTsB\naD0pHwDPstX1VNKmidFD2sHy+AOjGT6/4HUAPjz5VADmj+1e9V34Djk/8HvZHqbDYolmuDr207or\nrHJQMk+Ssch3RONgO0DV6xHfc/Df5/l75YQveI2OR3VvpWZ4iqtHab6UcyLG3L/Pm+navBWAtvdt\npUuktKfrz5Gcm6azfiI2NUJFURRFURRFURRFURSlhvhEhEpgrlgMM1xi9U90yC494afu8844WrcE\nYM7VEpnS3C4e1sf3dGHpeRKhkvtsWbX7XrO1PwHTF9dhyZUDMGTf+OKLewGQMbiCl078EoBBITP+\n/poW+2XrbiMWS29uB3dBQR0VVDlaosdLFFHWePnc/6LbsN+RCUg+lf1psSCczEFBALizsuqvkE0Y\n23Hibdn8WAAAybF5TOjwGQBxVvvpNmdbZxtV110dtQSAL0O6AXBG8noAvl3fpeqcmJX7zlfqF3tc\nbNUud66duwDYe51Ebr7yxBtV531SmAhAx7cKAQ6yN5dSWxgOGW4VPl3M3C4Ta3z9jGO/AsBuiB/M\nbR4osdbBErG77eTeGPOWH2lRlaPElb4TgN+v7EK38/sC8MiNEwC4PEzGrd7cHQB/3DgKgDGXpAHw\n4+Wye5O5Nf0AL61y9HhzKaQtgW5b7wJgyV0ig67SHfJSizkA2J6YV5UfJdMtc5Fl5ZLH4ZyQg7ed\nL285C4BAttZ6+ZW6wXNaN8b1fv9vv7tr3pW05496LlHTxtOnm/VO6n16eidiWP+P19kKG8a04RMG\nFRZIIqDXcyQO9r/x8vnX4z7lzCF3AxAzfBuwz5Di5dNlvXC+LMsMNp1WXVHybmkO7K2zYisHknGv\nDOyX3jv6gO9G7W1X7fOIaAlbduFmRqkY0dac9gEAp3x+OQCR56pBxVcI/mYh9l/FEJb2/G0A/K+f\nGNPGpMxk0ARZIuS4LhkA1470Bihl08G2VwzUlbulvrfsDmHghPsBiNgmbeauUyWkecWwffp6xoQH\nAGj7gBjMVlrHW6MTuIbE0Uq2EYz9LI8+UTJheP6rwQAsvP5lAEKMgKrz33tUvgtZrttf1wc7HhQn\nwp9d9unSJpcs9Rm5+bIDzh/T9gsAEu0hh/0bD8XK8sr8N4JZco8sVbDP1ElAQ+FetY4kKw/ih8+J\nY++Jl0TWT50vfd/g0Iyq82+JlMnCHT9uAqD/7bf97fJapfZIelYcdqfvlLlE9tnieF3eV5ZkBRt2\nvMH8iXZZyppoGVJsVY6GA4P9dy8Qg3VLNaj4DFuGw0mB1be6WFguMm73tm6BUd8UJwYe0XVmdMNs\nmKBLfhRFURRFURRFURRFUWqIb0SoWCy6y0oO9YVEqITYApj38tuHvGbzWe8dcKzzvKsBaLlJE7fV\nF94kYC/eNrba8VF72zF1RD8AAlfuACDsK7EEeyNUOn53Ox1HSGK3u5+QEDBXCwm9jKzjcivVscdL\nqCvh1pKQjVtqdL13iZZ3q8j3e0mi6Eu+/oDJHWTb5X6nSAhu+OcaoVKXeCOA2o84eD2nlFohl8P2\nHUv6TT01jZHy1nEAvJf6ddWxa2/wRkMEHHD+9f+R7Vv/00e2G2z/Wcnh/9gC3T6ypqROtaJh79h3\nbFl5CwAyP295wPlDBl8LwKxDLA96OLMHAFuKYwGY0OZHAP7b/A/WfiARZBd8cw8A7UZqgujGQNv7\nRA6fvn4KAI8/Fs+qc97423NPfGIRf06ut6I1aaI+mm+9yuczrx4JQElzG6cO/fsoLxuyqcW1cXOq\nlgpVYeqyV1/j/m4/7hd1JIx8WhrsmHm6EUZ9UxpXPebDMP55ExnPqcezYYDMM4enny4H62m8ohEq\niqIoiqIoiqIoiqIoNeQfI1QMw0gBPgTiARMYY5rmKMMwYoAJQCtgK3C5aZp1mpDEsVzWlbb7RPIv\nrLlyNE7jn7dDSreS2Z415kEAUp6RNZNNJQlfY5Bh0huS46Z/sESWDN0yAICicysJdEqUw/rXxEu3\noKV4VTt8dh8AHR9fhqdM1rW2fqTpWokbUo45N0vum843yoLwzBLLHdP/KG+8cMU/n+NHNAZdrAmZ\nvYIbugiNksYox8CtOQB0/vhOfhjyAgCpjoPL79oISZx57aWWd/zSw/+t85N6HFEZGxP1LUPPCsmP\n0fX3a1h+orjBLwmV217yr7+PUDgUQ7cMoEiCizBLJLqo70AZGzUbuZkv204H4MfBkqj/bJfkR2p7\nv39FqjRGXTwcNgxPAuDaXr8d9JzjQ7fzJyn1VaQGozHKMPJj0ZNIYNPLhz73zqlXMvv46lsnR278\nZ2+6v9EY5Xg4GD0loX7noI/wWFFHj++RPi72Y0nC31Sk2ZhkmPijbGLBI/LitB88Otqb9H3r7VTJ\ncO63XQFIYd5Br6tNDidCxQXcZ5pmZ6A3cIdhGJ2Bh4FfTNNsD/xifVYaJypD/0Dl6PuoDP0DlaPv\nozL0D1SOvo/K0D9QOfo+KsMj5B8jVEzTzAAyrPeFhmGsAZKAC4G+1mnjgZnAQ3VSSgtPoWTW9u4u\ncdL6Oxl05ywAbomWnAzeLZX3p89vkpOh3TP1Y6VqbDQGGcYEyNZ/yypcABSMlAzoNvtOHJMk2mFd\nO8l3c9aaocA+T1pTiST6JxpSjjknVwDwXqpsbf363vbyYyMGApAwqma6tfd6iXgpSpH1qk5jGV8U\nxQAQvtl/t4lsDLpYE0oSDvTLFCVKVGBQfRemEdEY5ejaIlGAbR7axl3jrpeDDpFV5imiWyfctOyg\n1ycESn6jx+P+rPLO5VVWj3BZ+IHk1GleTx6fuqTeZegR71rATxFwYs0vT5shiYyiZ4jmNf9yVVVO\nKi8hk2THptIfQujyoeRgWXHShwAM7LMUgE1Bcr036tPXaYy6aI+OBmDn9Z2qjvUaIrugjU2ZC0Cl\nuWT/K6pd7428fuehSwjG/3f5aYwyPFq8OVmaEr4qx3XDZTeZ/Xf4mbhK+rp2lUsbpEwNRWOSoXu9\nrEpZWykrG6Yf+ym9Hr8XgNRnpV20xcrYJuu9KABWd3uP/isvrXZOfUUX1SgprWEYrYBuwO9AvFXx\nALuR8KC/u+YW4BaAIA5/+7/DIXbsfOaNlcn491dLOOs1j04F4LqIDQAc99VI0h4UhWgqIVuHor5l\naLce9i4hkhTo5hXXABDRTAbqca/YmdBmGgBnrpZtPINvk8ApTX15cOpdF60Eax7LvHVH9DoArr1f\nluxMHt6W5/88C4CYyXLvnAskDD12iny+9V+TcJsi2zNCZElCC4d0ZJWmjfd2nCa/1USWATW29vRw\nCctQzdyfxihH97rqCdfjrK1bt445+DXpibK04Pgb+tDynbVyn5zcauf4gyHl76hPGSZ+s4W0U8Q4\nsr5f9aT5XX+X/rFk1z7HUKcXdgHQbrtlDDNlJHMoLfSUlJA6VJYYdX5Ckip+c/VLANwXdI6c5CcG\nlf2p9/FNWlsA1t4pCaFP7SnbVkc4pO+bmPjqAddUmnbrdZ8Esz3isJhdKsue33noEgDC5m9tcuOg\nxtie/hNF85thO756wP+ml3oD+5IQNzV8QY62rmLw/PoM75JLByN2SbLoDnduBpr2PKSxyHD4vZIg\n+vvXRrH8ttcB2HKz9F9BVqLaJLv81oO7exJ+q0jN5XLVyu8fLoedlNYwjDDgK2CkaZrV3CKmaZoc\nxF5hmuYY0zRPME3zBCdHtqe0UjuoDP0DlaPvozL0D1SOvo/K0D9QOfo+KkP/QOXo+6gMa85hRagY\nhuFEKvYT0zQnWYczDcNINE0zwzCMRGBPXRXycPAmkJrysWwfOAV5bc8CjUyh4WTo9XCuKBHv56Ie\nn8kX78pLqVlB76USmtzsAcvztnFDbRfDb2goOaa9LSF3nUruBOD1geMBCLeJlfjqiB1ce+r7AHhO\n/csiLSvwxIatKsIFq6H9vdwJwE2/X0fKWGmOHPj3dsm+0J4ejGx3KUGZpQ1djEaBL8vx73Bl7AYg\n+b+7m4xXriFk6MrYTburpa7PpXu175JYdeD5R/g7ZqVEPbR6XJYejHz8ZOub/CO8Y+OloXTx2m9/\nBeCCUEme6F2qs3/0ycFYUymvV0y6m9Dt4tv0Lp31LvNpKnoIvt2eNl9Sud/YRriwr8hwZUMUqAHx\nJTluGyRL844J2DcV3lMm0YHuvOwGKVNjoLHJ0LuU9RxGEHn3dgCiAmQcGmyXhnTBV5KANmnUEszy\n7fVVtGr8Y4SKYRgG8B6wxjTN/XNdTwGus95fB0yu/eIptYHK0D9QOfo+KkP/QOXo+6gM/QOVo++j\nMvQPVI6+j8rwyDmcCJVTgGuAFYZheDPaPQo8C3xhGMYwYBtwed0UUakFGlyGX82WLHznnS+J2cZn\nyTrFrU92JOaHRUDT8sYcIQ0nRyuvSXsrP94bnS8AoDJG1i3u7LtvreTYG2Xb6xOsBF+nLr0KgOIF\ncQfcNmmmrDVvPefgyTL9jAbXxZrQ+juJQOpkSB6GNpNKMRYub8giNRZ8So7K36Iy9A8anRy9OVEe\n3HHBAd8tWpQGQIc3xMHbdkPTzLHxFxqdDGtCyMJNvJrbGYCRMasBmPqd5FBp6ad5pw6CT8mxtMWB\n8X+bP5UNF5rRZCNUGq0MQyb9TqUVL5P1l+9aWHrWkCtSDmeXnzmAcZCv+9ducZS6QGXoH6gcfR+V\noX+gcvR9VIb+gcrR91EZ+gcqR99HZXjk1GiXH0U5UtqPEC/M/0YcZx2RLbADWNRAJVKOBvdq2UHC\nu2YwZc6+7/79TPW8ADGsr/ZAOIGRAAAgAElEQVSq+A62WbJDWttZDVwQRVGURsaLLwwB4H+hMv8o\nOVEiLtkuuxi2efjArXPbIWMhjcj1H9w5uXzy/gAARt63uoFLoxwuP5/nXdESXHUsakNFwxRG8XkO\ne5cfRVEURVEURVEURVEURdAIFUVRFEVRFEWpAbHvHhiBojRNEl+WHA4XvNwTaHK5U3yKvGtPAiDO\nXl1/H9/Tg+C1sgPbke6upjRd1KCiKIqiKIqiKIqi+DXZ3SR1aYgRUO34Dx+eTEK6GsKUI0OX/CiK\noiiKoiiKoiiKotQQjVBRFEVRFEVRFEVR/Jpjum+t9vmJPd0ASP5iqy71UY4YjVBRFEVRFEVRFEVR\nFEWpIYZpmvX3Y4aRBRQD2fX2o0dPHNXL29I0zWYNVZiGxk9kCCpHf5CjytD3ZQgqR3+Qo8rQ92UI\nKkd/kKPK0PdlCCpHf5CjytD3ZQiHKcd6NagAGIax2DTNE+r1R48CXytvfeBrdeJr5a0vfK1efK28\n9YGv1Ymvlbe+8LV68bXy1ge+Vie+Vt76wtfqxdfKWx/4Wp34WnnrC1+rF18rb33ga3VyNOXVJT+K\noiiKoiiKoiiKoig1RA0qiqIoiqIoiqIoiqIoNaQhDCpjGuA3jwZfK2994Gt14mvlrS98rV58rbz1\nga/Via+Vt77wtXrxtfLWB75WJ75W3vrC1+rF18pbH/hanfhaeesLX6sXXytvfeBrdXLE5a33HCqK\noiiKoiiKoiiKoii+ji75URRFURRFURRFURRFqSFqUFEURVEURVEURVEURakhR2VQMQxjoGEY6wzD\n2GgYxsO1dW5DYBhGimEYMwzDWG0YxirDMEZYx580DGOnYRjLrL9zG7qstY3K0fdRGfoHKkffR2Xo\nH6gcfR+VoX+gcvR9VIb+gcrxEJimeUR/gB3YBLQBAoDlQOejPbeh/oBEoLv1PhxYD3QGngTub+jy\n1eH/rXL08T+VoX/8qRx9/09l6B9/Kkff/1MZ+sefytH3/1SG/vGncjz03xEnpTUM4yTgSdM0z7Y+\nPwJgmub/DnauIyDkrMCQGNyxbgA6hmazoTQGAJfbjlwv19iLbbgDrOu9x0rl1RVi3dhh4syTt7bm\nlfJdnnWRx7qm0iQwsQyAokr5zpErv1UZYWIrN+R35YVWcXsA2JLfHICAfBNXnNzMLLFTmZeLu6TY\nONx6auzUVI624JB5zqgYwiJFGCZQ6ZH6jHBIPWeVhMv5FQbtYjMB2FQSB4BjjwRFVcTKPZ35RpXQ\nXTHyGhUo9y7eKYJ2NfMQ6HDJdd7npMgh10dUEGiX7wpKggEID5ZylOTKZ0+YB1uhFYxlPUul2enZ\npmk2q0ldNVaORBedkUFnBSdE4NntlC/iXQTbRYdC7eUA7M6Olms8ENcsH4CcilAAPAVS/6aIA3sZ\nmN4qlq9o1czSpWzRJWexSUWMVQ6bCMIolosCCtx4kkXP3HlSJneInOMoFHVzR3kIsJ6D8nInrpy9\nuAubsC6GhMxzRsdUtXVmgEnzsAIA9lg6iFuqx1Zh4AmU+rSXyjF7uXyuDJXPgXvdVESIQD3BlqJU\nyneWamOrNKmMs74rF9l55e4MqiTI0sUSl7S17jK5n7cND9xdhqu1PCBuj1xYvnlXk9ZFe1DIWc7I\nmCpd+juCw0UnK7ID8URJ/+kpt+pWPmKpLfYKDxXhlmyCPNZvyXe2IlvVNd4+Ly5edHtPbqR856Gq\nnYyMLQag2JJnRanoprMYKsP2lc+Vm4u7qOnqoj04ZJ4zMobASEtOuYEExlj9UGkgAGEh8rk4P7hK\nH2zhoi/e9tRRYulkuCFyABLi9gL72uOAPLkGlxtnOzmpqCC4WplMG7SMygZg+17pe4PCpGyl5fvG\nSIZDfs+02omKbTubtC46I4POCkkIx201agE2F0VF1es2oNCSUYhR1dcFFMgxd4DVV3nbT7tJ4FaR\ne0WiNXC1dDIgS86tDDWwSddLyxYyXtqcGw+ALdhdNSY2S0XfY6IKAcjLDq8qk8fqxm3Bbioy83AV\nlDRZXXREBM0LSoikolLqKy1sD+sLpD7xjjusvis4vJyyfNFPb/trWjph36858+qrK/wvYxK5FGex\nidlc9DIxUNrT7fnWmDfIhdsar3rl7L1fSPMSAIJslRS6ggCoyJGbNvUxqi0s5CxHbDTNwuR5z94b\nUfXs5+bJsx8XJeOdrNIw8IhMAqw5oVcXY5rJObl7IrBHiwCSg6RN3VpiTUSKRPg2F7gsdXcEizzt\nO+VZcQfZcEeJ7nrnE169C9xjtftxgfvGwTarXyxuuv2iIyJ4XkB8ZFUbZgBRATK/q7AUrjAvZN81\nXnOE9eoJsvStSKrQ8OwbdxhWN+gMt+b+BSKM6JhCSi3jQUKA6OI2y85g5DpITsoCYNd20c/4lFwA\n0nPlWXAWm3gc8nveZ6Ei/fB00fFPJxyCJGDHfp/TgRP/epJhGLcADwERNnsAXfuPIO8aUYpfe77H\noFVXArB7ryiI22oEwxcGU9RSHl67ZfSIXiOVm9Vd7m3GVpA8Sf6FwLsz5LvJKfKPlcq5obvdtHls\nDQC/p7cEIOpLkUjGmW5CNosQvI3puBtfB+DKabcDkPq9h6wbpdEzl0ay9d2XD7uCfIR/lKMlw1uA\naFtAIK1uupcTz18BgNs02FMqsuvXbD0Aby/qA0BgupOJ170IwMVLbgGg+ZvSaWy9RuSTMC0AR7nI\nefcQGXhcmCb3XvTYCQDk3lpEuxgZHG7Ll0Fl+VxRhuSzttEyTBTixz+PAaDvsesAWPppFzn3tEKC\nfxWZ2ywlXDrmvvQa1FFjp8a6aA9ycsqYKyh80dKXkbs5Jkp06MTwzQD87/0r5LsSuHn4twB8vK0X\nAEW/ygClIlLkGL3GrJqYl0fL6/jhrwJwzXsjAYhfXMm2IdZgMqRCXn+XZyfplzxKnxc9K/wqEYDc\nXtJQJvwsOp53cTEtY0XW6za2YPczr9WginyCGuti8t33VA3uSltWMuKUnwAYvbwvAO4C6VhCtzgo\nbiv1GfWntHmRW+Tz7hOlfltPyid9QBQAxceKLtozZHAXvVZ+P3S3i53XWzP3rWJcc4WITJM77KFD\nlBjRFu+W56pwveirXcRN2xfXkjVK+qX8QumtNg35vyatizZnIG2vuZeKiP2+9w4o7PKmS98NAGwb\n156KC2XEWLxRDCABeTKCi9wkcgjfXsaOMy1jdAfRKbtDrC5Bs0XfAgpMPFbvf8N93wHwxmeDAHAU\nUzWZP+va+QAsypa+M3256GbzxSaZveUc026S8fyof64Z36JGumg4A2k17F46DLTk9Ek72l4j/eGi\nP9sCcFo3UaLF3x1b1Q+F95UJdNEv0p42/0N0a2ffgCrD56PXTwD2tcepX4uOkVdA4niR75yfu1Qr\nvDsQXhv8LgB3fnETAJ1PkXZ9+SbRTSpsOKNEzyuLpJ3YPuzhJq2LjiAHp4+9nMJKafeSQvOYN7ez\nnGdN2JJmivD2dHdSFi96lfKj6GlRCxlI5h4vxx2RFbS9Qcaf227tAYCnU5Fc85YoYGavIEJ2y/Xv\nPCV95uWfSZ8ZesxeXJbh2bVc2ubLB88CYMrYPlX/Q3GSNd7tvJcN9753mNXjM9RIF+1BTrq/eTU7\nsqTvmXTKaM6cfjcANmuS7Ngm49AufTawZloaIA5WgErLORvzu7Pq/l6HQtYZ0pHFzRR9KWgjx+MX\nunHfKWPUx9tNBeCOb28AoHnHLPLniH4HZ8lveA0r3W9bBkCn0AxmZneQf+4juenSsU17jGoEBJDw\n+N3cfvKvAIybeDZDBs8E4NMp8uzfeNHPALy74hRcpaJPqV+LvhSmiC5ecbuc88XrZxJ+qYxxX2r/\nBQA3LL8OAHOuPCshmSY5XUVGzTrLxDvyX9KX5rcPJecCaW9DZ8t8oiRRzm07aiMA24a1x20ZAVwh\nJjtfefXwa8g3qNkYNcjJsa9fR3ml5TCwe7ggReZ3u8qlPZvxrUzoDfe+cYdXP4rS5E3z2ZYDvdRD\nxqmWAzBb5BzfZycA2T8lAXDpVTNZUdACgEeSpwFw84prAAiYEM3zT78NwL/uvBmAe0d9AsCDn8mz\nkLDARXG8/N7eY0WWW+69/7B08WgMKoeFaZpjDMPIBQY6g8KGuZ0GV7dbBMDxU0ZUnZf8k1RSyY1i\nOQw4t4AKqxF6+QbpILwNlMfy6gw9bjHH9RbZlpnS+L3QR0alpavltTjJweURWwFYZEsFIPsisZDZ\n00MoaWmNbKyB6yObLgbAuVeElXWcncoKqabK1ErMgKa3zbRpmmOAMYZhXBpSFvBlqwm7yThD6ndL\nViyBv0vj8lGADLqvHzoTgC/X9+XcOXcC4Lash3tOsAxgm+Tel/9rGmPWnApAxA8y2J/kOh4Axyly\nbti0KJYdJ7/Xq5sMWHPmy2TOHGBgt2YfhuXB2VIglkbvhMG1JYx2V8ngdvEmKSNjSDm6WvE99tdF\njzNy2LqFrXCdLwO/kF+S2dxG9G35lG4A3PacGFHe/mAQo7+QyVbXATIp+DNQzj174GIAki7MI84h\nhtLxj1wAwHM7z5EftuzznZ5eQWChyGbLXNHFoGyrQxq7iZ+3yMBm4C3SPqSXSIO7PLs9AK78QNLt\ncix1ikFuXi1Uio+xvy7aIkO+rIyrxKj0uidN3v5K6jxG5k5knW4ZUTa5cVjRBfEXbwMg+1ORQctT\npA094+I1jP/4bACCNsiA09uxXXCvDGo+WHES4cEyqDxjwEoAvpktRrYdO2I5sdlWAPIyRZcdSVYk\n23Yxnuwe0pG8TZYRrk2u999q0rpoRIYOK403aX6cTK5dn8Szp49UfOendgOweafoQEClSftYGehl\nBMnk+5JkGZSPnjEAgJwLPbj2iF4HrpfB4KnnLAdg5ubjAIi9cCfxwaKvY96xDCneaLM+udimygDz\nz70ySMmfIgMUd2drQnJjFm2flDY5v30IWSW1USu+xf66aA8N/dI0YNlWeZTNrm7KpsoEqdVSkeXO\niSLDBFs5g0fJID/SLhFAT3c/H4A9NpHX/10xgeffFgPK6M19AQg7XQwpa1qLbCLinXRyyiRu7JVv\nAfDQY8MBKI80mF8sv+d1SLUPl+vL3hBnxH8mvl810CzYWeUlbNK66LFHDVvzW5uq6Mjs9GSirIiU\noDwZ7e8Y4I0M82AGi54FjBTdDRktehLUWuSSEp7H4v9Kfxq+RX7vymPmAvDlA9ZE4pcgPJfnAPDy\nbtHh5JnyzGyNjCQqRTq6gF1Sjs+/PV1+40wZKxemRxCUIM9RQUEwbrffOMQPm/110QyK+HLrpnie\nO0MMkReNfhDaSrvV7pqlAKx/W/qsjV+kUd5b+qjQhdJHVcZI/Z12q4xDkgL38tYfMoEPXiP9YlYv\nuV/4Rmvi1cFBoEve371oCAApP8uzYfwQjduaeGf1lmO2UJHvttvF2PrT3Z0IsSLIIguq5hhNWhdT\nW1QMe6bPRF58UeozKAA+mC9zhU6jZQLxaRdxuhrbg7ElidUr5VExbizNkL4rzC7H955cQdEicQhc\nuVeMzMFzZe5Sfpr0hSkJGVwcJeOjsX/Kb0XnS4RL5kAnL3T/BoBHN10FQIvZ8hxsu0na2pi1bjJP\nkLnjCSetJ29sWa3Uiy+xvy4azogvc5Y3JzBHdGpvWxfffN0PgNh3xVnjvE+uM1wQuVXqM+tq0Umn\nS+pyTz/RiW7ttpGfIw654PaiQ9elyH2e6ylj1wkT++I+VozWN+VeC4DnV4lQibt5C9d/fysAQd2l\nHR/x09Xy2TJ0Z9jCuPFC6Z8/Hj/A+28dli4eTVLanX/5kWTr2OGcqzQeaipHfyG0oQtQi6gu+geq\ni76P6qJ/oLro+6gu+geqi76P6qJ/oLp4CI4mQmUR0N4wjNZIxQ0BrjzUuZWhkNXDYMIWscy3bLeH\nzLli0U8/34oJz5Aw5vO6/clvPSWs7vnhYkGKbm2tOQ2R4zOS2jN1umWxvEw85wG/yvVmP7Eq2hdG\n8M574oH7160S2jMzvxMArbtkMea7swCItCImdu2R8jx9xacAPDzvUuzWYvOwDU5sZX5n/a+RHMsS\nHax9LBpzvVj34pLzyDlOPN9Grrz+ltVOzu6VzyOdfgHgpU8l8if+TImc2rZUrMcl7kAqtoqVOPXK\n7QCU/yBRJG7L8V7SAkyneIc250mEw7vjxSP3R1kqT0+Te4dvE/vgtkCxYN5+43QA3px/Busmircw\nsl+VV7z0sGrHN6ixLmI3ccVWEj9DmoCss0uJmi8emu2DRRdj7WKxrYg0qYyU+l8yW+ox/lTxnM97\nWzwE190zjVffEzm0vkfCI7a+KxEngdaax1+/70bqD+LKPv/13wF4pPlsACYVtWdqoYStL/mPhEYH\n3bFLXtNkHWRRdij2WaLfvf49nw1X+p1bvGZytJnYAt2EbxdFSTtjE45jxAO26xHRweselCVA7/4x\niKCB4qHe+b3oV9BF4kX1/Fv0Zew5KbhSrIilRJF9/GjxyL3XWryi0cvthO+Q3/s1VdZ8XDRsAQCT\n/ujBNz/KsQ4TpP3d8rC0Ez1Ok/Y5qn8p0+dKBFrz/6vqfpq0LjpKIWYlFLSXujYHFxAzXaI/Njwv\n7V2beGkbu0bvZOJMqeOw7dLevdNb+kDvuuPLOy7l6+WnAVB+rFTtsnckMiXGCsjMTU+m5Dzxqnvz\nP5ScLzK7M+03vokQGZU+K/1h8fXiAU+IkuciuzAUx8nWkq9gqvKd+RE1kqNph4poDzcePw+Aj9b0\n4vvz3wRgwIcPAGDvKF5Qj8fg42dl04DAQtG3ROtGe7qLLKbnHkPcedJXZsxMBiAs3YrsstZ1l8VG\ns/DDngDkPiiy6DxCosZeTv6JU14X159hqdm8Pa0BCHhWZLiqvAUlZSK4dp9K2bY0cV20h7iI6JZD\n7l6pz8KAAAwrj1RQtrRl7Y4TXdwxI5XE46VNLaqQeswdItXX7mbpQxePSKvKXVXeV/Tru4f7A9Dt\nXxL6Pt9ohnuq6PmCVHld/4GMb1pPvoXyBXKs5WTxvBe9lgBAdIj8Vll+FK4CiQhM/MNDVt5Rbd7Z\nGKmRHG3lBmGbHLycKt7li6+exReTJMLEFi71FJ8q48Cu3XcyY7NEF0SfJ+MNz4/S5s1ZKLoVdFkm\noSulbe59iUT6edtTm0tkW9DGIPgLkZMjVZ6XS56XpT9Ow8Xra/oCkPaCPENZj0o0ypbB4jlvMdlD\nWbQsMytOqJpjNGld3JUTw78/Hkra1TI5WzW/DVErrcbsc3kt/V0ilsO2Qb8BIpvf3pAVKK2v3QrA\n1D4dAbCNcnPrhTIn+GiTRCiFpVtRRAES7bBkY0uWlFrruKxcOm//PB6A2zdfxspSaYu9KSJOf07a\n+0mbuwJQWBRFoASOUTgkBPfupq2LhgsCcwxKWsj84cKef7ColURGb25/EgBBaRKB5/w+ksyeUrFR\nU2TSsOc0GbC0/0Da06AXXJQUi544fhbZv1FyCQA9b1kFwLbxHSnOkPY7JEv0Nv8GEUqIo4KwJGmH\nY6fIORnDRBfDrRxnua0dVVFNjn0aeFi6eMQGFdM0XYZh3AlMR7L5jjNNc9WhznWUMjV6FRRZYf+x\nM0qoHGzlVMiQQfqoIeMAuO23a6oSV5rdpALLuskE6ox2kiNj1tZ2/Of2jwF4+BsJwQqxJuGlVkLS\nlP4ZlFoh8Q/PuxSAW3v8BsCnm3qS1F3W1O1MFeFE/yjXPbxAhGTLdmIi10+963ku+D67plXVqKmp\nHMOjkkn5ys52axOpgmWxxB4v4arNH5OBWv/pMnl664+zGf2zTLJLTxCFKH1XOquJ/5O1hVtdsbTu\nJobMdZvku5YrpXHLbyP1nte9grOOlSI9liATxL7fymAxYp0dK08YV90qjeXqIrnP2z9Lh2qLK8de\nJo96Oytc/s/q6wB9miPRxYA8pqZ+Y2PHmdbxYifFqVKRMQuk3p/aPhSAytZlpCbKACT7F6lb5ysy\nEHDeL4OQ9oG7qxK0rVhn5WWRfgxPoOi4J8JF0stibJnyk3R6rQbJszNqeT+cofKMpJ8vg9O4UkuH\no6TBXZMZRsWpMvDfVBRHmafOVyzWKzWVY0izFKJnBlE5SOpn0zsdKRksxqegNiKMNz8SY3KACTkr\nxHDSYo10UqV9Rd577pHOI8xWQn6BhP6XbpeBZ1GSDO6apUi7V7ytGa6zRQbNwmSS/fUqmXzbgl24\nrKTDO/8l9w75RTrGLTliiCtMtWFF07P9XGlzWdq0ddGMdk2tvDSXPolbAQh1lDP/Ypn8HhMh+jFn\ngxjIts1siaVOOC1DSNkGy3mSLPqzLC+5ai1y52Tp33YEyyDRq6OxqyrYtcRKFt3cWm5bIIOP57+7\nkLjlcu/My+VZSZkgF+a1kd+qTDCpjJdzmh+XSfrnlTWppkZPTeUY2DoZT7MKJnx0BgDJS8oZuPJB\nADydZNxSsVfaMyPAzYkjZKnkj5ulkYyYalmdO0ofGuaoqMppVWL1i7vPkjq2Wc4FZ4CL9GS5Z8Ym\neT5sVm6IC+d0Ivl+CV3fvFAGsJEjrESKYXLNh+EX4D5X5LrpCuufWdzEdbHAMdX1QxzxWVLHhS1t\nJP8gfV9uV1lqdVozMWzMHw9rEsQx5F3W2vZz0cH234jDYf1PKZjR1nKTZtKGbuoiuvjTApmUm2mV\n2K2kxAEFcqOuC6XvjVlqJ+IyGR9tP0Ha78R3RWZFLcQMF+qEsIvk97JLEnHNqWFFNXJqKsfghBRs\nLqhwyeRs0sd96H/5EgDmZouzpihPdHLurG4EWynBdkeLDjqsYcXV934PwOs/DaRZXzGc/bJW+rEg\nq1+ssBxNps0kMF/e5zvlBqO/2W+n1XbSV5Y+JX1neZH0syeeIf/G7IQ0opZYSTp7Vc3dmrQuBhR6\npib9Vsr2njJOaPNNMRuutKzJE1sBEHimjH0S3y7i57NFNrYQkc2mLCsZ9xUylhnccT7vThgIwPLh\nkivz2DJJI9E3UdrKGaXtqbSSvUcvFTl+dpIs2dtbFkyR1YFWRogh5sfnxHFR3NvKjdPSg81a5rwm\nNYmy5/zL01DjfjElhbJmJqZT6mfl/cfR8hlZ+5iXV90wXF4ayd0XSz63sW/JuNWbRPr28V8B4DZt\nbCmQQWZ+P+kPC8tlvrLqA8mhGTVyJ2U/ieErVHxG9EuRVBFz3uyJ8yIZU1WGiZxbvizPS+SL0s7b\nP44lv4uVg25fvtzD0sWjmpGYpjkNmHa454bGaRRXY6QmcvQj/GoGUFNdDI9KruMSKUeC6qLvU1Nd\nDGmf+M8nKvWO6qLvU2NdjNcxamNEddH3qakuRoQn1XGJlCNBdfHg1KuL1zTEQ1aaJBb7Tq+uIv1b\nCauL3CjW3du/vx6A8K12hlwny0U+CpXwrDYxErYz4xfxiHoc8NAOiTbyWFvRYYjVKWinWK1u7zOT\nJydIUiOsLa++3SlLC/J2RRAxVUKrQ62tPPPPEQ+RWSyWxcfPn8Rrr0q0ymf53ch1z62FmvBd3AEG\nhcl24lLEymek7kvSm3muLCV4e4q8Nl9mki2RcKROEu/Yl2+9BEDvKfcCcEHvJaTnigU6fI0VXmRa\nlseTxHLZLKqIlkFiPbzyXolMsVnLUqI2OdjRX2T+1q8SbuFoLtfFLBfLoycgiKgNcv6ilW2Ptgr8\ngoikIgb85zcmbJLld08fO4VX7hFdMtxWZJhNdCJicyDbeos3m/ZSjz/cIRFGZ78lXtjwm0txWitw\nLuopy3nm/p+1NGGFeFo3X5fC0mWie4aVpG3UNPHiuMPdxLcSD96uUvEEmpOt5MKL5blITYKiJPEi\nLevShlJrO9KmimmH8hiD2HekTlwjMnHPFqt/Tndra/pjxLC+9edWuBOtHUSGWksHrKWMBbkSdRCf\nkEfHRySCa8u14tWuvEz0vGCNyMKIMnF7t2j9TeQU2Ud0s6QsgOAV8l1pc1maVXSstPVt2okHNfSZ\nOJx7RT8z+kTXSj34Ou5KO3szIsiLk7qbtvoYUiaK7s25RLwxjkCpR8MNFZ2tUP/m0kdtvvQdADrO\nkQSjq7cnEmE5OVdsFMNpp19FrjsGiZc7+7gAIjfKcxC7VPrV4PMsD/qf7SloJc9G68/kHO9OUK2+\nEDnm9I7HZrUTxekJePL37YjRFAnIhVaf2Miwdj7aO7KIcmsHwcA14uYyrV1E3OEGP2yQnWPYJt+V\nXChhyBVWlNCesjCKRkri7/yB0nc2/7W6tzP3WOhxsnjesv8rEU3p/a0lCKkOEgKt5PvWMuVWH8sS\nohX/lU55Z1+jKnHmpc9KdOfIB46mFvwAEwyXSUErqfMbr/mBN4+X5Y6BK6UeJ74rUUgMBnuR1HfY\ndvku8z6J9tvxnYxZHZ0LcS4XD/mmKKs/s1TFuztXeaKH0PbiafduJV+yXsZESekuij+RCKXS46zk\nuJkS5bD7emv3nx2h5K+SZ8VsX4YZ2PQ2TtgfTwAUtnZjpksfdPM1vzDuR0mEGW8l63YtFb2LWeti\n+4UyLwjaLvqVdq7o1PoS6UuDMm1kBko7HNVC9DRslQgxp7OMPcuaeagIs3YdWWT1eY/L7k7rXj2G\n6Pfkug03ifE8sJNEkqY/Icuio9oFELlVxr1drpElLluPvip8mvIYg41DnfRtLlHQpS85cf4qUShR\nG2UcWlwkMl79SChRhhX9YwW+xkfK523WCoeJC3vSYq3IP+1H2XW04yhpI1emSZtYOdBFQI7I1LuD\nU6W1vidzZXOWfic375Qu/ema+6U/jZ/t3WmvhC0jpC1o2XE3OcF+ZROrOU4PnsQyYmZJv3bGqLl8\n+6zoYvMckWHRKZYuDSjnxd9kUwXjBCuZb5nU/aPLLwLA7bYROkPGqxcPl1C8n16SJc/edjV3UjLF\nPWWsW9xSrs/4SewMLa7IYPcC0UG3FWV2+uOS1H/aOLlPiOlh/LfSxgfUMMOH3y3wUhRFURRFURRF\nURRFqWvqNULFEVNBs1HTCsUAACAASURBVCHb6RspHutJS7sTaeUIzT7PskhVio0nep3Ju7P7AlRt\nTZc/Wbxtg+4RD/hvo0+koJ2YkNp1E09sYZpYIyu+EIv9Y9OuIKhUzjHjrbXHdrlffMtcXCGyzm74\nbZMBeGvMhQC4Wsm5b/33Ery+t6GRS/nM7neJMGuELaqSkAszybU84c6ee6m01qred+/XALz6vuRN\nMdwebJY8s2+QNaSnfHq/3ChU6nfW+72wW8vHT7xCkkr9OVrWFndvKfk2Hk2axuAfZftlo6+c67DJ\n9af+ZwG/PH8KAD3v/QOA38aLNTK7t7VlbEIhWzeLZblDmjwn24+yHnydnPxwPvyuH84O4jm5f8rV\ntM4THdx2vnjKHUWiN2UGjBoguYrunyh7tZ82TSKMOk4VBb4x5A5iM0Qmf+ZLqObem8RD4Aiz7rO9\nAvNYsRwf30K8Dqv3yHNkLoqkLMlqjqxtH5vPEG94v2/+BOCr7d0oXiSRMqf2WMP3If6Us63meAKh\nuKUbW4W0UOUL44k+VRaNVuRa25rPaAVAh7M2kVUqlv3MlVKHzVrIuaFfipfnkhcX8dEZEjH02LWy\n5eRHV8tn12mW5zTR5LIuome/TxA9y6sU713CFhd7xeFGZUuRc+xsaY/Ts+WZSH58J3u+lPedrxAP\n3opRR10Vvo3HwCi3se5dSZbubAO5nUQHQiMlYrLF4NUAhM+OY/lcSaJosxLjtf7uZgBiF4n+OItM\nQjNEl0sSpf7X3CvtX1yiRKHsXRtDRYSVbHOonLtpp3jCB161iG+Xi8duS6rIvf0H0n5vv0T0tey4\nElLHSQHy2gfisddKTfgu8S4892dzcwtpqz4YN5DofCvZr1QryTPFc93vv3MZ/5usvQ/PEBmc0FfW\n8G+ynoEsWxuMBCtHTprIJ2mOfM5/UJ6JwNnN2JYvume/S6KMjg+XSIfNabF0CBP9tg2Q65a+IpG9\neceJTANbFBK8S9rsV5b0t/6RX462JnyapPhsnr7/fTaXy/jxj8JUAlZLNENZnNRVpaU3houqqL+y\nUmubebflqbaiv7JaO4nZLe+9vZU7SD4nzbK2Rh5so3S16Kc3muiBobI966ezziPrJHluOrSXPrNg\noSxLOq2lJLWdvaYrbV6TvHXbbu3ojxsn1BzTICpJokAmjOtP/E4Z85fFiHy8SbR397YTsFsar/Jm\nck50gEhq0x3Szg5/fyrf3CXRz7H/lsiEjfESXRmSKbI03DaKLpHfy94sEUkdbZbcRq5i5lLR64AY\naUfdi0XeW8+X3+zdYw35V8hztudOXXYGSF4iu8lsKz9U24QsotbLV7ZKS79Okvpr1243Tpu839RF\nhLtth8ztAnbJ+Kgy0lOVW8wZZEV8Zkh/WNlVxkBxc5w4LpN8Obt3SNs6brYkNDYj3Gw9z0qI2lIm\nLYGWVmeeKmPmkssqCZwjz8aOxGAqSpp25CYeA0+Jgy43SbL0Tz7vT0yZtXHCqVI3YVMkcs+WZlYl\n8G79pbRhO/pbY5rV1kqSApPwndJuTv5Y+lBvRvfiY61cgBGlONeKfnlz3YR1lnnKrj8SGXSe2A/+\nzJNx6NIbjpV7PyvzjZ7NtrP2JomE2nlmZI3+XY1QURRFURRFURRFURRFqSH1GqFSXu5k3eZEMlZJ\njg1bqgdnsVikwueIddZxrlgM93SPI66VWIPtVjTC1Y8uBODPIolUuf6B73jt6/MB2GlttVvUQdZl\nBVipMoJbFnJRP/EaTVwvHpqsQrEutorJZXuyeGs/2i67jvS7Rn6j1Nqzd8X8rnS/fykAZ33wANtz\nXqmVuvBVXG47e/LCaD9ePGqBZ1ewapZYkCc3k/p1FopMK0NtnDxQ6j79blnjXdDWWuPdSrwC4+9/\nmQun3wXAz6tlx4NB1g4Ic96VLXn/faWN0GYSGdSzhcSWLB8vVsX1Kc1xWwvdvpsv+UAizxRrZI8Y\neZbS32nHQ49NkTIO7FE7FeHjGG4IyDcozpSohYDkEtxB8szHWjt85FrL/JNmV/L2xPMAaBMiES1B\nL0vdLr9N9M5RAJ889yIAg1+SvCplJ1le7XWWd3wdDL33VwA+f/FsABK2iodvb5pJ8VKxVAe5RJ7l\nb4t1eexKiUBq+0wFRaLurB53DKXZP9ZGVfgstnII3WanOFXax4Rj9rB3lkQQGF1EX665VDzOX2/v\nSna65QW1HJjtwqV9nXam5Et5fUUfgmLly2ffk20/yu6R+7jKRU4922/li2WiQ1vGjgXgtDtuBSCz\nl73KA3Rn95kAvLtBsuq7Lc/D1hUtMI+Rk5Z/16kWasH3sZdB1GobeeIUIbXHTnYsEu9J5V7pF1ss\nEK/nrHWpRG2V87zryFs+I2672fnSJtrcBq6rJIrBWC4etSBrq+uyDeK1s/cowtwsup81T1w8lUni\n+Zn9fk+MzqJ7gVnSTmc/LHqfFiVe8sL/S2brrXKOx12K6c1h1kQxsxx43mjOW1dIvg1PqofCcPGC\nNksQz/Xwq3+Wz44Cvt7QF4CeV0lU5i8LJLeU7SrRs9PabqR5oET43R4quyL8d7HsZDili+xQ8V7y\niewqE50udImcV02WPrTDoPVM+kg8qxWRonvNrpWIlfL1su7fzA6mxMpNbt/d1EOMhJ2743jiuRvw\nDJIxROHaGALkMaeZbBRD3M1bAVi1OgWb3YpQsLYjL7W29Cw9U+R4Tdff+ThXPKmhdtGRuy6Sscjr\nRRINfVKXNaz7UORW0Efa21EfSs6AitNMIlZLm3zqSZJb48sh4q39eaW0n7YID2v+I2OwxBke7OVH\nXw++TECBSep0N9sukHp65NZvmHjTWQDktZP2tCJGZBHaNp+u8bKL0uw1El6ZVS7zg/W3SaTD9vIY\n9twlkQjb58k5yRfvqvabrYKKWTFLIlqsR4LZ38ruMBFbPDiOFRnad8q9z7pY5hlOQx6uiQt7Yn9O\n2t/kj/1r98IjxVlg0OInO7mXSeRB1pcpxG63oqiHi/xCVoi+xXYpZm+ZyDYgQJQxIl7Otc2ScWXR\nhUWMe3kMAJf8IVGdue/LM+J2S66445plMGu9yDFitfSZFVZOFnegjYA8kWOxtcX1mV0kynbGdmm/\nQ76MxHOljKuajYokq6Bp5zOylRuEbnayuLlEXfUbvITf35J5WlxP6Y/ca2UOfn6/xWwslL6pcLqc\n704RGZ55tvSTs946kfJ7pW02Z8pY19u/xTeTfnbvwng8YdZ4xFrdUrRaoo2C9hgs+T8Zv+65UXQ6\n+GkZR3mmyP1m2hMovUTu6Siu2f9br5obEVLKOV1XMj1IOoKIBcHEXi0T8535ElqTlykPeOt55Wxp\nLe+dmdKwvfmbbKV07mXzAfijoCVmW/mPSyplQGDLkQc9ZJe1zGd3JF+ukWQz0d0sA837omA7r3YT\ncqaEdxWVyXXfzZRJfNRaS3E6GqwfKeUNOQ5sTTzHUKDDRbv4bDbcKRPp1H+VE2o94Oc3F+PJqGjJ\n5hSa6WbWbGlokp+RZV7lk6RDie4j4VVXvn0vAWHy8J4+QK7/dokYZsLOFgVZuro1CS3/n73vDo+q\nbNq/z/bNZrPpIT1ASELvvReRoiCKoCCgYEVERbHra8cGqKiooFJEpEhVQKT33ktCCOm9J5vtu+f3\nx/3A+73X910/9SOXnzFn/tmUPefszjwzzzwz98zQ4OkEjNIRSvkcT2uKng8R7uoopULUWgm/y/+R\nTkZNMxU+2kjnRTOXTiruunleNGSS/D3Q9ykDDvCAFbVChWt3cgO5ZzAbL/+ymI6gPVSDzLu4qxjF\nwUxbSYMVeJ4mxBEOPDp+OgAg0irGj91J+clP8725E5tj0VoesOVmlHlAFj+PPUKCOYt/e+9lNtl8\n+/GpAADPbQTSFfX3h9SdkPa6s4Hw/bMm0v158vdC1bsSkctoJx3JGng7cn2HmumYL9vA5lreRDs0\nVbSRGht1J60LjZnmDfI3JbIY6Xrq5/UGX6bD1+Hu/D1rTxLQl9e1XkB560RJQ+AV3w0o9eLUXgCA\nsDPU1/x7uWlprxqha1ctbqY0pQUAn9kHa38b9GcZ4FC9FYJhcxlUPlKcAAA4v5jBErm3C3Yx5tgZ\nzD0r6xgjn3IAeR0RXw55CZ0UmdWTcDQVpywxErLJBj/URfE+1wPgzqbU06p2bkiiVNPZjA5N5Kdc\nF1dn8O/aFAMi1vH91ig11HWNG+zqMUko7qaGp0YogMmLcZ0ow2Nl3Cu/yhHNTdUerH7mQwDAlBfZ\nZL3/TE6ePPoz98s99mRM7EJo8sevcoTue+98AwAY/QED1jXd7bg1hQ79qb2MxrmTqGfpPyVh2GT6\nSVVu6nDWszwMdpnDsb/PRm/D7C2PAwD8CpUyEQCQNYA9VIL7Ih1wU4GE2u50vN3tuN4lO/lpzNfA\nLpa9I5l6kvQJ9ezKEywBWrWhPzTihF1n5d/O1fGwoBZV7mXPxkH9ChMU4Uv43IDL3EOtSRaUdORD\nlpzvyQuKqPdSKGUdfgzQ11xvJO+D5GvchziPUUJZOy38ucwxR74NoS/Sb/Ccpm/oV0Ce2p2BuKwR\nZQGp1N1AMea8dzIDWNtzUqD5jf6PQYxRrXPxvc7tPAC6hheheW+eZcpstOP2PfxfSTcZ7dozKJq6\nh5ne335i02IRT0GAE9BVizOM21UPXGj45AvxwH5fJXznqRPW/jaEiSb505uwhORbI3Ui86tkuMy0\nYZGneCasFefH2OlMOFwujcAwMQxDX0Ze33YXE3wbP2aj1FRHMHrN4HkibQ/PfXVijLXFbENtHXVY\n7eX1O86JrKMoLSntoob/Dsq9YqgProuN3K7KgOQDApczIbQnqTNs/Wn4hoUwKHkwjOe2i1WRyMgn\n7/yTxECMI3zdAJYgmwIk1FTyXibmeGATAy4qTtHncUW6MbQD18fBdQxqBl+mb1Q8wQFPjhicUM7X\nhI+o72mPixHogW5E/cznVrb4c4mGxu0FKaSQQgoppJBCCimkkEIKKaSQQgr9L+gvRahIADSSF61j\niVZwvxeImmJiTl3tGdvRiASPtsaKhzoTFpds4PvnvE/I69oTRJFozS40CWaYqnIHYcvtRfO+1IuM\nLraddgFnigmfNr/LLFveIEadTD8HwSaimtZ2zCz4lfBzBF9klNMRYsLVh/h+o7kG8q/e+mBFgyWH\nTYcrJ+OgimdUT5dXA/2PbOD23SRGi4P6EX1SYoyAKpZ8XN1yBQBgTDkjxI41vMadIKP9AEaQdx5l\nFtYQyXtLB5gV8NMB9kuiSZyVUUjVbcxym1U+dLEwM3ByN2GzvjjKsrwtZauvAKIHcWRkzpGY+mFE\nAyevW42KAgvMPdnMUP1FHoy9RHa0iJDHmmaM2GpsEoJO01TcN2MbAGDFApbshJ0QKBS1CkUvMQps\nv0Tkgf5nZhZ8BJrAWCzD3YOlCM4aZtnqnuf1uq0GVPOxmP0RR9rd+cFuPmsdURa2JjL8txHJptYB\nAqzUaElXAES9LiHjHtos3f5QwCSatZmYJYs5TCYVSkYYKqgP+57h6PLxA8YCAKLfpr5ctTWHqz11\n79YWzHxv28sIf/O1/Hvusz6ocmlHY3ZTlp0Xcuzcmu29oUqgvvvtYnYo5w5m2/wuMLUXmO5FkZky\n9LVRMnEAEOVXhbc7b8DztrsBALV5evx8lLDYpsnc+8pEY2eV1oeog5RX1u2EERmKuT9prXytzG0C\nbxuuA4+AvjZdyevzBoqRkAkS5kxdAgCY9yT3VVsWddJtkm+UDYQc4IZcNZN2wnOM6E61BvBOY1bd\nIEuQtjRuZQwLrMZjd2zFolSWJ4Z964ftZ4jScg+mjbs1njq1c0kPTB3JEdePvbYWADDnx3EAAEmA\nC1o2L0CxkzpUlUj9fvwwx9oHCDRzYGAdnF7a5WbrqYtpU5l1q2nrws8Z3E8dIhM3dC6bmO44xGzf\nPWcT0WE2G7+HiO9xYd5NMqKBk09NCHmzrmxe73s9DF49+WdtTaUoyia3QgtleA3UJxcTqyjpJkoQ\n9vL3ivY+NGub/x/P2HqhNQDAKNAOpS844TrIG+jEfRxB3EMrerqgqqSe+53l5/DPE+UOxdTpzMku\n+FuYRffTueG73Lh9VJ+GY4yDxJhrlVeDygDuh2bBs5rmgocFKliPE36p6UkUy5HsBACAlMFr4nrk\noTCA8vD4CbRRFn2blKVElmX7tYZVdB2O/pk2O/1hvrdN+2xczOf5xBNGO6mr5LpxC3S2JtwO3yk+\nL6CxT0wQJFs1cB8IQfPhWQCA6oVxcFUTzfDdY0QpRHxEfSvtJMHFP6HyZQrC/DFt45k8nv98PhVk\nf8H/DPJ/52xWL1RO4TUxoVU4eJGOaIRTjLmvFmikc35Ae/o3LT64PsCB6yJ2J+2vT6+GVZTHOk6G\n3yiBbqwkawBXoIzC3tS7ZmvroPmJKOrd75LP7mjRImJuE2h7UWazHua++PZJthoI/4VyVnl8sHcR\nyFobr2u+hrIoeob+pDsnAKe/YJVD6ASiYLITeG5s9WwFrjxOhGCrt2mXS7+kXZVEaaX/WT1qWAUP\nzZ+cQaMgVBRSSCGFFFJIIYUUUkghhRRSSCGF/iT9pQiVmmo/bP+lK5xRrMF/4sedOFDBmsKqUkaQ\ndKKeMWd4MNZ9wsx0zGRmUd548TsAwOupowAA76Ssx9xsZsr9R2QBAI7vJDKl1zTWUI0OOY0nI9gM\n7pGmTwEA3CkMO/n3qYLudWZLe05hvfMv+1nnnD6ZEbGw2FL4rWYEuypZB9mhNG8DAO15RtO37FuK\nxN0PAAA0x4giUYnEs7pjDcIDGLntvZ+12ts/YnZ85PFHAQDSJTM0IoyrdjC+92yb3wAAb5dTzm1b\n5uD8VSJLjJmMFjsKmCVX21RYuod9ObqOIzrp7AbWNYYPFaiUk9G4msP1Zapt5DWN/5UkwCZqQnNm\ntEXTW1nne71Z6UYzM3EahxqOUMrGIZprVHSknkbey1rvS2fjEaLhz50GUff2prJmv01TRoJL6vzh\nq2Vazv8y5ViiYxZW7uxAsBixO+XpLQCAT7ZwZK++LTO8xh0BkEfyGe7DIZAbeTjYZVEjZ0Qg3KJ3\nxit3rMObp9i1t1McM6xpeeytoLEBwZeomN2WEiXmCqIMY5+nvF0VAegax/TYdWRKcEvWLHdYmA4A\nSD/bGRqnGGl3C3UwSzT71tglRCyhDHPGM5urLaSc75/wKwDg+6vdINeIOvbLhvphRAOn/NogPL93\nHNRV3I7bzzyD3bvI06JdtHt+Q4gGkVKDoanj/jVrKJsyf3KW+6R5K3Vr4OTj+DmN6AS9ljKOeYMo\nvsKt/Ls92YltVWywUtJJZMAL/917wS0QMYUDeH3IJjGeV7S96Xn/KRwuSAAAVGcFwuNo3I0US+oC\n8MnRIYjeQv8ge5QP0FDf1A7yd8Ne9k0YMuk08kTDzLn3EJnSdHQWAMDrU914TavinmWPpX4b0qk3\nXqN4qE+FXWfp75j78RkfDeZ4+4WPjIXHj/qVPYb765Vq0VcniJ/r5W5bsPwZ9qUrb93Ix3teJwmQ\ntTJ8rxMqcu1hQFVAvWiZQORB7pYEAID9tmqYfqMcuw4WfRqiyWNJxWva+tegW1AWAGD5Oo6m9hPo\nL49AExp0blgGMJOak0ofyhLPPe/euEvY/QGRThWtBfpQ9NUPP06Zhe2QEDKVqGD36xFQFzbujVGS\nAZVbQuBVZq5zn/DAV0LbGLGInYWrvyZKyO7VIW4bzyOVnURPqCrqjSGJvutdkaewsAf3Olu2QMjW\nksdZi5ntjgvORsYJprUru1GGoacpr0vRkdDp+QyfGLntEm5oUhLlbtS4kbuHvQczRwtbuuOmWdGg\nSeXvgaFfGQp/SgAAtHzyMk5vp71zCKTRgmVsjD/96+k3EMvVtTSQ5XeQj0bRmH32lLXIc3EfW57P\nnilZ91BGOtEw2qBxI3Mk79nnVzbbh4F7oCtQDdUVnntafMOzRt3b9JOybuffF4xfjJkr2PA2KN2H\nAkc9MKIBk87kQlyPPNQspR9T9qIDdid5ZdpOnQq5h77q1SZhaP4l9eTtZvT9NVoKVV9JWRZPdaDJ\nD6KxNC9H5hjqq7ec7wm8okL7x9jE9syX9HEswp+59EYTSBoaYGk55aoSSJf4jjynZGmiYCilfrsC\n/1w/qsZteRVSSCGFFFJIIYUUUkghhRRSSCGF/hf0l6aV9GYXEvtnIu1IAgBg6TfD4OrFeqq2kYzU\nntnPrHZ0n3wUVjL6f2UHUSzPu/l6xz37AQCP/joVYUcZE7JFMAJlEOiIg7uZidMM8mFkMKNVpV0Z\nhewisrfHLzdDaDJZsHU+kSnDnjoAAFjzC2vrBnZPxwE7M/U+rQS5sQMcZEDySDCJrE2rz6ejywh2\nxS5vwshj1klGI4OMThRVsbBRe4lRxenvPggAeHf9BgDAc+mTcDhNzLgOZDRyTSFTMJKTsr14MuFG\nR3tPG2YNLEYh6N+CoRcTZ9IWM3rt5lAF5JYF/vsz14pa8+FEO12ec5N8aOikkqEyeuBnYgjdlK1H\n3xC2xf/6LNe+2o/yaPFEGvanst7R6mW0P+gsM7EFUdRRVZgDOg3f3z2APD5znD1ZKjdRIOqHS6G6\nzLDy9WlZLWOZWUs9EQ+PH5Vry0T2ITAO5e9uG7NClW09CNnIDIOjre/GJJrGSip/D0y9SxG8mDx5\nP28c1O2pH8cvM9slWkRh3D17sM42AABg4GAz6CtEBieZwnBX65FeycxsQBJ7ZticRJioITqgO1VI\n7E60w5WTzMil9VkCAOi9eTo6v80MoP4xZgBrEimk7dtoXw1xBoTmU3fzBjbuaRTXSWOVEL5Pgwqq\nC05/0QHPP78eADDnFzE+tUZMFmleg5JOtGubJ5Gn3qdE/zE7ZbQ7twVU2czSaVKY6T7/PfdDZ7IY\nh2xy4ddU2ktfc2ZsVG7RQ8Usw5PAmvLAQ7xPpRh1bcrjgtq/qtMNHfbrXw2VrnH3bYBPgmRX487X\nia5ccGAIVGJ8p09MHhSgBVyrDUXRfcxsOzoRbZS5OwEA4FckJhb0dOHJbhx5vsItJoLEiv9dImLW\nl23B+8N+BAC84LmHr+vYD8c3WkZye6LNWo6joDK/YP8B2Ua9//a10bA+zHp/W62CFgO4L/kVqHB1\nEnnU/BsvikXNvu1D9mJQC3dlXOIJLL1EdFjFFNpgfWe+JjyRBgA4/VtL5OXSFmuFO+IMEsiUct63\n7Hw4Xh+9GgDwSs4YAEDUdMpl9+IWKOpH3UvYwFfVbBrw0gL6WY5QGZqFtMUFkz1wZjZuJ1XWynCH\nu3FNAAyaLtRCeoXnC8dm9jIx7uZ6bzI4DyVF5ONTSb8AAH6cNxwAkDGVe9eSd29HXUdxc4PoTRVI\nWYxP5Nli/aq+UHXg3lsWQ3scvZzXh2wzoKQ3f275EdGg1ybxc1TE0q7XOXRwthQjuIOU3mIAoMn2\nIvyhGlx7mPbuWHY8AsW5o88IIkRmfiwmDQ4pR1UVzx+GC3w1UhwwD6ePOcY/G7fP4H464Q02OVpy\nkugvpPOa/JP+eDuUvRjHvkFU7YLTRLN4/HxI+on2+pfm3E99LJCAQSBm3s8aDgNBvahMkeDdefN8\naMjkqdKhdEMsEqZxYlb6tuZQd6VPUi3sqPUg98Lxtx/EsOWc9DrjM8p14gPcT5c/xD0wPrAK+ZPF\nKORiniW0FTyLeH2UgdcA7DpIZ0ofw7/Zo7gXB57QwSH6VNmWcTyluxl93LipRNYXxFgQspd+j76M\nunjtD35fBaGikEIKKaSQQgoppJBCCimkkEIKKfQn6S9FqPh8EqwuPbRWRo1MBT7UZTHKdLyc4apn\nR/8MAFi08HY4OzI6tGXaRwCAmVPYhyNuCkOAI7qdwW+VnIbgx/JWVHXiNc1WMJJ5vn0kjv3ArvZS\nPKPK+Z8y255Q40Xxg0TIuK8RSbExk5GtkHO8vnaoAUUiiDl76Ca8/11VfbCi4ZIK8Jp8qEphLK7Z\n2hpE30meHD2XyPeEMiNWfSoUvubMdM6aSERKkyl87yfZQwAA3fpfxqET7PPQ5CDXRflhZluCRbiv\nvLcbUg0j/AGiHtJl5rrxBgDWWF6nq+LrfWM4W/6X9wYAAKrGWNE6nFmdAaHMHP1ys3xo4BThV4tZ\nnXdg4VLW0DuCgVANdcFykNmbKpGVPrK7NRDKCG+KkZmeDYP5XpfoiRIVWoVakf35QjcaABC9l/1O\nvtqyGAAw6qPnEDWKfW2qf2S2r/oTIesACW+++g0AIOcxIsJK3ES/LNk+AAAQGlcFexOuAznXXC98\naMjkcWhQkRoC/X2URcBaM4piKLvQOOpZuYtNL1atHgDLsGIAwGNNmZ15bwX7N5h1RClJRg+03zPD\n6g0UExJMfN1wiqglnV5GTTyf4TPSRrb7bAYAwNVOxr7PugMAJn/HPjircok2yzrLuvKYTgVwfsVM\nuTuwkbfAF6QKccNvcgEqBaLOWm3CZ5/fCQDwtqctHZvCTOiv3/eEyku+x36RBQDI2sb9rWgUba3G\nqUUgQYNwiHtasnif4DFcAyXbYqAXidDgS0SoeMQEhLzBElQCTlHVhW+K2EVXoZbJJDiDZchaMaHi\nckCj7y2m0nphbGLF56f7AwACLmvgX0DdqWhJ3jTpyxpt+xdRGPjicQDAg6FE285YPRMAUNaafF47\nYCFeuHYXACDoHWbLtJnMtLbdQGfnxKp2eHHLvQCAlC+Y+c4aRz3TFUjIiaPu2+cxAxe0nXrrEZlB\n/0dzYV9Lgfo6u+uFDw2dfH4+2DvbELKbPK9K1CBmB21p6qP0OcZ1PwQAWLxvAN66m8iSVX27AgA8\n31LmI0OZaT3ZMhamE+R7+D72QSrtSxnpa0QNv1uNefNoi9X9qcM5ExMAALr1MvyFLS7mI7At6QcA\nwKDTswEAfoUSqprzPXEbVKho5C4qPBI0ZVoYSrnucwfJ8JwTk16M3HNiB1CHrEuj4RUTPZrr6COW\ndKTs4wRqfs8Htoom6wAAIABJREFUGzDgApEN9uX0cUK20I/8eWJfAIClxAf/vWKCUDPKO3cC7aqv\nTkJYLBGf2Xfz+jtHEQl/vILo3bLcQMDI9RCxjT5u1k0zomGTO16DgnmBSA4kuuHapuaoGcQpO0cL\nKbRW47nR9Qi8hmt2Qg92+PE8YdhIHzE/l/5kRUsvCnuJ6T6v0Z/xb0F7a4sRPVTKJXy7lzZcFcy9\nT3+Z62H8uD1YouF1xsu8zh3APTCqD/3aWqce1u7UYe0VI9DIQbheg4zqlh6cvcR1rmlrg2Ur/fq4\nQzwfWOeRzwPMl/FWJs8j/3qUvcBmb+f+FpBGub3+9DI8+S/6m3a2R4HKTb0LjSLypdweDClE9Ekp\no+xaLOfvpR012DXtAwDAIOk5AIBL+KFnS4hY0Rw1wyF6p1SkiFFs+//Y9/1rAyrlWjiWNYEviR82\nfHomgibyI5QNpIL8tI4YqlveO4wduVSMe+c9CwAYu4AH5R3lhCrnfNkC3775OQDg+SscAVqVJRrI\nPs1dxazxoP3E0wCAnemEcpnTKUDrHDs0v9G5nzCF916XRee0ZBQFsO1IewSfp8DMwx1QSY1bQ7S1\nMmJ2ysi9nQ6Y37xirD9PPKShmLK0XBWlVU+ewvYMynDpa1SUol7kpc+fm0fZpRiMuo/jsfddosdQ\nN5BGM+4LKlHIhFKkXyAs0zyOm5zhGSqKZ34dMvJoSNffswAAMGop10vSI2yyWnqsKWzzGJ1psXxv\n/TCigVNxjQVzd4xA3AUR/IrXYu73PMQ5unPtx60iz+76aBu2lrCEY9FLdPITnmDZXOlyGso7Zp3F\n5tGUe/U6Gqa052iMxj9PebiaA9ln+T+kUI969OfI3UPfdMLjm0RzY7so9Ynl55h0C63Z2aoYqEQD\n44+7fIVRi8rqhRcNltQyvP5eGH7lBlXaWYZGjM4ty2eZVOhJyrDTo6dxvIg2dkMJ9dVYQhmcOcIA\ns8YN3PsqAyGf/8SmYFoBm9V0oUPotBqQL2zs2J7U2+t2WvdbMLx63nP+CTZgVJXRqVXFESpbcDQK\n6paUb/gRyjL75jnRsKlIA3leOALDaT9VXt+NchrjTspzc2EPAMCj07bgRHUCAODCxwz+q0QcO+AI\nbaLKI98YQR5wjfIovp9Bs4BlPFjUdfeiV0ceCjI/5r5Y14RrxZIKhDJ+g+oXhZMygpDo3X0+AwDc\n8dZsBF3mPad9sxGv/FBx02xoyKSqUsN/YwAsLvLbObEMBUXUQcs5ylCez4alNQ/XYHcuBbRvKfe8\nWe/xYD7nPMsN5uSNgH0hbeXMpSzrWfACy3oKS+jb1LRzIuQQ9at4Lp8RpGXQRfVVGBw6LqLeMQQs\n+7cSTqXo5nckqylMKupidAzl19gntkoOFTSpfpCFn1fTArAOJW+lIvJq63Jm2NY88TEmfctBB9cb\npDsHksf/OsGG+r4yPZzTuU8VnGcgxS+Z8isr5IEvIBUwFdIWVojmzuEneZ/KJB1if2ECMXU25fZS\nHhuPa2xifOjAMli+Y/DM+VhFox+bDI0MT5gLyT25s5TNaQqbsK3VLShL1UYxKvnJYjiKyLsPhrPc\nyvoaD8QBX1JerVOmw9acZwYt3SCUdqf+mnnWR1FPQNOB/k6TjtTBlMe4hlJnNkH5FR7q/bpR9gdf\noz1v/RoDb5mWEJiO8XrDFPq4+P5mGdGwyeuVUGs14mwB/RtdNyuMh0XJuIf70XkzZXdBToG9NeWW\n+DnXv/lDNtKvPEBZDft+NlSJ4mzRh5aubCsHWASfoy5VJ8mwxIkhCCuZjHCZKcet+a0QuU+MsH9j\nFQDgyxd57mw9mAG6nw91QugpvqfJA9dQvNJZP8xoyKQCzFeof9bmEso7UT62COqEXZSA6xK9uHqF\nAcf5CxlI8ZtAB1RzlmvgufSxcI2lLyqL5sPuMtGGQCQeKsYBLebznoOX7gMArKwcCgCI3FGCiWlP\nAgBMTSlXRyQ/T4CBssoPk2HOFWWZpX/6qyqkkEIKKaSQQgoppJBCCimkkEIKKfRn6C9FqEghbugm\nFcN9mZHfs1djEfAZI4bqbXyP08II8pbMVrCVMAqJVoQib5rL5kAVbRk92vPuR7j1ODtPRc0XHSon\nCahOCSNafj+rcWoaszg9mhKxAIJakH4sBYZejEYuOd8TAKC7xChX/EBCuDLLolDRkdmDt8+NQKEt\n96b50JDJL9KODq+cRtFVooQGh6ai4GtiiMsEn6yiKdcvp9phcg/CYzdMJD6raxgh52d2Mqs98+F1\n2FrGBk8PPMHs+LzDtwAArt4rMqYb4mBiEBIRXVjecHQ611DIyiCYBCR2wn4iISLHM8J/OZ/oI32z\nGqRPZfR/1qoHxDd55mZZ0aBJ0nlhiKrDsoVfAgBu/fY5JA9kuqXgOzbRa/cWR4l/cakfnMXkn6k5\n9TM/h7xtNTkLAPBdeg/80vlrAMBIf0LpmkczvKvbxwhwm6eqYfdSTw8fZVb87ByOh1WFAKZ8ynvq\nVK6Dag+fuW7JAABATZIH/xrI0rHbTj2EDNvi+mBFgyWVU4Jflha4XSB1akwID6E9s26jfPxKaTt3\n7ekAn4628WoC+dx9KrNj+3ZQN4M7leDLVEJarzf8fe/hbwEAM3ZPAgCYrmrhbE+0yaYtzLJ5/Hlf\nuaUXEaJsT6Pnc9U1hD87LLyhWg8k9GR2KKsNy4saeybO7S+hsLcGEV2Z2SzfEwlbM6ILTBnkmyuO\n2ZPPzg6A8Qz1wjqEWRijhUiRaiuNpKZAB08AbfGAGUQR7fqSsrKHivLIMBvSFtGGV/Si/HwBvF+L\nbzwo7iqgrr/xVR3M9/T3PgEAkDv4II3mOnhpxzgU1nxSL7xoqBQQYcXgpw+i3E2fZf+mjghkIg3W\nePJOa/23u1WXR3SC3Jp6opNoIzUavqb/lASnaFL86poJfO/d5LenUvhFTjWstzCD57PSb5nYlmi+\nNYahMH9KH+jQY00BAK6LRMz8MJGyOnTgSdia8LNNiiFS8PDNMOEfQJbAOowYfQSbdrJ0UYq1wVVK\nHQg+S7vpV0aZjd3zGPRCpGoxHjVxkUDethPNnDu7YdHzn7YS6t7ZSSsBAK1F40WPH1At9tXQ/XxG\nv3lE0q5bMgCeQN7L/xz1+6BMf0sK5bOCvw9GYU/e2+JVQW7cIGqYDQ4MaHkFZg353vyd41izj/bv\nOhKhoB8FF+xTQVPI84FmEc8iJ5otBwD0NjwMAJjZajcynURBdzMR7fXsTqLFQkfynFBXHAKDaPBf\neJp7b/6b/D35zVJUdeL15XXUwZLJ1FtVFf1YbaofnMLGVm6Jqh9GNHTyqOAr00PjpMzkahNs3WgD\nwzbRr3CZqS/m/sVwVNKmhszlGe1oZgIAwGehnnRsk4kzWSxxPHiZsE4plPukOYvPeOL2LZi/exgA\noGYk99wW0SwFK/8+DkGXaNTD1GwaXdSdz79cTZnD4kazh7IAAGd/TYGjRl8PjGi4pNL6YAqvw6ge\n9DV/ONsVURFEaRV6qBPNl1MGj+Y9AgibVtyFfI36lntd7lC+x70lEgMn06e5Wsvr0woSAACuQOqx\nHOzCtdkCFfogB1x4e4g1ZNCitCNlMva+PQCAjV+yxMsTz2cGtCpHpY0INmdTgTD6g0cNBaGikEIK\nKaSQQgoppJBCCimkkEIKKfQn6S9FqLidWuRdDceYvowwbV/ZAzXNGdMJHM7In0vNSFS8yYqSbcyw\ntJp6EQBwvIhIBjVL5TDkyGNwFTF7UJ0oYkMaZvYsZxitsoXLqBEjJ08fZL1c8nDW1pkzVKgFo5o+\nHZ/raMGIVGEVny2bPAg+xixhh6mZKNc17pq4GqcBv2a0hKecWZO5x29B+ARmyL1ZzDjbxHjHh3ru\nw6JDjP4Fn2LEMK+CfJVaizFjG8dAjqNA03/iyGwkMQMEA2Vi61UHTzkj0mUvJwAA/Hpy6ZZ3d8N0\nlfKpGsT7eDewT4BZNG+0xhshmXgvXavqeuBCwycJgCTJuGUpG9vJBhnuydQZ72es2Q7XEQ1k2u6P\nFvcR3VW9jX04goZQXysdXAd1OQF4I5KR/ahhRCBUOymzoCrWl+7Z2w5de7OJWOIqZhqu3C/Gderc\naN+cmYUf5/A+xim8zlBGOT79yAa8uZN1ztfGfIVupkbet8HsQUC/YlTXUQa6VCNUFylDiS2HkCdG\npoYEV0KnZvQ/P5PR9x1FLAj3FzohyxJebUN00JeLWRv8rHMqACBAZNunPLQNK+exz5VoZ4OqFOqy\nRwLCHskCAFTvIMrpej+lIjG+zhfuRN4OrqGut3P04dWbY0ODJ7WfB5aOZfDTEiHiKJChcdCm6QYL\n9JGLvxu2BsD/biLwTMuYFSvuJRpDN+N7S4oioKvgfvjTOTZtl7vSplrO8z76w2aE76K+WWOZtWvd\nOQsAcLlfElqPZH+VrEW0ybUtKEf/01xrYSPybvRDim9XiHJD425qWlHtjzXbe8Mo+mPYE1xQuamL\nYadov4pGUL5JATXwtmKGOiOfWbaXj7PpZdtYyvZsOz+YArmf1ZVRvqt6ME326jXawNzKQDzeikiG\nr75hj7Kv1czIuUc4ERZC+93Cn/p9xsy99+5NbIDboftVpG1lf4Hvlg0T32T7TfOiIVNdoR+Ov9UF\nFjFas1JjhF/xfyJT7A9RxgE7QxE8QjQadlOvyst5oeVOyrH2VBT6dqOFWx5MPZtZwL45sb9SLvYo\nE0qncD8MfZ3PWLVmAABAP6AC+Qb6VfYm1MHgI1xXFT24nsra6xCQzL1wb8dl6O9XXh+saLBkrTbi\n6Oa2cAr0QdBFCWjPn8t7kr9Ji8m7zAQTknpmAQDOX+Wm2eXULACArKbeLtT0Q20N7d620+yfs37m\nfADAo6+wH0PS6Qp4gqinskDGyKI/UfqD4Zg5UjRpf416VuHguaO6Gz+Hx1+GrlIgradxVOxL82+e\nFw2ZDEYXUtrlwDuQupT9Zk+ErCe6oEj0uNQbue90D8/GCYl+xeHLRHDFbqbeFvTl66jws7hQwB4d\nhrOiwbTonblrM+3mgk0jEH2ca8WvgP5SbSzXhanOg5hF3DOfXP4QAMArEBX5lUQe3d3uFHJt7Osy\nddyvWLC2pn6Y0UAp2liFd9uuxw8lRIjFr1SjJo6orNseJgK+2SAi2Tc9ORgl06k7nlLyM+cu8rdl\nAs8C16ri8es1Imt9Ykzy+KFs8PxLFhtESyoPwtdSX2sT/lOnvurQD2v6fwwAmPw1+1/ZW4kxzALh\n5PWqgBbce+9pfRIA8P4f/L4KQkUhhRRSSCGFFFJIIYUUUkghhRRS6E/SX4pQgSRDNnixYV83AECT\nIUWw5rDTb52dUffgzYzypvYIQpgAgxzfSmRKjxHnAQAHd/F3025/JNzNbHhwZ0b4rYVEJ7j9eb/O\no89j12X264gUo9Iy1jMrc/dDu7Dyx0EAALWDsSVPL94Hoqtw3xEXcCKQkc87Q0/ggKauPjjRYEmW\nAa9HDUmMqprbazVmbZ0IADBEknfNnmKGZOljg5C8gVmykq6M/vV8ieikcB0jt1//PBShQXxPWaTI\ntFcSzWLpwLS4xeDAwNZXAACPjebEprFTWMufHaKFryvv9VDyUQDA/ijWR+ZWsUu3+kwQlo3/AgAQ\nIeBNifXAi4ZMCX7lWNbpO3wT1w8AsH1vB6S+TeRC7Bc0C9tMRBeNf307thRS5255l12zvznA6yIS\nmBlrutGNfXYW/XvNAroghg303c/eLKnHvEirYAavYgqRKXFifnXRfS7UeaizpvuZkXB7uQ6uj8X+\n9MpATOnDHgFvlLZCgedPtuD+B5IsSwhbQpvZ9+39WO3HHiixO2g8I/dwvdvig6CrESiCSUI+Ipxu\nLWOdqvGgBYtyOOnJ/Rx1WDpIFIQtitm6pVe7I+ge2lG3jzeImccMak28Fql29muAGKlcMoKfI2Qf\n5a29swJFLsrV5Wvco3avk9enQlWtEZXVlMOUZ3ZjawHRlMViMkjkIVHrPTMb2dsTAADukdyLxomR\nygfmMAvkHybBHkH++wfSJptWMeNT3JNKGX5Uwvx9nB4zM4MjW6tfZCbONsmNEhvttfoe1o9PiiKy\nbE0RbUL+oWiYREb1yVE78ZyucWfioPNBjrXDdY57zoChFxDdhUiGvW8wq910KfkVP6cCh1Zz0paf\nUIHnHuCUn+ujzGP7FCI3k7ZSa6EOPXR2MgDgi3YrAACTch/CmjyOJf/gMY6cf2zHFACAJsCFKtFX\nxS7QTZHJlKVX6G21ywhHONeVuom9XtjQ0Kl5bDFWfToPA488BgCQHVo43SIr3p3C0hwW9fXxPrir\nqSc4Q3+xzWQiu05kcPqdpJWxKVdM4xK9IH45QJnhacr19pancOALolYKBvM9wanU0xJTEJoIv7V8\nD7PrHooV5vP8XOr+FbA7uXe+V9YVRZ7d9cCJBkxGH+QOtfCJ/hXWoU5MSGKPoD1vUxfVaUQaSKpY\nXN2XwOsE2iCuDfmdX06bGfm2Gs7nuHd2Gk90+51biPKS+wpke0go+k3mKPTuZvZZeWf5eADAHUMO\n4+ulIwEAr86h7s67yj6BZVVESsR2LID7K+61ZW7/+uBCgyePT4XSOn9U/sA+ewZDDQot1DdDGpUg\nOJXrftu4lugUzX42EOhcfQXlJ4v2Ju+tvgvNxHSfnJ6U2y/5ROkGvUyZ9zBY4etP+3jtU/b5K+1A\nnfzwrhV49Wva4NgD3HuzuAwQvJJ79477k2BeyOcuX7QTq683V2qklFcegheW3Q9PayIyx8w5jl35\nRL2ee5Vy3d2a+9PcLxdh9mdE/jjEGGujWUzeqSZP2wxMR8UbtK3ZI0U/xq8YT3DPIOLPV2FEaXvR\nD6Uj/dgVS6hv3e5MxUMfEFXWdDxR9xmltOcOUXUBjQ9SHc9APfz/HH5aQagopJBCCimkkEIKKaSQ\nQgoppJBCCv1JkuS/sCW4PjZWjn7qaahZNojgizLiHyfy4Oh5Ygbifub/codLaJ7CTPXwCPZQWTmX\ntfvlAxi1Ml0wwBHCz+/1F5kWK2NECV0ZrczfEwt9F2bRD3ZeBgBou5ZhRV2lCtcTa9eDwlrx+/WM\nrBxrh+4iM8BNl2ThUPGPqHYVSzfNjAZK8W3M8vNrO2Pu5lEAAH2FdCO79q89zG5LTspA1vuwcMhS\nAMBzF/g/l4uRP2cFo4GmLM2N7uZNN4peKvcz8tg+idHktJJw3Nb8AgBg43YxWSSSa8BwxYA7xrKG\nbvvnrIPUV3MtFPYRne/TVdCPYHbuekYg895XTsqy3KUeWNIgSd80Rm7y+gzoTVRGt0uDwP1EEehq\nKQ9bGOUYdNWNVz/ltJdH1rHzvTeAtciSnrzWFOmQsInZ8MJejNar+hBhVJvD7F3IaRXKO4nIc5SY\nTiGypcad/qhqyedG7edrwGVmeKet3woAmL11AhJXcY3UvmrFhSeWou5KYaPVRWNkrJwwbRbcYspO\n6FkZRf34szGP2VRzDvld1gkwZ5LX+ir+TQwWgc7K3/P7qxB8gezU1vFv17voVw4R/YmcasSv5t9y\nbuPrXT2JOtuxuCeq2jOTZyikDkftp55eu4/PUml8UOdynQUwkYfTi55p1LoYkBwhd/9yAq4WEpEg\nSQAKmV1tto58zyIgD7ozJgwfz1ksG7fRFrrDqItBEdy8qmv8YPJnZkzaw3puRyjXxfVJQlXbIzFm\nMvtv/JTBTJE9i9m/5MUViPiWGbvCx4jOLOtEHbbcy54R2cUhQAHl6NPJKPzwYzhzchuvLiZGyc3m\nPgRrMfcXyeC9MfGloA/9B0m4Ws5OVkQtpXyrm4r+Xx2pN8HHuT9W93fA7C96qFyiDLW1ZO87D9CP\nmXVwPAz+1C93BmW35G6OMHzqrcehvpsIvgqBfGrxAv2g6q7sfVPYW0KLZUSHpk/h9VlPPqvo4pcT\ncOUa0SBNdqvh1ZHvZZ1pEyMO83drtAqxI7IAAM/E/QoAeOggEUKyS0yp2K5BaSe+3x1KGcf/xN9r\nHqO+9ojMxm/pzIarRA/BuwSiYltuSzgEwihiEfWtKpFZeWOZ2EvL3Cjow/UUdtaDM7s/gbUyr9Hq\nol+LSDn546kwLqHehM7Mgk5NG5n/Cc8Zxd3JntgdXhT0ps417U1/89oR2jxPDHVLm6OHxiZ6gAlM\n/8Sx7L3hEOPwds3pjcLBYlJXFd90vbfDyj290bpjFgAgex17i0XtEU3JJN63pJsFNQOo7x4Hr895\n4IVGrYuJbf3kjza0wOvzqFPNJqTjviZHAAAfvUhUfHUz+jm6ahkVXShjrUA1BOyk3bMPo56FLDdh\n6JtEWO8tZZWC6mWukZyh/uI+QG1n7p3vdl8PAHjx4F0AAP9UHa4XKNRFy+L9lN/1/fWTMd/hiW38\nvH55amR+Nw/2wsa7L163p9d7LcprQ2GNE0jNXyiXa3dy7wk7I6NgEG1aUgue/ct/YN8pewSvUbkB\nbV+iTry7WN0SdIV2NW8g9SaqXRFcy4ns7fE0+7ScKSdsyfC8CS+s5ZS1Jz99lH+roOz6Pc219fPG\nnnDE8J6SjjqdPfmlP6SLf+3YZB/Hy1k6ieZ5cf5wrORG0uwOBkAGv0vI5LbX+qNDT/7t8+1DAQAq\nvhUPdjwIAFju1w2+fCpNVCKdh/JDxHe1DqRDWJMdA3dXMmxlLQ1lxjiOim1/7F6oVRSgfh1h66X9\nyMhxHSmIoy91Rc49VLCr80LhfOGvrZL6u1FFjgVrZgyDRgQr6pJcWPQiDU73ZwmHfCdmEwBgzKfP\nYX0nNkVMCqF8rkPJc0VApdnwa7gkxhtnz6QshiSwWWV6NQ8Y0Z9r4ZvL500buQPAvzeyFYX9kWKk\n8u26k89Qi+abKgHZjFjiRmYT3mv6HXR8ZtcDLxo8yRLcTq5nn0sNUzH55nuU+jlFjNJcdLk33rjK\nAJrchLqwvg91aFcdG0QtODYIJV2oi9MeEI1N144AABhF6Z4tEohP4YHuzijee8knfE9VKxmmPDqh\ntffR2Qiw0Nl/J3U4ACD522rYo7nxpQSVIEPduBth+nQybHGeGw2f3SYJKV+Qd2nT6Ch4jdQbtQNo\nPp7B65OXWJaz5VaOTx3xGyGQgadVKO1BpySxBe1n5W5uaL4KOuyGCBu6v8vSyz0RlOExJ+Vw4PZm\nSHyH+p1xNz9Tz7kMtvie5xjSop5GRPSivta0ESMFF908LxoyeXwqlNn84HNz/Uf+qr1xiJu+9CcA\nwKsXqH9BqR6cqaRzcD0g5RYNTmvSKXNZJ8N7UTgwfei06I4wIKJ/myUpbd67jGVHCX/vmJIFACjx\no0PfdWUa1qxmaY96CJ/Rfxybs+1fwXKFiAIfCgbRXkiuRusv3iCfUw17RgCkcAaoW75egbI+PJTP\nmLIRANDfj/vjhA+fRWFv+iTqFMonOZh6W3KKPoo604DaZnxPq16EJl/fJ19cTth50oBsWF3UoQdH\ns5ns54WDAQC2SAm2PK6H5K8p144bswAAP+xjQCXwsgT3h7SxHfXFAICsm+ZEwyYpVw3VLDOkB0VZ\nVFMV9IzrQ3W9LHwCHfouoYUYHcIS5Lcy2BRYKhejO/2oG1WJKhhTKFuz8EsqUwgxl/fwQHDYEQIp\nRgQ8O3N/3JzFElvDZgsee4b76dftWTYSeguDmvknKEdVcxfcLvpOdV3q4DvvrRdeNFTSFKgQ9qYe\nadPI057mEpyvIq+KxfhUdYxoERBpgiuCe97Vk9Q9bwh/N6YxgGWPcyMukXK5mk6dvl7yrCsX5au3\numEMoG9kl2h7azy8PvSUhAtalinEpXOvzHqFflfsxyIgUCsjJozrZH4iE5Sd64EXDZmK8kLw4XOT\nUHkr1/OZo4m4EEf+B2kpR6kXeWb6zoxlt3wKALhtOxPm2177CADQVfg33gl2/HiVXDXqKAd9DGUV\nPyAbAFC7MAaSTLm9puWee705eMCgCnzXkiO1Z9zLkee212g/h0XxzPLR4/ehy6ssE5lw61E8t7lx\nN4h21ulw7UQsdFWUl7OVjKDL1Mu0R0SJjVckHgYBnwz+HgAw6zhLX+WB1Kn72tKPXLmlH5pbaJA7\nPkA/9PD9LJ81FfC8V1kcCedt9InOvMb/zfqYpXZpyyMxbROTwiFVIpH/KGMOP2/oCQDQ2IBX+mwG\nALy79Y4/9X2Vkh+FFFJIIYUUUkghhRRSSCGFFFJIoT9Jf3FTWkBWA84dRAuEF/lgZQ9ZFO5m1u07\nM/+Q/FQWOpuyAADH2zC6615EGM+qImZhLKU+mLMYwcqczgycSmTrgjSMQAdeqcOVfEauTkYmAABM\nKqbMe0Rlwe4l0uHIMGZ6gnczy77Gxozqg+/vwc4SNrW9lt4EsqdxZ+NcQcC1sWr4i+xo/DoJtY8y\nSnz+Z0KIhoaz6ZBfnyoczCPE8d22hM89uYtQPegZlUw92BShFwUWeiKREbv3tQOAGzBLdU+gZAXh\n7bXXRyqLMdcGm4RCNzNx9l/DAQCVvUQ5SSkjzVeflWAhuh1ZjpCb5sE/gfQlPiR/br8BOc251YgK\nLnOEfUweLU0RY5DPOxH3DmUzIZaR4mensmmfI1iMYW2nRvRPWQCABc14naVQjK1O4H0D0oGKLdTv\necnUZZUo8zGUqODrwaZSsS9Tti9vZv3ftOUzAABpsxxIeYYLb8+Zlqi1Ne4Rn/BJUNlUCD/KiH3a\nwwGwh1F2Ac0FvD9LNGZ2SNCpqHOGAsrs2R4cv9p2FRv0le5OQMuPiPIquoX2WB7EDEzsMmYTcm4z\nYPVhNgHbFkV00oMtDgEAnJvC0f0TNuYrXsH3/Popm+TaOwpobIQHJfuZLbRHeeqHDw2cvG41Kgot\nN0Yam6/VIu956sDCSSyVtN3DspHY1DKsSFoFAOg6iFmy5u/xunzRTM8W40XoBfI2N4r7mdyKyAl7\nJjPoR84kIeAKs6PpVwh/Rm+uo5W/9EO4aIpZ3JX3/HUXkYYqAW0OHlmIhLm0t7XRapQ08p6mGjsQ\nck5CZQrpW+GtAAAgAElEQVT9CG+QCRXsd4hNo6gL5l/oq9giZRycwuxptw0c0Vqyg8guSyYzp5Vd\ngPjlYhSvOQEAkPio0FMzZVq2Mg7l3Sjnt0qJXkiMov7a4t0IOkX3zv0eUTBrNlMXNaIRbpvJF3F6\nPZEQ15opuggAjnAVUh/3h2QXvocLiNzDLLP2A4ECOUD/9UgnPY4XENWgEkhnTZQYapBOGUXvtUH7\nA+sEgn6gHM77iFDxiQStf74Pspp65pNFOVEF9V2nASo8vJe+krqXXUgbLwdSR+MDa1Gzjpl7eYSt\nPtjQoMkdKSP/JS9QSn5tXdUTRvGzVjS4j1gmGg1PsaJ9BMvBs6voR/p2Ea3uFfJJXmRHZSvuh8ZR\n3A81h3jeQD/6vkE6N+zbaQ8dHXi+2HyC2XGLRYImhAayYBKfrzlHfS8RMJTR0/Zi+VmeOV7TjRbf\n5LObY0QDJ0+wD8XjHNBdoy4kfpaJgrt4nihrT3kmmOnrexwmzMq4GwAQn0AbOPxfzwIAVINod112\nLfz3Ue72kdRFySRQ0V8SiVvUG1A7eG+NlvolH+f58fmp6/H41XsAAFmjxIjsCsrznD/92vHzt2LD\n5IEAgGcm3IeCqo/rhxkNlCQAKg/g7Ui9UcsSDIdFdUJToiJLtlO3VL0qb5wPJ/egT7n3RaJot4UQ\nEaaPkOC6k3vk+Q30I688TZkmfC9God8lIeE7+jnFXbmHvnKROuU9FgR1O66ZKjvt6skDPPhEH+d9\nh7y/H1/Mp2/ceTJR3Vl/8Pv+LkJFkqRYSZJ2S5J0SZKki5IkPSn+HixJ0m+SJKWL16A/+EyF/mJS\nZPjPIEWODZ8UGf4zSJFjwydFhv8MUuTY8EmR4T+DFDk2fFJk+L+nP4JQ8QB4RpblU5IkmQGclCTp\nNwD3A9gpy/J7kiS9AOAFAM///26kNbkR3bUA2amsBdZXqlCXwCjg3b058nbtBUZ1rxxJwCsaIlNM\nicxc194uGh6mMcVS08eHTXPYhO3zCmaBvt/dFwAQ2Z7ZtiuPaNF0BZ9xfnt7AEDE64xOnv68A2qb\nimY3KhHBrhNoCfGy+ExvRG0SWfi2akgNE6FSbzKEWoYm0AVnMGWQfQegqWW01k9MCBvZ9+SNt2+7\nyiz25zkcT+2fTl62HXMZAHDY2QLhD7MZ2PmrjFRqY5nR+bIr6+lUkg/vjmNXyzIxprB6MKORYWdV\nUI1jdsgtkgbBAbzeEsbM0qOxe/HiFdadH597vTJ15f/3a/5Nqd7kKGtUcIYaYShmRkvtAryipYVu\nNvtnLG3G/g1jdj2OzDRmsfNiiHjIGEfToStjTNYV6ULtt2KM4ymhI+Ll07FsaPv8gmkQrW8Qyz6z\nUM/gs/JKgyCJhsV+XzJy/UoGo8T+eVTGYaOPY83TzLJG7pFRbm3cuqhxAMEXJKQ9yIUf16IYtmPM\nVJYW8m/zR1CH5j07AcfUjMSHdmdGLmoEI/U7j3CspzzEg6Vz+P4385nxPlVAnVywgPXJY398GpEH\naU+9Oj5jyf3MrNnDpRvNv2pbMOMdsJMymvzMNgBAsTsAP15kb6/k+bTnOb/Ps78j1d++WCMh6jcV\nCoaQHyqPP5xZYux0F+qXyinqjl81Y8DJaQAA42na3Yy7+D9LClFkzoshyBku+ihoaBuDj1LxurxE\nBFEP/wwsXkP9qm5KvdUZacCL4/Uo9OPffCbK0S+YdqJ1BHsJnMiIh38rgajJ9ULVMNs21JsMpWA3\n9BOL8HwcGwYvunQHWvcgmq6iA+W09IHbAADeGXYM/JRdvAxMpOG+GdSPVR+w8b5K70DuUMrgzRFr\nAABvr+AY1vvv/g0A8GW3AYjbSP0KeY795vJqaZ+TEguhTqLsbR8ye+oewd8HdmGT/2KHGdYWXHMh\nx0QjzN9l2d+S6k2O6joJwSc0qBtC29iteybcd1IH838kmidoOHXAsSYC4UeIBMx9m/5Qwnze5+p4\n6mRBXz90GJUFAOgUQO6e1XMk+lcPE4Hw6MIZN5qd5mcSvaILpi4aKtU4WUUUTA2T81CpeW+fkUpX\nbTfAJBrUVh0KgWxtkH3+6s9HFRR+gHwI3Z6BS6/xLNG1XQYA4KKbe6FxrxbnWlE/x/SgbVzfkWeQ\nEa05CGFbaEdIZC8i/AUCKJvZ7YdTqIufpA+CJZO2MuIoXytaUbkr+jkQHsDr5JVEN3kNlGHi/ezf\ncLwiHpoC6nvukWZ/5Ov9Xan+/JtKFcLXGlA1kee1ug6xEG1p0L0fzw9ZNUQTwV+NvJNEmXjCxNST\nrtSPoUmpAIAD6zuioj/1qlMYfcwMf6JPOj7OfnCWuiAUreZaqfKjjE1ib3thyf2QBbovsBv32lui\nee8rVqKTjtY0Q0F/+kXxW5woq/7rhr7UI9XfOUMtwxXshdZLG6q5aIIsnAWjhntP7PAsAEDpsniE\nC6BkWhsi2E0XaWtzniMaJSBVDdtK8jdYzTO+bKOe3zaP/TUXrR2GUvbZR9xWxg4K6xj76TH+HJzC\n2J49RTusEq0YW73Jniy5jiCE3UNbXTgv8XeZ9V/pdxEqsiwXyrJ8SvxcC+AygGgAowEsFW9bCuDP\ndW9R6C8jRYb/DFLk2PBJkeE/gxQ5NnxSZPjPIEWODZ8UGf4zSJFjwydFhv97+lOhbEmSEgB0BHAU\nQIQsy4XiX0UAIn7verdVi8JD0Uhez8hS6iP+MBbwI5yfwAy45nVGr6aM2IUVP7JXSpCYPqDfwOzL\nd6/PBQDctu1J9Fz7DABg2uDdAICUT/iRPi6lrP18QNZ9YsyIxFDUT0sHAAC8UYC7hRgHWstsW60Y\nWZawmVHO/L56BD/BLvt3hlzBJ6tqfu9r/q3pZmWorlUhYI/xRqTWU66FT0PeGUsYxu/kz47Zb58a\nAXUG6+WuekQ3cyYDcH8ER8qd2J+C7EpGDzsnZQEA0jazB8vM4xxr5QoAXJN47ykD2Axl6V4ikVQe\nH1ZnCdSJCAbHmrm+LmxlFmJW+H0YPOIsACCnn0Cp/fB73/TvTTcrR5XLC2NeLfKGsR7b2sqJWd0Y\n4f32C6IT7kpkd/QBPS+izsNsy5k80fRIgEPuuo0Tt6L1lWihYzT5ZTcz35ZfOZFn+pb7AQDBNhkQ\nY+dyR4npWpWMNscs1aBmBnXr3AHaAlMr1ie7Qvmww2VNoW9N2frtMELlbpDR/xt0szL0agFrrITE\nH5h1KX9Jh07TzwEAsp6lDr13gMiuh97bgI0lDNuPCGMkfmk2+xKZs8T0ij52jFrCzLkzWkwr+ZD8\n/vp7Tn2J2eVGYS+uhfvvYnbu67OiN4NWRtlOZhKmT+I0rVUnOKFt2VwxzSkZ8IpJKHWxzADh+O99\n07833awcJY8MfaUHEftoR13jK+CrJG/C1lFPPAYazvBfVSjtQBtmi+X/dFWUn3c7s9uqJjJGdqW9\nO/MeZV7Wjjq0+TLRSFuzukLVX/QTOMZ98a5YIgsX/TYSksgU2Xtwf7y3Bf93YFpXAIA8TQN7uJhU\n41DfyLA3VLppXazRomp7JD4ys+fNLU8dx7GPuS85AyifwhnMUpsP+MFBUWH6+F8AAAt/oM1t9RBr\ntjULEhE9k1OBit3MomoEAnRVJvvZJCSUoDaC+mb7jpm0d15eDABYU94NaXPYxEX9NLOxcga/xvHV\n7FGmtcowBXENBGS7fu8rNgi6WTn6tIA9TIL+IJGwaatbwzuN2Wi9mArh9HCxd3v0NLZ2JS/VdtGj\nKIK2cewAjuBcu6cH0ipEbw0vrzOU8T6zXxSTQm63wyCmcgWfpp/k03DvtEYBheeZMU/pQb+qaBV/\nt0VSdtW1GlQNEnuhygOfrnHviz6vCnU1BsScY3b6ytPNEL2TtvKUhWgfXSf6GvZLZpii2N8hs46+\nkPk0kSI1yXzVN61Fj2jyfn9mc16fSDm9vZ59O/wKJAx+cw8AYMmx3gCAf/UlwnfeorFw9qPuR07l\nfa4W0wBkf8592j/XCf/ZRDtFfC/6cvzeF/2b083KEQB8Gglxr3IzuvyEBMnAs9z51UQXtB9PFNGV\naSpojlLPPAGUTdA5vha2ov3sf+cpZNSQ7+eEHxuTxvsdW0pUkn+hF1EzswAANQX8iNFt2JOluNYM\n7z4iYoxacZa8wv1Vc4b6quleCWeQ6MFi90KSG7cuShoZhlA7Yj6l7eu+YD/GWjhB984tnMbUqyNR\nWmntZJgzuVeeyad8zIPJV5Xoa6N2yXg0nmfArx/nXuvfgffeuoI+qvdxK4J/oj11v0/dtpXQtzrw\nW1uondQvexJt9s9DFgAA3sgjgvT4laY3Pn/LC2W/9xX/g/6wGyRJkj+AnwA8JctyjST9G24vy7Is\nSdL/uHIkSXoYwMMAYIk04r4xu7Dl0gAAgLZSglFAp7qPJQy1NJewyt+KUxB4lUYwJ5mLWCt6592x\nmo3cND4goDXLOtbPZ0mJfbxgVjwXvP8VLUyXaBidnUTD2nQqaGEvNbxOKl3gBbLCFsmvkTeVzI4N\nLcH5SzTCa0Ztxg9qxx9j2N+Q6kOGmsAgWOMBSwfKreZSKH4Yy3KAezazeeibO3igbtEyH5nZhOHp\nDZSHPYoL+4VLVAbE2+A6I5rKLuFp2/YU5a6t4efz6oG4bQy0bT/AQEqLbG5QQfPzkV5BI2nJ4HUn\nrolSsS48DM5ocQTffs9GqcGpDb/5Xn3IUR0ciMtPWBD3M/lhD9dhWSZLN5wi5uQL4P80kg+nDnPj\nj91BOcrPsGzk4Ks8lOsrnGi7gAd1q536Vi0aPSd/QzlkvqxBzELK/+4n9wMAlq6+BQBQkQIkB1GX\nX7qHo+nG7HocANB1JA3uqew4+ETJnb6pHt5zDbLkB0D9ydDRxIPSF+gUdAgrxJmv6ODb+vJ+thb8\n39yLQ9DkS8pj3hBuGImLGQCroU8I8zozxr3I0oOf3mQgpHSugGrauCjy+2qR+C3HdlrGUQeT36Fd\ntX1SCZub8l22hKUL8kg6tc1CKFtvtQXOw3RcfY/w+Vj3Bxj2N6X6kKPOLxAuiwY1Tclre4EFx0ey\ndmB4yAMAAGuR2J/a+6AR5UCRB8UBL4Cvtiai4aFVgklDuftnUTYV9/J/lp08KLafcgGZb7CJeMUj\nfM93nzHopXfKkO6ifa+rZmBnRRoDKe7x/H1st6NYv5v2whoPeHV/lGN/P6oXGRoDYbnmhekJlt4c\nKGgG2yjuZ00C6dT5tjD4UdXRjeQvqTvfVDGQEnWOfsWpJjywqTtL6GzgoW/r43QU7bdzf5MPUX9K\nulYjpJz7YqlwKl9670EAwGvPL8WRJJY4h4rmwf5t+J6Q4dTfbqHZeDWMB/+BZ1gSi1//CMf+nlQv\ncgwPgH+vUlSfok9h6+5EWxPlkC+Tj6UlTAJsy+gAScNbPt1hJwBgy2wGLFefYTAt5IKETr3ZTDj9\nFQa4avuIz6QWn69Mj7itorH4NOpn3BbKuqKl9sYI5isn6Nf4Won6E5HUMqer8fgjGwAA75+4FVA3\n3ENcveyLoRZIahmtvmE5Rl5OCkJ7cP8pPMm9L7Idf5cPGOA7SpvW5HWW6OVY+YiDGdTF8C16nJ/I\nUtrEt0T37c95yE7L55myuokKmz6nnibcXQAA+OD7sQAAVyc7gtTiLPNrAgDA25r3qUqkzS/rYIDv\nGvdny3xxZh3y+/z6u1J9yFFrDoI1RgWPUQQxQmthMZFvd0xjuOlwBcujKk+FIfowz2zxb9JfPFjF\ns+SlEwkAgFS3hMkjmHh3zqM8rTG0iVUdrwdEtajcyjXSbRTLiqrup+9jfTAIchMxaKGYf5PtVML2\nwkfNt1pgE2XtddEG+C423EG69SFDvd6CmM80uMZJxcjY3wtrK3iGkyzUiby3eLAPmVGOcpl7W6zY\nM3M6iURSM+qrIzscL+6lXoXGUXb2SN6npinrwbwex40SvbKfWYLui+MfmpzywSdGbkfvpV0dpWdg\nu2Us/dHE+GJcO8eAzpVHWKKHWb/PL+APjk2WJEkLMnaFLMvX3d9iSZIixf8jAZT8T9fKsvy1LMtd\nZFnuYgpqwF5XA6f6kqHaZPprPrBC/yPVmxz9FTn+X5Eiw38G1ZcctXr/v+YDK/TfSJHhP4PqS46a\nAL+/5gMr9N+o3vZFs7Iv/l9SvcnRT5Hj/xXVlwx12sYlw99FqEgMS30D4LIsy/P+y782AZgC4D3x\nuvH37lVZYsb6zwbC0VxkmasA3Q+MPq4cw4i+/hAj9EkTTsMzi1Ep/5mMGF4bywju7ns+5DU17VHt\n4Qb4Q2eOV5rUl6Uk3+9kFAy9q+CwM5ATEchMXMlkBtaaBlci6zARFH7FjFa5TSLsLyBcef4mBJTx\n8x5wGGD1NbyseH3KUPIyA+r5WaBCvMCs2USmDHueMPM9GwlJLrkUh8S7swAAFYuJ8rEPJurBdZCR\nyL2Pf4AFKZTdYTGONWov5VM9RWS3gyrg60G+17xPeWWITGkXAHNaciTz47dz5FbSfMoyczShfgsq\nBsFPhA5zh4ug6vrf+6Z/P6pfOUrQVqlRJ0B7fgUSHKWUaahAcBX5M9R+ckk7xN/FrKa5GzPfV35j\n9ubd+csAAK+cH40nLZcAAJdfSgAApE2njOwfMqvQM6Ac+x8kNH3xecJiAwRCbVB0Ooqd1P1xRx8C\nADRdSVmlJ7F0y88AOMTY1sqWgKcBZlPrVYYaGfoQO9yHaUOvZFig1ouGhSIKrymm7XNWa5E7lVlw\n/Vn+7/JsyrtnG2ZismuDEKOjzfXo+Z6qi9RTXXuuicHDTuNgW9rjrxeMAgCoCVJCZZkdP/RcBAB4\n/X6iHbIFrDC3mrpYFyvj0YlEwRyrTgAAHPm9L/o3pPqUo1cPVCap4Vd4HdaqwaD5LL2ytqfMLGcp\nx+r2LrhCaN+cZu5VZX2IGjt6yycAgO5bn7rReC1nOPluMlDPXEKu+zMSoetM/ZYFQlAj4MkVgxzo\nGsh1UJoTJD4jMzxqMdb1wEfdoUmmUXWGexpkVrw+Zah2eGC+UgXHe8x8Vt4DqPWUU3YGkQ2RheSh\nNV6NzDFEOWhECWTWw/zf5l4cs3nb1ifRwkh/9cAzRLOkBDCDVm6nz+NwaaGdwb850/hcr4EyeX75\n/ZC7MMuXLZpjWjjxHqU7mX07ej4cg8J6AgCOv7uQ3+P3vujfkOpTjh6PGmWlATAIWHjYT3pU2BIA\nAEYVbaAkstL6chUMnVmm8f3bRBrZ7laJz8R9sv/0o3gybB9/HkeEysQu9FFLXfQx9/zWAfmvU38i\n9NTT7HEsbw85IKN3ylUAQO67RIkW9OXzw0/ymvLWwIebORZUHVcHqeG5qPW7L0qAWu3Dnq+IoNOq\ngbNdqDMWUd5alUYdsPWU4AyjnhZlcq9ShwvZb+V5o8+zR2+UA3kWipIsUfMcEsQzhdWuh3MYbbXx\nbqKNWm2mcp872AIu038i25t9zVeXhWuqpIsGQUKeBZGW3/uKf1uqTzlCAnwaoLQXeaS7bEaFRB/x\ny0KigQJP0bYtmvUFnigj0iDzHJGXwQQcoUKMlvfZ1Pg+lUhLfSL3vusVCeoqMWI+QMbkO3YB+HfV\ng2sEZW3KA6Qh1Hev2AdtGq6R/9femYdHXV19/HtnJpNMMpnsCdlIIIRN9l2ggCwKomKlUq3bK4oU\nqxWwi11sRd9aa92qtoh9tWpLW8EFwRVQBEVAIptJICzZCNnJnkySWX7vH9+b2L5P3+eBQgd/M+fz\nPDyThF9mbu73d+69v3POPbfsedpmWLsfSTprLXLpKVj2m28r5fnU0OO0oGqSA/Hb2Cf143yw6goc\n3XrNUHkz+8hT60JULe2zehezOaO56x/RQ/hLjUkGVk3jw9szadRnaLQuw8GzDlBb2Be136K9pb/M\n+8OdTH0r5vlgr+HXfXUiWFQ0r219nM+WJ+cqRFWyHY8s5YEaV51hhsqZbPmZAuAmAF8qpQ7on/0U\n7NR1SqnbAJQBWHRmHylcAETD4EB0ND+iYXAgOpof0TA4EB3Nj2gYHIiO5kc0/DdRRgCL5sREphmT\nBi9BzYP0CCc9HI7uh5iFMLcPo9vP7aHnMSrejYlpLODU7OHeqH37GRV3VNNLbFiBjI8YvWnL4DUd\nKfQsNY/UnkGfQp9tvL5xiM6MGUEPcsvpKKgO/l/u3Ty2+fiTDLfa9JGsUSMb4M7TUdoxjTi+8n/g\nPl5pwhjA+SEiPdPIWroSVy34DADw/iuTYZvJqMr20cxWuLn4SgCA3erDga30Fn+6mFlFC5etAMD6\nNQDgifcheSe/TtAFu+Yls57OgVZ6DAsbU9Cyg6kUcdMZkav/nN87y4CoWt5P5fPYRlsr3y+atYQR\neXUNqk/T63/JABb9e3HCy18YhjHunDvEpDgTMo1hly2HJ5K3smEFnNd+dYQxAHwwlcWabij4LzS1\nscjTtkmMZH77brpsx/xiHwBg84YJSCigDrWj9VGvgxm98ZYwEueN9cLqZDR9RAYzXsrWMmNF+YH2\nS3m900FvdMdnzKCILtfHYkeq3qMjcyeWYdcdf0dzUU3I2mJ4ZqaRvnI5rB26PsYJIPt23t9Vj7Nf\nK+ay72yNNvii+PWwEbQz7zJGyQ+v5OvAO/bDu5mRu/Blen/qH6ipzcLf9f06GdYufn3iNuqcsJ1R\ngM4kBZ/eP+zuq8+i0xl99nrapGVwG9QBRpk6U3UE/3s/DGlbDM/MNNJXLMcl32ANor1/Gdn7f8Ys\nhmjaWjm/xcR0IEYfb9zUQZsMX89otjuJerT298F1lP3tKmd0rvlWZiukruI1RXdEwVlCjaMqqWfN\nVL46j9uQOb8UAHC4WB9XeIga23RxuMT9baieTB1ts+pRtPxFdByrCl1bzMow+vzsHlw5bj8AIO+x\nsXCdYIS68mfU4LK+rOnwev5oZKUyA+jUXvavXdcLm30t00gKlg9HyQJGP8Na9XiqlzRRldSgPV2h\noy/fO5NJXzg9hJqq8c3wHeKc1xVHXSMzeQ9EvUl7NyxA/QS92VwfDVp2d2jbYs8atWw+bSqsHTBm\naBssY38m6Ay/8EU18L/E7KP6BczCVCeYCdGti3rbK+1I386xsCOJ2tQzgReTLub9sHfbEIQ36KyI\neazB0/YKx2FnRTfKL6XtxbJMA5p1LUHLAM6XRpET/d7S2UjzXShb8wQ6K0+Gri1mZBoZ96yAN462\nEVUcBp1QgD6z2L8tnRxPW/cl9PZ9xwQ+S2S8wkms7DraRszn4b1FpO+5jkH5R/NYIyzlPWpTPdsL\nR6kuaaBNyq+zRbvj/Ejox3uosYjZpD4nDS7zvZ66Vz7YT3Ncr13Fdh+88lchbYuO1Eyj/y0rkTCb\nNWnCf+HC6eHMeu4ZC+vH6zkrowVWXdLDr8Vu1fbaU+nD1a8Jlne5th1yC7NyD780BADQOIzvE11s\nhWeqLlhcT1u26ucJn9MHawfH4qHjSgEAZW9wQRpTRs08DgvqxvDzY48ARzY8iY660LVFR2qmkb14\nJRIOs398dgvqRutn655u0bsHDAvQkUaxPCkU2HWAc+Cae/gscv3mZUjP5vNm07Y+AID2Qbw2qoj2\n54386ihkdw6fJeyV/D/lA3wDOFbHv8f1U/1ofd9E8B4Y9EI7ipbwPhs/jMesvz5lzRnZonkr5giC\nIAiCIAiCIAiCIFwgAnrYoTfSivrRMXDvo2fq+Lf9iF1PT9Dm47omxzC95z/Ojt179YkVA7S36gQ9\nhW69781nN1A1Re8nHkXvcuQefm9p5p/mj/Ei525GAnaXcO+/ZS+9lH9a/Bxu330LAGB2Pj38DaXc\nI9exk+1pKo9F2mRmRXT7rLD868LGIUNYq4HUT7uQ8i16cWOPeZF5Lfd6X34Xj8HqOXEg6YAXXfPp\nKhy/hUfwWmbShxfeyHvAAyAhjx7HkynZAIDnFV+jT9Jj6E62wKadmXV5zEzJWcsq6zds3IY1P14I\nAJg4knuND1XpqOoOeiAnpxzH32vpXCxuTTjnPggGvBFAwxALrMOYIeY/GIOGIvZtzBHa2fwTrOPQ\nHWNg0jiGx2av/hEAwD1TV+j+ktH0qZfno24jo2opK6jN/lJ9wpOOvn5v/ha89ktGdo4MZpjNewmj\nbPYDTiS/Qr0yf8JTEQ5NYDvau2mv3bFGb9SnaU1f+OpDu8i1xQM4qizAFGbcRW9z9I5PbWm6GoKP\n3xthBsKadfbeb1nPqOYBeuqT3mVErv6tHHS/z3Hvh5vWAQCefYRHALX21QY4GUgo9H3VAHwVMY26\nqAHhb2ut4nqiOrxP+q9nhK54URzSZjA7qUofmR3q2FsMZHzow84m2pJ7mAcRp6iJdRf7099Pn04W\nA5QdZWTGUcE+7mQyEsJGsY8jvFbEb2IE9pnnGdm57tl7AQAtg6jHggl52J7PmlURDdTTmaJPWet0\nwf0b2vIvn2JEdpXBejlhdWyX3xrde0y91WIgZENwGlu7QtIuKz5M4l56/8J2NOusLncT1ySvl7O/\n75q5Bc/s4f7vBL3Pv4MlULDldV5jHwkobburb34OAHD7Bh6VkPYpo27zV+7B6yW8ZxoG8T5ZcuO7\nAICDrZkofJOnXNTpuJrnMO0tolFnCP+4GD/vw0zTFytZc67sXDvC5HS7rKi8JBbdQzk2OlztMHTE\nO1GvW926xkbC/VEouk2Pb3/UNYt4OBqUrg/gizDQ0pc2M3gxo+K2x1kTrHQXXyMyFKwzmbE0K5nz\n7Mu5tL/GwXYkHuJ7tSzkGtXTQtvu/xznv6o7W1Fxkc6G6HDD0JHWUMUS4UPEwGb0eYLRbY8TCOvg\n+NnQwH7t0TC2xI/TNBOkrw3T13OejDjG71v7Gcj9C+fY38Yx+9qI5dwXWcNXe5UdEXVap28w0yR6\nF9czvgFdSLqVY3PLPXr9aeiMzW7aYnOOHS2L+fXU5HIAwMFz7glzY/EAkVUGGt/het42woAvgrop\nnQkbv59anQ5zYuhvuO7068LStRN0JsRCXXtqeyo6JvJZclcxnwUjEnlNdAn1uOSmz/FWHo9Qjkjg\nGJPhFEcAABLUSURBVDByONcrhW8MRnQ5NTqcyjnYNZvPLjUHqWvyPj8yRzHTuzwsFT4T1vk7n9ib\nvMh6oxZ1U3haTnx+Gypn0C7C9HNGRDP7dPz9eTjURPusfo/PDtZu2tT3Cq4HADiLbWg5pjNTBtL2\nep713X047v1y3mtYV81aOQVHecpPz46TsDbAUqXtUj8+aFNEVDnf59TPDcTbaO9HXx10Vn+vZKgI\ngiAIgiAIgiAIgiCcJQHNUPHZgfa0r6pqZ2w24I6ne8gbydfmi+h1iioJQ2cSPU5/u2QNAODhAayk\nfuqv9C629FfoGEEvoutTeiXdM+jFj/yc+7u70rqx90Puk5swk5kqJe/Q63Tvo0uhdE2GrX14jU/v\n+XfU0zMWN70OtfsZufcke+D1mLEO/vmj26Vwco4dz3w2CwBgv9iKqnJGvCf9mP1rXcX+PTXNhjA6\ncGHt1LUURtNT33WQEbWkXVbUTGNUPLyJfT5hKfeh5zcwbJe8ygXbr+h9PlHLa088SL0fOng5/JOo\nScV+hsotbl0pWnuoXysahcQE3hd1baF1jNf/R3RMB6bP24+dr9MbH+4GFi3k6QNbc6lfawMjmrlP\n+fBFK2vh+BJpk/kLnwYADF/HrKQ9Wy9CvI6SlXxEbS25zBrLfpURgk0zRiD5+wzJ3prEehFPvXAN\nAKAj3Y+TQ3XF+81DAQBDZh4DALTPo3ZHj6XBkcj3bGhxwbv9/PSFWXHEdWL4Nw9j10He976Lbaj4\nmNp5BnOMXXQxazK8vW5yb8ZXx3fpfZ+SyL3Je67KAgAorw3tWfy9/97AzBRQdhg688WX2oUFi9jx\nz22ZAwDw62hRu9sOzGKWw8AHGaUr+gnt7cjdHI/trna0dDJyOCad+9mPnWtHmBxrajdc951E83pO\nRhZvGDoyqEPOEGo0NZF7eTf+YToGb2NGYMUVnJfcg5ixYNORdO/RaHQs53h51RusWWVN7Dn9iWPj\nB8VDAAaN0DyUPxv4MMfRqikWlM3n9Q/vZ2EqayRt05uu63GMbkJNPutHJEd0wmoJ7ai4NaEbcTef\nRHM9ayRY851I/YQ20JLNUNhF380HADyXPxU5f2Z/nV7B+dDXyWs8bhqTG0CEk7r+upQnZs2dxnnx\n4zqeiPjqazOw/DsbAACNObSzNa/y2oRCHxx3MFJqb6LtqQK+1o3msq9qZy7uScrmH2DC0wv/Iyie\nutX/WX2qVbsFJQupad/jnHvqx3H8qpgZjYEvcm46cS3rhI2bzAyTg5s5cEZWGejUUfBjq7nGrJ1F\n2w5ror1ZuwGH/vgOP++DsPavIqo1l/E+cG1nTYiBC5jBUHwpo7iWw3a4+9A+bQ02oDu046RRYd2Y\nkFqObUuYLTYquxTlLzONL2wBx8XwV7iOrJruR+5fmLVw4lvM/InQJ414XHrMdPhxZFm0fnfeF5lp\nzGS3dFOTsFaFOUuZ7XVwCVNeyu/js4yvKgodE7IBADZd72zZN98DAPzeN0+/D+Btpe1/XHB2UfFg\nxevyo2GeG5YTtI6kg/7erMiWm5gh365PcO2zyYGGicxcSFjKPLulKR8DAB75fC4AIHFKHaL1Osb7\nJie/FT94FQDw8+1ch24tGwRriz4FZgDH5qK/0pYTi7pwejhtf3wWP+Pz0mwAgFOX8rvqwa34wy5m\nH7qym2Gx+85DT5gXj8uGysuS4ZnOTPj6CVGw6Do08YXMTj96E5/l/L8ej7ZUjold6TqzejJ17uvk\ntcdGRSLWxXHYu4s2PO4Kzqt736fd5bX1681MGTuEhTRPHOJY4HUAzTnUJO4APytthz6BaKT+zMIY\nROp6VS1z+Vl4+sz+3oA6VGABvNEGDH2cYO3YMFjduoDQPDpGwvWljvRmXJ3BFMkHFt4MAPjuOp1+\nbOXCM3JQI9pPcED7zp3MrVr9BYva+pLZOeH7nejI5Of1j+LTfWM+F6I1U+KQ8REHvWLFyUn1rAtn\nsiPbd/eB66Q+xjfLC1hCe8sPrAa8Lh9SMjjYuA8nQ1VzMXHyT7xpL3+cx46t3jYbVu3c6LuKk03f\nPVz47dFvVx/nQp/PeA/UzOfC4dhKLjxaR3Ag9f+iGnWf6Yc+PT45qnUxr04DTXr+iT6uj7TTqVzX\n3MAHvx21A2C38BdHpDJ9r+DcesH0tHZFYEfZAPidehCxKLyyjwWZB976BS9axO8rZlnQncAF26Dl\nTEQd6eUWLr+L/drtAFqyOJx0ZXCBEtazPe55Plh0dUVgfzl1zHHSFmNK9HHlLgtyfqftcyJTPL90\ncBEUqWtA537ejuPf0UXJcjtghIf2Q5xN+ZEY3gaEsZ8ddQb8dvZV6pPc/rbpOU4y3pFtaBmrF4MP\ncpTdcc1wAEDyPv5+7ViFeO3d6Irl+3TH9Ozr4EvGGzY8X8m89iGTSgEAVX/LBgBEjWhF1QEuahrG\n6K2Xuk5pIuc8xB3uRPlcToT57Ynn2gVBgdttx5eH+yLjFG3BuqwGnS+zH90f0xY2JHN+8kUBvtUc\nJzNv16nIw7g4TPsN7e+qF9/Gu/NGAQCiXqBDpqSQzumIOl2EPbkWznRuratoYwHOkqt5Tb+Ly5D0\nEOfIhkHUce7SnQCAja9OBQB4t6TAnk1tT6/PgLcxtLffdXXYcWJfZu9Rti2LmhH2JvVsnc4Htf1r\naW/ewT4k/4rFo5v+pB+8dSp7a5Z2XMZ7oPbRoV0axYe5Mi8DF5k7uVYqu9OPxzbwuNyedUvWh1y3\ntPSLQOd66hnr5nvOvJcO80grx+e3H52B6FtrAADFh9LPSz+YHX+4gfYcD06k6O2nm+2w81kA3bF8\n4I08RRvqjjNQvLwnwEZNDq+jnhF6h17LAMB1nP1fN4d22+d9Xch0Gu+PtLcMlMcxCLGugPbV7xGu\nl7rmj0fKbuqlijg4187RgSY9bteNUVDaieIc3AirI7Qf4lrbHNj+6TCkDaPj+Wh9MnwZer34EZ3A\nbl3M1H7aiuqJHONyf8jgQ8kD3C5gZFJT1EVgyCoGgkpX63FxGzXIbOU6eNVtb+C+PD6Up2ZQ34it\nukTBSB+a+uutknqYLHZzzDb0EbvetC7YSzlORI2ks6b8nHvC5HgsQIUD8YV6m/cAK/za3LqP87kv\nSS9VfTeeRkwE15nlGxlw/90UOkKhHVW+eIW23ez3J378AgDg++sWAwAcA+kYte6IgX8A7aeiiZ8R\nX8nvayaEI+Nh2uXRNh43b0/RgeIWtnHj/bPhGKqPVf80Fpam0A7A+xwGmi/ywNpFDRLyrL3bthou\n4nPjkCe4jmkfmgLnfDo82wpop8ZealCUzDkwos4CRyGNqHER57oeR0q4PmK58AfDsejJvQCA1wsZ\nMLZkUJ9x3ziCmvvpP/Dp8gEfv/BHAMDAl5YBAAY8UoDqG3jE/cwB9KwUn+HfG9qubEEQBEEQBEEQ\nBEEQhH+DgGaoOKPdmDIjH3s2MVIz75rdGBTJ7QDr72Cxyr6/pRd+jKsMz+TPAABYrqB36idrmanS\nOZzuf0txDHJHM8r24jr+vtJHBHrjdbpduA3w0SP29y30/jun6+JifQw0TaSXasiDTI9tHkMPdNdR\neq3ti6pRmc6CQ3E7HLC2hbgPSgEI82Nkoj729ko3Kt/O0l/TG3tHLMPR24cPRJePt1jN9ycDADof\nonbuqbzWZgDVlzNy44zWx8axfiI6C+ixb66NBWKoa2wB+z9uIbcLFB/tg3B9JGvqJwwlNQ3m/ZJ3\nOdtVen9K73GWW166WP8h68+xI8yNpcWCqA+cmL6MuUL7GjJxKo/R8OhPmDlQuYl9/fTta3DHpiUA\ngFN3M93c4qXdXDqWW3fqOp0YM4W22Oih7Ww5ydShkm3ZAABPtAG4qP+VsTp93cUsmOR9PhT+iFH5\npCwW6Mt6jDq2ZDGjoviaSCy7ZDMA2rvqDG1bbPfYsbsmu/dcwOZBBqJLdLG9t/RxyW8wyh0/pxqn\nTjFiUz6XHv7U0RzzcmYwMyhqZX8cu5l9bXUxKjo3l1mC7+xjgXB3nA05Exg7KzjMrAmM1wX6nkuG\nbwbb4k6kNv3eYhTh1DTeEy3Z0Vi8iNmEb98363x0g/mxGrA4PUi9l/16oCIdESnsv/ZM6jh+ArdT\n7snPgfcjZirYWR8RzkK+ll9GXT9pzMXJb1GbFEWbHDWKMZbqnTkAgLrf90NTGyNv1ZM4Rqfuom2W\n+PsiTqfcJh6ifm+t59zZe8ShFfA6eE1knR8W77l3g5mJd7XhujmfYq0ez2anl2L35fr4a72bpudI\nyKcu/TMeeJzF8Nt0QWELp0AYqZwDB964Hym7aLvlD3EcLWMyCkqW8X2sxZEwcrjFLmED7ct2hPdQ\nw12ZuH5oHgAg7zKmP3+opgAArvnRVgDA8vtfxWu1HM+tqR3n3AfBQFgzkPmOwsmr9PbTsXakfUpN\numNoJz1Htnoj/VA1XKP49bzWOpA2FVHFNYmjWsEbqd+8iVFa121cu9Qc0QUYL7bCcHIMjcvjNcee\nnggAUD4FZ5k+5noa183p1+4DAJSv4Dy95IoPcEvMIQDAxE0r4AvxLT8q3Ad7dhvGJdEWNhaMgG0w\n7WTJ8E8AAGs28HnhomnHUdLI9X3HCZ5nrQZye8GiXGbjvlk0FSefZ2ZDwivMkK2YQ72LVlDcB1ff\niCR9/Lzy6uOWS3mNqxxo0+Np51C+96FfMIMwicmBePnXz+LK0h8AAPxb5eAEAIiM6sToyUfxhY1b\nmjO3eFCzmLaYFce1ftcunVUZW4/Df2Nmu5fJXsh4nPb66Fru17jpwK2920OWr70NABAzlmufrBhm\nBe0bEAXXUV1+wkKtI2P1sfXjm4APOZZ6NurC1LpAf/31ekeDxY/OCq5b3Vk+eLeF+I4Gv4LFbcXA\ngfoggvBsOCv/OYOubjrHwfqZXRh8ry4Efanu8xnU5e5cbbfPLsCdv+Gz208+4rb0b8zjM8iOY5xM\nPdO7ezNT7IXMNOwpPPvFR4MRp+fhvt+lr2HanSz2nupju4Zsa8N42w4AwJ/3TzqrPze0R15BEARB\nEARBEARBEIR/A2UYgfOgKaXqALQDqA/Yh547ifjn9mYZhpF0oRpzoQkSDQHRMRh0FA3NryEgOgaD\njqKh+TUERMdg0FE0NL+GgOgYDDqKhubXEDhDHQPqUAEApVSeYRjjAvqh54DZ2hsIzNYnZmtvoDBb\nv5itvYHAbH1itvYGCrP1i9naGwjM1idma2+gMFu/mK29gcBsfWK29gYKs/WL2dobCMzWJ+fSXtny\nIwiCIAiCIAiCIAiCcJaIQ0UQBEEQBEEQBEEQBOEsuRAOlecvwGeeC2ZrbyAwW5+Yrb2Bwmz9Yrb2\nBgKz9YnZ2hsozNYvZmtvIDBbn5itvYHCbP1itvYGArP1idnaGyjM1i9ma28gMFuf/NvtDXgNFUEQ\nBEEQBEEQBEEQBLMjW34EQRAEQRAEQRAEQRDOkoA5VJRSc5VSRUqp40qp+wL1uWeKUipTKbVNKVWo\nlCpQSt2jf/6AUuqUUuqA/nf5hW7rhUR0ND+iYXAgOpof0TA4EB3Nj2gYHIiO5kc0DA5CTceAbPlR\nSlkBHAUwB0AFgL0ArjcMo/A//uFniFIqFUCqYRj7lFLRAL4AcDWARQDaDMN47II28GuA6Gh+RMPg\nQHQ0P6JhcCA6mh/RMDgQHc2PaBgchKKOgcpQmQDguGEYxYZhdAP4O4AFAfrsM8IwjCrDMPbpr1sB\nHAaQfmFb9bVDdDQ/omFwIDqaH9EwOBAdzY9oGByIjuZHNAwOQk7HQDlU0gGc/IfvK/A1vvmUUtkA\nRgPYo390l1LqkFLqRaVU3AVr2IVHdDQ/omFwIDqaH9EwOBAdzY9oGByIjuZHNAwOQk5HKUr7f1BK\nOQG8DmC5YRgtAFYDyAEwCkAVgMcvYPOEM0R0ND+iYXAgOpof0TA4EB3Nj2gYHIiO5kc0DA7Ol46B\ncqicApD5D99n6J99rVBKhYGdutYwjDcAwDCMGsMwfIZh+AH8EUxjClVER/MjGgYHoqP5EQ2DA9HR\n/IiGwYHoaH5Ew+Ag5HQMlENlL4BcpVQ/pZQdwHUANgbos88IpZQC8AKAw4ZhPPEPP0/9h8u+CSA/\n0G37GiE6mh/RMDgQHc2PaBgciI7mRzQMDkRH8yMaBgchp6Pt/DbvX2MYhlcpdReADwBYAbxoGEZB\nID77LJgC4CYAXyqlDuif/RTA9UqpUQAMAKUAll6Y5l14REfzIxoGB6Kj+RENgwPR0fyIhsGB6Gh+\nRMPgIBR1DMixyYIgCIIgCIIgCIIgCMGEFKUVBEEQBEEQBEEQBEE4S8ShIgiCIAiCIAiCIAiCcJaI\nQ0UQBEEQBEEQBEEQBOEsEYeKIAiCIAiCIAiCIAjCWSIOFUEQBEEQBEEQBEEQhLNEHCqCIAiCIAiC\nIAiCIAhniThUBEEQBEEQBEEQBEEQzhJxqAiCIAiCIAiCIAiCIJwl/wvj3w4qUrdnnwAAAABJRU5E\nrkJggg==\n",
"text/plain": [
"<Figure size 1440x144 with 20 Axes>"
]
},
"metadata": {
"tags": []
},
"output_type": "display_data"
}
],
"source": [
"images, labels = next(iter(mnist_test))\n",
"noise_dim = 128\n",
"\n",
"def get_noise_batch(batch_size):\n",
" noise_shape = [batch_size, noise_dim]\n",
" return tf.random.normal(noise_shape, dtype=images.dtype)\n",
"\n",
"def get_label_batch(batch_size):\n",
" label_shape = [batch_size]\n",
" return tf.random.uniform(label_shape, maxval=10, dtype=labels.dtype)\n",
"\n",
"logits = gan.discriminate(images)\n",
"noise = get_noise_batch(images.shape[0])\n",
"gen_images = gan.generate(noise)\n",
"\n",
"num_images = 10\n",
"plt.rcParams['figure.figsize'] = (2*num_images, 2)\n",
"for i in range(num_images):\n",
" plt.subplot(2, num_images, i+1)\n",
" plt.imshow(images[i])\n",
" plt.subplot(2, num_images, num_images+i+1)\n",
" plt.imshow(gen_images[i])"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "9yV4JM345J71"
},
"source": [
"Print the names, shapes, and other information about the variables in our little GAN model to get a rough idea of its structure."
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 615
},
"colab_type": "code",
"id": "9ti32zve4_QG",
"outputId": "4b3e61aa-a999-4c92-99a8-c83caef12ad7"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Variable Shape Type Trainable Device\n",
"little_gan/discriminator/block_0/spectrally_normed_linear/b 1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/discriminator/block_0/spectrally_normed_linear/spectral_normalizer/u 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/discriminator/block_0/spectrally_normed_linear/w 784x1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/discriminator/block_1/spectrally_normed_linear/b 1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/discriminator/block_1/spectrally_normed_linear/spectral_normalizer/u 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/discriminator/block_1/spectrally_normed_linear/w 1024x1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/discriminator/outputs/b 1 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/discriminator/outputs/spectral_normalizer/u 1x1 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/discriminator/outputs/w 1024x1 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_0/batch_norm/exponential_moving_average/average 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_0/batch_norm/exponential_moving_average/average 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_0/batch_norm/exponential_moving_average/counter int64 False /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_0/batch_norm/exponential_moving_average/counter int64 False /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_0/batch_norm/exponential_moving_average/hidden 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_0/batch_norm/exponential_moving_average/hidden 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_0/batch_norm/offset 1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_0/batch_norm/scale 1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_0/spectrally_normed_linear/b 1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_0/spectrally_normed_linear/spectral_normalizer/u 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_0/spectrally_normed_linear/w 128x1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_1/batch_norm/exponential_moving_average/average 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_1/batch_norm/exponential_moving_average/average 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_1/batch_norm/exponential_moving_average/counter int64 False /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_1/batch_norm/exponential_moving_average/counter int64 False /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_1/batch_norm/exponential_moving_average/hidden 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_1/batch_norm/exponential_moving_average/hidden 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_1/batch_norm/offset 1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_1/batch_norm/scale 1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_1/spectrally_normed_linear/b 1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_1/spectrally_normed_linear/spectral_normalizer/u 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/block_1/spectrally_normed_linear/w 1024x1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/outputs/b 784 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n",
"little_gan/generator/outputs/w 1024x784 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n"
]
}
],
"source": [
"print(snt.format_variables(gan.variables))"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "V297xpzfobXK"
},
"source": [
"## Training the model"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "WTrv-jn4pPSx"
},
"source": [
"To train the model we need a loss function for both the discriminator and generator, as well as an optimizer.\n",
"\n",
"We'll optimize the discriminator via the \"hinge loss\", defined by the function `hinge_loss_disc`. The generator simply maximizes the discriminator's error via a linear loss given by `loss_gen`.\n",
"\n",
"For the optimizer, we'll use the \"Adam\" optimizer (`snt.optimizers.Adam`). To compute gradients we'll use a `tf.GradientTape` which allows us to selectively record gradients only for the computation we want to back propagate through.\n",
"\n",
"We'll put all of this together below in a Sonnet Module called `GANOptimizer`, which takes as input the GAN to be optimized:"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"colab": {},
"colab_type": "code",
"id": "JgVCqok0usw-"
},
"outputs": [],
"source": [
"def hinge_loss_disc(preds_real, preds_gen):\n",
" loss_real = tf.reduce_mean(tf.nn.relu(1. - preds_real))\n",
" loss_gen = tf.reduce_mean(tf.nn.relu(1. + preds_gen))\n",
" return loss_real + loss_gen\n",
"\n",
"def loss_gen(preds_gen):\n",
" return -tf.reduce_mean(preds_gen)\n",
"\n",
"class GANOptimizer(snt.Module):\n",
"\n",
" def __init__(self,\n",
" gan,\n",
" gen_batch_size=100,\n",
" disc_lr=2e-4,\n",
" gen_lr=5e-5,\n",
" loss_type='hinge',\n",
" num_epochs=100,\n",
" decay_lr_start_epoch=50,\n",
" decay_disc_lr=True,\n",
" decay_gen_lr=True,\n",
" name=None):\n",
" super().__init__(name=name)\n",
" self.gan = gan\n",
" self.gen_batch_size = gen_batch_size\n",
" self.init_disc_lr = disc_lr\n",
" self.init_gen_lr = gen_lr\n",
" self.disc_lr = tf.Variable(\n",
" disc_lr, trainable=False, name='disc_lr', dtype=tf.float32)\n",
" self.gen_lr = tf.Variable(\n",
" gen_lr, trainable=False, name='gen_lr', dtype=tf.float32)\n",
" self.disc_opt = snt.optimizers.Adam(learning_rate=self.disc_lr, beta1=0.)\n",
" self.gen_opt = snt.optimizers.Adam(learning_rate=self.gen_lr, beta1=0.)\n",
" self.num_epochs = tf.constant(num_epochs, dtype=tf.int32)\n",
" self.decay_lr_start_epoch = tf.constant(decay_lr_start_epoch, dtype=tf.int32)\n",
" self.decay_disc_lr = decay_disc_lr\n",
" self.decay_gen_lr = decay_gen_lr\n",
"\n",
" def disc_step(self, images, labels, lr_mult=1.):\n",
" \"\"\"Updates the discriminator once on the given batch of (images, labels).\"\"\"\n",
" del labels\n",
" gan = self.gan\n",
" with tf.GradientTape() as tape:\n",
" gen_images = gan.generate(get_noise_batch(images.shape[0]))\n",
" preds_real = gan.discriminate(images)\n",
" preds_gen = gan.discriminate(gen_images)\n",
" loss = hinge_loss_disc(preds_real, preds_gen)\n",
" disc_params = gan.discriminator.trainable_variables\n",
" disc_grads = tape.gradient(loss, disc_params)\n",
" if self.decay_disc_lr:\n",
" self.disc_lr.assign(self.init_disc_lr * lr_mult)\n",
" self.disc_opt.apply(disc_grads, disc_params)\n",
" return loss\n",
"\n",
" def gen_step(self, lr_mult=1.):\n",
" \"\"\"Updates the generator once.\"\"\"\n",
" gan = self.gan\n",
" noise = get_noise_batch(self.gen_batch_size)\n",
" with tf.GradientTape() as tape:\n",
" gen_images = gan.generate(noise)\n",
" preds_gen = gan.discriminate(gen_images)\n",
" loss = loss_gen(preds_gen)\n",
" gen_params = gan.generator.trainable_variables\n",
" gen_grads = tape.gradient(loss, gen_params)\n",
" if self.decay_gen_lr:\n",
" self.gen_lr.assign(self.init_gen_lr * lr_mult)\n",
" self.gen_opt.apply(gen_grads, gen_params)\n",
" return loss\n",
"\n",
" def _get_lr_mult(self, epoch):\n",
" # Linear decay to 0.\n",
" decay_epoch = tf.cast(epoch - self.decay_lr_start_epoch, tf.float32)\n",
" if decay_epoch < tf.constant(0, dtype=tf.float32):\n",
" return tf.constant(1., dtype=tf.float32)\n",
" num_decay_epochs = tf.cast(self.num_epochs - self.decay_lr_start_epoch,\n",
" dtype=tf.float32)\n",
" return (num_decay_epochs - decay_epoch) / num_decay_epochs\n",
"\n",
" def step(self, train_batches, epoch):\n",
" \"\"\"Updates the discriminator and generator weights.\n",
"\n",
" The discriminator is updated `len(train_batches)` times and the generator is\n",
" updated once.\n",
"\n",
" Args:\n",
" train_batches: list of batches, where each item is an (image, label)\n",
" tuple. The discriminator is updated on each of these batches.\n",
" epoch: the epoch number, used to decide the learning rate multiplier for\n",
" learning rate decay.\n",
"\n",
" Returns:\n",
" loss: the generator loss.\n",
" lr_mult: the computed learning rate multiplier.\n",
" \"\"\"\n",
" lr_mult = self._get_lr_mult(epoch)\n",
" for train_batch in train_batches:\n",
" self.disc_step(*train_batch, lr_mult=lr_mult)\n",
" return self.gen_step(lr_mult=lr_mult), lr_mult"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 989
},
"colab_type": "code",
"id": "UUkshshiK6Eq",
"outputId": "6272bb79-7eff-46af-a913-b9e886772948"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r 0%| | 0/750000 [00:00<?, ?images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"WARNING:tensorflow:From /tensorflow-2.0.0-rc1/python3.6/tensorflow_core/python/ops/resource_variable_ops.py:1781: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"If using Keras pass *_constraint arguments to layers.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"WARNING:tensorflow:From /tensorflow-2.0.0-rc1/python3.6/tensorflow_core/python/ops/resource_variable_ops.py:1781: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"If using Keras pass *_constraint arguments to layers.\n",
" 4%|▍ | 31300/750000 [00:19<01:15, 9470.95images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 1/25 (lr_mult = 1.00, loss = 0.016450781375169754) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 8%|▊ | 62500/750000 [00:21<00:56, 12207.04images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 2/25 (lr_mult = 1.00, loss = 0.04956576228141785) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 12%|█▏ | 92400/750000 [00:24<00:53, 12197.78images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 3/25 (lr_mult = 1.00, loss = 0.10389317572116852) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 16%|█▋ | 122200/750000 [00:26<00:51, 12084.79images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 4/25 (lr_mult = 1.00, loss = 0.051943909376859665) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 20%|██ | 151900/750000 [00:28<00:49, 12153.37images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 5/25 (lr_mult = 1.00, loss = -0.21330127120018005) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 24%|██▍ | 181800/750000 [00:31<00:47, 11858.25images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 6/25 (lr_mult = 1.00, loss = 0.15133343636989594) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 28%|██▊ | 211500/750000 [00:33<00:44, 12015.89images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 7/25 (lr_mult = 1.00, loss = 0.2119884043931961) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 32%|███▏ | 242500/750000 [00:36<00:41, 12089.66images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 8/25 (lr_mult = 1.00, loss = 0.25304022431373596) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 36%|███▋ | 272400/750000 [00:38<00:39, 11972.72images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 9/25 (lr_mult = 1.00, loss = 0.31501442193984985) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 40%|████ | 302100/750000 [00:41<00:38, 11631.19images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 10/25 (lr_mult = 1.00, loss = 0.3819904327392578) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 44%|████▍ | 331900/750000 [00:43<00:35, 11762.71images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 11/25 (lr_mult = 1.00, loss = 0.359415739774704) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 48%|████▊ | 361700/750000 [00:46<00:33, 11701.33images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 12/25 (lr_mult = 1.00, loss = 0.3620455861091614) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 52%|█████▏ | 391300/750000 [00:48<00:30, 11574.55images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 13/25 (lr_mult = 1.00, loss = 0.34004005789756775) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 56%|█████▋ | 422000/750000 [00:51<00:27, 11761.06images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 14/25 (lr_mult = 1.00, loss = 0.4108557105064392) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 60%|██████ | 451400/750000 [00:53<00:25, 11703.31images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 15/25 (lr_mult = 1.00, loss = 0.4520989954471588) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 64%|██████▍ | 481900/750000 [00:56<00:23, 11577.90images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 16/25 (lr_mult = 1.00, loss = 0.46710577607154846) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 68%|██████▊ | 511200/750000 [00:58<00:20, 11707.29images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 17/25 (lr_mult = 1.00, loss = 0.4239480197429657) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 72%|███████▏ | 541700/750000 [01:01<00:17, 12108.87images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 18/25 (lr_mult = 1.00, loss = 0.40935561060905457) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 76%|███████▌ | 571500/750000 [01:03<00:15, 11730.40images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 19/25 (lr_mult = 1.00, loss = 0.46318596601486206) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 80%|████████ | 601400/750000 [01:06<00:12, 12025.96images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 20/25 (lr_mult = 1.00, loss = 0.45752182602882385) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 84%|████████▍ | 632400/750000 [01:08<00:09, 11889.06images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 21/25 (lr_mult = 1.00, loss = 0.3791620135307312) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 88%|████████▊ | 662100/750000 [01:11<00:07, 11857.17images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 22/25 (lr_mult = 1.00, loss = 0.2888197600841522) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 92%|█████████▏| 691900/750000 [01:13<00:04, 12035.37images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 23/25 (lr_mult = 1.00, loss = 0.4309738874435425) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 96%|█████████▌| 721800/750000 [01:16<00:02, 11965.87images/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Epoch = 24/25 (lr_mult = 1.00, loss = 0.28471168875694275) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 750000/750000 [01:18<00:00, 9546.01images/s] "
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch = 25/25 (lr_mult = 1.00, loss = 0.1431596875190735) done.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"import tqdm\n",
"\n",
"num_epochs = 25\n",
"num_disc_steps = 2\n",
"\n",
"# We'll turn the step function which updates our models into a tf.function using\n",
"# autograph. This makes training much faster. If debugging, you can turn this\n",
"# off by setting `debug = True`.\n",
"debug = False\n",
"\n",
"optimizer = GANOptimizer(gan, num_epochs=num_epochs)\n",
"step = optimizer.step\n",
"if not debug:\n",
" step = tf.function(step)\n",
"\n",
"train_dataset = iter(mnist_shuffled)\n",
"num_examples = mnist_train_info.splits['train'].num_examples\n",
"total_batch_size_per_step = batch_size * num_disc_steps\n",
"steps_per_epoch = num_examples // total_batch_size_per_step\n",
"\n",
"steps_with_progress = tqdm.tqdm(range(num_epochs * steps_per_epoch),\n",
" unit='images', unit_scale=batch_size,\n",
" position=0)\n",
"\n",
"for step_num in steps_with_progress:\n",
" epoch = tf.constant(int(step_num / steps_per_epoch))\n",
" train_batches = [train_dataset.next() for _ in range(num_disc_steps)]\n",
" loss, lr_mult = step(train_batches, epoch)\n",
"\n",
" if step_num and (step_num % steps_per_epoch == 0):\n",
" tqdm.tqdm.write(\n",
" '\\nEpoch = {}/{} (lr_mult = {:0.02f}, loss = {}) done.'.format(\n",
" epoch.numpy(), num_epochs, lr_mult.numpy(), loss.numpy()))\n",
"\n",
"print('Epoch = {}/{} (lr_mult = {:0.02f}, loss = {}) done.'.format(\n",
" num_epochs, num_epochs, lr_mult.numpy(), loss.numpy()))"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "2K0_eoR8og-G"
},
"source": [
"## Evaluating the model"
]
},
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "Cm_9RMJopgWc"
},
"source": [
"Having trained our little GAN, let's check what its generated images look like now."
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 99
},
"colab_type": "code",
"id": "PM7IPcOeXtxH",
"outputId": "4215ea70-f36a-43bd-fa9b-b232cea584ef"
},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAlMAAABSCAYAAABwglFkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXd4XNWZ/z/n3hmVUZcsy2q2bMsd\nTA89QMgGAkmAhADZEDYkgV96T5bsZtOWlE1I2fSwoaRCSAIxSQgdQrWxwQ1jLHdblmzLsnqbmXvP\n74/33NFYFrZslRmNz+d59Ei6unN133vKPef7vuc9SmuNxWKxWCwWi+XocFJ9AxaLxWKxWCyTGTuY\nslgsFovFYhkFdjBlsVgsFovFMgrsYMpisVgsFotlFNjBlMVisVgsFssosIMpi8VisVgsllFgB1MW\ni8VisVgso2BUgyml1MVKqQ1KqU1KqZvG6qbSCWvj5CfT7QNrY6aQ6TZmun1gbTxm0Vof1RfgApuB\nWUAWsBpYeLTXS8cva+Pk/8p0+6yNqb83a6O1z9qYWTYezddolKnXAZu01lu01lHgbuCyUVwvHbE2\nTn4y3T6wNmYKmW5jptsH1sZjltAoPlsN7Ez6vRE4fehJSqkbgRsBXNxTIhSO4l9OLDnk4RGjUJXq\nfnoAruMYtDFT7DOHuoBfDz0vU2w8luspZL6NmWKfOWTbItbGdKefHqJ6QB3uvNEMpkaE1vpW4FaA\nQlWqT1cXjve/HDP26EZa2c1CdSrL9GPEiA57XqbbmCn2ATyq/7RvuPMyxcZjuZ5C5tuYKfaBbYtY\nG8cepUCP7X7Dy/RjIzpvNIOpXUBt0u815ljGkE0u/fQlH0qNjY4Lvjcul04bG8eJYezLIoPsg8wv\nQ7A2ZgK2LWYG6WSjCmcB0P/GEwBovDZO4dM5AFQ+1ASA17QbAD0wMK73MpqYqeXAHKXUTKVUFnAN\ncP/Y3FZ6UEgJfXTTp3vQaLA2TjqS7fO1D1BKBtkHmV+GYG3MBGxbzAyOBRuPhqNWprTWcaXUR4GH\nkOj+27XW68bszsYTZdyfKmksOYzy4yiHefpEVvI0ffQC3DOhNqrDumlHTcptHGeS7TMNf38m2Qdp\nWoam7qqsLJz8PAB0v8wM/d5eOecI5Pi0tHGMyXQbbVvMDNLJRpWTDcCuC2Qoc8PiJ3m4YgEAW8tq\nAKj7aScAXhorU2itH9Baz9Vaz9Zaf32sbiqdmKIqOUtdTD5FWBsnJ4F9Z6s3A+xO9f2MB5lehmBt\nzARsW8wMjgUbj5RxD0BPF9w5swDY+e0cfrT4bgA6ffGtfupv1zHv1lYA9A7xs/o9PSm4S4OZ1Ycq\nppp76UVHJcgv+D7WQXbjRqCujfR+k9W4yWJjgFIo1wXAKSuVY0UFAPTNKiXUHUucBxBu2IW/vx0A\n7RlldJxi48YFR2xVjtjT+5aTaVksXcpXr/0dAIuydtOv5bx3Lb0BgNnfkjrsr14/obc75phydIuS\nVin5Ume9btN/aD+967EpQyc3J3FIhaUMVXERAN6UQtx9MrvXxi6/q3vwGqbuamP7pKrDmYrjDpap\nqX9+/8Brlk0QewSg47EDPpeuOAUFtFxzHACnnf0qAKdHNrO9qAyA3i1VAPjdE/Mut9vJWCwWi8Vi\nsYyCY0aZanzbNAD+ccq3qXQjAAxoUQXuuvxHXJ39EQDmfWJ7am4QUNni/23+0CkAdM2WWUTObpe6\nv4hyxh5ZTey3dwzOBCWY80BSOatIVmgKRJnR06eh4nKferM8Yz8aS3zEyQrL9/IpAPQsrmSgUK5R\nsnyPnL9TVMPxXpVxxASz+7wIarrMhgam5QOw419kxufl+bgyYcKLi6JRU5FHW6/YG19VDEDdLasB\nE1eU5jPDxCzXkW4kf2MHOy8Tu0/JlsU9daFI4vT3H/ccAPcdL0uki9eFBhW5dLc1Kc4yUOIC9dGv\nKmfrlaJORatEdZv3w345vWFbalTuZEV4iPrkTCllz5sknqTbrMeeda60yY/UPk652wVAuSPtbL+f\nRYsn7fjWptcDsOFB6aPKV8bIWyXpBnU8KEsfb1/reFlmGY5AJV0wB4CtV03hHZc/DcA/dkoMkf/Q\nFKoekD4UX/rifedWA9B2aQ+Fj0hs49RnWgDQjc2p9dC8Bm6htLW2Sxcy/T2bAPhIpaQvKHYG2NIl\nHW37PNGKSkzMptc2fAqHsSLzB1OmknXNjgNQ7IRwTeB5tjG/yw9Td7905sq81HVsfB/8UJxIhIab\nFwNwzxU/AKA2JIONB3pm8v2eKwEoaZCKlLdyJ7pHAnn9YHARyO3x+ITddzIqZNwDubn0XCgNuPFy\nuae3Hb+aVV88CYDcHWbg5Eg5OCXFNF1RB8C/fvAhAN5VeDcVbi4A3b7Y90Cv9Py/vfqiQRdRigeN\nMOgWcUqL0QNSb7Kb5YU057ttcu7AAMq4g2I10tg76qfh18o1yteZMgvsSffBRRLBoF67is+e/jAA\nNSGZGLjKwTOD/XPyNgDw4L7z5YOuCymqq0eM6TPckiJUgQwYvSlSnjtugs8sWgLAgC91+w+zLgag\nYL03/EKS8SrfoE6ayYz2NW6JuOuix9cBsO0GzYdPkHb2loK1ANS4ct/ZKoSsJ4IBLROBctdnXrgD\ngDkz7gNg1Xtl0rDyqhn8bumZAJSulM9Ne7gJZdyAaTfxORRHGpKQDgQD5eNkEHXir14B4NdTnqfI\nkcHzZ8qWAXBv/Sy+vkASlc+eL4Oq39d/B4BKN4snTpX6fNPP3wdA7Z1t0CeTgsSEPYXPJhAb2i5d\nCEDlhzZzY9VTAMS09MGf3vw2Om6XiULtVhkIJurgOKYYAuvms1gsFovFYhkVGa9MBTO0eXdIkrEN\nb3ZYnCWjUx8ZbW8cqKVzhjyK7CcmeKZsZkMbvrmYl9/xQwAijswIO8woekXXTLLbZEYwUCz2cHIt\neevE/eXkiYITBM/r7u4JnUEkFCnzXc+vY/91Miu496TbAYji8OxHZBFAbpPMHHpmiuug7t9f5cdV\n3wZgunELuSo/cf0S45a9Jl/k52/8l2LG9fJZr7NznKwaAeYZB4sC4jubEi6gYYNxzb2qRnGBla6J\nUGZULcyz8/oOSIY3KQiUuf0nlLAo+8DcfTHtJdpZuy9ltu84UUFqN1fjbdoqJ6arGmBm/qEqCRPQ\n/f0J18hpH38JgFsrHiPHtOMPbr0cgH2L5XOFj+eCcWcHfRHaHzf1OKFIBe5T5aBypX/onyLP/ZpF\nz1GTJW64AmX6FS33s88fVOSDmfaAhrA68JiHWUShPC47TZ7DktCJAJS/VAiNzWNr2NEw3AIBEIXY\nlEl8hizy6aiPsPd0eRala8TKwq3yLLqmZ1G2Wtqu0yh9kNfSmhaB9qFaUQjDPxIF/IvlKwDIVrmJ\nc7qMqrSyewZnnSTq8I3TngQG3fCucjgtW+pEdodpi8WFCYUx1V4PAMcowl3vFNX/i1VP81TXPACe\n+0/ZzSby/CaKe1cCg/2yH6RAGi4cZizvb1yvbrFYLBaLxZLhZK4yFSzbNrMy1S2+39/vP4OCKRKY\nl2NmZf/z1KUsvF9myPHoxMZKhWbOAOB7l/w2oUgFPNNfAsDWt5dT1iKjbYLZreNAqQQtx0pldqHL\nxW/uPrUa9MTNmoLZSuDTjpZkc/tJPwNgUZZUsQEdo7pA4i7aq6cD0DZX/va1qc9QZWJsghlyGBeH\nA2NN4ohNZfm9YGLb0oJEnJM3ssmPGowVi02XAHTnxVcPvNZkIJj5l0o9zX1PMydliyIZQsqzT0d5\nKSqxGy1xUQi8M4yauESSekL6pfwIVNYgyFwXyaz41c/V8qELJS7skvyXAShQDi9FRXXb2FoOQM2T\n0t/oaDQR+J1QpsKhRAJTbRSSsYrRPFg58NEx+R8DhVLvlrfOoKFbFJknsyW28fmmOgB6erMJr5WA\n3f6FopJev/h5FuSI6r20ezYAa9tFESnO7mN3j5RrTqOUpdPbheelSLUJ6mRxMe1vEtWi/pMSR/Th\niicAiDgxek1sW7krNnb5YaJGW+i9ROpusdNnzo/z67YzAFj+QYn7VK37x1voODxK4ZVLPNz5Zc8D\n4Br7fTTrolIXvrf7EgCeXjWfunrxZswPB4Hl8n6MaY9v7D0PgNJ1Eour9wxunZhQ2lNBEMtWXGh+\nlXtpiRfy8E/OBqDswRcA8IZTCyfoXThpB1NOnjR4lZszmPskaXWYayL4VcQEMc+WDn9qVlci781H\nNr0TgLIVLn5HatxFvXOl831T7n5kqypo86Qy/+T0NwDgtTYOfiAIMA2FwayeUaaity6STnvaqny8\n9o5xv/ehBC+GaKHLNDcIPJXnvz2u2f99GTjmL5MVGNn7xF3y+fPegetIz7S7WcqpZEoXH58rnd9p\nObLS6J4O2SC1ac005oWNO2wyBY2aAf6Wb7wOgB9ceQc/2P4v8qerTVBz8qAi3W0zg8KWi2YC8OVZ\nd5CjpEvxJcM1e7w4X9okrq8d68VVlrdLPqdz4wnp3u8Q6X6iF34Mi+PilEg9jM2VOto1Q16wFA/e\nX1hJnd0eD/GNrZcCUH2zaZ/rZXCsY/HBwZSxFWfQITDuAdpaJ1w0kRb5HvVd1rdUABBbKy/jymel\n7VY1tNA7T/qhgc1i85q6avJdGRwWhaTdNeyQsszZlE1ui5R11SYzQNy6c+JdYMn9IuDX13DuTUsB\n+NrU5QCEEoH1Di8MyHk/b5U+5d51J+I0Szldf/HjAHy8VFbWhlUWMfPO6KmR/ixvWepdfMp16amV\n91yRGRS2eFIGKwam8acWsW31XyRgO19DxWJpZ/2mTwkmqD9rn8PKL58MQO4L4rb1PS+t+h6/WGwN\nOTJIfLh1EVOXyLtk2EHUBGPdfBaLxWKxWCyjYPIoU2ZWr0+XjKfbLpJRasXyGLmPrT3gVOUolFkO\nHORTqbpW3HhvLhg8d9NWmZ0t+EvDYODvRI3EjT39n9gPcICL75KX3wNAYevmgz9n1ADlOolcIVvf\nZlyZM0Sh69o7n7z7XwQmNmAwyJzrxDRdvtjnaXmuVyz9MLMfFrdI8KzdmNxb+ftz8Y2SVuRJzhqV\nk813PirpIN7/7gcBuKdBZk6Vz/uJgO3E/ooT6NY8KpRi828k9cX6838MQFi5zK+/C4CPh8XWRP1L\nXk5v6ko6BLwCiXtz8sS9vH+RHJ4TbqXLuD46jFr6ue1X0LhGVIxpIhDgZSfZWCYKEClQUl8L5ShU\nyLTPcmmXRoRi2gNZ/NS7AICf9b0RgJwml9rHTCqAlbJFWaIX0T7aExUkKFHd0YlOlPM4z2eVSrgs\nWxfJ9/qsfjAiWWy1uCfdATEwNq2YaIHcU+vxcsdXFG+joVfK8MF1UtiFL4pqVdIQJdRjXPN7RN33\nYykIUk642sWO3qpc3lYkCkuXCaoP6uQXdl5G9/XyfvC37gCg3l9DqEK8BM+cJq7Mz5WJe3Cf18eS\njccDULfEBDeniWKz+3STssMs8lgdlbCBn+88jw0bxRVbaMTPzsVRLp8i9x8xbfhzzWcBsOm6meQ2\nyPNKZZD5ofDypB19Yp642b/6xOXM3bd85BdQalzf71aZslgsFovFYhkFk0OZUordH5Olj//90TsB\n+O0eSRTXfVcJXhArZWYlKpRN7wKZSb3pw88CcEa++Fb3exGe65FA7QXfluWk3v72CfcNB8GoX5zz\n98SxILlh0ZdMQHniZIVjgruDXbK159NzoiSx9KslnuHLJz4AwFf6L2NOk/jJnVUNck5//zhZkoR5\nhvnrW3mkRwJbg1iDWd/VktU7Ca9L/Pd0dR2syPRDdrscW9Ul6mJBRGyI5uUnBSynOgp0ZIRm1PLk\nuT8CIJyU9uGDm94FgLO78cAPHBAzlWY2BmVlAptz98p9/nTfeYmYmuf2SRqMxqdrKZDV5LTNl+/5\nO+Xzza8voXiTuUaTBMbqoE6kgkTcTSixwKHgVekjVJtRXaaW0F0japoJUyHUr4lH5Pzw0NQYWuOb\nxIfBjN/JzQETHBykNfHaxy9WrH+htJ/TLhdV/sopy9kdk8Ur3znhCgBqJEyIaHEW7fXSZqefKSpx\nkdvHxk5RbYqWS/9Tul7kjpwNzWBUPL/FZD1Ph/qq4KU+ieUrd0VhuvivnwFg/lc24e3bcvBnTL3+\nyax7AHAYzN4//fsmzi8dYvoCXJfIHqlvs7P2AlAVkvbzzsoXWZonO3y8WCnl/7fFdyY+umxAEgdv\nukb+5m3ZnD7K9xAcs4is5wuiXp+XK2WXsyd0yPd2sCBKBUH50di4ejCsMmWxWCwWi8UyCtJbmTIj\nyo53n85jn5W098Gc57NPzQVgdtPLgzOhIJ6oppKK/5J4ow+XyX5gHSaG59VoBb//vewNNr1D1KpU\njMiDbWvaPbNvkO5lc9zEEpkd2r2k7VmCVYnB9huqvCyRaPTfTxF165I8WfWWd9YfuWu2rBjru8bs\nH7araVztARKxPX0zS/jlBlmy2tsts4MF+1o5yBOfPKsIUlkE+57NrKXtBCmX1XtkRVX/WplNz3y5\nEx2svkyT2IXDEasqSdTBGnNsn9eDe4nMKIeLwUgspTekWyxDcD/Vj8uM8aWNJxPulDJrOUnK3cmC\njrnSPi8+axUACyKS0PEvzSfQ+Xsp21yjco13XMMhSfq/ulfUJD9QzMyKOLevj8geibc5/mOi9JxW\nuJXv/lm26Zi13CTQ7e5Ouq7Zk9Ks3NNKQdisOusbX8VYhcLsPVXK4mtT/wnAnHAfz5ntN7IWi3qx\nU0nbitZEufmsPwBQ7kob2xKdyvYWsxp6tzyH7J1tCZuUSTOQSNrreSnbJihQBHNaotzaIH3QHyMS\nazl1qVEoOoZRPx0Xdbe8P6Yn7ScJcM5TH6N+2ZrxuuWjx/MwGTkSaRwqXLHhsvzNnBuRd2BOpTyT\nHt/h/q4TAPjzLRLvV7JFUgqkqyoF4JjUK39c+GsApphtxvqnxYdd8exEjGdnniiTXqAab28h3rRb\nThoHe9N6MBWqkY72tpu/T4kjD7DVl0pTtjbIMO0nXGB6kQQOVv90G9+vlo0PHZNu4KdmCeyfXj2J\nun+aHBtJqRQmmiCNwG92Sf6Sc+v/kHCNxSuks3Y7pdGrcDix51uwzJlwiI758gzOj2yUQ2YweVZO\nE5uKZWD191MlWDZ3PAdTQUCycUF6uQ6xmAwEwtnSqfbOLSen2WxYPMQt6+TmDkqyRdI7bL+igki5\nvKTzc+QllGWSZTs7mvG9NHAljAQzSAxv3cOrUVnwMCMkA6grb/wk2QOvHUB5yI2s04DE4G6lpAHI\nXTlYB2pWmn203jyPgVKpH1eVSsdd6oq7t7y2k1vca4ZcNPWDYz8ag30mx85Q97Pr8t7/+CsANxSJ\nG8xH82fjEgsWhRxgx5BrOCXFaD8ISTCZ78c6RULgspw3i4uvkRxEC8MycAsrlzlh8b1eNUsCkucu\nlAHuidlNxLT0Iz1mwOUozayp4sLrDEtoQbRGBl/ZfQN45fKz02j6poGB1KX1MC/JUMMu8u+R98G+\nerOJ+HJpd772D9ov0Z0zk7vrfys/K0mRsNeT98S8T+5Ii6X3w5FjqmmQkd43zzusHIpNupktMXl3\nLu+bxW3rJDxmzkPiKksMeVM5iTkUSrH+ZgnZqQzlH/Cnb17wR34VkXCWYFLiFhXSe2Y9ALGPS509\nr0ImPQ/cfg5Vv5FJznikDrJuPovFYrFYLJZRkJ7KlJnNb/6AZMquD4dwjepSZFIIlNwoS1o3XDqX\n02duA+DrNb8AoCaUC4i0tzUuI9Z7HhXJt+pZj9CWbeb/DLOb+wQRpBHgJnHDffF/L2FXj8ygcnaL\n9O73yAze1xptlhsHrh+npY3pi0wQqMkIGzZJ6Qocxa4BuVZkh9k5ezxnHnpQJQRwoppQSGZy88pl\nNrj2gnqmRURizt0tZdJ4obg4s9uhb6pcI1oi1/jUG/6OY9ajFxgJ+8uvezsA5UuS9h5LcxJ7peVH\neLlPHHxPdIgCmfPISl6zRJJnzuk4Y0wmadaeWGRg0l+ULt2N8mVmeaP3/wCY9Tppuw07plFmPqrT\nQWk8xDMPynHPnVO4oehRgESfhPbZ+oIoNjN7hiwkGOYahELEayUAONRgzh+r9BdDklfGS3K5tEiS\nT0YcORbTHl2+9KMX5Etw9q64CapH45laud8TJWDAD1OcbTKFv1fa8+5eUTt6uysof8SkSQiZzP5b\nPLz9qQ3U9lr3U3y/1MUSo5b6JrmzTk5GaZ5Xf20RuerAHSguXvk+AMpbGybilo+K8pVi00Ndkrqh\nKCQ2d8QjhE1yyxc7JFly3HcofFT6XEzYiGPCTbTnD7qj0ymcQDl85nUPD/unk3MaufljsoCnYIdR\nej2IXyeK1I/nibu6zDGq7/tgxYsmg/3zxm07hn2rVaYsFovFYrFYRkFaKlNB8rTISTLCdJLGfMHP\nt86W5asVc3MJqyBQd9CnOqBF+XmyV9IgzLpPRuzu6k34QSBoKvcbMiNid6PMTNfcdRx9FXJsdp4k\n8gxmCNrXicBsFTYxFjOncVmV7DE4xah1MbM1wB7P58V9MlMuapSZpDee6kaw9NQ818gLWxi4UJ77\n6XMlTubCt73KivPrAHhylSgztTMlTmNKbjcvPyN+7qCom6NFvLdUYj2qzIx+/7kyQ3mo+gwcs7dZ\nIoA33WIa1IHlpeIedz5/DgDZe+TYDH/Za34O5aRtrNRICPbcQ2uKH5NA2JJ/SjnGa6V9V9eEKHhW\n/pZ2liaXA9D/LzKjfeHkXwwqUoaGWD+zvmSS5B7ymiYWaXElkQ37DvzbGNXf4LkH35vOzU0ovMls\njsnefF98UtTeKdUSQ1Ka20vDDonty9sgis78SxtYVCht9Z3lKwA4LWcwBnPVmXKtTz4oyYYXfDcf\n2kxMSqqS6foefo+JjQ3U0uH6QHNs95nZiS2QmuMSVzPteokrG9e+c5S4qyVe9pk3mGUtU0XxjJXl\nsec0UZ8WXynqY2NPMe0XSH8ZaZEyM/Hr6P1teJ1JiybSBOUo5mcPH+9b6sDl18g78C93nwtAuBva\nmyXmuGmuqK212RJ0/u6SZfz9ddIHV75gPAZjqMKl5WAqqODdvdKY93h95JmOaEtcbrnU9Gcx7SUG\nU8EAqsOPssq4uW77pqyyKVkunZ2fTnlCAN/k1al8qp3Ob0lF395tMkYvk8FhuKOfvkqRZ/fPF/v7\nTu7lorxXzFXkWODme7ynHm6TF5bulaDvcQ0wHHrdeJyI2X+tOSoVuy4nRp/JBH38AnHzNP9KVlus\nmwk1T0vZhdtlkPTKcZVEyoJgSrHrjFx58d5x4SVU95ts67ukoeh4POEKTQxCUtkJBpsZm4mBzgpT\nslrsyNtj9lR0XfTQl2giw72LjqfdEAMVCg1OQoYZAAQB1R1vl8HH7jfGmfczqdfOJgnSDu2QAX7h\nnqzBFW7x1C0GSZD07FWOrAjqfqMEuN70nV8BHDCQipmBwqff9j507NXDXt6tlna97/gwU2Py0stZ\nObYvsCD/WrAfYDyi6fWlH+3yZXCxbKCMW/73agAW/l3KRBtXGOSwwJeVekH+qPZl07nbTI5Ouvo3\nAFS7gyveSnNl8nfFWbKY4sUHTiFiAnxTsUfoQYwgF9GbL19KsyeuzGs/9mkAcltfGP97GwXa89DB\nzh3B91Ypi3BZKeVZdQC8cr4MjkOujzcgZbrzMpPTcIZkSa96KITqkWsc1CelEuVwT6usTD+/WnJG\nDu776fC7F2QBl1su9oS7HMJ75T2zob8SgHlhIyigKL10l1z3J2PvlLNuPovFYrFYLJZRkJbKlN8l\ns7X6L8rs560XfR4nZgKUi0V+711sgpJP/SvVYZlJPdYpe5+1xvJY2lQHQNWfJPgy3RSpgCCQur86\nj5Z2KY6qC0TWXPB2kdY749ns6S2U8x+RoPzjqpupNQGfwb5+vWYPqmK3l8INZkaYikDtUIjy1aI4\nPJwnswr/+C7CK0RULtwms4jyf4
gitextract_ho0gv3se/ ├── .github/ │ └── workflows/ │ └── ci.yml ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── WORKSPACE ├── docs/ │ ├── .gitignore │ ├── Makefile │ ├── api.rst │ ├── conf.py │ ├── ext/ │ │ ├── BUILD │ │ ├── link_tf_api.py │ │ └── link_tf_api_test.py │ ├── index.rst │ ├── modules.rst │ ├── references.bib │ └── requirements.txt ├── examples/ │ ├── BUILD │ ├── README.md │ ├── distributed_cifar10.ipynb │ ├── functional_mlp_mnist.py │ ├── little_gan_on_mnist.ipynb │ ├── mlp_on_mnist.ipynb │ ├── simple_mnist.py │ ├── simple_mnist_test.py │ └── vqvae_example.ipynb ├── readthedocs.yml ├── requirements-test.txt ├── requirements-tf.txt ├── requirements.txt ├── setup.py ├── sonnet/ │ ├── BUILD │ ├── __init__.py │ ├── distribute.py │ ├── functional.py │ ├── initializers.py │ ├── mixed_precision.py │ ├── nets/ │ │ ├── BUILD │ │ ├── __init__.py │ │ └── resnet.py │ ├── optimizers.py │ ├── pad.py │ ├── regularizers.py │ └── src/ │ ├── BUILD │ ├── __init__.py │ ├── axis_norm.py │ ├── axis_norm_test.py │ ├── base.py │ ├── base_test.py │ ├── batch_apply.py │ ├── batch_apply_test.py │ ├── batch_norm.py │ ├── batch_norm_test.py │ ├── bias.py │ ├── bias_test.py │ ├── build.py │ ├── build_defs.bzl │ ├── build_test.py │ ├── conformance/ │ │ ├── BUILD │ │ ├── __init__.py │ │ ├── api_test.py │ │ ├── build_test.py │ │ ├── checkpoint_test.py │ │ ├── checkpoints/ │ │ │ ├── BUILD │ │ │ ├── README.md │ │ │ ├── base_batch_norm_1x2x2x3/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── base_batch_norm_scale_offset_1x2x2x3/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── batch_norm_1x2x2x3/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── batch_norm_scale_offset_1x2x2x3/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── batch_norm_training_1x2x2x3/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── bias_3x3x3/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── cifar10_convnet_2x3_2x2_1x3x3x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── conv1d_3x3_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── conv1d_lstm_3x3_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── conv1d_transpose_3x3_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── conv2d_3x3_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── conv2d_lstm_3x3_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── conv2d_transpose_3x3_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── conv3d_3x3_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── conv3d_lstm_3x3_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── conv3d_transpose_3x3_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── cross_replica_batch_norm_1x2x2x3/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── depthwise_conv2d_3x3_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── dropout/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── ema_2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── embed_100_100/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── generate.py │ │ │ ├── group_norm_2_1x3x4/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── gru_1/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── instance_norm_1_1x3_2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── layer_norm_1_1x3_2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── linear_1x1/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── linear_nobias_1x1/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── lstm_1/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── lstm_8_projected_1/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── mean_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00002 │ │ │ │ ├── checkpoint-1.data-00001-of-00002 │ │ │ │ └── checkpoint-1.index │ │ │ ├── mlp_3x4x5_1x3/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── mlp_nobias_3x4x5_1x3/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── resnet50/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── sum_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00002 │ │ │ │ ├── checkpoint-1.data-00001-of-00002 │ │ │ │ └── checkpoint-1.index │ │ │ ├── trainable_state/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── unrolled_lstm_1/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── vanilla_rnn_8/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── vqvae/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── vqvae_ema_eval/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ └── vqvae_ema_train/ │ │ │ ├── checkpoint │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ └── checkpoint-1.index │ │ ├── copy_test.py │ │ ├── descriptors.py │ │ ├── descriptors_test.py │ │ ├── distribute_test.py │ │ ├── doctest_test.py │ │ ├── function_test.py │ │ ├── goldens.py │ │ ├── goldens_test.py │ │ ├── keras_test.py │ │ ├── optimizer_test.py │ │ ├── pickle_test.py │ │ ├── saved_model_test.py │ │ ├── tensorflow1_test.py │ │ └── xla_test.py │ ├── conv.py │ ├── conv_test.py │ ├── conv_transpose.py │ ├── conv_transpose_test.py │ ├── custom_getter.py │ ├── custom_getter_test.py │ ├── deferred.py │ ├── deferred_test.py │ ├── depthwise_conv.py │ ├── depthwise_conv_test.py │ ├── distribute/ │ │ ├── BUILD │ │ ├── __init__.py │ │ ├── distributed_batch_norm.py │ │ ├── distributed_batch_norm_test.py │ │ ├── replicator.py │ │ ├── replicator_test.py │ │ └── replicator_test_utils.py │ ├── dropout.py │ ├── dropout_test.py │ ├── embed.py │ ├── embed_test.py │ ├── functional/ │ │ ├── BUILD │ │ ├── __init__.py │ │ ├── haiku.py │ │ ├── haiku_test.py │ │ ├── jax.py │ │ ├── jax_test.py │ │ ├── optimizers.py │ │ ├── optimizers_test.py │ │ └── utils.py │ ├── group_norm.py │ ├── group_norm_test.py │ ├── initializers.py │ ├── initializers_test.py │ ├── leaky_clip_by_value.py │ ├── leaky_clip_by_value_test.py │ ├── linear.py │ ├── linear_test.py │ ├── metrics.py │ ├── metrics_test.py │ ├── mixed_precision.py │ ├── mixed_precision_test.py │ ├── moving_averages.py │ ├── moving_averages_test.py │ ├── nets/ │ │ ├── BUILD │ │ ├── __init__.py │ │ ├── cifar10_convnet.py │ │ ├── cifar10_convnet_test.py │ │ ├── dnc/ │ │ │ ├── BUILD │ │ │ ├── __init__.py │ │ │ ├── control.py │ │ │ ├── control_test.py │ │ │ ├── read.py │ │ │ ├── read_test.py │ │ │ ├── util.py │ │ │ ├── util_test.py │ │ │ ├── write.py │ │ │ └── write_test.py │ │ ├── mlp.py │ │ ├── mlp_test.py │ │ ├── resnet.py │ │ ├── resnet_test.py │ │ ├── vqvae.py │ │ └── vqvae_test.py │ ├── once.py │ ├── once_test.py │ ├── optimizers/ │ │ ├── BUILD │ │ ├── __init__.py │ │ ├── adam.py │ │ ├── adam_test.py │ │ ├── momentum.py │ │ ├── momentum_test.py │ │ ├── optimizer_tests.py │ │ ├── optimizer_utils.py │ │ ├── rmsprop.py │ │ ├── rmsprop_test.py │ │ ├── sgd.py │ │ └── sgd_test.py │ ├── pad.py │ ├── pad_test.py │ ├── parallel_linear.py │ ├── parallel_linear_test.py │ ├── recurrent.py │ ├── recurrent_test.py │ ├── regularizers.py │ ├── regularizers_test.py │ ├── reshape.py │ ├── reshape_test.py │ ├── scale_gradient.py │ ├── scale_gradient_test.py │ ├── sequential.py │ ├── sequential_test.py │ ├── test_utils.py │ ├── types.py │ ├── utils.py │ └── utils_test.py └── test.sh
SYMBOL INDEX (1523 symbols across 122 files)
FILE: docs/conf.py
function linkcode_resolve (line 130) | def linkcode_resolve(domain, info):
FILE: docs/ext/link_tf_api.py
function tf_role_fn (line 44) | def tf_role_fn(
function tf_doc_url (line 86) | def tf_doc_url(text):
function setup (line 122) | def setup(app):
FILE: docs/ext/link_tf_api_test.py
class LinkTfApiTest (line 23) | class LinkTfApiTest(absltest.TestCase):
method test_non_existent (line 25) | def test_non_existent(self):
method test_link_to_top_level (line 29) | def test_link_to_top_level(self):
method test_link_to_nested_package (line 34) | def test_link_to_nested_package(self):
method test_link_to_method_of_exported_class (line 39) | def test_link_to_method_of_exported_class(self):
method test_link_to_non_existent_method_of_exported_class (line 44) | def test_link_to_non_existent_method_of_exported_class(self):
FILE: examples/functional_mlp_mnist.py
function main (line 26) | def main(unused_argv):
FILE: examples/simple_mnist.py
function mnist (line 25) | def mnist(split: str, batch_size: int) -> tf.data.Dataset:
function train_step (line 52) | def train_step(
function train_epoch (line 71) | def train_epoch(
function test_accuracy (line 83) | def test_accuracy(
function main (line 97) | def main(unused_argv):
FILE: examples/simple_mnist_test.py
class SimpleMnistTest (line 23) | class SimpleMnistTest(test_utils.TestCase):
method setUp (line 25) | def setUp(self):
method test_train_epoch (line 29) | def test_train_epoch(self):
method test_test_accuracy (line 46) | def test_test_accuracy(self):
FILE: setup.py
function _get_sonnet_version (line 7) | def _get_sonnet_version():
function _parse_requirements (line 17) | def _parse_requirements(requirements_txt_path):
FILE: sonnet/src/axis_norm.py
class LayerNorm (line 27) | class LayerNorm(base.Module):
method __init__ (line 64) | def __init__(self,
method __call__ (line 129) | def __call__(self,
method _initialize (line 181) | def _initialize(self, inputs: tf.Tensor):
class InstanceNorm (line 210) | class InstanceNorm(LayerNorm):
method __init__ (line 222) | def __init__(self,
FILE: sonnet/src/axis_norm_test.py
class LayerNormTest (line 25) | class LayerNormTest(test_utils.TestCase, parameterized.TestCase):
method testSimpleCase (line 27) | def testSimpleCase(self):
method testSimpleCaseVar (line 35) | def testSimpleCaseVar(self):
method testSimpleCaseNCHWVar (line 48) | def testSimpleCaseNCHWVar(self):
method testDataFormatAgnosticVar (line 62) | def testDataFormatAgnosticVar(self):
method testSimpleCaseTensor (line 80) | def testSimpleCaseTensor(self):
method testSimpleCaseNCHWTensor (line 91) | def testSimpleCaseNCHWTensor(self):
method testDataFormatAgnosticTensor (line 105) | def testDataFormatAgnosticTensor(self):
method testInvalidDataFormat (line 128) | def testInvalidDataFormat(self, data_format):
method testValidDataFormatChannelsFirst (line 136) | def testValidDataFormatChannelsFirst(self, data_format):
method testValidDataFormatChannelsLast (line 143) | def testValidDataFormatChannelsLast(self, data_format):
method testInvalidAxis (line 150) | def testInvalidAxis(self, axis):
method testNoScaleAndInitProvided (line 155) | def testNoScaleAndInitProvided(self):
method testNoOffsetBetaInitProvided (line 164) | def testNoOffsetBetaInitProvided(self):
method testCreateScaleAndScaleProvided (line 173) | def testCreateScaleAndScaleProvided(self):
method testCreateOffsetAndOffsetProvided (line 180) | def testCreateOffsetAndOffsetProvided(self):
method testSliceAxis (line 188) | def testSliceAxis(self):
method testRankChanges (line 204) | def testRankChanges(self):
method testWorksWithFunction (line 218) | def testWorksWithFunction(self):
method testShapeAgnostic (line 231) | def testShapeAgnostic(self):
method test5DDataFormatAgnostic (line 254) | def test5DDataFormatAgnostic(self):
method test3DDataFormatAgnostic (line 277) | def test3DDataFormatAgnostic(self):
method testInstanceNormCorrectAxis (line 300) | def testInstanceNormCorrectAxis(self):
method testInstanceNormCorrectNCW (line 308) | def testInstanceNormCorrectNCW(self):
FILE: sonnet/src/base.py
function no_name_scope (line 36) | def no_name_scope(method: T) -> T:
class ModuleMetaclass (line 58) | class ModuleMetaclass(abc.ABCMeta):
method __new__ (line 61) | def __new__(
method __call__ (line 108) | def __call__(cls: Type[T], *args, **kwargs) -> T:
function safe_compare (line 147) | def safe_compare(a, b) -> bool:
function auto_repr (line 158) | def auto_repr(cls: Type[Any], *args, **kwargs) -> str:
function fancy_repr (line 214) | def fancy_repr(name: str, value: Any) -> str:
function indent (line 221) | def indent(amount: int, s: str) -> str:
function wrap_with_name_scope (line 228) | def wrap_with_name_scope(
function wrap_with_name_scope_no_exception (line 266) | def wrap_with_name_scope_no_exception(
function with_name_scope (line 284) | def with_name_scope(method: T) -> T:
function allow_empty_variables (line 337) | def allow_empty_variables(module_or_cls: T) -> T:
function assert_tf2 (line 359) | def assert_tf2():
class Module (line 368) | class Module(tf.Module, metaclass=ModuleMetaclass):
method __init__ (line 390) | def __init__(self, name: Optional[str] = None):
method variables (line 413) | def variables(self):
method trainable_variables (line 443) | def trainable_variables(self):
class Optimizer (line 474) | class Optimizer(Module):
method apply (line 478) | def apply(self, updates: Sequence[types.ParameterUpdate],
FILE: sonnet/src/base_test.py
class BaseTest (line 27) | class BaseTest(test_utils.TestCase):
method test_basic (line 29) | def test_basic(self):
method testWrappedMethod (line 33) | def testWrappedMethod(self):
method testControlFlow (line 39) | def testControlFlow(self):
class TestModuleNaming (line 46) | class TestModuleNaming(tf.test.TestCase):
method test_single_name (line 48) | def test_single_name(self):
method test_construct_in_scope (line 53) | def test_construct_in_scope(self):
method test_enters_name_scope_in_call (line 59) | def test_enters_name_scope_in_call(self):
method test_enters_name_scope_in_other_method (line 64) | def test_enters_name_scope_in_other_method(self):
method test_subclassed_module (line 69) | def test_subclassed_module(self):
method test_submodule_created_late (line 76) | def test_submodule_created_late(self):
method test_does_not_evaluate_property_methods (line 84) | def test_does_not_evaluate_property_methods(self):
method test_overridden_name_scope (line 89) | def test_overridden_name_scope(self):
method test_patched_callable (line 94) | def test_patched_callable(self):
method test_property (line 101) | def test_property(self):
method test_property_no_name_scope (line 108) | def test_property_no_name_scope(self):
method test_ctor_no_name_scope (line 115) | def test_ctor_no_name_scope(self):
method test_ctor_no_name_scope_no_super (line 120) | def test_ctor_no_name_scope_no_super(self):
method test_invalid_name (line 126) | def test_invalid_name(self):
method test_modules_not_numbered_in_eager (line 131) | def test_modules_not_numbered_in_eager(self):
method test_module_numbering_in_graph (line 140) | def test_module_numbering_in_graph(self):
method test_ctor_error_closes_name_scope (line 150) | def test_ctor_error_closes_name_scope(self):
method test_ctor_error_handles_ctor_not_opening_name_scope (line 159) | def test_ctor_error_handles_ctor_not_opening_name_scope(self):
method test_forward_method_closes_name_scope (line 168) | def test_forward_method_closes_name_scope(self):
method test_get_attr_doesnt_enter_name_scope (line 175) | def test_get_attr_doesnt_enter_name_scope(self):
method test_get_attribute_doesnt_enter_name_scope (line 189) | def test_get_attribute_doesnt_enter_name_scope(self):
class VariableNamingTest (line 204) | class VariableNamingTest(tf.test.TestCase):
method test_variable_names (line 206) | def test_variable_names(self):
class AutoReprTest (line 213) | class AutoReprTest(tf.test.TestCase):
method test_order_matches_argspec (line 215) | def test_order_matches_argspec(self):
method test_defaults_ignored (line 219) | def test_defaults_ignored(self):
method test_does_not_fail_with_hostile_input (line 223) | def test_does_not_fail_with_hostile_input(self):
method test_args_are_repred (line 230) | def test_args_are_repred(self):
method test_long_repr_multi_line (line 236) | def test_long_repr_multi_line(self):
method test_repr_wildcard (line 251) | def test_repr_wildcard(self):
method test_repr_non_bool_equality (line 259) | def test_repr_non_bool_equality(self):
class ForwardMethodsTest (line 274) | class ForwardMethodsTest(tf.test.TestCase):
method testFunctionType (line 276) | def testFunctionType(self):
method testEntersNameScope_call (line 281) | def testEntersNameScope_call(self):
method testEntersNameScope_concreteFunction (line 289) | def testEntersNameScope_concreteFunction(self):
class AbcTest (line 298) | class AbcTest(tf.test.TestCase):
method testAbstract (line 300) | def testAbstract(self):
method testConcrete (line 305) | def testConcrete(self):
method testCallMethodsOnParent (line 312) | def testCallMethodsOnParent(self):
class CustomGradientTest (line 317) | class CustomGradientTest(test_utils.TestCase):
method test_custom_gradient (line 319) | def test_custom_gradient(self):
class ZeroGradModule (line 331) | class ZeroGradModule(base.Module):
method __call__ (line 334) | def __call__(self, x):
class LambdaModule (line 349) | class LambdaModule(base.Module):
method __call__ (line 351) | def __call__(self, x):
function get_name_scope (line 355) | def get_name_scope():
function wrapt_decorator (line 361) | def wrapt_decorator(method, instance, args, kwargs):
class WraptModule (line 369) | class WraptModule(base.Module):
method __call__ (line 372) | def __call__(self, x):
class ControlFlowModule (line 376) | class ControlFlowModule(base.Module):
method __call__ (line 378) | def __call__(self, x):
class ErrorModuleError (line 385) | class ErrorModuleError(Exception):
class ErrorModule (line 389) | class ErrorModule(base.Module):
method __init__ (line 391) | def __init__(self, call_super, raise_in_constructor=True):
method __call__ (line 397) | def __call__(self):
class RecursiveModule (line 401) | class RecursiveModule(base.Module):
method __init__ (line 403) | def __init__(self, depth, trainable=True):
class AbstractModule (line 411) | class AbstractModule(base.Module, metaclass=abc.ABCMeta):
method __call__ (line 414) | def __call__(self, x):
method foo (line 417) | def foo(self):
class ConcreteModule (line 421) | class ConcreteModule(AbstractModule):
method __call__ (line 423) | def __call__(self, x):
class TreeModule (line 427) | class TreeModule(base.Module):
method __init__ (line 429) | def __init__(self, name=None):
method new_leaf (line 433) | def new_leaf(self, name=None):
class ReturnsNameScopeModule (line 439) | class ReturnsNameScopeModule(base.Module):
method alternative_forward (line 441) | def alternative_forward(self):
method __call__ (line 444) | def __call__(self):
class SubclassedReturnsNameScopeModule (line 448) | class SubclassedReturnsNameScopeModule(ReturnsNameScopeModule):
method alternative_alternative_forward (line 450) | def alternative_alternative_forward(self):
class PropertyThrowsWhenCalledModule (line 454) | class PropertyThrowsWhenCalledModule(base.Module):
method raise_assertion_error (line 457) | def raise_assertion_error(self):
class ModuleOverridingNameScope (line 461) | class ModuleOverridingNameScope(ReturnsNameScopeModule):
method name_scope (line 464) | def name_scope(self):
class CommonErrorsTest (line 468) | class CommonErrorsTest(test_utils.TestCase, parameterized.TestCase):
method test_not_calling_super_constructor (line 470) | def test_not_calling_super_constructor(self):
method test_calls_method_before_super (line 476) | def test_calls_method_before_super(self):
method test_annotated_method_is_allowed (line 481) | def test_annotated_method_is_allowed(self):
method test_requests_variables_before_they_exist (line 486) | def test_requests_variables_before_they_exist(self, property_name):
method test_allow_empty_variables_instance (line 496) | def test_allow_empty_variables_instance(self, property_name):
method test_allow_empty_variables_class (line 502) | def test_allow_empty_variables_class(self, property_name):
class NoopModule (line 507) | class NoopModule(base.Module):
method __init__ (line 509) | def __init__(self, a=None):
class RaisesOnEquality (line 514) | class RaisesOnEquality:
method __repr__ (line 518) | def __repr__(self):
method __eq__ (line 521) | def __eq__(self, other):
method __ne__ (line 525) | def __ne__(self, other):
class NeverCreatesVariables (line 531) | class NeverCreatesVariables(base.Module):
class ModuleWithFunctionAnnotatedCall (line 535) | class ModuleWithFunctionAnnotatedCall(base.Module):
method forward (line 538) | def forward(self):
method forward_ag (line 542) | def forward_ag(self):
class CtorNoNameScope (line 546) | class CtorNoNameScope(base.Module):
method __init__ (line 549) | def __init__(self):
class CtorNoNameScopeNoSuper (line 555) | class CtorNoNameScopeNoSuper(base.Module):
method __init__ (line 558) | def __init__(self):
class PropertyModule (line 562) | class PropertyModule(base.Module):
method __init__ (line 564) | def __init__(self):
method some_property (line 569) | def some_property(self):
method some_property (line 574) | def some_property(self, my_property):
method no_name_scope_property (line 579) | def no_name_scope_property(self):
method no_name_scope_property (line 585) | def no_name_scope_property(self, my_property):
class DoesNotCallSuperConstructorModule (line 589) | class DoesNotCallSuperConstructorModule(base.Module):
method __init__ (line 591) | def __init__(self):
class CallsMethodBeforeSuperConstructorModule (line 596) | class CallsMethodBeforeSuperConstructorModule(base.Module):
method __init__ (line 598) | def __init__(self, allowed_method):
method no_name_scope (line 606) | def no_name_scope(self):
method with_name_scope (line 609) | def with_name_scope(self):
class CustomMetaclass (line 613) | class CustomMetaclass(type):
method __new__ (line 617) | def __new__(cls, name, bases, clsdict):
class CombiningMetaclass (line 623) | class CombiningMetaclass(base.ModuleMetaclass, CustomMetaclass):
method __new__ (line 627) | def __new__(cls, name, bases, clsdict):
class ModuleWithCustomMetaclass (line 633) | class ModuleWithCustomMetaclass(base.Module, metaclass=CombiningMetaclass):
method __init__ (line 635) | def __init__(self):
class CustomMetaclassTest (line 640) | class CustomMetaclassTest(tf.test.TestCase):
method testSupportsCustomMetaclass (line 642) | def testSupportsCustomMetaclass(self):
class TakesSubmodules (line 649) | class TakesSubmodules(base.Module):
method __init__ (line 651) | def __init__(self, submodules, name=None):
class WildcardInit (line 655) | class WildcardInit(base.Module):
method __init__ (line 657) | def __init__(self, a, b, *args, **kwargs):
FILE: sonnet/src/batch_apply.py
class BatchApply (line 25) | class BatchApply(base.Module):
method __init__ (line 45) | def __init__(self,
method __call__ (line 53) | def __call__(self, *args, **kwargs):
function first_leaf (line 76) | def first_leaf(args, kwargs) -> Optional[Any]:
function split_leading_dim (line 86) | def split_leading_dim(
function maybe_prod (line 128) | def maybe_prod(s: Sequence[Union[int, None]]) -> Optional[int]:
function merge_leading_dims (line 136) | def merge_leading_dims(
FILE: sonnet/src/batch_apply_test.py
class BatchApplyTest (line 32) | class BatchApplyTest(test_utils.TestCase):
method test_simple (line 34) | def test_simple(self):
method test_no_output (line 40) | def test_no_output(self):
method test_kwargs (line 45) | def test_kwargs(self):
class MergeLeadingDimsTest (line 51) | class MergeLeadingDimsTest(test_utils.TestCase, parameterized.TestCase):
method test_x_not_tensor (line 54) | def test_x_not_tensor(self, x):
method test_static_shape (line 58) | def test_static_shape(self, x_shape, num_dims):
method test_dynamic_shape (line 65) | def test_dynamic_shape(self, x_shape, num_dims):
method test_dynamic_shape_has_static_info_in_graph (line 79) | def test_dynamic_shape_has_static_info_in_graph(self, x_shape, num_dims):
class SplitLeadingDimTest (line 99) | class SplitLeadingDimTest(test_utils.TestCase, parameterized.TestCase):
method test_x_not_tensor (line 102) | def test_x_not_tensor(self, x):
method test_static_shape (line 106) | def test_static_shape(self, i_shape, num_dims):
method test_dynamic_shape (line 115) | def test_dynamic_shape(self, i_shape, num_dims):
method test_dynamic_shape_has_static_info_in_graph (line 134) | def test_dynamic_shape_has_static_info_in_graph(self, i_shape, num_dims):
class NoOutputModule (line 154) | class NoOutputModule(base.Module):
method __call__ (line 156) | def __call__(self, x):
class KwargsModule (line 160) | class KwargsModule(base.Module):
method __call__ (line 162) | def __call__(self, x, is_training=None):
class AddOne (line 167) | class AddOne(base.Module):
method __call__ (line 169) | def __call__(self, x):
FILE: sonnet/src/batch_norm.py
class BaseBatchNorm (line 29) | class BaseBatchNorm(base.Module):
method __init__ (line 74) | def __init__(self,
method __call__ (line 127) | def __call__(self,
method _initialize (line 200) | def _initialize(self, inputs: tf.Tensor):
method _moments (line 239) | def _moments(self, inputs: tf.Tensor,
method _update_statistics (line 257) | def _update_statistics(self, mean, variance):
class BatchNorm (line 265) | class BatchNorm(BaseBatchNorm):
method __init__ (line 277) | def __init__(self,
FILE: sonnet/src/batch_norm_test.py
class BaseBatchNormTest (line 27) | class BaseBatchNormTest(test_utils.TestCase, parameterized.TestCase):
method testSimpleTraining (line 29) | def testSimpleTraining(self):
method testSimpleTrainingNCHW (line 44) | def testSimpleTrainingNCHW(self):
method testSimpleTraining3D (line 60) | def testSimpleTraining3D(self):
method testSimpleTraining3DNCDHW (line 75) | def testSimpleTraining3DNCDHW(self):
method testNoScaleAndOffset (line 91) | def testNoScaleAndOffset(self):
method testSingleBatchInference (line 103) | def testSingleBatchInference(self):
method testWithTfFunction (line 114) | def testWithTfFunction(self, autograph):
method testWithTfFunctionTfArgs (line 143) | def testWithTfFunctionTfArgs(self, autograph):
method testUsingTestStats (line 166) | def testUsingTestStats(self):
method testIsTrainingFalseFirstCall (line 183) | def testIsTrainingFalseFirstCall(self):
method testInvalidDataFormat (line 194) | def testInvalidDataFormat(self, data_format):
method testValidDataFormatChannelsFirst (line 206) | def testValidDataFormatChannelsFirst(self, data_format):
method testValidDataFormatChannelsLast (line 217) | def testValidDataFormatChannelsLast(self, data_format):
method testNoScaleAndInitProvided (line 227) | def testNoScaleAndInitProvided(self):
method testNoOffsetBetaInitProvided (line 237) | def testNoOffsetBetaInitProvided(self):
class BatchNormTest (line 248) | class BatchNormTest(test_utils.TestCase, parameterized.TestCase):
method testSimple (line 250) | def testSimple(self):
class TestMetric (line 261) | class TestMetric:
method __init__ (line 263) | def __init__(self):
method update (line 267) | def update(self, x):
method value (line 275) | def value(self):
method initialize (line 278) | def initialize(self, x):
FILE: sonnet/src/bias.py
class Bias (line 27) | class Bias(base.Module):
method __init__ (line 67) | def __init__(self,
method _initialize (line 91) | def _initialize(self, inputs):
method __call__ (line 106) | def __call__(self, inputs: tf.Tensor, multiplier: types.FloatLike = No...
function calculate_bias_shape (line 127) | def calculate_bias_shape(input_shape: types.ShapeLike,
FILE: sonnet/src/bias_test.py
class BiasTest (line 22) | class BiasTest(test_utils.TestCase):
method test_output_shape (line 24) | def test_output_shape(self):
method test_output_size_valid (line 29) | def test_output_size_valid(self):
method test_bias_dims_scalar (line 33) | def test_bias_dims_scalar(self):
method test_bias_dims_custom (line 38) | def test_bias_dims_custom(self):
method test_bias_dims_negative_out_of_order (line 45) | def test_bias_dims_negative_out_of_order(self):
method test_bias_dims_invalid (line 50) | def test_bias_dims_invalid(self):
method test_b_init_defaults_to_zeros (line 56) | def test_b_init_defaults_to_zeros(self):
method test_b_init_custom (line 61) | def test_b_init_custom(self):
method test_name (line 67) | def test_name(self):
method test_multiplier (line 73) | def test_multiplier(self):
FILE: sonnet/src/build.py
function _int_or_none (line 23) | def _int_or_none(o):
function _promote_shapes (line 27) | def _promote_shapes(o):
function _maybe_tensor_spec (line 34) | def _maybe_tensor_spec(shape, dtype):
function build (line 39) | def build(
FILE: sonnet/src/build_test.py
class BuildTest (line 22) | class BuildTest(test_utils.TestCase):
method test_call_with_shape_lke_object (line 24) | def test_call_with_shape_lke_object(self):
method test_output_spec (line 28) | def test_output_spec(self):
method test_does_not_trigger_sideeffects (line 35) | def test_does_not_trigger_sideeffects(self):
function tensor_identity (line 42) | def tensor_identity(x):
class IncrementsCounter (line 47) | class IncrementsCounter(tf.Module):
method __call__ (line 49) | def __call__(self):
FILE: sonnet/src/conformance/api_test.py
class PublicSymbolsTest (line 24) | class PublicSymbolsTest(test_utils.TestCase):
method test_src_not_exported (line 26) | def test_src_not_exported(self):
method test_supports_reload (line 29) | def test_supports_reload(self):
FILE: sonnet/src/conformance/build_test.py
function if_present (line 28) | def if_present(f):
class BuildTest (line 32) | class BuildTest(test_utils.TestCase, parameterized.TestCase):
method test_build (line 35) | def test_build(self, module_fn, input_shape, dtype):
method assertCompatible (line 44) | def assertCompatible(self, a: tf.TensorSpec, b: tf.TensorSpec):
FILE: sonnet/src/conformance/checkpoint_test.py
class TestCheckpoint (line 30) | class TestCheckpoint:
method __init__ (line 33) | def __init__(self, golden=None, **kwargs):
method save (line 43) | def save(self):
method restore_latest (line 46) | def restore_latest(self, assert_consumed):
function with_soft_placement (line 55) | def with_soft_placement(f):
class GoldenCheckpointsTest (line 65) | class GoldenCheckpointsTest(test_utils.TestCase, parameterized.TestCase):
method test_save_load (line 69) | def test_save_load(self, golden):
method test_save_then_load_new_instance (line 102) | def test_save_then_load_new_instance(self, golden):
method test_restore_on_create (line 132) | def test_restore_on_create(self, golden):
method test_restore_golden (line 159) | def test_restore_golden(self, golden):
class ReplicatorCheckpointTest (line 174) | class ReplicatorCheckpointTest(test_utils.TestCase, parameterized.TestCa...
method replicator_or_skip (line 176) | def replicator_or_skip(self, replicator_fn, use_function):
method test_save_restore (line 186) | def test_save_restore(self, golden, replicator_fn, use_function):
method test_restore_from_golden (line 240) | def test_restore_from_golden(self, golden, replicator_fn):
method test_restore_from_non_distributed (line 257) | def test_restore_from_non_distributed(self, golden, replicator_fn,
method test_restore_on_create (line 304) | def test_restore_on_create(self, golden, replicator_fn):
method test_restore_on_create_in_replica_context (line 332) | def test_restore_on_create_in_replica_context(self, golden, replicator...
function setUpModule (line 375) | def setUpModule():
FILE: sonnet/src/conformance/checkpoints/generate.py
function safe_mkdir (line 37) | def safe_mkdir(directory):
function safe_unlink (line 45) | def safe_unlink(path):
function main (line 53) | def main(unused_argv):
FILE: sonnet/src/conformance/copy_test.py
class CopyTest (line 27) | class CopyTest(test_utils.TestCase, parameterized.TestCase):
method test_copy (line 30) | def test_copy(self, golden):
FILE: sonnet/src/conformance/descriptors.py
class Wrapped (line 24) | class Wrapped(snt.Module):
method __init__ (line 27) | def __init__(self, wrapped: snt.Module):
class Training (line 32) | class Training(Wrapped):
method __call__ (line 35) | def __call__(self, x: tf.Tensor):
class Recurrent (line 39) | class Recurrent(Wrapped):
method __init__ (line 42) | def __init__(self,
method __call__ (line 49) | def __call__(self, x: tf.Tensor):
function unwrap (line 61) | def unwrap(module: snt.Module) -> snt.Module:
function recurrent_factory (line 205) | def recurrent_factory(
function unroll_descriptors (line 212) | def unroll_descriptors(descriptors, unroller=None):
FILE: sonnet/src/conformance/descriptors_test.py
class DescriptorsTest (line 28) | class DescriptorsTest(test_utils.TestCase):
method test_coverage (line 30) | def test_coverage(self):
FILE: sonnet/src/conformance/distribute_test.py
class TpuReplicatorTest (line 29) | class TpuReplicatorTest(test_utils.TestCase, parameterized.TestCase):
method test_variable_creation_in_replica_context (line 33) | def test_variable_creation_in_replica_context(self, golden, replicator...
method assertSameValuePerReplica (line 54) | def assertSameValuePerReplica(self, replicator, per_replica):
method test_unroll (line 63) | def test_unroll(
FILE: sonnet/src/conformance/doctest_test.py
class DoctestTest (line 27) | class DoctestTest(test_utils.TestCase, parameterized.TestCase):
method setUp (line 32) | def setUp(self):
method test_doctest (line 41) | def test_doctest(self, module):
FILE: sonnet/src/conformance/function_test.py
class FunctionTest (line 32) | class FunctionTest(test_utils.TestCase, parameterized.TestCase):
method test_trace (line 36) | def test_trace(
method test_create_variables_eagerly (line 49) | def test_create_variables_eagerly(
method test_trace_batch_agnostic (line 63) | def test_trace_batch_agnostic(
method test_trace_batch_apply_batch_agnostic (line 78) | def test_trace_batch_apply_batch_agnostic(
method test_optimizer_dense (line 99) | def test_optimizer_dense(
method test_optimizer_sparse (line 117) | def test_optimizer_sparse(
FILE: sonnet/src/conformance/goldens.py
function named_goldens (line 28) | def named_goldens() -> Sequence[Tuple[str, "Golden"]]:
function all_goldens (line 32) | def all_goldens(test_method):
function _register_golden (line 36) | def _register_golden(module_cls, golden_name):
function list_goldens (line 46) | def list_goldens():
function range_like (line 50) | def range_like(t, start=0):
class Golden (line 77) | class Golden(abc.ABC):
method create_module (line 81) | def create_module(self):
method create_all_variables (line 86) | def create_all_variables(self, module):
method forward (line 91) | def forward(self, module, x=None):
class AbstractGolden (line 96) | class AbstractGolden(Golden):
method input_spec (line 108) | def input_spec(self):
method num_variables (line 112) | def num_variables(self):
method forward (line 115) | def forward(self, module, x=None):
method create_all_variables (line 120) | def create_all_variables(self, module):
class Linear1x1Test (line 131) | class Linear1x1Test(AbstractGolden):
class LinearNoBias1x1 (line 138) | class LinearNoBias1x1(AbstractGolden):
class Conv1D (line 145) | class Conv1D(AbstractGolden):
class Conv2D (line 152) | class Conv2D(AbstractGolden):
class Conv3D (line 159) | class Conv3D(AbstractGolden):
class Conv1DTranspose (line 166) | class Conv1DTranspose(AbstractGolden):
class Conv2DTranspose (line 174) | class Conv2DTranspose(AbstractGolden):
class Conv3DTranspose (line 182) | class Conv3DTranspose(AbstractGolden):
class DepthwiseConv2D (line 190) | class DepthwiseConv2D(AbstractGolden):
class MLP (line 197) | class MLP(AbstractGolden):
class MLPNoBias (line 204) | class MLPNoBias(AbstractGolden):
class Cifar10ConvNet (line 211) | class Cifar10ConvNet(AbstractGolden):
method forward (line 217) | def forward(self, module, x=None):
class LayerNorm (line 224) | class LayerNorm(AbstractGolden):
class Instance (line 232) | class Instance(AbstractGolden):
class GroupNorm (line 240) | class GroupNorm(AbstractGolden):
class BaseBatchNorm (line 248) | class BaseBatchNorm(AbstractGolden):
method forward (line 254) | def forward(self, module, x=None):
class BaseBatchNormScaleOffset (line 261) | class BaseBatchNormScaleOffset(AbstractGolden):
method forward (line 267) | def forward(self, module, x=None):
class BatchNorm (line 274) | class BatchNorm(AbstractGolden):
method forward (line 279) | def forward(self, module, x=None):
class BatchNormScaleOffset (line 286) | class BatchNormScaleOffset(AbstractGolden):
method forward (line 291) | def forward(self, module, x=None):
class ExponentialMovingAverage (line 298) | class ExponentialMovingAverage(AbstractGolden):
method forward (line 304) | def forward(self, module, x=None):
class BatchNormTraining (line 311) | class BatchNormTraining(AbstractGolden):
method forward (line 317) | def forward(self, module, x=None):
class CrossReplicaBatchNorm (line 325) | class CrossReplicaBatchNorm(AbstractGolden):
method forward (line 331) | def forward(self, module, x=None):
class DropoutVariableRate (line 338) | class DropoutVariableRate(AbstractGolden):
method forward (line 344) | def forward(self, module, x=None):
class AbstractRNNGolden (line 351) | class AbstractRNNGolden(AbstractGolden):
method forward (line 353) | def forward(self, module, x=None):
class Conv1DLSTM (line 365) | class Conv1DLSTM(AbstractRNNGolden):
method create_module (line 369) | def create_module(self):
class Conv2DLSTM (line 377) | class Conv2DLSTM(AbstractRNNGolden):
method create_module (line 381) | def create_module(self):
class Conv3DLSTM (line 389) | class Conv3DLSTM(AbstractRNNGolden):
method create_module (line 393) | def create_module(self):
class GRU (line 401) | class GRU(AbstractRNNGolden):
class LSTM (line 408) | class LSTM(AbstractRNNGolden):
class LSTMWithProjection (line 415) | class LSTMWithProjection(AbstractRNNGolden):
class UnrolledLSTM (line 422) | class UnrolledLSTM(AbstractRNNGolden):
class VanillaRNN (line 429) | class VanillaRNN(AbstractRNNGolden):
class TrainableState (line 436) | class TrainableState(AbstractGolden):
class BiasTest (line 443) | class BiasTest(AbstractGolden):
class EmbedTest (line 450) | class EmbedTest(AbstractGolden):
class MeanTest (line 457) | class MeanTest(AbstractGolden):
class SumTest (line 465) | class SumTest(AbstractGolden):
class ResNet (line 473) | class ResNet(AbstractGolden):
method forward (line 479) | def forward(self, module, x=None):
class VectorQuantizerTest (line 486) | class VectorQuantizerTest(AbstractGolden):
method create_module (line 488) | def create_module(self):
method forward (line 495) | def forward(self, module, x=None):
class VectorQuantizerEMATrainTest (line 507) | class VectorQuantizerEMATrainTest(AbstractGolden):
method create_module (line 509) | def create_module(self):
method forward (line 516) | def forward(self, module, x=None):
class VectorQuantizerEMAEvalTest (line 529) | class VectorQuantizerEMAEvalTest(AbstractGolden):
method create_module (line 531) | def create_module(self):
method forward (line 538) | def forward(self, module, x=None):
class FooMetric (line 553) | class FooMetric(snt.Metric):
method initialize (line 556) | def initialize(self, x):
method reset (line 559) | def reset(self):
method update (line 562) | def update(self, x):
FILE: sonnet/src/conformance/goldens_test.py
class CoverageTest (line 25) | class CoverageTest(test_utils.TestCase):
method test_all_modules_covered (line 27) | def test_all_modules_covered(self):
FILE: sonnet/src/conformance/keras_test.py
class KerasTest (line 30) | class KerasTest(test_utils.TestCase, parameterized.TestCase):
method test_build_without_batch (line 33) | def test_build_without_batch(self, module_fn, input_shape, dtype):
method test_sonnet_module_as_layer (line 55) | def test_sonnet_module_as_layer(self, module_fn, input_shape, dtype):
method test_build_with_updating_module (line 78) | def test_build_with_updating_module(self):
method test_layer_with_model (line 89) | def test_layer_with_model(self):
method test_symbolic_model (line 107) | def test_symbolic_model(self, module_fn, input_shape, dtype):
method test_layer_adapter_custom_method (line 123) | def test_layer_adapter_custom_method(self):
method test_keras_layer_inside_sonnet_module (line 134) | def test_keras_layer_inside_sonnet_module(self):
method test_to_config (line 146) | def test_to_config(self):
method test_from_config (line 151) | def test_from_config(self):
class LayerAdapter (line 157) | class LayerAdapter(tf.keras.layers.Layer):
method __init__ (line 177) | def __init__(self, module, method="__call__", dtype=tf.float32):
method from_config (line 184) | def from_config(cls, config):
method to_config (line 187) | def to_config(self):
method _trace_and_initialize (line 190) | def _trace_and_initialize(self, input_shape):
method compute_output_shape (line 198) | def compute_output_shape(self, input_shape):
method build (line 202) | def build(self, input_shape):
method call (line 216) | def call(self, inputs):
class ModuleWithLayer (line 220) | class ModuleWithLayer(snt.Module):
method __init__ (line 222) | def __init__(self):
method __call__ (line 226) | def __call__(self, x):
class ModuleWithUpdateInCall (line 230) | class ModuleWithUpdateInCall(snt.Module):
method _init (line 233) | def _init(self, x):
method __call__ (line 236) | def __call__(self, x):
class ModuleWithCustomForward (line 242) | class ModuleWithCustomForward(snt.Module):
method _init (line 245) | def _init(self, x):
method forward (line 248) | def forward(self, x):
FILE: sonnet/src/conformance/optimizer_test.py
class OptimizerConformanceTest (line 26) | class OptimizerConformanceTest(test_utils.TestCase, parameterized.TestCa...
method test_variable_order_is_constant (line 32) | def test_variable_order_is_constant(self, module_fn, input_shape, dtype,
FILE: sonnet/src/conformance/pickle_test.py
class PickleTest (line 26) | class PickleTest(test_utils.TestCase, parameterized.TestCase):
method test_pickle (line 31) | def test_pickle(self, golden):
FILE: sonnet/src/conformance/saved_model_test.py
class SavedModelTest (line 28) | class SavedModelTest(test_utils.TestCase, parameterized.TestCase):
method test_save_restore_cycle (line 31) | def test_save_restore_cycle(self, golden):
FILE: sonnet/src/conformance/tensorflow1_test.py
class TensorFlow1Test (line 23) | class TensorFlow1Test(test_utils.TestCase):
method test_requires_tf2 (line 25) | def test_requires_tf2(self):
FILE: sonnet/src/conformance/xla_test.py
class XLATest (line 26) | class XLATest(test_utils.TestCase, parameterized.TestCase):
method test_compile (line 29) | def test_compile(self, golden):
method test_jit_scope (line 57) | def test_jit_scope(self, golden):
FILE: sonnet/src/conv.py
class ConvND (line 28) | class ConvND(base.Module):
method __init__ (line 31) | def __init__(self,
method __call__ (line 100) | def __call__(self, inputs: tf.Tensor) -> tf.Tensor:
method _initialize (line 129) | def _initialize(self, inputs: tf.Tensor):
method _make_w (line 150) | def _make_w(self):
class Conv1D (line 164) | class Conv1D(ConvND):
method __init__ (line 167) | def __init__(self,
class Conv2D (line 218) | class Conv2D(ConvND):
method __init__ (line 221) | def __init__(self,
class Conv3D (line 273) | class Conv3D(ConvND):
method __init__ (line 276) | def __init__(self,
FILE: sonnet/src/conv_test.py
function create_constant_initializers (line 25) | def create_constant_initializers(w, b, with_bias):
class ConvTest (line 35) | class ConvTest(test_utils.TestCase, parameterized.TestCase):
method testPaddingFunctionReached (line 37) | def testPaddingFunctionReached(self):
method testIncorrectN (line 60) | def testIncorrectN(self, n):
method testInitializerKeysInvalidWithoutBias (line 70) | def testInitializerKeysInvalidWithoutBias(self):
method testIncorrectRankInput (line 80) | def testIncorrectRankInput(self):
method testDefaultInitializers (line 90) | def testDefaultInitializers(self, dtype):
method testFunction (line 122) | def testFunction(self, with_bias, padding):
method testUnknownBatchSizeNHWC (line 152) | def testUnknownBatchSizeNHWC(self):
method testUnknownBatchSizeNCHW (line 168) | def testUnknownBatchSizeNCHW(self):
method testUnknownChannels (line 187) | def testUnknownChannels(self, autograph):
method testUnknownSpatialDims (line 201) | def testUnknownSpatialDims(self):
class Conv2DTest (line 222) | class Conv2DTest(test_utils.TestCase, parameterized.TestCase):
method testComputationPaddingSame (line 225) | def testComputationPaddingSame(self, with_bias):
method testComputationPaddingValid (line 247) | def testComputationPaddingValid(self, with_bias):
class Conv1DTest (line 268) | class Conv1DTest(test_utils.TestCase, parameterized.TestCase):
method testComputationPaddingSame (line 271) | def testComputationPaddingSame(self, with_bias):
method testComputationPaddingValid (line 292) | def testComputationPaddingValid(self, with_bias):
class Conv3DTest (line 313) | class Conv3DTest(test_utils.TestCase, parameterized.TestCase):
method testComputationPaddingSame (line 316) | def testComputationPaddingSame(self, with_bias):
method testComputationPaddingValid (line 344) | def testComputationPaddingValid(self, with_bias):
FILE: sonnet/src/conv_transpose.py
function smart_concat (line 28) | def smart_concat(v1, v2):
function smart_lambda (line 35) | def smart_lambda(func, v1, v2):
class ConvNDTranspose (line 42) | class ConvNDTranspose(base.Module):
method __init__ (line 52) | def __init__(self,
method __call__ (line 122) | def __call__(self, inputs):
method _initialize (line 149) | def _initialize(self, inputs):
method _make_w (line 171) | def _make_w(self):
method _get_output_shape (line 185) | def _get_output_shape(self, inputs):
class Conv1DTranspose (line 210) | class Conv1DTranspose(ConvNDTranspose):
method __init__ (line 213) | def __init__(self,
class Conv2DTranspose (line 267) | class Conv2DTranspose(ConvNDTranspose):
method __init__ (line 270) | def __init__(self,
class Conv3DTranspose (line 324) | class Conv3DTranspose(ConvNDTranspose):
method __init__ (line 327) | def __init__(self,
FILE: sonnet/src/conv_transpose_test.py
function create_constant_initializers (line 27) | def create_constant_initializers(w, b, with_bias):
class ConvTransposeTest (line 37) | class ConvTransposeTest(test_utils.TestCase, parameterized.TestCase):
method testIncorrectN (line 40) | def testIncorrectN(self, n):
method testIncorrectPadding (line 51) | def testIncorrectPadding(self):
method testBiasInitNoBias (line 58) | def testBiasInitNoBias(self):
method testIncorrectOutputShape (line 65) | def testIncorrectOutputShape(self):
method testGraphConv (line 79) | def testGraphConv(self, with_bias, padding):
method testUnknownBatchSizeNHWC (line 111) | def testUnknownBatchSizeNHWC(self):
method testUnknownBatchSizeNCHW (line 127) | def testUnknownBatchSizeNCHW(self):
method testUnknownShapeDims (line 146) | def testUnknownShapeDims(self):
method testGivenOutputShape (line 162) | def testGivenOutputShape(self):
method testUnknownChannels (line 174) | def testUnknownChannels(self, autograph):
method testInitializerVariance (line 192) | def testInitializerVariance(self, num_spatial_dims, kernel_shape,
class Conv2DTransposeTest (line 213) | class Conv2DTransposeTest(test_utils.TestCase, parameterized.TestCase):
method testComputationPaddingSame (line 216) | def testComputationPaddingSame(self, with_bias):
method testComputationPaddingValid (line 238) | def testComputationPaddingValid(self, with_bias):
method testShapeDilated (line 260) | def testShapeDilated(self):
class Conv1DTransposeTest (line 274) | class Conv1DTransposeTest(test_utils.TestCase, parameterized.TestCase):
method testComputationPaddingSame (line 277) | def testComputationPaddingSame(self, with_bias):
method testComputationPaddingValid (line 299) | def testComputationPaddingValid(self, with_bias):
class Conv3DTransposeTest (line 321) | class Conv3DTransposeTest(test_utils.TestCase, parameterized.TestCase):
method testComputationPaddingSame (line 324) | def testComputationPaddingSame(self, with_bias):
method testComputationPaddingValid (line 348) | def testComputationPaddingValid(self, with_bias):
FILE: sonnet/src/custom_getter.py
function _patch_getattribute (line 28) | def _patch_getattribute(cls, new_getattribute):
function _custom_getter (line 37) | def _custom_getter(
function custom_variable_getter (line 102) | def custom_variable_getter(
function _is_variable (line 159) | def _is_variable(x):
FILE: sonnet/src/custom_getter_test.py
class CustomVariableGetterTest (line 25) | class CustomVariableGetterTest(test_utils.TestCase):
method testDoesNotModifyNonVariables (line 27) | def testDoesNotModifyNonVariables(self):
class DoctestTest (line 44) | class DoctestTest(test_utils.TestCase):
method testDoctest (line 46) | def testDoctest(self):
FILE: sonnet/src/deferred.py
class Deferred (line 20) | class Deferred(base.Module):
method __init__ (line 47) | def __init__(self, constructor, call_methods=("__call__",), name=None):
method target (line 77) | def target(self):
method __call__ (line 92) | def __call__(self, *args, **kwargs):
method __str__ (line 95) | def __str__(self):
method __repr__ (line 98) | def __repr__(self):
method __getattr__ (line 101) | def __getattr__(self, name):
method __setattr__ (line 109) | def __setattr__(self, name, value):
method __delattr__ (line 117) | def __delattr__(self, name):
function _materialize_then_call (line 125) | def _materialize_then_call(module, method_name):
FILE: sonnet/src/deferred_test.py
class DeferredTest (line 23) | class DeferredTest(test_utils.TestCase):
method test_target (line 25) | def test_target(self):
method test_only_computes_target_once (line 30) | def test_only_computes_target_once(self):
method test_attr_forwarding_fails_before_construction (line 39) | def test_attr_forwarding_fails_before_construction(self):
method test_getattr (line 44) | def test_getattr(self):
method test_setattr (line 49) | def test_setattr(self):
method test_setattr_on_target (line 57) | def test_setattr_on_target(self):
method test_delattr (line 67) | def test_delattr(self):
method test_alternative_forward (line 74) | def test_alternative_forward(self):
method test_alternative_forward_call_type_error (line 78) | def test_alternative_forward_call_type_error(self):
method test_name_scope (line 84) | def test_name_scope(self):
method test_str (line 90) | def test_str(self):
method test_repr (line 95) | def test_repr(self):
class ExampleModule (line 101) | class ExampleModule(base.Module):
method __init__ (line 103) | def __init__(self):
method __str__ (line 107) | def __str__(self):
method __repr__ (line 110) | def __repr__(self):
method __call__ (line 113) | def __call__(self):
class AlternativeForwardModule (line 117) | class AlternativeForwardModule(base.Module):
method forward (line 119) | def forward(self):
FILE: sonnet/src/depthwise_conv.py
class DepthwiseConv2D (line 28) | class DepthwiseConv2D(base.Module):
method __init__ (line 35) | def __init__(self,
method __call__ (line 97) | def __call__(self, inputs: tf.Tensor) -> tf.Tensor:
method _initialize (line 113) | def _initialize(self, inputs: tf.Tensor):
FILE: sonnet/src/depthwise_conv_test.py
function create_constant_initializers (line 26) | def create_constant_initializers(w, b, with_bias):
class DepthwiseConvTest (line 36) | class DepthwiseConvTest(test_utils.TestCase, parameterized.TestCase):
method testInitializerKeysInvalidWithoutBias (line 38) | def testInitializerKeysInvalidWithoutBias(self):
method testDefaultInitializers (line 48) | def testDefaultInitializers(self, dtype):
method testFunction (line 69) | def testFunction(self, with_bias, padding):
method testUnknownBatchSizeNHWC (line 97) | def testUnknownBatchSizeNHWC(self):
method testUnknownBatchSizeNCHW (line 110) | def testUnknownBatchSizeNCHW(self):
method testUnknownSpatialDims (line 125) | def testUnknownSpatialDims(self):
method testUnknownChannels (line 143) | def testUnknownChannels(self, autograph):
method testComputationSame (line 155) | def testComputationSame(self, with_bias):
method testComputationValid (line 175) | def testComputationValid(self, with_bias):
method testComputationValidMultiChannel (line 193) | def testComputationValidMultiChannel(self, with_bias):
method testSharing (line 210) | def testSharing(self, with_bias):
FILE: sonnet/src/distribute/distributed_batch_norm.py
class CrossReplicaBatchNorm (line 28) | class CrossReplicaBatchNorm(batch_norm.BaseBatchNorm):
method __init__ (line 43) | def __init__(self,
method _initialize (line 91) | def _initialize(self, inputs: tf.Tensor):
method _moments (line 98) | def _moments(self, inputs: tf.Tensor,
FILE: sonnet/src/distribute/distributed_batch_norm_test.py
class CrossReplicaBatchNormTest (line 26) | class CrossReplicaBatchNormTest(test_utils.TestCase, parameterized.TestC...
method testDefaultReplicaContext (line 30) | def testDefaultReplicaContext(self):
method testWithMultipleDevicesMirrored (line 41) | def testWithMultipleDevicesMirrored(self):
method testWithTpuStrategy (line 80) | def testWithTpuStrategy(self):
class TestMetric (line 119) | class TestMetric:
method __init__ (line 121) | def __init__(self):
method update (line 125) | def update(self, x):
method value (line 133) | def value(self):
method initialize (line 136) | def initialize(self, x):
function setUpModule (line 141) | def setUpModule():
FILE: sonnet/src/distribute/replicator.py
function replica_local_creator (line 27) | def replica_local_creator(next_creator, **kwargs) -> tf.Variable:
class Replicator (line 38) | class Replicator(tf.distribute.MirroredStrategy):
method scope (line 79) | def scope(self):
class TpuReplicator (line 92) | class TpuReplicator(TPUStrategy):
method scope (line 138) | def scope(self):
function create_variables_eagerly (line 145) | def create_variables_eagerly(f: Callable[..., T]) -> Callable[..., T]:
function _eager_variable_creator (line 181) | def _eager_variable_creator(getter, initial_value, **kwargs):
function _eager_initial_values (line 199) | def _eager_initial_values():
FILE: sonnet/src/distribute/replicator_test.py
function _create_variable_in_cross_replica_context (line 28) | def _create_variable_in_cross_replica_context(replicator):
class TrainableVariable (line 34) | class TrainableVariable:
method __call__ (line 36) | def __call__(self):
function _create_variable_in_replica_context (line 42) | def _create_variable_in_replica_context(replicator):
function all_variable_creators (line 56) | def all_variable_creators():
class ReplicatorTest (line 61) | class ReplicatorTest(test_utils.TestCase, parameterized.TestCase):
method test_variable_synchronization_default (line 68) | def test_variable_synchronization_default(self, replicator_fn, create_...
method test_variable_aggregation_default (line 78) | def test_variable_aggregation_default(self, replicator_fn, create_var):
method test_variable_trainable_default (line 87) | def test_variable_trainable_default(self, replicator_fn, create_var):
method test_variable_trainable (line 96) | def test_variable_trainable(self, replicator_fn, trainable):
method test_assign (line 109) | def test_assign(self, replicator_fn, method_name, value, cross_replica):
method test_read_value (line 130) | def test_read_value(self, replicator_fn, cross_replica):
method test_falls_back_to_graph (line 152) | def test_falls_back_to_graph(self, autograph):
method test_requires_eager (line 163) | def test_requires_eager(self, autograph):
method test_eager_variable_creator (line 171) | def test_eager_variable_creator(self, autograph):
class MyOnesInitializer (line 193) | class MyOnesInitializer(initializers.Initializer):
method __call__ (line 195) | def __call__(self, shape, dtype):
class FailsInEagerMode (line 200) | class FailsInEagerMode(initializers.Initializer):
method __call__ (line 202) | def __call__(self, shape, dtype):
function setUpModule (line 208) | def setUpModule():
FILE: sonnet/src/distribute/replicator_test_utils.py
function _replicator_primary_device (line 25) | def _replicator_primary_device() -> snt_replicator.Replicator:
function _tpu_replicator_or_skip_test (line 40) | def _tpu_replicator_or_skip_test() -> snt_replicator.TpuReplicator:
function named_replicators (line 52) | def named_replicators() -> Sequence[Tuple[str, Callable[[], Strategy]]]:
FILE: sonnet/src/dropout.py
class Dropout (line 26) | class Dropout(base.Module):
method __init__ (line 35) | def __init__(self,
method __call__ (line 58) | def __call__(self, x: tf.Tensor, is_training: types.BoolLike) -> tf.Te...
FILE: sonnet/src/dropout_test.py
class DropoutTest (line 24) | class DropoutTest(test_utils.TestCase, parameterized.TestCase):
method test_sum_close (line 27) | def test_sum_close(self, rate):
method test_dropout_rate (line 37) | def test_dropout_rate(self, rate):
method test_dropout_is_actually_random (line 49) | def test_dropout_is_actually_random(self):
method test_with_tf_function_with_booleans (line 58) | def test_with_tf_function_with_booleans(self, autograph):
method test_with_tf_function_with_variables (line 72) | def test_with_tf_function_with_variables(self, autograph):
FILE: sonnet/src/embed.py
class Embed (line 27) | class Embed(base.Module):
method __init__ (line 30) | def __init__(self,
method __call__ (line 97) | def __call__(self, inputs):
function embedding_dim (line 105) | def embedding_dim(vocab_size: int):
function dense_gradient (line 125) | def dense_gradient(x: tf.Tensor):
FILE: sonnet/src/embed_test.py
class EmbedTest (line 24) | class EmbedTest(test_utils.TestCase, parameterized.TestCase):
method test_vocab_size (line 27) | def test_vocab_size(self, vocab_size):
method test_embed_dim (line 33) | def test_embed_dim(self, embed_dim):
method test_existing_vocab (line 39) | def test_existing_vocab(self, vocab_size, embed_dim):
method test_densify_gradients (line 47) | def test_densify_gradients(self, densify_gradients):
method test_initializer (line 57) | def test_initializer(self):
method test_pinned_to_cpu (line 61) | def test_pinned_to_cpu(self):
method test_trainable (line 68) | def test_trainable(self, trainable):
method test_dtype (line 73) | def test_dtype(self, dtype):
method test_name (line 79) | def test_name(self):
FILE: sonnet/src/functional/haiku.py
class TensorVariableCallbacks (line 33) | class TensorVariableCallbacks(threading.local):
method __init__ (line 38) | def __init__(self):
method notify (line 43) | def notify(self, variable):
method __call__ (line 50) | def __call__(self, callback):
function notify (line 63) | def notify(f):
function defer_property (line 72) | def defer_property(name):
function safe_read_tensor_value (line 76) | def safe_read_tensor_value(variable):
function defer_read (line 105) | def defer_read():
function defer_raise_notimplemented (line 110) | def defer_raise_notimplemented():
function defer_indexed (line 117) | def defer_indexed(f):
function defer_assign (line 121) | def defer_assign(map_fn=None):
class TensorVariable (line 136) | class TensorVariable(tf.Variable):
method __init__ (line 139) | def __init__(self, value, trainable, name=None):
method __repr__ (line 200) | def __repr__(self):
function tv_to_tensor (line 215) | def tv_to_tensor(value, dtype=None, name=None, as_ref=None):
function create_tensor_variables (line 227) | def create_tensor_variables():
function track_tensor_variables (line 253) | def track_tensor_variables():
function track_new_variables (line 260) | def track_new_variables():
function track_initial_state (line 272) | def track_initial_state():
function initial_value_by_ref (line 283) | def initial_value_by_ref(tf_variables):
function final_value_by_ref (line 288) | def final_value_by_ref(tf_variables):
function transform (line 293) | def transform(f) -> Transformed:
function transform_with_state (line 349) | def transform_with_state(f) -> TransformedWithState:
function without_state (line 444) | def without_state(with_state: TransformedWithState) -> Transformed:
FILE: sonnet/src/functional/haiku_test.py
class TensorVariableTest (line 25) | class TensorVariableTest(test_utils.TestCase, parameterized.TestCase):
method test_initial_value (line 27) | def test_initial_value(self):
method test_trainable (line 36) | def test_trainable(self, trainable):
method test_name (line 44) | def test_name(self):
method test_name_with_scope (line 49) | def test_name_with_scope(self):
method test_shape (line 55) | def test_shape(self, shape):
method test_dtype (line 61) | def test_dtype(self, dtype):
method test_attributes_do_not_notify (line 66) | def test_attributes_do_not_notify(self):
method test_read_captured_variables_included (line 88) | def test_read_captured_variables_included(self):
method test_captured_variable_from_other_function_raises (line 99) | def test_captured_variable_from_other_function_raises(self):
method test_assign (line 116) | def test_assign(self):
method test_assign_add (line 124) | def test_assign_add(self):
method test_assign_sub (line 132) | def test_assign_sub(self):
class NetworkTest (line 141) | class NetworkTest(test_utils.TestCase, parameterized.TestCase):
method test_transform (line 143) | def test_transform(self):
method test_initial_values_preserved (line 163) | def test_initial_values_preserved(self):
method test_variables_in_transform_set_to_none (line 180) | def test_variables_in_transform_set_to_none(self):
method test_disallows_variables_in_apply (line 192) | def test_disallows_variables_in_apply(self):
method test_state_returns_initial_value (line 198) | def test_state_returns_initial_value(self):
method test_state_counter (line 213) | def test_state_counter(self):
method test_state_ema (line 225) | def test_state_ema(self):
FILE: sonnet/src/functional/jax.py
function device_put (line 24) | def device_put(t, device=None):
function device_get (line 28) | def device_get(t):
function jit (line 33) | def jit(f, device=None):
function grad (line 40) | def grad(f, argnums=0, has_aux=False):
function value_and_grad (line 54) | def value_and_grad(f, argnums=0, has_aux=False):
FILE: sonnet/src/functional/jax_test.py
class JaxTest (line 23) | class JaxTest(test_utils.TestCase, parameterized.TestCase):
method test_jit_copies_to_device (line 25) | def test_jit_copies_to_device(self):
method test_device_put (line 39) | def test_device_put(self):
class GradTest (line 52) | class GradTest(test_utils.TestCase, parameterized.TestCase):
method test_grad (line 54) | def test_grad(self):
method test_argnums (line 60) | def test_argnums(self):
method test_has_aux (line 69) | def test_has_aux(self):
function get_accelerators (line 78) | def get_accelerators():
FILE: sonnet/src/functional/optimizers.py
function optimizer (line 30) | def optimizer(cls: Type[base.Optimizer]) -> Callable[..., TransformedOpt...
function _split_on_trainable (line 102) | def _split_on_trainable(opt_state):
function _merge (line 113) | def _merge(a, b):
function _wrap_optimizer (line 120) | def _wrap_optimizer(opt: base.Optimizer) -> TransformedOptimizer:
FILE: sonnet/src/functional/optimizers_test.py
class OptimizersTest (line 29) | class OptimizersTest(test_utils.TestCase, parameterized.TestCase):
method test_sgd (line 31) | def test_sgd(self):
method test_adam (line 43) | def test_adam(self):
method test_adam_with_variable_lr (line 54) | def test_adam_with_variable_lr(self, trainable_lr):
FILE: sonnet/src/functional/utils.py
function get_first_accelerator (line 24) | def get_first_accelerator():
function run_on_device (line 33) | def run_on_device(f, device):
function get_name_scope (line 47) | def get_name_scope():
function first_non_none (line 52) | def first_non_none(*args):
function compose (line 56) | def compose(f0, *fs):
FILE: sonnet/src/group_norm.py
class GroupNorm (line 27) | class GroupNorm(base.Module):
method __init__ (line 66) | def __init__(self,
method __call__ (line 132) | def __call__(self,
method _initialize (line 188) | def _initialize(self, inputs: tf.Tensor):
FILE: sonnet/src/group_norm_test.py
class GroupNormTest (line 25) | class GroupNormTest(test_utils.TestCase, parameterized.TestCase):
method testSimpleCase (line 27) | def testSimpleCase(self):
method testSimpleCaseVar (line 36) | def testSimpleCaseVar(self):
method testSimpleCaseNCHWVar (line 50) | def testSimpleCaseNCHWVar(self):
method testDataFormatAgnosticVar (line 65) | def testDataFormatAgnosticVar(self):
method testSimpleCaseTensor (line 80) | def testSimpleCaseTensor(self):
method testSimpleCaseNCHWTensor (line 92) | def testSimpleCaseNCHWTensor(self):
method testDataFormatAgnosticTensor (line 104) | def testDataFormatAgnosticTensor(self):
method testInvalidDataFormat (line 124) | def testInvalidDataFormat(self, data_format):
method testValidDataFormatChannelsFirst (line 135) | def testValidDataFormatChannelsFirst(self, data_format):
method testValidDataFormatChannelsLast (line 145) | def testValidDataFormatChannelsLast(self, data_format):
method testInvalidAxis (line 155) | def testInvalidAxis(self, axis):
method testNoScaleAndInitProvided (line 161) | def testNoScaleAndInitProvided(self):
method testNoOffsetBetaInitProvided (line 170) | def testNoOffsetBetaInitProvided(self):
method testCreateScaleAndScaleProvided (line 179) | def testCreateScaleAndScaleProvided(self):
method testCreateOffsetAndOffsetProvided (line 187) | def testCreateOffsetAndOffsetProvided(self):
method testSliceAxis (line 196) | def testSliceAxis(self):
method testRankChanges (line 211) | def testRankChanges(self):
method testIncompatibleGroupsAndTensor (line 227) | def testIncompatibleGroupsAndTensor(self, shape):
method testWorksWithFunction (line 238) | def testWorksWithFunction(self):
method testBatchSizeAgnostic (line 252) | def testBatchSizeAgnostic(self):
method test5DDataFormatAgnostic (line 276) | def test5DDataFormatAgnostic(self):
method test3DDataFormatAgnostic (line 296) | def test3DDataFormatAgnostic(self):
FILE: sonnet/src/initializers.py
class Initializer (line 25) | class Initializer(abc.ABC):
method __call__ (line 29) | def __call__(self, shape: types.ShapeLike, dtype: tf.DType) -> tf.Tensor:
class Zeros (line 34) | class Zeros(Initializer):
method __call__ (line 37) | def __call__(self, shape: types.ShapeLike, dtype: tf.DType) -> tf.Tensor:
class Ones (line 42) | class Ones(Initializer):
method __call__ (line 45) | def __call__(self, shape: types.ShapeLike, dtype: tf.DType) -> tf.Tensor:
class Constant (line 50) | class Constant(Initializer):
method __init__ (line 53) | def __init__(self, value: Union[float, int]):
method __call__ (line 59) | def __call__(self, shape: types.ShapeLike, dtype: tf.DType) -> tf.Tensor:
class RandomUniform (line 65) | class RandomUniform(Initializer):
method __init__ (line 72) | def __init__(self,
method __call__ (line 89) | def __call__(self, shape: types.ShapeLike, dtype: tf.DType):
class RandomNormal (line 99) | class RandomNormal(Initializer):
method __init__ (line 102) | def __init__(self,
method __call__ (line 119) | def __call__(self, shape: types.ShapeLike, dtype: tf.DType) -> tf.Tensor:
class TruncatedNormal (line 129) | class TruncatedNormal(Initializer):
method __init__ (line 137) | def __init__(self,
method __call__ (line 154) | def __call__(self, shape: types.ShapeLike, dtype: tf.DType):
class Identity (line 164) | class Identity(Initializer):
method __init__ (line 170) | def __init__(self, gain: float = 1.0):
method __call__ (line 178) | def __call__(self, shape: types.ShapeLike, dtype: tf.DType) -> tf.Tensor:
class Orthogonal (line 195) | class Orthogonal(Initializer):
method __init__ (line 214) | def __init__(self, gain: float = 1.0, seed: Optional[int] = None):
method __call__ (line 224) | def __call__(self, shape: types.ShapeLike, dtype: tf.DType) -> tf.Tensor:
class VarianceScaling (line 252) | class VarianceScaling(Initializer):
method __init__ (line 287) | def __init__(self,
method __call__ (line 317) | def __call__(self, shape: types.ShapeLike, dtype: tf.DType) -> tf.Tensor:
function check_initializers (line 345) | def check_initializers(initializers: Mapping[str, Initializer],
function _compute_fans (line 364) | def _compute_fans(shape: types.ShapeLike):
function _as_floating_dtype (line 391) | def _as_floating_dtype(dtype: tf.DType) -> tf.DType:
function _as_numerical_dtype (line 398) | def _as_numerical_dtype(dtype: tf.DType) -> tf.DType:
FILE: sonnet/src/initializers_test.py
class InitializersTest (line 26) | class InitializersTest(test_utils.TestCase, parameterized.TestCase):
method assertDifferentInitializerValues (line 28) | def assertDifferentInitializerValues(self,
method assertRange (line 40) | def assertRange(self,
class ConstantInitializersTest (line 61) | class ConstantInitializersTest(InitializersTest):
method testZeros (line 64) | def testZeros(self, dtype):
method testOnes (line 73) | def testOnes(self, dtype):
method testConstantInvalidValue (line 85) | def testConstantInvalidValue(self, value, value_type):
method testConstantValidValue (line 92) | def testConstantValidValue(self, value, dtype):
method testInvalidDataType (line 101) | def testInvalidDataType(self, initializer):
method testInvalidDataTypeConstant (line 107) | def testInvalidDataTypeConstant(self):
method testTFFunction (line 113) | def testTFFunction(self):
method testBatchAgnostic (line 121) | def testBatchAgnostic(self):
class RandomUniformInitializerTest (line 132) | class RandomUniformInitializerTest(InitializersTest):
method testRangeInitializer (line 134) | def testRangeInitializer(self):
method testDifferentInitializer (line 144) | def testDifferentInitializer(self, dtype):
method testInvalidDataType (line 148) | def testInvalidDataType(self):
method testTFFunction (line 154) | def testTFFunction(self):
method testBatchAgnostic (line 164) | def testBatchAgnostic(self):
class RandomNormalInitializerTest (line 177) | class RandomNormalInitializerTest(InitializersTest):
method testRangeInitializer (line 179) | def testRangeInitializer(self):
method testDifferentInitializer (line 186) | def testDifferentInitializer(self):
method testInvalidDataType (line 191) | def testInvalidDataType(self, dtype):
method testTFFunction (line 197) | def testTFFunction(self):
method testBatchAgnostic (line 207) | def testBatchAgnostic(self):
class TruncatedNormalInitializerTest (line 220) | class TruncatedNormalInitializerTest(InitializersTest):
method testRangeInitializer (line 222) | def testRangeInitializer(self):
method testDifferentInitializer (line 230) | def testDifferentInitializer(self):
method testInvalidDataType (line 235) | def testInvalidDataType(self, dtype):
method testTFFunction (line 241) | def testTFFunction(self):
method testBatchAgnostic (line 251) | def testBatchAgnostic(self):
class IdentityInitializerTest (line 264) | class IdentityInitializerTest(InitializersTest):
method testRange (line 269) | def testRange(self, shape, gain, dtype):
method testInvalidDataType (line 280) | def testInvalidDataType(self):
method testInvalidShape (line 287) | def testInvalidShape(self, dtype):
method testTFFunction (line 294) | def testTFFunction(self):
method testTFFunction4D (line 302) | def testTFFunction4D(self):
method testBatchAgnostic (line 310) | def testBatchAgnostic(self):
class OrthogonalInitializerTest (line 321) | class OrthogonalInitializerTest(InitializersTest):
method testRangeInitializer (line 323) | def testRangeInitializer(self):
method testDuplicatedInitializer (line 327) | def testDuplicatedInitializer(self):
method testInvalidDataType (line 332) | def testInvalidDataType(self, dtype):
method testInvalidShape (line 338) | def testInvalidShape(self):
method testShapesValues (line 349) | def testShapesValues(self, shape):
method testTFFunctionSimple (line 364) | def testTFFunctionSimple(self):
method testTFFunction (line 371) | def testTFFunction(self):
method testBatchAgnostic (line 382) | def testBatchAgnostic(self):
class VarianceScalingInitializerTest (line 396) | class VarianceScalingInitializerTest(InitializersTest):
method testTruncatedNormalDistribution (line 398) | def testTruncatedNormalDistribution(self):
method testNormalDistribution (line 405) | def testNormalDistribution(self):
method testUniformDistribution (line 412) | def testUniformDistribution(self):
method testGlorotUniform (line 419) | def testGlorotUniform(self):
method test_GlorotNormal (line 430) | def test_GlorotNormal(self):
method testLecunUniform (line 444) | def testLecunUniform(self):
method testLecunNormal (line 455) | def testLecunNormal(self):
method testHeUniform (line 467) | def testHeUniform(self):
method testHeNormal (line 478) | def testHeNormal(self):
method testMixedShape (line 493) | def testMixedShape(self, mode, distribution):
method testWithTFFunction (line 506) | def testWithTFFunction(self, mode, distribution):
method testBatchAgnostic (line 519) | def testBatchAgnostic(self, mode, distribution):
method testInvalidDataType (line 533) | def testInvalidDataType(self, dtype):
method testCheckInitializersInvalidType (line 539) | def testCheckInitializersInvalidType(self):
method testCheckInitalizersEmpty (line 544) | def testCheckInitalizersEmpty(self):
method testCheckInitalizersValid (line 550) | def testCheckInitalizersValid(self, keys):
method testCheckInitalizersInvalid (line 556) | def testCheckInitalizersInvalid(self):
FILE: sonnet/src/leaky_clip_by_value.py
function leaky_clip_by_value (line 23) | def leaky_clip_by_value(t: tf.Tensor,
FILE: sonnet/src/leaky_clip_by_value_test.py
class LeakyClipByValueTest (line 23) | class LeakyClipByValueTest(test_utils.TestCase, parameterized.TestCase):
method test_leaky_clip_by_value_forward (line 25) | def test_leaky_clip_by_value_forward(self):
method test_leaky_clip_by_value_backward (line 46) | def test_leaky_clip_by_value_backward(self, init, fn, expected_grad):
FILE: sonnet/src/linear.py
class Linear (line 27) | class Linear(base.Module):
method __init__ (line 30) | def __init__(self,
method _initialize (line 59) | def _initialize(self, inputs: tf.Tensor):
method __call__ (line 82) | def __call__(self, inputs: tf.Tensor) -> tf.Tensor:
FILE: sonnet/src/linear_test.py
class LinearTest (line 24) | class LinearTest(test_utils.TestCase, parameterized.TestCase):
method testInitW (line 26) | def testInitW(self):
method testInitB (line 31) | def testInitB(self):
method testInitializerKeysInvalidWithoutBias (line 36) | def testInitializerKeysInvalidWithoutBias(self):
method testParametersCreatedOnce (line 40) | def testParametersCreatedOnce(self):
method testParameterShape (line 48) | def testParameterShape(self):
method testParameterDtype (line 58) | def testParameterDtype(self, dtype):
method testBiasZeroInitialized (line 70) | def testBiasZeroInitialized(self):
method testCall (line 75) | def testCall(self):
method testCallMultiBatch (line 95) | def testCallMultiBatch(self):
method testFunction (line 109) | def testFunction(self, with_bias):
method testUnknownBatchSize (line 125) | def testUnknownBatchSize(self):
method testUnknownInputSize (line 141) | def testUnknownInputSize(self):
method testMultiBatchOutputDimensions (line 151) | def testMultiBatchOutputDimensions(self):
method testIncorrectDims (line 168) | def testIncorrectDims(self, shape):
method testInputSize (line 173) | def testInputSize(self):
method testOutputSize (line 181) | def testOutputSize(self):
FILE: sonnet/src/metrics.py
class Metric (line 25) | class Metric(base.Module, metaclass=abc.ABCMeta):
method initialize (line 29) | def initialize(self, value):
method update (line 33) | def update(self, value):
method value (line 37) | def value(self):
method reset (line 41) | def reset(self):
method __call__ (line 44) | def __call__(self, value):
class Sum (line 50) | class Sum(Metric):
method __init__ (line 53) | def __init__(self, name: Optional[str] = None):
method initialize (line 58) | def initialize(self, value: tf.Tensor):
method update (line 62) | def update(self, value: tf.Tensor):
method _checked_sum (line 68) | def _checked_sum(self):
method value (line 74) | def value(self) -> tf.Tensor:
method reset (line 78) | def reset(self):
class Mean (line 85) | class Mean(Metric):
method __init__ (line 88) | def __init__(self, name: Optional[str] = None):
method initialize (line 94) | def initialize(self, value: tf.Tensor):
method update (line 98) | def update(self, value: tf.Tensor):
method _checked_sum (line 105) | def _checked_sum(self) -> tf.Variable:
method value (line 111) | def value(self) -> tf.Tensor:
method reset (line 118) | def reset(self):
FILE: sonnet/src/metrics_test.py
class SumTest (line 22) | class SumTest(test_utils.TestCase):
method testSimple (line 24) | def testSimple(self):
method testInitialize (line 29) | def testInitialize(self):
method testReset (line 34) | def testReset(self):
class MeanTest (line 42) | class MeanTest(test_utils.TestCase):
method testSimple (line 44) | def testSimple(self):
method testInitialize (line 49) | def testInitialize(self):
method testReset (line 54) | def testReset(self):
FILE: sonnet/src/mixed_precision.py
function enable (line 30) | def enable(dtype):
function disable (line 40) | def disable():
function _get_mixed_precision_mode (line 45) | def _get_mixed_precision_mode():
function _maybe_cast_element (line 50) | def _maybe_cast_element(x, dtype):
function _maybe_cast_structure (line 56) | def _maybe_cast_structure(x, dtype: tf.DType):
function _cast_call (line 60) | def _cast_call(f, new_dtype, args, kwargs):
function modes (line 73) | def modes(valid_types):
function scope (line 141) | def scope(dtype: tf.DType):
FILE: sonnet/src/mixed_precision_test.py
class DummyVar (line 25) | class DummyVar(base.Module, test_utils.TestCase):
method __init__ (line 27) | def __init__(self, x):
method check_type (line 32) | def check_type(self, _, dtype):
method check_type_structure (line 38) | def check_type_structure(self, _, dtype):
method runTest (line 43) | def runTest(self):
class DummyInput (line 47) | class DummyInput(test_utils.TestCase):
method __init__ (line 49) | def __init__(self, _):
method check_type (line 53) | def check_type(self, x, dtype):
method check_type_structure (line 57) | def check_type_structure(self, x, dtype):
method runTest (line 61) | def runTest(self):
class MixedPrecisionClassTest (line 66) | class MixedPrecisionClassTest(test_utils.TestCase):
method test_float16_mode_variable_eligible_class (line 68) | def test_float16_mode_variable_eligible_class(self, test_class):
method test_float16_mode_disable_class (line 81) | def test_float16_mode_disable_class(self, test_class):
method test_float16_mode_nested_eligible_class (line 94) | def test_float16_mode_nested_eligible_class(self, test_class):
method test_float16_mode_eligible_multiple_instances_class (line 117) | def test_float16_mode_eligible_multiple_instances_class(self, test_cla...
method test_float16_mode_ineligible_multiple_instances_class (line 134) | def test_float16_mode_ineligible_multiple_instances_class(self, test_c...
method test_float16_mode_multiple_instances_different_eligibility_class (line 152) | def test_float16_mode_multiple_instances_different_eligibility_class(
method test_bfloat16_input_float16_mode_eligible_class (line 171) | def test_bfloat16_input_float16_mode_eligible_class(self, test_class):
method test_float16_input_float32_mode_eligible_class (line 182) | def test_float16_input_float32_mode_eligible_class(self, test_class):
method test_function_create_module_eligible (line 195) | def test_function_create_module_eligible(self, test_class):
method test_function_create_module_ineligible (line 210) | def test_function_create_module_ineligible(self, test_class):
method test_function_create_module_not_decorated (line 225) | def test_function_create_module_not_decorated(self, test_class):
method test_scoping_option (line 237) | def test_scoping_option(self, test_class):
method test_scoping_disable (line 250) | def test_scoping_disable(self, test_class):
method test_nested_scoping (line 264) | def test_nested_scoping(self, test_class):
class MixedPrecisionTest (line 282) | class MixedPrecisionTest(test_utils.TestCase):
method test_float16_mode_eligible_func (line 284) | def test_float16_mode_eligible_func(self):
method test_float32_mode_eligible_func (line 300) | def test_float32_mode_eligible_func(self):
method test_float16_mode_ineligible_func (line 314) | def test_float16_mode_ineligible_func(self):
method test_dont_cast_non_floats_func (line 329) | def test_dont_cast_non_floats_func(self):
method test_non_tensor_variable_input_no_cast_func (line 344) | def test_non_tensor_variable_input_no_cast_func(self):
method test_float16_mode_enabled_call_function (line 359) | def test_float16_mode_enabled_call_function(self):
method test_float16_mode_tensor_eligible_class (line 389) | def test_float16_mode_tensor_eligible_class(self):
FILE: sonnet/src/moving_averages.py
class ExponentialMovingAverage (line 25) | class ExponentialMovingAverage(metrics.Metric):
method __init__ (line 50) | def __init__(self, decay: types.FloatLike, name: Optional[str] = None):
method update (line 67) | def update(self, value: tf.Tensor):
method value (line 78) | def value(self) -> tf.Tensor:
method reset (line 82) | def reset(self):
method initialize (line 91) | def initialize(self, value: tf.Tensor):
FILE: sonnet/src/moving_averages_test.py
class ExponentialMovingAverageTest (line 23) | class ExponentialMovingAverageTest(test_utils.TestCase, parameterized.Te...
method testCall (line 25) | def testCall(self):
method testUpdateAndValue (line 31) | def testUpdateAndValue(self):
method testReset (line 39) | def testReset(self):
method testResetVector (line 49) | def testResetVector(self):
method testValueEqualsLatestUpdate (line 58) | def testValueEqualsLatestUpdate(self):
method testWithTFFunction (line 68) | def testWithTFFunction(self, autograph):
method testResetWithTFFunction (line 79) | def testResetWithTFFunction(self, autograph):
method testAlternativeShape (line 90) | def testAlternativeShape(self, shape):
FILE: sonnet/src/nets/cifar10_convnet.py
class Cifar10ConvNet (line 28) | class Cifar10ConvNet(base.Module):
method __init__ (line 36) | def __init__(self,
method __call__ (line 91) | def __call__(
FILE: sonnet/src/nets/cifar10_convnet_test.py
class ModelTest (line 24) | class ModelTest(parameterized.TestCase, test_utils.TestCase):
method testModelCreation (line 26) | def testModelCreation(self):
method testFailedModelCreation (line 31) | def testFailedModelCreation(self):
method testModelForwards (line 39) | def testModelForwards(self, batch_size):
method testModelForwardsFunction (line 53) | def testModelForwardsFunction(self, batch_size):
method testDifferentSizedImages (line 66) | def testDifferentSizedImages(self):
method testDefunBackProp (line 80) | def testDefunBackProp(self):
FILE: sonnet/src/nets/dnc/control.py
function get_controller_ctor (line 29) | def get_controller_ctor(controller_name):
class FeedForward (line 40) | class FeedForward(recurrent.RNNCore):
method __init__ (line 51) | def __init__(self,
method __call__ (line 69) | def __call__(self, inputs, prev_state):
method initial_state (line 86) | def initial_state(self, batch_size):
function deep_core (line 90) | def deep_core(control_name,
FILE: sonnet/src/nets/dnc/control_test.py
class CoreTest (line 26) | class CoreTest(test_utils.TestCase, parameterized.TestCase):
method testShape (line 30) | def testShape(self, constructor):
class FeedForwardTest (line 44) | class FeedForwardTest(test_utils.TestCase):
method testShape (line 46) | def testShape(self):
method testValues (line 60) | def testValues(self):
class DeepCore (line 79) | class DeepCore(test_utils.TestCase, parameterized.TestCase):
method testShape (line 94) | def testShape(self, control_name, num_layers):
FILE: sonnet/src/nets/dnc/read.py
function read (line 20) | def read(memory,
FILE: sonnet/src/nets/dnc/read_test.py
class ReadTest (line 23) | class ReadTest(test_utils.TestCase):
method testShape (line 25) | def testShape(self):
method testValues (line 37) | def testValues(self):
FILE: sonnet/src/nets/dnc/util.py
function segment_dim (line 22) | def segment_dim(inputs, dim, shapes):
function batch_invert_permutation (line 81) | def batch_invert_permutation(permutations):
function batch_gather (line 90) | def batch_gather(values, indices):
function one_hot (line 97) | def one_hot(length, index):
function apply_linear (line 104) | def apply_linear(inputs, linear_modules, activation=tf.identity):
function apply_split_linear (line 132) | def apply_split_linear(lin_module_1,
FILE: sonnet/src/nets/dnc/util_test.py
class SegmentDimTest (line 26) | class SegmentDimTest(test_utils.TestCase, parameterized.TestCase):
method testShape (line 30) | def testShape(self, initial_shape, final_shape):
method testShapeNegative (line 50) | def testShapeNegative(self, initial_shape, final_shape):
method testValues (line 68) | def testValues(self):
method testInvalidDims (line 77) | def testInvalidDims(self):
class BatchInvertPermutationTest (line 84) | class BatchInvertPermutationTest(test_utils.TestCase):
method testCorrectOutput (line 86) | def testCorrectOutput(self):
class BatchGatherTest (line 104) | class BatchGatherTest(test_utils.TestCase):
method testCorrectOutput (line 106) | def testCorrectOutput(self):
class LinearTest (line 114) | class LinearTest(test_utils.TestCase, parameterized.TestCase):
method testLinearOutputOneModule (line 116) | def testLinearOutputOneModule(self):
method testLinearOutputTwoModules (line 128) | def testLinearOutputTwoModules(self):
method testDifferentOutputSizeBreaks (line 144) | def testDifferentOutputSizeBreaks(self):
method testNonMatchingStructureBreaks (line 168) | def testNonMatchingStructureBreaks(self, input_sizes, module_hidden_si...
method testListMustBeLengthTwo (line 188) | def testListMustBeLengthTwo(self, input_sizes, module_hidden_sizes):
FILE: sonnet/src/nets/dnc/write.py
function additive_write (line 20) | def additive_write(memory, address, values):
function erase (line 38) | def erase(memory, address, reset_weights):
function erase_rows (line 65) | def erase_rows(memory, address, reset_row_weights):
function erase_and_write (line 89) | def erase_and_write(memory, address, reset_weights, values):
FILE: sonnet/src/nets/dnc/write_test.py
class EraseRowsTest (line 23) | class EraseRowsTest(test_utils.TestCase):
method testShape (line 25) | def testShape(self):
method testValues (line 38) | def testValues(self):
class EraseTest (line 79) | class EraseTest(test_utils.TestCase):
method testShape (line 81) | def testShape(self):
method testValues (line 94) | def testValues(self):
class EraseAndWriteTest (line 132) | class EraseAndWriteTest(test_utils.TestCase):
method testShape (line 134) | def testShape(self):
method testValues (line 148) | def testValues(self):
class AdditiveWriteTest (line 173) | class AdditiveWriteTest(test_utils.TestCase):
method testShape (line 175) | def testShape(self):
method testValues (line 188) | def testValues(self):
FILE: sonnet/src/nets/mlp.py
class MLP (line 25) | class MLP(base.Module):
method __init__ (line 28) | def __init__(self,
method __call__ (line 75) | def __call__(self, inputs: tf.Tensor, is_training=None) -> tf.Tensor:
method reverse (line 106) | def reverse(self,
FILE: sonnet/src/nets/mlp_test.py
class MLPTest (line 24) | class MLPTest(test_utils.TestCase, parameterized.TestCase):
method test_b_init_when_with_bias_false (line 26) | def test_b_init_when_with_bias_false(self):
method test_submodules (line 31) | def test_submodules(self, num_layers, dropout_rate):
method test_applies_activation (line 36) | def test_applies_activation(self, num_layers):
method test_activate_final (line 43) | def test_activate_final(self, num_layers):
method test_adds_index_to_layer_names (line 50) | def test_adds_index_to_layer_names(self, num_layers):
method test_passes_with_bias_to_layers (line 56) | def test_passes_with_bias_to_layers(self, with_bias):
method test_repeat_initializer (line 61) | def test_repeat_initializer(self):
method test_default_name (line 69) | def test_default_name(self):
method test_custom_name (line 73) | def test_custom_name(self):
method test_reverse_default_name (line 77) | def test_reverse_default_name(self):
method test_reverse_custom_name (line 81) | def test_reverse_custom_name(self):
method test_reverse_override_name (line 85) | def test_reverse_override_name(self):
method test_reverse (line 91) | def test_reverse(self):
method test_reverse_passed_with_bias (line 96) | def test_reverse_passed_with_bias(self, with_bias):
method test_reverse_w_init (line 101) | def test_reverse_w_init(self):
method test_reverse_b_init (line 107) | def test_reverse_b_init(self):
method test_reverse_activation (line 113) | def test_reverse_activation(self):
method test_dropout_requires_is_training (line 120) | def test_dropout_requires_is_training(self):
method test_no_dropout_rejects_is_training (line 126) | def test_no_dropout_rejects_is_training(self, is_training):
method test_reverse_activate_final (line 132) | def test_reverse_activate_final(self, activate_final):
method test_applies_activation_with_dropout (line 140) | def test_applies_activation_with_dropout(self, use_dropout, is_training):
method test_repr (line 148) | def test_repr(self):
function reversed_mlp (line 156) | def reversed_mlp(**kwargs):
class CountingActivation (line 162) | class CountingActivation:
method __init__ (line 164) | def __init__(self):
method __call__ (line 167) | def __call__(self, x):
class CountingInitializer (line 172) | class CountingInitializer:
method __init__ (line 174) | def __init__(self):
method __call__ (line 177) | def __call__(self, shape, dtype=tf.float32):
FILE: sonnet/src/nets/resnet.py
class BottleNeckBlockV1 (line 28) | class BottleNeckBlockV1(base.Module):
method __init__ (line 31) | def __init__(self,
method __call__ (line 91) | def __call__(self, inputs, is_training):
class BottleNeckBlockV2 (line 107) | class BottleNeckBlockV2(base.Module):
method __init__ (line 110) | def __init__(self,
method __call__ (line 166) | def __call__(self, inputs, is_training):
class BlockGroup (line 182) | class BlockGroup(base.Module):
method __init__ (line 185) | def __init__(self,
method __call__ (line 213) | def __call__(self, inputs, is_training):
class ResNet (line 220) | class ResNet(base.Module):
method __init__ (line 223) | def __init__(self,
method __call__ (line 301) | def __call__(self, inputs, is_training):
class ResNet50 (line 321) | class ResNet50(ResNet):
method __init__ (line 324) | def __init__(self,
FILE: sonnet/src/nets/resnet_test.py
class ResnetTest (line 23) | class ResnetTest(test_utils.TestCase, parameterized.TestCase):
method test_simple (line 26) | def test_simple(self, resnet_v2):
method test_tf_function (line 35) | def test_tf_function(self, resnet_v2):
method test_error_incorrect_args_block_list (line 50) | def test_error_incorrect_args_block_list(self, list_length):
method test_error_incorrect_args_channel_list (line 58) | def test_error_incorrect_args_channel_list(self, list_length):
FILE: sonnet/src/nets/vqvae.py
class VectorQuantizer (line 25) | class VectorQuantizer(base.Module):
method __init__ (line 50) | def __init__(self,
method __call__ (line 77) | def __call__(self, inputs, is_training):
method quantize (line 134) | def quantize(self, encoding_indices):
class VectorQuantizerEMA (line 142) | class VectorQuantizerEMA(base.Module):
method __init__ (line 176) | def __init__(self,
method __call__ (line 221) | def __call__(self, inputs, is_training):
method quantize (line 297) | def quantize(self, encoding_indices):
FILE: sonnet/src/nets/vqvae_test.py
class VqvaeTest (line 26) | class VqvaeTest(parameterized.TestCase, test_utils.TestCase):
method testConstruct (line 38) | def testConstruct(self, constructor, kwargs):
method testShapeChecking (line 84) | def testShapeChecking(self, constructor, kwargs):
method testNoneBatch (line 102) | def testNoneBatch(self, constructor, kwargs):
method testEmaUpdating (line 112) | def testEmaUpdating(self, use_tf_function, dtype):
method testEmbeddingsNotTrainable (line 147) | def testEmbeddingsNotTrainable(self):
FILE: sonnet/src/once.py
function _check_no_output (line 24) | def _check_no_output(output):
function once (line 29) | def once(f):
FILE: sonnet/src/once_test.py
class OnceTest (line 24) | class OnceTest(parameterized.TestCase):
method test_runs_once (line 26) | def test_runs_once(self):
method test_always_returns_none (line 38) | def test_always_returns_none(self):
method test_does_not_cache_on_error (line 43) | def test_does_not_cache_on_error(self):
method test_method (line 54) | def test_method(self):
method test_method_does_not_cache_on_error (line 64) | def test_method_does_not_cache_on_error(self):
method test_pickle_method_before_evaluation (line 78) | def test_pickle_method_before_evaluation(self):
method test_pickle_method_already_evaluated (line 88) | def test_pickle_method_already_evaluated(self):
method test_inline (line 97) | def test_inline(self):
method test_adds_property (line 109) | def test_adds_property(self, factory):
function nop (line 114) | def nop():
class NoOpCallable (line 118) | class NoOpCallable:
method nop (line 120) | def nop(self):
method __call__ (line 123) | def __call__(self):
class Counter (line 127) | class Counter:
method increment (line 131) | def increment(self):
FILE: sonnet/src/optimizers/adam.py
function adam_update (line 27) | def adam_update(g, alpha, beta_1, beta_2, epsilon, t, m, v):
class Adam (line 37) | class Adam(base.Optimizer):
method __init__ (line 56) | def __init__(self,
method _initialize (line 82) | def _initialize(self, parameters: Sequence[tf.Variable]):
method apply (line 90) | def apply(self, updates: Sequence[types.ParameterUpdate],
FILE: sonnet/src/optimizers/adam_test.py
class ComparisonTest (line 29) | class ComparisonTest(optimizer_tests.AbstractFuzzTest):
method _make_tf (line 32) | def _make_tf(self, learning_rate, beta_1, beta_2, epsilon):
method _make_snt (line 39) | def _make_snt(self, learning_rate, beta_1, beta_2, epsilon):
method testComparingSonnetAndTensorFlow (line 47) | def testComparingSonnetAndTensorFlow(self, config):
class AdamTest (line 52) | class AdamTest(optimizer_tests.OptimizerTestBase):
method make_optimizer (line 54) | def make_optimizer(self, **kwargs):
method testDense (line 59) | def testDense(self):
method testSparse (line 76) | def testSparse(self):
method testVariableHyperParams (line 117) | def testVariableHyperParams(self):
method testHyperParamDTypeConversion (line 132) | def testHyperParamDTypeConversion(self):
method testAuxVariablesColocatedWithOriginal (line 147) | def testAuxVariablesColocatedWithOriginal(self):
class ReferenceAdamTest (line 156) | class ReferenceAdamTest(optimizer_tests.OptimizerTestBase):
method make_optimizer (line 158) | def make_optimizer(self, **kwargs):
FILE: sonnet/src/optimizers/momentum.py
function momentum_update (line 27) | def momentum_update(update, learning_rate, mu, momentum, use_nesterov):
class Momentum (line 37) | class Momentum(base.Optimizer):
method __init__ (line 47) | def __init__(self,
method _initialize (line 67) | def _initialize(self, parameters):
method apply (line 72) | def apply(self, updates: Sequence[types.ParameterUpdate],
FILE: sonnet/src/optimizers/momentum_test.py
class ComparisonTest (line 28) | class ComparisonTest(optimizer_tests.AbstractFuzzTest):
method _make_tf (line 31) | def _make_tf(self, learning_rate, momentum, use_nesterov):
method _make_snt (line 37) | def _make_snt(self, learning_rate, momentum, use_nesterov):
method testComparingSonnetAndTensorFlow (line 44) | def testComparingSonnetAndTensorFlow(self, config):
class MomentumTest (line 49) | class MomentumTest(optimizer_tests.OptimizerTestBase):
method make_optimizer (line 51) | def make_optimizer(self, **kwargs):
method testDense (line 58) | def testDense(self):
method testDenseNesterov (line 75) | def testDenseNesterov(self):
method testSparse (line 93) | def testSparse(self):
method testSparseNesterov (line 121) | def testSparseNesterov(self):
method testVariableHyperParams (line 150) | def testVariableHyperParams(self):
method testHyperParamDTypeConversion (line 171) | def testHyperParamDTypeConversion(self):
method testAuxVariablesColocatedWithOriginal (line 183) | def testAuxVariablesColocatedWithOriginal(self):
class ReferenceMomentumTest (line 194) | class ReferenceMomentumTest(MomentumTest):
method make_optimizer (line 196) | def make_optimizer(self, **kwargs):
FILE: sonnet/src/optimizers/optimizer_tests.py
class WrappedTFOptimizer (line 27) | class WrappedTFOptimizer(base.Optimizer):
method __init__ (line 32) | def __init__(self, optimizer: tf.optimizers.Optimizer):
method __getattr__ (line 36) | def __getattr__(self, name):
method apply (line 39) | def apply(self, updates, params):
function is_tf_optimizer (line 43) | def is_tf_optimizer(optimizer):
class OptimizerTestBase (line 47) | class OptimizerTestBase(test_utils.TestCase):
method make_optimizer (line 50) | def make_optimizer(self, *args, **kwargs):
method testNoneUpdate (line 53) | def testNoneUpdate(self):
method testDifferentLengthUpdatesParams (line 60) | def testDifferentLengthUpdatesParams(self):
method testEmptyParams (line 71) | def testEmptyParams(self):
method testAllUpdatesNone (line 79) | def testAllUpdatesNone(self):
method testInconsistentDTypes (line 90) | def testInconsistentDTypes(self):
method testUnsuppportedStrategyError (line 101) | def testUnsuppportedStrategyError(self):
class AbstractFuzzTest (line 116) | class AbstractFuzzTest(test_utils.TestCase, parameterized.TestCase):
method _make_tf (line 119) | def _make_tf(self, learning_rate, momentum, use_nesterov):
method _make_snt (line 122) | def _make_snt(self, learning_rate, momentum, use_nesterov):
method assertParametersRemainClose (line 125) | def assertParametersRemainClose(self, seed, config, num_steps=100, ato...
function _generate_dense_data (line 143) | def _generate_dense_data(seed, num_steps):
function _apply_optimizer (line 162) | def _apply_optimizer(data, apply_fn):
function named_product (line 174) | def named_product(**config):
FILE: sonnet/src/optimizers/optimizer_utils.py
function check_distribution_strategy (line 34) | def check_distribution_strategy():
function check_updates_parameters (line 44) | def check_updates_parameters(updates: Sequence[types.ParameterUpdate],
function check_same_dtype (line 54) | def check_same_dtype(update: types.ParameterUpdate, parameter: tf.Variab...
function deduplicate_indexed_slices (line 61) | def deduplicate_indexed_slices(indexed_slice: tf.IndexedSlices):
FILE: sonnet/src/optimizers/rmsprop.py
function rmsprop_update (line 28) | def rmsprop_update(update, decay, learning_rate, epsilon, mu, mom, ms, mg):
class RMSProp (line 40) | class RMSProp(base.Optimizer):
method __init__ (line 73) | def __init__(self,
method _initialize (line 104) | def _initialize(self, parameters: Sequence[tf.Variable]):
method apply (line 114) | def apply(self, updates: Sequence[types.ParameterUpdate],
FILE: sonnet/src/optimizers/rmsprop_test.py
class ComparisonTest (line 30) | class ComparisonTest(optimizer_tests.AbstractFuzzTest):
method _make_tf (line 33) | def _make_tf(self, learning_rate, decay, momentum, epsilon, centered):
method _make_snt (line 41) | def _make_snt(self, learning_rate, decay, momentum, epsilon, centered):
method testComparingSonnetAndTensorFlow (line 50) | def testComparingSonnetAndTensorFlow(self, config):
class RMSPropTest (line 55) | class RMSPropTest(optimizer_tests.OptimizerTestBase):
method make_optimizer (line 57) | def make_optimizer(self, **kwargs):
method testDense (line 62) | def testDense(self):
method testDenseCentered (line 79) | def testDenseCentered(self):
method testSparse (line 96) | def testSparse(self):
method testSparseCentered (line 124) | def testSparseCentered(self):
method testVariableHyperParams (line 152) | def testVariableHyperParams(self):
method testHyperParamDTypeConversion (line 166) | def testHyperParamDTypeConversion(self):
method testAuxVariablesColocatedWithOriginal (line 186) | def testAuxVariablesColocatedWithOriginal(self):
class ReferenceRMSPropTest (line 198) | class ReferenceRMSPropTest(RMSPropTest):
method make_optimizer (line 200) | def make_optimizer(self, **kwargs):
FILE: sonnet/src/optimizers/sgd.py
class SGD (line 25) | class SGD(base.Optimizer):
method __init__ (line 32) | def __init__(self,
method apply (line 44) | def apply(self, updates: Sequence[types.ParameterUpdate],
FILE: sonnet/src/optimizers/sgd_test.py
class SGDTest (line 22) | class SGDTest(optimizer_tests.OptimizerTestBase):
method make_optimizer (line 24) | def make_optimizer(self, *args, **kwargs):
method testDense (line 29) | def testDense(self):
method testSparse (line 37) | def testSparse(self):
method testVariableLearningRate (line 55) | def testVariableLearningRate(self):
method testLearningRateDTypeConversion (line 69) | def testLearningRateDTypeConversion(self):
class ReferenceSGDTest (line 80) | class ReferenceSGDTest(SGDTest):
method make_optimizer (line 82) | def make_optimizer(self, *args, **kwargs):
FILE: sonnet/src/pad.py
function valid (line 25) | def valid(effective_kernel_size: int): # pylint: disable=unused-argument
function same (line 30) | def same(effective_kernel_size: int):
function full (line 35) | def full(effective_kernel_size: int):
function causal (line 40) | def causal(effective_kernel_size: int):
function reverse_causal (line 45) | def reverse_causal(effective_kernel_size: int):
function create (line 50) | def create(
FILE: sonnet/src/pad_test.py
class PadTest (line 23) | class PadTest(test_utils.TestCase, parameterized.TestCase):
method test_padding_2d (line 25) | def test_padding_2d(self):
method test_padding_1d (line 29) | def test_padding_1d(self):
method test_padding_3d (line 33) | def test_padding_3d(self):
method test_padding_incorrect_input (line 39) | def test_padding_incorrect_input(self, kernel_size, rate):
method test_padding_valid (line 45) | def test_padding_valid(self):
method test_padding_same (line 49) | def test_padding_same(self):
method test_padding_full (line 53) | def test_padding_full(self):
method test_padding_causal (line 57) | def test_padding_causal(self):
method test_padding_reverse_causal (line 61) | def test_padding_reverse_causal(self):
method test_same_padding (line 67) | def test_same_padding(self, kernel_size, stride, rate):
method test_valid_padding (line 80) | def test_valid_padding(self, kernel_size, stride, rate):
FILE: sonnet/src/parallel_linear.py
class ParallelLinears (line 27) | class ParallelLinears(base.Module):
method __init__ (line 39) | def __init__(self,
method _initialize (line 68) | def _initialize(self, inputs: tf.Tensor):
method __call__ (line 95) | def __call__(self, inputs: tf.Tensor) -> tf.Tensor:
FILE: sonnet/src/parallel_linear_test.py
class ParallelLinearTest (line 23) | class ParallelLinearTest(test_utils.TestCase):
method test_output_size_correct (line 25) | def test_output_size_correct(self):
method test_behaves_same_as_stacked_linears (line 31) | def test_behaves_same_as_stacked_linears(self):
FILE: sonnet/src/recurrent.py
class RNNCore (line 40) | class RNNCore(base.Module, metaclass=abc.ABCMeta):
method __call__ (line 54) | def __call__(self, inputs: types.TensorNest, prev_state):
method initial_state (line 72) | def initial_state(self, batch_size: types.IntegerLike, **kwargs):
class UnrolledRNN (line 84) | class UnrolledRNN(base.Module, metaclass=abc.ABCMeta):
method __call__ (line 92) | def __call__(self, input_sequence: types.TensorNest,
method initial_state (line 111) | def initial_state(self, batch_size: types.IntegerLike, **kwargs):
class TrainableState (line 123) | class TrainableState(base.Module):
method for_core (line 137) | def for_core(cls,
method __init__ (line 157) | def __init__(self,
method __call__ (line 191) | def __call__(self, batch_size: int) -> types.TensorNest:
function static_unroll (line 198) | def static_unroll(
class _ListWrapper (line 279) | class _ListWrapper:
method __init__ (line 288) | def __init__(self, data):
function dynamic_unroll (line 295) | def dynamic_unroll(
function _unstack_input_sequence (line 386) | def _unstack_input_sequence(input_sequence):
function _safe_where (line 427) | def _safe_where(condition, x, y): # pylint: disable=g-doc-args
function _rnn_step (line 437) | def _rnn_step(core, input_tas, sequence_length, t, prev_outputs, prev_st...
class VanillaRNN (line 457) | class VanillaRNN(RNNCore):
method __init__ (line 475) | def __init__(self,
method input_to_hidden (line 512) | def input_to_hidden(self) -> tf.Variable:
method hidden_to_hidden (line 516) | def hidden_to_hidden(self) -> tf.Variable:
method __call__ (line 519) | def __call__(self, inputs: types.TensorNest,
method initial_state (line 531) | def initial_state(self, batch_size: int) -> tf.Tensor:
method _initialize (line 536) | def _initialize(self, inputs: tf.Tensor):
class _LegacyDeepRNN (line 541) | class _LegacyDeepRNN(RNNCore):
method __init__ (line 548) | def __init__(self,
method __call__ (line 566) | def __call__(self, inputs, prev_state):
method initial_state (line 595) | def initial_state(self, batch_size, **kwargs):
class DeepRNN (line 603) | class DeepRNN(_LegacyDeepRNN):
method __init__ (line 626) | def __init__(self, layers, name: Optional[str] = None):
function deep_rnn_with_skip_connections (line 630) | def deep_rnn_with_skip_connections(
class _ResidualWrapper (line 672) | class _ResidualWrapper(RNNCore):
method __init__ (line 679) | def __init__(self, base_core: RNNCore):
method __call__ (line 683) | def __call__(self, inputs: types.TensorNest, prev_state: types.TensorN...
method initial_state (line 689) | def initial_state(self, batch_size, **kwargs):
function deep_rnn_with_residual_connections (line 693) | def deep_rnn_with_residual_connections(
class LSTM (line 738) | class LSTM(RNNCore):
method __init__ (line 781) | def __init__(self,
method __call__ (line 829) | def __call__(self, inputs, prev_state):
method initial_state (line 835) | def initial_state(self, batch_size: int) -> LSTMState:
method input_to_hidden (line 842) | def input_to_hidden(self):
method hidden_to_hidden (line 846) | def hidden_to_hidden(self):
method _initialize (line 850) | def _initialize(self, inputs):
function _lstm_fn (line 882) | def _lstm_fn(inputs, prev_state, w_i, w_h, b, projection=None):
class UnrolledLSTM (line 901) | class UnrolledLSTM(UnrolledRNN):
method __init__ (line 909) | def __init__(self,
method __call__ (line 943) | def __call__(self, input_sequence, initial_state):
method initial_state (line 949) | def initial_state(self, batch_size):
method input_to_hidden (line 956) | def input_to_hidden(self):
method hidden_to_hidden (line 960) | def hidden_to_hidden(self):
method _initialize (line 964) | def _initialize(self, input_sequence):
function _specialize_per_device (line 985) | def _specialize_per_device(api_name, specializations, default):
function _fallback_unrolled_lstm (line 1045) | def _fallback_unrolled_lstm(input_sequence, initial_state, w_i, w_h, b):
function _block_unrolled_lstm (line 1052) | def _block_unrolled_lstm(input_sequence, initial_state, w_i, w_h, b):
function _cudnn_unrolled_lstm (line 1070) | def _cudnn_unrolled_lstm(input_sequence, initial_state, w_i, w_h, b):
class _RecurrentDropoutWrapper (line 1111) | class _RecurrentDropoutWrapper(RNNCore):
method __init__ (line 1123) | def __init__(self, base_core: RNNCore, rates, seed: Optional[int] = No...
method __call__ (line 1137) | def __call__(self, inputs, prev_state):
method initial_state (line 1147) | def initial_state(self, batch_size, **kwargs):
function lstm_with_recurrent_dropout (line 1161) | def lstm_with_recurrent_dropout(hidden_size, dropout=0.5, seed=None, **k...
class _ConvNDLSTM (line 1203) | class _ConvNDLSTM(RNNCore):
method __init__ (line 1247) | def __init__(self,
method __call__ (line 1314) | def __call__(self, inputs, prev_state):
method input_to_hidden (line 1332) | def input_to_hidden(self):
method hidden_to_hidden (line 1336) | def hidden_to_hidden(self):
method initial_state (line 1339) | def initial_state(self, batch_size):
method _initialize (line 1349) | def _initialize(self, inputs):
class Conv1DLSTM (line 1357) | class Conv1DLSTM(_ConvNDLSTM): # pylint: disable=missing-docstring,empt...
method __init__ (line 1360) | def __init__(self,
class Conv2DLSTM (line 1408) | class Conv2DLSTM(_ConvNDLSTM): # pylint: disable=missing-docstring,empt...
method __init__ (line 1411) | def __init__(self,
class Conv3DLSTM (line 1459) | class Conv3DLSTM(_ConvNDLSTM): # pylint: disable=missing-docstring,empt...
method __init__ (line 1462) | def __init__(self,
class GRU (line 1510) | class GRU(RNNCore):
method __init__ (line 1538) | def __init__(self,
method __call__ (line 1568) | def __call__(self, inputs, prev_state):
method initial_state (line 1587) | def initial_state(self, batch_size):
method input_to_hidden (line 1592) | def input_to_hidden(self):
method hidden_to_hidden (line 1596) | def hidden_to_hidden(self):
method _initialize (line 1600) | def _initialize(self, inputs):
class CuDNNGRU (line 1613) | class CuDNNGRU(RNNCore):
method __init__ (line 1627) | def __init__(self,
method __call__ (line 1657) | def __call__(self, inputs, prev_state):
method input_to_hidden (line 1706) | def input_to_hidden(self):
method hidden_to_hidden (line 1710) | def hidden_to_hidden(self):
method initial_state (line 1713) | def initial_state(self, batch_size):
method _initialize (line 1718) | def _initialize(self, inputs):
function _check_inputs_dtype (line 1730) | def _check_inputs_dtype(inputs, expected_dtype):
FILE: sonnet/src/recurrent_test.py
class VanillaRNNTest (line 29) | class VanillaRNNTest(test_utils.TestCase, parameterized.TestCase):
method setUp (line 31) | def setUp(self):
method testComputationAgainstNumPy (line 38) | def testComputationAgainstNumPy(self, use_tf_function):
method testDtypeMismatch (line 57) | def testDtypeMismatch(self):
method testInitialization (line 66) | def testInitialization(self):
class DeepRNNTest (line 80) | class DeepRNNTest(test_utils.TestCase, parameterized.TestCase):
method setUp (line 82) | def setUp(self):
method testComputationAgainstNumPy (line 89) | def testComputationAgainstNumPy(self, use_tf_function):
method testComputationAgainstNumPyWithCallables (line 111) | def testComputationAgainstNumPyWithCallables(self, use_tf_function):
method testInitialState (line 123) | def testInitialState(self):
method testWithSkipConnectionsOutputs (line 132) | def testWithSkipConnectionsOutputs(self, use_tf_function):
method testWithConnectionsValidation (line 148) | def testWithConnectionsValidation(self):
class LSTMTest (line 155) | class LSTMTest(test_utils.TestCase, parameterized.TestCase):
method setUp (line 157) | def setUp(self):
method testComputationAgainstNumPy (line 165) | def testComputationAgainstNumPy(self, use_tf_function, projection_size,
method testDtypeMismatch (line 197) | def testDtypeMismatch(self):
method testInitialization (line 207) | def testInitialization(self):
method testRecurrentDropout (line 225) | def testRecurrentDropout(self, rate):
method testRecurrentDropoutInvalid (line 248) | def testRecurrentDropoutInvalid(self):
class UnrolledLSTMTest (line 254) | class UnrolledLSTMTest(test_utils.TestCase, parameterized.TestCase):
method setUp (line 256) | def setUp(self):
method testComputationAgainstLSTM (line 264) | def testComputationAgainstLSTM(self, num_steps, use_tf_function):
method testNumStepsPolymorphism (line 300) | def testNumStepsPolymorphism(self, use_tf_function):
method testDtypeMismatch (line 322) | def testDtypeMismatch(self):
method testInitialization (line 333) | def testInitialization(self):
class ConvNDLSTMTest (line 348) | class ConvNDLSTMTest(test_utils.TestCase, parameterized.TestCase):
method setUp (line 350) | def setUp(self):
method testComputationAgainstNumPy (line 362) | def testComputationAgainstNumPy(self, use_tf_function, core_cls):
method testDtypeMismatch (line 403) | def testDtypeMismatch(self):
method testInitialization (line 421) | def testInitialization(self):
class GRUTest (line 442) | class GRUTest(test_utils.TestCase, parameterized.TestCase):
method setUp (line 444) | def setUp(self):
method testComputationAgainstNumPy (line 451) | def testComputationAgainstNumPy(self, use_tf_function):
method testDtypeMismatch (line 473) | def testDtypeMismatch(self):
method testInitialization (line 482) | def testInitialization(self):
function expit (line 496) | def expit(x):
class CuDNNGRUTest (line 500) | class CuDNNGRUTest(test_utils.TestCase, parameterized.TestCase):
method setUp (line 502) | def setUp(self):
method testComputationAgainstTF (line 513) | def testComputationAgainstTF(self, num_steps):
method testDtypeMismatch (line 550) | def testDtypeMismatch(self):
method testInitialization (line 559) | def testInitialization(self):
class Counter (line 573) | class Counter(recurrent.RNNCore):
method __init__ (line 583) | def __init__(self, hidden_size, name=None):
method __call__ (line 588) | def __call__(self, inputs, prev_state):
method initial_state (line 598) | def initial_state(self, batch_size):
class Replicate (line 602) | class Replicate(recurrent.RNNCore):
method __init__ (line 605) | def __init__(self, base_core, n, name=None):
method __call__ (line 610) | def __call__(self, inputs, prev_state):
method initial_state (line 614) | def initial_state(self, batch_size, **kwargs):
class TrainableStateTest (line 618) | class TrainableStateTest(test_utils.TestCase, parameterized.TestCase):
method testUnmasked (line 631) | def testUnmasked(self, initial_values_shape):
method testMasked (line 644) | def testMasked(self):
method testForCore (line 659) | def testForCore(self):
class UnrollTest (line 684) | class UnrollTest(test_utils.TestCase, parameterized.TestCase):
method setUp (line 686) | def setUp(self):
method testFlat (line 694) | def testFlat(self, use_tf_function, unroll_fn):
method testNestedInputs (line 708) | def testNestedInputs(self, use_tf_function, unroll_fn):
method testNestedOutputs (line 725) | def testNestedOutputs(self, use_tf_function, unroll_fn):
method testEmptyOutputs (line 742) | def testEmptyOutputs(self, use_tf_function, unroll_fn):
method testZeroSteps (line 755) | def testZeroSteps(self, use_tf_function, unroll_fn):
method testInconsistentSteps (line 766) | def testInconsistentSteps(self, use_tf_function, unroll_fn):
method testVariableLengthOneZeroLength (line 778) | def testVariableLengthOneZeroLength(self, use_tf_function, unroll_fn):
method testVariableLengthRange (line 794) | def testVariableLengthRange(self, use_tf_function, unroll_fn):
method assertConsistentWithLength (line 809) | def assertConsistentWithLength(self, output_sequence, sequence_length):
method testVariableLengthAllFull (line 819) | def testVariableLengthAllFull(self, use_tf_function, unroll_fn):
method testVariableLengthAllEmpty (line 835) | def testVariableLengthAllEmpty(self, use_tf_function, unroll_fn):
class UnknownStepsUnrollTest (line 852) | class UnknownStepsUnrollTest(test_utils.TestCase):
method setUp (line 854) | def setUp(self):
method testStaticUnroll (line 862) | def testStaticUnroll(self):
method testDynamicUnroll (line 873) | def testDynamicUnroll(self):
method testDynamicUnrollInconsistentSteps (line 886) | def testDynamicUnrollInconsistentSteps(self):
FILE: sonnet/src/regularizers.py
class Regularizer (line 24) | class Regularizer(abc.ABC):
method __call__ (line 28) | def __call__(self, tensors: Sequence[tf.Tensor]) -> tf.Tensor:
class L1 (line 39) | class L1(Regularizer):
method __init__ (line 47) | def __init__(self, scale: types.FloatLike):
method __repr__ (line 59) | def __repr__(self):
method __call__ (line 65) | def __call__(self, tensors: Sequence[tf.Tensor]) -> tf.Tensor:
class L2 (line 73) | class L2(Regularizer):
method __init__ (line 81) | def __init__(self, scale: types.FloatLike):
method __repr__ (line 93) | def __repr__(self):
method __call__ (line 99) | def __call__(self, tensors: Sequence[tf.Tensor]) -> tf.Tensor:
class OffDiagonalOrthogonal (line 107) | class OffDiagonalOrthogonal(Regularizer):
method __init__ (line 136) | def __init__(self, scale: types.FloatLike):
method __repr__ (line 147) | def __repr__(self):
method __call__ (line 153) | def __call__(self, tensors: Sequence[tf.Tensor]) -> tf.Tensor:
function _check_scale (line 168) | def _check_scale(scale: types.FloatLike) -> types.FloatLike:
FILE: sonnet/src/regularizers_test.py
class L1Test (line 23) | class L1Test(test_utils.TestCase):
method testAgainstNumPy (line 25) | def testAgainstNumPy(self):
method testNegativeScale (line 36) | def testNegativeScale(self):
method testEmpty (line 40) | def testEmpty(self):
class L2Test (line 44) | class L2Test(test_utils.TestCase):
method testAgainstNumPy (line 46) | def testAgainstNumPy(self):
method testNegativeScale (line 57) | def testNegativeScale(self):
method testEmpty (line 61) | def testEmpty(self):
class OffDiagonalOrthogonalTest (line 65) | class OffDiagonalOrthogonalTest(test_utils.TestCase):
method testAgainstNumPy (line 67) | def testAgainstNumPy(self):
method testNegativeScale (line 81) | def testNegativeScale(self):
method testEmpty (line 85) | def testEmpty(self):
FILE: sonnet/src/reshape.py
function reshape (line 26) | def reshape(inputs: tf.Tensor,
function flatten (line 34) | def flatten(inputs: tf.Tensor, name: str = "flatten") -> tf.Tensor:
function _infer_shape (line 39) | def _infer_shape(output_shape: types.ShapeLike, dimensions: Sequence[int]):
class Reshape (line 61) | class Reshape(base.Module):
method __init__ (line 88) | def __init__(self,
method _initialize (line 115) | def _initialize(self, inputs: tf.Tensor):
method __call__ (line 123) | def __call__(self, inputs: tf.Tensor) -> tf.Tensor:
method reversed (line 162) | def reversed(self, name: Optional[str] = None) -> "Reshape":
class Flatten (line 173) | class Flatten(Reshape):
method __init__ (line 183) | def __init__(self, preserve_dims: int = 1, name: Optional[str] = None):
FILE: sonnet/src/reshape_test.py
class ReshapeTest (line 26) | class ReshapeTest(test_utils.TestCase, parameterized.TestCase):
method testReshape (line 34) | def testReshape(self, preserve_dims, expected_output_shape):
method testInvalid_multipleWildcard (line 39) | def testInvalid_multipleWildcard(self):
method testInvalid_negativeSize (line 44) | def testInvalid_negativeSize(self):
method testInvalid_type (line 50) | def testInvalid_type(self):
method testIncompatibleShape (line 55) | def testIncompatibleShape(self):
method testInferShape (line 65) | def testInferShape(self):
method testAddDimensions (line 72) | def testAddDimensions(self):
method testFlatten (line 85) | def testFlatten(self):
method testUnknownBatchSize (line 92) | def testUnknownBatchSize(self):
method testReverse (line 99) | def testReverse(self):
method testReverse_name (line 120) | def testReverse_name(self):
method testInvalidPreserveDimsError (line 126) | def testInvalidPreserveDimsError(self):
method testBuildDimError (line 130) | def testBuildDimError(self):
method testPreserve (line 144) | def testPreserve(self, preserve):
method testRun (line 166) | def testRun(self, preserve, trailing_in, trailing_out):
class FlattenTest (line 185) | class FlattenTest(test_utils.TestCase, parameterized.TestCase):
method testFlatten (line 188) | def testFlatten(self, batch_size):
method testFlatten_unknownBatchSize (line 196) | def testFlatten_unknownBatchSize(self):
method testFlatten_unknownNonBatchSize (line 205) | def testFlatten_unknownNonBatchSize(self):
method testPreserveDimsOk (line 215) | def testPreserveDimsOk(self, preserve_dims):
method testPreserveDimsError (line 226) | def testPreserveDimsError(self, preserve_dims):
method testFlattenWithZeroDim (line 233) | def testFlattenWithZeroDim(self):
method testInvalidFlattenFromError (line 238) | def testInvalidFlattenFromError(self):
method testBuildDimError (line 242) | def testBuildDimError(self):
method testReverse (line 249) | def testReverse(self, batch_size):
FILE: sonnet/src/scale_gradient.py
function scale_gradient (line 24) | def scale_gradient(
FILE: sonnet/src/scale_gradient_test.py
class ScaleGradientTest (line 25) | class ScaleGradientTest(test_utils.TestCase, parameterized.TestCase):
method test_scale (line 29) | def test_scale(self, t_, scale):
FILE: sonnet/src/sequential.py
class Sequential (line 22) | class Sequential(base.Module):
method __init__ (line 57) | def __init__(self,
method __call__ (line 63) | def __call__(self, inputs, *args, **kwargs):
FILE: sonnet/src/sequential_test.py
class SequentialTest (line 26) | class SequentialTest(test_utils.TestCase, parameterized.TestCase):
method test_empty (line 29) | def test_empty(self, value):
method test_empty_drops_varargs_varkwargs (line 34) | def test_empty_drops_varargs_varkwargs(self, value):
method test_identity_chain (line 39) | def test_identity_chain(self, value):
method test_call (line 43) | def test_call(self):
method test_varargs_varkwargs_to_call (line 47) | def test_varargs_varkwargs_to_call(self):
function identity (line 54) | def identity(v):
function append_character (line 58) | def append_character(c):
FILE: sonnet/src/test_utils.py
class TestCase (line 35) | class TestCase(tf.test.TestCase):
method setUp (line 40) | def setUp(self):
method tearDown (line 63) | def tearDown(self):
method primary_device (line 70) | def primary_device(self):
method device_types (line 79) | def device_types(self):
method get_atol (line 82) | def get_atol(self):
function find_all_sonnet_modules (line 100) | def find_all_sonnet_modules(
function find_sonnet_python_modules (line 114) | def find_sonnet_python_modules(
function combined_named_parameters (line 135) | def combined_named_parameters(*parameters):
function named_bools (line 162) | def named_bools(name) -> Sequence[Tuple[str, bool]]:
FILE: sonnet/src/utils.py
function replicate (line 32) | def replicate(
function _is_object (line 49) | def _is_object(f: Any) -> bool:
function decorator (line 54) | def decorator(
function get_channel_index (line 103) | def get_channel_index(data_format: str) -> int:
function assert_rank (line 133) | def assert_rank(inputs, rank: int):
function assert_minimum_rank (line 141) | def assert_minimum_rank(inputs, rank: int):
function _synchronized (line 149) | def _synchronized(f: Callable[..., T]) -> Callable[..., T]:
function smart_autograph (line 159) | def smart_autograph(f: Callable[..., T]) -> Callable[..., T]:
function variable_like (line 204) | def variable_like(inputs: Union[tf.Tensor, tf.Variable],
function _render_spec (line 218) | def _render_spec(shape: tf.TensorShape, dtype: tf.DType) -> str:
function _simple_device (line 248) | def _simple_device(var: tf.Variable) -> str:
function _name_scope_then_rank (line 258) | def _name_scope_then_rank(var: tf.Variable):
function format_variables (line 264) | def format_variables(variables: Sequence[tf.Variable],
function log_variables (line 280) | def log_variables(variables: Sequence[tf.Variable]):
class CompareById (line 295) | class CompareById(Generic[T]):
method __init__ (line 298) | def __init__(self, wrapped: T):
method __hash__ (line 301) | def __hash__(self):
method __eq__ (line 309) | def __eq__(self, other):
method __lt__ (line 314) | def __lt__(self, other):
FILE: sonnet/src/utils_test.py
class ReplicateTest (line 34) | class ReplicateTest(test_utils.TestCase, parameterized.TestCase):
method testSingleValue (line 37) | def testSingleValue(self, value):
method testListLengthOne (line 44) | def testListLengthOne(self, value):
method testTupleLengthN (line 51) | def testTupleLengthN(self, value):
method testListLengthN (line 59) | def testListLengthN(self, value):
method testIncorrectLength (line 65) | def testIncorrectLength(self):
class DecoratorTest (line 73) | class DecoratorTest(test_utils.TestCase):
method test_callable_object (line 75) | def test_callable_object(self):
method test_function (line 91) | def test_function(self):
method test_unbound_method (line 101) | def test_unbound_method(self):
method test_bound_method (line 117) | def test_bound_method(self):
class ChannelIndexTest (line 133) | class ChannelIndexTest(test_utils.TestCase, parameterized.TestCase):
method test_returns_index_channels_first (line 136) | def test_returns_index_channels_first(self, data_format):
method test_returns_index_channels_last (line 140) | def test_returns_index_channels_last(self, data_format):
method test_invalid_strings (line 144) | def test_invalid_strings(self, data_format):
class AssertRankTest (line 151) | class AssertRankTest(test_utils.TestCase, parameterized.TestCase):
method test_valid_rank (line 158) | def test_valid_rank(self, input_fn):
method test_invalid_rank (line 165) | def test_invalid_rank(self, rank):
class SmartAutographTest (line 180) | class SmartAutographTest(test_utils.TestCase):
method test_smart_ag (line 182) | def test_smart_ag(self):
class VariableLikeTest (line 204) | class VariableLikeTest(test_utils.TestCase, parameterized.TestCase):
method test_copies_shape (line 208) | def test_copies_shape(self, a):
method test_copies_dtype (line 217) | def test_copies_dtype(self, a):
method test_copies_device (line 223) | def test_copies_device(self, a):
method test_default_initializer_is_zero (line 229) | def test_default_initializer_is_zero(self):
method test_override_initializer (line 234) | def test_override_initializer(self):
method test_copies_variable_trainable (line 240) | def test_copies_variable_trainable(self, trainable):
method test_default_trainable_for_tensor (line 245) | def test_default_trainable_for_tensor(self):
method test_override_trainable (line 251) | def test_override_trainable(self, trainable):
method test_copies_variable_name (line 256) | def test_copies_variable_name(self):
method test_default_name_for_tensor (line 261) | def test_default_name_for_tensor(self):
method test_override_name (line 267) | def test_override_name(self, a):
class FormatVariablesTest (line 273) | class FormatVariablesTest(test_utils.TestCase):
method test_format_variables (line 275) | def test_format_variables(self):
method test_log_variables (line 285) | def test_log_variables(self):
class NotHashable (line 295) | class NotHashable:
method __hash__ (line 297) | def __hash__(self):
class CompareByIdTest (line 301) | class CompareByIdTest(test_utils.TestCase):
method test_access (line 303) | def test_access(self):
method test_hash (line 308) | def test_hash(self):
method test_eq (line 313) | def test_eq(self):
Condensed preview — 299 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (1,816K chars).
[
{
"path": ".github/workflows/ci.yml",
"chars": 903,
"preview": "name: pytest\n\non:\n pull_request:\n branches:\n - v2\n push:\n branches:\n - v2\n\njobs:\n test-ubuntu:\n na"
},
{
"path": "CONTRIBUTING.md",
"chars": 1762,
"preview": "# Contributing guidelines\n\n## How to become a contributor and submit your own code\n\n### Contributor License Agreements\n\n"
},
{
"path": "LICENSE",
"chars": 11358,
"preview": "\n Apache License\n Version 2.0, January 2004\n "
},
{
"path": "MANIFEST.in",
"chars": 34,
"preview": "include README.md\ninclude LICENSE\n"
},
{
"path": "README.md",
"chars": 11646,
"preview": "\n\n# Sonnet\n\n[**Documentation**](https://sonnet.readthedocs.io/) | [*"
},
{
"path": "WORKSPACE",
"chars": 27,
"preview": "workspace(name = \"sonnet\")\n"
},
{
"path": "docs/.gitignore",
"chars": 7,
"preview": "_build\n"
},
{
"path": "docs/Makefile",
"chars": 581,
"preview": "# Minimal makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHI"
},
{
"path": "docs/api.rst",
"chars": 5856,
"preview": ".. TODO(slebedev): add a title, e.g. \"API docs\"?\n\nBase\n----\n\n.. currentmodule:: sonnet\n\nModule\n~~~~~~\n\n.. autoclass:: Mo"
},
{
"path": "docs/conf.py",
"chars": 5308,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "docs/ext/BUILD",
"chars": 435,
"preview": "load(\"//sonnet/src:build_defs.bzl\", \"snt_py_library\", \"snt_py_test\")\n\nlicenses([\"notice\"])\n\nsnt_py_library(\n name = \""
},
{
"path": "docs/ext/link_tf_api.py",
"chars": 3895,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "docs/ext/link_tf_api_test.py",
"chars": 1792,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "docs/index.rst",
"chars": 1075,
"preview": ":github_url: https://github.com/deepmind/sonnet/tree/v2/docs\n\nSonnet Documentation\n====================\n\nSonnet is a lib"
},
{
"path": "docs/modules.rst",
"chars": 3751,
"preview": ".. currentmodule:: sonnet\n\nModules\n=======\n\n:class:`Module` is the core abstraction provided by Sonnet.\n\nBy organising y"
},
{
"path": "docs/references.bib",
"chars": 3935,
"preview": "@article{chung2014empirical,\n title={Empirical evaluation of gated recurrent neural networks on sequence modeling},\n a"
},
{
"path": "docs/requirements.txt",
"chars": 129,
"preview": "sphinx>=2.0.1\nsphinx_rtd_theme>=0.4.3\nsphinxcontrib-katex>=0.4.1\nsphinxcontrib-bibtex>=0.4.2,<2\nsphinx-autodoc-typehints"
},
{
"path": "examples/BUILD",
"chars": 1216,
"preview": "# buildifier: disable=out-of-order-load - Breaks copybara otherwise\nload(\"//third_party/bazel_rules/rules_python/python:"
},
{
"path": "examples/README.md",
"chars": 373,
"preview": "Examples\n========\n\nGoogle Colab notebooks:\n\n- [Predicting MNIST with an MLP](https://colab.research.google.com/github/de"
},
{
"path": "examples/distributed_cifar10.ipynb",
"chars": 14321,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"8Kc"
},
{
"path": "examples/functional_mlp_mnist.py",
"chars": 3958,
"preview": "# Copyright 2020 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "examples/little_gan_on_mnist.ipynb",
"chars": 135538,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"Yu7"
},
{
"path": "examples/mlp_on_mnist.ipynb",
"chars": 75097,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"Yu7"
},
{
"path": "examples/simple_mnist.py",
"chars": 3859,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "examples/simple_mnist_test.py",
"chars": 1887,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "examples/vqvae_example.ipynb",
"chars": 624679,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"cqCt_GhvCnwY\"\n },\n"
},
{
"path": "readthedocs.yml",
"chars": 357,
"preview": "# Read the Docs configuration file\n# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details\n\nversion:"
},
{
"path": "requirements-test.txt",
"chars": 46,
"preview": "mock>=3.0.5\ntensorflow-datasets>1,<4\ndocutils\n"
},
{
"path": "requirements-tf.txt",
"chars": 53,
"preview": "tensorflow==2.12.0rc0\ntensorflow-probability==0.12.2\n"
},
{
"path": "requirements.txt",
"chars": 74,
"preview": "absl-py>=0.7.1\nnumpy>=1.16.3\ndm-tree>=0.1.1\nwrapt>=1.11.1\ntabulate>=0.7.5\n"
},
{
"path": "setup.py",
"chars": 2080,
"preview": "\"\"\"Setup for pip package.\"\"\"\n\nfrom setuptools import find_namespace_packages\nfrom setuptools import setup\n\n\ndef _get_son"
},
{
"path": "sonnet/BUILD",
"chars": 2487,
"preview": "load(\"//sonnet/src:build_defs.bzl\", \"snt_py_library\")\n\npackage(default_visibility = [\"//visibility:private\"])\n\nlicenses("
},
{
"path": "sonnet/__init__.py",
"chars": 5170,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/distribute.py",
"chars": 1135,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/functional.py",
"chars": 1861,
"preview": "# Copyright 2020 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/initializers.py",
"chars": 1371,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/mixed_precision.py",
"chars": 990,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/nets/BUILD",
"chars": 399,
"preview": "load(\"//sonnet/src:build_defs.bzl\", \"snt_py_library\")\n\npackage(default_visibility = [\"//sonnet:__pkg__\"])\n\nlicenses([\"no"
},
{
"path": "sonnet/nets/__init__.py",
"chars": 1206,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/nets/resnet.py",
"chars": 944,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/optimizers.py",
"chars": 1047,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/pad.py",
"chars": 1009,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/regularizers.py",
"chars": 971,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/BUILD",
"chars": 12265,
"preview": "load(\"//sonnet/src:build_defs.bzl\", \"snt_py_library\", \"snt_py_test\")\n\npackage(default_visibility = [\"//sonnet:__subpacka"
},
{
"path": "sonnet/src/__init__.py",
"chars": 683,
"preview": "# Copyright 2021 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/axis_norm.py",
"chars": 10074,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/axis_norm_test.py",
"chars": 11968,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/base.py",
"chars": 17906,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/base_test.py",
"chars": 19554,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/batch_apply.py",
"chars": 6361,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/batch_apply_test.py",
"chars": 5770,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/batch_norm.py",
"chars": 12674,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/batch_norm_test.py",
"chars": 9414,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/bias.py",
"chars": 5997,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/bias_test.py",
"chars": 2712,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/build.py",
"chars": 2561,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/build_defs.bzl",
"chars": 1221,
"preview": "\"\"\"Sonnet specific build rules.\"\"\"\n\ndef snt_py_library(name, **kwargs):\n \"\"\"Proxy for py_library.\n\n Internally we "
},
{
"path": "sonnet/src/build_test.py",
"chars": 1792,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conformance/BUILD",
"chars": 5004,
"preview": "load(\"//sonnet/src:build_defs.bzl\", \"snt_py_library\", \"snt_py_test\")\n\npackage(\n default_testonly = True,\n default_"
},
{
"path": "sonnet/src/conformance/__init__.py",
"chars": 683,
"preview": "# Copyright 2021 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conformance/api_test.py",
"chars": 1148,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conformance/build_test.py",
"chars": 1857,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conformance/checkpoint_test.py",
"chars": 15121,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conformance/checkpoints/BUILD",
"chars": 462,
"preview": "load(\"//third_party/bazel_rules/rules_python/python:py_binary.bzl\", \"py_binary\")\n\npackage(default_testonly = True)\n\nlice"
},
{
"path": "sonnet/src/conformance/checkpoints/README.md",
"chars": 1556,
"preview": "# Golden checkpoints\n\nGolden checkpoints represent checkpoints generated from stable Sonnet code. We\nhave unit tests tha"
},
{
"path": "sonnet/src/conformance/checkpoints/base_batch_norm_1x2x2x3/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/base_batch_norm_scale_offset_1x2x2x3/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/batch_norm_1x2x2x3/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/batch_norm_scale_offset_1x2x2x3/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/batch_norm_training_1x2x2x3/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/bias_3x3x3/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/cifar10_convnet_2x3_2x2_1x3x3x2/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/conv1d_3x3_2x2/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/conv1d_lstm_3x3_2x2/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/conv1d_transpose_3x3_2x2/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/conv2d_3x3_2x2/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/conv2d_lstm_3x3_2x2/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/conv2d_transpose_3x3_2x2/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/conv3d_3x3_2x2/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/conv3d_lstm_3x3_2x2/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/conv3d_transpose_3x3_2x2/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/cross_replica_batch_norm_1x2x2x3/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/depthwise_conv2d_3x3_2x2/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/dropout/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/ema_2/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/embed_100_100/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/generate.py",
"chars": 2946,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conformance/checkpoints/group_norm_2_1x3x4/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/gru_1/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/instance_norm_1_1x3_2/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/layer_norm_1_1x3_2/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/linear_1x1/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/linear_nobias_1x1/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/lstm_1/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/lstm_8_projected_1/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/mean_2x2/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/mlp_3x4x5_1x3/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/mlp_nobias_3x4x5_1x3/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/resnet50/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/sum_2x2/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/trainable_state/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/unrolled_lstm_1/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/vanilla_rnn_8/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/vqvae/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/vqvae_ema_eval/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/checkpoints/vqvae_ema_train/checkpoint",
"chars": 81,
"preview": "model_checkpoint_path: \"checkpoint-1\"\nall_model_checkpoint_paths: \"checkpoint-1\"\n"
},
{
"path": "sonnet/src/conformance/copy_test.py",
"chars": 1622,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conformance/descriptors.py",
"chars": 8620,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conformance/descriptors_test.py",
"chars": 1471,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conformance/distribute_test.py",
"chars": 3519,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conformance/doctest_test.py",
"chars": 2651,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conformance/function_test.py",
"chars": 4934,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conformance/goldens.py",
"chars": 16495,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conformance/goldens_test.py",
"chars": 2419,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conformance/keras_test.py",
"chars": 8393,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conformance/optimizer_test.py",
"chars": 2252,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conformance/pickle_test.py",
"chars": 1702,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conformance/saved_model_test.py",
"chars": 2986,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conformance/tensorflow1_test.py",
"chars": 1167,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conformance/xla_test.py",
"chars": 2337,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conv.py",
"chars": 13827,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conv_test.py",
"chars": 11770,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conv_transpose.py",
"chars": 16137,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/conv_transpose_test.py",
"chars": 12358,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/custom_getter.py",
"chars": 4876,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/custom_getter_test.py",
"chars": 1688,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/deferred.py",
"chars": 4692,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/deferred_test.py",
"chars": 3570,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/depthwise_conv.py",
"chars": 5634,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/depthwise_conv_test.py",
"chars": 8357,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/distribute/BUILD",
"chars": 1554,
"preview": "load(\"//sonnet/src:build_defs.bzl\", \"snt_py_library\", \"snt_py_test\")\n\npackage(default_visibility = [\"//sonnet:__subpacka"
},
{
"path": "sonnet/src/distribute/__init__.py",
"chars": 683,
"preview": "# Copyright 2021 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/distribute/distributed_batch_norm.py",
"chars": 5303,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/distribute/distributed_batch_norm_test.py",
"chars": 5505,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/distribute/replicator.py",
"chars": 8058,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/distribute/replicator_test.py",
"chars": 8100,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/distribute/replicator_test_utils.py",
"chars": 2039,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/dropout.py",
"chars": 2607,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/dropout_test.py",
"chars": 3097,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/embed.py",
"chars": 5598,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/embed_test.py",
"chars": 3232,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/functional/BUILD",
"chars": 1573,
"preview": "load(\"//sonnet/src:build_defs.bzl\", \"snt_py_library\", \"snt_py_test\")\n\npackage(default_visibility = [\"//sonnet:__subpacka"
},
{
"path": "sonnet/src/functional/__init__.py",
"chars": 683,
"preview": "# Copyright 2021 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/functional/haiku.py",
"chars": 14600,
"preview": "# Copyright 2020 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/functional/haiku_test.py",
"chars": 7294,
"preview": "# Copyright 2020 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/functional/jax.py",
"chars": 2253,
"preview": "# Copyright 2020 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/functional/jax_test.py",
"chars": 2585,
"preview": "# Copyright 2020 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/functional/optimizers.py",
"chars": 5364,
"preview": "# Copyright 2020 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/functional/optimizers_test.py",
"chars": 2588,
"preview": "# Copyright 2020 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/functional/utils.py",
"chars": 2158,
"preview": "# Copyright 2020 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/group_norm.py",
"chars": 9115,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/group_norm_test.py",
"chars": 11108,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/initializers.py",
"chars": 14358,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/initializers_test.py",
"chars": 19087,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/leaky_clip_by_value.py",
"chars": 2331,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/leaky_clip_by_value_test.py",
"chars": 2314,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/linear.py",
"chars": 3157,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/linear_test.py",
"chars": 6043,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/metrics.py",
"chars": 3468,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/metrics_test.py",
"chars": 2092,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/mixed_precision.py",
"chars": 5398,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/mixed_precision_test.py",
"chars": 13935,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/moving_averages.py",
"chars": 3312,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/moving_averages_test.py",
"chars": 3591,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/nets/BUILD",
"chars": 2241,
"preview": "load(\"//sonnet/src:build_defs.bzl\", \"snt_py_library\", \"snt_py_test\")\n\npackage(default_visibility = [\"//sonnet:__subpacka"
},
{
"path": "sonnet/src/nets/__init__.py",
"chars": 683,
"preview": "# Copyright 2021 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/nets/cifar10_convnet.py",
"chars": 4981,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/nets/cifar10_convnet_test.py",
"chars": 4147,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/nets/dnc/BUILD",
"chars": 1753,
"preview": "# Description:\n# Differentiable Neural Computer\n\nload(\"//sonnet/src:build_defs.bzl\", \"snt_py_library\", \"snt_py_test\")\n"
},
{
"path": "sonnet/src/nets/dnc/__init__.py",
"chars": 683,
"preview": "# Copyright 2021 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/nets/dnc/control.py",
"chars": 3688,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/nets/dnc/control_test.py",
"chars": 3898,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/nets/dnc/read.py",
"chars": 1508,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/nets/dnc/read_test.py",
"chars": 1882,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/nets/dnc/util.py",
"chars": 4681,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/nets/dnc/util_test.py",
"chars": 7386,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/nets/dnc/write.py",
"chars": 4078,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/nets/dnc/write_test.py",
"chars": 7853,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/nets/mlp.py",
"chars": 5572,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/nets/mlp_test.py",
"chars": 6103,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/nets/resnet.py",
"chars": 11081,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/nets/resnet_test.py",
"chars": 2516,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/nets/vqvae.py",
"chars": 12852,
"preview": "# Copyright 2018 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/nets/vqvae_test.py",
"chars": 6336,
"preview": "# Copyright 2018 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/once.py",
"chars": 2764,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/once_test.py",
"chars": 3218,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/optimizers/BUILD",
"chars": 2532,
"preview": "load(\"//sonnet/src:build_defs.bzl\", \"snt_py_library\", \"snt_py_test\")\n\npackage(default_visibility = [\"//sonnet:__subpacka"
},
{
"path": "sonnet/src/optimizers/__init__.py",
"chars": 683,
"preview": "# Copyright 2021 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/optimizers/adam.py",
"chars": 6137,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/optimizers/adam_test.py",
"chars": 7057,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/optimizers/momentum.py",
"chars": 4691,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/optimizers/momentum_test.py",
"chars": 8919,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/optimizers/optimizer_tests.py",
"chars": 6422,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/optimizers/optimizer_utils.py",
"chars": 3156,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/optimizers/rmsprop.py",
"chars": 6379,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/optimizers/rmsprop_test.py",
"chars": 9155,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/optimizers/sgd.py",
"chars": 2386,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/optimizers/sgd_test.py",
"chars": 3689,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/pad.py",
"chars": 3402,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "sonnet/src/pad_test.py",
"chars": 3681,
"preview": "# Copyright 2019 The Sonnet Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
}
]
// ... and 99 more files (download for full content)
About this extraction
This page contains the full source code of the google-deepmind/sonnet GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 299 files (32.5 MB), approximately 740.2k tokens, and a symbol index with 1523 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.