Repository: google-deepmind/sonnet Branch: v2 Commit: e501ca17bf9a Files: 299 Total size: 32.5 MB Directory structure: gitextract_ho0gv3se/ ├── .github/ │ └── workflows/ │ └── ci.yml ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── WORKSPACE ├── docs/ │ ├── .gitignore │ ├── Makefile │ ├── api.rst │ ├── conf.py │ ├── ext/ │ │ ├── BUILD │ │ ├── link_tf_api.py │ │ └── link_tf_api_test.py │ ├── index.rst │ ├── modules.rst │ ├── references.bib │ └── requirements.txt ├── examples/ │ ├── BUILD │ ├── README.md │ ├── distributed_cifar10.ipynb │ ├── functional_mlp_mnist.py │ ├── little_gan_on_mnist.ipynb │ ├── mlp_on_mnist.ipynb │ ├── simple_mnist.py │ ├── simple_mnist_test.py │ └── vqvae_example.ipynb ├── readthedocs.yml ├── requirements-test.txt ├── requirements-tf.txt ├── requirements.txt ├── setup.py ├── sonnet/ │ ├── BUILD │ ├── __init__.py │ ├── distribute.py │ ├── functional.py │ ├── initializers.py │ ├── mixed_precision.py │ ├── nets/ │ │ ├── BUILD │ │ ├── __init__.py │ │ └── resnet.py │ ├── optimizers.py │ ├── pad.py │ ├── regularizers.py │ └── src/ │ ├── BUILD │ ├── __init__.py │ ├── axis_norm.py │ ├── axis_norm_test.py │ ├── base.py │ ├── base_test.py │ ├── batch_apply.py │ ├── batch_apply_test.py │ ├── batch_norm.py │ ├── batch_norm_test.py │ ├── bias.py │ ├── bias_test.py │ ├── build.py │ ├── build_defs.bzl │ ├── build_test.py │ ├── conformance/ │ │ ├── BUILD │ │ ├── __init__.py │ │ ├── api_test.py │ │ ├── build_test.py │ │ ├── checkpoint_test.py │ │ ├── checkpoints/ │ │ │ ├── BUILD │ │ │ ├── README.md │ │ │ ├── base_batch_norm_1x2x2x3/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── base_batch_norm_scale_offset_1x2x2x3/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── batch_norm_1x2x2x3/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── batch_norm_scale_offset_1x2x2x3/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── batch_norm_training_1x2x2x3/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── bias_3x3x3/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── cifar10_convnet_2x3_2x2_1x3x3x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── conv1d_3x3_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── conv1d_lstm_3x3_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── conv1d_transpose_3x3_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── conv2d_3x3_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── conv2d_lstm_3x3_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── conv2d_transpose_3x3_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── conv3d_3x3_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── conv3d_lstm_3x3_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── conv3d_transpose_3x3_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── cross_replica_batch_norm_1x2x2x3/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── depthwise_conv2d_3x3_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── dropout/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── ema_2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── embed_100_100/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── generate.py │ │ │ ├── group_norm_2_1x3x4/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── gru_1/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── instance_norm_1_1x3_2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── layer_norm_1_1x3_2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── linear_1x1/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── linear_nobias_1x1/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── lstm_1/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── lstm_8_projected_1/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── mean_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00002 │ │ │ │ ├── checkpoint-1.data-00001-of-00002 │ │ │ │ └── checkpoint-1.index │ │ │ ├── mlp_3x4x5_1x3/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── mlp_nobias_3x4x5_1x3/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── resnet50/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── sum_2x2/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00002 │ │ │ │ ├── checkpoint-1.data-00001-of-00002 │ │ │ │ └── checkpoint-1.index │ │ │ ├── trainable_state/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── unrolled_lstm_1/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── vanilla_rnn_8/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── vqvae/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ ├── vqvae_ema_eval/ │ │ │ │ ├── checkpoint │ │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ │ └── checkpoint-1.index │ │ │ └── vqvae_ema_train/ │ │ │ ├── checkpoint │ │ │ ├── checkpoint-1.data-00000-of-00001 │ │ │ └── checkpoint-1.index │ │ ├── copy_test.py │ │ ├── descriptors.py │ │ ├── descriptors_test.py │ │ ├── distribute_test.py │ │ ├── doctest_test.py │ │ ├── function_test.py │ │ ├── goldens.py │ │ ├── goldens_test.py │ │ ├── keras_test.py │ │ ├── optimizer_test.py │ │ ├── pickle_test.py │ │ ├── saved_model_test.py │ │ ├── tensorflow1_test.py │ │ └── xla_test.py │ ├── conv.py │ ├── conv_test.py │ ├── conv_transpose.py │ ├── conv_transpose_test.py │ ├── custom_getter.py │ ├── custom_getter_test.py │ ├── deferred.py │ ├── deferred_test.py │ ├── depthwise_conv.py │ ├── depthwise_conv_test.py │ ├── distribute/ │ │ ├── BUILD │ │ ├── __init__.py │ │ ├── distributed_batch_norm.py │ │ ├── distributed_batch_norm_test.py │ │ ├── replicator.py │ │ ├── replicator_test.py │ │ └── replicator_test_utils.py │ ├── dropout.py │ ├── dropout_test.py │ ├── embed.py │ ├── embed_test.py │ ├── functional/ │ │ ├── BUILD │ │ ├── __init__.py │ │ ├── haiku.py │ │ ├── haiku_test.py │ │ ├── jax.py │ │ ├── jax_test.py │ │ ├── optimizers.py │ │ ├── optimizers_test.py │ │ └── utils.py │ ├── group_norm.py │ ├── group_norm_test.py │ ├── initializers.py │ ├── initializers_test.py │ ├── leaky_clip_by_value.py │ ├── leaky_clip_by_value_test.py │ ├── linear.py │ ├── linear_test.py │ ├── metrics.py │ ├── metrics_test.py │ ├── mixed_precision.py │ ├── mixed_precision_test.py │ ├── moving_averages.py │ ├── moving_averages_test.py │ ├── nets/ │ │ ├── BUILD │ │ ├── __init__.py │ │ ├── cifar10_convnet.py │ │ ├── cifar10_convnet_test.py │ │ ├── dnc/ │ │ │ ├── BUILD │ │ │ ├── __init__.py │ │ │ ├── control.py │ │ │ ├── control_test.py │ │ │ ├── read.py │ │ │ ├── read_test.py │ │ │ ├── util.py │ │ │ ├── util_test.py │ │ │ ├── write.py │ │ │ └── write_test.py │ │ ├── mlp.py │ │ ├── mlp_test.py │ │ ├── resnet.py │ │ ├── resnet_test.py │ │ ├── vqvae.py │ │ └── vqvae_test.py │ ├── once.py │ ├── once_test.py │ ├── optimizers/ │ │ ├── BUILD │ │ ├── __init__.py │ │ ├── adam.py │ │ ├── adam_test.py │ │ ├── momentum.py │ │ ├── momentum_test.py │ │ ├── optimizer_tests.py │ │ ├── optimizer_utils.py │ │ ├── rmsprop.py │ │ ├── rmsprop_test.py │ │ ├── sgd.py │ │ └── sgd_test.py │ ├── pad.py │ ├── pad_test.py │ ├── parallel_linear.py │ ├── parallel_linear_test.py │ ├── recurrent.py │ ├── recurrent_test.py │ ├── regularizers.py │ ├── regularizers_test.py │ ├── reshape.py │ ├── reshape_test.py │ ├── scale_gradient.py │ ├── scale_gradient_test.py │ ├── sequential.py │ ├── sequential_test.py │ ├── test_utils.py │ ├── types.py │ ├── utils.py │ └── utils_test.py └── test.sh ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/workflows/ci.yml ================================================ name: pytest on: pull_request: branches: - v2 push: branches: - v2 jobs: test-ubuntu: name: "pytest on ${{ matrix.python-version }} on ${{ matrix.os }}" runs-on: "${{ matrix.os }}" strategy: matrix: python-version: [3.9, '3.10', '3.11'] os: [ubuntu-latest] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt pip install -r requirements-tf.txt pip install -r requirements-test.txt pip install . - name: Test with pytest run: | pip install pytest pytest-xdist pytest -n auto sonnet --ignore=sonnet/src/conformance/ ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing guidelines ## How to become a contributor and submit your own code ### Contributor License Agreements We'd love to accept your patches! Before we can take them, we have to jump a couple of legal hurdles. Please fill out either the individual or corporate Contributor License Agreement (CLA). * If you are an individual writing original source code and you're sure you own the intellectual property, then you'll need to sign an [individual CLA](http://code.google.com/legal/individual-cla-v1.0.html). * If you work for a company that wants to allow you to contribute your work, then you'll need to sign a [corporate CLA](http://code.google.com/legal/corporate-cla-v1.0.html). Follow either of the two links above to access the appropriate CLA and instructions for how to sign and return it. Once we receive it, we'll be able to accept your pull requests. ***NOTE***: Only original source code from you and other people that have signed the CLA can be accepted into the main repository. ### Contributing code If you have improvements to Sonnet, send us your pull requests! For those just getting started, Github has a [howto](https://help.github.com/articles/using-pull-requests/). If you want to contribute but you're not sure where to start, take a look at the [issues with the "contributions welcome" label](https://github.com/deepmind/sonnet/labels/stat%3Acontributions%20welcome). These are issues that we believe are particularly well suited for outside contributions, often because we probably won't get to them right now. If you decide to start on an issue, leave a comment so that other people know that you're working on it. If you want to help out, but not alone, use the issue comment thread to coordinate. ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: MANIFEST.in ================================================ include README.md include LICENSE ================================================ FILE: README.md ================================================ ![Sonnet](https://sonnet.dev/images/sonnet_logo.png) # Sonnet [**Documentation**](https://sonnet.readthedocs.io/) | [**Examples**](#examples) Sonnet is a library built on top of [TensorFlow 2](https://www.tensorflow.org/) designed to provide simple, composable abstractions for machine learning research. # Introduction Sonnet has been designed and built by researchers at DeepMind. It can be used to construct neural networks for many different purposes (un/supervised learning, reinforcement learning, ...). We find it is a successful abstraction for our organization, you might too! More specifically, Sonnet provides a simple but powerful programming model centered around a single concept: `snt.Module`. Modules can hold references to parameters, other modules and methods that apply some function on the user input. Sonnet ships with many predefined modules (e.g. `snt.Linear`, `snt.Conv2D`, `snt.BatchNorm`) and some predefined networks of modules (e.g. `snt.nets.MLP`) but users are also encouraged to build their own modules. Unlike many frameworks Sonnet is extremely unopinionated about **how** you will use your modules. Modules are designed to be self contained and entirely decoupled from one another. Sonnet does not ship with a training framework and users are encouraged to build their own or adopt those built by others. Sonnet is also designed to be simple to understand, our code is (hopefully!) clear and focussed. Where we have picked defaults (e.g. defaults for initial parameter values) we try to point out why. # Getting Started ## Examples The easiest way to try Sonnet is to use Google Colab which offers a free Python notebook attached to a GPU or TPU. - [Predicting MNIST with an MLP](https://colab.research.google.com/github/deepmind/sonnet/blob/v2/examples/mlp_on_mnist.ipynb) - [Training a Little GAN on MNIST](https://colab.research.google.com/github/deepmind/sonnet/blob/v2/examples/little_gan_on_mnist.ipynb) - [Distributed training with `snt.distribute`](https://colab.research.google.com/github/deepmind/sonnet/blob/v2/examples/distributed_cifar10.ipynb) ## Installation To get started install TensorFlow 2.0 and Sonnet 2: ```shell $ pip install tensorflow tensorflow-probability $ pip install dm-sonnet ``` You can run the following to verify things installed correctly: ```python import tensorflow as tf import sonnet as snt print("TensorFlow version {}".format(tf.__version__)) print("Sonnet version {}".format(snt.__version__)) ``` ### Using existing modules Sonnet ships with a number of built in modules that you can trivially use. For example to define an MLP we can use the `snt.Sequential` module to call a sequence of modules, passing the output of a given module as the input for the next module. We can use `snt.Linear` and `tf.nn.relu` to actually define our computation: ```python mlp = snt.Sequential([ snt.Linear(1024), tf.nn.relu, snt.Linear(10), ]) ``` To use our module we need to "call" it. The `Sequential` module (and most modules) define a `__call__` method that means you can call them by name: ```python logits = mlp(tf.random.normal([batch_size, input_size])) ``` It is also very common to request all the parameters for your module. Most modules in Sonnet create their parameters the first time they are called with some input (since in most cases the shape of the parameters is a function of the input). Sonnet modules provide two properties for accessing parameters. The `variables` property returns **all** `tf.Variable`s that are referenced by the given module: ```python all_variables = mlp.variables ``` It is worth noting that `tf.Variable`s are not just used for parameters of your model. For example they are used to hold state in metrics used in `snt.BatchNorm`. In most cases users retrieve the module variables to pass them to an optimizer to be updated. In this case non-trainable variables should typically not be in that list as they are updated via a different mechanism. TensorFlow has a built in mechanism to mark variables as "trainable" (parameters of your model) vs. non-trainable (other variables). Sonnet provides a mechanism to gather all trainable variables from your module which is probably what you want to pass to an optimizer: ```python model_parameters = mlp.trainable_variables ``` ### Building your own module Sonnet strongly encourages users to subclass `snt.Module` to define their own modules. Let's start by creating a simple `Linear` layer called `MyLinear`: ```python class MyLinear(snt.Module): def __init__(self, output_size, name=None): super(MyLinear, self).__init__(name=name) self.output_size = output_size @snt.once def _initialize(self, x): initial_w = tf.random.normal([x.shape[1], self.output_size]) self.w = tf.Variable(initial_w, name="w") self.b = tf.Variable(tf.zeros([self.output_size]), name="b") def __call__(self, x): self._initialize(x) return tf.matmul(x, self.w) + self.b ``` Using this module is trivial: ```python mod = MyLinear(32) mod(tf.ones([batch_size, input_size])) ``` By subclassing `snt.Module` you get many nice properties for free. For example a default implementation of `__repr__` which shows constructor arguments (very useful for debugging and introspection): ```python >>> print(repr(mod)) MyLinear(output_size=10) ``` You also get the `variables` and `trainable_variables` properties: ```python >>> mod.variables (, ) ``` You may notice the `my_linear` prefix on the variables above. This is because Sonnet modules also enter the modules name scope whenever methods are called. By entering the module name scope we provide a much more useful graph for tools like TensorBoard to consume (e.g. all operations that occur inside my_linear will be in a group called my_linear). Additionally your module will now support TensorFlow checkpointing and saved model which are advanced features covered later. # Serialization Sonnet supports multiple serialization formats. The simplest format we support is Python's `pickle`, and all built in modules are tested to make sure they can be saved/loaded via pickle in the same Python process. In general we discourage the use of pickle, it is not well supported by many parts of TensorFlow and in our experience can be quite brittle. ## TensorFlow Checkpointing **Reference:** https://www.tensorflow.org/alpha/guide/checkpoints TensorFlow checkpointing can be used to save the value of parameters periodically during training. This can be useful to save the progress of training in case your program crashes or is stopped. Sonnet is designed to work cleanly with TensorFlow checkpointing: ```python checkpoint_root = "/tmp/checkpoints" checkpoint_name = "example" save_prefix = os.path.join(checkpoint_root, checkpoint_name) my_module = create_my_sonnet_module() # Can be anything extending snt.Module. # A `Checkpoint` object manages checkpointing of the TensorFlow state associated # with the objects passed to it's constructor. Note that Checkpoint supports # restore on create, meaning that the variables of `my_module` do **not** need # to be created before you restore from a checkpoint (their value will be # restored when they are created). checkpoint = tf.train.Checkpoint(module=my_module) # Most training scripts will want to restore from a checkpoint if one exists. This # would be the case if you interrupted your training (e.g. to use your GPU for # something else, or in a cloud environment if your instance is preempted). latest = tf.train.latest_checkpoint(checkpoint_root) if latest is not None: checkpoint.restore(latest) for step_num in range(num_steps): train(my_module) # During training we will occasionally save the values of weights. Note that # this is a blocking call and can be slow (typically we are writing to the # slowest storage on the machine). If you have a more reliable setup it might be # appropriate to save less frequently. if step_num and not step_num % 1000: checkpoint.save(save_prefix) # Make sure to save your final values!! checkpoint.save(save_prefix) ``` ## TensorFlow Saved Model **Reference:** https://www.tensorflow.org/alpha/guide/saved_model TensorFlow saved models can be used to save a copy of your network that is decoupled from the Python source for it. This is enabled by saving a TensorFlow graph describing the computation and a checkpoint containing the value of weights. The first thing to do in order to create a saved model is to create a `snt.Module` that you want to save: ```python my_module = snt.nets.MLP([1024, 1024, 10]) my_module(tf.ones([1, input_size])) ``` Next, we need to create another module describing the specific parts of our model that we want to export. We advise doing this (rather than modifying the original model in-place) so you have fine grained control over what is actually exported. This is typically important to avoid creating very large saved models, and such that you only share the parts of your model you want to (e.g. you only want to share the generator for a GAN but keep the discriminator private). ```python @tf.function(input_signature=[tf.TensorSpec([None, input_size])]) def inference(x): return my_module(x) to_save = snt.Module() to_save.inference = inference to_save.all_variables = list(my_module.variables) tf.saved_model.save(to_save, "/tmp/example_saved_model") ``` We now have a saved model in the `/tmp/example_saved_model` folder: ```shell $ ls -lh /tmp/example_saved_model total 24K drwxrwsr-t 2 tomhennigan 154432098 4.0K Apr 28 00:14 assets -rw-rw-r-- 1 tomhennigan 154432098 14K Apr 28 00:15 saved_model.pb drwxrwsr-t 2 tomhennigan 154432098 4.0K Apr 28 00:15 variables ``` Loading this model is simple and can be done on a different machine without any of the Python code that built the saved model: ```python loaded = tf.saved_model.load("/tmp/example_saved_model") # Use the inference method. Note this doesn't run the Python code from `to_save` # but instead uses the TensorFlow Graph that is part of the saved model. loaded.inference(tf.ones([1, input_size])) # The all_variables property can be used to retrieve the restored variables. assert len(loaded.all_variables) > 0 ``` Note that the loaded object is not a Sonnet module, it is a container object that has the specific methods (e.g. `inference`) and properties (e.g. `all_variables`) that we added in the previous block. ## Distributed training **Example:** https://github.com/deepmind/sonnet/blob/v2/examples/distributed_cifar10.ipynb Sonnet has support for distributed training using [custom TensorFlow distribution strategies](https://sonnet.readthedocs.io/en/latest/api.html#module-sonnet.distribute). A key difference between Sonnet and distributed training using `tf.keras` is that Sonnet modules and optimizers do not behave differently when run under distribution strategies (e.g. we do not average your gradients or sync your batch norm stats). We believe that users should be in full control of these aspects of their training and they should not be baked into the library. The trade off here is that you need to implement these features in your training script (typically this is just 2 lines of code to all reduce your gradients before applying your optimizer) or swap in modules that are explicitly distribution aware (e.g. `snt.distribute.CrossReplicaBatchNorm`). Our [distributed Cifar-10](https://github.com/deepmind/sonnet/blob/v2/examples/distributed_cifar10.ipynb) example walks through doing multi-GPU training with Sonnet. ================================================ FILE: WORKSPACE ================================================ workspace(name = "sonnet") ================================================ FILE: docs/.gitignore ================================================ _build ================================================ FILE: docs/Makefile ================================================ # Minimal makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build SOURCEDIR = . BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) ================================================ FILE: docs/api.rst ================================================ .. TODO(slebedev): add a title, e.g. "API docs"? Base ---- .. currentmodule:: sonnet Module ~~~~~~ .. autoclass:: Module :members: once ~~~~ .. autofunction:: once no_name_scope ~~~~~~~~~~~~~ .. autofunction:: no_name_scope Deferred ~~~~~~~~ .. autoclass:: Deferred :members: :exclude-members: __getattr__, __setattr__ Linear modules -------------- Linear ~~~~~~ .. autoclass:: Linear :members: Bias ~~~~ .. autoclass:: Bias :members: Convolutional modules --------------------- .. currentmodule:: sonnet Conv1D ~~~~~~ .. autoclass:: Conv1D :members: Conv2D ~~~~~~ .. autoclass:: Conv2D :members: Conv3D ~~~~~~ .. autoclass:: Conv3D :members: Conv1DTranspose ~~~~~~~~~~~~~~~ .. autoclass:: Conv1DTranspose :members: Conv2DTranspose ~~~~~~~~~~~~~~~ .. autoclass:: Conv2DTranspose :members: Conv3DTranspose ~~~~~~~~~~~~~~~ .. autoclass:: Conv3DTranspose :members: DepthwiseConv2D ~~~~~~~~~~~~~~~ .. autoclass:: DepthwiseConv2D :members: Normalization modules --------------------- .. currentmodule:: sonnet LayerNorm ~~~~~~~~~ .. autoclass:: LayerNorm :members: InstanceNorm ~~~~~~~~~~~~ .. autoclass:: InstanceNorm :members: BaseBatchNorm ~~~~~~~~~~~~~ .. autoclass:: BaseBatchNorm :members: BatchNorm ~~~~~~~~~ .. autoclass:: BatchNorm :members: CrossReplicaBatchNorm ~~~~~~~~~~~~~~~~~~~~~ .. currentmodule:: sonnet.distribute .. autoclass:: CrossReplicaBatchNorm :members: GroupNorm ~~~~~~~~~ .. currentmodule:: sonnet .. autoclass:: GroupNorm :members: Recurrent modules ----------------- .. currentmodule:: sonnet RNNCore ~~~~~~~ .. autoclass:: RNNCore :members: UnrolledRNN ~~~~~~~~~~~ .. autoclass:: UnrolledRNN :members: TrainableState ~~~~~~~~~~~~~~ .. autoclass:: TrainableState :members: dynamic_unroll ~~~~~~~~~~~~~~ .. autofunction:: dynamic_unroll static_unroll ~~~~~~~~~~~~~ .. autofunction:: static_unroll VanillaRNN ~~~~~~~~~~ .. autoclass:: VanillaRNN :members: :special-members: DeepRNN ~~~~~~~ .. autoclass:: DeepRNN :members: .. autofunction:: deep_rnn_with_skip_connections .. autofunction:: deep_rnn_with_residual_connections LSTM ~~~~ .. autoclass:: LSTM :members: .. autoclass:: LSTMState lstm_with_recurrent_dropout ~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. autofunction:: lstm_with_recurrent_dropout UnrolledLSTM ~~~~~~~~~~~~ .. autoclass:: UnrolledLSTM :members: Conv1DLSTM ~~~~~~~~~~ .. autoclass:: Conv1DLSTM :members: Conv2DLSTM ~~~~~~~~~~ .. autoclass:: Conv2DLSTM :members: Conv3DLSTM ~~~~~~~~~~ .. autoclass:: Conv3DLSTM :members: GRU ~~~ .. autoclass:: GRU :members: Batch ----- .. currentmodule:: sonnet reshape ~~~~~~~ .. autofunction:: reshape Reshape ~~~~~~~ .. autoclass:: Reshape :members: flatten ~~~~~~~ .. autofunction:: flatten Flatten ~~~~~~~ .. autoclass:: Flatten :members: BatchApply ~~~~~~~~~~ .. autoclass:: BatchApply :members: Embedding modules ----------------- .. currentmodule:: sonnet Embed ~~~~~ .. autoclass:: Embed :members: Optimizers ---------- .. automodule:: sonnet.optimizers Adam ~~~~ .. autoclass:: Adam :members: Momentum ~~~~~~~~ .. autoclass:: Momentum :members: RMSProp ~~~~~~~ .. autoclass:: RMSProp :members: SGD ~~~ .. autoclass:: SGD :members: Initializers ------------ .. automodule:: sonnet.initializers Initializer ~~~~~~~~~~~ .. autoclass:: Initializer :members: Constant ~~~~~~~~ .. autoclass:: Constant :members: Identity ~~~~~~~~ .. autoclass:: Identity :members: Ones ~~~~ .. autoclass:: Ones :members: Orthogonal ~~~~~~~~~~ .. autoclass:: Orthogonal :members: RandomNormal ~~~~~~~~~~~~ .. autoclass:: RandomNormal :members: RandomUniform ~~~~~~~~~~~~~ .. autoclass:: RandomUniform :members: TruncatedNormal ~~~~~~~~~~~~~~~ .. autoclass:: TruncatedNormal :members: VarianceScaling ~~~~~~~~~~~~~~~ .. autoclass:: VarianceScaling :members: Zeros ~~~~~ .. autoclass:: Zeros :members: Regularizers ------------ .. automodule:: sonnet.regularizers Regularizer ~~~~~~~~~~~ .. autoclass:: Regularizer :members: L1 ~~ .. autoclass:: L1 :members: L2 ~~ .. autoclass:: L2 :members: OffDiagonalOrthogonal ~~~~~~~~~~~~~~~~~~~~~ .. autoclass:: OffDiagonalOrthogonal :members: Paddings -------- .. automodule:: sonnet.pad causal ~~~~~~ .. autofunction:: causal create ~~~~~~ .. autofunction:: create full ~~~~ .. autofunction:: full reverse_causal ~~~~~~~~~~~~~~ .. autofunction:: reverse_causal same ~~~~ .. autofunction:: same valid ~~~~~ .. autofunction:: valid .. TODO(petebu): better title? Distribution ------------ .. automodule:: sonnet.distribute Replicator ~~~~~~~~~~ .. autoclass:: Replicator :members: TpuReplicator ~~~~~~~~~~~~~ .. autoclass:: TpuReplicator :members: Metrics ------- .. currentmodule:: sonnet Metric ~~~~~~ .. autoclass:: Metric :members: Mean ~~~~~~ .. autoclass:: Mean :members: Sum ~~~~~~ .. autoclass:: Sum :members: .. TODO(tomhennigan): rename to something more appropriate. Nets ---- .. automodule:: sonnet.nets MLP ~~~ .. autoclass:: MLP :members: Cifar10ConvNet ~~~~~~~~~~~~~~ .. autoclass:: Cifar10ConvNet :members: ResNet ~~~~~~~~~~~~~~ .. autoclass:: ResNet :members: ResNet50 ~~~~~~~~~~~~~~ .. autoclass:: ResNet50 :members: VectorQuantizer ~~~~~~~~~~~~~~~ .. autoclass:: VectorQuantizer :members: VectorQuantizerEMA ~~~~~~~~~~~~~~~~~~ .. autoclass:: VectorQuantizerEMA :members: Mixed Precision --------------- .. automodule:: sonnet.mixed_precision modes ~~~~~ .. autofunction:: modes enable ~~~~~~ .. autofunction:: enable disable ~~~~~~~ .. autofunction:: disable scope ~~~~~ .. autofunction:: scope References ---------- .. bibliography:: references.bib :style: unsrt ================================================ FILE: docs/conf.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Configuration file for the Sphinx documentation builder.""" # This file only contains a selection of the most common options. For a full # list see the documentation: # http://www.sphinx-doc.org/en/master/config # -- Path setup -------------------------------------------------------------- # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # pylint: disable=g-bad-import-order # pylint: disable=g-import-not-at-top import doctest import inspect import os import sys sys.path.insert(0, os.path.abspath('../')) sys.path.append(os.path.abspath('ext')) import sonnet as snt import sphinxcontrib.katex as katex # -- Project information ----------------------------------------------------- project = 'Sonnet' copyright = '2019, DeepMind' # pylint: disable=redefined-builtin author = 'Sonnet Contributors' # -- General configuration --------------------------------------------------- master_doc = 'index' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'link_tf_api', 'sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.doctest', 'sphinx.ext.inheritance_diagram', 'sphinx.ext.linkcode', 'sphinx.ext.napoleon', 'sphinxcontrib.bibtex', 'sphinxcontrib.katex', 'sphinx_autodoc_typehints', ] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] # -- Options for autodoc ----------------------------------------------------- autodoc_default_options = { 'member-order': 'bysource', 'special-members': True, 'exclude-members': '__repr__, __str__, __weakref__', } # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = 'sphinx_rtd_theme' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] html_favicon = '_static/favicon.ico' # -- Options for doctest ----------------------------------------------------- doctest_test_doctest_blocks = 'true' doctest_global_setup = """ import tensorflow as tf import sonnet as snt import tree # `TpuReplicator` cannot be constructed without a TPU, however it has exactly # the same API as `Replicator` so we can run doctests using that instead. snt.distribute.TpuReplicator = snt.distribute.Replicator """ doctest_default_flags = ( doctest.ELLIPSIS | doctest.IGNORE_EXCEPTION_DETAIL | doctest.DONT_ACCEPT_TRUE_FOR_1 | doctest.NORMALIZE_WHITESPACE) # -- Options for katex ------------------------------------------------------ # See: https://sphinxcontrib-katex.readthedocs.io/en/0.4.1/macros.html latex_macros = r""" \def \d #1{\operatorname{#1}} """ # Translate LaTeX macros to KaTeX and add to options for HTML builder katex_macros = katex.latex_defs_to_katex_macros(latex_macros) katex_options = 'macros: {' + katex_macros + '}' # Add LaTeX macros for LATEX builder latex_elements = {'preamble': latex_macros} # -- Source code links ------------------------------------------------------- def linkcode_resolve(domain, info): """Resolve a GitHub URL corresponding to Python object.""" if domain != 'py': return None try: mod = sys.modules[info['module']] except ImportError: return None obj = mod try: for attr in info['fullname'].split('.'): obj = getattr(obj, attr) except AttributeError: return None else: obj = inspect.unwrap(obj) try: filename = inspect.getsourcefile(obj) except TypeError: return None try: source, lineno = inspect.getsourcelines(obj) except OSError: return None # TODO(slebedev): support tags after we release an initial version. return 'https://github.com/deepmind/sonnet/blob/v2/sonnet/%s#L%d#L%d' % ( os.path.relpath(filename, start=os.path.dirname( snt.__file__)), lineno, lineno + len(source) - 1) ================================================ FILE: docs/ext/BUILD ================================================ load("//sonnet/src:build_defs.bzl", "snt_py_library", "snt_py_test") licenses(["notice"]) snt_py_library( name = "link_tf_api", srcs = ["link_tf_api.py"], deps = [ # pip: docutils # pip: tensorflow ], ) snt_py_test( name = "link_tf_api_test", srcs = ["link_tf_api_test.py"], gpu = False, tpu = False, deps = [ ":link_tf_api", # pip: absl/testing:absltest ], ) ================================================ FILE: docs/ext/link_tf_api.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Reference TensorFlow API symbols. This extension allows to reference TensorFlow API symbols using the ``:tf:`` role. For example, the following:: Sonnet :py:`~base.Module` is based on :tf:`Module`. generates a link to ``tf.Module``. """ import functools from typing import Any, List, Tuple from urllib import parse as urlparse from docutils import nodes from docutils.parsers.rst import states import tensorflow as tf from tensorflow.python.util import tf_export # pylint: disable=g-direct-tensorflow-import __version__ = "0.1" # TODO(slebedev): make the version configurable or infer from ``tf``? TF_VERSION = "2.0" TF_API_BASE_URL = ( "https://www.tensorflow.org/versions/r%s/api_docs/python/tf/" % TF_VERSION) def tf_role_fn( typ: str, rawtext: str, text: str, lineno: int, inliner: states.Inliner, options: Any = None, content: Any = None) -> Tuple[List[nodes.Node], List[nodes.system_message]]: """Generates a reference to a given TensorFlow API symbol. Only exported API symbols can be referenced. For example, non-exported :tf:`float32` will not produce a reference and will be rendered as plain-text. Args: typ: Type of the role. Fixed to ``"tf"``. rawtext: Raw contents of the role, e.g. ``":tf:`Module``"`. text: The `contents` of the role e.g. ``"Module"``. lineno: Line number of the parsed role. inliner: Inline reST markup parser. Used for error reporting. options: Unused. content: Unused. Returns: Generated reST nodes and system messages. """ del options, content # Unused. canonical_url = tf_doc_url(text) xref = nodes.literal(rawtext, typ + "." + text, classes=["xref"]) if not canonical_url: warning = ( "unable to expand :%s:`%s`; symbol is not exported by TensorFlow." % (typ, text)) inliner.reporter.warning(warning, line=lineno) return [xref], [] else: node = nodes.reference( rawtext, "", xref, internal=False, refuri=canonical_url) return [node], [] def tf_doc_url(text): """Retrieves the TensorFlow doc URL for the given symbol. Args: text: A string for a symbol inside TF (e.g. ``"optimizers.Adam"``). Returns: A string URL linking to the TensorFlow doc site or ``None`` if a URL could not be resolved. """ get_tf_name = functools.partial( tf_export.get_canonical_name_for_symbol, add_prefix_to_v1_names=True) try: prev_symbol = None symbol = tf for chunk in text.split("."): prev_symbol = symbol symbol = getattr(prev_symbol, chunk) except AttributeError: return None canonical_name = get_tf_name(symbol) # Check if we're looking at a method reference (e.g. "TensorArray.read"). if prev_symbol and not canonical_name: prev_canonical_name = get_tf_name(prev_symbol) if prev_canonical_name: canonical_name = prev_canonical_name + "#" + text.split(".")[-1] if not canonical_name: return None return urlparse.urljoin(TF_API_BASE_URL, canonical_name.replace(".", "/")) def setup(app): app.add_role("tf", tf_role_fn) return { "version": __version__, "parallel_read_safe": True, "parallel_write_safe": True, } ================================================ FILE: docs/ext/link_tf_api_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for ``:tf:`` Sphinx role.""" from absl.testing import absltest from docs.ext import link_tf_api DOC_BASE_URL = "https://www.tensorflow.org/versions/r2.0/api_docs/python/tf" class LinkTfApiTest(absltest.TestCase): def test_non_existent(self): self.assertIsNone(link_tf_api.tf_doc_url("tomhennigan")) self.assertIsNone(link_tf_api.tf_doc_url("autograph.1")) def test_link_to_top_level(self): self.assertEqual( link_tf_api.tf_doc_url("function"), DOC_BASE_URL + "/function") self.assertEqual(link_tf_api.tf_doc_url("Module"), DOC_BASE_URL + "/Module") def test_link_to_nested_package(self): self.assertEqual( link_tf_api.tf_doc_url("autograph.to_code"), DOC_BASE_URL + "/autograph/to_code") def test_link_to_method_of_exported_class(self): self.assertEqual( link_tf_api.tf_doc_url("TensorArray.read"), DOC_BASE_URL + "/TensorArray#read") def test_link_to_non_existent_method_of_exported_class(self): self.assertIsNone(link_tf_api.tf_doc_url("TensorArray.tomhennigan")) if __name__ == "__main__": absltest.main() ================================================ FILE: docs/index.rst ================================================ :github_url: https://github.com/deepmind/sonnet/tree/v2/docs Sonnet Documentation ==================== Sonnet is a library built on top of TensorFlow designed to provide simple, composable abstractions for machine learning research. .. code-block:: python import sonnet as snt import tensorflow as tf mlp = snt.nets.MLP([1024, 1024, 10]) logits = mlp(tf.ones([8, 28 * 28])) Installation ------------ Install Sonnet by running:: $ pip install tensorflow $ pip install dm-sonnet .. toctree:: :caption: Guides :maxdepth: 1 modules .. toctree:: :caption: Package Reference :maxdepth: 1 api Contribute ---------- - Issue tracker: https://github.com/deepmind/sonnet/issues - Source code: https://github.com/deepmind/sonnet/tree/v2 Support ------- If you are having issues, please let us know by filing an issue on our `issue tracker `_. License ------- Sonnet is licensed under the Apache 2.0 License. Indices and tables ================== * :ref:`genindex` * :ref:`modindex` ================================================ FILE: docs/modules.rst ================================================ .. currentmodule:: sonnet Modules ======= :class:`Module` is the core abstraction provided by Sonnet. By organising your code into :class:`Module` subclasses, it is easy to keep track of variables and deal with common tasks such as locating model parameters and checkpointing state. Module also helps with debugging, adding a :tf:`name_scope` around each method, making tools like TensorBoard even more useful. Sonnet ships with many predefined modules (e.g. :class:`Linear`, :class:`Conv2D`, :class:`BatchNorm`) and some predefined networks of modules (e.g. :class:`nets.MLP`). If you can't find what you're looking for then we encourage you to subclass :class:`Module` and implement your ideas. Using built-in modules ---------------------- Using :doc:`built in modules ` is as easy as using any other Python object: >>> linear = snt.Linear(output_size=10) >>> linear(tf.ones([8, 28 * 28])) You can get access to the modules parameters using the ``trainable_variables`` property, note that most modules only create parameters the first time they are called with an input: >>> linear.trainable_variables (, ) Some modules contain references to other modules, Sonnet provides a convenient way to find these referenced modules: >>> mlp = snt.nets.MLP([1000, 10]) >>> mlp(tf.ones([1, 1])) >>> [s.name for s in mlp.submodules] ['linear_0', 'linear_1'] Writing your own modules ------------------------ To create your own module simply subclass :class:`Module` and implement your logic. For example we can build our own simple multi-layer perceptron module by reusing the built in :class:`Linear` modules and :tf:`nn.relu` to add a non-linearity: >>> class MyMLP(snt.Module): ... def __init__(self, name=None): ... super(MyMLP, self).__init__(name=name) ... self.hidden1 = snt.Linear(1024, name="hidden1") ... self.output = snt.Linear(10, name="output") ... ... def __call__(self, x): ... x = self.hidden1(x) ... x = tf.nn.relu(x) ... x = self.output(x) ... return x You can use your module like you would any other Python object: >>> mlp = MyMLP() >>> mlp(tf.random.normal([8, 28 * 28])) Additionally, the variable and submodule tracking features of :class:`Module` will work without any additional code: >>> mlp.trainable_variables (, , , ) >>> mlp.submodules (Linear(output_size=1024, name='hidden1'), Linear(output_size=10, name='output')) It is often useful to defer some one-time initialization until your module is first used. For example in a linear layer the shape of the weights matrix depends on the input shape and the desired output shape. Sonnet provides the :func:`once` dectorator that means a given method is evaluated once and only once per instance, regardless of other arguments. For example we can build a simple linear layer like so: .. code-block:: python :emphasize-lines: 6-10,13 class MyLinear(snt.Module): def __init__(self, output_size): super(MyLinear, self).__init__() self.output_size = output_size @snt.once def _initialize(self, inputs): input_size = inputs.shape[1] self.w = tf.Variable(tf.random.normal([input_size, self.output_size])) self.b = tf.Variable(tf.zeros([self.output_size])) def __call__(self, inputs): self._initialize(inputs) return tf.matmul(inputs, self.w) + self.b ================================================ FILE: docs/references.bib ================================================ @article{chung2014empirical, title={Empirical evaluation of gated recurrent neural networks on sequence modeling}, author={Chung, Junyoung and Gulcehre, Caglar and Cho, KyungHyun and Bengio, Yoshua}, journal={arXiv preprint arXiv:1412.3555}, year={2014}, url={https://arxiv.org/abs/1412.3555} } @article{zaremba2014recurrent, title={Recurrent neural network regularization}, author={Zaremba, Wojciech and Sutskever, Ilya and Vinyals, Oriol}, journal={arXiv preprint arXiv:1409.2329}, year={2014}, url={https://arxiv.org/abs/1409.2329} } @inproceedings{jozefowicz2015empirical, title={An empirical exploration of recurrent network architectures}, author={Jozefowicz, Rafal and Zaremba, Wojciech and Sutskever, Ilya}, booktitle={International Conference on Machine Learning}, pages={2342--2350}, year={2015} } @article{sak2014long, title={Long short-term memory based recurrent neural network architectures for large vocabulary speech recognition}, author={Sak, Ha{\c{s}}im and Senior, Andrew and Beaufays, Fran{\c{c}}oise}, journal={arXiv preprint arXiv:1402.1128}, year={2014}, url={https://arxiv.org/abs/1402.1128} } @inproceedings{gal2016theoretically, title={A theoretically grounded application of dropout in recurrent neural networks}, author={Gal, Yarin and Ghahramani, Zoubin}, booktitle={Advances in neural information processing systems}, pages={1019--1027}, year={2016} } @inproceedings{xingjian2015convolutional, title={Convolutional LSTM network: A machine learning approach for precipitation nowcasting}, author={Xingjian, SHI and Chen, Zhourong and Wang, Hao and Yeung, Dit-Yan and Wong, Wai-Kin and Woo, Wang-chun}, booktitle={Advances in neural information processing systems}, pages={802--810}, year={2015} } @article{buchlovsky2019tf, title={{TF}-{R}eplicator: {D}istributed Machine Learning for Researchers}, author={Buchlovsky, Peter and Budden, David and Grewe, Dominik and Jones, Chris and Aslanides, John and Besse, Frederic and Brock, Andy and Clark, Aidan and Colmenarejo, Sergio G{\'o}mez and Pope, Aedan and others}, journal={arXiv preprint arXiv:1902.00465}, year={2019}, url={https://arxiv.org/abs/1902.00465} } @article{buchlovsky2019distribution, author={Buchlovsky, Peter and Grewe, Dominik and Gupta, Priya and Hennigan, Tom and Hseu, Jonathan and Jones, Chris and Levenberg, Josh}, title={Distribution {S}trategy - {R}evised {API}}, journal={TensorFlow Community RFCs, Google / DeepMind}, year={2018}, url={https://github.com/tensorflow/community/pull/25} } @article{agarwal2019stateful, author={Agarwal, Ashish and Berthelot, David and Hennigan, Tom and Passos, Alex and Reynolds, Malcolm}, title={Stateful Containers with tf.{M}odule}, journal={TensorFlow Community RFCs, Google / DeepMind}, year={2019}, url={https://github.com/tensorflow/community/pull/56} } @article{saxe2013exact, title={Exact solutions to the nonlinear dynamics of learning in deep linear neural networks}, author={Saxe, Andrew M and McClelland, James L and Ganguli, Surya}, journal={arXiv preprint arXiv:1312.6120}, year={2013}, url={https://arxiv.org/abs/1312.6120} } @article{blundell2015weight, title={Weight uncertainty in neural networks}, author={Blundell, Charles and Cornebise, Julien and Kavukcuoglu, Koray and Wierstra, Daan}, journal={arXiv preprint arXiv:1505.05424}, year={2015}, url={https://arxiv.org/abs/1505.05424} } @article{fortunato2017bayesian, title={Bayesian recurrent neural networks}, author={Fortunato, Meire and Blundell, Charles and Vinyals, Oriol}, journal={arXiv preprint arXiv:1704.02798}, year={2017}, url={https://arxiv.org/abs/1704.02798} } @misc{kingma2014adam, title={Adam: A Method for Stochastic Optimization}, author={Diederik P. Kingma and Jimmy Ba}, year={2014}, eprint={1412.6980}, archivePrefix={arXiv}, primaryClass={cs.LG} } ================================================ FILE: docs/requirements.txt ================================================ sphinx>=2.0.1 sphinx_rtd_theme>=0.4.3 sphinxcontrib-katex>=0.4.1 sphinxcontrib-bibtex>=0.4.2,<2 sphinx-autodoc-typehints>=1.10.3 ================================================ FILE: examples/BUILD ================================================ # buildifier: disable=out-of-order-load - Breaks copybara otherwise load("//third_party/bazel_rules/rules_python/python:py_binary.bzl", "py_binary") load("//sonnet/src:build_defs.bzl", "snt_py_library", "snt_py_test") package(default_visibility = ["//visibility:private"]) licenses(["notice"]) py_binary( name = "simple_mnist", srcs = ["simple_mnist.py"], strict_deps = False, deps = [ # pip: absl:app "//sonnet", # pip: tensorflow # pip: tensorflow_datasets ], ) snt_py_library( name = "simple_mnist_library", srcs = ["simple_mnist.py"], deps = [ # pip: absl:app "//sonnet", # pip: tensorflow # pip: tensorflow_datasets ], ) snt_py_test( name = "simple_mnist_test", srcs = ["simple_mnist_test.py"], deps = [ ":simple_mnist_library", "//sonnet", "//sonnet/src:test_utils", # pip: tensorflow ], ) py_binary( name = "functional_mlp_mnist", srcs = ["functional_mlp_mnist.py"], strict_deps = False, deps = [ # pip: absl:app # pip: absl/logging "//sonnet", # pip: tensorflow # pip: tensorflow_datasets ], ) ================================================ FILE: examples/README.md ================================================ Examples ======== Google Colab notebooks: - [Predicting MNIST with an MLP](https://colab.research.google.com/github/deepmind/sonnet/blob/v2/examples/mlp_on_mnist.ipynb) - [Training a Little GAN on MNIST](https://colab.research.google.com/github/deepmind/sonnet/blob/v2/examples/little_gan_on_mnist.ipynb) Training scripts: - [Simple ConvNet on MNIST](simple_mnist.py) ================================================ FILE: examples/distributed_cifar10.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "8KcfZ0oRSJ-I" }, "source": [ "**Copyright 2019 The Sonnet Authors. All Rights Reserved.**\n", "\n", "Licensed under the Apache License, Version 2.0 (the \"License\");\n", "you may not use this file except in compliance with the License.\n", "You may obtain a copy of the License at\n", "\n", " http://www.apache.org/licenses/LICENSE-2.0\n", "\n", "Unless required by applicable law or agreed to in writing, software\n", "distributed under the License is distributed on an \"AS IS\" BASIS,\n", "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "See the License for the specific language governing permissions and\n", "limitations under the License.\n", "\n", "---" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "PUWojJ3iTgKw" }, "source": [ "# Introduction\n", "\n", "This tutorial assumes you have already completed (and understood!) the Sonnet 2 \"Hello, world!\" example (MLP on MNIST).\n", "\n", "In this tutorial, we're going to scale things up with a bigger model and bigger dataset, and we're going to distribute the computation across multiple devices." ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "y4TOXSlnTcSB" }, "source": [ "# Preamble" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "9f811iaTTbmI" }, "outputs": [], "source": [ "import sys\n", "assert sys.version_info >= (3, 6), \"Sonnet 2 requires Python >=3.6\"" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "5R3-sAFoTiyB" }, "outputs": [], "source": [ "!pip install dm-sonnet tqdm" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "RdSVvLnkTmWY" }, "outputs": [], "source": [ "import sonnet as snt\n", "import tensorflow as tf\n", "import tensorflow_datasets as tfds" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "uPtNKJwhTnze" }, "outputs": [], "source": [ "print(\"TensorFlow version: {}\".format(tf.__version__))\n", "print(\" Sonnet version: {}\".format(snt.__version__))" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "Ab-nCz85sIkh" }, "source": [ "Finally lets take a quick look at the GPUs we have available:" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "elmPgnWJsIUI" }, "outputs": [], "source": [ "!grep Model: /proc/driver/nvidia/gpus/*/information | awk '{$1=\"\";print$0}'" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "-T4gfAHytWrk" }, "source": [ "# Distribution strategy\n", "\n", "We need a strategy to distribute our computation across several devices. Since Google Colab only provides a single GPU we'll split it into four virtual GPUs:" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 34 }, "colab_type": "code", "id": "14VQpLastTlW", "outputId": "f8d1bcf2-ca45-43bb-8485-5e6589f0b2e3" }, "outputs": [ { "data": { "text/plain": [ "[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]" ] }, "execution_count": 8, "metadata": { "tags": [] }, "output_type": "execute_result" } ], "source": [ "physical_gpus = tf.config.experimental.list_physical_devices(\"GPU\")\n", "physical_gpus" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "aoEY-15BtYaf" }, "outputs": [], "source": [ "tf.config.experimental.set_virtual_device_configuration(\n", " physical_gpus[0],\n", " [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=2000)] * 4\n", ")" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 85 }, "colab_type": "code", "id": "risJlPoDtx6-", "outputId": "522e0953-de6f-47f8-f06d-5dc14ad32465" }, "outputs": [ { "data": { "text/plain": [ "[LogicalDevice(name='/job:localhost/replica:0/task:0/device:GPU:0', device_type='GPU'),\n", " LogicalDevice(name='/job:localhost/replica:0/task:0/device:GPU:1', device_type='GPU'),\n", " LogicalDevice(name='/job:localhost/replica:0/task:0/device:GPU:2', device_type='GPU'),\n", " LogicalDevice(name='/job:localhost/replica:0/task:0/device:GPU:3', device_type='GPU')]" ] }, "execution_count": 12, "metadata": { "tags": [] }, "output_type": "execute_result" } ], "source": [ "gpus = tf.config.experimental.list_logical_devices(\"GPU\")\n", "gpus" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "WhdyolCktTGU" }, "source": [ "When using Sonnet optimizers, we must use either `Replicator` or `TpuReplicator` from `snt.distribute`, or we can use `tf.distribute.OneDeviceStrategy`. `Replicator` is equivalent to `MirroredStrategy` and `TpuReplicator` is equivalent to `TPUStrategy`." ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "cellView": "both", "colab": {}, "colab_type": "code", "id": "J82G9zqxtb1c" }, "outputs": [], "source": [ "strategy = snt.distribute.Replicator(\n", " [\"/device:GPU:{}\".format(i) for i in range(4)],\n", " tf.distribute.ReductionToOneDevice(\"GPU:0\"))" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "smiRXbgmT9SD" }, "source": [ "# Dataset\n", "\n", "Basically the same as the MNIST example, but this time we're using CIFAR-10. CIFAR-10 contains 32x32 pixel color images in 10 different classes (airplanes, cars, birds, cats, deer, dogs, frogs, horses, ships, and trucks)." ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "1xOwe9y_T_A4" }, "outputs": [], "source": [ "# NOTE: This is the batch size across all GPUs.\n", "batch_size = 100 * 4\n", "\n", "def process_batch(images, labels):\n", " images = tf.cast(images, dtype=tf.float32)\n", " images = ((images / 255.) - .5) * 2.\n", " return images, labels\n", "\n", "def cifar10(split):\n", " dataset = tfds.load(\"cifar10\", split=split, as_supervised=True)\n", " dataset = dataset.map(process_batch)\n", " dataset = dataset.batch(batch_size)\n", " dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)\n", " dataset = dataset.cache()\n", " return dataset\n", "\n", "cifar10_train = cifar10(\"train\").shuffle(10)\n", "cifar10_test = cifar10(\"test\")" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "BT2fP7jjpUGe" }, "source": [ "# Model & Optimizer\n", "\n", "Conveniently, there is a pre-built model in `snt.nets` designed specifically for this dataset.\n", "\n", "We must build our model and optimizer within the strategy scope, to ensure that any variables created are distributed correctly. Alternatively, we could enter the scope for the entire program using `tf.distribute.experimental_set_strategy`." ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "vEk6eJUPpWB-" }, "outputs": [], "source": [ "learning_rate = 0.1\n", "\n", "with strategy.scope():\n", " model = snt.nets.Cifar10ConvNet()\n", " optimizer = snt.optimizers.Momentum(learning_rate, 0.9)" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "YMgcImzms2V0" }, "source": [ "# Training the model\n", "\n", "The Sonnet optimizers are designed to be as clean and simple as possible. They do not contain any code to deal with distributed execution. It therefore requires a few additional lines of code.\n", "\n", "We must aggregate the gradients calculated on the different devices. This can be done using `ReplicaContext.all_reduce`.\n", "\n", "Note that when using `Replicator` / `TpuReplicator` it is the user's responsibility to ensure that the values remain identical in all replicas." ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "RUkuAJsxsjvt" }, "outputs": [], "source": [ "def step(images, labels):\n", " \"\"\"Performs a single training step, returning the cross-entropy loss.\"\"\"\n", " with tf.GradientTape() as tape:\n", " logits = model(images, is_training=True)[\"logits\"]\n", " loss = tf.reduce_mean(\n", " tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels,\n", " logits=logits))\n", "\n", " grads = tape.gradient(loss, model.trainable_variables)\n", "\n", " # Aggregate the gradients from the full batch.\n", " replica_ctx = tf.distribute.get_replica_context()\n", " grads = replica_ctx.all_reduce(\"mean\", grads)\n", "\n", " optimizer.apply(grads, model.trainable_variables)\n", " return loss\n", "\n", "@tf.function\n", "def train_step(images, labels):\n", " per_replica_loss = strategy.run(step, args=(images, labels))\n", " return strategy.reduce(\"sum\", per_replica_loss, axis=None)\n", "\n", "def train_epoch(dataset):\n", " \"\"\"Performs one epoch of training, returning the mean cross-entropy loss.\"\"\"\n", " total_loss = 0.0\n", " num_batches = 0\n", "\n", " # Loop over the entire training set.\n", " for images, labels in dataset:\n", " total_loss += train_step(images, labels).numpy()\n", " num_batches += 1\n", "\n", " return total_loss / num_batches\n", "\n", "cifar10_train_dist = strategy.experimental_distribute_dataset(cifar10_train)\n", "\n", "for epoch in range(20):\n", " print(\"Training epoch\", epoch, \"...\", end=\" \")\n", " print(\"loss :=\", train_epoch(cifar10_train_dist))" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "cHbiCxJ81wZo" }, "source": [ "# Evaluating the model\n", "\n", "Note the use of the `axis` parameter with `strategy.reduce` to reduce across the batch dimension." ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "cellView": "both", "colab": {}, "colab_type": "code", "id": "hxL8-GOB1yAq" }, "outputs": [], "source": [ "num_cifar10_test_examples = 10000\n", "\n", "def is_predicted(images, labels):\n", " logits = model(images, is_training=False)[\"logits\"]\n", " # The reduction over the batch happens in `strategy.reduce`, below.\n", " return tf.cast(tf.equal(labels, tf.argmax(logits, axis=1)), dtype=tf.int32)\n", "\n", "cifar10_test_dist = strategy.experimental_distribute_dataset(cifar10_test)\n", "\n", "@tf.function\n", "def evaluate():\n", " \"\"\"Returns the top-1 accuracy over the entire test set.\"\"\"\n", " total_correct = 0\n", "\n", " for images, labels in cifar10_test_dist:\n", " per_replica_correct = strategy.run(is_predicted, args=(images, labels))\n", " total_correct += strategy.reduce(\"sum\", per_replica_correct, axis=0)\n", "\n", " return tf.cast(total_correct, tf.float32) / num_cifar10_test_examples\n", "\n", "print(\"Testing...\", end=\" \")\n", "print(\"top-1 accuracy =\", evaluate().numpy())" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "Multi-GPU training with Sonnet 2", "provenance": [], "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: examples/functional_mlp_mnist.py ================================================ # Copyright 2020 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Toy MLP on MNIST example of using TF2 JAX/HK shims.""" from absl import app from absl import logging import sonnet as snt import tensorflow as tf import tensorflow_datasets as tfds fn = snt.functional def main(unused_argv): del unused_argv with fn.variables(): net = snt.nets.MLP([1000, 100, 10]) def loss_fn(images, labels): images = snt.flatten(images) logits = net(images) loss = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=logits)) return loss loss_fn = fn.transform(loss_fn) def preprocess(images, labels): images = tf.image.convert_image_dtype(images, tf.float32) return images, labels # _ _ # | |_ _ __ __ _(_)_ __ # | __| '__/ _` | | '_ \ # | |_| | | (_| | | | | | # \__|_| \__,_|_|_| |_| # batch_size = 100 dataset = tfds.load("mnist", split="train", as_supervised=True) dataset = dataset.map(preprocess) dataset = dataset.cache() dataset = dataset.shuffle(batch_size * 8) dataset = dataset.batch(batch_size, drop_remainder=True) dataset = dataset.prefetch() # As before we want to unzip our loss_fn into init and apply. optimizer = fn.adam(0.01) # To get our initial state we need to pull a record from our dataset and pass # it to our init function. We'll also be sure to use `device_put` such that # the parameters are on the accelerator. images, labels = next(iter(dataset)) params = fn.device_put(loss_fn.init(images, labels)) opt_state = fn.device_put(optimizer.init(params)) # Our training loop is to iterate through 10 epochs of the train dataset, and # use sgd after each minibatch to update our parameters according to the # gradient from our loss function. grad_apply_fn = fn.jit(fn.value_and_grad(loss_fn.apply)) apply_opt_fn = fn.jit(optimizer.apply) for epoch in range(10): for images, labels in dataset: loss, grads = grad_apply_fn(params, images, labels) params, opt_state = apply_opt_fn(opt_state, grads, params) logging.info("[Epoch %s] loss=%s", epoch, loss.numpy()) # _ _ # | |_ ___ ___| |_ # | __/ _ \/ __| __| # | || __/\__ \ |_ # \__\___||___/\__| # def accuracy_fn(images, labels): images = snt.flatten(images) predictions = tf.argmax(net(images), axis=1) correct = tf.math.count_nonzero(tf.equal(predictions, labels)) total = tf.shape(labels)[0] return correct, total accuracy_fn = fn.transform(accuracy_fn) batch_size = 10000 dataset = tfds.load("mnist", split="test", as_supervised=True) dataset = dataset.batch(batch_size, drop_remainder=True) dataset = dataset.map(preprocess) # Note that while we still unzip our accuracy function, we can ignore the # init_fn here since we already have all the state we need from our training # function. apply_fn = fn.jit(accuracy_fn.apply) # Compute top-1 accuracy. num_correct = num_total = 0 for images, labels in dataset: correct, total = apply_fn(params, images, labels) num_correct += correct num_total += total accuracy = (int(num_correct) / int(num_total)) * 100 logging.info("Accuracy %.5f%%", accuracy) if __name__ == "__main__": app.run(main) ================================================ FILE: examples/little_gan_on_mnist.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "Yu7u1mNfnxVP" }, "source": [ "**Copyright 2019 The Sonnet Authors. All Rights Reserved.**\n", "\n", "Licensed under the Apache License, Version 2.0 (the \"License\");\n", "you may not use this file except in compliance with the License.\n", "You may obtain a copy of the License at\n", "\n", " http://www.apache.org/licenses/LICENSE-2.0\n", "\n", "Unless required by applicable law or agreed to in writing, software\n", "distributed under the License is distributed on an \"AS IS\" BASIS,\n", "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "See the License for the specific language governing permissions and\n", "limitations under the License.\n", "\n", "---" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "YlYRfLOzyuW9" }, "source": [ "## Generative Adversarial Networks (GANs)\n", "\n", "In this notebook we'll use Sonnet 2 and TensorFlow 2 to train a small image generator using the Generative Adversarial Nets (GAN) [1] framework. GANs consist of two modules:\n", "\n", "1. A **generator**, which takes randomly sampled noise or latents as inputs and produces data (in this case, images) as output.\n", "2. A **discriminator**, which provides the learning signal for the generator. Its inputs are real images and generated images, and it's trained to predict whether each input is real or generated. The generator is trained to \"fool\" the discriminator into believing its outputs are real.\n", "\n", "Typically both the generator and discriminator are deep neural networks.\n", "\n", "For an extended tutorial on GANs, see Ian Goodfellow's [GAN Tutorial](https://arxiv.org/abs/1701.00160) [2].\n", "\n", "[1] I. Goodfellow, J. Pouget-Abadie, M. Mirza, B. Xu, D. Warde-Farley, S. Ozair, A. Courville, and Y. Bengio. [Generative Adversarial Nets](https://papers.nips.cc/paper/5423-generative-adversarial-nets.pdf). *NeurIPS*, 2014.\n", "\n", "[2] I. Goodfellow. [NIPS 2016 Tutorial: Generative Adversarial Networks](https://arxiv.org/abs/1701.00160). *arXiv:1701.00160*, 2017." ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "WAfR3cvnoGMB" }, "source": [ "# Preamble" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "4FqOAJb_jJR9" }, "outputs": [], "source": [ "import sys\n", "assert sys.version_info >= (3, 6), \"Sonnet 2 requires Python >=3.6\"" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "FZP3Wut53PTb" }, "outputs": [], "source": [ "!pip install dm-sonnet tqdm" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "ermIoeaTel6V" }, "outputs": [], "source": [ "import functools\n", "import time\n", "import numpy as np\n", "import sonnet as snt\n", "import tensorflow as tf\n", "import tensorflow_datasets as tfds\n", "import tqdm" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 51 }, "colab_type": "code", "id": "Rpp_houJEHr9", "outputId": "4edf64da-e887-4f06-90d7-e1e7e51e31fc" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "TensorFlow version: 2.0.0-rc1\n", " Sonnet version: 2.0.0b0\n" ] } ], "source": [ "print(\"TensorFlow version: {}\".format(tf.__version__))\n", "print(\" Sonnet version: {}\".format(snt.__version__))" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "ZAC0cJhzqE-a" }, "source": [ "Finally lets take a quick look at the GPUs we have available:" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 34 }, "colab_type": "code", "id": "6fOFkKYtqH8s", "outputId": "0f1bb7bb-0df2-492d-fd9e-67733a20cb28" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " Tesla T4\n" ] } ], "source": [ "!grep Model: /proc/driver/nvidia/gpus/*/information | awk '{$1=\"\";print$0}'" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "UYYmqvOKfNbk" }, "source": [ "# Dataset\n", "\n", "We need to get our dataset in a state where we can iterate over it easily. The TensorFlow Datasets package provides a simple API for this. It will download the dataset and prepare it for us to speedily process on a GPU. We can also add our own pre-processing functions to mutate the dataset before our model sees it:" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "UkBRriaQEr4z" }, "outputs": [], "source": [ "def process_batch(images, labels):\n", " images = tf.squeeze(images, axis=[-1])\n", " images = tf.cast(images, dtype=tf.float32)\n", " images /= 255.\n", " images = tf.clip_by_value(images, 0., 1.)\n", " return images, labels\n", "\n", "batch_size = 100\n", "def mnist(split, batch_size=batch_size):\n", " dataset, ds_info = tfds.load('mnist:3.*.*', split=split, as_supervised=True,\n", " with_info=True)\n", " dataset = dataset.map(process_batch)\n", " dataset = dataset.batch(batch_size)\n", " dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)\n", " dataset = dataset.cache()\n", " return dataset, ds_info\n", "\n", "mnist_train, mnist_train_info = mnist('train')\n", "mnist_test, mnist_test_info = mnist('test')\n", "\n", "mnist_shuffled = mnist_train.shuffle(10000).repeat()" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "JfOCWVGEfgcq" }, "source": [ "MNIST contains `28x28` greyscale handwritten digits. Let's take a look at one:" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 286 }, "colab_type": "code", "id": "I_yM0TVjFCZq", "outputId": "24c80259-d3e5-476d-e238-296506c02d62" }, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 8, "metadata": { "tags": [] }, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADfxJREFUeJzt3X2MXXWdx/HPt53pVAqYVpY6W0Yo\nWMxWVlsy1oftEk2FAIsp/kOolBQlDmskSkJUUo1i1ii7i3UJGMIglcLyoBFIm1gfsJhFQCrDUwvO\nagu2sXXoAKPyoJRO+/WPOdUR5vzu7T3n3nNnvu9XcjP3nu95+ObCp+fe87v3/szdBSCeaVU3AKAa\nhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFAdrTzYDOvymZrVykMCobyil/Wq77V61i0UfjM7\nXdJVkqZL+pa7X5Faf6Zm6d22rMghASRs9k11r9vwy34zmy7pm5LOkLRQ0gozW9jo/gC0VpH3/Esk\nbXf3p939VUm3S1peTlsAmq1I+OdJ+u24x7uyZX/HzPrMbMDMBvZpb4HDAShT06/2u3u/u/e6e2+n\nupp9OAB1KhL+3ZJ6xj0+JlsGYBIoEv6HJC0ws/lmNkPSuZI2lNMWgGZreKjP3UfN7GJJP9LYUN9a\nd3+ytM4ANFWhcX533yhpY0m9AGghPt4LBEX4gaAIPxAU4QeCIvxAUIQfCKql3+fH5DNtUfqLml1X\nPZ+s/3m0M7+4bFcjLaEknPmBoAg/EBThB4Ii/EBQhB8IivADQTHUF9z0I49M1uddtzNZv77n/mT9\n+O9dlFtbIIb6qsSZHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCYpw/uO3XzU/WN/asS9b7//iPyfr8\n9aOH3BNagzM/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRVaJzfzHZIelHSfkmj7t5bRlMoz+8+875k\nffCUa2rsIX1+uO5/lifrR236eY39oyplfMjnA+7+XAn7AdBCvOwHgioafpf0YzN72Mz6ymgIQGsU\nfdm/1N13m9nRku42s/9393vHr5D9o9AnSTN1WMHDAShLoTO/u+/O/g5LukvSkgnW6Xf3Xnfv7VRX\nkcMBKFHD4TezWWZ2xMH7kk6T9ERZjQForiIv++dKusvMDu7nVnf/YSldAWi6hsPv7k9LemeJvaBB\n02bOzK1d8rE7k9tOt/SLvy89+/Zk/eibH0/WDySrqBJDfUBQhB8IivADQRF+ICjCDwRF+IGg+Onu\nKWDPBYtzaxe+8cFC+/7BmlOS9dl/4iu7kxVnfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IinH+SaBj\nXnoa7O+v/u9E9fDktieu+0SyPv+mYp8TQPvizA8ERfiBoAg/EBThB4Ii/EBQhB8IivADQTHOPwkM\nfq4nWe/uyB/Lf27/y8lt569P1+WermPS4swPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0HVHOc3s7WS\nzpI07O4nZcvmSPqOpOMk7ZB0jrv/vnltTm0db56brN/+oWtq7KEzt3L1yJL0pg9uqbFvTFX1nPlv\nlHT6a5ZdJmmTuy+QtCl7DGASqRl+d79X0shrFi+XtC67v07S2SX3BaDJGn3PP9fdh7L7z0hKv24F\n0HYKX/Bzd5eU+wFwM+szswEzG9invUUPB6AkjYZ/j5l1S1L2dzhvRXfvd/ded+/tVFeDhwNQtkbD\nv0HSquz+Kknry2kHQKvUDL+Z3Sbp55LeZma7zOxCSVdIOtXMtkn6YPYYwCRSc5zf3VfklJaV3Etc\nh70hWV7SlT+OX8sDn0qP80/Tow3vu9k6eo5J1g/MOSJdf3ywzHamHD7hBwRF+IGgCD8QFOEHgiL8\nQFCEHwiKn+5uA785Lz0Fdy17fV9ubdqfRwvtuyjryv9U587/PTG57VWLb0/WF3Smv0V+/qWX5tZm\nfW9zctsIOPMDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCM87dAR/ebk/WrL7iu0P6/OPyu/OIvthba\ndy2pcXxJev6OY3NrgyffXPDo+VOTS9KX//NbubU195+a3HZ06JmGOppMOPMDQRF+ICjCDwRF+IGg\nCD8QFOEHgiL8QFCM87fAy4t7kvVlb9hfaP9PvXRUovpcoX3XPPaXT07Wt518bcP7Hnz1T8n6P804\nLFlPPa//sSj936SLcX4AUxXhB4Ii/EBQhB8IivADQRF+ICjCDwRVc5zfzNZKOkvSsLuflC27XNLH\nJT2brbba3Tc2q0mkDf4g//fvjyk4zv+br703Wb//vCtr7GFWbuWbf0iPtd945VnJ+kNfSX+GYL8f\nyK3ZAU9uG0E9Z/4bJZ0+wfJvuPui7EbwgUmmZvjd/V5JIy3oBUALFXnPf7GZbTGztWY2u7SOALRE\no+G/VtIJkhZJGpL09bwVzazPzAbMbGCf9jZ4OABlayj87r7H3fe7+wFJ10takli339173b23U+kf\newTQOg2F38y6xz38sKQnymkHQKvUM9R3m6T3SzrKzHZJ+pKk95vZIkkuaYeki5rYI4AmqBl+d18x\nweIbmtDLlNU1kr7WMTT6UrLe3ZH+ffojlg4fck8HdczP/119SbpvZXoc/+jp+eP4kvSF4X/OrT36\nb+lx/pErXknWazl/x7Lc2owfDRTa91TAJ/yAoAg/EBThB4Ii/EBQhB8IivADQfHT3a3w4JZk+ern\n35esf3Vuevt73nFrbu2D534que2cf9+ZrNcayqvl1sdyP/ypzjX7kts+9a/fLnTsP/Qdnaj+vtC+\npwLO/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QlLm37ieMj7Q5/m7L/5plVAeWLkrW7/7uja1pZJJZ\n+MDKZP3YldtzawdeKfZ14Xa12TfpBR+xetblzA8ERfiBoAg/EBThB4Ii/EBQhB8IivADQfF9/jbQ\n8fhTyfpbb/lEsj74kWtya502vaGeWmFXjZ8sP63/s8l6z1ceSNbzJ+iGxJkfCIvwA0ERfiAowg8E\nRfiBoAg/EBThB4Kq+X1+M+uRdJOkuZJcUr+7X2VmcyR9R9JxknZIOsfdkz+Gzvf5m+P5j783t/ah\ni/8vuW3f7F8k67WmBy/ihHs+mqy/deWjTTv2VFX29/lHJV3q7gslvUfSJ81soaTLJG1y9wWSNmWP\nAUwSNcPv7kPu/kh2/0VJg5LmSVouaV222jpJZzerSQDlO6T3/GZ2nKTFkjZLmuvuQ1npGY29LQAw\nSdQdfjM7XNIdki5x9xfG13zswsGEFw/MrM/MBsxsYJ/2FmoWQHnqCr+ZdWos+Le4+53Z4j1m1p3V\nuyUNT7Stu/e7e6+793aqq4yeAZSgZvjNzCTdIGnQ3deMK22QtCq7v0rS+vLbA9As9Qz1LZX0M0lb\n9bdvSa7W2Pv+70p6i6SdGhvqG0nti6G+9vPHle9J1s9f/f1kfdWR25L1d9xxSW7txM+mh/J8L28T\nD9WhDPXV/D6/u98nKW9nJBmYpPiEHxAU4QeCIvxAUIQfCIrwA0ERfiAopugGphCm6AZQE+EHgiL8\nQFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii\n/EBQhB8IivADQRF+ICjCDwRVM/xm1mNmPzWzX5rZk2b26Wz55Wa228wey25nNr9dAGXpqGOdUUmX\nuvsjZnaEpIfN7O6s9g13v7J57QFolprhd/chSUPZ/RfNbFDSvGY3BqC5Duk9v5kdJ2mxpM3ZoovN\nbIuZrTWz2Tnb9JnZgJkN7NPeQs0CKE/d4TezwyXdIekSd39B0rWSTpC0SGOvDL4+0Xbu3u/uve7e\n26muEloGUIa6wm9mnRoL/i3ufqckufsed9/v7gckXS9pSfPaBFC2eq72m6QbJA26+5pxy7vHrfZh\nSU+U3x6AZqnnav+/SDpf0lYzeyxbtlrSCjNbJMkl7ZB0UVM6BNAU9Vztv0/SRPN9byy/HQCtwif8\ngKAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQZm7t+5gZs9K\n2jlu0VGSnmtZA4emXXtr174kemtUmb0d6+7/UM+KLQ3/6w5uNuDuvZU1kNCuvbVrXxK9Naqq3njZ\nDwRF+IGgqg5/f8XHT2nX3tq1L4neGlVJb5W+5wdQnarP/AAqUkn4zex0M/uVmW03s8uq6CGPme0w\ns63ZzMMDFfey1syGzeyJccvmmNndZrYt+zvhNGkV9dYWMzcnZpau9LlrtxmvW/6y38ymS/q1pFMl\n7ZL0kKQV7v7LljaSw8x2SOp198rHhM3sFEkvSbrJ3U/Klv2XpBF3vyL7h3O2u3+uTXq7XNJLVc/c\nnE0o0z1+ZmlJZ0u6QBU+d4m+zlEFz1sVZ/4lkra7+9Pu/qqk2yUtr6CPtufu90oaec3i5ZLWZffX\naex/npbL6a0tuPuQuz+S3X9R0sGZpSt97hJ9VaKK8M+T9Ntxj3epvab8dkk/NrOHzayv6mYmMDeb\nNl2SnpE0t8pmJlBz5uZWes3M0m3z3DUy43XZuOD3ekvd/WRJZ0j6ZPbyti352Hu2dhquqWvm5laZ\nYGbpv6ryuWt0xuuyVRH+3ZJ6xj0+JlvWFtx9d/Z3WNJdar/Zh/ccnCQ1+ztccT9/1U4zN080s7Ta\n4Llrpxmvqwj/Q5IWmNl8M5sh6VxJGyro43XMbFZ2IUZmNkvSaWq/2Yc3SFqV3V8laX2Fvfyddpm5\nOW9maVX83LXdjNfu3vKbpDM1dsX/KUmfr6KHnL6Ol/R4dnuy6t4k3aaxl4H7NHZt5EJJb5K0SdI2\nST+RNKeNertZ0lZJWzQWtO6KeluqsZf0WyQ9lt3OrPq5S/RVyfPGJ/yAoLjgBwRF+IGgCD8QFOEH\ngiL8QFCEHwiK8ANBEX4gqL8A74xLCC0psmEAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "tags": [] }, "output_type": "display_data" } ], "source": [ "import matplotlib.pyplot as plt\n", "\n", "images, _ = next(iter(mnist_test))\n", "plt.imshow(images[0])" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "d7bsizs5gK3K" }, "source": [ "# Sonnet\n", "\n", "The next step is to define a model. In Sonnet everything that contains TensorFlow variables (`tf.Variable`) extends `snt.Module`, this includes low level neural network components (e.g. `snt.Linear`, `snt.Conv2D`), larger nets containing subcomponents (e.g. `snt.nets.MLP`), optimizers (e.g. `snt.optimizers.Adam`) and whatever else you can think of.\n", "\n", "Modules provide a simple abstraction for storing parameters (and `Variable`s used for other purposes, like for storing moving averages in `BatchNorm`).\n", "\n", "To find all the parameters for a given module, simply do: `module.variables`. This will return a `tuple` of all the parameters that exist for this module, or any module it references:" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "GrN37pi1o4HT" }, "source": [ "## Building the model" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "c6XoN56S2lSW" }, "source": [ "In Sonnet you build neural networks out of `snt.Module`s. In this simple example we'll build multi-layer perceptron (MLP) based generators and discriminators.\n", "\n", "We'll make use of \"Spectral Normalization\" [1] in both modules, which serves to regularize them.\n", "\n", "[1] T. Miyato, T. Kataoka, M. Koyama, and Y. Yoshida. [Spectral Normalization for Generative Adversarial Networks](https://arxiv.org/abs/1802.05957). *arXiv:1802.05957*, 2018." ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "hgjyB9yhFclD" }, "outputs": [], "source": [ "class SpectralNormalizer(snt.Module):\n", "\n", " def __init__(self, epsilon=1e-12, name=None):\n", " super().__init__(name=name)\n", " self.l2_normalize = functools.partial(tf.math.l2_normalize, epsilon=epsilon)\n", "\n", " @snt.once\n", " def _initialize(self, weights):\n", " init = self.l2_normalize(snt.initializers.TruncatedNormal()(\n", " shape=[1, weights.shape[-1]], dtype=weights.dtype))\n", " # 'u' tracks our estimate of the first spectral vector for the given weight.\n", " self.u = tf.Variable(init, name='u', trainable=False)\n", "\n", " def __call__(self, weights, is_training=True):\n", " self._initialize(weights)\n", " if is_training:\n", " # Do a power iteration and update u and weights.\n", " weights_matrix = tf.reshape(weights, [-1, weights.shape[-1]])\n", " v = self.l2_normalize(self.u @ tf.transpose(weights_matrix))\n", " v_w = v @ weights_matrix\n", " u = self.l2_normalize(v_w)\n", " sigma = tf.stop_gradient(tf.reshape(v_w @ tf.transpose(u), []))\n", " self.u.assign(u)\n", " weights.assign(weights / sigma)\n", " return weights\n", "\n", "\n", "class SpectrallyNormedLinear(snt.Linear):\n", "\n", " def __init__(self, *args, **kwargs):\n", " super().__init__(*args, **kwargs)\n", " self.spectral_normalizer = SpectralNormalizer()\n", "\n", " def __call__(self, inputs, is_training=True):\n", " self._initialize(inputs)\n", "\n", " normed_w = self.spectral_normalizer(self.w, is_training=is_training)\n", " outputs = tf.matmul(inputs, normed_w)\n", " if self.with_bias:\n", " outputs = tf.add(outputs, self.b)\n", " return outputs\n", "\n", "\n", "class SimpleBlock(snt.Module):\n", "\n", " def __init__(self, embed_dim, with_batch_norm=False, name=None):\n", " super().__init__(name=name)\n", " self.embed_dim = embed_dim\n", " self.hidden = SpectrallyNormedLinear(self.embed_dim)\n", " if with_batch_norm:\n", " self.bn = snt.BatchNorm(create_scale=True, create_offset=True)\n", " else:\n", " self.bn = None\n", "\n", " def __call__(self, inputs, is_training=True):\n", " output = self.hidden(inputs, is_training=is_training)\n", " if self.bn:\n", " output = self.bn(output, is_training=is_training)\n", " output = tf.nn.relu(output)\n", " return output\n", "\n", "\n", "class Generator(snt.Module):\n", "\n", " def __init__(self, output_shape, num_layers=1, embed_dim=1024, name=None):\n", " super().__init__(name=name)\n", " self.layers = [\n", " SimpleBlock(embed_dim, with_batch_norm=True, name='block_'+str(index))\n", " for index in range(num_layers)\n", " ]\n", " self.output_shape = tuple(output_shape)\n", " output_size = np.prod(self.output_shape, dtype=int)\n", " self.outputs = snt.Linear(output_size, name='outputs')\n", "\n", " def __call__(self, inputs, is_training=True):\n", " inputs = tf.convert_to_tensor(inputs)\n", " output = snt.Flatten()(inputs)\n", " for layer in self.layers:\n", " output = layer(output, is_training=is_training)\n", " output = self.outputs(output)\n", " output = tf.reshape(output, [-1] + list(self.output_shape))\n", " output = tf.sigmoid(output)\n", " return output\n", "\n", "\n", "class Discriminator(snt.Module):\n", "\n", " def __init__(self, num_layers=1, embed_dim=1024, name=None):\n", " super().__init__(name=name)\n", " self.layers = [\n", " SimpleBlock(embed_dim, with_batch_norm=False, name='block_'+str(index))\n", " for index in range(num_layers)\n", " ]\n", " self.outputs = SpectrallyNormedLinear(1, name='outputs')\n", "\n", " def __call__(self, inputs, is_training=True):\n", " inputs = tf.convert_to_tensor(inputs)\n", " output = snt.Flatten()(inputs)\n", " for layer in self.layers:\n", " output = layer(output, is_training=is_training)\n", " output = self.outputs(output)\n", " return tf.reshape(output, [-1])\n", "\n", "\n", "class LittleGAN(snt.Module):\n", "\n", " def __init__(self, num_layers=2, embed_dim=1024, name=None):\n", " super().__init__(name=name)\n", " self.generator = Generator(\n", " [28, 28], num_layers=num_layers, embed_dim=embed_dim)\n", " self.discriminator = Discriminator(\n", " num_layers=num_layers, embed_dim=embed_dim)\n", "\n", " def generate(self, noise, is_training=True):\n", " return self.generator(noise, is_training=is_training)\n", "\n", " def discriminate(self, images):\n", " return self.discriminator(images)" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "0i03px8y8gf7" }, "source": [ "Now we'll create an instance of our class whose weights will be randomly initialized. We'll train this MLP such that it learns to recognize digits in the MNIST dataset." ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 34 }, "colab_type": "code", "id": "XqL8oIMqGAnU", "outputId": "6471fe0d-0db3-4da0-c4ce-cdf672da67d9" }, "outputs": [ { "data": { "text/plain": [ "LittleGAN(num_layers=2)" ] }, "execution_count": 10, "metadata": { "tags": [] }, "output_type": "execute_result" } ], "source": [ "gan = LittleGAN(num_layers=2)\n", "gan" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "snzkUUh9oXPy" }, "source": [ "## Using the model" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "On8wI6VwpDPm" }, "source": [ "Let's feed some random noise through the generator and see what it generates. Since the model is randomly initialized and not trained yet, the images it produces should look like noise.\n", "\n", "Below, the top row of images are real MNIST digits; the bottom row are the outputs of our randomly initialized generator." ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 160 }, "colab_type": "code", "id": "4T-qmIc0GHfP", "outputId": "35babc71-6172-4756-f3fb-bb2f6f5cf363" }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAABFQAAACPCAYAAADUS4+vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzsnWd4VNXWgN8zJb0nkIQUeiiKCAhi\nBUQUCypWsCsWrGCv31Wv3nvtimIDRbGjiIKCYgOkShGQ3lsghBTS68yc78c6E4gIEkibyXqfJ8/M\nnDnnzM5eZ+2y1tprG6ZpoiiKoiiKoiiKoiiKohw+toYugKIoiqIoiqIoiqIoiq+hBhVFURRFURRF\nURRFUZQaogYVRVEURVEURVEURVGUGqIGFUVRFEVRFEVRFEVRlBqiBhVFURRFURRFURRFUZQaogYV\nRVEURVEURVEURVGUGqIGFUVRFEVRFEVRFEVRlBpyVAYVwzAGGoaxzjCMjYZhPFxbhVLqF5Wj76My\n9A9Ujr6PytA/UDn6PipD/0Dl6PuoDP0DlePBMUzTPLILDcMOrAcGAOnAImCoaZqra694Sl2jcvR9\nVIb+gcrR91EZ+gcqR99HZegfqBx9H5Whf6ByPDSOo7i2F7DRNM3NAIZhfA5cCBy0YgOMQDOI0KP4\nyYanjGIqzHKjoctRi9RIjv4gQ4BC9mabptmsoctRS6gu+geqi76P6qJ/oLro+6gu+geqi76P6qJ/\noLp4CI7GoJIE7Njvczpw4l9PMgzjFuAWgCBCONHofxQ/2fD8bv7S0EWobf5Rjv4mQ4CfzYnbGroM\ntYjqon+guuj7qC76B6qLvo/qon+guuj7qC76B6qLh6DOk9KapjnGNM0TTNM8wUlgXf+cUgeoDP0D\nlaPvozL0D1SOvo/K0D9QOfo+KkP/QOXo+zRlGR5NhMpOIGW/z8nWsQbFdnxnAAJH5QBQ6nJC//SG\nLFJjp1HKUakRKkP/QOXo+6gM/YMGl+MFq2UMc0fUjmrH234xHIAOT67BnZdfn0XyNRpchkqtoHL0\nfVSG/oHK8RAcjUFlEdDeMIzWSIUOAa6slVIdAfaICACS3pHInLEpcwFoM/FW2qMGlUPQqOSoHBEq\nQ/9A5ej7qAz9gwaX4xufDQJg+PDR1Y5vuvxtANIvLmLIvfcBEDrx9/osmq/Q4DJUagWVo++jMvQP\nfEKO5ef0BODh1z8E4Ovc7gAsGXM8se/Or7PfPWKDimmaLsMw7gSmA3ZgnGmaq2qtZEq9oHL0fVSG\n/oHK0fdRGfoHKkffR2XoH6gcfR+VoX+gcjw0RxOhgmma04BptVSWo2LjO60BmJYyHoAx+S0AaD3Z\n1WBl8hUajRx7dQGgxWtbAXg/dTYAnxdG887dlwAQMH1xgxStsVPXMiy9qBcA2V2kySjvUArA2n7v\nVp3jNOwAVJpuADr+ehMAgeuDAYhb4SL4m4V1VUS/oNHoonLEqAz9g4aW48NXfXHI7xPtIYx/6SUA\nbs4bAYDz5yV1Xi5foqFlqNQOKkffR2XoHzR2OaY/cjITb30RgHZOyeEywynzlWaL8/HU4W/XeVJa\nRVEURVEURVEURVEUf+OoIlQaA7seOBmANad71xmLjeidVy8EIO6XulsvpdQO3vVur731OgDHOAMA\ncJvy/WVhORS/+h0AX10g8nZv2FzPpWx6OFKSASgZZ2d0+9cA6OC0Vztnf2tvpek9JkdXnzFGDpwh\nLxsrXdxx+1AAQodJ5Jhrh+Y3UhRF+SvPfH0ZAE+3FO9a5AyJ9Lvs7p8BuD9mHa0cIQDs7i2euJSf\n67uUyv5M37UM2BeluT+dZw0DIGRRSLXjzf8oxTZrad0XTjksvPkYPaVlAJiVFf94jS0khE3vtQdg\n1enjADj14TsBiPpI5yCKUtdsefYkAOZd9QK51sSk2+t3AZD65koAPAWr67QMGqGiKIqiKIqiKIqi\nKIpSQ3w6QsUWFMTIGycBYDfENvRE1jEANP9oOUCdrpdSjh57XCz/fmMsAM1sErVw/MIbAUh4Wbxu\nWy4MZN3QNwEom7wIgKmntAPAvXdvvZa3KVHYQ/IQvZ/2MjvdYfI+PwmAd96UCLDg7H0aZhryaliR\nKgE37gZgSLLIrEvQDqYf8yUAj0+WnCyrrFxH7n676urfUJQmQeEVvQEIn7CAjPskks8VJN+lPie5\ni0yX5hTzFVo/8vee7VkTEgHImBHJSwki129vfh6AIbseACBmnHrF6xOvd7TSXGK9Hhihsvx0idh0\n9qmea+zr4kS+z+lS7dx5izsA0OGRlXiKi+um0MrfsmXksQAk95Htym39dxzqdACyhnZlxekSxesd\nET32L8nn+MZHabVfSEVp4hjWSob1Y6Xt3DDgDQCey+nOjNulPU6aMw+AA1vjusGnDSqZ13djWOSC\nase+f/l0AKJLdEDhC6x9qh2nBP4EwAn/fRCAFm/Mq3ZO2zkGXVpdC8DSk94HYPRdsqVk6r+rn6vU\nHt4EskMjHyByk4Sd2+ZISHNzDqPeP5eXr2kGwFenncWT498D4Jl4KzltvLyccvPdxL5r6bJp1kLp\nFcX/MBwOjI5iTC5LEiPnMc+sAODqWBlQfHz/yXyeIEnZwmxilL5vsBhbCl3hAOy5LFKX2/ko7oIC\nADacn8BPs2UZ0AB54fhb/wRg+7gGKVqTJWrtkV87ODSDwaEZ1Y45W/4CQP9fbiN4siZyrw9yhskk\nbPKwFwBo6ZAJ2wX0PKL7JTnyALCntcW9flMtlFBRFC+ZE9sAsO4EMVTPLnMCMPfCjti2LGuQMumS\nH0VRFEVRFEVRFEVRlBrikxEqjiRZJjD10RcA8dKljb8NgNYfLjjYZUojpOOTm+m1/A4AEj4Wq+IB\ny7RMk5RLJanQA4sllP1/13wIwJh3TsaduadeytpUiR5fO9FettlL+Xeb7gDYZ4gOf502GYD5T46m\nR4QkkEp8SaOOjpa914u3LXy7JNRz/LpvO9Wy82W5VWVodXt65Jo8PH8ehatVqXPscbF8M/2TQ57T\no8U8ILDasZcSq/eLj0/pwZTJ0pamPqn65ou4MnZz9xeyPHbNdRKd9H+J0wG4se/d2Gf+0WBla2pE\nfyB95IULhx7wXV6XGABS714PwClREq0wLHLDP94397pikibXVimVv8MeK/IZdPcsYF9kSk1oviCX\nWaWSbLhPcAkAOe5QAMwMHZ/WNbk3nkRO33IAnj5RFObZ1WcDULYu8oDzk36Tpa87Tz9wCpw0y/qu\nj3x3Up9VAOy+vw3G3IaJfFD2UXClRNvO7vEqAOnWMuYX+lwAgCt9W8MUDI1QURRFURRFURRFURRF\nqTE+GaGy5qEUABIdYWS7JWFX68lW4i7Nv+BTuLOyiBuTBRxeAuEZn8t61lfu+R2AdxJiQSNUfAbv\nVsydI3ce8F3syn/enlA5NFetlbwYV4RLgjy31R6muyurzikzJVrBjnyXZiX3KjErKDdFC72W9vfy\njgfgq+3yWv5TM5J+ypF7r1pXV/+G8hcGr5Y20mbsa+v2emRbz9dyxGPjsbJC2wyz2nuATsGS9Pny\nMLn+meZLePQm8ar3ct4LQKvHNO+YrxG8x6j2eVl5cwACsorrLRGfsg/36vUHHAu3durcO0Fe373n\nPACG3ffqP94vZnxorZVN+Xt2XdkRgEfjfrKOSO/X8QuJnG7HP0e9u1et4zurr+wXLOdXIsmHPYWF\ntVlc5W/I7uVmff+x1Y5d3usjedPrwPOLrpJoFm+Osf3Jv0r61UhbULXjX4xbxYcdUmqhtMqRYju2\nIw899TEATkP0a+hjIwGITG/41SkaoaIoiqIoiqIoiqIoilJDfCpCxZEgW4J8Pmi0dcTJ67mW+XHB\nnw1TKKVeidimfjdfwRYkFn6zU1sAdvaPZOCV4gWv2uXH4vT77yB6vrjyVMJHzoe3yTrS0anieQnO\nldoMzigBt0QrmEtlTTA2sfCXndej6vodA8TjbYbLutT7T/wRgOnHyRaQYccHknmv7Ph0wf9ki9Zm\nb2lkQ10zPEoiuipNN7eny052C77sCkCLF/85B8ribgMBeOryCABWXTuaEEMikypiVeN8leSvtgPw\n0fAEAK4Jl63qH7osltRVDVYsZT8crVsC0OLzbACmp7wJQKVpP+g1XedfB0DqzhI05rpuuf32bwDw\nWDHS9+w6DYC0f62yjh8eHtNW7T5uU/3V9cX9p39fo/Oj7ZLv5u+2N/9rZIqXKLtuX97QrL09gvNC\n8gE45rebAGj9ccNHpnhRjVcURVEURVEURVEURakhPhWhQkgwAL0CnVWH5t0tESo2lv7j5d78DZ6Y\ncHldvqa2S6jUMZm9xAa4tlLWQBrFZQ1ZnKZL7+MAWH/LfhnxrXwNWPkbQiIlkuGP3h8AYMNW5b35\nvVx0ePi42wFI+WyeRqbUAvYZsrNH9F+O/62X0yM1HvTtvmih9t9WP2UKsQC89sx9AKy8YTTxdmmH\ni2WjJpodVYmVw2HIljMA2P56GlE/y+4gLbIPf3ceb1RSmxXS5XdqOYw1fd6r5VIq9Y1rh+RMenuz\nRC1d0/ULAFr32artaQNiWHmptj12ApcPlt1jHoqVMao3MsXrHV9TCWsrEgF48uvLAWjzsET9aXRK\n3XNDxA5gXyTKosxUAGIKD8yHozQucm+Q3QyvCH8RqB5Zcl+G5Bb7I/vAvCdh/ye5iTJOlblg+Nm7\nDzjnubSJAPQKVC1saAqGWjv7nP8iIONP1x55NXp2AcCeXSDHtzTcLj8+ZVDZclWLap/LzUpspa5D\nXmMEBrLt4zQARnX7HID2zr0AXHPffYRO/L0OSqrUFS27Sej7uJxTAHBv3NKQxWk6GGIkKZjWBoDf\njnv/gFO8SaIODKMUI9gLOZ355LP+ACT/TyaDKeiWrb7A/tHpXqNY86WHGwytHC17T8kFIJwFRzVR\nNoJlEKLGFD/BWrYX5JBxkN2QtrbU5aTmm78qtYUtrTUAS246MPHsM9nijPjkZ1la0mK2h+BvxKjd\nBl0+WZ/sevBkYElDF0M5QioiZFy6/zKdAasuASD0WnHohe7efMB1XhNJwiLrzSsH3nvUnAEAfNL6\nx9oprHLEzHlRlkl6LGMKwLpL35A3l8pLvpWk/8P8Lry3TgxtKUPE+WRW1s+GF7rkR1EURVEURVEU\nRVEUpYb4RISKI1ESrr1+/TvVjv9rT09YuOJvrzECJSljzlctWdP9o798GwbAU8+9y8tzxQrpyjgw\n5EtpPJgnSQLGyR3lGej223AA2rCswcrUFCn+WRJDp3cW638Lx75t5yots7/nIGncfsrsSMLC8rot\noFInuBL3WfjfzZcopZBJGt2nKA3JjkdOBGDFMZKo/8MC2TY5+DabLvlppMx69GQA2k5tPMkUmyqV\nYeZ+kbVyLGx0ZI3vU3hFb15pYXnMLT+13dAIzrrmhKEHbkayd5qsZAjcfWTRz+5+3QF4NOkt64hP\nTJP9ksIhstTHbljzPNPDoPXnA5D3jizNy5RTeP+CtwEYEb2REb03ApD2vKQUaHdP/bS1GqGiKIqi\nKIqiKIqiKIpSQ3zC9FbcTZIK9Q+u7nPZVBQHZP/tNZueEivjhu5vVR1bU1ECQKeAkKr7PX283DtQ\nI1QaL4ZB5kPiIZ9WIhES7Z+QBETqhasnTHHfJL4kVv/rNt4LgCt4n0122BOy/aA36d6A0HUAJFtR\nLD92nsT8sfLdbUuvAiDlGev2S3WPz8aIPUK22X2w1w9Vx9aVJFjvKhugRIriPzjatAIgv5v0azlD\nZIzSfHwwwTurb9NpLl8r18RbaaAdDqJPqz5uCbVZkWT2g2/JqzQs1780GYCnBl0EQNrwhYc6XalL\nTKMq55s3snbbOaI77X846FUH4Lo254DIXN02ue6ptOr47bw2fJMhUeyRWw+dV/OfKI2TDROOCag+\nPb5n8eW05sCIGKXuiPxWVqC4XxLdKvCU4XpcojDD50rUSbikRuW516Q9XXdHAsuukNxVyy+T10s/\nHgaAuaRu5xmq8YqiKIqiKIqiKIqiKDXEJyJUDsaa79NI/kuEypb/SXbfuVe9aB0J5Y08iUL54EVZ\ne7XoGYlacZseDI9uidXQ2KNkzWpB/44A7DlB7HxnninbDC7IaMkfPT4BoNP4OwBovUGz4TckwZMP\n9Kp98XlCtc/jrrsAgLRbZXvy91v+womBEtXg3UqZ7+TlgqSedVNQ5Yiwhcq2gltGHgvAsMgZVd8l\nBEh02A+f9T7o9d4t7dqP0DwBjYm8QcdY72Y2ZDGaNEbPLuQ9JTmonkyTfcoHBJdWP+nkA6/rt1J2\nr7goaTkAcY4CrgrfU+2cH/bKFpLudRtrs8hKDTHyiwB4Ied4Ho2rnufv+giR2VWDrOjpQdDlvTsB\naD0pHwDPstX1VNKmidFD2sHy+AOjGT6/4HUAPjz5VADmj+1e9V34Djk/8HvZHqbDYolmuDr207or\nrHJQMk+Ssch3RONgO0DV6xHfc/Df5/l75YQveI2OR3VvpWZ4iqtHab6UcyLG3L/Pm+navBWAtvdt\npUuktKfrz5Gcm6azfiI2NUJFURRFURRFURRFURSlhvhEhEpgrlgMM1xi9U90yC494afu8844WrcE\nYM7VEpnS3C4e1sf3dGHpeRKhkvtsWbX7XrO1PwHTF9dhyZUDMGTf+OKLewGQMbiCl078EoBBITP+\n/poW+2XrbiMWS29uB3dBQR0VVDlaosdLFFHWePnc/6LbsN+RCUg+lf1psSCczEFBALizsuqvkE0Y\n23Hibdn8WAAAybF5TOjwGQBxVvvpNmdbZxtV110dtQSAL0O6AXBG8noAvl3fpeqcmJX7zlfqF3tc\nbNUud66duwDYe51Ebr7yxBtV531SmAhAx7cKAQ6yN5dSWxgOGW4VPl3M3C4Ta3z9jGO/AsBuiB/M\nbR4osdbBErG77eTeGPOWH2lRlaPElb4TgN+v7EK38/sC8MiNEwC4PEzGrd7cHQB/3DgKgDGXpAHw\n4+Wye5O5Nf0AL61y9HhzKaQtgW5b7wJgyV0ig67SHfJSizkA2J6YV5UfJdMtc5Fl5ZLH4ZyQg7ed\nL285C4BAttZ6+ZW6wXNaN8b1fv9vv7tr3pW05496LlHTxtOnm/VO6n16eidiWP+P19kKG8a04RMG\nFRZIIqDXcyQO9r/x8vnX4z7lzCF3AxAzfBuwz5Di5dNlvXC+LMsMNp1WXVHybmkO7K2zYisHknGv\nDOyX3jv6gO9G7W1X7fOIaAlbduFmRqkY0dac9gEAp3x+OQCR56pBxVcI/mYh9l/FEJb2/G0A/K+f\nGNPGpMxk0ARZIuS4LhkA1470Bihl08G2VwzUlbulvrfsDmHghPsBiNgmbeauUyWkecWwffp6xoQH\nAGj7gBjMVlrHW6MTuIbE0Uq2EYz9LI8+UTJheP6rwQAsvP5lAEKMgKrz33tUvgtZrttf1wc7HhQn\nwp9d9unSJpcs9Rm5+bIDzh/T9gsAEu0hh/0bD8XK8sr8N4JZco8sVbDP1ElAQ+FetY4kKw/ih8+J\nY++Jl0TWT50vfd/g0Iyq82+JlMnCHT9uAqD/7bf97fJapfZIelYcdqfvlLlE9tnieF3eV5ZkBRt2\nvMH8iXZZyppoGVJsVY6GA4P9dy8Qg3VLNaj4DFuGw0mB1be6WFguMm73tm6BUd8UJwYe0XVmdMNs\nmKBLfhRFURRFURRFURRFUWqIb0SoWCy6y0oO9YVEqITYApj38tuHvGbzWe8dcKzzvKsBaLlJE7fV\nF94kYC/eNrba8VF72zF1RD8AAlfuACDsK7EEeyNUOn53Ox1HSGK3u5+QEDBXCwm9jKzjcivVscdL\nqCvh1pKQjVtqdL13iZZ3q8j3e0mi6Eu+/oDJHWTb5X6nSAhu+OcaoVKXeCOA2o84eD2nlFohl8P2\nHUv6TT01jZHy1nEAvJf6ddWxa2/wRkMEHHD+9f+R7Vv/00e2G2z/Wcnh/9gC3T6ypqROtaJh79h3\nbFl5CwAyP295wPlDBl8LwKxDLA96OLMHAFuKYwGY0OZHAP7b/A/WfiARZBd8cw8A7UZqgujGQNv7\nRA6fvn4KAI8/Fs+qc97423NPfGIRf06ut6I1aaI+mm+9yuczrx4JQElzG6cO/fsoLxuyqcW1cXOq\nlgpVYeqyV1/j/m4/7hd1JIx8WhrsmHm6EUZ9UxpXPebDMP55ExnPqcezYYDMM4enny4H62m8ohEq\niqIoiqIoiqIoiqIoNeQfI1QMw0gBPgTiARMYY5rmKMMwYoAJQCtgK3C5aZp1mpDEsVzWlbb7RPIv\nrLlyNE7jn7dDSreS2Z415kEAUp6RNZNNJQlfY5Bh0huS46Z/sESWDN0yAICicysJdEqUw/rXxEu3\noKV4VTt8dh8AHR9fhqdM1rW2fqTpWokbUo45N0vum843yoLwzBLLHdP/KG+8cMU/n+NHNAZdrAmZ\nvYIbugiNksYox8CtOQB0/vhOfhjyAgCpjoPL79oISZx57aWWd/zSw/+t85N6HFEZGxP1LUPPCsmP\n0fX3a1h+orjBLwmV217yr7+PUDgUQ7cMoEiCizBLJLqo70AZGzUbuZkv204H4MfBkqj/bJfkR2p7\nv39FqjRGXTwcNgxPAuDaXr8d9JzjQ7fzJyn1VaQGozHKMPJj0ZNIYNPLhz73zqlXMvv46lsnR278\nZ2+6v9EY5Xg4GD0loX7noI/wWFFHj++RPi72Y0nC31Sk2ZhkmPijbGLBI/LitB88Otqb9H3r7VTJ\ncO63XQFIYd5Br6tNDidCxQXcZ5pmZ6A3cIdhGJ2Bh4FfTNNsD/xifVYaJypD/0Dl6PuoDP0DlaPv\nozL0D1SOvo/K0D9QOfo+KsMj5B8jVEzTzAAyrPeFhmGsAZKAC4G+1mnjgZnAQ3VSSgtPoWTW9u4u\ncdL6Oxl05ywAbomWnAzeLZX3p89vkpOh3TP1Y6VqbDQGGcYEyNZ/yypcABSMlAzoNvtOHJMk2mFd\nO8l3c9aaocA+T1pTiST6JxpSjjknVwDwXqpsbf363vbyYyMGApAwqma6tfd6iXgpSpH1qk5jGV8U\nxQAQvtl/t4lsDLpYE0oSDvTLFCVKVGBQfRemEdEY5ejaIlGAbR7axl3jrpeDDpFV5imiWyfctOyg\n1ycESn6jx+P+rPLO5VVWj3BZ+IHk1GleTx6fuqTeZegR71rATxFwYs0vT5shiYyiZ4jmNf9yVVVO\nKi8hk2THptIfQujyoeRgWXHShwAM7LMUgE1Bcr036tPXaYy6aI+OBmDn9Z2qjvUaIrugjU2ZC0Cl\nuWT/K6pd7428fuehSwjG/3f5aYwyPFq8OVmaEr4qx3XDZTeZ/Xf4mbhK+rp2lUsbpEwNRWOSoXu9\nrEpZWykrG6Yf+ym9Hr8XgNRnpV20xcrYJuu9KABWd3uP/isvrXZOfUUX1SgprWEYrYBuwO9AvFXx\nALuR8KC/u+YW4BaAIA5/+7/DIXbsfOaNlcn491dLOOs1j04F4LqIDQAc99VI0h4UhWgqIVuHor5l\naLce9i4hkhTo5hXXABDRTAbqca/YmdBmGgBnrpZtPINvk8ApTX15cOpdF60Eax7LvHVH9DoArr1f\nluxMHt6W5/88C4CYyXLvnAskDD12iny+9V+TcJsi2zNCZElCC4d0ZJWmjfd2nCa/1USWATW29vRw\nCctQzdyfxihH97rqCdfjrK1bt445+DXpibK04Pgb+tDynbVyn5zcauf4gyHl76hPGSZ+s4W0U8Q4\nsr5f9aT5XX+X/rFk1z7HUKcXdgHQbrtlDDNlJHMoLfSUlJA6VJYYdX5Ckip+c/VLANwXdI6c5CcG\nlf2p9/FNWlsA1t4pCaFP7SnbVkc4pO+bmPjqAddUmnbrdZ8Esz3isJhdKsue33noEgDC5m9tcuOg\nxtie/hNF85thO756wP+ml3oD+5IQNzV8QY62rmLw/PoM75JLByN2SbLoDnduBpr2PKSxyHD4vZIg\n+vvXRrH8ttcB2HKz9F9BVqLaJLv81oO7exJ+q0jN5XLVyu8fLoedlNYwjDDgK2CkaZrV3CKmaZoc\nxF5hmuYY0zRPME3zBCdHtqe0UjuoDP0DlaPvozL0D1SOvo/K0D9QOfo+KkP/QOXo+6gMa85hRagY\nhuFEKvYT0zQnWYczDcNINE0zwzCMRGBPXRXycPAmkJrysWwfOAV5bc8CjUyh4WTo9XCuKBHv56Ie\nn8kX78pLqVlB76USmtzsAcvztnFDbRfDb2goOaa9LSF3nUruBOD1geMBCLeJlfjqiB1ce+r7AHhO\n/csiLSvwxIatKsIFq6H9vdwJwE2/X0fKWGmOHPj3dsm+0J4ejGx3KUGZpQ1djEaBL8vx73Bl7AYg\n+b+7m4xXriFk6MrYTburpa7PpXu175JYdeD5R/g7ZqVEPbR6XJYejHz8ZOub/CO8Y+OloXTx2m9/\nBeCCUEme6F2qs3/0ycFYUymvV0y6m9Dt4tv0Lp31LvNpKnoIvt2eNl9Sud/YRriwr8hwZUMUqAHx\nJTluGyRL844J2DcV3lMm0YHuvOwGKVNjoLHJ0LuU9RxGEHn3dgCiAmQcGmyXhnTBV5KANmnUEszy\n7fVVtGr8Y4SKYRgG8B6wxjTN/XNdTwGus95fB0yu/eIptYHK0D9QOfo+KkP/QOXo+6gM/QOVo++j\nMvQPVI6+j8rwyDmcCJVTgGuAFYZheDPaPQo8C3xhGMYwYBtwed0UUakFGlyGX82WLHznnS+J2cZn\nyTrFrU92JOaHRUDT8sYcIQ0nRyuvSXsrP94bnS8AoDJG1i3u7LtvreTYG2Xb6xOsBF+nLr0KgOIF\ncQfcNmmmrDVvPefgyTL9jAbXxZrQ+juJQOpkSB6GNpNKMRYub8giNRZ8So7K36Iy9A8anRy9OVEe\n3HHBAd8tWpQGQIc3xMHbdkPTzLHxFxqdDGtCyMJNvJrbGYCRMasBmPqd5FBp6ad5pw6CT8mxtMWB\n8X+bP5UNF5rRZCNUGq0MQyb9TqUVL5P1l+9aWHrWkCtSDmeXnzmAcZCv+9ducZS6QGXoH6gcfR+V\noX+gcvR9VIb+gcrR91EZ+gcqR99HZXjk1GiXH0U5UtqPEC/M/0YcZx2RLbADWNRAJVKOBvdq2UHC\nu2YwZc6+7/79TPW8ADGsr/ZAOIGRAAAgAElEQVSq+A62WbJDWttZDVwQRVGURsaLLwwB4H+hMv8o\nOVEiLtkuuxi2efjArXPbIWMhjcj1H9w5uXzy/gAARt63uoFLoxwuP5/nXdESXHUsakNFwxRG8XkO\ne5cfRVEURVEURVEURVEURdAIFUVRFEVRFEWpAbHvHhiBojRNEl+WHA4XvNwTaHK5U3yKvGtPAiDO\nXl1/H9/Tg+C1sgPbke6upjRd1KCiKIqiKIqiKIqi+DXZ3SR1aYgRUO34Dx+eTEK6GsKUI0OX/CiK\noiiKoiiKoiiKotQQjVBRFEVRFEVRFEVR/Jpjum+t9vmJPd0ASP5iqy71UY4YjVBRFEVRFEVRFEVR\nFEWpIYZpmvX3Y4aRBRQD2fX2o0dPHNXL29I0zWYNVZiGxk9kCCpHf5CjytD3ZQgqR3+Qo8rQ92UI\nKkd/kKPK0PdlCCpHf5CjytD3ZQiHKcd6NagAGIax2DTNE+r1R48CXytvfeBrdeJr5a0vfK1efK28\n9YGv1Ymvlbe+8LV68bXy1ge+Vie+Vt76wtfqxdfKWx/4Wp34WnnrC1+rF18rb33ga3VyNOXVJT+K\noiiKoiiKoiiKoig1RA0qiqIoiqIoiqIoiqIoNaQhDCpjGuA3jwZfK2994Gt14mvlrS98rV58rbz1\nga/Via+Vt77wtXrxtfLWB75WJ75W3vrC1+rF18pbH/hanfhaeesLX6sXXytvfeBrdXLE5a33HCqK\noiiKoiiKoiiKoii+ji75URRFURRFURRFURRFqSFqUFEURVEURVEURVEURakhR2VQMQxjoGEY6wzD\n2GgYxsO1dW5DYBhGimEYMwzDWG0YxirDMEZYx580DGOnYRjLrL9zG7qstY3K0fdRGfoHKkffR2Xo\nH6gcfR+VoX+gcvR9VIb+gcrxEJimeUR/gB3YBLQBAoDlQOejPbeh/oBEoLv1PhxYD3QGngTub+jy\n1eH/rXL08T+VoX/8qRx9/09l6B9/Kkff/1MZ+sefytH3/1SG/vGncjz03xEnpTUM4yTgSdM0z7Y+\nPwJgmub/DnauIyDkrMCQGNyxbgA6hmazoTQGAJfbjlwv19iLbbgDrOu9x0rl1RVi3dhh4syTt7bm\nlfJdnnWRx7qm0iQwsQyAokr5zpErv1UZYWIrN+R35YVWcXsA2JLfHICAfBNXnNzMLLFTmZeLu6TY\nONx6auzUVI624JB5zqgYwiJFGCZQ6ZH6jHBIPWeVhMv5FQbtYjMB2FQSB4BjjwRFVcTKPZ35RpXQ\nXTHyGhUo9y7eKYJ2NfMQ6HDJdd7npMgh10dUEGiX7wpKggEID5ZylOTKZ0+YB1uhFYxlPUul2enZ\npmk2q0ldNVaORBedkUFnBSdE4NntlC/iXQTbRYdC7eUA7M6Olms8ENcsH4CcilAAPAVS/6aIA3sZ\nmN4qlq9o1czSpWzRJWexSUWMVQ6bCMIolosCCtx4kkXP3HlSJneInOMoFHVzR3kIsJ6D8nInrpy9\nuAubsC6GhMxzRsdUtXVmgEnzsAIA9lg6iFuqx1Zh4AmU+rSXyjF7uXyuDJXPgXvdVESIQD3BlqJU\nyneWamOrNKmMs74rF9l55e4MqiTI0sUSl7S17jK5n7cND9xdhqu1PCBuj1xYvnlXk9ZFe1DIWc7I\nmCpd+juCw0UnK7ID8URJ/+kpt+pWPmKpLfYKDxXhlmyCPNZvyXe2IlvVNd4+Ly5edHtPbqR856Gq\nnYyMLQag2JJnRanoprMYKsP2lc+Vm4u7qOnqoj04ZJ4zMobASEtOuYEExlj9UGkgAGEh8rk4P7hK\nH2zhoi/e9tRRYulkuCFyABLi9gL72uOAPLkGlxtnOzmpqCC4WplMG7SMygZg+17pe4PCpGyl5fvG\nSIZDfs+02omKbTubtC46I4POCkkIx201agE2F0VF1es2oNCSUYhR1dcFFMgxd4DVV3nbT7tJ4FaR\ne0WiNXC1dDIgS86tDDWwSddLyxYyXtqcGw+ALdhdNSY2S0XfY6IKAcjLDq8qk8fqxm3Bbioy83AV\nlDRZXXREBM0LSoikolLqKy1sD+sLpD7xjjusvis4vJyyfNFPb/trWjph36858+qrK/wvYxK5FGex\nidlc9DIxUNrT7fnWmDfIhdsar3rl7L1fSPMSAIJslRS6ggCoyJGbNvUxqi0s5CxHbDTNwuR5z94b\nUfXs5+bJsx8XJeOdrNIw8IhMAqw5oVcXY5rJObl7IrBHiwCSg6RN3VpiTUSKRPg2F7gsdXcEizzt\nO+VZcQfZcEeJ7nrnE169C9xjtftxgfvGwTarXyxuuv2iIyJ4XkB8ZFUbZgBRATK/q7AUrjAvZN81\nXnOE9eoJsvStSKrQ8OwbdxhWN+gMt+b+BSKM6JhCSi3jQUKA6OI2y85g5DpITsoCYNd20c/4lFwA\n0nPlWXAWm3gc8nveZ6Ei/fB00fFPJxyCJGDHfp/TgRP/epJhGLcADwERNnsAXfuPIO8aUYpfe77H\noFVXArB7ryiI22oEwxcGU9RSHl67ZfSIXiOVm9Vd7m3GVpA8Sf6FwLsz5LvJKfKPlcq5obvdtHls\nDQC/p7cEIOpLkUjGmW5CNosQvI3puBtfB+DKabcDkPq9h6wbpdEzl0ay9d2XD7uCfIR/lKMlw1uA\naFtAIK1uupcTz18BgNs02FMqsuvXbD0Aby/qA0BgupOJ170IwMVLbgGg+ZvSaWy9RuSTMC0AR7nI\nefcQGXhcmCb3XvTYCQDk3lpEuxgZHG7Ll0Fl+VxRhuSzttEyTBTixz+PAaDvsesAWPppFzn3tEKC\nfxWZ2ywlXDrmvvQa1FFjp8a6aA9ycsqYKyh80dKXkbs5Jkp06MTwzQD87/0r5LsSuHn4twB8vK0X\nAEW/ygClIlLkGL3GrJqYl0fL6/jhrwJwzXsjAYhfXMm2IdZgMqRCXn+XZyfplzxKnxc9K/wqEYDc\nXtJQJvwsOp53cTEtY0XW6za2YPczr9WginyCGuti8t33VA3uSltWMuKUnwAYvbwvAO4C6VhCtzgo\nbiv1GfWntHmRW+Tz7hOlfltPyid9QBQAxceKLtozZHAXvVZ+P3S3i53XWzP3rWJcc4WITJM77KFD\nlBjRFu+W56pwveirXcRN2xfXkjVK+qX8QumtNg35vyatizZnIG2vuZeKiP2+9w4o7PKmS98NAGwb\n156KC2XEWLxRDCABeTKCi9wkcgjfXsaOMy1jdAfRKbtDrC5Bs0XfAgpMPFbvf8N93wHwxmeDAHAU\nUzWZP+va+QAsypa+M3256GbzxSaZveUc026S8fyof64Z36JGumg4A2k17F46DLTk9Ek72l4j/eGi\nP9sCcFo3UaLF3x1b1Q+F95UJdNEv0p42/0N0a2ffgCrD56PXTwD2tcepX4uOkVdA4niR75yfu1Qr\nvDsQXhv8LgB3fnETAJ1PkXZ9+SbRTSpsOKNEzyuLpJ3YPuzhJq2LjiAHp4+9nMJKafeSQvOYN7ez\nnGdN2JJmivD2dHdSFi96lfKj6GlRCxlI5h4vxx2RFbS9Qcaf227tAYCnU5Fc85YoYGavIEJ2y/Xv\nPCV95uWfSZ8ZesxeXJbh2bVc2ubLB88CYMrYPlX/Q3GSNd7tvJcN9753mNXjM9RIF+1BTrq/eTU7\nsqTvmXTKaM6cfjcANmuS7Ngm49AufTawZloaIA5WgErLORvzu7Pq/l6HQtYZ0pHFzRR9KWgjx+MX\nunHfKWPUx9tNBeCOb28AoHnHLPLniH4HZ8lveA0r3W9bBkCn0AxmZneQf+4juenSsU17jGoEBJDw\n+N3cfvKvAIybeDZDBs8E4NMp8uzfeNHPALy74hRcpaJPqV+LvhSmiC5ecbuc88XrZxJ+qYxxX2r/\nBQA3LL8OAHOuPCshmSY5XUVGzTrLxDvyX9KX5rcPJecCaW9DZ8t8oiRRzm07aiMA24a1x20ZAVwh\nJjtfefXwa8g3qNkYNcjJsa9fR3ml5TCwe7ggReZ3u8qlPZvxrUzoDfe+cYdXP4rS5E3z2ZYDvdRD\nxqmWAzBb5BzfZycA2T8lAXDpVTNZUdACgEeSpwFw84prAAiYEM3zT78NwL/uvBmAe0d9AsCDn8mz\nkLDARXG8/N7eY0WWW+69/7B08WgMKoeFaZpjDMPIBQY6g8KGuZ0GV7dbBMDxU0ZUnZf8k1RSyY1i\nOQw4t4AKqxF6+QbpILwNlMfy6gw9bjHH9RbZlpnS+L3QR0alpavltTjJweURWwFYZEsFIPsisZDZ\n00MoaWmNbKyB6yObLgbAuVeElXWcncoKqabK1ErMgKa3zbRpmmOAMYZhXBpSFvBlqwm7yThD6ndL\nViyBv0vj8lGADLqvHzoTgC/X9+XcOXcC4Lash3tOsAxgm+Tel/9rGmPWnApAxA8y2J/kOh4Axyly\nbti0KJYdJ7/Xq5sMWHPmy2TOHGBgt2YfhuXB2VIglkbvhMG1JYx2V8ngdvEmKSNjSDm6WvE99tdF\njzNy2LqFrXCdLwO/kF+S2dxG9G35lG4A3PacGFHe/mAQo7+QyVbXATIp+DNQzj174GIAki7MI84h\nhtLxj1wAwHM7z5EftuzznZ5eQWChyGbLXNHFoGyrQxq7iZ+3yMBm4C3SPqSXSIO7PLs9AK78QNLt\ncix1ikFuXi1Uio+xvy7aIkO+rIyrxKj0uidN3v5K6jxG5k5knW4ZUTa5cVjRBfEXbwMg+1ORQctT\npA094+I1jP/4bACCNsiA09uxXXCvDGo+WHES4cEyqDxjwEoAvpktRrYdO2I5sdlWAPIyRZcdSVYk\n23Yxnuwe0pG8TZYRrk2u999q0rpoRIYOK403aX6cTK5dn8Szp49UfOendgOweafoQEClSftYGehl\nBMnk+5JkGZSPnjEAgJwLPbj2iF4HrpfB4KnnLAdg5ubjAIi9cCfxwaKvY96xDCneaLM+udimygDz\nz70ySMmfIgMUd2drQnJjFm2flDY5v30IWSW1USu+xf66aA8N/dI0YNlWeZTNrm7KpsoEqdVSkeXO\niSLDBFs5g0fJID/SLhFAT3c/H4A9NpHX/10xgeffFgPK6M19AQg7XQwpa1qLbCLinXRyyiRu7JVv\nAfDQY8MBKI80mF8sv+d1SLUPl+vL3hBnxH8mvl810CzYWeUlbNK66LFHDVvzW5uq6Mjs9GSirIiU\noDwZ7e8Y4I0M82AGi54FjBTdDRktehLUWuSSEp7H4v9Kfxq+RX7vymPmAvDlA9ZE4pcgPJfnAPDy\nbtHh5JnyzGyNjCQqRTq6gF1Sjs+/PV1+40wZKxemRxCUIM9RQUEwbrffOMQPm/110QyK+HLrpnie\nO0MMkReNfhDaSrvV7pqlAKx/W/qsjV+kUd5b+qjQhdJHVcZI/Z12q4xDkgL38tYfMoEPXiP9YlYv\nuV/4Rmvi1cFBoEve371oCAApP8uzYfwQjduaeGf1lmO2UJHvttvF2PrT3Z0IsSLIIguq5hhNWhdT\nW1QMe6bPRF58UeozKAA+mC9zhU6jZQLxaRdxuhrbg7ElidUr5VExbizNkL4rzC7H955cQdEicQhc\nuVeMzMFzZe5Sfpr0hSkJGVwcJeOjsX/Kb0XnS4RL5kAnL3T/BoBHN10FQIvZ8hxsu0na2pi1bjJP\nkLnjCSetJ29sWa3Uiy+xvy4azogvc5Y3JzBHdGpvWxfffN0PgNh3xVnjvE+uM1wQuVXqM+tq0Umn\nS+pyTz/RiW7ttpGfIw654PaiQ9elyH2e6ylj1wkT++I+VozWN+VeC4DnV4lQibt5C9d/fysAQd2l\nHR/x09Xy2TJ0Z9jCuPFC6Z8/Hj/A+28dli4eTVLanX/5kWTr2OGcqzQeaipHfyG0oQtQi6gu+geq\ni76P6qJ/oLro+6gu+geqi76P6qJ/oLp4CI4mQmUR0N4wjNZIxQ0BrjzUuZWhkNXDYMIWscy3bLeH\nzLli0U8/34oJz5Aw5vO6/clvPSWs7vnhYkGKbm2tOQ2R4zOS2jN1umWxvEw85wG/yvVmP7Eq2hdG\n8M574oH7160S2jMzvxMArbtkMea7swCItCImdu2R8jx9xacAPDzvUuzWYvOwDU5sZX5n/a+RHMsS\nHax9LBpzvVj34pLzyDlOPN9Grrz+ltVOzu6VzyOdfgHgpU8l8if+TImc2rZUrMcl7kAqtoqVOPXK\n7QCU/yBRJG7L8V7SAkyneIc250mEw7vjxSP3R1kqT0+Te4dvE/vgtkCxYN5+43QA3px/Busmircw\nsl+VV7z0sGrHN6ixLmI3ccVWEj9DmoCss0uJmi8emu2DRRdj7WKxrYg0qYyU+l8yW+ox/lTxnM97\nWzwE190zjVffEzm0vkfCI7a+KxEngdaax1+/70bqD+LKPv/13wF4pPlsACYVtWdqoYStL/mPhEYH\n3bFLXtNkHWRRdij2WaLfvf49nw1X+p1bvGZytJnYAt2EbxdFSTtjE45jxAO26xHRweselCVA7/4x\niKCB4qHe+b3oV9BF4kX1/Fv0Zew5KbhSrIilRJF9/GjxyL3XWryi0cvthO+Q3/s1VdZ8XDRsAQCT\n/ujBNz/KsQ4TpP3d8rC0Ez1Ok/Y5qn8p0+dKBFrz/6vqfpq0LjpKIWYlFLSXujYHFxAzXaI/Njwv\n7V2beGkbu0bvZOJMqeOw7dLevdNb+kDvuuPLOy7l6+WnAVB+rFTtsnckMiXGCsjMTU+m5Dzxqnvz\nP5ScLzK7M+03vokQGZU+K/1h8fXiAU+IkuciuzAUx8nWkq9gqvKd+RE1kqNph4poDzcePw+Aj9b0\n4vvz3wRgwIcPAGDvKF5Qj8fg42dl04DAQtG3ROtGe7qLLKbnHkPcedJXZsxMBiAs3YrsstZ1l8VG\ns/DDngDkPiiy6DxCosZeTv6JU14X159hqdm8Pa0BCHhWZLiqvAUlZSK4dp9K2bY0cV20h7iI6JZD\n7l6pz8KAAAwrj1RQtrRl7Y4TXdwxI5XE46VNLaqQeswdItXX7mbpQxePSKvKXVXeV/Tru4f7A9Dt\nXxL6Pt9ohnuq6PmCVHld/4GMb1pPvoXyBXKs5WTxvBe9lgBAdIj8Vll+FK4CiQhM/MNDVt5Rbd7Z\nGKmRHG3lBmGbHLycKt7li6+exReTJMLEFi71FJ8q48Cu3XcyY7NEF0SfJ+MNz4/S5s1ZKLoVdFkm\noSulbe59iUT6edtTm0tkW9DGIPgLkZMjVZ6XS56XpT9Ow8Xra/oCkPaCPENZj0o0ypbB4jlvMdlD\nWbQsMytOqJpjNGld3JUTw78/Hkra1TI5WzW/DVErrcbsc3kt/V0ilsO2Qb8BIpvf3pAVKK2v3QrA\n1D4dAbCNcnPrhTIn+GiTRCiFpVtRRAES7bBkY0uWlFrruKxcOm//PB6A2zdfxspSaYu9KSJOf07a\n+0mbuwJQWBRFoASOUTgkBPfupq2LhgsCcwxKWsj84cKef7ColURGb25/EgBBaRKB5/w+ksyeUrFR\nU2TSsOc0GbC0/0Da06AXXJQUi544fhbZv1FyCQA9b1kFwLbxHSnOkPY7JEv0Nv8GEUqIo4KwJGmH\nY6fIORnDRBfDrRxnua0dVVFNjn0aeFi6eMQGFdM0XYZh3AlMR7L5jjNNc9WhznWUMjV6FRRZYf+x\nM0qoHGzlVMiQQfqoIeMAuO23a6oSV5rdpALLuskE6ox2kiNj1tZ2/Of2jwF4+BsJwQqxJuGlVkLS\nlP4ZlFoh8Q/PuxSAW3v8BsCnm3qS1F3W1O1MFeFE/yjXPbxAhGTLdmIi10+963ku+D67plXVqKmp\nHMOjkkn5ys52axOpgmWxxB4v4arNH5OBWv/pMnl664+zGf2zTLJLTxCFKH1XOquJ/5O1hVtdsbTu\nJobMdZvku5YrpXHLbyP1nte9grOOlSI9liATxL7fymAxYp0dK08YV90qjeXqIrnP2z9Lh2qLK8de\nJo96Oytc/s/q6wB9miPRxYA8pqZ+Y2PHmdbxYifFqVKRMQuk3p/aPhSAytZlpCbKACT7F6lb5ysy\nEHDeL4OQ9oG7qxK0rVhn5WWRfgxPoOi4J8JF0stibJnyk3R6rQbJszNqeT+cofKMpJ8vg9O4UkuH\no6TBXZMZRsWpMvDfVBRHmafOVyzWKzWVY0izFKJnBlE5SOpn0zsdKRksxqegNiKMNz8SY3KACTkr\nxHDSYo10UqV9Rd577pHOI8xWQn6BhP6XbpeBZ1GSDO6apUi7V7ytGa6zRQbNwmSS/fUqmXzbgl24\nrKTDO/8l9w75RTrGLTliiCtMtWFF07P9XGlzWdq0ddGMdk2tvDSXPolbAQh1lDP/Ypn8HhMh+jFn\ngxjIts1siaVOOC1DSNkGy3mSLPqzLC+5ai1y52Tp33YEyyDRq6OxqyrYtcRKFt3cWm5bIIOP57+7\nkLjlcu/My+VZSZkgF+a1kd+qTDCpjJdzmh+XSfrnlTWppkZPTeUY2DoZT7MKJnx0BgDJS8oZuPJB\nADydZNxSsVfaMyPAzYkjZKnkj5ulkYyYalmdO0ofGuaoqMppVWL1i7vPkjq2Wc4FZ4CL9GS5Z8Ym\neT5sVm6IC+d0Ivl+CV3fvFAGsJEjrESKYXLNh+EX4D5X5LrpCuufWdzEdbHAMdX1QxzxWVLHhS1t\nJP8gfV9uV1lqdVozMWzMHw9rEsQx5F3W2vZz0cH234jDYf1PKZjR1nKTZtKGbuoiuvjTApmUm2mV\n2K2kxAEFcqOuC6XvjVlqJ+IyGR9tP0Ha78R3RWZFLcQMF+qEsIvk97JLEnHNqWFFNXJqKsfghBRs\nLqhwyeRs0sd96H/5EgDmZouzpihPdHLurG4EWynBdkeLDjqsYcXV934PwOs/DaRZXzGc/bJW+rEg\nq1+ssBxNps0kMF/e5zvlBqO/2W+n1XbSV5Y+JX1neZH0syeeIf/G7IQ0opZYSTp7Vc3dmrQuBhR6\npib9Vsr2njJOaPNNMRuutKzJE1sBEHimjH0S3y7i57NFNrYQkc2mLCsZ9xUylhnccT7vThgIwPLh\nkivz2DJJI9E3UdrKGaXtqbSSvUcvFTl+dpIs2dtbFkyR1YFWRogh5sfnxHFR3NvKjdPSg81a5rwm\nNYmy5/zL01DjfjElhbJmJqZT6mfl/cfR8hlZ+5iXV90wXF4ayd0XSz63sW/JuNWbRPr28V8B4DZt\nbCmQQWZ+P+kPC8tlvrLqA8mhGTVyJ2U/ieErVHxG9EuRVBFz3uyJ8yIZU1WGiZxbvizPS+SL0s7b\nP44lv4uVg25fvtzD0sWjmpGYpjkNmHa454bGaRRXY6QmcvQj/GoGUFNdDI9KruMSKUeC6qLvU1Nd\nDGmf+M8nKvWO6qLvU2NdjNcxamNEddH3qakuRoQn1XGJlCNBdfHg1KuL1zTEQ1aaJBb7Tq+uIv1b\nCauL3CjW3du/vx6A8K12hlwny0U+CpXwrDYxErYz4xfxiHoc8NAOiTbyWFvRYYjVKWinWK1u7zOT\nJydIUiOsLa++3SlLC/J2RRAxVUKrQ62tPPPPEQ+RWSyWxcfPn8Rrr0q0ymf53ch1z62FmvBd3AEG\nhcl24lLEymek7kvSm3muLCV4e4q8Nl9mki2RcKROEu/Yl2+9BEDvKfcCcEHvJaTnigU6fI0VXmRa\nlseTxHLZLKqIlkFiPbzyXolMsVnLUqI2OdjRX2T+1q8SbuFoLtfFLBfLoycgiKgNcv6ilW2Ptgr8\ngoikIgb85zcmbJLld08fO4VX7hFdMtxWZJhNdCJicyDbeos3m/ZSjz/cIRFGZ78lXtjwm0txWitw\nLuopy3nm/p+1NGGFeFo3X5fC0mWie4aVpG3UNPHiuMPdxLcSD96uUvEEmpOt5MKL5blITYKiJPEi\nLevShlJrO9KmimmH8hiD2HekTlwjMnHPFqt/Tndra/pjxLC+9edWuBOtHUSGWksHrKWMBbkSdRCf\nkEfHRySCa8u14tWuvEz0vGCNyMKIMnF7t2j9TeQU2Ud0s6QsgOAV8l1pc1maVXSstPVt2okHNfSZ\nOJx7RT8z+kTXSj34Ou5KO3szIsiLk7qbtvoYUiaK7s25RLwxjkCpR8MNFZ2tUP/m0kdtvvQdADrO\nkQSjq7cnEmE5OVdsFMNpp19FrjsGiZc7+7gAIjfKcxC7VPrV4PMsD/qf7SloJc9G68/kHO9OUK2+\nEDnm9I7HZrUTxekJePL37YjRFAnIhVaf2Miwdj7aO7KIcmsHwcA14uYyrV1E3OEGP2yQnWPYJt+V\nXChhyBVWlNCesjCKRkri7/yB0nc2/7W6tzP3WOhxsnjesv8rEU3p/a0lCKkOEgKt5PvWMuVWH8sS\nohX/lU55Z1+jKnHmpc9KdOfIB46mFvwAEwyXSUErqfMbr/mBN4+X5Y6BK6UeJ74rUUgMBnuR1HfY\ndvku8z6J9tvxnYxZHZ0LcS4XD/mmKKs/s1TFuztXeaKH0PbiafduJV+yXsZESekuij+RCKXS46zk\nuJkS5bD7emv3nx2h5K+SZ8VsX4YZ2PQ2TtgfTwAUtnZjpksfdPM1vzDuR0mEGW8l63YtFb2LWeti\n+4UyLwjaLvqVdq7o1PoS6UuDMm1kBko7HNVC9DRslQgxp7OMPcuaeagIs3YdWWT1eY/L7k7rXj2G\n6Pfkug03ifE8sJNEkqY/Icuio9oFELlVxr1drpElLluPvip8mvIYg41DnfRtLlHQpS85cf4qUShR\nG2UcWlwkMl79SChRhhX9YwW+xkfK523WCoeJC3vSYq3IP+1H2XW04yhpI1emSZtYOdBFQI7I1LuD\nU6W1vidzZXOWfic375Qu/ema+6U/jZ/t3WmvhC0jpC1o2XE3OcF+ZROrOU4PnsQyYmZJv3bGqLl8\n+6zoYvMckWHRKZYuDSjnxd9kUwXjBCuZb5nU/aPLLwLA7bYROkPGqxcPl1C8n16SJc/edjV3UjLF\nPWWsW9xSrs/4SewMLa7IYPcC0UG3FWV2+uOS1H/aOLlPiOlh/LfSxgfUMMOH3y3wUhRFURRFURRF\nURRFqWvqNULFEVNBs1HTCsUAACAASURBVCHb6RspHutJS7sTaeUIzT7PskhVio0nep3Ju7P7AlRt\nTZc/Wbxtg+4RD/hvo0+koJ2YkNp1E09sYZpYIyu+EIv9Y9OuIKhUzjHjrbXHdrlffMtcXCGyzm74\nbZMBeGvMhQC4Wsm5b/33Ery+t6GRS/nM7neJMGuELaqSkAszybU84c6ee6m01qred+/XALz6vuRN\nMdwebJY8s2+QNaSnfHq/3ChU6nfW+72wW8vHT7xCkkr9OVrWFndvKfk2Hk2axuAfZftlo6+c67DJ\n9af+ZwG/PH8KAD3v/QOA38aLNTK7t7VlbEIhWzeLZblDmjwn24+yHnydnPxwPvyuH84O4jm5f8rV\ntM4THdx2vnjKHUWiN2UGjBoguYrunyh7tZ82TSKMOk4VBb4x5A5iM0Qmf+ZLqObem8RD4Aiz7rO9\nAvNYsRwf30K8Dqv3yHNkLoqkLMlqjqxtH5vPEG94v2/+BOCr7d0oXiSRMqf2WMP3If6Us63meAKh\nuKUbW4W0UOUL44k+VRaNVuRa25rPaAVAh7M2kVUqlv3MlVKHzVrIuaFfipfnkhcX8dEZEjH02LWy\n5eRHV8tn12mW5zTR5LIuome/TxA9y6sU713CFhd7xeFGZUuRc+xsaY/Ts+WZSH58J3u+lPedrxAP\n3opRR10Vvo3HwCi3se5dSZbubAO5nUQHQiMlYrLF4NUAhM+OY/lcSaJosxLjtf7uZgBiF4n+OItM\nQjNEl0sSpf7X3CvtX1yiRKHsXRtDRYSVbHOonLtpp3jCB161iG+Xi8duS6rIvf0H0n5vv0T0tey4\nElLHSQHy2gfisddKTfgu8S4892dzcwtpqz4YN5DofCvZr1QryTPFc93vv3MZ/5usvQ/PEBmc0FfW\n8G+ynoEsWxuMBCtHTprIJ2mOfM5/UJ6JwNnN2JYvume/S6KMjg+XSIfNabF0CBP9tg2Q65a+IpG9\neceJTANbFBK8S9rsV5b0t/6RX462JnyapPhsnr7/fTaXy/jxj8JUAlZLNENZnNRVpaU3houqqL+y\nUmubebflqbaiv7JaO4nZLe+9vZU7SD4nzbK2Rh5so3S16Kc3muiBobI966ezziPrJHluOrSXPrNg\noSxLOq2lJLWdvaYrbV6TvHXbbu3ojxsn1BzTICpJokAmjOtP/E4Z85fFiHy8SbR397YTsFsar/Jm\nck50gEhq0x3Szg5/fyrf3CXRz7H/lsiEjfESXRmSKbI03DaKLpHfy94sEUkdbZbcRq5i5lLR64AY\naUfdi0XeW8+X3+zdYw35V8hztudOXXYGSF4iu8lsKz9U24QsotbLV7ZKS79Okvpr1243Tpu839RF\nhLtth8ztAnbJ+Kgy0lOVW8wZZEV8Zkh/WNlVxkBxc5w4LpN8Obt3SNs6brYkNDYj3Gw9z0qI2lIm\nLYGWVmeeKmPmkssqCZwjz8aOxGAqSpp25CYeA0+Jgy43SbL0Tz7vT0yZtXHCqVI3YVMkcs+WZlYl\n8G79pbRhO/pbY5rV1kqSApPwndJuTv5Y+lBvRvfiY61cgBGlONeKfnlz3YR1lnnKrj8SGXSe2A/+\nzJNx6NIbjpV7PyvzjZ7NtrP2JomE2nlmZI3+XY1QURRFURRFURRFURRFqSH1GqFSXu5k3eZEMlZJ\njg1bqgdnsVikwueIddZxrlgM93SPI66VWIPtVjTC1Y8uBODPIolUuf6B73jt6/MB2GlttVvUQdZl\nBVipMoJbFnJRP/EaTVwvHpqsQrEutorJZXuyeGs/2i67jvS7Rn6j1Nqzd8X8rnS/fykAZ33wANtz\nXqmVuvBVXG47e/LCaD9ePGqBZ1ewapZYkCc3k/p1FopMK0NtnDxQ6j79blnjXdDWWuPdSrwC4+9/\nmQun3wXAz6tlx4NB1g4Ic96VLXn/faWN0GYSGdSzhcSWLB8vVsX1Kc1xWwvdvpsv+UAizxRrZI8Y\neZbS32nHQ49NkTIO7FE7FeHjGG4IyDcozpSohYDkEtxB8szHWjt85FrL/JNmV/L2xPMAaBMiES1B\nL0vdLr9N9M5RAJ889yIAg1+SvCplJ1le7XWWd3wdDL33VwA+f/FsABK2iodvb5pJ8VKxVAe5RJ7l\nb4t1eexKiUBq+0wFRaLurB53DKXZP9ZGVfgstnII3WanOFXax4Rj9rB3lkQQGF1EX665VDzOX2/v\nSna65QW1HJjtwqV9nXam5Et5fUUfgmLly2ffk20/yu6R+7jKRU4922/li2WiQ1vGjgXgtDtuBSCz\nl73KA3Rn95kAvLtBsuq7Lc/D1hUtMI+Rk5Z/16kWasH3sZdB1GobeeIUIbXHTnYsEu9J5V7pF1ss\nEK/nrHWpRG2V87zryFs+I2672fnSJtrcBq6rJIrBWC4etSBrq+uyDeK1s/cowtwsup81T1w8lUni\n+Zn9fk+MzqJ7gVnSTmc/LHqfFiVe8sL/S2brrXKOx12K6c1h1kQxsxx43mjOW1dIvg1PqofCcPGC\nNksQz/Xwq3+Wz44Cvt7QF4CeV0lU5i8LJLeU7SrRs9PabqR5oET43R4quyL8d7HsZDili+xQ8V7y\niewqE50udImcV02WPrTDoPVM+kg8qxWRonvNrpWIlfL1su7fzA6mxMpNbt/d1EOMhJ2743jiuRvw\nDJIxROHaGALkMaeZbBRD3M1bAVi1OgWb3YpQsLYjL7W29Cw9U+R4Tdff+ThXPKmhdtGRuy6Sscjr\nRRINfVKXNaz7UORW0Efa21EfSs6AitNMIlZLm3zqSZJb48sh4q39eaW0n7YID2v+I2OwxBke7OVH\nXw++TECBSep0N9sukHp65NZvmHjTWQDktZP2tCJGZBHaNp+u8bKL0uw1El6ZVS7zg/W3SaTD9vIY\n9twlkQjb58k5yRfvqvabrYKKWTFLIlqsR4LZ38ruMBFbPDiOFRnad8q9z7pY5hlOQx6uiQt7Yn9O\n2t/kj/1r98IjxVlg0OInO7mXSeRB1pcpxG63oqiHi/xCVoi+xXYpZm+ZyDYgQJQxIl7Otc2ScWXR\nhUWMe3kMAJf8IVGdue/LM+J2S66445plMGu9yDFitfSZFVZOFnegjYA8kWOxtcX1mV0kynbGdmm/\nQ76MxHOljKuajYokq6Bp5zOylRuEbnayuLlEXfUbvITf35J5WlxP6Y/ca2UOfn6/xWwslL6pcLqc\n704RGZ55tvSTs946kfJ7pW02Z8pY19u/xTeTfnbvwng8YdZ4xFrdUrRaoo2C9hgs+T8Zv+65UXQ6\n+GkZR3mmyP1m2hMovUTu6Siu2f9br5obEVLKOV1XMj1IOoKIBcHEXi0T8535ElqTlykPeOt55Wxp\nLe+dmdKwvfmbbKV07mXzAfijoCVmW/mPSyplQGDLkQc9ZJe1zGd3JF+ukWQz0d0sA837omA7r3YT\ncqaEdxWVyXXfzZRJfNRaS3E6GqwfKeUNOQ5sTTzHUKDDRbv4bDbcKRPp1H+VE2o94Oc3F+PJqGjJ\n5hSa6WbWbGlokp+RZV7lk6RDie4j4VVXvn0vAWHy8J4+QK7/dokYZsLOFgVZuro1CS3/n73vDo+q\nbNq/z/bNZrPpIT1ASELvvReRoiCKoCCgYEVERbHra8cGqKiooFJEpEhVQKT33ktCCOm9J5vtu+f3\nx/3A+73X910/9SOXnzFn/tmUPefszjwzzzwz98zQ4OkEjNIRSvkcT2uKng8R7uoopULUWgm/y/+R\nTkZNMxU+2kjnRTOXTiruunleNGSS/D3Q9ykDDvCAFbVChWt3cgO5ZzAbL/+ymI6gPVSDzLu4qxjF\nwUxbSYMVeJ4mxBEOPDp+OgAg0irGj91J+clP8725E5tj0VoesOVmlHlAFj+PPUKCOYt/e+9lNtl8\n+/GpAADPbQTSFfX3h9SdkPa6s4Hw/bMm0v158vdC1bsSkctoJx3JGng7cn2HmumYL9vA5lreRDs0\nVbSRGht1J60LjZnmDfI3JbIY6Xrq5/UGX6bD1+Hu/D1rTxLQl9e1XkB560RJQ+AV3w0o9eLUXgCA\nsDPU1/x7uWlprxqha1ctbqY0pQUAn9kHa38b9GcZ4FC9FYJhcxlUPlKcAAA4v5jBErm3C3Yx5tgZ\nzD0r6xgjn3IAeR0RXw55CZ0UmdWTcDQVpywxErLJBj/URfE+1wPgzqbU06p2bkiiVNPZjA5N5Kdc\nF1dn8O/aFAMi1vH91ig11HWNG+zqMUko7qaGp0YogMmLcZ0ow2Nl3Cu/yhHNTdUerH7mQwDAlBfZ\nZL3/TE6ePPoz98s99mRM7EJo8sevcoTue+98AwAY/QED1jXd7bg1hQ79qb2MxrmTqGfpPyVh2GT6\nSVVu6nDWszwMdpnDsb/PRm/D7C2PAwD8CpUyEQCQNYA9VIL7Ih1wU4GE2u50vN3tuN4lO/lpzNfA\nLpa9I5l6kvQJ9ezKEywBWrWhPzTihF1n5d/O1fGwoBZV7mXPxkH9ChMU4Uv43IDL3EOtSRaUdORD\nlpzvyQuKqPdSKGUdfgzQ11xvJO+D5GvchziPUUJZOy38ucwxR74NoS/Sb/Ccpm/oV0Ce2p2BuKwR\nZQGp1N1AMea8dzIDWNtzUqD5jf6PQYxRrXPxvc7tPAC6hheheW+eZcpstOP2PfxfSTcZ7dozKJq6\nh5ne335i02IRT0GAE9BVizOM21UPXGj45AvxwH5fJXznqRPW/jaEiSb505uwhORbI3Ui86tkuMy0\nYZGneCasFefH2OlMOFwujcAwMQxDX0Ze33YXE3wbP2aj1FRHMHrN4HkibQ/PfXVijLXFbENtHXVY\n7eX1O86JrKMoLSntoob/Dsq9YqgProuN3K7KgOQDApczIbQnqTNs/Wn4hoUwKHkwjOe2i1WRyMgn\n7/yTxECMI3zdAJYgmwIk1FTyXibmeGATAy4qTtHncUW6MbQD18fBdQxqBl+mb1Q8wQFPjhicUM7X\nhI+o72mPixHogW5E/cznVrb4c4mGxu0FKaSQQgoppJBCCimkkEIKKaSQQgr9L+gvRahIADSSF61j\niVZwvxeImmJiTl3tGdvRiASPtsaKhzoTFpds4PvnvE/I69oTRJFozS40CWaYqnIHYcvtRfO+1IuM\nLraddgFnigmfNr/LLFveIEadTD8HwSaimtZ2zCz4lfBzBF9klNMRYsLVh/h+o7kG8q/e+mBFgyWH\nTYcrJ+OgimdUT5dXA/2PbOD23SRGi4P6EX1SYoyAKpZ8XN1yBQBgTDkjxI41vMadIKP9AEaQdx5l\nFtYQyXtLB5gV8NMB9kuiSZyVUUjVbcxym1U+dLEwM3ByN2GzvjjKsrwtZauvAKIHcWRkzpGY+mFE\nAyevW42KAgvMPdnMUP1FHoy9RHa0iJDHmmaM2GpsEoJO01TcN2MbAGDFApbshJ0QKBS1CkUvMQps\nv0Tkgf5nZhZ8BJrAWCzD3YOlCM4aZtnqnuf1uq0GVPOxmP0RR9rd+cFuPmsdURa2JjL8txHJptYB\nAqzUaElXAES9LiHjHtos3f5QwCSatZmYJYs5TCYVSkYYKqgP+57h6PLxA8YCAKLfpr5ctTWHqz11\n79YWzHxv28sIf/O1/Hvusz6ocmlHY3ZTlp0Xcuzcmu29oUqgvvvtYnYo5w5m2/wuMLUXmO5FkZky\n9LVRMnEAEOVXhbc7b8DztrsBALV5evx8lLDYpsnc+8pEY2eV1oeog5RX1u2EERmKuT9prXytzG0C\nbxuuA4+AvjZdyevzBoqRkAkS5kxdAgCY9yT3VVsWddJtkm+UDYQc4IZcNZN2wnOM6E61BvBOY1bd\nIEuQtjRuZQwLrMZjd2zFolSWJ4Z964ftZ4jScg+mjbs1njq1c0kPTB3JEdePvbYWADDnx3EAAEmA\nC1o2L0CxkzpUlUj9fvwwx9oHCDRzYGAdnF7a5WbrqYtpU5l1q2nrws8Z3E8dIhM3dC6bmO44xGzf\nPWcT0WE2G7+HiO9xYd5NMqKBk09NCHmzrmxe73s9DF49+WdtTaUoyia3QgtleA3UJxcTqyjpJkoQ\n9vL3ivY+NGub/x/P2HqhNQDAKNAOpS844TrIG+jEfRxB3EMrerqgqqSe+53l5/DPE+UOxdTpzMku\n+FuYRffTueG73Lh9VJ+GY4yDxJhrlVeDygDuh2bBs5rmgocFKliPE36p6UkUy5HsBACAlMFr4nrk\noTCA8vD4CbRRFn2blKVElmX7tYZVdB2O/pk2O/1hvrdN+2xczOf5xBNGO6mr5LpxC3S2JtwO3yk+\nL6CxT0wQJFs1cB8IQfPhWQCA6oVxcFUTzfDdY0QpRHxEfSvtJMHFP6HyZQrC/DFt45k8nv98PhVk\nf8H/DPJ/52xWL1RO4TUxoVU4eJGOaIRTjLmvFmikc35Ae/o3LT64PsCB6yJ2J+2vT6+GVZTHOk6G\n3yiBbqwkawBXoIzC3tS7ZmvroPmJKOrd75LP7mjRImJuE2h7UWazHua++PZJthoI/4VyVnl8sHcR\nyFobr2u+hrIoeob+pDsnAKe/YJVD6ASiYLITeG5s9WwFrjxOhGCrt2mXS7+kXZVEaaX/WT1qWAUP\nzZ+cQaMgVBRSSCGFFFJIIYUUUkghhRRSSCGF/iT9pQiVmmo/bP+lK5xRrMF/4sedOFDBmsKqUkaQ\ndKKeMWd4MNZ9wsx0zGRmUd548TsAwOupowAA76Ssx9xsZsr9R2QBAI7vJDKl1zTWUI0OOY0nI9gM\n7pGmTwEA3CkMO/n3qYLudWZLe05hvfMv+1nnnD6ZEbGw2FL4rWYEuypZB9mhNG8DAO15RtO37FuK\nxN0PAAA0x4giUYnEs7pjDcIDGLntvZ+12ts/YnZ85PFHAQDSJTM0IoyrdjC+92yb3wAAb5dTzm1b\n5uD8VSJLjJmMFjsKmCVX21RYuod9ObqOIzrp7AbWNYYPFaiUk9G4msP1Zapt5DWN/5UkwCZqQnNm\ntEXTW1nne71Z6UYzM3EahxqOUMrGIZprVHSknkbey1rvS2fjEaLhz50GUff2prJmv01TRoJL6vzh\nq2Vazv8y5ViiYxZW7uxAsBixO+XpLQCAT7ZwZK++LTO8xh0BkEfyGe7DIZAbeTjYZVEjZ0Qg3KJ3\nxit3rMObp9i1t1McM6xpeeytoLEBwZeomN2WEiXmCqIMY5+nvF0VAegax/TYdWRKcEvWLHdYmA4A\nSD/bGRqnGGl3C3UwSzT71tglRCyhDHPGM5urLaSc75/wKwDg+6vdINeIOvbLhvphRAOn/NogPL93\nHNRV3I7bzzyD3bvI06JdtHt+Q4gGkVKDoanj/jVrKJsyf3KW+6R5K3Vr4OTj+DmN6AS9ljKOeYMo\nvsKt/Ls92YltVWywUtJJZMAL/917wS0QMYUDeH3IJjGeV7S96Xn/KRwuSAAAVGcFwuNo3I0US+oC\n8MnRIYjeQv8ge5QP0FDf1A7yd8Ne9k0YMuk08kTDzLn3EJnSdHQWAMDrU914TavinmWPpX4b0qk3\nXqN4qE+FXWfp75j78RkfDeZ4+4WPjIXHj/qVPYb765Vq0VcniJ/r5W5bsPwZ9qUrb93Ix3teJwmQ\ntTJ8rxMqcu1hQFVAvWiZQORB7pYEAID9tmqYfqMcuw4WfRqiyWNJxWva+tegW1AWAGD5Oo6m9hPo\nL49AExp0blgGMJOak0ofyhLPPe/euEvY/QGRThWtBfpQ9NUPP06Zhe2QEDKVqGD36xFQFzbujVGS\nAZVbQuBVZq5zn/DAV0LbGLGInYWrvyZKyO7VIW4bzyOVnURPqCrqjSGJvutdkaewsAf3Olu2QMjW\nksdZi5ntjgvORsYJprUru1GGoacpr0vRkdDp+QyfGLntEm5oUhLlbtS4kbuHvQczRwtbuuOmWdGg\nSeXvgaFfGQp/SgAAtHzyMk5vp71zCKTRgmVsjD/96+k3EMvVtTSQ5XeQj0bRmH32lLXIc3EfW57P\nnilZ91BGOtEw2qBxI3Mk79nnVzbbh4F7oCtQDdUVnntafMOzRt3b9JOybuffF4xfjJkr2PA2KN2H\nAkc9MKIBk87kQlyPPNQspR9T9qIDdid5ZdpOnQq5h77q1SZhaP4l9eTtZvT9NVoKVV9JWRZPdaDJ\nD6KxNC9H5hjqq7ec7wm8okL7x9jE9syX9HEswp+59EYTSBoaYGk55aoSSJf4jjynZGmiYCilfrsC\n/1w/qsZteRVSSCGFFFJIIYUUUkghhRRSSCGF/hf0l6aV9GYXEvtnIu1IAgBg6TfD4OrFeqq2kYzU\nntnPrHZ0n3wUVjL6f2UHUSzPu/l6xz37AQCP/joVYUcZE7JFMAJlEOiIg7uZidMM8mFkMKNVpV0Z\nhewisrfHLzdDaDJZsHU+kSnDnjoAAFjzC2vrBnZPxwE7M/U+rQS5sQMcZEDySDCJrE2rz6ejywh2\nxS5vwshj1klGI4OMThRVsbBRe4lRxenvPggAeHf9BgDAc+mTcDhNzLgOZDRyTSFTMJKTsr14MuFG\nR3tPG2YNLEYh6N+CoRcTZ9IWM3rt5lAF5JYF/vsz14pa8+FEO12ec5N8aOikkqEyeuBnYgjdlK1H\n3xC2xf/6LNe+2o/yaPFEGvanst7R6mW0P+gsM7EFUdRRVZgDOg3f3z2APD5znD1ZKjdRIOqHS6G6\nzLDy9WlZLWOZWUs9EQ+PH5Vry0T2ITAO5e9uG7NClW09CNnIDIOjre/GJJrGSip/D0y9SxG8mDx5\nP28c1O2pH8cvM9slWkRh3D17sM42AABg4GAz6CtEBieZwnBX65FeycxsQBJ7ZticRJioITqgO1VI\n7E60w5WTzMil9VkCAOi9eTo6v80MoP4xZgBrEimk7dtoXw1xBoTmU3fzBjbuaRTXSWOVEL5Pgwqq\nC05/0QHPP78eADDnFzE+tUZMFmleg5JOtGubJ5Gn3qdE/zE7ZbQ7twVU2czSaVKY6T7/PfdDZ7IY\nh2xy4ddU2ktfc2ZsVG7RQ8Usw5PAmvLAQ7xPpRh1bcrjgtq/qtMNHfbrXw2VrnH3bYBPgmRX487X\nia5ccGAIVGJ8p09MHhSgBVyrDUXRfcxsOzoRbZS5OwEA4FckJhb0dOHJbhx5vsItJoLEiv9dImLW\nl23B+8N+BAC84LmHr+vYD8c3WkZye6LNWo6joDK/YP8B2Ua9//a10bA+zHp/W62CFgO4L/kVqHB1\nEnnU/BsvikXNvu1D9mJQC3dlXOIJLL1EdFjFFNpgfWe+JjyRBgA4/VtL5OXSFmuFO+IMEsiUct63\n7Hw4Xh+9GgDwSs4YAEDUdMpl9+IWKOpH3UvYwFfVbBrw0gL6WY5QGZqFtMUFkz1wZjZuJ1XWynCH\nu3FNAAyaLtRCeoXnC8dm9jIx7uZ6bzI4DyVF5ONTSb8AAH6cNxwAkDGVe9eSd29HXUdxc4PoTRVI\nWYxP5Nli/aq+UHXg3lsWQ3scvZzXh2wzoKQ3f275EdGg1ybxc1TE0q7XOXRwthQjuIOU3mIAoMn2\nIvyhGlx7mPbuWHY8AsW5o88IIkRmfiwmDQ4pR1UVzx+GC3w1UhwwD6ePOcY/G7fP4H464Q02OVpy\nkugvpPOa/JP+eDuUvRjHvkFU7YLTRLN4/HxI+on2+pfm3E99LJCAQSBm3s8aDgNBvahMkeDdefN8\naMjkqdKhdEMsEqZxYlb6tuZQd6VPUi3sqPUg98Lxtx/EsOWc9DrjM8p14gPcT5c/xD0wPrAK+ZPF\nKORiniW0FTyLeH2UgdcA7DpIZ0ofw7/Zo7gXB57QwSH6VNmWcTyluxl93LipRNYXxFgQspd+j76M\nunjtD35fBaGikEIKKaSQQgoppJBCCimkkEIKKfQn6S9FqPh8EqwuPbRWRo1MBT7UZTHKdLyc4apn\nR/8MAFi08HY4OzI6tGXaRwCAmVPYhyNuCkOAI7qdwW+VnIbgx/JWVHXiNc1WMJJ5vn0kjv3ArvZS\nPKPK+Z8y255Q40Xxg0TIuK8RSbExk5GtkHO8vnaoAUUiiDl76Ca8/11VfbCi4ZIK8Jp8qEphLK7Z\n2hpE30meHD2XyPeEMiNWfSoUvubMdM6aSERKkyl87yfZQwAA3fpfxqET7PPQ5CDXRflhZluCRbiv\nvLcbUg0j/AGiHtJl5rrxBgDWWF6nq+LrfWM4W/6X9wYAAKrGWNE6nFmdAaHMHP1ys3xo4BThV4tZ\nnXdg4VLW0DuCgVANdcFykNmbKpGVPrK7NRDKCG+KkZmeDYP5XpfoiRIVWoVakf35QjcaABC9l/1O\nvtqyGAAw6qPnEDWKfW2qf2S2r/oTIesACW+++g0AIOcxIsJK3ES/LNk+AAAQGlcFexOuAznXXC98\naMjkcWhQkRoC/X2URcBaM4piKLvQOOpZuYtNL1atHgDLsGIAwGNNmZ15bwX7N5h1RClJRg+03zPD\n6g0UExJMfN1wiqglnV5GTTyf4TPSRrb7bAYAwNVOxr7PugMAJn/HPjircok2yzrLuvKYTgVwfsVM\nuTuwkbfAF6QKccNvcgEqBaLOWm3CZ5/fCQDwtqctHZvCTOiv3/eEyku+x36RBQDI2sb9rWgUba3G\nqUUgQYNwiHtasnif4DFcAyXbYqAXidDgS0SoeMQEhLzBElQCTlHVhW+K2EVXoZbJJDiDZchaMaHi\nckCj7y2m0nphbGLF56f7AwACLmvgX0DdqWhJ3jTpyxpt+xdRGPjicQDAg6FE285YPRMAUNaafF47\nYCFeuHYXACDoHWbLtJnMtLbdQGfnxKp2eHHLvQCAlC+Y+c4aRz3TFUjIiaPu2+cxAxe0nXrrEZlB\n/0dzYV9Lgfo6u+uFDw2dfH4+2DvbELKbPK9K1CBmB21p6qP0OcZ1PwQAWLxvAN66m8iSVX27AgA8\n31LmI0OZaT3ZMhamE+R7+D72QSrtSxnpa0QNv1uNefNoi9X9qcM5ExMAALr1MvyFLS7mI7At6QcA\nwKDTswEAfoUSqprzPXEbVKho5C4qPBI0ZVoYSrnucwfJ8JwTk16M3HNiB1CHrEuj4RUTPZrr6COW\ndKTs4wRqfs8Htoom6wAAIABJREFUGzDgApEN9uX0cUK20I/8eWJfAIClxAf/vWKCUDPKO3cC7aqv\nTkJYLBGf2Xfz+jtHEQl/vILo3bLcQMDI9RCxjT5u1k0zomGTO16DgnmBSA4kuuHapuaoGcQpO0cL\nKbRW47nR9Qi8hmt2Qg92+PE8YdhIHzE/l/5kRUsvCnuJ6T6v0Z/xb0F7a4sRPVTKJXy7lzZcFcy9\nT3+Z62H8uD1YouF1xsu8zh3APTCqD/3aWqce1u7UYe0VI9DIQbheg4zqlh6cvcR1rmlrg2Ur/fq4\nQzwfWOeRzwPMl/FWJs8j/3qUvcBmb+f+FpBGub3+9DI8+S/6m3a2R4HKTb0LjSLypdweDClE9Ekp\no+xaLOfvpR012DXtAwDAIOk5AIBL+KFnS4hY0Rw1wyF6p1SkiFFs+//Y9/1rAyrlWjiWNYEviR82\nfHomgibyI5QNpIL8tI4YqlveO4wduVSMe+c9CwAYu4AH5R3lhCrnfNkC3775OQDg+SscAVqVJRrI\nPs1dxazxoP3E0wCAnemEcpnTKUDrHDs0v9G5nzCF916XRee0ZBQFsO1IewSfp8DMwx1QSY1bQ7S1\nMmJ2ysi9nQ6Y37xirD9PPKShmLK0XBWlVU+ewvYMynDpa1SUol7kpc+fm0fZpRiMuo/jsfddosdQ\nN5BGM+4LKlHIhFKkXyAs0zyOm5zhGSqKZ34dMvJoSNffswAAMGop10vSI2yyWnqsKWzzGJ1psXxv\n/TCigVNxjQVzd4xA3AUR/IrXYu73PMQ5unPtx60iz+76aBu2lrCEY9FLdPITnmDZXOlyGso7Zp3F\n5tGUe/U6Gqa052iMxj9PebiaA9ln+T+kUI969OfI3UPfdMLjm0RzY7so9Ynl55h0C63Z2aoYqEQD\n44+7fIVRi8rqhRcNltQyvP5eGH7lBlXaWYZGjM4ty2eZVOhJyrDTo6dxvIg2dkMJ9dVYQhmcOcIA\ns8YN3PsqAyGf/8SmYFoBm9V0oUPotBqQL2zs2J7U2+t2WvdbMLx63nP+CTZgVJXRqVXFESpbcDQK\n6paUb/gRyjL75jnRsKlIA3leOALDaT9VXt+NchrjTspzc2EPAMCj07bgRHUCAODCxwz+q0QcO+AI\nbaLKI98YQR5wjfIovp9Bs4BlPFjUdfeiV0ceCjI/5r5Y14RrxZIKhDJ+g+oXhZMygpDo3X0+AwDc\n8dZsBF3mPad9sxGv/FBx02xoyKSqUsN/YwAsLvLbObEMBUXUQcs5ylCez4alNQ/XYHcuBbRvKfe8\nWe/xYD7nPMsN5uSNgH0hbeXMpSzrWfACy3oKS+jb1LRzIuQQ9at4Lp8RpGXQRfVVGBw6LqLeMQQs\n+7cSTqXo5nckqylMKupidAzl19gntkoOFTSpfpCFn1fTArAOJW+lIvJq63Jm2NY88TEmfctBB9cb\npDsHksf/OsGG+r4yPZzTuU8VnGcgxS+Z8isr5IEvIBUwFdIWVojmzuEneZ/KJB1if2ECMXU25fZS\nHhuPa2xifOjAMli+Y/DM+VhFox+bDI0MT5gLyT25s5TNaQqbsK3VLShL1UYxKvnJYjiKyLsPhrPc\nyvoaD8QBX1JerVOmw9acZwYt3SCUdqf+mnnWR1FPQNOB/k6TjtTBlMe4hlJnNkH5FR7q/bpR9gdf\noz1v/RoDb5mWEJiO8XrDFPq4+P5mGdGwyeuVUGs14mwB/RtdNyuMh0XJuIf70XkzZXdBToG9NeWW\n+DnXv/lDNtKvPEBZDft+NlSJ4mzRh5aubCsHWASfoy5VJ8mwxIkhCCuZjHCZKcet+a0QuU+MsH9j\nFQDgyxd57mw9mAG6nw91QugpvqfJA9dQvNJZP8xoyKQCzFeof9bmEso7UT62COqEXZSA6xK9uHqF\nAcf5CxlI8ZtAB1RzlmvgufSxcI2lLyqL5sPuMtGGQCQeKsYBLebznoOX7gMArKwcCgCI3FGCiWlP\nAgBMTSlXRyQ/T4CBssoPk2HOFWWZpX/6qyqkkEIKKaSQQgoppJBCCimkkEIKKfRn6C9FqEghbugm\nFcN9mZHfs1djEfAZI4bqbXyP08II8pbMVrCVMAqJVoQib5rL5kAVbRk92vPuR7j1ODtPRc0XHSon\nCahOCSNafj+rcWoaszg9mhKxAIJakH4sBYZejEYuOd8TAKC7xChX/EBCuDLLolDRkdmDt8+NQKEt\n96b50JDJL9KODq+cRtFVooQGh6ai4GtiiMsEn6yiKdcvp9phcg/CYzdMJD6raxgh52d2Mqs98+F1\n2FrGBk8PPMHs+LzDtwAArt4rMqYb4mBiEBIRXVjecHQ611DIyiCYBCR2wn4iISLHM8J/OZ/oI32z\nGqRPZfR/1qoHxDd55mZZ0aBJ0nlhiKrDsoVfAgBu/fY5JA9kuqXgOzbRa/cWR4l/cakfnMXkn6k5\n9TM/h7xtNTkLAPBdeg/80vlrAMBIf0LpmkczvKvbxwhwm6eqYfdSTw8fZVb87ByOh1WFAKZ8ynvq\nVK6Dag+fuW7JAABATZIH/xrI0rHbTj2EDNvi+mBFgyWVU4Jflha4XSB1akwID6E9s26jfPxKaTt3\n7ekAn4628WoC+dx9KrNj+3ZQN4M7leDLVEJarzf8fe/hbwEAM3ZPAgCYrmrhbE+0yaYtzLJ5/Hlf\nuaUXEaJsT6Pnc9U1hD87LLyhWg8k9GR2KKsNy4saeybO7S+hsLcGEV2Z2SzfEwlbM6ILTBnkmyuO\n2ZPPzg6A8Qz1wjqEWRijhUiRaiuNpKZAB08AbfGAGUQR7fqSsrKHivLIMBvSFtGGV/Si/HwBvF+L\nbzwo7iqgrr/xVR3M9/T3PgEAkDv4II3mOnhpxzgU1nxSL7xoqBQQYcXgpw+i3E2fZf+mjghkIg3W\nePJOa/23u1WXR3SC3Jp6opNoIzUavqb/lASnaFL86poJfO/d5LenUvhFTjWstzCD57PSb5nYlmi+\nNYahMH9KH+jQY00BAK6LRMz8MJGyOnTgSdia8LNNiiFS8PDNMOEfQJbAOowYfQSbdrJ0UYq1wVVK\nHQg+S7vpV0aZjd3zGPRCpGoxHjVxkUDethPNnDu7YdHzn7YS6t7ZSSsBAK1F40WPH1At9tXQ/XxG\nv3lE0q5bMgCeQN7L/xz1+6BMf0sK5bOCvw9GYU/e2+JVQW7cIGqYDQ4MaHkFZg353vyd41izj/bv\nOhKhoB8FF+xTQVPI84FmEc8iJ5otBwD0NjwMAJjZajcynURBdzMR7fXsTqLFQkfynFBXHAKDaPBf\neJp7b/6b/D35zVJUdeL15XXUwZLJ1FtVFf1YbaofnMLGVm6Jqh9GNHTyqOAr00PjpMzkahNs3WgD\nwzbRr3CZqS/m/sVwVNKmhszlGe1oZgIAwGehnnRsk4kzWSxxPHiZsE4plPukOYvPeOL2LZi/exgA\noGYk99wW0SwFK/8+DkGXaNTD1GwaXdSdz79cTZnD4kazh7IAAGd/TYGjRl8PjGi4pNL6YAqvw6ge\n9DV/ONsVURFEaRV6qBPNl1MGj+Y9AgibVtyFfI36lntd7lC+x70lEgMn06e5Wsvr0woSAACuQOqx\nHOzCtdkCFfogB1x4e4g1ZNCitCNlMva+PQCAjV+yxMsTz2cGtCpHpY0INmdTgTD6g0cNBaGikEIK\nKaSQQgoppJBCCimkkEIKKfQn6S9FqLidWuRdDceYvowwbV/ZAzXNGdMJHM7In0vNSFS8yYqSbcyw\ntJp6EQBwvIhIBjVL5TDkyGNwFTF7UJ0oYkMaZvYsZxitsoXLqBEjJ08fZL1c8nDW1pkzVKgFo5o+\nHZ/raMGIVGEVny2bPAg+xixhh6mZKNc17pq4GqcBv2a0hKecWZO5x29B+ARmyL1ZzDjbxHjHh3ru\nw6JDjP4Fn2LEMK+CfJVaizFjG8dAjqNA03/iyGwkMQMEA2Vi61UHTzkj0mUvJwAA/Hpy6ZZ3d8N0\nlfKpGsT7eDewT4BZNG+0xhshmXgvXavqeuBCwycJgCTJuGUpG9vJBhnuydQZ72es2Q7XEQ1k2u6P\nFvcR3VW9jX04goZQXysdXAd1OQF4I5KR/ahhRCBUOymzoCrWl+7Z2w5de7OJWOIqZhqu3C/Gderc\naN+cmYUf5/A+xim8zlBGOT79yAa8uZN1ztfGfIVupkbet8HsQUC/YlTXUQa6VCNUFylDiS2HkCdG\npoYEV0KnZvQ/P5PR9x1FLAj3FzohyxJebUN00JeLWRv8rHMqACBAZNunPLQNK+exz5VoZ4OqFOqy\nRwLCHskCAFTvIMrpej+lIjG+zhfuRN4OrqGut3P04dWbY0ODJ7WfB5aOZfDTEiHiKJChcdCm6QYL\n9JGLvxu2BsD/biLwTMuYFSvuJRpDN+N7S4oioKvgfvjTOTZtl7vSplrO8z76w2aE76K+WWOZtWvd\nOQsAcLlfElqPZH+VrEW0ybUtKEf/01xrYSPybvRDim9XiHJD425qWlHtjzXbe8Mo+mPYE1xQuamL\nYadov4pGUL5JATXwtmKGOiOfWbaXj7PpZdtYyvZsOz+YArmf1ZVRvqt6ME326jXawNzKQDzeikiG\nr75hj7Kv1czIuUc4ERZC+93Cn/p9xsy99+5NbIDboftVpG1lf4Hvlg0T32T7TfOiIVNdoR+Ov9UF\nFjFas1JjhF/xfyJT7A9RxgE7QxE8QjQadlOvyst5oeVOyrH2VBT6dqOFWx5MPZtZwL45sb9SLvYo\nE0qncD8MfZ3PWLVmAABAP6AC+Qb6VfYm1MHgI1xXFT24nsra6xCQzL1wb8dl6O9XXh+saLBkrTbi\n6Oa2cAr0QdBFCWjPn8t7kr9Ji8m7zAQTknpmAQDOX+Wm2eXULACArKbeLtT0Q20N7d620+yfs37m\nfADAo6+wH0PS6Qp4gqinskDGyKI/UfqD4Zg5UjRpf416VuHguaO6Gz+Hx1+GrlIgradxVOxL82+e\nFw2ZDEYXUtrlwDuQupT9Zk+ErCe6oEj0uNQbue90D8/GCYl+xeHLRHDFbqbeFvTl66jws7hQwB4d\nhrOiwbTonblrM+3mgk0jEH2ca8WvgP5SbSzXhanOg5hF3DOfXP4QAMArEBX5lUQe3d3uFHJt7Osy\nddyvWLC2pn6Y0UAp2liFd9uuxw8lRIjFr1SjJo6orNseJgK+2SAi2Tc9ORgl06k7nlLyM+cu8rdl\nAs8C16ri8es1Imt9Ykzy+KFs8PxLFhtESyoPwtdSX2sT/lOnvurQD2v6fwwAmPw1+1/ZW4kxzALh\n5PWqgBbce+9pfRIA8P4f/L4KQkUhhRRSSCGFFFJIIYUUUkghhRRS6E/SX4pQgSRDNnixYV83AECT\nIUWw5rDTb52dUffgzYzypvYIQpgAgxzfSmRKjxHnAQAHd/F3025/JNzNbHhwZ0b4rYVEJ7j9eb/O\no89j12X264gUo9Iy1jMrc/dDu7Dyx0EAALWDsSVPL94Hoqtw3xEXcCKQkc87Q0/ggKauPjjRYEmW\nAa9HDUmMqprbazVmbZ0IADBEknfNnmKGZOljg5C8gVmykq6M/vV8ieikcB0jt1//PBShQXxPWaTI\ntFcSzWLpwLS4xeDAwNZXAACPjebEprFTWMufHaKFryvv9VDyUQDA/ijWR+ZWsUu3+kwQlo3/AgAQ\nIeBNifXAi4ZMCX7lWNbpO3wT1w8AsH1vB6S+TeRC7Bc0C9tMRBeNf307thRS5255l12zvznA6yIS\nmBlrutGNfXYW/XvNAroghg303c/eLKnHvEirYAavYgqRKXFifnXRfS7UeaizpvuZkXB7uQ6uj8X+\n9MpATOnDHgFvlLZCgedPtuD+B5IsSwhbQpvZ9+39WO3HHiixO2g8I/dwvdvig6CrESiCSUI+Ipxu\nLWOdqvGgBYtyOOnJ/Rx1WDpIFIQtitm6pVe7I+ge2lG3jzeImccMak28Fql29muAGKlcMoKfI2Qf\n5a29swJFLsrV5Wvco3avk9enQlWtEZXVlMOUZ3ZjawHRlMViMkjkIVHrPTMb2dsTAADukdyLxomR\nygfmMAvkHybBHkH++wfSJptWMeNT3JNKGX5Uwvx9nB4zM4MjW6tfZCbONsmNEhvttfoe1o9PiiKy\nbE0RbUL+oWiYREb1yVE78ZyucWfioPNBjrXDdY57zoChFxDdhUiGvW8wq910KfkVP6cCh1Zz0paf\nUIHnHuCUn+ujzGP7FCI3k7ZSa6EOPXR2MgDgi3YrAACTch/CmjyOJf/gMY6cf2zHFACAJsCFKtFX\nxS7QTZHJlKVX6G21ywhHONeVuom9XtjQ0Kl5bDFWfToPA488BgCQHVo43SIr3p3C0hwW9fXxPrir\nqSc4Q3+xzWQiu05kcPqdpJWxKVdM4xK9IH45QJnhacr19pancOALolYKBvM9wanU0xJTEJoIv7V8\nD7PrHooV5vP8XOr+FbA7uXe+V9YVRZ7d9cCJBkxGH+QOtfCJ/hXWoU5MSGKPoD1vUxfVaUQaSKpY\nXN2XwOsE2iCuDfmdX06bGfm2Gs7nuHd2Gk90+51biPKS+wpke0go+k3mKPTuZvZZeWf5eADAHUMO\n4+ulIwEAr86h7s67yj6BZVVESsR2LID7K+61ZW7/+uBCgyePT4XSOn9U/sA+ewZDDQot1DdDGpUg\nOJXrftu4lugUzX42EOhcfQXlJ4v2Ju+tvgvNxHSfnJ6U2y/5ROkGvUyZ9zBY4etP+3jtU/b5K+1A\nnfzwrhV49Wva4NgD3HuzuAwQvJJ79477k2BeyOcuX7QTq683V2qklFcegheW3Q9PayIyx8w5jl35\nRL2ee5Vy3d2a+9PcLxdh9mdE/jjEGGujWUzeqSZP2wxMR8UbtK3ZI0U/xq8YT3DPIOLPV2FEaXvR\nD6Uj/dgVS6hv3e5MxUMfEFXWdDxR9xmltOcOUXUBjQ9SHc9APfz/HH5aQagopJBCCimkkEIKKaSQ\nQgoppJBCCv1JkuS/sCW4PjZWjn7qaahZNojgizLiHyfy4Oh5Ygbifub/codLaJ7CTPXwCPZQWTmX\ntfvlAxi1Ml0wwBHCz+/1F5kWK2NECV0ZrczfEwt9F2bRD3ZeBgBou5ZhRV2lCtcTa9eDwlrx+/WM\nrBxrh+4iM8BNl2ThUPGPqHYVSzfNjAZK8W3M8vNrO2Pu5lEAAH2FdCO79q89zG5LTspA1vuwcMhS\nAMBzF/g/l4uRP2cFo4GmLM2N7uZNN4peKvcz8tg+idHktJJw3Nb8AgBg43YxWSSSa8BwxYA7xrKG\nbvvnrIPUV3MtFPYRne/TVdCPYHbuekYg895XTsqy3KUeWNIgSd80Rm7y+gzoTVRGt0uDwP1EEehq\nKQ9bGOUYdNWNVz/ltJdH1rHzvTeAtciSnrzWFOmQsInZ8MJejNar+hBhVJvD7F3IaRXKO4nIc5SY\nTiGypcad/qhqyedG7edrwGVmeKet3woAmL11AhJXcY3UvmrFhSeWou5KYaPVRWNkrJwwbRbcYspO\n6FkZRf34szGP2VRzDvld1gkwZ5LX+ir+TQwWgc7K3/P7qxB8gezU1vFv17voVw4R/YmcasSv5t9y\nbuPrXT2JOtuxuCeq2jOTZyikDkftp55eu4/PUml8UOdynQUwkYfTi55p1LoYkBwhd/9yAq4WEpEg\nSQAKmV1tto58zyIgD7ozJgwfz1ksG7fRFrrDqItBEdy8qmv8YPJnZkzaw3puRyjXxfVJQlXbIzFm\nMvtv/JTBTJE9i9m/5MUViPiWGbvCx4jOLOtEHbbcy54R2cUhQAHl6NPJKPzwYzhzchuvLiZGyc3m\nPgRrMfcXyeC9MfGloA/9B0m4Ws5OVkQtpXyrm4r+Xx2pN8HHuT9W93fA7C96qFyiDLW1ZO87D9CP\nmXVwPAz+1C93BmW35G6OMHzqrcehvpsIvgqBfGrxAv2g6q7sfVPYW0KLZUSHpk/h9VlPPqvo4pcT\ncOUa0SBNdqvh1ZHvZZ1pEyMO83drtAqxI7IAAM/E/QoAeOggEUKyS0yp2K5BaSe+3x1KGcf/xN9r\nHqO+9ojMxm/pzIarRA/BuwSiYltuSzgEwihiEfWtKpFZeWOZ2EvL3Cjow/UUdtaDM7s/gbUyr9Hq\nol+LSDn546kwLqHehM7Mgk5NG5n/Cc8Zxd3JntgdXhT0ps417U1/89oR2jxPDHVLm6OHxiZ6gAlM\n/8Sx7L3hEOPwds3pjcLBYlJXFd90vbfDyj290bpjFgAgex17i0XtEU3JJN63pJsFNQOo7x4Hr895\n4IVGrYuJbf3kjza0wOvzqFPNJqTjviZHAAAfvUhUfHUz+jm6ahkVXShjrUA1BOyk3bMPo56FLDdh\n6JtEWO8tZZWC6mWukZyh/uI+QG1n7p3vdl8PAHjx4F0AAP9UHa4XKNRFy+L9lN/1/fWTMd/hiW38\nvH55amR+Nw/2wsa7L163p9d7LcprQ2GNE0jNXyiXa3dy7wk7I6NgEG1aUgue/ct/YN8pewSvUbkB\nbV+iTry7WN0SdIV2NW8g9SaqXRFcy4ns7fE0+7ScKSdsyfC8CS+s5ZS1Jz99lH+roOz6Pc219fPG\nnnDE8J6SjjqdPfmlP6SLf+3YZB/Hy1k6ieZ5cf5wrORG0uwOBkAGv0vI5LbX+qNDT/7t8+1DAQAq\nvhUPdjwIAFju1w2+fCpNVCKdh/JDxHe1DqRDWJMdA3dXMmxlLQ1lxjiOim1/7F6oVRSgfh1h66X9\nyMhxHSmIoy91Rc49VLCr80LhfOGvrZL6u1FFjgVrZgyDRgQr6pJcWPQiDU73ZwmHfCdmEwBgzKfP\nYX0nNkVMCqF8rkPJc0VApdnwa7gkxhtnz6QshiSwWWV6NQ8Y0Z9r4ZvL500buQPAvzeyFYX9kWKk\n8u26k89Qi+abKgHZjFjiRmYT3mv6HXR8ZtcDLxo8yRLcTq5nn0sNUzH55nuU+jlFjNJcdLk33rjK\nAJrchLqwvg91aFcdG0QtODYIJV2oi9MeEI1N144AABhF6Z4tEohP4YHuzijee8knfE9VKxmmPDqh\ntffR2Qiw0Nl/J3U4ACD522rYo7nxpQSVIEPduBth+nQybHGeGw2f3SYJKV+Qd2nT6Ch4jdQbtQNo\nPp7B65OXWJaz5VaOTx3xGyGQgadVKO1BpySxBe1n5W5uaL4KOuyGCBu6v8vSyz0RlOExJ+Vw4PZm\nSHyH+p1xNz9Tz7kMtvie5xjSop5GRPSivta0ESMFF908LxoyeXwqlNn84HNz/Uf+qr1xiJu+9CcA\nwKsXqH9BqR6cqaRzcD0g5RYNTmvSKXNZJ8N7UTgwfei06I4wIKJ/myUpbd67jGVHCX/vmJIFACjx\no0PfdWUa1qxmaY96CJ/Rfxybs+1fwXKFiAIfCgbRXkiuRusv3iCfUw17RgCkcAaoW75egbI+PJTP\nmLIRANDfj/vjhA+fRWFv+iTqFMonOZh6W3KKPoo604DaZnxPq16EJl/fJ19cTth50oBsWF3UoQdH\ns5ns54WDAQC2SAm2PK6H5K8p144bswAAP+xjQCXwsgT3h7SxHfXFAICsm+ZEwyYpVw3VLDOkB0VZ\nVFMV9IzrQ3W9LHwCHfouoYUYHcIS5Lcy2BRYKhejO/2oG1WJKhhTKFuz8EsqUwgxl/fwQHDYEQIp\nRgQ8O3N/3JzFElvDZgsee4b76dftWTYSeguDmvknKEdVcxfcLvpOdV3q4DvvrRdeNFTSFKgQ9qYe\nadPI057mEpyvIq+KxfhUdYxoERBpgiuCe97Vk9Q9bwh/N6YxgGWPcyMukXK5mk6dvl7yrCsX5au3\numEMoG9kl2h7azy8PvSUhAtalinEpXOvzHqFflfsxyIgUCsjJozrZH4iE5Sd64EXDZmK8kLw4XOT\nUHkr1/OZo4m4EEf+B2kpR6kXeWb6zoxlt3wKALhtOxPm2177CADQVfg33gl2/HiVXDXqKAd9DGUV\nPyAbAFC7MAaSTLm9puWee705eMCgCnzXkiO1Z9zLkee212g/h0XxzPLR4/ehy6ssE5lw61E8t7lx\nN4h21ulw7UQsdFWUl7OVjKDL1Mu0R0SJjVckHgYBnwz+HgAw6zhLX+WB1Kn72tKPXLmlH5pbaJA7\nPkA/9PD9LJ81FfC8V1kcCedt9InOvMb/zfqYpXZpyyMxbROTwiFVIpH/KGMOP2/oCQDQ2IBX+mwG\nALy79Y4/9X2Vkh+FFFJIIYUUUkghhRRSSCGFFFJIoT9Jf3FTWkBWA84dRAuEF/lgZQ9ZFO5m1u07\nM/+Q/FQWOpuyAADH2zC6615EGM+qImZhLKU+mLMYwcqczgycSmTrgjSMQAdeqcOVfEauTkYmAABM\nKqbMe0Rlwe4l0uHIMGZ6gnczy77Gxozqg+/vwc4SNrW9lt4EsqdxZ+NcQcC1sWr4i+xo/DoJtY8y\nSnz+Z0KIhoaz6ZBfnyoczCPE8d22hM89uYtQPegZlUw92BShFwUWeiKREbv3tQOAGzBLdU+gZAXh\n7bXXRyqLMdcGm4RCNzNx9l/DAQCVvUQ5SSkjzVeflWAhuh1ZjpCb5sE/gfQlPiR/br8BOc251YgK\nLnOEfUweLU0RY5DPOxH3DmUzIZaR4mensmmfI1iMYW2nRvRPWQCABc14naVQjK1O4H0D0oGKLdTv\necnUZZUo8zGUqODrwaZSsS9Tti9vZv3ftOUzAABpsxxIeYYLb8+Zlqi1Ne4Rn/BJUNlUCD/KiH3a\nwwGwh1F2Ac0FvD9LNGZ2SNCpqHOGAsrs2R4cv9p2FRv0le5OQMuPiPIquoX2WB7EDEzsMmYTcm4z\nYPVhNgHbFkV00oMtDgEAnJvC0f0TNuYrXsH3/Popm+TaOwpobIQHJfuZLbRHeeqHDw2cvG41Kgot\nN0Yam6/VIu956sDCSSyVtN3DspHY1DKsSFoFAOg6iFmy5u/xunzRTM8W40XoBfI2N4r7mdyKyAl7\nJjPoR84kIeAKs6PpVwh/Rm+uo5W/9EO4aIpZ3JX3/HUXkYYqAW0OHlmIhLm0t7XRapQ08p6mGjsQ\nck5CZQrpW+GtAAAgAElEQVT9CG+QCRXsd4hNo6gL5l/oq9giZRycwuxptw0c0Vqyg8guSyYzp5Vd\ngPjlYhSvOQEAkPio0FMzZVq2Mg7l3Sjnt0qJXkiMov7a4t0IOkX3zv0eUTBrNlMXNaIRbpvJF3F6\nPZEQ15opuggAjnAVUh/3h2QXvocLiNzDLLP2A4ECOUD/9UgnPY4XENWgEkhnTZQYapBOGUXvtUH7\nA+sEgn6gHM77iFDxiQStf74Pspp65pNFOVEF9V2nASo8vJe+krqXXUgbLwdSR+MDa1Gzjpl7eYSt\nPtjQoMkdKSP/JS9QSn5tXdUTRvGzVjS4j1gmGg1PsaJ9BMvBs6voR/p2Ea3uFfJJXmRHZSvuh8ZR\n3A81h3jeQD/6vkE6N+zbaQ8dHXi+2HyC2XGLRYImhAayYBKfrzlHfS8RMJTR0/Zi+VmeOV7TjRbf\n5LObY0QDJ0+wD8XjHNBdoy4kfpaJgrt4nihrT3kmmOnrexwmzMq4GwAQn0AbOPxfzwIAVINod112\nLfz3Ue72kdRFySRQ0V8SiVvUG1A7eG+NlvolH+f58fmp6/H41XsAAFmjxIjsCsrznD/92vHzt2LD\n5IEAgGcm3IeCqo/rhxkNlCQAKg/g7Ui9UcsSDIdFdUJToiJLtlO3VL0qb5wPJ/egT7n3RaJot4UQ\nEaaPkOC6k3vk+Q30I688TZkmfC9God8lIeE7+jnFXbmHvnKROuU9FgR1O66ZKjvt6skDPPhEH+d9\nh7y/H1/Mp2/ceTJR3Vl/8Pv+LkJFkqRYSZJ2S5J0SZKki5IkPSn+HixJ0m+SJKWL16A/+EyF/mJS\nZPjPIEWODZ8UGf4zSJFjwydFhv8MUuTY8EmR4T+DFDk2fFJk+L+nP4JQ8QB4RpblU5IkmQGclCTp\nNwD3A9gpy/J7kiS9AOAFAM///26kNbkR3bUA2amsBdZXqlCXwCjg3b058nbtBUZ1rxxJwCsaIlNM\nicxc194uGh6mMcVS08eHTXPYhO3zCmaBvt/dFwAQ2Z7ZtiuPaNF0BZ9xfnt7AEDE64xOnv68A2qb\nimY3KhHBrhNoCfGy+ExvRG0SWfi2akgNE6FSbzKEWoYm0AVnMGWQfQegqWW01k9MCBvZ9+SNt2+7\nyiz25zkcT+2fTl62HXMZAHDY2QLhD7MZ2PmrjFRqY5nR+bIr6+lUkg/vjmNXyzIxprB6MKORYWdV\nUI1jdsgtkgbBAbzeEsbM0qOxe/HiFdadH597vTJ15f/3a/5Nqd7kKGtUcIYaYShmRkvtAryipYVu\nNvtnLG3G/g1jdj2OzDRmsfNiiHjIGEfToStjTNYV6ULtt2KM4ymhI+Ll07FsaPv8gmkQrW8Qyz6z\nUM/gs/JKgyCJhsV+XzJy/UoGo8T+eVTGYaOPY83TzLJG7pFRbm3cuqhxAMEXJKQ9yIUf16IYtmPM\nVJYW8m/zR1CH5j07AcfUjMSHdmdGLmoEI/U7j3CspzzEg6Vz+P4385nxPlVAnVywgPXJY398GpEH\naU+9Oj5jyf3MrNnDpRvNv2pbMOMdsJMymvzMNgBAsTsAP15kb6/k+bTnOb/Ps78j1d++WCMh6jcV\nCoaQHyqPP5xZYux0F+qXyinqjl81Y8DJaQAA42na3Yy7+D9LClFkzoshyBku+ihoaBuDj1LxurxE\nBFEP/wwsXkP9qm5KvdUZacCL4/Uo9OPffCbK0S+YdqJ1BHsJnMiIh38rgajJ9ULVMNs21JsMpWA3\n9BOL8HwcGwYvunQHWvcgmq6iA+W09IHbAADeGXYM/JRdvAxMpOG+GdSPVR+w8b5K70DuUMrgzRFr\nAABvr+AY1vvv/g0A8GW3AYjbSP0KeY795vJqaZ+TEguhTqLsbR8ye+oewd8HdmGT/2KHGdYWXHMh\nx0QjzN9l2d+S6k2O6joJwSc0qBtC29iteybcd1IH838kmidoOHXAsSYC4UeIBMx9m/5Qwnze5+p4\n6mRBXz90GJUFAOgUQO6e1XMk+lcPE4Hw6MIZN5qd5mcSvaILpi4aKtU4WUUUTA2T81CpeW+fkUpX\nbTfAJBrUVh0KgWxtkH3+6s9HFRR+gHwI3Z6BS6/xLNG1XQYA4KKbe6FxrxbnWlE/x/SgbVzfkWeQ\nEa05CGFbaEdIZC8i/AUCKJvZ7YdTqIufpA+CJZO2MuIoXytaUbkr+jkQHsDr5JVEN3kNlGHi/ezf\ncLwiHpoC6nvukWZ/5Ov9Xan+/JtKFcLXGlA1kee1ug6xEG1p0L0fzw9ZNUQTwV+NvJNEmXjCxNST\nrtSPoUmpAIAD6zuioj/1qlMYfcwMf6JPOj7OfnCWuiAUreZaqfKjjE1ib3thyf2QBbovsBv32lui\nee8rVqKTjtY0Q0F/+kXxW5woq/7rhr7UI9XfOUMtwxXshdZLG6q5aIIsnAWjhntP7PAsAEDpsniE\nC6BkWhsi2E0XaWtzniMaJSBVDdtK8jdYzTO+bKOe3zaP/TUXrR2GUvbZR9xWxg4K6xj76TH+HJzC\n2J49RTusEq0YW73Jniy5jiCE3UNbXTgv8XeZ9V/pdxEqsiwXyrJ8SvxcC+AygGgAowEsFW9bCuDP\ndW9R6C8jRYb/DFLk2PBJkeE/gxQ5NnxSZPjPIEWODZ8UGf4zSJFjwydFhv97+lOhbEmSEgB0BHAU\nQIQsy4XiX0UAIn7verdVi8JD0Uhez8hS6iP+MBbwI5yfwAy45nVGr6aM2IUVP7JXSpCYPqDfwOzL\nd6/PBQDctu1J9Fz7DABg2uDdAICUT/iRPi6lrP18QNZ9YsyIxFDUT0sHAAC8UYC7hRgHWstsW60Y\nWZawmVHO/L56BD/BLvt3hlzBJ6tqfu9r/q3pZmWorlUhYI/xRqTWU66FT0PeGUsYxu/kz47Zb58a\nAXUG6+WuekQ3cyYDcH8ER8qd2J+C7EpGDzsnZQEA0jazB8vM4xxr5QoAXJN47ykD2Axl6V4ikVQe\nH1ZnCdSJCAbHmrm+LmxlFmJW+H0YPOIsACCnn0Cp/fB73/TvTTcrR5XLC2NeLfKGsR7b2sqJWd0Y\n4f32C6IT7kpkd/QBPS+izsNsy5k80fRIgEPuuo0Tt6L1lWihYzT5ZTcz35ZfOZFn+pb7AQDBNhkQ\nY+dyR4npWpWMNscs1aBmBnXr3AHaAlMr1ie7Qvmww2VNoW9N2frtMELlbpDR/xt0szL0agFrrITE\nH5h1KX9Jh07TzwEAsp6lDr13gMiuh97bgI0lDNuPCGMkfmk2+xKZs8T0ij52jFrCzLkzWkwr+ZD8\n/vp7Tn2J2eVGYS+uhfvvYnbu67OiN4NWRtlOZhKmT+I0rVUnOKFt2VwxzSkZ8IpJKHWxzADh+O99\n07833awcJY8MfaUHEftoR13jK+CrJG/C1lFPPAYazvBfVSjtQBtmi+X/dFWUn3c7s9uqJjJGdqW9\nO/MeZV7Wjjq0+TLRSFuzukLVX/QTOMZ98a5YIgsX/TYSksgU2Xtwf7y3Bf93YFpXAIA8TQN7uJhU\n41DfyLA3VLppXazRomp7JD4ys+fNLU8dx7GPuS85AyifwhnMUpsP+MFBUWH6+F8AAAt/oM1t9RBr\ntjULEhE9k1OBit3MomoEAnRVJvvZJCSUoDaC+mb7jpm0d15eDABYU94NaXPYxEX9NLOxcga/xvHV\n7FGmtcowBXENBGS7fu8rNgi6WTn6tIA9TIL+IJGwaatbwzuN2Wi9mArh9HCxd3v0NLZ2JS/VdtGj\nKIK2cewAjuBcu6cH0ipEbw0vrzOU8T6zXxSTQm63wyCmcgWfpp/k03DvtEYBheeZMU/pQb+qaBV/\nt0VSdtW1GlQNEnuhygOfrnHviz6vCnU1BsScY3b6ytPNEL2TtvKUhWgfXSf6GvZLZpii2N8hs46+\nkPk0kSI1yXzVN61Fj2jyfn9mc16fSDm9vZ59O/wKJAx+cw8AYMmx3gCAf/UlwnfeorFw9qPuR07l\nfa4W0wBkf8592j/XCf/ZRDtFfC/6cvzeF/2b083KEQB8Gglxr3IzuvyEBMnAs9z51UQXtB9PFNGV\naSpojlLPPAGUTdA5vha2ov3sf+cpZNSQ7+eEHxuTxvsdW0pUkn+hF1EzswAANQX8iNFt2JOluNYM\n7z4iYoxacZa8wv1Vc4b6quleCWeQ6MFi90KSG7cuShoZhlA7Yj6l7eu+YD/GWjhB984tnMbUqyNR\nWmntZJgzuVeeyad8zIPJV5Xoa6N2yXg0nmfArx/nXuvfgffeuoI+qvdxK4J/oj11v0/dtpXQtzrw\nW1uondQvexJt9s9DFgAA3sgjgvT4laY3Pn/LC2W/9xX/g/6wGyRJkj+AnwA8JctyjST9G24vy7Is\nSdL/uHIkSXoYwMMAYIk04r4xu7Dl0gAAgLZSglFAp7qPJQy1NJewyt+KUxB4lUYwJ5mLWCt6592x\nmo3cND4goDXLOtbPZ0mJfbxgVjwXvP8VLUyXaBidnUTD2nQqaGEvNbxOKl3gBbLCFsmvkTeVzI4N\nLcH5SzTCa0Ztxg9qxx9j2N+Q6kOGmsAgWOMBSwfKreZSKH4Yy3KAezazeeibO3igbtEyH5nZhOHp\nDZSHPYoL+4VLVAbE2+A6I5rKLuFp2/YU5a6t4efz6oG4bQy0bT/AQEqLbG5QQfPzkV5BI2nJ4HUn\nrolSsS48DM5ocQTffs9GqcGpDb/5Xn3IUR0ciMtPWBD3M/lhD9dhWSZLN5wi5uQL4P80kg+nDnPj\nj91BOcrPsGzk4Ks8lOsrnGi7gAd1q536Vi0aPSd/QzlkvqxBzELK/+4n9wMAlq6+BQBQkQIkB1GX\nX7qHo+nG7HocANB1JA3uqew4+ETJnb6pHt5zDbLkB0D9ydDRxIPSF+gUdAgrxJmv6ODb+vJ+thb8\n39yLQ9DkS8pj3hBuGImLGQCroU8I8zozxr3I0oOf3mQgpHSugGrauCjy+2qR+C3HdlrGUQeT36Fd\ntX1SCZub8l22hKUL8kg6tc1CKFtvtQXOw3RcfY/w+Vj3Bxj2N6X6kKPOLxAuiwY1Tclre4EFx0ey\ndmB4yAMAAGuR2J/a+6AR5UCRB8UBL4Cvtiai4aFVgklDuftnUTYV9/J/lp08KLafcgGZb7CJeMUj\nfM93nzHopXfKkO6ifa+rZmBnRRoDKe7x/H1st6NYv5v2whoPeHV/lGN/P6oXGRoDYbnmhekJlt4c\nKGgG2yjuZ00C6dT5tjD4UdXRjeQvqTvfVDGQEnWOfsWpJjywqTtL6GzgoW/r43QU7bdzf5MPUX9K\nulYjpJz7YqlwKl9670EAwGvPL8WRJJY4h4rmwf5t+J6Q4dTfbqHZeDWMB/+BZ1gSi1//CMf+nlQv\ncgwPgH+vUlSfok9h6+5EWxPlkC+Tj6UlTAJsy+gAScNbPt1hJwBgy2wGLFefYTAt5IKETr3ZTDj9\nFQa4avuIz6QWn69Mj7itorH4NOpn3BbKuqKl9sYI5isn6Nf4Won6E5HUMqer8fgjGwAA75+4FVA3\n3ENcveyLoRZIahmtvmE5Rl5OCkJ7cP8pPMm9L7Idf5cPGOA7SpvW5HWW6OVY+YiDGdTF8C16nJ/I\nUtrEt0T37c95yE7L55myuokKmz6nnibcXQAA+OD7sQAAVyc7gtTiLPNrAgDA25r3qUqkzS/rYIDv\nGvdny3xxZh3y+/z6u1J9yFFrDoI1RgWPUQQxQmthMZFvd0xjuOlwBcujKk+FIfowz2zxb9JfPFjF\ns+SlEwkAgFS3hMkjmHh3zqM8rTG0iVUdrwdEtajcyjXSbRTLiqrup+9jfTAIchMxaKGYf5PtVML2\nwkfNt1pgE2XtddEG+C423EG69SFDvd6CmM80uMZJxcjY3wtrK3iGkyzUiby3eLAPmVGOcpl7W6zY\nM3M6iURSM+qrIzscL+6lXoXGUXb2SN6npinrwbwex40SvbKfWYLui+MfmpzywSdGbkfvpV0dpWdg\nu2Us/dHE+GJcO8eAzpVHWKKHWb/PL+APjk2WJEkLMnaFLMvX3d9iSZIixf8jAZT8T9fKsvy1LMtd\nZFnuYgpqwF5XA6f6kqHaZPprPrBC/yPVmxz9FTn+X5Eiw38G1ZcctXr/v+YDK/TfSJHhP4PqS46a\nAL+/5gMr9N+o3vZFs7Iv/l9SvcnRT5Hj/xXVlwx12sYlw99FqEgMS30D4LIsy/P+y782AZgC4D3x\nuvH37lVZYsb6zwbC0VxkmasA3Q+MPq4cw4i+/hAj9EkTTsMzi1Ep/5mMGF4bywju7ns+5DU17VHt\n4Qb4Q2eOV5rUl6Uk3+9kFAy9q+CwM5ATEchMXMlkBtaaBlci6zARFH7FjFa5TSLsLyBcef4mBJTx\n8x5wGGD1NbyseH3KUPIyA+r5WaBCvMCs2USmDHueMPM9GwlJLrkUh8S7swAAFYuJ8rEPJurBdZCR\nyL2Pf4AFKZTdYTGONWov5VM9RWS3gyrg60G+17xPeWWITGkXAHNaciTz47dz5FbSfMoyczShfgsq\nBsFPhA5zh4ug6vrf+6Z/P6pfOUrQVqlRJ0B7fgUSHKWUaahAcBX5M9R+ckk7xN/FrKa5GzPfV35j\n9ubd+csAAK+cH40nLZcAAJdfSgAApE2njOwfMqvQM6Ac+x8kNH3xecJiAwRCbVB0Ooqd1P1xRx8C\nADRdSVmlJ7F0y88AOMTY1sqWgKcBZlPrVYYaGfoQO9yHaUOvZFig1ouGhSIKrymm7XNWa5E7lVlw\n/Vn+7/JsyrtnG2ZismuDEKOjzfXo+Z6qi9RTXXuuicHDTuNgW9rjrxeMAgCoCVJCZZkdP/RcBAB4\n/X6iHbIFrDC3mrpYFyvj0YlEwRyrTgAAHPm9L/o3pPqUo1cPVCap4Vd4HdaqwaD5LL2ytqfMLGcp\nx+r2LrhCaN+cZu5VZX2IGjt6yycAgO5bn7rReC1nOPluMlDPXEKu+zMSoetM/ZYFQlAj4MkVgxzo\nGsh1UJoTJD4jMzxqMdb1wEfdoUmmUXWGexpkVrw+Zah2eGC+UgXHe8x8Vt4DqPWUU3YGkQ2RheSh\nNV6NzDFEOWhECWTWw/zf5l4cs3nb1ifRwkh/9cAzRLOkBDCDVm6nz+NwaaGdwb850/hcr4EyeX75\n/ZC7MMuXLZpjWjjxHqU7mX07ej4cg8J6AgCOv7uQ3+P3vujfkOpTjh6PGmWlATAIWHjYT3pU2BIA\nAEYVbaAkstL6chUMnVmm8f3bRBrZ7laJz8R9sv/0o3gybB9/HkeEysQu9FFLXfQx9/zWAfmvU38i\n9NTT7HEsbw85IKN3ylUAQO67RIkW9OXzw0/ymvLWwIebORZUHVcHqeG5qPW7L0qAWu3Dnq+IoNOq\ngbNdqDMWUd5alUYdsPWU4AyjnhZlcq9ShwvZb+V5o8+zR2+UA3kWipIsUfMcEsQzhdWuh3MYbbXx\nbqKNWm2mcp872AIu038i25t9zVeXhWuqpIsGQUKeBZGW3/uKf1uqTzlCAnwaoLQXeaS7bEaFRB/x\ny0KigQJP0bYtmvUFnigj0iDzHJGXwQQcoUKMlvfZ1Pg+lUhLfSL3vusVCeoqMWI+QMbkO3YB+HfV\ng2sEZW3KA6Qh1Hev2AdtGq6R/9femYdHXV19/HtnJpNMMpnsCdlIIIRN9l2ggCwKomKlUq3bK4oU\nqxWwi11sRd9aa92qtoh9tWpLW8EFwRVQBEVAIptJICzZCNnJnkySWX7vH9+b2L5P3+eBQgd/M+fz\nPDyThF9mbu73d+69v3POPbfsedpmWLsfSTprLXLpKVj2m28r5fnU0OO0oGqSA/Hb2Cf143yw6goc\n3XrNUHkz+8hT60JULe2zehezOaO56x/RQ/hLjUkGVk3jw9szadRnaLQuw8GzDlBb2Be136K9pb/M\n+8OdTH0r5vlgr+HXfXUiWFQ0r219nM+WJ+cqRFWyHY8s5YEaV51hhsqZbPmZAuAmAF8qpQ7on/0U\n7NR1SqnbAJQBWHRmHylcAETD4EB0ND+iYXAgOpof0TA4EB3Nj2gYHIiO5kc0/DdRRgCL5sREphmT\nBi9BzYP0CCc9HI7uh5iFMLcPo9vP7aHnMSrejYlpLODU7OHeqH37GRV3VNNLbFiBjI8YvWnL4DUd\nKfQsNY/UnkGfQp9tvL5xiM6MGUEPcsvpKKgO/l/u3Ty2+fiTDLfa9JGsUSMb4M7TUdoxjTi+8n/g\nPl5pwhjA+SEiPdPIWroSVy34DADw/iuTYZvJqMr20cxWuLn4SgCA3erDga30Fn+6mFlFC5etAMD6\nNQDgifcheSe/TtAFu+Yls57OgVZ6DAsbU9Cyg6kUcdMZkav/nN87y4CoWt5P5fPYRlsr3y+atYQR\neXUNqk/T63/JABb9e3HCy18YhjHunDvEpDgTMo1hly2HJ5K3smEFnNd+dYQxAHwwlcWabij4LzS1\nscjTtkmMZH77brpsx/xiHwBg84YJSCigDrWj9VGvgxm98ZYwEueN9cLqZDR9RAYzXsrWMmNF+YH2\nS3m900FvdMdnzKCILtfHYkeq3qMjcyeWYdcdf0dzUU3I2mJ4ZqaRvnI5rB26PsYJIPt23t9Vj7Nf\nK+ay72yNNvii+PWwEbQz7zJGyQ+v5OvAO/bDu5mRu/Blen/qH6ipzcLf9f06GdYufn3iNuqcsJ1R\ngM4kBZ/eP+zuq8+i0xl99nrapGVwG9QBRpk6U3UE/3s/DGlbDM/MNNJXLMcl32ANor1/Gdn7f8Ys\nhmjaWjm/xcR0IEYfb9zUQZsMX89otjuJerT298F1lP3tKmd0rvlWZiukruI1RXdEwVlCjaMqqWfN\nVL46j9uQOb8UAHC4WB9XeIga23RxuMT9baieTB1ts+pRtPxFdByrCl1bzMow+vzsHlw5bj8AIO+x\nsXCdYIS68mfU4LK+rOnwev5oZKUyA+jUXvavXdcLm30t00gKlg9HyQJGP8Na9XiqlzRRldSgPV2h\noy/fO5NJXzg9hJqq8c3wHeKc1xVHXSMzeQ9EvUl7NyxA/QS92VwfDVp2d2jbYs8atWw+bSqsHTBm\naBssY38m6Ay/8EU18L/E7KP6BczCVCeYCdGti3rbK+1I386xsCOJ2tQzgReTLub9sHfbEIQ36KyI\neazB0/YKx2FnRTfKL6XtxbJMA5p1LUHLAM6XRpET/d7S2UjzXShb8wQ6K0+Gri1mZBoZ96yAN462\nEVUcBp1QgD6z2L8tnRxPW/cl9PZ9xwQ+S2S8wkms7DraRszn4b1FpO+5jkH5R/NYIyzlPWpTPdsL\nR6kuaaBNyq+zRbvj/Ejox3uosYjZpD4nDS7zvZ66Vz7YT3Ncr13Fdh+88lchbYuO1Eyj/y0rkTCb\nNWnCf+HC6eHMeu4ZC+vH6zkrowVWXdLDr8Vu1fbaU+nD1a8Jlne5th1yC7NyD780BADQOIzvE11s\nhWeqLlhcT1u26ucJn9MHawfH4qHjSgEAZW9wQRpTRs08DgvqxvDzY48ARzY8iY660LVFR2qmkb14\nJRIOs398dgvqRutn655u0bsHDAvQkUaxPCkU2HWAc+Cae/gscv3mZUjP5vNm07Y+AID2Qbw2qoj2\n54386ihkdw6fJeyV/D/lA3wDOFbHv8f1U/1ofd9E8B4Y9EI7ipbwPhs/jMesvz5lzRnZonkr5giC\nIAiCIAiCIAiCIFwgAnrYoTfSivrRMXDvo2fq+Lf9iF1PT9Dm47omxzC95z/Ojt179YkVA7S36gQ9\nhW69781nN1A1Re8nHkXvcuQefm9p5p/mj/Ei525GAnaXcO+/ZS+9lH9a/Bxu330LAGB2Pj38DaXc\nI9exk+1pKo9F2mRmRXT7rLD868LGIUNYq4HUT7uQ8i16cWOPeZF5Lfd6X34Xj8HqOXEg6YAXXfPp\nKhy/hUfwWmbShxfeyHvAAyAhjx7HkynZAIDnFV+jT9Jj6E62wKadmXV5zEzJWcsq6zds3IY1P14I\nAJg4knuND1XpqOoOeiAnpxzH32vpXCxuTTjnPggGvBFAwxALrMOYIeY/GIOGIvZtzBHa2fwTrOPQ\nHWNg0jiGx2av/hEAwD1TV+j+ktH0qZfno24jo2opK6jN/lJ9wpOOvn5v/ha89ktGdo4MZpjNewmj\nbPYDTiS/Qr0yf8JTEQ5NYDvau2mv3bFGb9SnaU1f+OpDu8i1xQM4qizAFGbcRW9z9I5PbWm6GoKP\n3xthBsKadfbeb1nPqOYBeuqT3mVErv6tHHS/z3Hvh5vWAQCefYRHALX21QY4GUgo9H3VAHwVMY26\nqAHhb2ut4nqiOrxP+q9nhK54URzSZjA7qUofmR3q2FsMZHzow84m2pJ7mAcRp6iJdRf7099Pn04W\nA5QdZWTGUcE+7mQyEsJGsY8jvFbEb2IE9pnnGdm57tl7AQAtg6jHggl52J7PmlURDdTTmaJPWet0\nwf0b2vIvn2JEdpXBejlhdWyX3xrde0y91WIgZENwGlu7QtIuKz5M4l56/8J2NOusLncT1ySvl7O/\n75q5Bc/s4f7vBL3Pv4MlULDldV5jHwkobburb34OAHD7Bh6VkPYpo27zV+7B6yW8ZxoG8T5ZcuO7\nAICDrZkofJOnXNTpuJrnMO0tolFnCP+4GD/vw0zTFytZc67sXDvC5HS7rKi8JBbdQzk2OlztMHTE\nO1GvW926xkbC/VEouk2Pb3/UNYt4OBqUrg/gizDQ0pc2M3gxo+K2x1kTrHQXXyMyFKwzmbE0K5nz\n7Mu5tL/GwXYkHuJ7tSzkGtXTQtvu/xznv6o7W1Fxkc6G6HDD0JHWUMUS4UPEwGb0eYLRbY8TCOvg\n+NnQwH7t0TC2xI/TNBOkrw3T13OejDjG71v7Gcj9C+fY38Yx+9qI5dwXWcNXe5UdEXVap28w0yR6\nF9czvgFdSLqVY3PLPXr9aeiMzW7aYnOOHS2L+fXU5HIAwMFz7glzY/EAkVUGGt/het42woAvgrop\nnQkbv59anQ5zYuhvuO7068LStRN0JsRCXXtqeyo6JvJZclcxnwUjEnlNdAn1uOSmz/FWHo9Qjkjg\nGJPhFEcAABLUSURBVDByONcrhW8MRnQ5NTqcyjnYNZvPLjUHqWvyPj8yRzHTuzwsFT4T1vk7n9ib\nvMh6oxZ1U3haTnx+Gypn0C7C9HNGRDP7dPz9eTjURPusfo/PDtZu2tT3Cq4HADiLbWg5pjNTBtL2\nep713X047v1y3mtYV81aOQVHecpPz46TsDbAUqXtUj8+aFNEVDnf59TPDcTbaO9HXx10Vn+vZKgI\ngiAIgiAIgiAIgiCcJQHNUPHZgfa0r6pqZ2w24I6ne8gbydfmi+h1iioJQ2cSPU5/u2QNAODhAayk\nfuqv9C629FfoGEEvoutTeiXdM+jFj/yc+7u70rqx90Puk5swk5kqJe/Q63Tvo0uhdE2GrX14jU/v\n+XfU0zMWN70OtfsZufcke+D1mLEO/vmj26Vwco4dz3w2CwBgv9iKqnJGvCf9mP1rXcX+PTXNhjA6\ncGHt1LUURtNT33WQEbWkXVbUTGNUPLyJfT5hKfeh5zcwbJe8ygXbr+h9PlHLa088SL0fOng5/JOo\nScV+hsotbl0pWnuoXysahcQE3hd1baF1jNf/R3RMB6bP24+dr9MbH+4GFi3k6QNbc6lfawMjmrlP\n+fBFK2vh+BJpk/kLnwYADF/HrKQ9Wy9CvI6SlXxEbS25zBrLfpURgk0zRiD5+wzJ3prEehFPvXAN\nAKAj3Y+TQ3XF+81DAQBDZh4DALTPo3ZHj6XBkcj3bGhxwbv9/PSFWXHEdWL4Nw9j10He976Lbaj4\nmNp5BnOMXXQxazK8vW5yb8ZXx3fpfZ+SyL3Je67KAgAorw3tWfy9/97AzBRQdhg688WX2oUFi9jx\nz22ZAwDw62hRu9sOzGKWw8AHGaUr+gnt7cjdHI/trna0dDJyOCad+9mPnWtHmBxrajdc951E83pO\nRhZvGDoyqEPOEGo0NZF7eTf+YToGb2NGYMUVnJfcg5ixYNORdO/RaHQs53h51RusWWVN7Dn9iWPj\nB8VDAAaN0DyUPxv4MMfRqikWlM3n9Q/vZ2EqayRt05uu63GMbkJNPutHJEd0wmoJ7ai4NaEbcTef\nRHM9ayRY851I/YQ20JLNUNhF380HADyXPxU5f2Z/nV7B+dDXyWs8bhqTG0CEk7r+upQnZs2dxnnx\n4zqeiPjqazOw/DsbAACNObSzNa/y2oRCHxx3MFJqb6LtqQK+1o3msq9qZy7uScrmH2DC0wv/Iyie\nutX/WX2qVbsFJQupad/jnHvqx3H8qpgZjYEvcm46cS3rhI2bzAyTg5s5cEZWGejUUfBjq7nGrJ1F\n2w5ror1ZuwGH/vgOP++DsPavIqo1l/E+cG1nTYiBC5jBUHwpo7iWw3a4+9A+bQ02oDu046RRYd2Y\nkFqObUuYLTYquxTlLzONL2wBx8XwV7iOrJruR+5fmLVw4lvM/InQJ414XHrMdPhxZFm0fnfeF5lp\nzGS3dFOTsFaFOUuZ7XVwCVNeyu/js4yvKgodE7IBADZd72zZN98DAPzeN0+/D+Btpe1/XHB2UfFg\nxevyo2GeG5YTtI6kg/7erMiWm5gh365PcO2zyYGGicxcSFjKPLulKR8DAB75fC4AIHFKHaL1Osb7\nJie/FT94FQDw8+1ch24tGwRriz4FZgDH5qK/0pYTi7pwejhtf3wWP+Pz0mwAgFOX8rvqwa34wy5m\nH7qym2Gx+85DT5gXj8uGysuS4ZnOTPj6CVGw6Do08YXMTj96E5/l/L8ej7ZUjold6TqzejJ17uvk\ntcdGRSLWxXHYu4s2PO4Kzqt736fd5bX1681MGTuEhTRPHOJY4HUAzTnUJO4APytthz6BaKT+zMIY\nROp6VS1z+Vl4+sz+3oA6VGABvNEGDH2cYO3YMFjduoDQPDpGwvWljvRmXJ3BFMkHFt4MAPjuOp1+\nbOXCM3JQI9pPcED7zp3MrVr9BYva+pLZOeH7nejI5Of1j+LTfWM+F6I1U+KQ8REHvWLFyUn1rAtn\nsiPbd/eB66Q+xjfLC1hCe8sPrAa8Lh9SMjjYuA8nQ1VzMXHyT7xpL3+cx46t3jYbVu3c6LuKk03f\nPVz47dFvVx/nQp/PeA/UzOfC4dhKLjxaR3Ag9f+iGnWf6Yc+PT45qnUxr04DTXr+iT6uj7TTqVzX\n3MAHvx21A2C38BdHpDJ9r+DcesH0tHZFYEfZAPidehCxKLyyjwWZB976BS9axO8rZlnQncAF26Dl\nTEQd6eUWLr+L/drtAFqyOJx0ZXCBEtazPe55Plh0dUVgfzl1zHHSFmNK9HHlLgtyfqftcyJTPL90\ncBEUqWtA537ejuPf0UXJcjtghIf2Q5xN+ZEY3gaEsZ8ddQb8dvZV6pPc/rbpOU4y3pFtaBmrF4MP\ncpTdcc1wAEDyPv5+7ViFeO3d6Irl+3TH9Ozr4EvGGzY8X8m89iGTSgEAVX/LBgBEjWhF1QEuahrG\n6K2Xuk5pIuc8xB3uRPlcToT57Ynn2gVBgdttx5eH+yLjFG3BuqwGnS+zH90f0xY2JHN+8kUBvtUc\nJzNv16nIw7g4TPsN7e+qF9/Gu/NGAQCiXqBDpqSQzumIOl2EPbkWznRuratoYwHOkqt5Tb+Ly5D0\nEOfIhkHUce7SnQCAja9OBQB4t6TAnk1tT6/PgLcxtLffdXXYcWJfZu9Rti2LmhH2JvVsnc4Htf1r\naW/ewT4k/4rFo5v+pB+8dSp7a5Z2XMZ7oPbRoV0axYe5Mi8DF5k7uVYqu9OPxzbwuNyedUvWh1y3\ntPSLQOd66hnr5nvOvJcO80grx+e3H52B6FtrAADFh9LPSz+YHX+4gfYcD06k6O2nm+2w81kA3bF8\n4I08RRvqjjNQvLwnwEZNDq+jnhF6h17LAMB1nP1fN4d22+d9Xch0Gu+PtLcMlMcxCLGugPbV7xGu\nl7rmj0fKbuqlijg4187RgSY9bteNUVDaieIc3AirI7Qf4lrbHNj+6TCkDaPj+Wh9MnwZer34EZ3A\nbl3M1H7aiuqJHONyf8jgQ8kD3C5gZFJT1EVgyCoGgkpX63FxGzXIbOU6eNVtb+C+PD6Up2ZQ34it\nukTBSB+a+uutknqYLHZzzDb0EbvetC7YSzlORI2ks6b8nHvC5HgsQIUD8YV6m/cAK/za3LqP87kv\nSS9VfTeeRkwE15nlGxlw/90UOkKhHVW+eIW23ez3J378AgDg++sWAwAcA+kYte6IgX8A7aeiiZ8R\nX8nvayaEI+Nh2uXRNh43b0/RgeIWtnHj/bPhGKqPVf80Fpam0A7A+xwGmi/ywNpFDRLyrL3bthou\n4nPjkCe4jmkfmgLnfDo82wpop8ZealCUzDkwos4CRyGNqHER57oeR0q4PmK58AfDsejJvQCA1wsZ\nMLZkUJ9x3ziCmvvpP/Dp8gEfv/BHAMDAl5YBAAY8UoDqG3jE/cwB9KwUn+HfG9qubEEQBEEQBEEQ\nBEEQhH+DgGaoOKPdmDIjH3s2MVIz75rdGBTJ7QDr72Cxyr6/pRd+jKsMz+TPAABYrqB36idrmanS\nOZzuf0txDHJHM8r24jr+vtJHBHrjdbpduA3w0SP29y30/jun6+JifQw0TaSXasiDTI9tHkMPdNdR\neq3ti6pRmc6CQ3E7HLC2hbgPSgEI82Nkoj729ko3Kt/O0l/TG3tHLMPR24cPRJePt1jN9ycDADof\nonbuqbzWZgDVlzNy44zWx8axfiI6C+ixb66NBWKoa2wB+z9uIbcLFB/tg3B9JGvqJwwlNQ3m/ZJ3\nOdtVen9K73GWW166WP8h68+xI8yNpcWCqA+cmL6MuUL7GjJxKo/R8OhPmDlQuYl9/fTta3DHpiUA\ngFN3M93c4qXdXDqWW3fqOp0YM4W22Oih7Ww5ydShkm3ZAABPtAG4qP+VsTp93cUsmOR9PhT+iFH5\npCwW6Mt6jDq2ZDGjoviaSCy7ZDMA2rvqDG1bbPfYsbsmu/dcwOZBBqJLdLG9t/RxyW8wyh0/pxqn\nTjFiUz6XHv7U0RzzcmYwMyhqZX8cu5l9bXUxKjo3l1mC7+xjgXB3nA05Exg7KzjMrAmM1wX6nkuG\nbwbb4k6kNv3eYhTh1DTeEy3Z0Vi8iNmEb98363x0g/mxGrA4PUi9l/16oCIdESnsv/ZM6jh+ArdT\n7snPgfcjZirYWR8RzkK+ll9GXT9pzMXJb1GbFEWbHDWKMZbqnTkAgLrf90NTGyNv1ZM4Rqfuom2W\n+PsiTqfcJh6ifm+t59zZe8ShFfA6eE1knR8W77l3g5mJd7XhujmfYq0ez2anl2L35fr4a72bpudI\nyKcu/TMeeJzF8Nt0QWELp0AYqZwDB964Hym7aLvlD3EcLWMyCkqW8X2sxZEwcrjFLmED7ct2hPdQ\nw12ZuH5oHgAg7zKmP3+opgAArvnRVgDA8vtfxWu1HM+tqR3n3AfBQFgzkPmOwsmr9PbTsXakfUpN\numNoJz1Htnoj/VA1XKP49bzWOpA2FVHFNYmjWsEbqd+8iVFa121cu9Qc0QUYL7bCcHIMjcvjNcee\nnggAUD4FZ5k+5noa183p1+4DAJSv4Dy95IoPcEvMIQDAxE0r4AvxLT8q3Ad7dhvGJdEWNhaMgG0w\n7WTJ8E8AAGs28HnhomnHUdLI9X3HCZ5nrQZye8GiXGbjvlk0FSefZ2ZDwivMkK2YQ72LVlDcB1ff\niCR9/Lzy6uOWS3mNqxxo0+Np51C+96FfMIMwicmBePnXz+LK0h8AAPxb5eAEAIiM6sToyUfxhY1b\nmjO3eFCzmLaYFce1ftcunVUZW4/Df2Nmu5fJXsh4nPb66Fru17jpwK2920OWr70NABAzlmufrBhm\nBe0bEAXXUV1+wkKtI2P1sfXjm4APOZZ6NurC1LpAf/31ekeDxY/OCq5b3Vk+eLeF+I4Gv4LFbcXA\ngfoggvBsOCv/OYOubjrHwfqZXRh8ry4Efanu8xnU5e5cbbfPLsCdv+Gz208+4rb0b8zjM8iOY5xM\nPdO7ezNT7IXMNOwpPPvFR4MRp+fhvt+lr2HanSz2nupju4Zsa8N42w4AwJ/3TzqrPze0R15BEARB\nEARBEARBEIR/A2UYgfOgKaXqALQDqA/Yh547ifjn9mYZhpF0oRpzoQkSDQHRMRh0FA3NryEgOgaD\njqKh+TUERMdg0FE0NL+GgOgYDDqKhubXEDhDHQPqUAEApVSeYRjjAvqh54DZ2hsIzNYnZmtvoDBb\nv5itvYHAbH1itvYGCrP1i9naGwjM1idma2+gMFu/mK29gcBsfWK29gYKs/WL2dobCMzWJ+fSXtny\nIwiCIAiCIAiCIAiCcJaIQ0UQBEEQBEEQBEEQBOEsuRAOlecvwGeeC2ZrbyAwW5+Yrb2Bwmz9Yrb2\nBgKz9YnZ2hsozNYvZmtvIDBbn5itvYHCbP1itvYGArP1idnaGyjM1i9ma28gMFuf/NvtDXgNFUEQ\nBEEQBEEQBEEQBLMjW34EQRAEQRAEQRAEQRDOkoA5VJRSc5VSRUqp40qp+wL1uWeKUipTKbVNKVWo\nlCpQSt2jf/6AUuqUUuqA/nf5hW7rhUR0ND+iYXAgOpof0TA4EB3Nj2gYHIiO5kc0DA5CTceAbPlR\nSlkBHAUwB0AFgL0ArjcMo/A//uFniFIqFUCqYRj7lFLRAL4AcDWARQDaDMN47II28GuA6Gh+RMPg\nQHQ0P6JhcCA6mh/RMDgQHc2PaBgchKKOgcpQmQDguGEYxYZhdAP4O4AFAfrsM8IwjCrDMPbpr1sB\nHAaQfmFb9bVDdDQ/omFwIDqaH9EwOBAdzY9oGByIjuZHNAwOQk7HQDlU0gGc/IfvK/A1vvmUUtkA\nRgPYo390l1LqkFLqRaVU3AVr2IVHdDQ/omFwIDqaH9EwOBAdzY9oGByIjuZHNAwOQk5HKUr7f1BK\nOQG8DmC5YRgtAFYDyAEwCkAVgMcvYPOEM0R0ND+iYXAgOpof0TA4EB3Nj2gYHIiO5kc0DA7Ol46B\ncqicApD5D99n6J99rVBKhYGdutYwjDcAwDCMGsMwfIZh+AH8EUxjClVER/MjGgYHoqP5EQ2DA9HR\n/IiGwYHoaH5Ew+Ag5HQMlENlL4BcpVQ/pZQdwHUANgbos88IpZQC8AKAw4ZhPPEPP0/9h8u+CSA/\n0G37GiE6mh/RMDgQHc2PaBgciI7mRzQMDkRH8yMaBgchp6Pt/DbvX2MYhlcpdReADwBYAbxoGEZB\nID77LJgC4CYAXyqlDuif/RTA9UqpUQAMAKUAll6Y5l14REfzIxoGB6Kj+RENgwPR0fyIhsGB6Gh+\nRMPgIBR1DMixyYIgCIIgCIIgCIIgCMGEFKUVBEEQBEEQBEEQBEE4S8ShIgiCIAiCIAiCIAiCcJaI\nQ0UQBEEQBEEQBEEQBOEsEYeKIAiCIAiCIAiCIAjCWSIOFUEQBEEQBEEQBEEQhLNEHCqCIAiCIAiC\nIAiCIAhniThUBEEQBEEQBEEQBEEQzhJxqAiCIAiCIAiCIAiCIJwl/wvj3w4qUrdnnwAAAABJRU5E\nrkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "tags": [] }, "output_type": "display_data" } ], "source": [ "images, labels = next(iter(mnist_test))\n", "noise_dim = 128\n", "\n", "def get_noise_batch(batch_size):\n", " noise_shape = [batch_size, noise_dim]\n", " return tf.random.normal(noise_shape, dtype=images.dtype)\n", "\n", "def get_label_batch(batch_size):\n", " label_shape = [batch_size]\n", " return tf.random.uniform(label_shape, maxval=10, dtype=labels.dtype)\n", "\n", "logits = gan.discriminate(images)\n", "noise = get_noise_batch(images.shape[0])\n", "gen_images = gan.generate(noise)\n", "\n", "num_images = 10\n", "plt.rcParams['figure.figsize'] = (2*num_images, 2)\n", "for i in range(num_images):\n", " plt.subplot(2, num_images, i+1)\n", " plt.imshow(images[i])\n", " plt.subplot(2, num_images, num_images+i+1)\n", " plt.imshow(gen_images[i])" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "9yV4JM345J71" }, "source": [ "Print the names, shapes, and other information about the variables in our little GAN model to get a rough idea of its structure." ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 615 }, "colab_type": "code", "id": "9ti32zve4_QG", "outputId": "4b3e61aa-a999-4c92-99a8-c83caef12ad7" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Variable Shape Type Trainable Device\n", "little_gan/discriminator/block_0/spectrally_normed_linear/b 1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/discriminator/block_0/spectrally_normed_linear/spectral_normalizer/u 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/discriminator/block_0/spectrally_normed_linear/w 784x1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/discriminator/block_1/spectrally_normed_linear/b 1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/discriminator/block_1/spectrally_normed_linear/spectral_normalizer/u 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/discriminator/block_1/spectrally_normed_linear/w 1024x1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/discriminator/outputs/b 1 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/discriminator/outputs/spectral_normalizer/u 1x1 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/discriminator/outputs/w 1024x1 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_0/batch_norm/exponential_moving_average/average 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_0/batch_norm/exponential_moving_average/average 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_0/batch_norm/exponential_moving_average/counter int64 False /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_0/batch_norm/exponential_moving_average/counter int64 False /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_0/batch_norm/exponential_moving_average/hidden 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_0/batch_norm/exponential_moving_average/hidden 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_0/batch_norm/offset 1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_0/batch_norm/scale 1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_0/spectrally_normed_linear/b 1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_0/spectrally_normed_linear/spectral_normalizer/u 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_0/spectrally_normed_linear/w 128x1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_1/batch_norm/exponential_moving_average/average 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_1/batch_norm/exponential_moving_average/average 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_1/batch_norm/exponential_moving_average/counter int64 False /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_1/batch_norm/exponential_moving_average/counter int64 False /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_1/batch_norm/exponential_moving_average/hidden 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_1/batch_norm/exponential_moving_average/hidden 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_1/batch_norm/offset 1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_1/batch_norm/scale 1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_1/spectrally_normed_linear/b 1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_1/spectrally_normed_linear/spectral_normalizer/u 1x1024 float32 False /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/block_1/spectrally_normed_linear/w 1024x1024 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/outputs/b 784 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n", "little_gan/generator/outputs/w 1024x784 float32 True /job:localhost/replica:0/task:0/device:GPU:0\n" ] } ], "source": [ "print(snt.format_variables(gan.variables))" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "V297xpzfobXK" }, "source": [ "## Training the model" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "WTrv-jn4pPSx" }, "source": [ "To train the model we need a loss function for both the discriminator and generator, as well as an optimizer.\n", "\n", "We'll optimize the discriminator via the \"hinge loss\", defined by the function `hinge_loss_disc`. The generator simply maximizes the discriminator's error via a linear loss given by `loss_gen`.\n", "\n", "For the optimizer, we'll use the \"Adam\" optimizer (`snt.optimizers.Adam`). To compute gradients we'll use a `tf.GradientTape` which allows us to selectively record gradients only for the computation we want to back propagate through.\n", "\n", "We'll put all of this together below in a Sonnet Module called `GANOptimizer`, which takes as input the GAN to be optimized:" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "JgVCqok0usw-" }, "outputs": [], "source": [ "def hinge_loss_disc(preds_real, preds_gen):\n", " loss_real = tf.reduce_mean(tf.nn.relu(1. - preds_real))\n", " loss_gen = tf.reduce_mean(tf.nn.relu(1. + preds_gen))\n", " return loss_real + loss_gen\n", "\n", "def loss_gen(preds_gen):\n", " return -tf.reduce_mean(preds_gen)\n", "\n", "class GANOptimizer(snt.Module):\n", "\n", " def __init__(self,\n", " gan,\n", " gen_batch_size=100,\n", " disc_lr=2e-4,\n", " gen_lr=5e-5,\n", " loss_type='hinge',\n", " num_epochs=100,\n", " decay_lr_start_epoch=50,\n", " decay_disc_lr=True,\n", " decay_gen_lr=True,\n", " name=None):\n", " super().__init__(name=name)\n", " self.gan = gan\n", " self.gen_batch_size = gen_batch_size\n", " self.init_disc_lr = disc_lr\n", " self.init_gen_lr = gen_lr\n", " self.disc_lr = tf.Variable(\n", " disc_lr, trainable=False, name='disc_lr', dtype=tf.float32)\n", " self.gen_lr = tf.Variable(\n", " gen_lr, trainable=False, name='gen_lr', dtype=tf.float32)\n", " self.disc_opt = snt.optimizers.Adam(learning_rate=self.disc_lr, beta1=0.)\n", " self.gen_opt = snt.optimizers.Adam(learning_rate=self.gen_lr, beta1=0.)\n", " self.num_epochs = tf.constant(num_epochs, dtype=tf.int32)\n", " self.decay_lr_start_epoch = tf.constant(decay_lr_start_epoch, dtype=tf.int32)\n", " self.decay_disc_lr = decay_disc_lr\n", " self.decay_gen_lr = decay_gen_lr\n", "\n", " def disc_step(self, images, labels, lr_mult=1.):\n", " \"\"\"Updates the discriminator once on the given batch of (images, labels).\"\"\"\n", " del labels\n", " gan = self.gan\n", " with tf.GradientTape() as tape:\n", " gen_images = gan.generate(get_noise_batch(images.shape[0]))\n", " preds_real = gan.discriminate(images)\n", " preds_gen = gan.discriminate(gen_images)\n", " loss = hinge_loss_disc(preds_real, preds_gen)\n", " disc_params = gan.discriminator.trainable_variables\n", " disc_grads = tape.gradient(loss, disc_params)\n", " if self.decay_disc_lr:\n", " self.disc_lr.assign(self.init_disc_lr * lr_mult)\n", " self.disc_opt.apply(disc_grads, disc_params)\n", " return loss\n", "\n", " def gen_step(self, lr_mult=1.):\n", " \"\"\"Updates the generator once.\"\"\"\n", " gan = self.gan\n", " noise = get_noise_batch(self.gen_batch_size)\n", " with tf.GradientTape() as tape:\n", " gen_images = gan.generate(noise)\n", " preds_gen = gan.discriminate(gen_images)\n", " loss = loss_gen(preds_gen)\n", " gen_params = gan.generator.trainable_variables\n", " gen_grads = tape.gradient(loss, gen_params)\n", " if self.decay_gen_lr:\n", " self.gen_lr.assign(self.init_gen_lr * lr_mult)\n", " self.gen_opt.apply(gen_grads, gen_params)\n", " return loss\n", "\n", " def _get_lr_mult(self, epoch):\n", " # Linear decay to 0.\n", " decay_epoch = tf.cast(epoch - self.decay_lr_start_epoch, tf.float32)\n", " if decay_epoch < tf.constant(0, dtype=tf.float32):\n", " return tf.constant(1., dtype=tf.float32)\n", " num_decay_epochs = tf.cast(self.num_epochs - self.decay_lr_start_epoch,\n", " dtype=tf.float32)\n", " return (num_decay_epochs - decay_epoch) / num_decay_epochs\n", "\n", " def step(self, train_batches, epoch):\n", " \"\"\"Updates the discriminator and generator weights.\n", "\n", " The discriminator is updated `len(train_batches)` times and the generator is\n", " updated once.\n", "\n", " Args:\n", " train_batches: list of batches, where each item is an (image, label)\n", " tuple. The discriminator is updated on each of these batches.\n", " epoch: the epoch number, used to decide the learning rate multiplier for\n", " learning rate decay.\n", "\n", " Returns:\n", " loss: the generator loss.\n", " lr_mult: the computed learning rate multiplier.\n", " \"\"\"\n", " lr_mult = self._get_lr_mult(epoch)\n", " for train_batch in train_batches:\n", " self.disc_step(*train_batch, lr_mult=lr_mult)\n", " return self.gen_step(lr_mult=lr_mult), lr_mult" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 989 }, "colab_type": "code", "id": "UUkshshiK6Eq", "outputId": "6272bb79-7eff-46af-a913-b9e886772948" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\r 0%| | 0/750000 [00:00" ] }, "metadata": { "tags": [] }, "output_type": "display_data" } ], "source": [ "num_images = 10\n", "noise = get_noise_batch(num_images)\n", "gen_images = gan.generate(noise)\n", "plt.rcParams['figure.figsize'] = (num_images, 1)\n", "for i in range(num_images):\n", " plt.subplot(1, num_images, i+1)\n", " plt.imshow(gen_images[i])" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "Lnkc55PtqA_I" }, "source": [ "Whew, those should already look much closer to actual handwritten digits! But there's still a ways to go. There are many potential ways to improve the results further, including (roughly in order of implementation difficulty):\n", "\n", "* Train the model for more steps (e.g., try setting `num_epochs = 100` or higher).\n", "* Add learning rate decay to make the step size smaller after a certain amount of time. (For example, try `optimizer = GANOptimizer(num_epochs=100, decay_lr_start_epoch=50)` to begin decaying the learning rate halfway through training.)\n", "* Increase the size of the model -- make it deeper (`num_layers=4`) and/or wider (`embed_dim=2048`).\n", "* Tweak the learning hyperparameters, such as the discriminator & generator learning rates or the training batch size.\n", "* Make the models **convolutional**, using `snt.Conv2D` layers to process the inputs and outputs. The generator and discriminator defined above use MLPs, which naively treat the digit images as flat vectors.\n", "* Make the model **conditional** on labels by feeding the digit label (e.g., \"7\") into the generator and discriminator. (See [1, 2] for some ideas on how to do this.)\n", "\n", "[1] M. Mirza and S. Osindero. [*Conditional Generative Adversarial Networks*](https://arxiv.org/abs/1411.1784). *arXiv:1411.1784*, 2014.\n", "\n", "[2] T. Miyato and M. Koyama. [*cGANs with Projection Discriminator*](https://arxiv.org/abs/1802.05637). *arXiv:1802.05637*, 2018." ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "little_gan_on_mnist.ipynb", "provenance": [], "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: examples/mlp_on_mnist.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "Yu7u1mNfnxVP" }, "source": [ "**Copyright 2019 The Sonnet Authors. All Rights Reserved.**\n", "\n", "Licensed under the Apache License, Version 2.0 (the \"License\");\n", "you may not use this file except in compliance with the License.\n", "You may obtain a copy of the License at\n", "\n", " http://www.apache.org/licenses/LICENSE-2.0\n", "\n", "Unless required by applicable law or agreed to in writing, software\n", "distributed under the License is distributed on an \"AS IS\" BASIS,\n", "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "See the License for the specific language governing permissions and\n", "limitations under the License.\n", "\n", "---" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "WAfR3cvnoGMB" }, "source": [ "# Preamble" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "4FqOAJb_jJR9" }, "outputs": [], "source": [ "import sys\n", "assert sys.version_info >= (3, 6), \"Sonnet 2 requires Python >=3.6\"" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 136 }, "colab_type": "code", "id": "XnWX2azUDuCl", "outputId": "8516864f-06cb-4dd5-946c-adeae1e17f3a" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: dm-sonnet==2.0.0b0 in /usr/local/lib/python3.6/dist-packages (2.0.0b0)\n", "Requirement already satisfied: gast==0.2.2 in /usr/local/lib/python3.6/dist-packages (0.2.2)\n", "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (4.28.1)\n", "Requirement already satisfied: wrapt>=1.11.1 in /tensorflow-2.0.0-rc1/python3.6 (from dm-sonnet==2.0.0b0) (1.11.2)\n", "Requirement already satisfied: numpy>=1.16.3 in /tensorflow-2.0.0-rc1/python3.6 (from dm-sonnet==2.0.0b0) (1.17.2)\n", "Requirement already satisfied: absl-py>=0.7.1 in /tensorflow-2.0.0-rc1/python3.6 (from dm-sonnet==2.0.0b0) (0.8.0)\n", "Requirement already satisfied: six>=1.12.0 in /tensorflow-2.0.0-rc1/python3.6 (from dm-sonnet==2.0.0b0) (1.12.0)\n" ] } ], "source": [ "!pip install dm-sonnet tqdm" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "mn5ofK4-D1Qk" }, "outputs": [], "source": [ "import sonnet as snt\n", "import tensorflow as tf\n", "import tensorflow_datasets as tfds" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 51 }, "colab_type": "code", "id": "Rpp_houJEHr9", "outputId": "9e7597ba-d8a5-483e-b415-5729ef3102e8" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "TensorFlow version: 2.0.0-rc1\n", " Sonnet version: 2.0.0b0\n" ] } ], "source": [ "print(\"TensorFlow version: {}\".format(tf.__version__))\n", "print(\" Sonnet version: {}\".format(snt.__version__))" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "5RmHUmz1padR" }, "source": [ "Finally lets take a quick look at the GPUs we have available:" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 34 }, "colab_type": "code", "id": "TXoxEvKepdw2", "outputId": "da76b78b-274e-462d-fa47-89d626a4370f" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " Tesla K80\n" ] } ], "source": [ "!grep Model: /proc/driver/nvidia/gpus/*/information | awk '{$1=\"\";print$0}'" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "UYYmqvOKfNbk" }, "source": [ "# Dataset\n", "\n", "We need to get our dataset in a state where we can iterate over it easily. The TensorFlow Datasets package provides a simple API for this. It will download the dataset and prepare it for us to speedily process on a GPU. We can also add our own pre-processing functions to mutate the dataset before our model sees it:" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "UkBRriaQEr4z" }, "outputs": [], "source": [ "batch_size = 100\n", "\n", "def process_batch(images, labels):\n", " images = tf.squeeze(images, axis=[-1])\n", " images = tf.cast(images, dtype=tf.float32)\n", " images = ((images / 255.) - .5) * 2.\n", " return images, labels\n", "\n", "def mnist(split):\n", " dataset = tfds.load(\"mnist\", split=split, as_supervised=True)\n", " dataset = dataset.map(process_batch)\n", " dataset = dataset.batch(batch_size)\n", " dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)\n", " dataset = dataset.cache()\n", " return dataset\n", "\n", "mnist_train = mnist(\"train\").shuffle(10)\n", "mnist_test = mnist(\"test\")" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "JfOCWVGEfgcq" }, "source": [ "MNIST contains `28x28` greyscale handwritten digits. Let's take a look at one:" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 269 }, "colab_type": "code", "id": "I_yM0TVjFCZq", "outputId": "54959260-c8fc-4e39-ea54-e566e9122893" }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADfxJREFUeJzt3X2MXXWdx/HPt53pVAqYVpY6W0Yo\nWMxWVlsy1oftEk2FAIsp/kOolBQlDmskSkJUUo1i1ii7i3UJGMIglcLyoBFIm1gfsJhFQCrDUwvO\nagu2sXXoAKPyoJRO+/WPOdUR5vzu7T3n3nNnvu9XcjP3nu95+ObCp+fe87v3/szdBSCeaVU3AKAa\nhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFAdrTzYDOvymZrVykMCobyil/Wq77V61i0UfjM7\nXdJVkqZL+pa7X5Faf6Zm6d22rMghASRs9k11r9vwy34zmy7pm5LOkLRQ0gozW9jo/gC0VpH3/Esk\nbXf3p939VUm3S1peTlsAmq1I+OdJ+u24x7uyZX/HzPrMbMDMBvZpb4HDAShT06/2u3u/u/e6e2+n\nupp9OAB1KhL+3ZJ6xj0+JlsGYBIoEv6HJC0ws/lmNkPSuZI2lNMWgGZreKjP3UfN7GJJP9LYUN9a\nd3+ytM4ANFWhcX533yhpY0m9AGghPt4LBEX4gaAIPxAU4QeCIvxAUIQfCKql3+fH5DNtUfqLml1X\nPZ+s/3m0M7+4bFcjLaEknPmBoAg/EBThB4Ii/EBQhB8IivADQTHUF9z0I49M1uddtzNZv77n/mT9\n+O9dlFtbIIb6qsSZHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCYpw/uO3XzU/WN/asS9b7//iPyfr8\n9aOH3BNagzM/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRVaJzfzHZIelHSfkmj7t5bRlMoz+8+875k\nffCUa2rsIX1+uO5/lifrR236eY39oyplfMjnA+7+XAn7AdBCvOwHgioafpf0YzN72Mz6ymgIQGsU\nfdm/1N13m9nRku42s/9393vHr5D9o9AnSTN1WMHDAShLoTO/u+/O/g5LukvSkgnW6Xf3Xnfv7VRX\nkcMBKFHD4TezWWZ2xMH7kk6T9ERZjQForiIv++dKusvMDu7nVnf/YSldAWi6hsPv7k9LemeJvaBB\n02bOzK1d8rE7k9tOt/SLvy89+/Zk/eibH0/WDySrqBJDfUBQhB8IivADQRF+ICjCDwRF+IGg+Onu\nKWDPBYtzaxe+8cFC+/7BmlOS9dl/4iu7kxVnfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IinH+SaBj\nXnoa7O+v/u9E9fDktieu+0SyPv+mYp8TQPvizA8ERfiBoAg/EBThB4Ii/EBQhB8IivADQTHOPwkM\nfq4nWe/uyB/Lf27/y8lt569P1+WermPS4swPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0HVHOc3s7WS\nzpI07O4nZcvmSPqOpOMk7ZB0jrv/vnltTm0db56brN/+oWtq7KEzt3L1yJL0pg9uqbFvTFX1nPlv\nlHT6a5ZdJmmTuy+QtCl7DGASqRl+d79X0shrFi+XtC67v07S2SX3BaDJGn3PP9fdh7L7z0hKv24F\n0HYKX/Bzd5eU+wFwM+szswEzG9invUUPB6AkjYZ/j5l1S1L2dzhvRXfvd/ded+/tVFeDhwNQtkbD\nv0HSquz+Kknry2kHQKvUDL+Z3Sbp55LeZma7zOxCSVdIOtXMtkn6YPYYwCRSc5zf3VfklJaV3Etc\nh70hWV7SlT+OX8sDn0qP80/Tow3vu9k6eo5J1g/MOSJdf3ywzHamHD7hBwRF+IGgCD8QFOEHgiL8\nQFCEHwiKn+5uA785Lz0Fdy17fV9ubdqfRwvtuyjryv9U587/PTG57VWLb0/WF3Smv0V+/qWX5tZm\nfW9zctsIOPMDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCM87dAR/ebk/WrL7iu0P6/OPyu/OIvthba\ndy2pcXxJev6OY3NrgyffXPDo+VOTS9KX//NbubU195+a3HZ06JmGOppMOPMDQRF+ICjCDwRF+IGg\nCD8QFOEHgiL8QFCM87fAy4t7kvVlb9hfaP9PvXRUovpcoX3XPPaXT07Wt518bcP7Hnz1T8n6P804\nLFlPPa//sSj936SLcX4AUxXhB4Ii/EBQhB8IivADQRF+ICjCDwRVc5zfzNZKOkvSsLuflC27XNLH\nJT2brbba3Tc2q0mkDf4g//fvjyk4zv+br703Wb//vCtr7GFWbuWbf0iPtd945VnJ+kNfSX+GYL8f\nyK3ZAU9uG0E9Z/4bJZ0+wfJvuPui7EbwgUmmZvjd/V5JIy3oBUALFXnPf7GZbTGztWY2u7SOALRE\no+G/VtIJkhZJGpL09bwVzazPzAbMbGCf9jZ4OABlayj87r7H3fe7+wFJ10takli339173b23U+kf\newTQOg2F38y6xz38sKQnymkHQKvUM9R3m6T3SzrKzHZJ+pKk95vZIkkuaYeki5rYI4AmqBl+d18x\nweIbmtDLlNU1kr7WMTT6UrLe3ZH+ffojlg4fck8HdczP/119SbpvZXoc/+jp+eP4kvSF4X/OrT36\nb+lx/pErXknWazl/x7Lc2owfDRTa91TAJ/yAoAg/EBThB4Ii/EBQhB8IivADQfHT3a3w4JZk+ern\n35esf3Vuevt73nFrbu2D534que2cf9+ZrNcayqvl1sdyP/ypzjX7kts+9a/fLnTsP/Qdnaj+vtC+\npwLO/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QlLm37ieMj7Q5/m7L/5plVAeWLkrW7/7uja1pZJJZ\n+MDKZP3YldtzawdeKfZ14Xa12TfpBR+xetblzA8ERfiBoAg/EBThB4Ii/EBQhB8IivADQfF9/jbQ\n8fhTyfpbb/lEsj74kWtya502vaGeWmFXjZ8sP63/s8l6z1ceSNbzJ+iGxJkfCIvwA0ERfiAowg8E\nRfiBoAg/EBThB4Kq+X1+M+uRdJOkuZJcUr+7X2VmcyR9R9JxknZIOsfdkz+Gzvf5m+P5j783t/ah\ni/8vuW3f7F8k67WmBy/ihHs+mqy/deWjTTv2VFX29/lHJV3q7gslvUfSJ81soaTLJG1y9wWSNmWP\nAUwSNcPv7kPu/kh2/0VJg5LmSVouaV222jpJZzerSQDlO6T3/GZ2nKTFkjZLmuvuQ1npGY29LQAw\nSdQdfjM7XNIdki5x9xfG13zswsGEFw/MrM/MBsxsYJ/2FmoWQHnqCr+ZdWos+Le4+53Z4j1m1p3V\nuyUNT7Stu/e7e6+793aqq4yeAZSgZvjNzCTdIGnQ3deMK22QtCq7v0rS+vLbA9As9Qz1LZX0M0lb\n9bdvSa7W2Pv+70p6i6SdGhvqG0nti6G+9vPHle9J1s9f/f1kfdWR25L1d9xxSW7txM+mh/J8L28T\nD9WhDPXV/D6/u98nKW9nJBmYpPiEHxAU4QeCIvxAUIQfCIrwA0ERfiAopugGphCm6AZQE+EHgiL8\nQFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii\n/EBQhB8IivADQRF+ICjCDwRVM/xm1mNmPzWzX5rZk2b26Wz55Wa228wey25nNr9dAGXpqGOdUUmX\nuvsjZnaEpIfN7O6s9g13v7J57QFolprhd/chSUPZ/RfNbFDSvGY3BqC5Duk9v5kdJ2mxpM3ZoovN\nbIuZrTWz2Tnb9JnZgJkN7NPeQs0CKE/d4TezwyXdIekSd39B0rWSTpC0SGOvDL4+0Xbu3u/uve7e\n26muEloGUIa6wm9mnRoL/i3ufqckufsed9/v7gckXS9pSfPaBFC2eq72m6QbJA26+5pxy7vHrfZh\nSU+U3x6AZqnnav+/SDpf0lYzeyxbtlrSCjNbJMkl7ZB0UVM6BNAU9Vztv0/SRPN9byy/HQCtwif8\ngKAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQZm7t+5gZs9K\n2jlu0VGSnmtZA4emXXtr174kemtUmb0d6+7/UM+KLQ3/6w5uNuDuvZU1kNCuvbVrXxK9Naqq3njZ\nDwRF+IGgqg5/f8XHT2nX3tq1L4neGlVJb5W+5wdQnarP/AAqUkn4zex0M/uVmW03s8uq6CGPme0w\ns63ZzMMDFfey1syGzeyJccvmmNndZrYt+zvhNGkV9dYWMzcnZpau9LlrtxmvW/6y38ymS/q1pFMl\n7ZL0kKQV7v7LljaSw8x2SOp198rHhM3sFEkvSbrJ3U/Klv2XpBF3vyL7h3O2u3+uTXq7XNJLVc/c\nnE0o0z1+ZmlJZ0u6QBU+d4m+zlEFz1sVZ/4lkra7+9Pu/qqk2yUtr6CPtufu90oaec3i5ZLWZffX\naex/npbL6a0tuPuQuz+S3X9R0sGZpSt97hJ9VaKK8M+T9Ntxj3epvab8dkk/NrOHzayv6mYmMDeb\nNl2SnpE0t8pmJlBz5uZWes3M0m3z3DUy43XZuOD3ekvd/WRJZ0j6ZPbyti352Hu2dhquqWvm5laZ\nYGbpv6ryuWt0xuuyVRH+3ZJ6xj0+JlvWFtx9d/Z3WNJdar/Zh/ccnCQ1+ztccT9/1U4zN080s7Ta\n4Llrpxmvqwj/Q5IWmNl8M5sh6VxJGyro43XMbFZ2IUZmNkvSaWq/2Yc3SFqV3V8laX2Fvfyddpm5\nOW9maVX83LXdjNfu3vKbpDM1dsX/KUmfr6KHnL6Ol/R4dnuy6t4k3aaxl4H7NHZt5EJJb5K0SdI2\nST+RNKeNertZ0lZJWzQWtO6KeluqsZf0WyQ9lt3OrPq5S/RVyfPGJ/yAoLjgBwRF+IGgCD8QFOEH\ngiL8QFCEHwiK8ANBEX4gqL8A74xLCC0psmEAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "tags": [] }, "output_type": "display_data" } ], "source": [ "import matplotlib.pyplot as plt\n", "\n", "images, _ = next(iter(mnist_test))\n", "plt.imshow(images[0]);" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "d7bsizs5gK3K" }, "source": [ "# Sonnet\n", "\n", "The next step is to define a model. In Sonnet everything that contains TensorFlow variables (`tf.Variable`) extends `snt.Module`, this includes low level neural network components (e.g. `snt.Linear`, `snt.Conv2D`), larger nets containing subcomponents (e.g. `snt.nets.MLP`), optimizers (e.g. `snt.optimizers.Adam`) and whatever else you can think of.\n", "\n", "Modules provide a simple abstraction for storing parameters (and `Variable`s used for other purposes, like for storing moving avergages in `BatchNorm`).\n", "\n", "To find all the parameters for a given module, simply do: `module.variables`. This will return a `tuple` of all the parameters that exist for this module, or any module it references:" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "GrN37pi1o4HT" }, "source": [ "## Building the model" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "c6XoN56S2lSW" }, "source": [ "In Sonnet you build neural networks out of `snt.Module`s. In this case we'll build a multi-layer perceptron as a new class with a `__call__` method that computes the logits by passing the input through a number of fully connected layers, with a ReLU non-linearity." ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab": {}, "colab_type": "code", "id": "hgjyB9yhFclD" }, "outputs": [], "source": [ "class MLP(snt.Module):\n", "\n", " def __init__(self):\n", " super(MLP, self).__init__()\n", " self.flatten = snt.Flatten()\n", " self.hidden1 = snt.Linear(1024, name=\"hidden1\")\n", " self.hidden2 = snt.Linear(1024, name=\"hidden2\")\n", " self.logits = snt.Linear(10, name=\"logits\")\n", "\n", " def __call__(self, images):\n", " output = self.flatten(images)\n", " output = tf.nn.relu(self.hidden1(output))\n", " output = tf.nn.relu(self.hidden2(output))\n", " output = self.logits(output)\n", " return output" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "0i03px8y8gf7" }, "source": [ "Now we'll create an instance of our class whose weights will be randomly initialized. We'll train this MLP such that it learns to recognize digits in the MNIST dataset." ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 34 }, "colab_type": "code", "id": "XqL8oIMqGAnU", "outputId": "96efc445-d7bb-45db-f6b1-5aa4a8b72dcb" }, "outputs": [ { "data": { "text/plain": [ "MLP()" ] }, "execution_count": 11, "metadata": { "tags": [] }, "output_type": "execute_result" } ], "source": [ "mlp = MLP()\n", "mlp" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "snzkUUh9oXPy" }, "source": [ "## Using the model" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "On8wI6VwpDPm" }, "source": [ "Let's feed an example input through the model and see what it predicts. Since the model is randomly initialized there is a 1/10 chance that it will predict the right class!" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 286 }, "colab_type": "code", "id": "4T-qmIc0GHfP", "outputId": "316f41f2-319c-452e-a132-5006b732e86b" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Predicted class: 0 actual class: 6\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADfxJREFUeJzt3X2MXXWdx/HPt53pVAqYVpY6W0Yo\nWMxWVlsy1oftEk2FAIsp/kOolBQlDmskSkJUUo1i1ii7i3UJGMIglcLyoBFIm1gfsJhFQCrDUwvO\nagu2sXXoAKPyoJRO+/WPOdUR5vzu7T3n3nNnvu9XcjP3nu95+ObCp+fe87v3/szdBSCeaVU3AKAa\nhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFAdrTzYDOvymZrVykMCobyil/Wq77V61i0UfjM7\nXdJVkqZL+pa7X5Faf6Zm6d22rMghASRs9k11r9vwy34zmy7pm5LOkLRQ0gozW9jo/gC0VpH3/Esk\nbXf3p939VUm3S1peTlsAmq1I+OdJ+u24x7uyZX/HzPrMbMDMBvZpb4HDAShT06/2u3u/u/e6e2+n\nupp9OAB1KhL+3ZJ6xj0+JlsGYBIoEv6HJC0ws/lmNkPSuZI2lNMWgGZreKjP3UfN7GJJP9LYUN9a\nd3+ytM4ANFWhcX533yhpY0m9AGghPt4LBEX4gaAIPxAU4QeCIvxAUIQfCKql3+fH5DNtUfqLml1X\nPZ+s/3m0M7+4bFcjLaEknPmBoAg/EBThB4Ii/EBQhB8IivADQTHUF9z0I49M1uddtzNZv77n/mT9\n+O9dlFtbIIb6qsSZHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCYpw/uO3XzU/WN/asS9b7//iPyfr8\n9aOH3BNagzM/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRVaJzfzHZIelHSfkmj7t5bRlMoz+8+875k\nffCUa2rsIX1+uO5/lifrR236eY39oyplfMjnA+7+XAn7AdBCvOwHgioafpf0YzN72Mz6ymgIQGsU\nfdm/1N13m9nRku42s/9393vHr5D9o9AnSTN1WMHDAShLoTO/u+/O/g5LukvSkgnW6Xf3Xnfv7VRX\nkcMBKFHD4TezWWZ2xMH7kk6T9ERZjQForiIv++dKusvMDu7nVnf/YSldAWi6hsPv7k9LemeJvaBB\n02bOzK1d8rE7k9tOt/SLvy89+/Zk/eibH0/WDySrqBJDfUBQhB8IivADQRF+ICjCDwRF+IGg+Onu\nKWDPBYtzaxe+8cFC+/7BmlOS9dl/4iu7kxVnfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IinH+SaBj\nXnoa7O+v/u9E9fDktieu+0SyPv+mYp8TQPvizA8ERfiBoAg/EBThB4Ii/EBQhB8IivADQTHOPwkM\nfq4nWe/uyB/Lf27/y8lt569P1+WermPS4swPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0HVHOc3s7WS\nzpI07O4nZcvmSPqOpOMk7ZB0jrv/vnltTm0db56brN/+oWtq7KEzt3L1yJL0pg9uqbFvTFX1nPlv\nlHT6a5ZdJmmTuy+QtCl7DGASqRl+d79X0shrFi+XtC67v07S2SX3BaDJGn3PP9fdh7L7z0hKv24F\n0HYKX/Bzd5eU+wFwM+szswEzG9invUUPB6AkjYZ/j5l1S1L2dzhvRXfvd/ded+/tVFeDhwNQtkbD\nv0HSquz+Kknry2kHQKvUDL+Z3Sbp55LeZma7zOxCSVdIOtXMtkn6YPYYwCRSc5zf3VfklJaV3Etc\nh70hWV7SlT+OX8sDn0qP80/Tow3vu9k6eo5J1g/MOSJdf3ywzHamHD7hBwRF+IGgCD8QFOEHgiL8\nQFCEHwiKn+5uA785Lz0Fdy17fV9ubdqfRwvtuyjryv9U587/PTG57VWLb0/WF3Smv0V+/qWX5tZm\nfW9zctsIOPMDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCM87dAR/ebk/WrL7iu0P6/OPyu/OIvthba\ndy2pcXxJev6OY3NrgyffXPDo+VOTS9KX//NbubU195+a3HZ06JmGOppMOPMDQRF+ICjCDwRF+IGg\nCD8QFOEHgiL8QFCM87fAy4t7kvVlb9hfaP9PvXRUovpcoX3XPPaXT07Wt518bcP7Hnz1T8n6P804\nLFlPPa//sSj936SLcX4AUxXhB4Ii/EBQhB8IivADQRF+ICjCDwRVc5zfzNZKOkvSsLuflC27XNLH\nJT2brbba3Tc2q0mkDf4g//fvjyk4zv+br703Wb//vCtr7GFWbuWbf0iPtd945VnJ+kNfSX+GYL8f\nyK3ZAU9uG0E9Z/4bJZ0+wfJvuPui7EbwgUmmZvjd/V5JIy3oBUALFXnPf7GZbTGztWY2u7SOALRE\no+G/VtIJkhZJGpL09bwVzazPzAbMbGCf9jZ4OABlayj87r7H3fe7+wFJ10takli339173b23U+kf\newTQOg2F38y6xz38sKQnymkHQKvUM9R3m6T3SzrKzHZJ+pKk95vZIkkuaYeki5rYI4AmqBl+d18x\nweIbmtDLlNU1kr7WMTT6UrLe3ZH+ffojlg4fck8HdczP/119SbpvZXoc/+jp+eP4kvSF4X/OrT36\nb+lx/pErXknWazl/x7Lc2owfDRTa91TAJ/yAoAg/EBThB4Ii/EBQhB8IivADQfHT3a3w4JZk+ern\n35esf3Vuevt73nFrbu2D534que2cf9+ZrNcayqvl1sdyP/ypzjX7kts+9a/fLnTsP/Qdnaj+vtC+\npwLO/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QlLm37ieMj7Q5/m7L/5plVAeWLkrW7/7uja1pZJJZ\n+MDKZP3YldtzawdeKfZ14Xa12TfpBR+xetblzA8ERfiBoAg/EBThB4Ii/EBQhB8IivADQfF9/jbQ\n8fhTyfpbb/lEsj74kWtya502vaGeWmFXjZ8sP63/s8l6z1ceSNbzJ+iGxJkfCIvwA0ERfiAowg8E\nRfiBoAg/EBThB4Kq+X1+M+uRdJOkuZJcUr+7X2VmcyR9R9JxknZIOsfdkz+Gzvf5m+P5j783t/ah\ni/8vuW3f7F8k67WmBy/ihHs+mqy/deWjTTv2VFX29/lHJV3q7gslvUfSJ81soaTLJG1y9wWSNmWP\nAUwSNcPv7kPu/kh2/0VJg5LmSVouaV222jpJZzerSQDlO6T3/GZ2nKTFkjZLmuvuQ1npGY29LQAw\nSdQdfjM7XNIdki5x9xfG13zswsGEFw/MrM/MBsxsYJ/2FmoWQHnqCr+ZdWos+Le4+53Z4j1m1p3V\nuyUNT7Stu/e7e6+793aqq4yeAZSgZvjNzCTdIGnQ3deMK22QtCq7v0rS+vLbA9As9Qz1LZX0M0lb\n9bdvSa7W2Pv+70p6i6SdGhvqG0nti6G+9vPHle9J1s9f/f1kfdWR25L1d9xxSW7txM+mh/J8L28T\nD9WhDPXV/D6/u98nKW9nJBmYpPiEHxAU4QeCIvxAUIQfCIrwA0ERfiAopugGphCm6AZQE+EHgiL8\nQFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii\n/EBQhB8IivADQRF+ICjCDwRVM/xm1mNmPzWzX5rZk2b26Wz55Wa228wey25nNr9dAGXpqGOdUUmX\nuvsjZnaEpIfN7O6s9g13v7J57QFolprhd/chSUPZ/RfNbFDSvGY3BqC5Duk9v5kdJ2mxpM3ZoovN\nbIuZrTWz2Tnb9JnZgJkN7NPeQs0CKE/d4TezwyXdIekSd39B0rWSTpC0SGOvDL4+0Xbu3u/uve7e\n26muEloGUIa6wm9mnRoL/i3ufqckufsed9/v7gckXS9pSfPaBFC2eq72m6QbJA26+5pxy7vHrfZh\nSU+U3x6AZqnnav+/SDpf0lYzeyxbtlrSCjNbJMkl7ZB0UVM6BNAU9Vztv0/SRPN9byy/HQCtwif8\ngKAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQZm7t+5gZs9K\n2jlu0VGSnmtZA4emXXtr174kemtUmb0d6+7/UM+KLQ3/6w5uNuDuvZU1kNCuvbVrXxK9Naqq3njZ\nDwRF+IGgqg5/f8XHT2nX3tq1L4neGlVJb5W+5wdQnarP/AAqUkn4zex0M/uVmW03s8uq6CGPme0w\ns63ZzMMDFfey1syGzeyJccvmmNndZrYt+zvhNGkV9dYWMzcnZpau9LlrtxmvW/6y38ymS/q1pFMl\n7ZL0kKQV7v7LljaSw8x2SOp198rHhM3sFEkvSbrJ3U/Klv2XpBF3vyL7h3O2u3+uTXq7XNJLVc/c\nnE0o0z1+ZmlJZ0u6QBU+d4m+zlEFz1sVZ/4lkra7+9Pu/qqk2yUtr6CPtufu90oaec3i5ZLWZffX\naex/npbL6a0tuPuQuz+S3X9R0sGZpSt97hJ9VaKK8M+T9Ntxj3epvab8dkk/NrOHzayv6mYmMDeb\nNl2SnpE0t8pmJlBz5uZWes3M0m3z3DUy43XZuOD3ekvd/WRJZ0j6ZPbyti352Hu2dhquqWvm5laZ\nYGbpv6ryuWt0xuuyVRH+3ZJ6xj0+JlvWFtx9d/Z3WNJdar/Zh/ccnCQ1+ztccT9/1U4zN080s7Ta\n4Llrpxmvqwj/Q5IWmNl8M5sh6VxJGyro43XMbFZ2IUZmNkvSaWq/2Yc3SFqV3V8laX2Fvfyddpm5\nOW9maVX83LXdjNfu3vKbpDM1dsX/KUmfr6KHnL6Ol/R4dnuy6t4k3aaxl4H7NHZt5EJJb5K0SdI2\nST+RNKeNertZ0lZJWzQWtO6KeluqsZf0WyQ9lt3OrPq5S/RVyfPGJ/yAoLjgBwRF+IGgCD8QFOEH\ngiL8QFCEHwiK8ANBEX4gqL8A74xLCC0psmEAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "tags": [] }, "output_type": "display_data" } ], "source": [ "images, labels = next(iter(mnist_test))\n", "logits = mlp(images)\n", " \n", "prediction = tf.argmax(logits[0]).numpy()\n", "actual = labels[0].numpy()\n", "print(\"Predicted class: {} actual class: {}\".format(prediction, actual))\n", "plt.imshow(images[0]);" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "V297xpzfobXK" }, "source": [ "## Training the model" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "WTrv-jn4pPSx" }, "source": [ "To train the model we need an optimizer. For this simple example we'll use Stochastic Gradient Descent which is implemented in the `SGD` optimizer. To compute gradients we'll use a `tf.GradientTape` which allows us to selectively record gradients only for the computation we want to back propagate through:" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "cellView": "form", "colab": {}, "colab_type": "code", "id": "V7gi8NQ-WZOl" }, "outputs": [], "source": [ "#@title Utility function to show progress bar.\n", "from tqdm import tqdm\n", "\n", "# MNIST training set has 60k images.\n", "num_images = 60000\n", "\n", "def progress_bar(generator):\n", " return tqdm(\n", " generator,\n", " unit='images',\n", " unit_scale=batch_size,\n", " total=(num_images // batch_size) * num_epochs)\n" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 85 }, "colab_type": "code", "id": "UUkshshiK6Eq", "outputId": "edf09c6d-cc08-482d-98d8-45778082ed0b" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 600000/600000 [01:02<00:00, 9660.48images/s] " ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "\n", "Final loss: 0.039747316390275955\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "opt = snt.optimizers.SGD(learning_rate=0.1)\n", "\n", "num_epochs = 10\n", "\n", "def step(images, labels):\n", " \"\"\"Performs one optimizer step on a single mini-batch.\"\"\"\n", " with tf.GradientTape() as tape:\n", " logits = mlp(images)\n", " loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,\n", " labels=labels)\n", " loss = tf.reduce_mean(loss)\n", "\n", " params = mlp.trainable_variables\n", " grads = tape.gradient(loss, params)\n", " opt.apply(grads, params)\n", " return loss\n", "\n", "for images, labels in progress_bar(mnist_train.repeat(num_epochs)):\n", " loss = step(images, labels)\n", "\n", "print(\"\\n\\nFinal loss: {}\".format(loss.numpy()))" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "2K0_eoR8og-G" }, "source": [ "## Evaluating the model" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "Cm_9RMJopgWc" }, "source": [ "We'll do very simple analysis of the model to get a feeling for how well it does against this dataset:" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 34 }, "colab_type": "code", "id": "PM7IPcOeXtxH", "outputId": "fc983d03-f92c-4002-f7df-514c251bd300" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Got 9767/10000 (97.67%) correct\n" ] } ], "source": [ "total = 0\n", "correct = 0\n", "for images, labels in mnist_test:\n", " predictions = tf.argmax(mlp(images), axis=1)\n", " correct += tf.math.count_nonzero(tf.equal(predictions, labels))\n", " total += images.shape[0]\n", "\n", "print(\"Got %d/%d (%.02f%%) correct\" % (correct, total, correct / total * 100.))" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "Lnkc55PtqA_I" }, "source": [ "To understand the result a bit better, lets take a look at a small sample of where the model correctly identified the digits:" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "cellView": "form", "colab": {}, "colab_type": "code", "id": "eFro_RB4YR-X" }, "outputs": [], "source": [ "#@title Utility function to show a sample of images.\n", "def sample(correct, rows, cols):\n", " n = 0\n", "\n", " f, ax = plt.subplots(rows, cols)\n", " if rows > 1: \n", " ax = tf.nest.flatten([tuple(ax[i]) for i in range(rows)])\n", " f.set_figwidth(14)\n", " f.set_figheight(4 * rows)\n", "\n", "\n", " for images, labels in mnist_test:\n", " predictions = tf.argmax(mlp(images), axis=1)\n", " eq = tf.equal(predictions, labels)\n", " for i, x in enumerate(eq):\n", " if x.numpy() == correct:\n", " label = labels[i]\n", " prediction = predictions[i]\n", " image = images[i]\n", "\n", " ax[n].imshow(image)\n", " ax[n].set_title(\"Prediction:{}\\nActual:{}\".format(prediction, label))\n", "\n", " n += 1\n", " if n == (rows * cols):\n", " break\n", "\n", " if n == (rows * cols):\n", " break" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 214 }, "colab_type": "code", "id": "PSamdka2dodW", "outputId": "e5c521a6-3d5b-4be6-f371-a2c7a87ae124" }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzIAAADFCAYAAACPSscRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmYXGWZ/vH7SXdnIwtZIHtCgCCE\nkWGJEWRTtkGU7afjwLAEZRcQRkAZ5RoigwPOAI6sEgwkAoIoSxBRNkFECCESthCWsJmELCxBQjbS\nnWf+qBN/Xf1Up6u7q6vOqfp+rquu9HnqrXPeqrpzqt8+9Z5j7i4AAAAAyJJule4AAAAAALQXAxkA\nAAAAmcNABgAAAEDmMJABAAAAkDkMZAAAAABkDgMZAAAAAJnDQKZEzGwLM3Mzq0+Wf2dmkzqwntFm\n9rGZ1ZW+l6hmZBBpQA5RaWQQaUAOy6PmBjJm9paZrU5CsdTMpplZn1Jvx92/6O7Ti+zPfs0e91d3\n7+PuTaXuU7K9I8xsnpmtNLPXzWzPrtgOWlerGTSzHmY21czeNrMVZvasmX2xlNtA8Wo1h8m2bjaz\nxWb2kZm9amYnlHobaFuNZ/DjFrcmM7uy1NtB22o1h9XymVxzA5nEwe7eR9LOkiZIOr/5nZZTda+N\nme0v6UeSvi6pr6S9JL1R0U7VrlrMYL2kBZL2ltRfued8u5ltUcE+1bpazKEkXSxpC3fvJ+kQSReZ\n2S4V7lOtqskMJr+Y9kme+1BJqyX9qsLdqmW1mMOq+EyutjelXdx9kaTfSfoHM3vUzH5oZn+WtErS\nlmbWPxmtLjazRWZ20YZDe2ZWZ2aXmtl7ZvaGpC81X3eyvhOaLZ+YHAlZYWYvmdnOZnaTpNGSfpP8\nJeA7Fg9FDjeze8zsAzObb2YnNlvnZDO73cx+nqx3rplN2MhT/oGkC919pruvd/dFyWuACqmlDLr7\nSnef7O5vJfm7V9KbkvgFssJqKYfJ853r7ms3LCa3rUrxWqJjai2DLXxF0jJJf+r4K4hSqKUcVs1n\nsrvX1E3SW5L2S34eJWmupP+U9Kikv0raXrlRaoOkuyRdJ2kTSZtLmiXp5OSxp0h6OVnHQEmPKPdh\nWJ/c/6ikE5Kf/1nSIkmfkWSStpY0pmV/kuUtWqznMUnXSOopaUdJ70raJ7lvsqQ1kg6SVKfcXxln\nNlvXNZKuSX6uk/SJpPMkzZe0UNJVknpV+j2ptVutZrDA6zAkeey2lX5PavFW6zlMaquSbTwjqU+l\n35Nau9V6Bpvd9wdJkyv9ftTqjRz+/b5MfiZXvAMVCuzHkj6U9HbypvZKAnZhizd0rZr9oi/pSEmP\nJD//QdIpze47YCOBvV/SmRvpT8HAJv8ZmiT1bXb/xZKmJT9PlvRQs/vGS1rdynaGJ+udLWmYpMGS\n/izph5V+T2rtVqsZbLHNBkkPSbqu0u9Hrd7I4d//wLOHcl+paKj0e1JrNzLokjQmWe/YSr8ftXoj\nh9n+TK5XbTrM3R9qXjAzKfddwQ3GKPfGLk7uk3JfxdvQZniL9m9vZHujJL3egX4Ol/SBu69osZ3m\nhwmXNPt5laSeZlbv7o0t1rU6+fdKd18sSWZ2uXIf4N/vQN/QObWYQUmS5b5nfJNyRwhP70CfUDo1\nm0NJ8tzk2cfN7GhJp0q6ogN9Q+fUdAYlHSPpcXd/swN9QunUbA6z/plcqwOZ1niznxcoN/Ie3Mqb\nv1i5IG4weiPrXaDWv3/trdQl6R1JA82sb7PQjlbucGS7uPtyM1vYYnsb2zYqo2ozKOUmTEqaqtxf\ntg5y93UdWQ+6XFXnsID6jfQLlVErGTxW0iWdXAe6TlXnsBo+k2t6sv/GJEctHpB0mZn1M7NuZraV\nme2dNLld0rfMbKSZDVBu7klrfibpHDPbxXK2NrMxyX1LJW3ZSh8WSHpC0sVm1tPMdpB0vKSbO/i0\nbpR0hpltnvT53yTd28F1oYtVaQavlbSdcmeIWd1WY1ReteUw2f8dYWZ9ksm5/6Tc10Mebu+6UB7V\nlsENzOxzkkaIs5VlQpXmMPOfyQxkNu5YSd0lvSRpuaRfKze/RJKuV+47js8pN1H0ztZW4u6/kvRD\nSb+QtELS3cpNBJNy320838w+NLNzCjz8SOW+H/mOcpPMLmh5+LM1ZvZTM/tps9J/Snpa0quS5kma\nk/QL6VU1GUx20icrNzlxif3/6yccVcy6UFFVk0Pl/tp5qnInPFku6VJJZ7n7PcWsCxVTTRncYJKk\nO1t8TQjpVjU5rJbPZEsm+QAAAABAZnBEBgAAAEDmMJABAAAAkDkMZAAAAABkDgMZAAAAAJnDQCbl\nzOzzyfVfgIogg0gDcohKI4NIA3KYj4FMEczsUTNbbmY9imi7hZm5mZXtYqPJNRHmmdlKM3vdzPYs\n17ZRHmnNoJn1MLOpZva2ma0ws2fN7ItdvV1URlpzmGzvZjNbbGYfmdmrZnZCObaL8kp5Bj9ucWsy\nsyvLsW2UV1pzWIufyQxk2mBmW0jaU7lrDxxS0c4UYGb7S/qRpK9L6itpL0lvVLRTKKmUZ7BeuSsU\n7y2pv6TzJd2e9BlVJOU5lHLXXtjC3fsp17+LzGyXCvcJJZT2DLp7nw03SUMlrRYXu6w6Kc9hzX0m\nM5Bp27GSZkqaptzFqyRJZtbLzC5LRr1/M7PHzayXpMeSJh8mf5HZzcwmm9nNzR6bNzo3s68nR1RW\nmNkbZnZyO/r3A0kXuvtMd1/v7ovcfVFnnzRSJbUZdPeV7j7Z3d9K8nevpDcl8Qtk9UltDiXJ3ee6\n+9oNi8ltq049Y6RNqjPYwlckLZP0pw4+HumV2hzW4mcyA5m2HSvpluT2T2Y2JKlfqlwwPqfc1Vi/\nI2m9ckdEJGnT5C8zTxaxjWWSviypn3JHVn5sZjsXamhm15jZNcnPdZImSNrMzOab2UIzuyr5j4Pq\nkdoMFrhviKRtJM0t5okhU1Kfw6S2StLLkhZLuq8dzw/pl/oMNjNJ0s+dq45Xo8zksBY+k8s2jyOL\nzGwPSWMk3e7u75nZ65L+1cx+IukbknZtdvTjieQx7d6Ou/+22eIfzewB5Q5bPlOg7TebLQ6R1CDp\nq0n7dZJmKHco8fvt7ghSJwMZbN7XBuV27NPd/eV2dwKplZUcuvs3zewMSbtJ+ryktS3bIJuyksFk\nu2OU+2rP8e3uAFItYzmsic9kjshs3CRJD7j7e8nyL5LaYEk9Jb1eio2Y2RfNbKaZfWBmH0o6KNlG\nW1Yn/17p7ouTfl6ePB7VIe0Z3PD4bpJukvSJpNNL0SekSiZyKEnu3uTuj0saKenUUvQLqZCZDEo6\nRtLj7v5mKfqEVMlEDmvpM5kjMq1Ivp71NUl1ZrYkKfeQtKmkYZLWKPf96+daPLTQYeSVkno3Wx7a\nbDs9JN2h3KHKGe6+zszultTmEN7dl1vuFHzNt8lh7CqRhQwmjzdJU5U7QniQu68r5nHIhqzksIB6\nMUemKmQwg8dKuqSdj0HKZSWHtfaZzBGZ1h0mqUnSeEk7JrftlJu4d6ykGyRdbmbDzawumbzVQ9K7\nyn0ncstm63pW0l5mNtrM+kv692b3dVfuP8K7khotd5q8A9rRzxslnWFmm5vZAEn/June9j9dpFBW\nMnht0q+D3X11W42ROanPYbL/O8LM+iR9+CdJR0p6uONPGymS+gxuYGafkzRCnK2sGmUlh7X1mezu\n3ArcJP1e0mUF6l+TtES5Ux3/r6RFkv6m3FkpeiVtLlQugB8q931JSbo6WZ4v6UTlRuj1yX2nSVqa\n3H+TpNskXZTc93lJC5tt/6eSftpsuUHSNcljl0i6QlLPSr9+3Gojg8p9V9iV+0vUx81uR1X69eNW\nUzncTNIfk8d9JOkFSSdW+rXjVjsZbFa7TtJNlX7NuNVmDlWDn8mWPHEAAAAAyAy+WgYAAAAgcxjI\nAAAAAMgcBjIAAAAAMoeBDAAAAIDM6dRAxswONLNXzGy+mZ1Xqk4B7UEOkQbkEJVGBpEG5BDl1OGz\nlplZnaRXJe0vaaGkpyUd6e4vtfaY7tbDe2qTDm0P1W2Flr/n7pu193HkEKWyRiv1ia/t0MUX25tD\nMojWsC9EGpQrh2QQrSk2g/Wd2MZESfPd/Q1JMrPbJB0qqdWdZk9tos/avp3YJKrVQ/7rtzv4UHKI\nknjKO3XtxHblkAyiNewLkQblyiEZRGuKzWBnvlo2QtKCZssLkxpQTuQQaUAOUWlkEGlADlFWnTki\nUxQzO0nSSZLUU727enNAQeQQlUYGkQbkEJVGBlFKnTkis0jSqGbLI5NaHnef4u4T3H1Cg3p0YnNA\nQeQQadBmDskguhj7QqQB+0KUVWcGMk9LGmdmY82su6QjJN1Tmm4BRSOHSANyiEojg0gDcoiy6vBX\ny9y90cxOl3S/pDpJN7j73JL1DCgCOUQakENUGhlEGpBDlFun5si4+32S7itRX4AOIYdIA3KISiOD\nSANyiHLq1AUxAQAAAKASGMgAAAAAyJwuP/0yCuu24/hQ6/GT90NtdWNDfPC+C7uiSwAAAEBmcEQG\nAAAAQOYwkAEAAACQOQxkAAAAAGQOc2TKpK5fv7zlEde9HdpcP+rPobblr08OtXFijgwAAABqG0dk\nAAAAAGQOAxkAAAAAmcNABgAAAEDmMJABAAAAkDlM9i+T+deNzVu+b9T00GbK34aH2tgZjV3WJ9Sg\niZ/OWxx+xVuhyY2j/xRqt60YEGrXfesrodb9/tkd7xvKbvVhE0PtvU/nfyys/dTq0OblL/ysqPU3\nWF2orfOmvOVt/3BCaNPj1V6hNviFuC/sdfesovoBAKhOHJEBAAAAkDkMZAAAAABkDgMZAAAAAJnT\nqTkyZvaWpBWSmiQ1uvuEUnQKaA9yiDQgh6g0Mog0IIcop1JM9v+Cu79XgvVUjXfO/VyozdvrqhaV\neDDsuv89NNQGP/xkqbpV7chhC2u/+JlQu+LaK/OWt2/oHto0eVzXP/d5P9RW/u+9oXbHITH7Ta+9\nsbFuVptU5LB+1MhQW3VDnHh/1bgrQu1TDbFdS+uL7Me6Alla3+LRL+0zJTbaJ5bmr4uT/U/75pGh\ntsnxsV3jgoWtd7L6pCKDqHnkEGXBV8sAAAAAZE5nBzIu6QEz+4uZnVSKDgEdQA6RBuQQlUYGkQbk\nEGXT2a+W7eHui8xsc0kPmtnL7v5Y8wZJiE+SpJ7q3cnNAQWRQ6TBRnNIBlEG7AuRBuwLUTadOiLj\n7ouSf5dJuktSuLqau09x9wnuPqFBPTqzOaAgcog0aCuHZBBdjX0h0oB9Icqpw0dkzGwTSd3cfUXy\n8wGSLixZzzKiW8+eoXbWN+4MtTrLHzNe8O72oc3mNz0XasVOqq1V5DCnbvCgULvw6utDbbNu+ROh\nd5z1jdBm6OXxg+XNQ2PtlSOvCbU1M54Otd/uvnXectPy5aFN1qUthyt2GR5qN25zeagtauoT2/1t\nRN7yddfEk5D0eq+4PZNbrFmLEwB0/8aS0OaIkTFHn+65INTu3/5XoXb+jPC7u+b+Lf/1aPrCO7Fj\nGZe2DKI21VoOV/zLrqHW95czQ23x2fFEOI0tfn0c/aNZoY03xpOXIF9nvlo2RNJdZrZhPb9w99+X\npFdA8cgh0oAcotLIINKAHKKsOjyQcfc3JP1jCfsCtBs5RBqQQ1QaGUQakEOUG6dfBgAAAJA5DGQA\nAAAAZE5nT79c85Yet1OoHd8/TvRq6XeX7xVqA1Y9WZI+ofa8/IOtQ233Hg+G2oT/+k7e8vCrnyhq\n/Vs9Hmdtf3qLY0Ntzm43htpVZxyctzz6wuK2iY7rdXecNHpk/3NDrf/rq0Ot2+PP5i1vri5+v26L\npbu0WajdsecBoTZ5+tRQu2hIfO4akr+4+4nfCk0G/azAfts91gBkmtXHX31t2/gZumZE/slQtr/o\nhdDm6EFXh9rN58SJ/bcNvTTU+nTLP4nO2YfHEwesaOwbasv+uX+oNS5YGGq1giMyAAAAADKHgQwA\nAACAzGEgAwAAACBzGMgAAAAAyBwm+7dD/Yh4tezffu9/CrSMV8veZvqpectjf972CQGAYm07+Y1Q\nm/jcaaE29Ob8idzFXZ9dBSc9j/rqi6F27uw4yfHiY36etzzlutimaemyYnuCDhowPdsnE+n2pzmh\nduGWO4da3SNxP33XNjPylp+cfFVos0u/M0Jt2GWcmKJYy4/bLdT6/vWTUKv/w1+KWt+aL0/MW163\nSXF/d+0/78NQW//8y0U9FrWhbvCgULv7/ltKtv5dhhfab/QoUMt32bDifi88/55dQu2eGfFzdfTk\n2th/cUQGAAAAQOYwkAEAAACQOQxkAAAAAGQOc2TaYd53R4XasPo4H+a9ppWhNnZGixoXWkMJNb37\nbqgNnhJrRc+J6aBHbvtMqP34357KW75uaPx+spgjgw6oHzUy1Mb3X9ShdQ16Mc7nQGFHvRwvvvcv\nfa8ItaYCn3MLm9YVtY01nj9foE5xXds0dA+1VR7fx7Ue93wt/4o79cMdQ5s7/hprax+MF2sd8eD7\nodY095VQQ/kd/lL8HOxmxX3eLF+/Jm/5ivfjBSvXe7xYdDeLWS2m3Xa93gltvtYn9vWizeM8s++d\nEOdATmz4dqht8f1sz5UshCMyAAAAADKHgQwAAACAzGEgAwAAACBz2hzImNkNZrbMzF5sVhtoZg+a\n2WvJvwO6tpuodeQQaUAOUWlkEGlADpEWxUz2nybpKknNr2p3nqSH3f0SMzsvWf5u6btXOfVDh4Ta\nbQfHi6hJDaFy5QcTY7OZz5egVzVtmmowh1nT7+2mSnehq00TOSypbj17hppvt1WoLdq3f6gd+K9x\n4upFQ2a1uc29zokXix3w5EuhltI0T1OFM/jzUw8JtatGxwv+9fogvoK9Fq+KK2yKk6N9ztz8Qre6\n0GbNl+KFAQtZsH+caO19G/OWz/nsA6HN/TtMD7U+O8bnufTbq0PtkIvPDbXNrq2qidbTlIF94Smb\nxpN/rPOYy28u3CvUZv7qH/OWh1/atReYnL3TgaH2g6/1C7W5x8bfRXtbPPHFJ4NSugcrsTaPyLj7\nY5I+aFE+VNKG/+HTJR1W4n4Becgh0oAcotLIINKAHCItOjpHZoi7L05+XiIpHr4Auh45RBqQQ1Qa\nGUQakEOUXacn+7u7SwVO8J4ws5PMbLaZzV6ntZ3dHFAQOUQabCyHZBDlwL4QacC+EOXS0YHMUjMb\nJknJv61eXcjdp7j7BHef0KD43VKgE8gh0qCoHJJBdCH2hUgD9oUou2Im+xdyj6RJki5J/p1Rsh6l\nRe9eoTSxR5zYX8gT34qT/btpTqe7tDGFrnC9fmDf/OXn5nVpHyqg+nOYMUsnxr+NvLwu/y9utnJN\naJNx1Z3DXXcIpVdPihNLCypwhWu1uMJ17/5xovQzu04LtW4F/u62XvGK7U+tjfvpU274Zt7yqFvj\npN2MT4stawbrHnkm1Io9PVWrh4rasj6+Qz1/0/aJHSRp3G/abnOPBoXaFRedHWovfj1OtB5SF39f\nWDk8bmOztruRdanbFx7x5j6h9tcrtwm1TR96LdSGv9e1k/tbCie4kLTlC/HX9O3GHB9q8/ae2iV9\nyoJiTr98q6QnJX3KzBaa2fHKhXR/M3tN0n7JMtBlyCHSgByi0sgg0oAcIi3aPCLj7ke2cte+Je4L\n0CpyiDQgh6g0Mog0IIdIi05P9gcAAACAcmMgAwAAACBzOjrZv+q9eVSBmXoFrPV1odZtdWOBlh1j\nPeIZPd6+OU5U+8lOt4XauIblecvHnB0nLm7y66c60Tsg35id4lWUb3h/97zlpvlvlqs7aIvFK55/\ndN+WecuP7XBjh1ffYPFq7IWuqh3Fv7H9z/vjQ+2WW+O3WEZeHCfojlJ5J+2iOniMb0GFTjqx+ZxY\nQ/kt373lNTulvpoZamk92Yf1iieSqOWJ/YVwRAYAAABA5jCQAQAAAJA5DGQAAAAAZA4DGQAAAACZ\nw2T/RP2woXnLVx53XVGP+49ln4nFWS90qA+FJva/f8eYUJu3801FrrFP3tIPfvSz0OLyP+8fao2L\nlxS5ftQy3+0fQ23GtvH/zU6PnZK3vKWe7bI+ofNWPjQkb3nh+NWhzfD6uK8qZF2By7gXmhhdjAeX\nbhtqQ2et7dC6gGI0DvukqHY/+9uWodb7Tk6kA5QDR2QAAAAAZA4DGQAAAACZw0AGAAAAQOYwRyax\ncqdRecv79iru8kivfzy4QPW9DvXh9R/sHGqv7XxtUY+d98mqUNuue++85ULP6T93HBVqPZgjg5YK\nXDhx6Xfj98fvWzUk1MZd8FHeclovPFaTPE5iGXZZ/sUjJ83/dmjT2Ku4v4Edf8HdobauxVUG99/k\nldBmZIE5OA+MvzPUnrw+XrHw1DlHhdqoi/KXfc7c0Aa1ra5fv1D7zsTfF/XYV1YNLVCNF8sGUHoc\nkQEAAACQOQxkAAAAAGQOAxkAAAAAmdPmQMbMbjCzZWb2YrPaZDNbZGbPJreDurabqHXkEGlADlFp\nZBBpQA6RFsVM9p8m6SpJP29R/7G7X1ryHmXMvN9tE2oji5js/+bFu4Xan48q9HJuEipXfxgn6E+7\n9Muh9vRF+ScKaPJ4ITpbX+CKdek0TeSwXeo27R9qH+2bf1HBZRPi3zL2229OqM1cHC/M+swut4Ta\ndtNPC7Wxrz250X5mzDTVWA57zZjV4cfefluhSdD5bph0SKhtc/K8ULtxzMOh9tkecUL1M7tOixu5\nN3/xkBEFLmScHdNUYxnsCt02yf9sffOsfwhtju//SFHrGtr9o1D7/a27dqxjBTQu6xVq486cWbL1\nd9A0kcMu9+HB2xeoPlrubqRam0dk3P0xSR+UoS9Aq8gh0oAcotLIINKAHCItOjNH5nQzez45vDig\nZD0C2occIg3IISqNDCINyCHKqqMDmWslbSVpR0mLJV3WWkMzO8nMZpvZ7HVa28HNAQWRQ6RBUTkk\ng+hC7AuRBuwLUXYdGsi4+1J3b3L39ZKulzRxI22nuPsEd5/QoHiRM6CjyCHSoNgckkF0FfaFSAP2\nhaiEYib7B2Y2zN0XJ4uHS3pxY+2zoMcH+X8VWNz4cWgzrL5PqPXdY1lR668fmz9Z+vGj41y4zevi\nxP7zl3061OZ8KU72/+CSNW324Zi39g217vfPbvNxaVWNOQzMQmnl/4ufDYsP/yTULvvsr0Lt4N7F\nTV4Nhj/RdhtJ2nJlKLW8YnbTR3FibJbVRA670IDp8WQQ706P7fY97NRQqzttaag9MP7ONrc5fGbf\nUFt6cM9Qa3r33TbXlQa1kMFuO2wbam98v3uojRz0Yaj98lO3htrgFp+3Tf6nAluN+99Cjt70L6H2\nq947hdo+I1/NW/7Nq/HzvZCBLxbXj0qrhRwWo27woFCzHnHA1rjonbzl5ZPiSaB+fMHVRW3zlhXD\nQm3ba1eEWjzlU/a1OZAxs1slfV7SYDNbKOkCSZ83sx0luaS3JJ3chX0EyCFSgRyi0sgg0oAcIi3a\nHMi4+5EFylO7oC9Aq8gh0oAcotLIINKAHCItOnPWMgAAAACoCAYyAAAAADKnQ5P9q9LM5/MWr3z/\nc6HJfw15PtT+sMMvQm2/I74VagNPeTtvudDE/kJ+8Wyc2N1webya9et73tjmuj48afMC1eVF9QOV\nsfjbcfLfnG9f1eH1/WT51m22OXPA/FBrVFOoPbI6nvxi3p7TQm33276Wt9z/oOqa7I/y6HX3rFCr\n+0O/UNvmv+NJAS7+Qv6JL6aMejS0OfiXh4Ra/aSRoda4YOHGuoku0m15PAHPuiXx/XlzSe9QO/CX\n54Rav7fzP0ff2aMhtHnh+OL2tfv88txQ2+rceBKLljPfx+q5otaP9KrfYnSoDbo1nnBi703nhtp/\n33F43vKs4y4PbXpbPKFFIVO/d3io9X7uqaIem3UckQEAAACQOQxkAAAAAGQOAxkAAAAAmcNABgAA\nAEDmMNm/FU+fsXMs3h4n+/fuFidiPXH5T0vWjzcO6Php2cc/cXTe8pjX4yRupIvtsn3e8qWnXl/U\n4wpN4v/tmV8ItR4vLshb7nNHnMRfaLL/tvd+M9bOfCHUvnVBvJp14/C1ecv9QwukSd2QFicF6RtP\nTNI0/80y9Wbjmj6KJ47Y5pR4UoAbJ345b/krd00LbWZ86u5Q+8LuZ4Ra39uY7F8JhU6yMO7M0r0X\no1bHfZeOL+6xIx6L+1HUhrVjB4fa1NF3FfXYY7/e8mQSxU3sL+S4H84ItR/ufViojbt1VYe30WEz\n4+/OpcQRGQAAAACZw0AGAAAAQOYwkAEAAACQOQxkAAAAAGQOk/1bUf/c66G29S3xitHz/jVe+bfB\n6rqkTxssbIxXOD5gyndCbdRFT+Qtr++yHqFURlz9dt7yvr3WhjZHvrl/qH180LpQ69EQJ2S/esWY\nvOWZY2J+P3Xr2aG27fnPhtr6NWtCbey/x6tZI73eP3G3UBv/jfwrUC9dVWAC6r5d1aMuMiuemAJo\nbunEXpXuAjKox1vvh9r4m08Ptd8f8T+hNrq+dJk7tt+iWPvq1bHhV0u2yaJ9ecQuXbp+jsgAAAAA\nyBwGMgAAAAAyp82BjJmNMrNHzOwlM5trZmcm9YFm9qCZvZb8O6Dru4taRQ5RaWQQaUAOkQbkEGlR\nzByZRklnu/szZtZX0l/M7EFJx0l62N0vMbPzJJ0n6btd19XyWr9iRahtdW78/v9ur8bvQh58+h9D\n7aQB+RdpG1bfp8N92/uxeJG2rVvMh6lCNZHDgd1X5i0/+0ljaPPRWcNCrVtd/H5s/Z1xbsMrW+df\nYPWAeUeGNludMzPUmF8lqQoz+P7nPgm1qaMfyVu+cvm40Gb6mQeG2tCfpGMftPy4OO/n41GWt9xg\ncc7X7R8PDLW+b6wMtRSouhymwaqh3uHHfjwszovt2ZnOZAM5lNT45tuhtuV3Y+2MG46LD67Pz83S\n3eM+aMIJcV/VGUN75F9E+PzB8WKV5y+Lc1o+XFfcfJ5Z0+KFZTdX1342tHlExt0Xu/szyc8rJM2T\nNELSoZKmJ82mS4qXEAVKhByi0sgg0oAcIg3IIdKiXXNkzGwLSTtJekrSEHdfnNy1RNKQkvYMaAU5\nRKWRQaQBOUQakENUUtEDGTMDLiOeAAAIz0lEQVTrI+kOSWe5e96xKXd3SQWPy5rZSWY228xmr1M8\nlSzQHuQQlUYGkQbkEGnQkRySQZRSUQMZM2tQLqi3uPudSXmpmQ1L7h8maVmhx7r7FHef4O4TGtSj\nFH1GjSKHqDQyiDQgh0iDjuaQDKKU2pzsb2Ymaaqkee5+ebO77pE0SdIlyb8zuqSHKTfo+ngCgCeu\nj5Osf3f0OXnLx3zvt6HNpH6vhdoOd5wVatt8Z06odXyaYjZUYw7rBsWJfZ/unT/x7sQXjglt+m0W\nJ90N/nGcbPrLLe8Ltf1eOjxvudep8W8ZTbGrUHVmUG6htL7FqR1OG/BKaHPsOfECkzNO2SrU/vv5\nA0Jt4IzeecvvH7IqtBl0T+9QO/k/7gy1Jo/53ad3vPDc8Pr8X5bWFXjc1AV7hloaL6RZlTnMuD6L\na2+vSQ7bp+mV+W22GTw31t6aUtp+LBw2Km95x6/vHdqMue7lUGt6/4Oi1t/VE/sLKeasZbtLOkbS\nC2Z/P9XL95QL6e1mdryktyV9rWu6CEgih6g8Mog0IIdIA3KIVGhzIOPuj0uKf7bL2be03QEKI4eo\nNDKINCCHSANyiLRo11nLAAAAACANGMgAAAAAyJxi5sigBPrfnH+19HtuHhTa3KNYG6d4lfVqn9hf\nKwpNnnthVf5EvKd3uTU+8GextNrjFdp3nXNsqG12bn56mubHE0ygdmzz03jq0+1WnZ63fOWB00Ob\nvt3WhNrR/RaE2rF73Bhq6/dYH2pBgXn33Qr83a3liQly4lmQnlrbkLd8wlOTQptR18ePw3otbL2P\nqEnvNa0OtZ5LYw1Io8bFS/KWR/7XktAma6eu4IgMAAAAgMxhIAMAAAAgcxjIAAAAAMgcBjIAAAAA\nMofJ/kCK3PGnz+Ytf+nLz4U209/dPdTemrxtqA38/dOhlrVJfOhiBa5cP25W/vLV4w8JbdYN7B1q\niz4fa4Vc/42r8pYn9Iip3GPOUaG2cubgotZfyIhHV+Utj3382VZaolaNvTeewGI7Oy3UtrwzTuy3\nWXE/DaA8OCIDAAAAIHMYyAAAAADIHAYyAAAAADKHgQwAAACAzGGyP5Ai486cmbd88Zk7FGi1IlS6\nK07sB0qh6aVXQ63QX8BGPV7c+i68aOc22wxU3GahGlAq3f44J9S2+mMFOgKgXTgiAwAAACBzGMgA\nAAAAyJw2BzJmNsrMHjGzl8xsrpmdmdQnm9kiM3s2uR3U9d1FrSKHqDQyiDQgh6g0Mog0KWaOTKOk\ns939GTPrK+kvZvZgct+P3f3Sruse8HfkEJVGBpEG5BCVRgaRGm0OZNx9saTFyc8rzGyepBFd3TGg\nOXKISiODSANyiEojg0iTds2RMbMtJO0k6amkdLqZPW9mN5jZgBL3DSiIHKLSyCDSgByi0sggKq3o\ngYyZ9ZF0h6Sz3P0jSddK2krSjsqNzC9r5XEnmdlsM5u9TmtL0GXUMnKISiODSANyiEojg0iDogYy\nZtagXFhvcfc7Jcndl7p7k7uvl3S9pImFHuvuU9x9grtPaFCPUvUbNYgcotLIINKAHKLSyCDSopiz\nlpmkqZLmufvlzerDmjU7XNKLpe8ekEMOUWlkEGlADlFpZBBpUsxZy3aXdIykF8zs2aT2PUlHmtmO\nklzSW5JO7pIeAjnkEJVGBpEG5BCVRgaRGsWctexxSVbgrvtK3x2gMHKISiODSANyiEojg0iTdp21\nDAAAAADSgIEMAAAAgMxhIAMAAAAgcxjIAAAAAMgcBjIAAAAAMoeBDAAAAIDMYSADAAAAIHPM3cu3\nMbN3Jb0tabCk98q24dLLev+l9D2HMe6+WTk2RA5TI239r0QGpfS9Du1F/0uLfWH70f/SK0sO2Rem\nStr6X1QGyzqQ+ftGzWa7+4Syb7hEst5/qTqeQ2dl/TWg/9Uh668D/c++rL8G9L86ZP11oP+VwVfL\nAAAAAGQOAxkAAAAAmVOpgcyUCm23VLLef6k6nkNnZf01oP/VIeuvA/3Pvqy/BvS/OmT9daD/FVCR\nOTIAAAAA0Bl8tQwAAABA5pR9IGNmB5rZK2Y238zOK/f228vMbjCzZWb2YrPaQDN70MxeS/4dUMk+\nboyZjTKzR8zsJTOba2ZnJvXMPIdSy1oGJXJYjchheZHBwrKWwyxnUCKHhWQtgxI5TJOyDmTMrE7S\n1ZK+KGm8pCPNbHw5+9AB0yQd2KJ2nqSH3X2cpIeT5bRqlHS2u4+XtKuk05LXPEvPoWQymkGJHFYV\nclgRZLCFjOZwmrKbQYkc5sloBiVymBrlPiIzUdJ8d3/D3T+RdJukQ8vch3Zx98ckfdCifKik6cnP\n0yUdVtZOtYO7L3b3Z5KfV0iaJ2mEMvQcSixzGZTIYRUih2VGBgvKXA6znEGJHBaQuQxK5DBNyj2Q\nGSFpQbPlhUkta4a4++Lk5yWShlSyM8Uysy0k7STpKWX0OZRAtWRQyuh7SA4lkcOKIoN/Vy05zOR7\nSA4lVU8GpYy+h1nPIZP9O8lzp31L/anfzKyPpDskneXuHzW/LyvPAa3LyntIDqtbFt5DMljdsvIe\nksPqlpX3sBpyWO6BzCJJo5otj0xqWbPUzIZJUvLvsgr3Z6PMrEG5oN7i7ncm5Uw9hxKqlgxKGXsP\nyWEeclgBZDColhxm6j0kh3mqJYNSxt7DaslhuQcyT0saZ2Zjzay7pCMk3VPmPpTCPZImJT9PkjSj\ngn3ZKDMzSVMlzXP3y5vdlZnnUGLVkkEpQ+8hOQzIYZmRwYKqJYeZeQ/JYVAtGZQy9B5WVQ7dvaw3\nSQdJelXS65K+X+7td6C/t0paLGmdct/dPF7SIOXO5vCapIckDax0PzfS/z2UOzT4vKRnk9tBWXoO\nXfCaZCqDSZ/JYZXdyGHZ+04GC78umcphljOY9J8cxtckUxlM+kwOU3Kz5AkBAAAAQGYw2R8AAABA\n5jCQAQAAAJA5DGQAAAAAZA4DGQAAAACZw0AGAAAAQOYwkAEAAACQOQxkAAAAAGQOAxkAAAAAmfN/\n5MF1f//wc0EAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "tags": [] }, "output_type": "display_data" } ], "source": [ "sample(correct=True, rows=1, cols=5)" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "hzHp02F_pzdh" }, "source": [ "Now lets take a look at where it incorrectly classifies the input. MNIST has some rather dubious handwriting, I'm sure you'll agree that some of the samples below are a little ambiguous:" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 451 }, "colab_type": "code", "id": "KQe5Q9LNdnb0", "outputId": "7f28c377-2226-468e-e388-a1cb649a26f5" }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzIAAAGyCAYAAAA/GvHcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xm8XPP9x/H3597smwgS2SMbUioI\nRSm1lNBFF36W2kqDolRLtT+/n9TSamupIogiaS2tqiVF7VvtgliSEEEiiUSEREIkubn3+/tjTvqb\nuZ+5uXPnzp1zzszr+XjMI/d85izfmXnn3Pudc77nWAhBAAAAAJAmNXE3AAAAAABaio4MAAAAgNSh\nIwMAAAAgdejIAAAAAEgdOjIAAAAAUoeODAAAAIDUoSNTImY2xMyCmbWLpv9lZkcVsZ5BZvapmdWW\nvpWoZGQQSUAOETcyiCQgh+VRdR0ZM5tjZp9HofjAzCaZWbdSbyeEMDaEMLnA9uydtdx7IYRuIYT6\nUrbHzDqa2XVmNtfMVpjZNDMbW8ptoDDVmsFoW582etSb2eWl3g6aV605ZF+YHNWawWhbN5rZQjNb\nbmazzOy4Um8DhanmHGZtc4SZrTKzG9tqG22l6joykW+EELpJ2k7SGElnZz9pGZX23rSTNE/S7pI2\nUOY132pmQ2JsUzWrxgwq2hl3i177ppI+l/T3mJtVzaoxh+wLk6UaMyhJv5E0JITQQ9I3JZ1vZtvH\n3KZqVq05XOdKSS/E3YhiVPKH0qwQwgJJ/5K0lZk9ZmYXmNlTklZKGmpmG0Tf3C00swVmdv66Q3tm\nVmtmF5nZEjN7R9IB2euO1ndc1vQPzWxm9A3gDDPbzsz+ImmQpH9G3wScmedQZD8zm2JmH5vZbDP7\nYdY6x5vZrWb252i9081sTBOv9bMQwvgQwpwQQkMI4W5J70pixxmjaspgHt+VtFjSv4t/B1EK1ZRD\n9oXJVE0ZjF7v9BDC6nWT0WNYKd5LFK/achgtc4ikZZIeLsmbWG4hhKp6SJojae/o54GSpks6T9Jj\nkt6T9AVlvrFrL+kOSddI6iqpt6TnJR0fLXuCpDeidfSS9KgyO6J20fOPSTou+vkgSQsk7SDJJA2X\nNLhxe6LpIY3W84SkCZI6SRot6UNJe0bPjZe0StL+kmqV+Ybn2ax1TZA0oYn3oU+07BZxfybV9iCD\n/3nuEUnj4/48qvVBDv/zHPtCMhhLBqPaymgbL0nqFvdnUo2Pas6hpB6SZkkaEC17Y9yfR4s/v7gb\nEFNgP1Wm9zk3+lA7RwE7N2u+PpJWS+qcVTtU0qPRz49IOiHrua+tJ7D3Szp1Pe3JG9joP0O9pO5Z\nz/9G0qSswD6U9dwoSZ8X8B60l/SQpGvi/jyq8UEGgyQNjta7WdyfR7U+yCH7wrgfZDBImT82d1Xm\nVKb2cX8m1fio5hxKukzSz7OWTV1Hpp2q04EhhIeyC2YmZc6bXmewMr/kFkbPSZlT8dbN06/R/HPX\ns72Bkt4uop39JH0cQljRaDvZhwkXZf28UlInM2sXQlibb4WWOcfzL5LWSDq5iDahNKo2g5EjJD0Z\nQni3iDahdKo2h+wLE6NqMyhJITOA+0kz+76kEyX9sYi2ofWqLodmNlrS3pK2LaIdiVGtHZmmhKyf\n5ynT8964iZ3QQmWCuM6g9ax3npo+9zU0UZek9yX1MrPuWaEdpMzhyBazzP+865T5VmH/EEJdMetB\nm6roDGY5UtKFrVwH2k5F55B9YSpUdAbzaLeediE+lZzDPZQ52vNe1DHrJqnWzEaFELYrYn2xqOrB\n/usTQlgo6QFJF5tZDzOrMbNhZrZ7NMutkn5sZgPMbENJZ61ndX+S9DMz294yhpvZ4Oi5DyQNbaIN\n8yQ9Lek3ZtbJzL4o6VhJxV4e7ypJWypzdY7Pi1wHyqRCMygz20VSf3G1slSo0ByyL0yRSsugmfU2\ns0PMrFs0QHxfZU5RSudg6ypRaTmUNFGZztTo6HG1pHsk7VvEumJDR2b9jpTUQdIMSUsl3Sapb/Tc\ntcqc4/iKMoP0bm9qJSGEv0u6QNLNklZIulOZgWBS5tzGs81smZn9LM/ihyrTY35fmUFm5zQ+/NkU\nM7vazK6Ofh4s6XhlwrrI/v8+HocXsi7EpmIymOUoSbc3OjSOZKuYHLIvTK2KyaAy37ifKGl+9Fou\nknRaCGFKIetCrComhyGElSGEReseyowTWhVC+LCQdSWFRQN8AAAAACA1OCIDAAAAIHXoyAAAAABI\nHToyAAAAAFKHjgwAAACA1KEjk3BmtoeZzY+7HaheZBBJQA6RBOQQcSODuejIFMDMHjOzpWbWsYB5\nh5hZMLOy3Gw02t69UfsWmdkV5do2yiepGTSzjmZ2nZnNNbMVZjbNzMa29XYRj6TmMNrejWa20MyW\nm9ksMzuuHNtF+SU8h73M7A4z+yzaLx5Wju2ivJKawWr8nUxHphlmNkTSbspc9/2bsTYmvwmSFitz\nHfPRknaX9KNYW4SSSngG2ylzh+LdJW0g6WxJt0ZtRgVJeA6lzL0XhoQQeijTvvPNbPuY24QSS0EO\nr5S0RlIfSYdLusrMvhBvk1BKCc9g1f1OpiPTvCMlPStpkjI38pMkmVlnM7s46vV+YmZPmllnSU9E\nsyyLbrK2s5mNN7Mbs5bN6Z2b2TFmNjPqPb9jZse3oH2bSbo1hLAquqHRfZLYaVaWxGYwhPBZCGF8\nCGFOCKEhhHC3pHcl8Qdk5UlsDiUphDA9hLB63WT0GNaqV4wkSmwOzayrpO9K+p8QwqchhCclTZF0\nRCleOBIjsRmsxt/JdGSad6Skm6LHvmbWJ6pfpEwwdlHmbqxnSmqQ9JXo+Z4hhG4hhGcK2MZiSV+X\n1EPSMZIuNbPt8s1oZhPMbEJW6Q+SDjGzLmbWX9JYZTozqBxJz2D2c30kjZQ0vZAXhlRJfA6j2kpJ\nb0haKOneFrw+pEOSczhS0toQwqysWV4RXy5WmiRnsPFzFf87mbEU62Fmu0oarMwRjyVm9rakw8zs\nMkk/kLRTCGFBNPvT0TIt3k4I4Z6sycfN7AFlDlu+lGfexqeNPSFpnKTlkmolTZZ0Z4sbgURKSQbX\ntbW9Mjv2ySGEN1rcCCRWWnIYQviRmZ0iaWdJe0ha3XgepFcKcthNmd/F2T6R1L3FjUAipSCD2W2t\nit/JHJFZv6MkPRBCWBJN3xzVNpbUSdLbpdiImY01s2fN7GMzWyZp/2gbzS1Xo8zRl9sldY2W2VDS\nb0vRLiRCojOYtXyNpL8oc274yaVoExIlFTmUpBBCfXRKzwBJJ5aiXUiMpOfwU2W+Qc/WQ9KKUrQL\niZD0DK5bvmp+J3NEpgnReY0HS6o1s0VRuaOknsoMrF+lzPnXrzRaNORZ3WeSumRNb5q1nY6S/qHM\nocq7Qgh1ZnanpEK68L0kDZJ0RXRu+Gozu0HS+coc0kSKpSSDsszXTdcpM7h1/xBCXSHLIR3SksM8\n2okxMhUjJTmcJamdmY0IIbwV1bZRBZ/WU01SksGq+53MEZmmHSipXtIoZa4GNlrSlpL+rUy4rpd0\niZn1M7PaaPBWR0kfKnNO5NCsdU2T9BUzG2RmG0j6RdZzHZT5j/ChpLWWuUze1wppYPSNwLuSTjSz\ndmbWU5lvBl4t9kUjURKfwchVUbu+EUL4vIjXiWRLfA7NrLeZHWJm3aI27CvpUEkPF/+ykTCJz2EI\n4TNlzpA418y6mtmXJX1LmW/GkX6Jz2Ckun4nhxB45Hkoc8rWxXnqB0tapMw5r3+QtECZc2CfkNQ5\nmudcZQK4TJnzJaXMJRmXSZot6YfK9NDbRc+dJOmD6Pm/SPqrpPOj5/aQND9r+1dLujprerSkxyQt\nlbRE0q2S+sT9/vGojgwqc65wUOabqE+zHofH/f7xqKocbiLp8Wi55ZJek/TDuN87HtWVw2i6lzLj\nVD+T9J6kw+J+73hUTwZVhb+TLXrhAAAAAJAanFoGAAAAIHXoyAAAAABIHToyAAAAAFKHjgwAAACA\n1GlVR8bM9jOzN81stpmdVapGAS1BDpEE5BBxI4NIAnKIcir6qmVmVqvMzZ/2kTRf0guSDg0hzGhq\nmQ7WMXRS16K2h8q2QkuXhBA2aely5BClskqfaU1YXdTNF1uaQzKIprAvRBKUK4dkEE0pNIPtWrGN\nHSXNDiG8I0lm9ldlbvzU5E6zk7rqS7ZXKzaJSvVQuG1ukYuSQ5TEc6FV905sUQ7JIJrCvhBJUK4c\nkkE0pdAMtubUsv6S5mVNz49qOcxsnJlNNbOpdVrdis0BeZFDJEGzOSSDaGPsC5EE7AtRVm0+2D+E\nMDGEMCaEMKa9Orb15oC8yCHiRgaRBOQQcSODKKXWdGQWSBqYNT0gqgHlRA6RBOQQcSODSAJyiLJq\nTUfmBUkjzGwzM+sg6RBJU0rTLKBg5BBJQA4RNzKIJCCHKKuiB/uHENaa2cmS7pdUK+n6EML0krUM\nKAA5RBKQQ8SNDCIJyCHKrTVXLVMI4V5J95aoLUBRyCGSgBwibmQQSUAOUU5tPtgfAAAAAEqNjgwA\nAACA1KEjAwAAACB16MgAAAAASB06MgAAAABSh44MAAAAgNRp1eWXAQAAKpHtsLWr3fCPq1xt7O/P\ndLU+lz/dJm0CkIsjMgAAAABSh44MAAAAgNShIwMAAAAgdejIAAAAAEgdBvvH5N4FL7naV177nqtZ\nnmXfn7WJq4348XOlaBaAlKnp0sXVrHOngpZtWP6pq4W6Na1uE5A2c8/d2dXq8/w3OvKIU1ytz7/5\n/VsNws7buNrbB3V2tSF317lau0debJM2gSMyAAAAAFKIjgwAAACA1KEjAwAAACB1WjVGxszmSFoh\nqV7S2hDCmFI0CmgJcogkIIeIGxlEEpBDlFMpBvt/NYSwpATrqSoNCq72yNZ/c7WaPAfNGrZucLUv\nv/JjV9voumeKbF0qkUMkQdlz+MWnVrraub0fL2jZb715oKutvrhfznTXl+a5edYuXFRg6xAD9oVF\n2Oy2pa52zd1/crWxy850tQGP17dJm1Ku4nL4+ab+6g/PHXSxq804sKurnXPica7W4f6ppWlYlePU\nMgAAAACp09qOTJD0gJm9aGbj8s1gZuPMbKqZTa3T6lZuDsiLHCIJ1ptDMogyYF+IJGBfiLJp7all\nu4YQFphZb0kPmtkbIYQnsmcIIUyUNFGSelgvfz4V0HrkEEmw3hySQZQB+0IkAftClE2rjsiEEBZE\n/y6WdIekHUvRKKAlyCGSgBwibmQQSUAOUU5FH5Exs66SakIIK6Kfvybp3JK1rMJt+0d/d+B8rj/h\nMr9sB9///NrJT7nai9dV/hCoasxhbZ/errZq64Gu9u6h1uy6Zu830dXyXYiiUE+s6uBqF3/juznT\n9TNmFb3+pIozh+f39neM9pcDye+uze/0xUaRuH/lBm6WS0473NU6ve8vOqDX33KlULemwNahJapx\nX1hKDa++4WonfPEAVxv46fOuxiGF/1fJOexyx3Ou9tXhZ7ja1NP8320Dx/vfex/cX5p2VbvWnFrW\nR9IdZrZuPTeHEO4rSauAwpFDJAE5RNzIIJKAHKKsiu7IhBDekbRNCdsCtBg5RBKQQ8SNDCIJyCHK\nrfLPPQIAAABQcejIAAAAAEid1l5+GUXq/9unC5rvsG1+6GrTd7+21M1BQi0+eRdX+8GJ97jauJ6+\nVoiGPN9l/HrJ1q62QTs/kPvEnn4g966dVrna+QN75Ex3mNGSFqI5Wzzi7xg9Y09/EYeP6/39Gq78\neGdXO7Rn7mDmfbt84ubZd+KEgtq234zvudqip/rnTA8aX9i+ECi3+mU++0C2fr/3+6+9Z5/sao9c\n4feZW9yYu+8e/v2XS9ewKsIRGQAAAACpQ0cGAAAAQOrQkQEAAACQOnRkAAAAAKQOg/1TqCZP//OW\nF3d0tZGaWo7moIQWn5Q7uP+uM3/n5ulT29HVXl/j7y19yDPjXK3D611ypvs/9pmbp92b81zN2rd3\nte89/7qrbZKnbXMOyW3bSO5mXFIjfjDd1b581I9dbWUfc7WBF/iBqvf/4PSc6Un/e4mbZ3j7wn51\n3DfqNl8clTu535f9BQEa/tjH1bq9tsjV1s55r6B2AMWo3XKEq302fENX6/LucldreP2NNmkT0qlB\nDa7WaUbnGFpSeTgiAwAAACB16MgAAAAASB06MgAAAABShzEyCXfjTte5Wr5zLQfdRZ80bWp79HC1\nLQ+bmTN9zvtj3TyzLh7laj3u83eZHLpiWlHtqs9Tq+ne3dV8CvMbfBvZbEuhbo2rbfSnZ3ytwPX1\nuj532dNmnejmWdOzg6stPW6Fqx09/DlXO2nDN3OmHxh1u5un4WqfrjMW+pvDvnrODq7W8Z4XXA1o\nrPH4l5ln+v3x43td5mqD2nVztedX17na+K9/39Xqp7/pagBah78wAAAAAKQOHRkAAAAAqUNHBgAA\nAEDqNNuRMbPrzWyxmb2eVetlZg+a2VvRv/7C6kAJkUMkATlE3MggkoAcIikKGew/SdIVkv6cVTtL\n0sMhhAvN7Kxo+uelbx6+/+yxrjZzd38BgE7/fL4czYnTJFVYDuuX+5uoffTl5pfrJj+AutCB98V6\n63+3crU+tY+42uTlg12t67QFOdNrS9esOExSheWwOTVP+otGdMozX9+7fe3hgf7CFP8aunvO9Pw9\n/E3hfnLIna72+77+5p0fXPWoqx15wk9creO9FXUBgEmqsgy2RO2oka7W5ZqPXe33g27Imd6svR/E\nv7Tef9e7uN7fRHjHjl1d7ePR/m/4Dfy9a9Nskshhq3RZ6G9kjZZr9ohMCOEJSY33At+SNDn6ebKk\nA0vcLiAHOUQSkEPEjQwiCcghkqLYyy/3CSEsjH5eJKlPUzOa2ThJ4ySpk7oUuTkgL3KIJCgoh2QQ\nbYh9IZKAfSHKrtWD/UMIQVKTx8dCCBNDCGNCCGPaq2NrNwfkRQ6RBOvLIRlEObAvRBKwL0S5FNuR\n+cDM+kpS9O/i0jUJKBg5RBKQQ8SNDCIJyCHKrthTy6ZIOkrShdG/d5WsRVWi3cABrjbjnE1d7Zad\nrnG1K5cNa5M2pRA5bAN1e2/vao/81+/zzOm/Sbv0r/6U6EEL/CDtCkMOm7B23nxXq2lUG/S4X+7O\n23bzxdt86agec11t9Qa1rlYF3/mSwUjd5Z+72m3DHnK1JfWWM33se7u6eabe8kVXu+bUy12tt4+c\nLj5vgquNn5978Z6ax1/2C6Zb1eWwpnt3V+tz+tsFLdvrhmdK3ZyqVMjll2+R9Iykzc1svpkdq0xI\n9zGztyTtHU0DbYYcIgnIIeJGBpEE5BBJ0ewRmRDCoU08tVeJ2wI0iRwiCcgh4kYGkQTkEEnR6sH+\nAAAAAFBudGQAAAAApE6xg/2xHvnuLKz5i3Im3z5ukJtl1lg/iPCDej9w8b6j8wyE1WsFtw/4jxo/\nSnXe1zq4Wp9aP1z6zbp6Vxvw8MrStAtV7eNt/V3R9+s6K8+cVTCMH01675xdXG3aFpe52o8W+N+Z\nc44enDNdP/1NN0/Dz/w2u9escbXhN//E1U4ee5+rnfKnv+VMX73DDm6e+mWf+I0iscLmg13tlqGT\nXW2v1w92tc56t03aVG04IgMAAAAgdejIAAAAAEgdOjIAAAAAUoeODAAAAIDUYbB/G/jfu29xtRMv\nOSVnetIRfmB/gxpc7au3nOFqQ1/gbrAojfd/9iVXe/1wP1g2n8OuOt3V+j/5dKvbhOpSu6Ef2H/3\nby52te41fmD/k6s6udoGs1a4WiiybUiOTw/y+6pp4/y+6k+fDHW1OUcOdLX6mbmD+2s32cTNs6q3\nT84px53sasMeetbV7rvpy6725ykTc6Yn3N7NzaM9GeyfJm8d5j/DZ1b7i+h0O8UfN/CXy0ExOCID\nAAAAIHXoyAAAAABIHToyAAAAAFKHMTKt9NGxO7vaDh1fcrUfnHhPo3nMzTNh2XBXG/pzxsOg7fT+\n2vyC5rvrs41dbeDl01zNj/JCNavp5MewfPKt0TnTT196tZunLvjl8qnNM/pl1tF5xh0cnTu+YtIB\n17hZdu7oz1gfed/xrjbqbP9/Zu3CRa6G1qnt0SNnepszX3HzPPq5/6zv+e5OrlY/861mt1f/4Yeu\nNvRMXytUeHm6q+365I9ypu/Y2Wf/dPm/KZAMNV27utrF37jR1Y6ZcoKrDZ/lx1GhNDgiAwAAACB1\n6MgAAAAASB06MgAAAABSp9mOjJldb2aLzez1rNp4M1tgZtOix/5t20xUO3KIJCCHiBsZRBKQQyRF\nIYP9J0m6QtKfG9UvDSFcVPIWJcWOW7vSO9/zAwsfPuT3rtagzq526UNjc6b/uvliN88jW//N1f74\nux+72tAzq/ICAJNUjTkssfo9tsuZvm/La908+Qbs/2rS4a42YGVV3vxyksiharp397VNNnK1+j/V\nudqjm+feDLgu+O/T8t0cOJ+dO612tZnfuaKgZf02vTf2u8oX9/Olb/bfoahtFmmSqiCDy8aOypme\n0N8PjB9+sx9UPWxmcgdV16+tqJNgJqkKcpjt7euGudrLK5e52vDTkpvBStTs/6oQwhOSPi5DW4Am\nkUMkATlE3MggkoAcIila8/XAyWb2anR4ccOmZjKzcWY21cym1sl/ewa0EjlEEjSbQzKINsa+EEnA\nvhBlVWxH5ipJwySNlrRQ0sVNzRhCmBhCGBNCGNNeHYvcHJAXOUQSFJRDMog2xL4QScC+EGVXVEcm\nhPBBCKE+hNAg6VpJO5a2WUDzyCGSgBwibmQQSUAOEYdCBvs7ZtY3hLAwmvy2pNfXN38a1I4amTO9\n/NzP3DxvbD3Z1Y6bN9bVvrXRy6428sbc9dV190dctzzwZFd76/ArXW3Me36+3ldU38DrSsxhKdX2\n3MDV1p69JGe6vdW6eQ5+52uuNuDX1ZevQlV6Dj8/0P8tMuAMf6f0G4b8vaj1v7zGD7M/5HE/iLu2\nQ72rdXmhi6t91t+vb+T27+VML5gyxM1z3HH3uNq4nrNdLYkqPYNNGf43/3s6xNCOfNoNHOBqD30l\n90IX57yf76Jey9uoRW2v0nL48TE750zfv7O/uNM+t/3M1YYp3YP9a0cMdbW5B23qasH8slvs6383\nTH98eM50u8/zLJjPBbcVNFuzHRkzu0XSHpI2NrP5ks6RtIeZjVZmnzFH0vGFtQooDjlEEpBDxI0M\nIgnIIZKi2Y5MCOHQPOXr2qAtQJPIIZKAHCJuZBBJQA6RFBV1UXMAAAAA1YGODAAAAIDUKWqwfyXq\ne/37OdN3DHzUzfPCat/vm/eLEa527Vs9XC3Mfy1numMPP88WU3279r3jOFfr8vNFrvbR5zu72kbX\nPeNXiKox56QvuNrLW16WM71wrb+G/1u3bO5qvcVg/7RrfEETSdKaOleaefomOdO35Lm7/bYd/YD6\nQl24ZJuc6WeO297NM+KFF4tefz6NLxOwqd5389x5wDaulpbB/tWqdtFSV1sbRzs26uVrN/qWrAq5\nf0MsObBTnrWld7B/pTn4pw/kTD/+uR8EP/gevw/99OCdXG3h19cU1YbvbuUvHvXrPnn+WMwj38V8\n6oK/aEpjC+ufdLXbln/R1WrM/x5oCP7v5F2//XbO9OVP7+Xm6TKnfbPtagpHZAAAAACkDh0ZAAAA\nAKlDRwYAAABA6tCRAQAAAJA6VTnYP98ddycOnJIzfdy8Pd087++0wtVq9ZKrFTLYsH55YQP6ah/1\n6+/mr0OgS9/9m6v98p3ce1HlWxcqg23rB/b/c9zv8szZMWdq19v9XYmHX8nA/jSxdn43/tZFY1zt\n0gP+7Gof13dztUO7LyiqHb9c9CVXe/wqX9t4cu5A/lD3mpunreX7HXDS4IcKWnanqUe4Wm+90eo2\nIVdtXciZrg9+YPGi/Qe62sbXzG+zNklSTffurjbzvOGu9vRml7jaXteemTM98AP2tUl22oazcqYb\n5DN46F+uLnr9NY2OJeRb/x+XbuFqu7yc7xY+xVk+bSNXG3T/Kler+be/6ECxRuqFguYrdK/KERkA\nAAAAqUNHBgAAAEDq0JEBAAAAkDp0ZAAAAACkTlUO9p972CBXa1DuwMJn/7W1m2dQgu9uftovTnG1\nlVvl9lP75LlIAFLIzJXm/tJ/JzGgXUdXa2zw3f6uxEgX69zZ1S4Y6y/+sW+XT/Isna+W67nV/o7L\np19woqttcuvrrrbRimdcLbhK26vZZsuc6RHXv+Xmyf/+eH3O9+9HHK+p0nW5/bmc6WkX+8vodDhw\nsV/wmtK1od2A/q627d3vudq1vfzA/m+OP8PVBl6f3L8h4O18zsk508u2LP5/es8Z/vd2nwfmNbtc\nWOEvMtVr2aw8cxanV8nWFB+OyAAAAABIHToyAAAAAFKn2Y6MmQ00s0fNbIaZTTezU6N6LzN70Mze\niv7dsO2bi2pFDhE3MogkIIdIAnKIpChkjMxaST8NIbxkZt0lvWhmD0o6WtLDIYQLzewsSWdJ+nnb\nNbV0Oi3x5zm+vCb3RkQ/OvgeN8/k+fu72kbX+XPA29pHx+7salPPu8rV6kJ9zvTXL9++zdpUBhWX\nw2J9dNxOrjZtlz8WtOwXnzw2Z3rIQy82MSfySGQGG/KcQ/3ra/wN037/1SWu9vGHPVxt+OTc/Ua7\npZ+7eTZ61e/3/K3c4pHvZpdvHdozZ/qaTR7Ls6QfU/aFm3/sasNfLuxmbm0okTlsa8dddJqrnXrK\nba5260ZbuVr9Rx/7FdbU5kx++l1/E9njz/uHq+3Uea6r7XP9ma42qPLHw1R8Djf6U+5+zt86snUK\nuXk6mtfsEZkQwsIQwkvRzyskzZTUX9K3JE2OZpss6cC2aiRADhE3MogkIIdIAnKIpGjRGBkzGyJp\nW0nPSeoTQlgYPbVIUp+StgxoAjlE3MggkoAcIgnIIeJUcEfGzLpJ+oek00IIy7OfCyEENXEFSjMb\nZ2ZTzWxqnVa3qrEAOUTcyCCSgBwiCYrJIRlEKRXUkTGz9soE9aYQwu1R+QMz6xs931dSngu6SyGE\niSGEMSGEMe3znIMMFIocIm5kEElADpEExeaQDKKUmh3sb2Ym6TpJM0MI2Xd9miLpKEkXRv/e1SYt\nbAP5BugfNfDUnOnjD7rXzXNGUdm2AAAgAElEQVT7Ob93tb1G+pteDf156S4A8M7v/MD+hw/x7agL\n/qZ4m992Us70CD1bsnaVWyXmsFifDC9+2aHnrcmZbs0A7Q9P8Nnc5OryX/yiXNKUwb6X5Blo7O/Z\np40LWFdSBvHns/I7X3K1zc6Y6Wq3D2p8MQz/x9Nu0w5zteG/9BfDCGvjHaKbphyWUr+7/CD7D07Y\nwNX63rPG1V68aRdX+3TnlTnTs/fwd9Kc8lkXVzvloBNcbdDzFT+w36nWHCJ5Crlq2ZclHSHpNTOb\nFtV+qUxIbzWzYyXNlXRw2zQRkEQOET8yiCQgh0gCcohEaLYjE0J4UpI18fRepW0OkB85RNzIIJKA\nHCIJyCGSokVXLQMAAACAJKAjAwAAACB1ChkjUxUGjc8drPeve3dz8wy5yd8Ze+Tl77lasUNBF965\npasdNPgpV3thVT9X+81vD3e1EXkuaoD06zt6UUHzjbr1FFcb8cbUnOmaLn4w66KjR7vaD066x9X+\ndk6Sh4Gj0rTru6mrzd/HX2F4/oyRrrZFnlpjW1683NXq6/zAccRj7fwFrvbEWP+5rp3sz3aa9osJ\nrjZzTe5g/+E3n+7mGXnhW74hS15bXzMBlBlHZAAAAACkDh0ZAAAAAKlDRwYAAABA6tCRAQAAAJA6\nDPZvyvN+QN+lp/s7P3eTH4BYkB23dqW7t7vK1fa65QxXe+Xyga620XwG9leL80fcUdB8YcM6V2v4\n0lY50/tc8283z3e6/87Vxt7kczj0rjx3PS+oZUDLrV3oL3Ix8sTCLnxRiPqSrQnlku8CAPnuYLKv\n/AVMGhumZ12NTADJxxEZAAAAAKlDRwYAAABA6tCRAQAAAJA6dGQAAAAApA6D/Vug0z+fd7W1xa4s\nz8UEfjhoV1cbKj+Iv+htoiIc89QxrjZjz4muNnOfq/3C++RO1uT5LmPLx092tWG/9DlkYD8AAIgT\nR2QAAAAApA4dGQAAAACp02xHxswGmtmjZjbDzKab2alRfbyZLTCzadFj/7ZvLqoVOUTcyCCSgBwi\nbmQQSVLIGJm1kn4aQnjJzLpLetHMHoyeuzSEcFHbNQ/4D3KIuJFBJAE5RNzIIBKj2Y5MCGGhpIXR\nzyvMbKak/m3dMCAbOfx/W/zyQ1f7w72jXO20XjNc7bwPt8uZvv+P/gITI2+b7mrc4ZoMIhnIIeJG\nBpEkLRojY2ZDJG0r6bmodLKZvWpm15vZhiVuG5AXOUTcyCCSgBwibmQQcSu4I2Nm3ST9Q9JpIYTl\nkq6SNEzSaGV65hc3sdw4M5tqZlPrtLoETUY1I4eIGxlEEpBDxI0MIgkK6siYWXtlwnpTCOF2SQoh\nfBBCqA8hNEi6VtKO+ZYNIUwMIYwJIYxpr46lajeqEDlE3MggkoAcIm5kEEnR7BgZMzNJ10maGUK4\nJKveNzpPUpK+Len1tmkiQA6zrZ0339Ue2bqrr2mHZtfVK88NVxkPkx8ZRBKQQ8SNDCJJCrlq2Zcl\nHSHpNTObFtV+KelQMxutzA2+50g6vk1aCGSQQ8SNDCIJyCHiRgaRGIVctexJSZbnqXtL3xwgP3KI\nuJFBJAE5RNzIIJKkRVctAwAAAIAkoCMDAAAAIHXoyAAAAABIHToyAAAAAFKHjgwAAACA1KEjAwAA\nACB16MgAAAAASB0LIZRvY2YfSporaWNJS8q24dJLe/ul5L2GwSGETcqxIXKYGElrfxwZlJL3PrQU\n7S8t9oUtR/tLryw5ZF+YKElrf0EZLGtH5j8bNZsaQhhT9g2XSNrbL1XGa2ittL8HtL8ypP19oP3p\nl/b3gPZXhrS/D7Q/HpxaBgAAACB16MgAAAAASJ24OjITY9puqaS9/VJlvIbWSvt7QPsrQ9rfB9qf\nfml/D2h/ZUj7+0D7YxDLGBkAAAAAaA1OLQMAAACQOmXvyJjZfmb2ppnNNrOzyr39ljKz681ssZm9\nnlXrZWYPmtlb0b8bxtnG9TGzgWb2qJnNMLPpZnZqVE/Nayi1tGVQIoeViByWFxnML205THMGJXKY\nT9oyKJHDJClrR8bMaiVdKWmspFGSDjWzUeVsQxEmSdqvUe0sSQ+HEEZIejiaTqq1kn4aQhglaSdJ\nJ0XveZpeQ8mkNIMSOawo5DAWZLCRlOZwktKbQYkc5khpBiVymBjlPiKzo6TZIYR3QghrJP1V0rfK\n3IYWCSE8IenjRuVvSZoc/TxZ0oFlbVQLhBAWhhBein5eIWmmpP5K0WsosdRlUCKHFYgclhkZzCt1\nOUxzBiVymEfqMiiRwyQpd0emv6R5WdPzo1ra9AkhLIx+XiSpT5yNKZSZDZG0raTnlNLXUAKVkkEp\npZ8hOZREDmNFBv+jUnKYys+QHEqqnAxKKf0M055DBvu3Ushc9i3xl34zs26S/iHptBDC8uzn0vIa\n0LS0fIbksLKl4TMkg5UtLZ8hOaxsafkMKyGH5e7ILJA0MGt6QFRLmw/MrK8kRf8ujrk962Vm7ZUJ\n6k0hhNujcqpeQwlVSgallH2G5DAHOYwBGXQqJYep+gzJYY5KyaCUss+wUnJY7o7MC5JGmNlmZtZB\n0iGSppS5DaUwRdJR0c9HSborxrasl5mZpOskzQwhXJL1VGpeQ4lVSgalFH2G5NAhh2VGBvOqlBym\n5jMkh06lZFBK0WdYUTkMIZT1IWl/SbMkvS3pv8u9/SLae4ukhZLqlDl381hJGylzNYe3JD0kqVfc\n7VxP+3dV5tDgq5KmRY/90/Qa2uA9SVUGozaTwwp7kMOyt50M5n9fUpXDNGcwaj859O9JqjIYtZkc\nJuRh0QsCAAAAgNRgsD8AAACA1KEjAwAAACB16MgAAAAASB06MgAAAABSh44MAAAAgNShIwMAAAAg\ndejIAAAAAEgdOjIAAAAAUoeODAAAAIDUoSMDAAAAIHXoyAAAAABIHToyAAAAAFKHjgwAAACA1KEj\nAwAAACB16MgAAAAASB06MgAAAABSh44MAAAAgNShIwMAAAAgdejIAAAAAEgdOjIAAAAAUoeODAAA\nAIDUoSMDAAAAIHXoyAAAAABIHToyAAAAAFKHjgwAAACA1KEjAwAAACB16MgAAAAASB06MgAAAABS\nh44MAAAAgNShIwMAAAAgdejIAAAAAEgdOjIAAAAAUoeODAAAAIDUoSMDAAAAIHXoyAAAAABIHToy\nAAAAAFKHjgwAAACA1KEjAwAAACB16MgAAAAASB06MgAAAABSh44MAAAAgNShIwMAAAAgdejIAAAA\nAEgdOjIAAAAAUoeODAAAAIDUoSMDAAAAIHXoyAAAAABIHToyAAAAAFKHjgwAAACA1KEjAwAAACB1\n6MgAAAAASB06MgAAAABSh44MAAAAgNShIwMAAAAgdejIAAAAAEgdOjIlYmZDzCyYWbto+l9mdlQR\n6xlkZp+aWW3pW4lKRgaRBOQQcSODSAJyWB5V15Exszlm9nkUig/MbJKZdSv1dkIIY0MIkwtsz95Z\ny70XQugWQqgvdZui/1T3mtlSM1tkZles+w+G8qnyDD5mZqui1/6pmb1Z6m2gMNWcw2h7h5jZTDP7\nzMzeNrPd2mI7aFq1ZtDMOprZdWY218xWmNk0Mxtbym2gcNWaw2hbnzZ61JvZ5aXeTluquo5M5Bsh\nhG6StpM0RtLZ2U9aRiW+NxMkLZbUV9JoSbtL+lGsLape1ZpBSTo52il3CyFsHndjqlxV5tDM9pH0\nW0nHSOou6SuS3om1UdWrGjPYTtI8ZX4Hb6DMa77VzIbE2KZqV405VNbv4m6SNpX0uaS/x9ysFqm4\nD6UlQggLJP1L0lbRN8UXmNlTklZKGmpmG0Tfmiw0swVmdv66Q3tmVmtmF5nZEjN7R9IB2euO1ndc\n1vQPo2//VpjZDDPbzsz+ImmQpH9GPeEz8xyK7GdmU8zsYzObbWY/zFrneDO71cz+HK13upmNWc9L\n3kzSrSGEVSGERZLuk/SFkryZKEoVZhAJVIU5/JWkc0MIz4YQGkIIC6L3ADGppgyGED4LIYwPIcyJ\n8ne3pHclbV/SNxUtVk05zOO7ynzZ/e/i38EYhBCq6iFpjqS9o58HSpou6TxJj0l6T5k/7NtJai/p\nDknXSOoqqbek5yUdHy17gqQ3onX0kvSopCCpXfT8Y5KOi34+SNICSTtIMknDJQ1u3J5oekij9Tyh\nzJGUTsocRflQ0p7Rc+MlrZK0v6RaSb+R9GzWuiZImpA1fbykP0vqIqm/pNclfTvuz6TaHlWewcei\n5ZdIekrSHnF/HtX6qNYcRs+vkXSWpNmS5ku6QlLnuD+TantUawbzvA99omW3iPszqcYHOfzPc49I\nGh/359Hizy/uBsQU2E8lLZM0N/pQO0cBOzdrvj6SVivrl5ukQyU9mvWBn5D13NfWE9j7JZ26nvbk\nDWz0n6FeUves538jaVL083hJD2U9N0rS5+t57VtKelHS2mgbkyRZ3J9JtT2qPINfUuZUno6SjpK0\nQtKwuD+TanxUaw4l9YvWO1WZ02w3VqZTfUHcn0m1Pao1g4222V7SQ5KuifvzqNYHOQySNDha72Zx\nfx4tfVTrQO8DQwgPZRfMTMqcs7rOYGV2MAuj56TMqXjr5unXaP6569neQElvF9HOfpI+DiGsaLSd\n7MOEi7J+Ximpk5m1CyGszV6RZc7tvE/SREm7SOom6XplzhM/s4i2oXWqLoOSFEJ4Lmtyspkdqsw3\nR6kaXFhBqjGHn0f/Xh5CWChJZnaJMufE/3cRbUPrVGMGJf3n9/JflDlCeHIRbULpVG0OI0dIejKE\n8G4RbYpVtXZkmhKyfp6nTM974yY+/IXKBHGdQetZ7zxJwwrYZmPvS+plZt2zQjtImcORLdUrWvaK\nEMJqSavN7AZJ54uOTJJUcgab2rY1OxfKrWJzGEJYambzG21vfdtGPCo2g1Jm8Lik65T5ln//EEJd\nMetBm6voHGY5UtKFrVxHLKp6sP/6RN/UPSDpYjPrYWY1ZjbMzHaPZrlV0o/NbICZbajM+dZN+ZOk\nn5nZ9pYx3MwGR899IGloE22YJ+lpSb8xs05m9kVJx0q6sYjXs0SZwYQnmlk7M+upzKk9r7Z0XSiP\nSsugmfU0s32j9bQzs8OVuVrUfS1dF8qn0nIYuUHSKWbWO2rzTyTdXeS60MYqNINXKXO69zdCCJ83\nNzPiV6E5lJntosy46VRdrWwdOjLrd6SkDpJmSFoq6TZlzqmWpGuVOcfxFUkvSbq9qZWEEP4u6QJJ\nNyszJuBOZY6QSJlzG882s2Vm9rM8ix+qzPmR7yszyOycxoc/m2JmV5vZ1Vml70jaT5mBYbMl1Snz\nCxzJVUkZbK/MEcB1g/1PUeZw/qxC1oVYVVIOpcxA3hckzZI0U9LLUbuQXBWTwegP1uOVGai9yP7/\nHh6HF7IuxKpicpjlKEm3NzpdLTUsGuQDAAAAAKnBERkAAAAAqUNHBgAAAEDq0JEBAAAAkDp0ZAAA\nAACkDh2ZhDOzPaJ7HgCxIINIAnKIuJFBJAE5zEVHpgBm9piZLTWzjgXMO8TMgpmV5WajZnaymU01\ns9VmNqkc20T5JTyDW5rZI2b2iZnNNrNvl2O7KL8k5zBruyPMbJWZFX1fBSRXkjMYbe/eqH2LzOyK\ncucf5UEOk4OOTDPMbIik3ZS50+o3Y21Mfu8rc2+O6+NuCNpGkjMY7RzvUuZmgr0kjZN0o5mNjLVh\nKLkk57CRK5W5RwwqTAoyOEHSYmXuKzJa0u6SfhRri1By5DBZ6Mg070hJz0qapMxNgyRJZtbZzC42\ns7nRN9FPmllnSU9EsyyLbnC1s5mNz/52sHHv3MyOMbOZZrbCzN4xs+MLbVwI4fYQwp2SPirBa0Uy\nJTmDW0jqJ+nSEEJ9COERSU9JOqLVrxpJk+QcrlvfIZKWSXq4dS8VCZX0DG4m6dYQwqoQwiJJ90n6\nQuteMhKIHCYIHZnmHSnppuixr5n1ieoXSdpe0i7KfBN9pqQGSV+Jnu8ZQugWQnimgG0slvR1ST0k\nHSPpUjPbLt+MZjbBzCYU+2KQSmnLoEnaqoBtIl0SnUMz6yHpXEmnt/SFITUSnUFJf5B0iJl1MbP+\nksYq80ckKgs5TJCKPWeuFMxsV0mDlenZLjGztyUdZmaXSfqBpJ1CCAui2Z+OlmnxdkII92RNPm5m\nDyhz2PKlPPNW7OFBeCnI4JvK7HDPMLNLJX1VmcPYj7a4EUisFORQks6TdF0IYX4x20aypSSDTyhz\neu1ySbWSJku6s8WNQGKRw+ThiMz6HSXpgRDCkmj65qi2saROkt4uxUbMbKyZPWtmH5vZMkn7R9sA\nEp3BEEKdpAMlHSBpkaSfSrpVEldUqSyJzqGZjZa0t6RLS9EOJFLSM1ijzLfet0vqGi2zoaTflqJd\nSAxymDAckWlCdF7jwZJqzWxRVO4oqacyA6hWSRom6ZVGi4Y8q/tMUpes6U2zttNR0j+UOVR5Vwih\nzszuVOb0HFSxtGQwhPCqMkdh1q3vaWW+AUIFSEkO95A0RNJ70bef3aL2jgoh5D0dA+mRkgz2kjRI\n0hUhhNWSVpvZDcpcjOfMApZHwpHDZOKITNMOlFQvaZQyV30YLWlLSf9WJlzXS7rEzPqZWW00eKuj\npA+VOSdyaNa6pkn6ipkNMrMNJP0i67kOyvxH+FDSWjMbK+lrhTbSzNqZWSdlDh/Wmlknq+DL7FWZ\ntGTwi1HuupjZz5TZoU8q6hUjidKQw4nK/AGxrn1XS7pH0r5FvF4kT+IzGH1D/66kE6Pfyz2V+ab+\n1WJfNBKHHCYQHZmmHSXphhDCeyGEResekq6QdLiksyS9psxlPj9W5rBdTQhhpaQLJD1lZsvMbKcQ\nwoOS/qZMkF5U5lK1kqQQwgpJP1bmdJylkg6TNKWpRpnZ1WZ2dVbpbEmfR+35fvTz2aV4AxC7tGTw\nCEkLlRkrs5ekfaJvglAZEp/DEMLKRm37VNKqEMKHpX0rEJPEZzDyHUn7KfMH6GxJdZJ+Uoo3AIlA\nDhPIQsh3xAsAAAAAkosjMgAAAABSh44MAAAAgNShIwMAAAAgdejIAAAAAEidVnVkzGw/M3vTzGab\n2VmlahTQEuQQSUAOETcyiCQghyinoq9aZma1kmZJ2keZu3i/IOnQEMKMppbpYB1DJ3UtanuobCu0\ndEkIYZOWLkcOUSqr9JnWhNVF3Yi2pTkkg2gK+0IkQblySAbRlEIz2JobJ+4oaXYI4R1JMrO/SvqW\npCZ3mp3UVV+yvVqxSVSqh8Jtc4tclByiJJ4LD7dm8RblkAyiKewLkQTlyiEZRFMKzWBrTi3rL2le\n1vT8qAaUEzlEEpBDxI0MIgnIIcqqNUdkCmJm4ySNk6RO6tLWmwPyIoeIGxlEEpBDxI0MopRac0Rm\ngaSBWdMDolqOEMLEEMKYEMKY9urYis0BeZFDJEGzOSSDaGPsC5EE7AtRVq3pyLwgaYSZbWZmHSQd\nImlKaZoFFIwcIgnIIeJGBpEE5BBlVfSpZSGEtWZ2sqT7JdVKuj6EML1kLQMKQA6RBOQQcSODSAJy\niHJr1RiZEMK9ku4tUVuAopBDJAE5RNzIIJKAHKKcWnVDTAAAAACIAx0ZAAAAAKlDRwYAAABA6tCR\nAQAAAJA6dGQAAAAApA4dGQAAAACpQ0cGAAAAQOrQkQEAAACQOnRkAAAAAKQOHRkAAAAAqUNHBgAA\nAEDqtIu7AWlS07Wrq82dvJmr3bPD1a520n7H5EzXz3yrdA1DVWk3cICrvfnbjV3txp2uc7VrF+/e\n7Pr7dFzuai9uy3ceKJ99X/cZPL3XO6428okjXW2zQ15tkzYhXtbO/7liW410tfe/2tPVdj/8BVd7\n9ooxOdMb3znDzVO/7JOWNBFADPjrBAAAAEDq0JEBAAAAkDp0ZAAAAACkTqvGyJjZHEkrJNVLWhtC\nGLP+JYDSI4dIAnKIuJFBJAE5RDmVYrD/V0MIS0qwnsSr23FzV3tt5z/lmbOLq2x549s5069vX6pW\nIVKZOdxxa1f6wY13udo3uy51tQY1uNrVAx9vdp4P6le72l6/PcPVhv78GVdDheawDc2+dCdXu6Pn\nH12tPvhfV9N3u8HVdjnmZFfrdUNVZTV1GWzXv5+rvXv0kJzpb3zvaTfPr3vfWPxGL3guZ/Lf/+Pz\n9btvHuRq9dPfLH6b1SV1OYxbzehRrvbeWH/xim2+PtPVxg+429W+cePPXG3If1fevpBTywAAAACk\nTms7MkHSA2b2opmNK0WDgCKQQyQBOUTcyCCSgByibFp7atmuIYQFZtZb0oNm9kYI4YnsGaIQj5Ok\nTnlOuQJKgBwiCdabQzKIMmBfiCRgX4iyadURmRDCgujfxZLukLRjnnkmhhDGhBDGtFfH1mwOyIsc\nIgmayyEZRFtjX4gkYF+Icir6iIyZdZVUE0JYEf38NUnnlqxlJdKw27auVvPvl8vejrN7P5kzffBu\nJ7l54mhX2qUlh4VqN3BAzvQn537m5sk3sP+elRu42v+8/k1X+2x+95zpN78zwc1z9Ue7uBoD+9ev\n0nJYTtbbX1yioxX2q+nlNf5iFRu9tsLVQsublTppyWDYeRtX++Nfr3S1Ie1yv6l/YbX/FIf/63i/\ngTX++9khU/yyC47Ozd3M3Sa5eQbfe52rjX3uRFcb+qP3Xa1+yUe+bVUgLTlsS7Ujh7nau4f0cbVd\nD3glZ/qsTSe6eYa171bgVru6ystHXeZqX3/8RznT7R+YWuD6k6s1p5b1kXSHma1bz80hhPtK0iqg\ncOQQSUAOETcyiCQghyirojsyIYR3JPmvVoAyIodIAnKIuJFBJAE5RLlx+WUAAAAAqUNHBgAAAEDq\ntPbyy4mXlAH0PWo65UzXd65189CrxIxzNs2ZnrX1NW6eBvkBztd+dXdX6zd/hqu9dfmXml3XlL/u\n6mpDH3/H1eq/73cfcw8b5Gr9H200+Pr519w8qB6LTs29mMSLu1+cZ67CrmR0zEtHu9qAqa8X0Sq0\nhXaDB7raYZP8Hcg3qDFX2+6Fw3Om+/9irZtn5MziByoPfTQ3Y1ufebKbp+/u811t+pcnu9qDz3Z2\ntTMmHutq/X73dEuaiIQJu/gz5t75jv/sZx7qL17R3vzffF6hA/u9cz78gqvd+e4XXa3DgPY5072K\n3mJy8LczAAAAgNShIwMAAAAgdejIAAAAAEgdOjIAAAAAUqfiB/uXUvuPVrraa2vqXG3rDu1drbGl\nwzu4Wu8HimtXoWq22dLVGl6Z2bYbRZNqR410tVv2zB3cXyM/CHbzf/hBqSPmP1fQNn+6170503u+\n9l9unhOP+qerjdtgjqvVPOvb1pDnHurb6pSc6f7PN9dKVArbYWtXO+/kSTnT3aywgf1nLhrjaoN+\nVe9q/vIViMu8y/zg5UO6fehq2/zxDFfr/9vcgfH+k26dsHp1zvTA8/xA/JqLu7jargf+yNUO+qX/\n5f3SqZe72ncPOCBneu33/Kuq/9C/P0iGt471f9u9O/bqPHMWMrDfW7j2U1fb/Wmft43u8rns8bcX\nXG3Thur4+44jMgAAAABSh44MAAAAgNShIwMAAAAgdRgj0wINr77hahOX+BsRXt6v+Zterd1nmS9O\nKKpZBdvqBt/+177kx+qEujVt2xBIklZv2t3Vtu2Ye4Z/Q4m/a2g81uW4rf2NLmvybPMrrx7sak98\n8VZXy3eDTdspN+vtBvR386ydv8DVkC41Xfx524OunO1qB3Tx54E3lm+s1b1TdvLrf5UbDCbZOaPu\ncbWtnznS1Qb94UVX8wkov4aVflxsj5ufdbUHb+/tavffM8rV7t1iSs70F648ys0z+GDGyCTVOV+e\n0vxMrfBmXQ9X6/mvrq7W45ZnXK12441cre5vfp88d3HuLTBHnPmRm2ftPH8j2CTjiAwAAACA1KEj\nAwAAACB16MgAAAAASJ1mOzJmdr2ZLTaz17NqvczsQTN7K/p3w7ZtJqodOUQSkEPEjQwiCcghkqKQ\nwf6TJF0h6c9ZtbMkPRxCuNDMzoqmf1765iXfjKWb+mK/8rejEP07LnW112r6xNCSokxSheVw7nH+\nZmiNB9rnuyHmT/a6z9Uuu/mrrnbjTtflWX/u+l5c7b/LOOeQo12tx/Ovudrml/sbdb35HX/Fimk7\n3pi73G+PdfMMOzw1g/0nqcJyWAxr5391bPXk5652YR8/iLsQ2z6XZ0D4r/IM7K/xN56bO35HV6sb\nuipnesSl/oIm4cXpLWhhrCYpJRn8brflrnbJ3/zg5cY3p0ybhlWrXM32XeRqX7g5d3D/dWMmu3ku\n6LOfq9V/sLgVrWszk5SSHJbKlMXbuNrRPR4s2fr36OwvlvPYBZe52mE/+IarHdHXXwAg3/8/Nbov\n+siTTnSzbHZWhQ32DyE8IenjRuVvSVr3P3CypANL3C4gBzlEEpBDxI0MIgnIIZKi2DEyfUIIC6Of\nF0lKzdf6qCjkEElADhE3MogkIIcou1YP9g8hBK3nku9mNs7MpprZ1Dql+/AxkoscIgnWl0MyiHJg\nX4gkYF+Icim2I/OBmfWVpOjfJk/gDCFMDCGMCSGMaa+ORW4OyIscIgkKyiEZRBtiX4gkYF+Isitk\nsH8+UyQdJenC6N+7StailFl+R19f/ELzy127zV9c7dwN93a1+qV+gD7+I9U57D2lk6s17N54sJ//\nrmFcT3+39BN2f8evS37g4HHz9syZnveLEW6e2udfcrV8tvjvma525Z7DXO2knm/nTO82zLf//YK2\nmFipzmFzarr4u0Mvv91f5OTCPrcVtf6dp/2Xqw0+Kc/dpvMs++75fmD/zKOubHabP90yz3LbN7tY\nkiUyg3d+1s3VPjpopat1v7ODq4U6f0GGNAlrfWJrXumeM73Tl/1ydVv098slc7B/PonMYams3McP\nnt93W39hkvl7+twX4qvf9hdH+d9NH3G1O0fcX9T6JWlW3Wc5050X+QsKpU0hl1++RdIzkjY3s/lm\ndqwyId3HzN6StHc0DbSnTfgAABOnSURBVLQZcogkIIeIGxlEEpBDJEWzR2RCCIc28dReJW4L0CRy\niCQgh4gbGUQSkEMkRasH+wMAAABAudGRAQAAAJA6xQ72Ryvt0DHPAKsO7cvfEMRmw6f83XN/tTh3\nxPF5vaflWdJ//3DO4m1d7Z9ztnK1fr/JvRN6oQP786lf7gc+Ll7Tw9VqlJv1iQMfc/N8XekeaV1J\narrnDkh+4/dbunlmb3110ev/6aLcgfabjPvMzbN2ob8r+qcH7+RqLxx5SZ4tNH8VpNo8F8JA6V04\n/vt5aje62l3/9vuvl24ZkzPdd8JUN09SLgjQrq+/+MU744a62i1HX5ozfdlS/3+r/etzXa2+FW1D\n6YTVeS4V/eyrrjTg2eLW/+5NA11txiPdXa135+L3Xwf94Yyc6U3/8HTR60oKjsgAAAAASB06MgAA\nAABSh44MAAAAgNShIwMAAAAgdRjsD8Rk7Tw/2P+Vb+QO9tt3xBg3Tz61j/pB+/00o7iGtcLfH/C3\nqv7V91/OmW5goHWizZs8KGd69peKH9g/Z62/i/u0s3MHdndc8IKbp2arLVztqF9NcbVu1vzA/nxu\nf8H/vxqp54taF5q2wU1+1PN/D/J3Qn/oxN+5Wu8zH8+Z/u2xfmD84x+OcLV3XvQDpgds+76r7bix\nH1RfrO/2/KerbdPBz7fv9P/Kme5yqL9gSv1HH5esXUiXmedt4mp7tGJg/+J6fyGVTS9N/+D+xjgi\nAwAAACB16MgAAAAASB06MgAAAABShzEyQIKsnb8gZ7q20XTSDf3Hp65W8/3GN3/1358svNOf/973\nwJmlahaasOzInV3tT6OvaFTJc/PePBbU+/EwR53+U1fr+q/ncqZrOnVy8yy50J8XfmwPP6asWKfv\ndr+r3XDi/q7W54aXXa1h1aqStaMaDfiNP0f/2Lt+4Gpzvr1xzvR3Dvq3m+feLfy4KfnhVW3ugDcP\ndLVPrxzgal3/kZt9bnRZ3RaevkvO9Dt7Tyjp+ruZv8l6/R7b5UzXPlb8TbGTgiMyAAAAAFKHjgwA\nAACA1KEjAwAAACB1mu3ImNn1ZrbYzF7Pqo03swVmNi16+JOLgRIih0gCcoi4kUEkATlEUhQy2H+S\npCsk/blR/dIQwkUlbxGQ3ySRw+R7/jVXalBoNO0Hcn998HRXezGZB4wnqYJyeN7//snVdujY/OD+\nhXkG9h+Wb2B/o8HN+cwev62rvbHtlc0u1xo/6vmur53tt7nl7ke72maHvNoWTWqJSaqgDEpS/YxZ\nrjawUe12283N86sTXylo/SMe/KGrbfHrZblt6NnFzfP2qf5PpGe/0vhiGNLbi/yNDPvWB1erMJNU\nYTlsS7Ubb+RqPzj23jbdZpcaf1fWpZvn3kR448fatAll0exfCiGEJyRxq1nEihwiCcgh4kYGkQTk\nEEnRmq88TzazV6PDixuWrEVAy5BDJAE5RNzIIJKAHKKsiu3IXCVpmKTRkhZKuripGc1snJlNNbOp\ndVpd5OaAvMghkqCgHJJBtCH2hUgC9oUou6I6MiGED0II9SGEBknXStpxPfNODCGMCSGMaa+OTc0G\ntBg5RBIUmkMyiLbCvhBJwL4QcShksL9jZn1DCAujyW9Len1988N7YXWegYBr6tp0mze96/cpG9W/\n06bbbEvkMB1q3J3h/fcnO3bzOXxlwO6utnb+glI1q2TSksOGXUe72uiOT+WZs3Oz6zrgkjNdre8/\nX/QzdvR/pCz9r9w7Sz9x2O/zbMEPvI7DRdvf5mpXamQMLVm/tGSwNTrt+FFB8+0949uutsVvPnG1\n+llvN7uuYYf72nOz/aDtN3a/3tVWf2Wtq33xoONzpode4S98Ys8UdgGDJKqGHBbrjYuGuNq9Gz7c\nptusC/Wu1m2Br6Vdsx0ZM7tF0h6SNjaz+ZLOkbSHmY2WFCTNkXR8kysASoAcIgnIIeJGBpEE5BBJ\n0WxHJoRwaJ7ydW3QFqBJ5BBJQA4RNzKIJCCHSIpE3qgBAAAAANaHjgwAAACA1ClqsD9a74evHOFq\n/ZbOaNNtHr7Z8652X20fP+NaP0gRKFaDQqNpP8D1+U+HuloSB/anmeW503h9KO7u4y+d4e9urjMK\nXfqZRtPJGNg/Z+1KV/ufK092tU31dDmaU/Xm/2KXnOlp21/u5hn9h1Ncrd/Fz7lafUPpBjhfvsVW\nrjb+iB1c7aSf/8PVXtx9Qs50pz38n2A/eX83V5t96uauZk+n96IAlS7fhVVub/TZZ7TtFduWNqxy\ntU53+78D044jMgAAAPi/9u4/yK66vOP450lMdpRNJAk0iSECiUGIRRMNKY4dZfih/HAEpqigplGC\nQKcoKK1mbDsjbZ1qwdQZDQISSIahYCtg8MeME2JUIJiShA1hs0IwmlJM+JW0yWDNj92nf+wtszfP\nye7dzdlzzvfs+zWzs/c89+ye55zzyYXv3vs9B0gOAxkAAAAAyWEgAwAAACA5DGQAAAAAJIfJ/iVZ\nd9ryULvk2AtCrfullwroBhg+p/x8UdNy1/virQamte0OtY7xJ4Za9549+TU2wmTdMXzprneH2g3H\n1m8S8YLfntW0vHX3sWGd8UvGhdqUnzKxvyyf/PhPmpYf/cOYsM7027tCLc+J/Vk842I4E+889AIW\n0j13vinUvjvn7Kbl578cL7axdt6dobb9X9eE2odvvz7Upv8jea2CbVdbqM1pG96J/VnefV/MyFv0\ny8L7GG68IwMAAAAgOQxkAAAAACSHgQwAAACA5DCQAQAAAJAcJvsXZLQ1jxnbsg79qDhBDCPM/FOb\nFrdd0h5W+fA5j4bahrnV/ZvE+2Y827Tco56wTo9Xt/8623jZyaF29fLxTcu3HPdwUe3069F9MSOf\n6/xIqNnKSaE2aVnzBNeJHi8ugWq7/EefDrVZu9eV0MnQ9XRsaVqeelFc58yFnwu1FTd8PdTuvWJJ\nqP3VmqtDzR7tGESHyMOMqS+X3YIk6fgfHii7hULwfw8AAAAAksNABgAAAEByBhzImNl0M1tjZlvM\nrNPMrm3UJ5rZKjPb2vg+YfjbxUhFDlE2MogqIIeoAnKIqmhljsxBSde7+0YzGydpg5mtkvRJSavd\n/atmtljSYklfHL5W09btcV4ABmVE5LDtxuYboG6ddVdY54DHG76d9pnPhNqkzn2h9rqfbhiwh9Gz\nTwq1fVPizQL/84rYx12nx5tdntbWPPerJ+PvJ6tePiXUuvfs7LfPEtQug91dW0Ptufc237jtvHd+\nKqyzcPkPQ+3S9vxu3vuWlfGz/rP/6Xehdsxzz+S2zYTULoetaH9zvBmujRkban5gfxHtDJsJK+LN\nNS/f//lQe+Smm0Pt1b+Lx6j93Hz6yjAic3io0cfEOXn3nvTdjDXfMKx9/PXOuaE29uGnQi3egjV9\nA74j4+473H1j4/FeSV2Spkm6UNKKxmorJGVMWwPyQQ5RNjKIKiCHqAJyiKoY1BwZMztB0lxJ6yRN\ndvcdjad2Spqca2fAYZBDlI0MogrIIaqAHKJMLQ9kzKxd0n2SrnP3pvcv3d11mHeszOxKM1tvZusP\nKH7UBRgMcoiykUFUATlEFQwlh2QQeWppIGNmY9Qb1Lvd/f5G+QUzm9p4fqqkF7N+1t1vc/d57j5v\njNqyVgFaQg5RNjKIKiCHqIKh5pAMIk8DTvY3M5O0TFKXu/e9A9ODkhZK+mrj+8ph6XAEeeX9M0Pt\n6Lsy/1s04oyUHG778Yym5QOfjRPqs24o+fjib7a03g0vvmvAHj70xntCbW5b/F2jMv4Oknmzy0PW\nW/rfMefdHx89YF9lGykZ9H3NfyG1xzaFdVa+NCfULm1fNaTtnb3l4lA7+YtdoXZw794h/f66GSk5\nvHntmU3Lz15wa1jnjA/+Rai94YG0bpLZCm/x5fGi4+K/1YcUL9SSh5GSw4HsvCReHGfC6OGd2J91\n8ajNV/1xqPm+zcPaR1W0ctWy90haIGmzmf3/LWK/pN6Q/puZLZK0XVK8xTKQH3KIspFBVAE5RBWQ\nQ1TCgAMZd39Ekh3m6bPybQfIRg5RNjKIKiCHqAJyiKoY1FXLAAAAAKAKGMgAAAAASE4rc2TQj/Yd\nGXc33zsl1BaMG/gu5a+8I14t8+h4Y/chW7rpjFCb2bMlvw3giE372tqm5dmTrgnrrL70xvhzmZML\n498p/uGPOpqWezKu0Doq49MCh07YP9x6G/bF9a75SvM+TFoW71wtPZ9RQxWMOuqoUDv96N8M+ff9\n6PftTct/uGNqWGfs3u1D/v2oh1Nu2tW03HHOwbDOiV+IF4V4+akZoda9dVt+jeVo1Lg4Ef/ppbNC\n7Wdn3JTx0/E1/9ZN7w21mXpiSL2hGvb5gVCbf+O1oTbl8bWhNlLwjgwAAACA5DCQAQAAAJAcBjIA\nAAAAksNABgAAAEBymOx/hF7//f8ItZuP/rNQW/CVpUW0068ZH+sItTjVG1Uy4wtxYvyV/x7vZr39\ngjhp9PY//1aozW9rPuM9incIzpqw/4nHrgi1nlfGhtrJ394VapO6sib3IxU9r74aanfdcm6ovfW6\nO0LtbzsvCrXJf998m/LxG355BN2hrrqf+XXT8kfv/2xY5/GPLAm1H/zgzaH2z1s+EDew7o1Ni2/6\nRcy5PbZpoDYlSQfPfFeo/c/M+PpoF77StPzA2+O/mamjfx5qm/ePCbUzH14UarO+9r+hFl/hkadj\nNv0+1Dr3x/PwtrGvH9Lv/9CvLg61Kd8YuRP7s/CODAAAAIDkMJABAAAAkBwGMgAAAACSw0AGAAAA\nQHLMvbjp3uNtov+JnVXY9pCOh/x7G9x9XhHbIofIss5Xa4/vsiK2RQZxOLwWogqKymEdM2hz3xZq\nT18TJ/u//9TOpuVHHpgb1jn+lq5Q6969+wi6S0erGeQdGQAAAADJYSADAAAAIDkDDmTMbLqZrTGz\nLWbWaWbXNupfNrPnzayj8XX+8LeLkYocomxkEFVADlE2MogqaeWGmAclXe/uG81snKQNZraq8dy/\nuPtNw9ce8BpyiLKRQVQBOUTZyCAqY8CBjLvvkLSj8XivmXVJmjbcjQF9kUOUjQyiCsghykYG++dP\ndIbaSYvier89ZPk4rQ3rdOfTUq0Nao6MmZ0gaa6kdY3SNWb2pJndYWYTcu4NyEQOUTYyiCoghygb\nGUTZWh7ImFm7pPskXefueyR9W9JMSXPUOzL/+mF+7kozW29m6w9oXw4tYyQjhygbGUQVkEOUjQyi\nCloayJjZGPWG9W53v1+S3P0Fd+929x5J35E0P+tn3f02d5/n7vPGqC2vvjECkUOUjQyiCsghykYG\nURWtXLXMJC2T1OXuS/rUp/ZZ7WJJT+XfHtCLHKJsZBBVQA5RNjKIKmnlqmXvkbRA0mYz62jUviTp\nMjObI8nVO2fpqmHpEOhFDlE2MogqIIcoGxlEZbRy1bJHJFnGUz/Ovx0gGzlE2cggqoAcomxkEFUy\nqKuWAQAAAEAVMJABAAAAkBwGMgAAAACSw0AGAAAAQHIYyAAAAABIDgMZAAAAAMlhIAMAAAAgOebu\nxW3M7CVJ2yUdI+nlwjacv9T7l6q3D8e7+7FFbIgcVkbV+i8jg1L1jsNg0X++eC0cPPrPXyE55LWw\nUqrWf0sZLHQg89pGzda7+7zCN5yT1PuX6rEPRyr1Y0D/9ZD6caD/9KV+DOi/HlI/DvRfDj5aBgAA\nACA5DGQAAAAAJKesgcxtJW03L6n3L9VjH45U6seA/ush9eNA/+lL/RjQfz2kfhzovwSlzJEBAAAA\ngCPBR8sAAAAAJKfwgYyZnWtmT5vZs2a2uOjtD5aZ3WFmL5rZU31qE81slZltbXyfUGaP/TGz6Wa2\nxsy2mFmnmV3bqCezD3lLLYMSOawjclgsMpgttRymnEGJHGZJLYMSOaySQgcyZjZa0lJJ50maLeky\nM5tdZA9DsFzSuYfUFkta7e6zJK1uLFfVQUnXu/tsSadL+svGMU9pH3KTaAYlclgr5LAUZPAQieZw\nudLNoEQOmySaQYkcVkbR78jMl/Ssu29z9/2S7pV0YcE9DIq7/0LSrkPKF0pa0Xi8QtJFhTY1CO6+\nw903Nh7vldQlaZoS2oecJZdBiRzWEDksGBnMlFwOU86gRA4zJJdBiRxWSdEDmWmSnuuz/F+NWmom\nu/uOxuOdkiaX2UyrzOwESXMlrVOi+5CDumRQSvQckkNJ5LBUZPA1dclhkueQHEqqTwalRM9h6jlk\nsv8R8t7LvlX+0m9m1i7pPknXufuevs+lsg84vFTOITmstxTOIRmst1TOITmst1TOYR1yWPRA5nlJ\n0/ssH9eopeYFM5sqSY3vL5bcT7/MbIx6g3q3u9/fKCe1DzmqSwalxM4hOWxCDktABoO65DCpc0gO\nm9Qlg1Ji57AuOSx6IPO4pFlmdqKZjZV0qaQHC+4hDw9KWth4vFDSyhJ76ZeZmaRlkrrcfUmfp5LZ\nh5zVJYNSQueQHAbksGBkMFNdcpjMOSSHQV0yKCV0DmuVQ3cv9EvS+ZKekfRrSX9T9PaH0O89knZI\nOqDez24ukjRJvVdz2CrpIUkTy+6zn/7/VL1vDT4pqaPxdX5K+zAMxySpDDZ6Joc1+yKHhfdOBrOP\nS1I5TDmDjf7JYTwmSWWw0TM5rMiXNXYIAAAAAJLBZH8AAAAAyWEgAwAAACA5DGQAAAAAJIeBDAAA\nAIDkMJABAAAAkBwGMgAAAACSw0AGAAAAQHIYyAAAAABIzv8BBEECVApBPZUAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "tags": [] }, "output_type": "display_data" } ], "source": [ "sample(correct=False, rows=2, cols=5)" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "Sonnet 2 \"Hello, world!\" (aka. MLP on MNIST)", "provenance": [], "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: examples/simple_mnist.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Trivial convnet learning MNIST.""" from typing import Dict from absl import app import sonnet as snt import tensorflow as tf import tensorflow_datasets as tfds def mnist(split: str, batch_size: int) -> tf.data.Dataset: """Returns a tf.data.Dataset with MNIST image/label pairs.""" def preprocess_dataset(images, labels): # Mnist images are int8 [0, 255], we cast and rescale to float32 [-1, 1]. images = ((tf.cast(images, tf.float32) / 255.) - .5) * 2. return images, labels dataset = tfds.load( name="mnist", split=split, shuffle_files=split == "train", as_supervised=True) dataset = dataset.map(preprocess_dataset) dataset = dataset.shuffle(buffer_size=4 * batch_size) dataset = dataset.batch(batch_size) # Cache the result of the data pipeline to avoid recomputation. The pipeline # is only ~100MB so this should not be a significant cost and will afford a # decent speedup. dataset = dataset.cache() # Prefetching batches onto the GPU will help avoid us being too input bound. # We allow tf.data to determine how much to prefetch since this will vary # between GPUs. dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE) return dataset def train_step( model: snt.Module, optimizer: snt.Optimizer, images: tf.Tensor, labels: tf.Tensor, ) -> tf.Tensor: """Runs a single training step of the model on the given input.""" with tf.GradientTape() as tape: logits = model(images) loss = tf.nn.sparse_softmax_cross_entropy_with_logits( labels=labels, logits=logits) loss = tf.reduce_mean(loss) variables = model.trainable_variables gradients = tape.gradient(loss, variables) optimizer.apply(gradients, variables) return loss @tf.function def train_epoch( model: snt.Module, optimizer: snt.Optimizer, dataset: tf.data.Dataset, ) -> tf.Tensor: loss = 0. for images, labels in dataset: loss = train_step(model, optimizer, images, labels) return loss @tf.function def test_accuracy( model: snt.Module, dataset: tf.data.Dataset, ) -> Dict[str, tf.Tensor]: """Computes accuracy on the test set.""" correct, total = 0, 0 for images, labels in dataset: preds = tf.argmax(model(images), axis=1) correct += tf.math.count_nonzero(tf.equal(preds, labels), dtype=tf.int32) total += tf.shape(labels)[0] accuracy = (correct / tf.cast(total, tf.int32)) * 100. return {"accuracy": accuracy, "incorrect": total - correct} def main(unused_argv): del unused_argv model = snt.Sequential([ snt.Conv2D(32, 3, 1), tf.nn.relu, snt.Conv2D(32, 3, 1), tf.nn.relu, snt.Flatten(), snt.Linear(10), ]) optimizer = snt.optimizers.SGD(0.1) train_data = mnist("train", batch_size=128) test_data = mnist("test", batch_size=1000) for epoch in range(5): train_loss = train_epoch(model, optimizer, train_data) test_metrics = test_accuracy(model, test_data) print("[Epoch %d] train loss: %.05f, test acc: %.02f%% (%d wrong)" % (epoch, train_loss, test_metrics["accuracy"], test_metrics["incorrect"])) if __name__ == "__main__": app.run(main) ================================================ FILE: examples/simple_mnist_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.examples.simple_mnist.""" import sonnet as snt from examples import simple_mnist from sonnet.src import test_utils import tensorflow as tf class SimpleMnistTest(test_utils.TestCase): def setUp(self): self.ENTER_PRIMARY_DEVICE = False # pylint: disable=invalid-name super().setUp() def test_train_epoch(self): model = snt.Sequential([ snt.Flatten(), snt.Linear(10), ]) optimizer = snt.optimizers.SGD(0.1) dataset = tf.data.Dataset.from_tensor_slices( (tf.random.normal([2, 8, 8, 1]), tf.ones([2], dtype=tf.int64))).batch(2).repeat(4) for _ in range(3): loss = simple_mnist.train_epoch(model, optimizer, dataset) self.assertEqual(loss.shape, []) self.assertEqual(loss.dtype, tf.float32) def test_test_accuracy(self): model = snt.Sequential([ snt.Flatten(), snt.Linear(10), ]) dataset = tf.data.Dataset.from_tensor_slices( (tf.random.normal([2, 8, 8, 1]), tf.ones([2], dtype=tf.int64))).batch(2).repeat(4) outputs = simple_mnist.test_accuracy(model, dataset) self.assertEqual(len(outputs), 2) if __name__ == "__main__": tf.test.main() ================================================ FILE: examples/vqvae_example.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "cqCt_GhvCnwY" }, "source": [ "# VQ-VAE training example\n", "\n", "Demonstration of how to train the model specified in https://arxiv.org/abs/1711.00937, using TF 2 / Sonnet 2.\n", "\n", "On Mac and Linux, simply execute each cell in turn." ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "colab": {}, "colab_type": "code", "id": "cG_1Oe4TQli2" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: dm-sonnet in /tmp/sonnet-nb-env/lib/python3.7/site-packages (2.0.0)\r\n", "Requirement already satisfied: dm-tree in /tmp/sonnet-nb-env/lib/python3.7/site-packages (0.1.5)\r\n", "Requirement already satisfied: six>=1.12.0 in /tmp/sonnet-nb-env/lib/python3.7/site-packages (from dm-sonnet) (1.14.0)\r\n", "Requirement already satisfied: tabulate>=0.7.5 in /tmp/sonnet-nb-env/lib/python3.7/site-packages (from dm-sonnet) (0.8.7)\r\n", "Requirement already satisfied: absl-py>=0.7.1 in /tmp/sonnet-nb-env/lib/python3.7/site-packages (from dm-sonnet) (0.9.0)\r\n", "Requirement already satisfied: numpy>=1.16.3 in /tmp/sonnet-nb-env/lib/python3.7/site-packages (from dm-sonnet) (1.18.3)\r\n", "Requirement already satisfied: wrapt>=1.11.1 in /tmp/sonnet-nb-env/lib/python3.7/site-packages (from dm-sonnet) (1.12.1)\r\n" ] } ], "source": [ "!pip install dm-sonnet dm-tree" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 51 }, "colab_type": "code", "id": "95YuC82P35Of", "outputId": "2247702b-178a-4c87-b2fe-da195901da6d" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "TensorFlow version 2.1.0\n", "Sonnet version 2.0.0\n" ] } ], "source": [ "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import tensorflow.compat.v2 as tf\n", "import tensorflow_datasets as tfds\n", "import tree\n", "\n", "try:\n", " import sonnet.v2 as snt\n", " tf.enable_v2_behavior()\n", "except ImportError:\n", " import sonnet as snt\n", "\n", "print(\"TensorFlow version {}\".format(tf.__version__))\n", "print(\"Sonnet version {}\".format(snt.__version__))" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "DT8fKmqQC35h" }, "source": [ "# Download Cifar10 data\n", "This requires a connection to the internet and will download ~160MB.\n" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 34 }, "colab_type": "code", "id": "mR0lkHXDC3Pz", "outputId": "fbbb16d5-0ad3-466d-bd03-50e9872c35f7" }, "outputs": [ { "data": { "text/plain": [ "{'image': 'uint8[60000, 32, 32, 3]'}" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "cifar10 = tfds.as_numpy(tfds.load(\"cifar10:3.0.2\", split=\"train+test\", batch_size=-1))\n", "cifar10.pop(\"id\", None)\n", "cifar10.pop(\"label\")\n", "tree.map_structure(lambda x: f'{x.dtype.name}{list(x.shape)}', cifar10)" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "lUgvEhfJyQLZ" }, "source": [ "# Load the data into Numpy\n", "We compute the variance of the whole training set to normalise the Mean Squared Error below.\n" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "colab": {}, "colab_type": "code", "id": "9C-V2D6RSQwl" }, "outputs": [], "source": [ "train_data_dict = tree.map_structure(lambda x: x[:40000], cifar10)\n", "valid_data_dict = tree.map_structure(lambda x: x[40000:50000], cifar10)\n", "test_data_dict = tree.map_structure(lambda x: x[50000:], cifar10)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 34 }, "colab_type": "code", "id": "cIRl2ZtxoKNz", "outputId": "4d590492-9381-4202-f4c2-f5d91a80b8ce" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "train data variance: 0.06327039811675479\n" ] } ], "source": [ "def cast_and_normalise_images(data_dict):\n", " \"\"\"Convert images to floating point with the range [-0.5, 0.5]\"\"\"\n", " images = data_dict['image']\n", " data_dict['image'] = (tf.cast(images, tf.float32) / 255.0) - 0.5\n", " return data_dict\n", "\n", "train_data_variance = np.var(train_data_dict['image'] / 255.0)\n", "print('train data variance: %s' % train_data_variance)" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "Jse__pEBAkvI" }, "source": [ "# Encoder & Decoder Architecture\n" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "colab": {}, "colab_type": "code", "id": "1gwD36Vr6KqA" }, "outputs": [], "source": [ "class ResidualStack(snt.Module):\n", " def __init__(self, num_hiddens, num_residual_layers, num_residual_hiddens,\n", " name=None):\n", " super(ResidualStack, self).__init__(name=name)\n", " self._num_hiddens = num_hiddens\n", " self._num_residual_layers = num_residual_layers\n", " self._num_residual_hiddens = num_residual_hiddens\n", "\n", " self._layers = []\n", " for i in range(num_residual_layers):\n", " conv3 = snt.Conv2D(\n", " output_channels=num_residual_hiddens,\n", " kernel_shape=(3, 3),\n", " stride=(1, 1),\n", " name=\"res3x3_%d\" % i)\n", " conv1 = snt.Conv2D(\n", " output_channels=num_hiddens,\n", " kernel_shape=(1, 1),\n", " stride=(1, 1),\n", " name=\"res1x1_%d\" % i)\n", " self._layers.append((conv3, conv1))\n", "\n", " def __call__(self, inputs):\n", " h = inputs\n", " for conv3, conv1 in self._layers:\n", " conv3_out = conv3(tf.nn.relu(h))\n", " conv1_out = conv1(tf.nn.relu(conv3_out))\n", " h += conv1_out\n", " return tf.nn.relu(h) # Resnet V1 style\n", "\n", "\n", "class Encoder(snt.Module):\n", " def __init__(self, num_hiddens, num_residual_layers, num_residual_hiddens,\n", " name=None):\n", " super(Encoder, self).__init__(name=name)\n", " self._num_hiddens = num_hiddens\n", " self._num_residual_layers = num_residual_layers\n", " self._num_residual_hiddens = num_residual_hiddens\n", "\n", " self._enc_1 = snt.Conv2D(\n", " output_channels=self._num_hiddens // 2,\n", " kernel_shape=(4, 4),\n", " stride=(2, 2),\n", " name=\"enc_1\")\n", " self._enc_2 = snt.Conv2D(\n", " output_channels=self._num_hiddens,\n", " kernel_shape=(4, 4),\n", " stride=(2, 2),\n", " name=\"enc_2\")\n", " self._enc_3 = snt.Conv2D(\n", " output_channels=self._num_hiddens,\n", " kernel_shape=(3, 3),\n", " stride=(1, 1),\n", " name=\"enc_3\")\n", " self._residual_stack = ResidualStack(\n", " self._num_hiddens,\n", " self._num_residual_layers,\n", " self._num_residual_hiddens)\n", "\n", " def __call__(self, x):\n", " h = tf.nn.relu(self._enc_1(x))\n", " h = tf.nn.relu(self._enc_2(h))\n", " h = tf.nn.relu(self._enc_3(h))\n", " return self._residual_stack(h)\n", "\n", "\n", "class Decoder(snt.Module):\n", " def __init__(self, num_hiddens, num_residual_layers, num_residual_hiddens,\n", " name=None):\n", " super(Decoder, self).__init__(name=name)\n", " self._num_hiddens = num_hiddens\n", " self._num_residual_layers = num_residual_layers\n", " self._num_residual_hiddens = num_residual_hiddens\n", "\n", " self._dec_1 = snt.Conv2D(\n", " output_channels=self._num_hiddens,\n", " kernel_shape=(3, 3),\n", " stride=(1, 1),\n", " name=\"dec_1\")\n", " self._residual_stack = ResidualStack(\n", " self._num_hiddens,\n", " self._num_residual_layers,\n", " self._num_residual_hiddens)\n", " self._dec_2 = snt.Conv2DTranspose(\n", " output_channels=self._num_hiddens // 2,\n", " output_shape=None,\n", " kernel_shape=(4, 4),\n", " stride=(2, 2),\n", " name=\"dec_2\")\n", " self._dec_3 = snt.Conv2DTranspose(\n", " output_channels=3,\n", " output_shape=None,\n", " kernel_shape=(4, 4),\n", " stride=(2, 2),\n", " name=\"dec_3\")\n", " \n", " def __call__(self, x):\n", " h = self._dec_1(x)\n", " h = self._residual_stack(h)\n", " h = tf.nn.relu(self._dec_2(h))\n", " x_recon = self._dec_3(h)\n", " return x_recon\n", " \n", "\n", "class VQVAEModel(snt.Module):\n", " def __init__(self, encoder, decoder, vqvae, pre_vq_conv1, \n", " data_variance, name=None):\n", " super(VQVAEModel, self).__init__(name=name)\n", " self._encoder = encoder\n", " self._decoder = decoder\n", " self._vqvae = vqvae\n", " self._pre_vq_conv1 = pre_vq_conv1\n", " self._data_variance = data_variance\n", "\n", " def __call__(self, inputs, is_training):\n", " z = self._pre_vq_conv1(self._encoder(inputs))\n", " vq_output = self._vqvae(z, is_training=is_training)\n", " x_recon = self._decoder(vq_output['quantize'])\n", " recon_error = tf.reduce_mean((x_recon - inputs) ** 2) / self._data_variance\n", " loss = recon_error + vq_output['loss']\n", " return {\n", " 'z': z,\n", " 'x_recon': x_recon,\n", " 'loss': loss,\n", " 'recon_error': recon_error,\n", " 'vq_output': vq_output,\n", " }" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "FF7WaOn-s7En" }, "source": [ "# Build Model and train" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000 }, "colab_type": "code", "id": "owGEoOkO4ttk", "outputId": "0208f600-ecea-43b1-c5d3-32e9eccdeb9c" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "WARNING:tensorflow:AutoGraph could not transform and will run it as-is.\n", "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n", "Cause: Unable to locate the source code of . Note that functions defined in certain environments, like the interactive Python shell do not expose their source code. If that is the case, you should to define them in a .py source file. If you are certain the code is graph-compatible, wrap the call using @tf.autograph.do_not_convert. Original error: could not get source code\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "WARNING:tensorflow:AutoGraph could not transform and will run it as-is.\n", "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n", "Cause: Unable to locate the source code of . Note that functions defined in certain environments, like the interactive Python shell do not expose their source code. If that is the case, you should to define them in a .py source file. If you are certain the code is graph-compatible, wrap the call using @tf.autograph.do_not_convert. Original error: could not get source code\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "WARNING: AutoGraph could not transform and will run it as-is.\n", "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n", "Cause: Unable to locate the source code of . Note that functions defined in certain environments, like the interactive Python shell do not expose their source code. If that is the case, you should to define them in a .py source file. If you are certain the code is graph-compatible, wrap the call using @tf.autograph.do_not_convert. Original error: could not get source code\n", "WARNING:tensorflow:From /tmp/sonnet-nb-env/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1786: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "If using Keras pass *_constraint arguments to layers.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "WARNING:tensorflow:From /tmp/sonnet-nb-env/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1786: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "If using Keras pass *_constraint arguments to layers.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "100 train loss: 0.523625 recon_error: 0.483 perplexity: 10.356 vqvae loss: 0.041\n", "200 train loss: 0.248232 recon_error: 0.223 perplexity: 18.294 vqvae loss: 0.026\n", "300 train loss: 0.215068 recon_error: 0.190 perplexity: 23.106 vqvae loss: 0.025\n", "400 train loss: 0.191891 recon_error: 0.164 perplexity: 29.139 vqvae loss: 0.028\n", "500 train loss: 0.180945 recon_error: 0.147 perplexity: 34.253 vqvae loss: 0.033\n", "600 train loss: 0.167115 recon_error: 0.134 perplexity: 39.961 vqvae loss: 0.033\n", "700 train loss: 0.157724 recon_error: 0.124 perplexity: 46.521 vqvae loss: 0.033\n", "800 train loss: 0.153761 recon_error: 0.119 perplexity: 53.559 vqvae loss: 0.035\n", "900 train loss: 0.145033 recon_error: 0.112 perplexity: 62.442 vqvae loss: 0.033\n", "1000 train loss: 0.137589 recon_error: 0.105 perplexity: 71.831 vqvae loss: 0.033\n", "1100 train loss: 0.133044 recon_error: 0.101 perplexity: 79.135 vqvae loss: 0.032\n", "1200 train loss: 0.129990 recon_error: 0.098 perplexity: 87.959 vqvae loss: 0.032\n", "1300 train loss: 0.126507 recon_error: 0.095 perplexity: 96.704 vqvae loss: 0.031\n", "1400 train loss: 0.122403 recon_error: 0.092 perplexity: 104.202 vqvae loss: 0.031\n", "1500 train loss: 0.122003 recon_error: 0.091 perplexity: 112.476 vqvae loss: 0.031\n", "1600 train loss: 0.120192 recon_error: 0.089 perplexity: 122.269 vqvae loss: 0.032\n", "1700 train loss: 0.117041 recon_error: 0.086 perplexity: 129.887 vqvae loss: 0.031\n", "1800 train loss: 0.115004 recon_error: 0.083 perplexity: 138.603 vqvae loss: 0.032\n", "1900 train loss: 0.114134 recon_error: 0.082 perplexity: 147.545 vqvae loss: 0.032\n", "2000 train loss: 0.112840 recon_error: 0.081 perplexity: 153.993 vqvae loss: 0.032\n", "2100 train loss: 0.108815 recon_error: 0.077 perplexity: 161.729 vqvae loss: 0.031\n", "2200 train loss: 0.108596 recon_error: 0.078 perplexity: 171.971 vqvae loss: 0.031\n", "2300 train loss: 0.108132 recon_error: 0.077 perplexity: 181.157 vqvae loss: 0.031\n", "2400 train loss: 0.106273 recon_error: 0.076 perplexity: 186.200 vqvae loss: 0.031\n", "2500 train loss: 0.105936 recon_error: 0.075 perplexity: 194.301 vqvae loss: 0.031\n", "2600 train loss: 0.103880 recon_error: 0.073 perplexity: 201.674 vqvae loss: 0.030\n", "2700 train loss: 0.101655 recon_error: 0.072 perplexity: 207.131 vqvae loss: 0.030\n", "2800 train loss: 0.102564 recon_error: 0.072 perplexity: 216.983 vqvae loss: 0.030\n", "2900 train loss: 0.101613 recon_error: 0.072 perplexity: 219.649 vqvae loss: 0.030\n", "3000 train loss: 0.101227 recon_error: 0.071 perplexity: 226.789 vqvae loss: 0.030\n", "3100 train loss: 0.100786 recon_error: 0.071 perplexity: 235.522 vqvae loss: 0.030\n", "3200 train loss: 0.100130 recon_error: 0.070 perplexity: 243.282 vqvae loss: 0.030\n", "3300 train loss: 0.097764 recon_error: 0.067 perplexity: 249.584 vqvae loss: 0.030\n", "3400 train loss: 0.100630 recon_error: 0.069 perplexity: 260.551 vqvae loss: 0.031\n", "3500 train loss: 0.099929 recon_error: 0.068 perplexity: 266.012 vqvae loss: 0.032\n", "3600 train loss: 0.099245 recon_error: 0.067 perplexity: 272.031 vqvae loss: 0.032\n", "3700 train loss: 0.097812 recon_error: 0.066 perplexity: 279.691 vqvae loss: 0.032\n", "3800 train loss: 0.097137 recon_error: 0.064 perplexity: 284.240 vqvae loss: 0.033\n", "3900 train loss: 0.099217 recon_error: 0.066 perplexity: 293.507 vqvae loss: 0.034\n", "4000 train loss: 0.098570 recon_error: 0.065 perplexity: 300.891 vqvae loss: 0.034\n", "4100 train loss: 0.099238 recon_error: 0.065 perplexity: 306.762 vqvae loss: 0.034\n", "4200 train loss: 0.098172 recon_error: 0.064 perplexity: 311.918 vqvae loss: 0.035\n", "4300 train loss: 0.096449 recon_error: 0.063 perplexity: 316.246 vqvae loss: 0.034\n", "4400 train loss: 0.096487 recon_error: 0.062 perplexity: 319.591 vqvae loss: 0.034\n", "4500 train loss: 0.096092 recon_error: 0.062 perplexity: 322.313 vqvae loss: 0.034\n", "4600 train loss: 0.096474 recon_error: 0.062 perplexity: 324.620 vqvae loss: 0.035\n", "4700 train loss: 0.097075 recon_error: 0.063 perplexity: 324.357 vqvae loss: 0.035\n", "4800 train loss: 0.094709 recon_error: 0.060 perplexity: 326.024 vqvae loss: 0.034\n", "4900 train loss: 0.096557 recon_error: 0.061 perplexity: 327.701 vqvae loss: 0.035\n", "5000 train loss: 0.096185 recon_error: 0.061 perplexity: 326.664 vqvae loss: 0.035\n", "5100 train loss: 0.095646 recon_error: 0.060 perplexity: 327.617 vqvae loss: 0.035\n", "5200 train loss: 0.094689 recon_error: 0.059 perplexity: 328.692 vqvae loss: 0.035\n", "5300 train loss: 0.097047 recon_error: 0.061 perplexity: 327.988 vqvae loss: 0.036\n", "5400 train loss: 0.096259 recon_error: 0.060 perplexity: 327.075 vqvae loss: 0.036\n", "5500 train loss: 0.094588 recon_error: 0.059 perplexity: 327.083 vqvae loss: 0.036\n", "5600 train loss: 0.095947 recon_error: 0.060 perplexity: 328.213 vqvae loss: 0.036\n", "5700 train loss: 0.095466 recon_error: 0.059 perplexity: 329.375 vqvae loss: 0.036\n", "5800 train loss: 0.094849 recon_error: 0.059 perplexity: 326.821 vqvae loss: 0.036\n", "5900 train loss: 0.093799 recon_error: 0.058 perplexity: 328.409 vqvae loss: 0.036\n", "6000 train loss: 0.095373 recon_error: 0.059 perplexity: 326.791 vqvae loss: 0.036\n", "6100 train loss: 0.093989 recon_error: 0.059 perplexity: 325.959 vqvae loss: 0.035\n", "6200 train loss: 0.095549 recon_error: 0.059 perplexity: 330.829 vqvae loss: 0.036\n", "6300 train loss: 0.094730 recon_error: 0.058 perplexity: 330.906 vqvae loss: 0.036\n", "6400 train loss: 0.095038 recon_error: 0.058 perplexity: 329.353 vqvae loss: 0.037\n", "6500 train loss: 0.095891 recon_error: 0.059 perplexity: 330.197 vqvae loss: 0.037\n", "6600 train loss: 0.094342 recon_error: 0.058 perplexity: 331.240 vqvae loss: 0.036\n", "6700 train loss: 0.095096 recon_error: 0.058 perplexity: 330.618 vqvae loss: 0.037\n", "6800 train loss: 0.095581 recon_error: 0.059 perplexity: 324.493 vqvae loss: 0.037\n", "6900 train loss: 0.094467 recon_error: 0.058 perplexity: 328.868 vqvae loss: 0.037\n", "7000 train loss: 0.092967 recon_error: 0.057 perplexity: 328.276 vqvae loss: 0.036\n", "7100 train loss: 0.094339 recon_error: 0.058 perplexity: 327.318 vqvae loss: 0.037\n", "7200 train loss: 0.095227 recon_error: 0.058 perplexity: 326.306 vqvae loss: 0.037\n", "7300 train loss: 0.093832 recon_error: 0.057 perplexity: 328.262 vqvae loss: 0.037\n", "7400 train loss: 0.093331 recon_error: 0.057 perplexity: 327.987 vqvae loss: 0.037\n", "7500 train loss: 0.094718 recon_error: 0.058 perplexity: 328.948 vqvae loss: 0.037\n", "7600 train loss: 0.094199 recon_error: 0.058 perplexity: 328.468 vqvae loss: 0.037\n", "7700 train loss: 0.094603 recon_error: 0.058 perplexity: 327.501 vqvae loss: 0.037\n", "7800 train loss: 0.092299 recon_error: 0.056 perplexity: 327.630 vqvae loss: 0.037\n", "7900 train loss: 0.095228 recon_error: 0.058 perplexity: 329.946 vqvae loss: 0.037\n", "8000 train loss: 0.094291 recon_error: 0.058 perplexity: 326.790 vqvae loss: 0.037\n", "8100 train loss: 0.094481 recon_error: 0.057 perplexity: 328.667 vqvae loss: 0.037\n", "8200 train loss: 0.093992 recon_error: 0.057 perplexity: 329.655 vqvae loss: 0.037\n", "8300 train loss: 0.093976 recon_error: 0.057 perplexity: 323.950 vqvae loss: 0.037\n", "8400 train loss: 0.093422 recon_error: 0.057 perplexity: 324.523 vqvae loss: 0.036\n", "8500 train loss: 0.092898 recon_error: 0.056 perplexity: 325.402 vqvae loss: 0.037\n", "8600 train loss: 0.094298 recon_error: 0.057 perplexity: 329.251 vqvae loss: 0.037\n", "8700 train loss: 0.094489 recon_error: 0.057 perplexity: 331.027 vqvae loss: 0.037\n", "8800 train loss: 0.093022 recon_error: 0.056 perplexity: 327.495 vqvae loss: 0.037\n", "8900 train loss: 0.093427 recon_error: 0.057 perplexity: 328.008 vqvae loss: 0.037\n", "9000 train loss: 0.094884 recon_error: 0.058 perplexity: 327.057 vqvae loss: 0.037\n", "9100 train loss: 0.093559 recon_error: 0.056 perplexity: 331.800 vqvae loss: 0.037\n", "9200 train loss: 0.093282 recon_error: 0.056 perplexity: 328.689 vqvae loss: 0.037\n", "9300 train loss: 0.092217 recon_error: 0.056 perplexity: 323.903 vqvae loss: 0.036\n", "9400 train loss: 0.093902 recon_error: 0.057 perplexity: 326.350 vqvae loss: 0.037\n", "9500 train loss: 0.093772 recon_error: 0.057 perplexity: 325.627 vqvae loss: 0.037\n", "9600 train loss: 0.093123 recon_error: 0.056 perplexity: 327.352 vqvae loss: 0.037\n", "9700 train loss: 0.092934 recon_error: 0.056 perplexity: 328.674 vqvae loss: 0.037\n", "9800 train loss: 0.093284 recon_error: 0.056 perplexity: 329.437 vqvae loss: 0.037\n", "9900 train loss: 0.094147 recon_error: 0.057 perplexity: 330.146 vqvae loss: 0.037\n", "10000 train loss: 0.092876 recon_error: 0.056 perplexity: 326.349 vqvae loss: 0.037\n", "CPU times: user 1h 47min 46s, sys: 14min 12s, total: 2h 1min 59s\n", "Wall time: 4min 29s\n" ] } ], "source": [ "%%time\n", "\n", "# Set hyper-parameters.\n", "batch_size = 32\n", "image_size = 32\n", "\n", "# 100k steps should take < 30 minutes on a modern (>= 2017) GPU.\n", "# 10k steps gives reasonable accuracy with VQVAE on Cifar10.\n", "num_training_updates = 10000\n", "\n", "num_hiddens = 128\n", "num_residual_hiddens = 32\n", "num_residual_layers = 2\n", "# These hyper-parameters define the size of the model (number of parameters and layers).\n", "# The hyper-parameters in the paper were (For ImageNet):\n", "# batch_size = 128\n", "# image_size = 128\n", "# num_hiddens = 128\n", "# num_residual_hiddens = 32\n", "# num_residual_layers = 2\n", "\n", "# This value is not that important, usually 64 works.\n", "# This will not change the capacity in the information-bottleneck.\n", "embedding_dim = 64\n", "\n", "# The higher this value, the higher the capacity in the information bottleneck.\n", "num_embeddings = 512\n", "\n", "# commitment_cost should be set appropriately. It's often useful to try a couple\n", "# of values. It mostly depends on the scale of the reconstruction cost\n", "# (log p(x|z)). So if the reconstruction cost is 100x higher, the\n", "# commitment_cost should also be multiplied with the same amount.\n", "commitment_cost = 0.25\n", "\n", "# Use EMA updates for the codebook (instead of the Adam optimizer).\n", "# This typically converges faster, and makes the model less dependent on choice\n", "# of the optimizer. In the VQ-VAE paper EMA updates were not used (but was\n", "# developed afterwards). See Appendix of the paper for more details.\n", "vq_use_ema = True\n", "\n", "# This is only used for EMA updates.\n", "decay = 0.99\n", "\n", "learning_rate = 3e-4\n", "\n", "\n", "# # Data Loading.\n", "train_dataset = (\n", " tf.data.Dataset.from_tensor_slices(train_data_dict)\n", " .map(cast_and_normalise_images)\n", " .shuffle(10000)\n", " .repeat(-1) # repeat indefinitely\n", " .batch(batch_size, drop_remainder=True)\n", " .prefetch(-1))\n", "\n", "valid_dataset = (\n", " tf.data.Dataset.from_tensor_slices(valid_data_dict)\n", " .map(cast_and_normalise_images)\n", " .repeat(1) # 1 epoch\n", " .batch(batch_size)\n", " .prefetch(-1))\n", "\n", "# # Build modules.\n", "encoder = Encoder(num_hiddens, num_residual_layers, num_residual_hiddens)\n", "decoder = Decoder(num_hiddens, num_residual_layers, num_residual_hiddens)\n", "pre_vq_conv1 = snt.Conv2D(output_channels=embedding_dim,\n", " kernel_shape=(1, 1),\n", " stride=(1, 1),\n", " name=\"to_vq\")\n", "\n", "if vq_use_ema:\n", " vq_vae = snt.nets.VectorQuantizerEMA(\n", " embedding_dim=embedding_dim,\n", " num_embeddings=num_embeddings,\n", " commitment_cost=commitment_cost,\n", " decay=decay)\n", "else:\n", " vq_vae = snt.nets.VectorQuantizer(\n", " embedding_dim=embedding_dim,\n", " num_embeddings=num_embeddings,\n", " commitment_cost=commitment_cost)\n", " \n", "model = VQVAEModel(encoder, decoder, vq_vae, pre_vq_conv1,\n", " data_variance=train_data_variance)\n", "\n", "optimizer = snt.optimizers.Adam(learning_rate=learning_rate)\n", "\n", "@tf.function\n", "def train_step(data):\n", " with tf.GradientTape() as tape:\n", " model_output = model(data['image'], is_training=True)\n", " trainable_variables = model.trainable_variables\n", " grads = tape.gradient(model_output['loss'], trainable_variables)\n", " optimizer.apply(grads, trainable_variables)\n", "\n", " return model_output\n", "\n", "train_losses = []\n", "train_recon_errors = []\n", "train_perplexities = []\n", "train_vqvae_loss = []\n", "\n", "for step_index, data in enumerate(train_dataset):\n", " train_results = train_step(data)\n", " train_losses.append(train_results['loss'])\n", " train_recon_errors.append(train_results['recon_error'])\n", " train_perplexities.append(train_results['vq_output']['perplexity'])\n", " train_vqvae_loss.append(train_results['vq_output']['loss'])\n", "\n", " if (step_index + 1) % 100 == 0:\n", " print('%d train loss: %f ' % (step_index + 1,\n", " np.mean(train_losses[-100:])) +\n", " ('recon_error: %.3f ' % np.mean(train_recon_errors[-100:])) +\n", " ('perplexity: %.3f ' % np.mean(train_perplexities[-100:])) +\n", " ('vqvae loss: %.3f' % np.mean(train_vqvae_loss[-100:])))\n", " if step_index == num_training_updates:\n", " break" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "m2hNyAnhs-1f" }, "source": [ "# Plot loss" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 516 }, "colab_type": "code", "id": "2vo-lDyRomKD", "outputId": "47aab497-c2f2-4052-b1da-cfbe5bfbe865" }, "outputs": [ { "data": { "text/plain": [ "Text(0.5, 1.0, 'Average codebook usage (perplexity).')" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6oAAAHiCAYAAADh6DE2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd5hU5fn/8c/N0tvSVqQXQREbArEX7AVRE3vXxGDs/hKTYIuaqDHWmMQYa+z6NTZUEEWl2ABBkaYo4Cq9s/SF3X1+f5wzy+zuzO7M7syc2Tnv13Xtxcwpz3PP7LBz7vM0c84JAAAAAIBs0SDoAAAAAAAAiEaiCgAAAADIKiSqAAAAAICsQqIKAAAAAMgqJKoAAAAAgKxCogoAAAAAyCokqgAAAKgXzOw2M3s+wWPHm9mlaYjhaTO7I9XlBsXMCszsWzNrFkDdPc3MmVnDOpZzo5k9Uctz9zazz+pSP9KDRBUIkJkVmtkKM2sRte1SMxvvP3b+/oZR+xv521zUtj3M7H0zW2Nm68xsmpmd6O8bYmZlZrax0s+BGXypAIAk+YnWWjNrEnQsyGkjJD3tnNsSdCC15Zy7yzl3qZR88uucmyFpnZkNS2uQSBqJKhC8PEnXVrN/raQTop6f4G+L9raksZJ2lrSTpGskrY/av8Q517LSz+d1Dx0AkA5m1lPSoZKcpJPTUH6dWrCQG/ybIBdJSqiVOsmyzczqS67xgqTLgg4CFdWXDw+Qy+6VdL2ZtYmz/zlJF0Y9v1DSs5EnZtZBUi9Jjzvntvk/nzrnPklbxACAdLtQ0iRJT8tLJGRmTfxeM3tGDvK7bW4xs5385yeZ2XT/uM/MbO+oYwvN7I9mNkPSJjNraGYjzGy+mW0wszlm9vOo4/PM7H4zW2VmP5jZVdEtVWaWb2ZPmtlSM1tsZneYWV6sF+OXdWNUXdPMrJu/7yAz+8LMivx/D4o6r5eZTfDPGSupQ6VyD/Bf5zoz+9rMhlSqehczm2Jm681spJm1izr3ZDOb7Z873sx2j9q3u79tnX9MzJsFZtbKzMaZ2T/MzGLsLzSzo6Oel3ddNrOmZva8ma326/nCzDr6+y4xs2/8173AzC6rVO4f/Pd9iXk9sZyZ9fH3NTGz+8zsJzNbbmb/sfjdeveXtM45tyiq7PFm9tdq3re477l/7p1m9qmkzZJ611RepdcV8zNlZo39z/XV/nF5Zvapmf2p8vsqaaL/7zrzepAdbl6Ps72i6tnJzDabWYG/abyko4zeC1mFRBUI3lR5fyCvj7P/TUmHmVkbM2sr7w77yKj9qyXNk/S8mZ0a+ZIDANRrF8pr5XlB0nFm1tE5VyzpdUnnRB13pqQJzrkVZravpKfktQy1l/SopLcqXXyfI2mopDbOuRJJ8+V9r+RLul3ed0kn/9hfy+vFM0DSQEmnVorxaUklkvpI2lfSsZLijQn9rV/3iZJaS/qlpM1+wjJK0j/8mB+QNMrM2vvnvShpmrwE9S/yk3ZJMrMu/rl3SGon73v0tajkI/I+/lJSJz/Wf/jn7irpJUnXSSqQNFrS235C1EheT6X35fVSulrSC2a2W/QL8mP8UNKnzrlrnHNOyblI3vvezX/tv5EU6X67QtJJ/nt1iaQHzWygX+/x8t7Po+W990MqlXu3pF3l/d76SOoi6U9xYthL0twY2+O9b4m85xdIGi6plaQfqysvhqcV4zPlnNsm6XxJf/ZvKIyQ1yPtzhhlHOb/28bvQTZB0sv++RHnSPrQObdSkpxziyVtl1Thd4xgkagC2eFPkq6u9Ic+Yqu8L8yz/J+3/G2SJP+L8QhJhZLul7TUzCaaWd+oMjr7dz6jf1oIAJB1zOwQST0kveKcmyYvmTzX3/2ipLOjDj/X3yZ5ycGjzrnJzrlS59wzkoolHRB1/D+ccwsj4xGdc/9zzi1xzpU55/5P0veS9vOPPVPSQ865Rc65tfISoEiMHeUlndc55zY551ZIerBSbNEulXSzc26u83ztnFstL2n+3jn3nHOuxDn3kqRvJQ0zs+6SfibpFudcsXNuorzvw4jzJY12zo324x8r7+bviVHHPOecm+Wc2yTpFklnmtfqe5akUc65sc657ZLuk9RM0kH++9VS0t1+L6WPJL2jijcIOkuaIOl/zrmb47zmmmyXl6D28X9f05xz6yXJOTfKOTfff68myEuaD/XPO1PSf51zs51zmyXdFinQb9UdLun/OefWOOc2SLpL8X8vbSRtiLE93vuWyHv+tB9bif/eVldeuZo+U865WfIS5DflJcgXOOdK4767FT0j6ZyoVu8L5PVYi7bBfz+QJUhUgSzg//F9R94dwlielXc3skK336jzFznnrnLO7SLv4mZTpeOWOOfaVPrZlNpXAQBIkYskve+cW+U/f1E7WhLHSWpuZvubN451gKQ3/H09JP0u+qakvNa6zlFlL4yuyMwutB1dhddJ2lM7utd2rnR89OMekhrJuzkaOfdReS2QsXSTl3BX1lk7Wt0ifpTXCthZ0tpK31fRx/aQdEal13uIvFa7WDH/6MfcoXK9zrky/9hIvQv9bZVjihgqL7H9T8xXm5jnJL0n6WW/C+89fmuuzOwEM5vkd1ldJy+BS+T3UiCpuaRpUe/JGH97LGvltXxWFu99S/Y9r6m8aIl8pp7xjxvtnPs+zmuqwjk3WV5X5CFm1k9ei+1blQ5rJWldomUi/RhID2SPWyV9Ka9VtLKP5X0JOEmfSNolXiHOuYVm9rC8Lk0AgHrEH0t4pqQ8M1vmb24iqY2Z7eOc+9rMXpHXurdc0jt+q5nkJQN3OudidYeMiJ4xvoekxyUdJelz51ypmU2XFGl1Wiqpa9S53aIeL5TXWtvB70Jck4XyvrtmVdq+RF7iEa27vORqqaS2ZtYiKlntHvUaFsprqft1NfVGx9xdXivmKr/e6DGL5h+7WFKppG5m1iAqWe0u6buosh6X1FbSaDM7vpqbv5vkJY4RO0ce+K2Nt0u63b/pMFrSXH+s5Wvybk6PdM5tN7M3ldjvZZW87sN7+N1ZazJD0v+LsT3e+5bIex6rC3S88pL9TP1b3o3948zskDjzccTrgv2MvBbhZZJedc6V907zuzQ3Vuxu0AgILapAlnDOzZP0f/Jm7K28z0kaJunkymNgzKytmd1uZn3MrIF5kyv9Ut4kHACA+uVUeYlSf3mtpQMk7S7vhmVkYr0X5XVdPU87uv1KXvL0G7+11cyshZkNNbNYLWaS1ELeRf1KyZvAR16LasQrkq41sy7mTfj3x8gO59xSed1R7zez1v73zy5mdnicup6Q9Bcz6+vHtrc/xnO0pF3N7FzzJnc6y3/t7zjnfpTXrfR2f+zoIfK+CyOel9dF+DjzJtdpat6SbNFJ3Plm1t/Mmkv6s7wEpdR/bUPN7Ci/FfN38pKkzyRFWt/+YN6ScEP8el+u9JqukpfYvG3xJyuaLulsv5zBkk6P7DCzI8xsL78L7Hp5yVuZvISpibzfS4mZnSBvrGbEK5IuMW/Cp+byutJKKm8ZflzemNbIBFtdzOy4OPFNkXcTpEul7fHet0Te81jilVeups+UmV0gaZCki+VdKz1jZi1j1LVS3vvYu9L25yX9XF6yWrl32uGSPvLHgSNLkKgC2eXP8i4cqvDHe8yOsWubpJ6SPpD3RTdL3pftxVHHdLaq66ieJknmzQZYl65LAIDUuUje+MOfnHPLIj+S/iXpPDNr6Hdj3CSvC+i7kROdc1PlTYD0L3ldOuep4ndBBc65OfJ68Xwur3V2L0mfRh3yuLzEYYakr+QllSXyEmnJS5wbS5rj1/eqKnYBjfaAvATrfXnfVU9KauaPUz1JXqK4WtIfJJ0U1e35XHkz066R1/OoPMFwzi2UdIqkG+UlJwsl/V4Vr2+fkzdBzzJJTeXfDHbOzZWXsPxTXsveMEnD/DGp2/znJ/j7/i3pQufct5XePydvPOgiSSPNrGmM132LvJbktfJaT6NvLOzsv2frJX0jb8zrc34L+TX++7XWfw/Ku6k6596VNxnROHm/48iN6UiS9cfIdjNbL+/6IOYkQf5rfVoVJxqq7n1L5D2PJWZ5McT8TJk3Xvnv8n4PG51zL8q7ifFgjNe0Wd4kS5/6XYgPiIr9S3k3Zz6udNp5iurGbd4M1e8KgTKX9ARlAAAACBu/Ze8/zrnKXXURIPNmwZ0lqUmC3bArn18gL3Hb1zm3xczGS3reOfdEiuJLaXl1jOUpefN23By1bW95k5AdGFxkiIUWVQAAAFRhZs3M7ES/S24XeS2ab9R0HtLPzH5u3nqpbSX9TdLbtUlSJck5t9I518/5M0HnKn8c8C/kteaXc87NIEnNTiSqAAAAiMXkdVddK6/r7zeKvx4nMusyeWutzpfXFfvyYMPJbmb2F3mtzvc6534IOh4khq6/AAAAAICsQosqAAAAACCrkKgCAAAAALJKw6ADqE6HDh1cz549gw4DAJAjpk2btso5VxB0HOnkr8k4VdJi59xJZtZL3vqP7SVNk3SBc26bmTWRt9THIHnLgpzlnCusqXy+mwEAqVLd93JWJ6o9e/bU1KlTgw4DAJAjzOzHoGPIgGvlTXrT2n/+N0kPOude9tdM/pWkR/x/1zrn+pjZ2f5xZ9VUON/NAIBUqe57ma6/AADkCDPrKmmopCf85ybpSEmv+oc8I+lU//Ep/nP5+4/yjwcAIHAkqgAA5I6/S/qDpDL/eXtJ66LWV1wkqYv/uIukhZLk7y/yj6/CzIab2VQzm7py5cp0xQ4AQDkSVQAAcoCZnSRphXNuWqrLds495pwb7JwbXFCQ00N8AQBZIqvHqAIAgIQdLOlkMztRUlN5Y1QfktTGzBr6raZdJS32j18sqZukRWbWUFK+vEmVAAAIHC2qAADkAOfcDc65rs65npLOlvSRc+48SeMkne4fdpGkkf7jt/zn8vd/5JxzGQwZAIC4SFQBAMhtf5T0WzObJ28M6pP+9icltfe3/1bSiIDiAwCgCrr+AgCQY5xz4yWN9x8vkLRfjGO2Sjojo4EBAJAgWlQBAAAAAFmFRBUAAAAAkFVIVAEAAAAAWYVEFQAAAACQVUhUAQAAAABZhUQVAAAAAJBVSFQBAAAAAFmFRBUAAAAAkFVIVAEAAAAAWYVEFQAAAACQVXI+UX143Dz1HDFK20rKgg4FAAAAiGn1xmL1HDFK/5u6MOhQgKyQ84nqoxPmS5K2bCsNOBIAAAAgtsLVmyVJL075KeBIgOyQ84mqCzoAAAAAAKHknNM7M5aouIRGs2TlfKJazoIOAAAAAECYfDpvta568Sv97d25QYdS74QnUQUAAACADCrasl2StGz9loAjqX9yPlHdsLUk6BAAAACAuDZs3a7THvks6DCArJLziSoAAACQzeat2Bh0CEDWIVEFAAAAgAwaO2e5LntuatBhVOCc06gZS1Valh3T0YYmUTUmUwIAAACQQS7OGiS/fnaq3pu9PMPRVO/N6Yt15Ytf6qlPfgg6FEkZTFTNrIWZPWNmj5vZeZmqN8Jlx40BAAAAIC6uWXOT1WEJku2lZXrr6yVyaf5wrNqwTZK0fP3WtNaTqDolqmb2lJmtMLNZlbYfb2ZzzWyemY3wN/9C0qvOuV9LOrku9dYK/+kBAACQRVas36rtpWUyuv5lRFmZ09pN2wKpO17LaiL++eH3uualr/T+nOBbYEdOX6zJC1ZnpK66tqg+Len46A1mlifpYUknSOov6Rwz6y+pq6SF/mGseAsAAIB6bfXGYm0srt0KE8Ulpdrvrg/1x1dnVNie7TlrSWmZtpWUBR1Grdz7/lzt+5exgSWrtbXMb+Fctzn4uK99ebrOemxSRuqqU6LqnJsoaU2lzftJmuecW+Cc2ybpZUmnSFokL1mttl4zG25mU81s6sqVK+sSXsVYaVIFAABACg264wMNuXdcrc7dXupdm743e1nau3Sm0rEPTtSuN7+b9noG/WWs7ntvbkrLHDNrmSRpbQAJX227/s5Zsl6vTF2U4mjqh3SMUe2iHS2nkpegdpH0uqTTzOwRSW/HO9k595hzbrBzbnBBQUGdg7nqiD5+uXUuCgAAAKhg1UYv6dm6PRwdBhes2pSRelZv2qZ/jZuX0jKDvCFQ20aza1/+qvxxXca51kcZm0zJObfJOXeJc+5y59wLmaq3Q8vGXv2ZqhAAAAChMvG7lep3yxh9UVi5o2FisnWM6vbSsqzobhpL0ebt6jliVHkraSIi+UC2vt/Rnp/0o8bPXZHROrOtB2o6EtXFkrpFPe/qbwtE5INYn7pUAAAAIHttK6mYwH06f5UkaWrh2kDimfDdSq3fur1OZTjn9N3yDRW2XfPSVxrw57Hlz+taRyp9v8KL9fGPFyR8TiQdCCJNjW4NfTKB5V9ufnOWLv7vF5ULCZV0JKpfSOprZr3MrLGksyW9lYZ6ElIPbpgAAACgHrn02akVErhEfLN0vYpL6tY9eNbiIvUcMUqffL+qfNucJet10VNTtPdt79e63Le+XqLHP16gYx+cqHHf7mjFezeqtXLNpm011vHPD7+vcH62ibQYBpEfODmt2lisSQtW6y/vzKlVGU998oN+9fQXNR9YS7G6Fv+0erNWbAhmuZq6Lk/zkqTPJe1mZovM7FfOuRJJV0l6T9I3kl5xzs2ue6h1Q3sqAAAAUmHid8lN+Lli/Vad8NDHuvmNWTH3J5o3TfKXBfkoKhms68RAazdt0zUvfaW7Rn8rSZq3YmPM41ZtLK6xrPvHfqdL0phI1dWOFtWq7/i9732rf49P7ZjYyo57cKLOrsOMud8u26APk7gR8NKUn/T1wnVx9386b5VmLymqtozD7h2n/e78MOE6U6lhXU52zp0TZ/toSaPrUnaqRD6G9PwFAABAECJdZr/8KfVdg+t6jVtSVveL5JvemKnmjfPqXE66RZbVidWi+vC4+ZKkK4b0SUvdo2cmPpY2VW54faYkqfDuoTH3n/fE5Jj7P8/QOqk1ydhkSoGJjFGlTRUAAABZKB1Xqec9MUk9R4yq8bhEu8FWPuz0Rz4rn+n4hck/6fGPax53KUm//9/XgXUPXrHBaxV+d9bSjNVZ1xsJQWQws5esD6DWqnI+US3/T0WeCgAAgDSKNIxc+swXOur+8XGP+2z+Kt1Ri3GKm4q9xHDu8poTiU/nea1iW7aV6rlJP6rniFGaHNVS9uZXi/XqtKrrcz79WWHM8uavrNgleOqPa/XN0uQTmv9NW5TS7sG1mTA10s05UZuKSzRyenrmhq0p/nhdsdPt++UbNOK1GYHUHVGnrr/1AZMpAQAAIJ0eneDNPHvPmLm6YkgfffBNxRbDrdu9LqfzV3prkJ77+OQK+xO9XJ38g5doRpJQqeZegxf9d4qm/OAtm/P3D77XS8PbS5Ku+7/pkqSpNx9d4fjF67bELOc3z3+ZYJT13+qNxdpUXKru7ZtLkm4ZOUuvf7lY3ds1177d2yZVViZzEeecSsucGuYl1xa5y42jdfnhu6h1sx2p4W+en1b+eQ1KzreoRtCgCgAAgHSL1UJ205uxJ1FKhdveqn7O0kiSKsVOahPJo0pKy2JuD/L6OtKNNx1rou5/14c67N5x5c+XFXmz3m7elvyszTU1+Ebvn/LDGo2dszzpOiJuf3uO+tz0rsqSHHdcWub0r3Hz9ODY72tddzrkfKIamdWLyZQAAACQbgtWVWyF+vPbcyrMvPpF4ZrKp1QQfc26ZtM27XP7+5qxyDu/ck62vbQsqVavWNfDicwiG2/86ebiUq3dVLdZh2vrihe8Ft4NW7fr8uenaU1UHMvXb9XCNZtrXXYiE0xt3laia1/+KqHZkBN15qOf69fPTq31+c9+Xiip5hsIC1bG7k68ZfuORDwdNwCSlfOJauTOUV3XrQIAAEBuKiktS9m1YuUW1ac+rZjknfGfzxMu69N5q1S0Zbsenbgg5v773p+bXGwxtn2QQAvex9/HXo7n/Ccna9+/JLeebCImRC3/s3pjsUa8NqN84qbKvlu+Ue/OWqZD/vaRijZ7syvvf9eHOvSecXp43Dx9Nn+Vtmwr1T63x18Ddu6yDUnH+MZXizVy+hLd//531R5XU75X27a0lRuK474nlT+DkVmnJWnk9MU68v4JNU5oVTnsJeu26N73khvbW1dZmaia2TAze6yoqPp1fRIxcvoSSdI/PkzvukgAAACon859YrJ2u3lM0GFIip3YjJrhzVJbuUV03vIkJ9pJMCuqnAB9Nj+zy5Vc9NSU8sd3jv5GL3+xUO/MWKqphWt07ctfxexevXlbqS7875QKEzzd+95cnfv4ZP20ZrOKtmyvck7E14virzUaT3QIb3+9RH1vGh0zcaxNr86valjG6MoXv9TP7vxA/W4Zo/3v+qB8u5WvdlLRne98U/541mIvv/puefXJ+fZK3b2vevHL8iV8MiUrE1Xn3NvOueH5+fl1Lmvj1hJJUtGWYLolAAAAIFhL1m1RzxGj9Nn8VTH3R4/jnLRgdcwxfl/+tFaTElhf8ugHJiYV26ZajHuUpL43jU6o2260RJdr7HdLapP2f3z4vd6fXbt1RCOJnkk6/T+fa+T0Jbr97dgzJn+9cJ1OeOjjqmXEed2lZa58bdVYEhnraSZd/dJX2l7qtHJD8t2AYyXdNa1jGrlxIUnL1++oM17j7drNVfOgmlp6C1dX7DpdXM37lC5ZmaimR/D9rAEAAJB5kXGhL01ZWGXfc5N+LH88ds5ynf3YJD3zeWGV437x78909mOT0hVi0raXJt9UF6t17/06TN5TndEzl2qMv17pA2O/0/DnptV4zumPfKZBcboSRydW8ZbQiSfW6y5ctUnnPDZJu978btzzpi9ap8JVm2Ken03T3+z75/e1eVtJ+fNFa7fowqhW6ejfcX2atyfnl6fZoR79VgAAAJBysVqvbomakXfxWq8VqXBV/AmK3viq6tqj8cyoY5fSVEum6Hkrkh+3GS0y2VHh3UMTPmfqj1W7vEZ+Z7995etax/LPj6rOZjvkvvHlj+M1Z907Zm7c1s27R39TZVvl3915T0yqsJRQLLF+J8l+BtZu3q7CVZvLk/n73p+rid/FHlccKdqSbMQLIsHN+RbVyC+sPt09AAAAQOpEj92rPPYuWiKXi//v/xJPmE7+16cJHRfdWjh94bqYy8EsK9pa5+6XpUksW5JsF+ZEbIia1GdapaQ03vqtazbHH1uaqNEzq+92vC3OZyJekjru2xUxu2zPXFxxfp2aktRM+8W/P9WTn3iTe2XBpL41Ck2iCgAAgHCKXA6OmrFUfW96VxuLS6o9PsI5p5HTF1c7jjEVKueP//yo6iSgB/z1wyrJXTxFcZK76QvXaX6cpUlS6Z4xsWeHveH1meWPT3vkM0ley+320jJd9/JXMc+J1zKYSje9saNVPZHf9SVPf1H+ODrVGDl9se4Z823Mlvt41m7apic+XpDUOTWKUdSaTdv05U87WviTuWkhSUuKYt9ISKecT1RpSQUAAEC0dTEml4k2auYyTS1co/fnLNe1L0/XpXVY2zIRsyq1xL06zetenGhiGjG1cI0mLVitE/9RdUKhiKPun5DapCiGf4+PPTts5TVHZy0u0tEPTNSf356jZeu3pjWmRN361uxan/v+nOX69/j5+jaJ5W5+97+vdceob/T1orqtduLkyrvzjpq5tMr+gZXG/v713eSWmlmXgpbtZIVojCoAAADCKNEedpH8bdXGYp3+n8/1t9P2kpT+Vr3bKiVHkUQy3izF8Zye4Bqtj0zI3DIjPUeMKn88acGaCvtO+ucnkqRPk3yd6fTSlJ/i7lu7eVuFNUklqSxG0p/MfYBIAhiru3ey4nVhrq9yvkU18odpw9bEungAAFBfmVlTM5tiZl+b2Wwzu93f/rSZ/WBm0/2fAf52M7N/mNk8M5thZgODfQVAahRt3q5/ffR93OVF4iUSP66uOIlSshPO1FZJpTjT3SHwnjFz01xDchas3JSx97ournrxK93w2swK22LNJL1yY7H2vPW9TIWVs3K+RXX1Rq9rx5TCNTUcCQBAvVcs6Ujn3EYzayTpEzOLrL3we+fcq5WOP0FSX/9nf0mP+P8C9drtb8/W618tVv/OrXVkv44JJ0HPfP5jhed3jIq9Xme6OSfNW7FRK2qxLifSa8WGmrsoXxS1NExNNsUYL12bBrYFK+PPVF1f5XyL6tbttVtEGQCA+sZ5IjOlNPJ/qmucOUXSs/55kyS1MbNO6Y4TSLfIZEmRiXHidf19b3b1s8GuD6hHnpPT0Q9MCGRcYFBcPVlKckYdx5JWtsBfCim6Uf3JTxYkXc7VL8WejCpdikvSn2PlfKKa5IRWAADUa2aWZ2bTJa2QNNY5N9nfdaffvfdBM2vib+siKbrf2iJ/W+Uyh5vZVDObunJl+mfgBOqqpuUJI8u8vD97eYYiSk4Yr18Xrsn8rLK1UdclghKxvTT7PwCZmLA25xPVdM9qBgBANnHOlTrnBkjqKmk/M9tT0g2S+kn6maR2kv6YZJmPOecGO+cGFxQUpDxmINVq6up785veOMNsbcVbSZdfZLlMLAGa84kqAABh5JxbJ2mcpOOdc0v97r3Fkv4raT//sMWSukWd1tXfBuSEdVu8rrOVr6mnFq7VPWO+1etf8nEHauOVqYvSXkdWJqpmNszMHisqSm0fcAAAcpmZFZhZG/9xM0nHSPo2Mu7UzEzSqZIiq9u/JelCf/bfAyQVOeeqLsAH1DOR1p4bXp+pV76oOitrSZmLu9YnEIQLnpxc80FZ5MXJ8ZfxSZWsTFSdc28754bn5+fXuawGDbJ/qmsAAFKkk6RxZjZD0hfyxqi+I+kFM5spaaakDpLu8I8fLWmBpHmSHpd0ReZDBlIvulviH16boW+Wrg8uGCABmRj7Wt/k/PI0AACEhXNuhqR9Y2w/Ms7xTtKV6Y4LyLTKY1Tfn5OdkyYBiC8rW1QBAACAVPl22YagQwCQpJxPVBtkYkoqAAAAAAiJTHSnz/lElTQVAAAgZLgABOq93E9U+UMFAACQsx4eN0/TflwbdBgAUozJlAAAAFBv3fveXElS4d1Dy7fRTgHUfznfogoAAIDc13PEKG3dXipJMoAbykoAACAASURBVLrUAfVezieqA7u3DToEAAAAZMCqjcV67vNCFW3ZHnQoAOoo5xPV8w/oEXQIAAAAyIBZi9frlpGzNfG7lUGHAqCOcj5RpecHAABAOGwrLQs6BAApkvOJKgAAAMJhU3FJ0CEASBESVQAAAOSEG16fGXQIAFKERBUAAAD1SlmZ0+tfLlJpmQs6FABpkpXrqJrZMEnD+vTpU+eyurdrXveAAAAAkDVe/mKhbnxjptZtZnZfIFdlZYuqc+5t59zw/Pz8OpfVo32LFEQEAACAbLF6Y7H376bigCMBkC5Z2aIKAAAAVPbT6s16ccpPat44T5L08Lj5AUcEIF1IVAEAAFAvHP3ABG0rLdOwfToHHQqANMvKrr8AAABAZZF1Ur9btiHgSACkGy2qAAAAyGoD/zJWfQpalj+fu5xEFch1JKoAAADIams2bdOUTWuCDgNABtH1FwAAAFnr64Xrgg4BQABIVAEAAJC1Hpu4IOgQAASARBUAAABZa9TMpUGHACAAJKoAAAAAgKxCogoAAAAAyCokqgAAAMhKFzw5OegQAASERBUAAABZ6ePvVwUdAoCAkKgCAAAAALIKiSoAAAAAIKuQqAIAAAAAsgqJKgAAAAAgq2Rlompmw8zssaKioqBDAQAAAABkWFYmqs65t51zw/Pz84MOBQAAABlUVua0563v6eUpPwUdCoAAZWWimmpH775T0CEAAAAgAdtKy7SxuES3vjU76FAABCgUiWrnNs3UpnmjoMMAAABAgopLyoIOAUCAQpGomiTngo4CAAAAAJCIcCSqZnJkqgAAAABQL4QkUaVFFQAAIFvd//5cXfnCl0GHASCLNAw6gEwwmchTAQAAstM/P5onSXo44DgAZI9QtKg2MNH1FwAAAADqiVAkqmZSGXkqAABAVus5YhTDtQBICk2ianJ0/gUAAMh6E75bEXQIyDGv/ubAoENALYQiUX3qkx+0dbu3eDQAAACy12+eZ1IlpFbzxqGYlifnhCJRLfH7/a7eWBxwJAAAAACAmoQiUY3YXkr3XwBA7jKzpmY2xcy+NrPZZna7v72XmU02s3lm9n9m1tjf3sR/Ps/f3zPI+BE+t7w5S8c8MCHoMOqlhg0s6BByxvXH7hp0CIHq2rZZ0CHEFKpE9d2ZS4MOAQCAdCqWdKRzbh9JAyQdb2YHSPqbpAedc30krZX0K//4X0la629/0D8OSDvnnJxzem7Sj/p+xcagw6mX7jh1Tz109oCgw8gJVx3ZN+gQAtWlDYlq4GhPBQDkMueJXPU38n+cpCMlvepvf0bSqf7jU/zn8vcfZWY00yDtet0wWqc98lnQYdRbvz60l34xsKtOGdAlJeU9fcnPUlIOau/qI/sEWn+rptk3jjdUiSrfvACAXGdmeWY2XdIKSWMlzZe0zjkXmVFwkaTI1W0XSQslyd9fJKl9jDKHm9lUM5u6cuXKdL8E5LiiLdslSV/+tC7gSLLPPaftrWd/uV+Nx900tL8aN0zdZfy+3drWfEz3NimrT5IuPqhnSstLxjH9OwZWdzz9O7UOtP6Ztx2n53+1f6AxVBaqRLW4pCzoEAAASCvnXKlzboCkrpL2k9QvBWU+5pwb7JwbXFBQUOcYES6bt1VcdeEfH34fUCTBadE4L6HjmjRqoEZ5mb88z2/eSKcN7FrtMXkp7mwRZHfTxy8cHFjdsZhJXds2D7R+STqkb4fAYoglVInqw+PnBR0CAAAZ4ZxbJ2mcpAMltTGzSL+urpIW+48XS+omSf7+fEmrMxwqcti0H9eq/5/e07hvd6yNWlpWvwZj7dM1v85lvHPNoQkdZ2YKqvP9lUfsktH60vE6W6e5++qoaw5Ja/moKFSJqqtffxcBAEiKmRWYWRv/cTNJx0j6Rl7Cerp/2EWSRvqP3/Kfy9//kXN8WyJ1vvxxrSTp03mrJEkr1m/Vpnq2rv1TF9d9/GavDi1SEEl69S5oqcK7h8bdHy+xbNUk+8Y2pssenet+0yIZ/XZuJUnauXXTjNabLUKVqAIAkOM6SRpnZjMkfSFprHPuHUl/lPRbM5snbwzqk/7xT0pq72//raQRAcSMENnvrg/1v2mLgg4jKU0b5aVkKZjPRhxZq/N+1rPm8aOZ9OtDe1V4fuWRfXTLSf2TLidV87bt16tdUsf/fN/YE1B99LvD9ch5A2sdx5jrEms1j/jLqXuWP776iNgTKe3ZxUuMD01Tl9yjd8++sbrRQpWoHtA7uQ8yAAD1iXNuhnNuX+fc3s65PZ1zf/a3L3DO7eec6+OcO8M5V+xv3+o/7+PvXxDsKwAy55oMz7LavmXjGo+JlbqZTB//4YiE6hh+WO8ko0p8rGifnVpKkg7apWLS1DivQa0mRkpV543rjk5uaZkHz4q9pE/vgpY6Ya9OtY6j387JTYbUvNGOccuH9I099j/yeUhXN5fObZr69cS+aXD4rsHOSZCViaqZDTOzx4qKilJabrIfIAAAANSeq6eLA+6dgnGplTVpmNiESrFShm7t4k+0c8nBPcsfn7d/96RievjcgXr18gMTOvbWYXvohUv3165+d9REXXZ4b826/Th1yk+++2q/BOqKNVvuoB7Z1QpdV+kakNGghlbtZxKYgTqdsjJRdc697Zwbnp+f2X7gAAAACIkYF+k/69lWpw7orIfPTbwLaCLJVKIO6F1ldaiktG/ZJKnjh+7dSZ3yE2tRbdooTwf3Sb4L6g0n7K6WMcaxmlmNS96Mue6waiddKrx7qNo0r9pSffcv9qoxrkyNrU1Ft/F06NWhhfb3u03Heo+zoVtwViaq6dIkhetNAQAAoHrxuhRmhRjNVA0bNNDfz963QgtmTUMpI0nYwX2qJpn/PGdfTfx91W678dZALWgVI9H06483DjTyHt88NHZCGMuuHVvq5eEH1HhcfrNGVWNs2UTtWjRWx9Y7Yq3Nb9kkvXHFwbU4s2Y7tWqqPbtU35Pyw+sPj7n96UvqPnlWbVT3Oattz4Tqyhx3/RC1alr19xtx+qDYY3kzKRSZW+SXFMS6VAAAAGFVX7r+vnDp/pJqt2RK5JxrjuxbYYzo747ZVcP26azu7at22/3ujhMSL7+G/bV5jzu2bppQ6+171x1WZVvjhg305S3H6MEzvbGe+9dyDphULk9zz+l766mLY6+N2qJxXvn42mg7tYrdFXnIbjulLrBqJPJbszoOUh1xfO2X0T5+T2+87qMXDNJltRj7nAqhmE/alL5ByAAAAKiemfTDqk1Bh1FRVKYUSWROG9g15qHVXUfGazW++qjkJvnZEVbtMrhUzaIbbedqxpUe1KdD+XI2ZQGvjXvm4G5VtkUazF8efmDcFuxMaN2skdZs2lbtMb0LWmhZ0dYMRVRVdR+d4/bYWQWtmujRiZmfay9UTYxBLaAMAAAQZm99vURH3Dc+sPpjdcuN7vrbsXVTFd49VKcNip2oJsIpNTPZNm3kXZ63be51y6xtktW7QwvFGx55wp7xZ7d96OzYs+JWpzbX2PFaNCPuiFq+JVH/Pm+QDu7TXq2a7miLC/r6v3nj6ifR+sXALupQw9jifbpVP5Y3HjNVO9460db4gd2DmZwqFIkqrakAAACZF2ltXL6+ONA4Yk4Mk6oMppbFdIizXM1eXfJ118/30rjrh+g3h++i+8/Yp1bld2/fXNcft1vMfefsV7UFMqKuEzqlwhmDuup0/6ZBMrn/IX076IVLD1CDBpa2mXLr6qGzB+jD38UeHxtPk4YN9K9z961VfUP3rnnJndqMJa8pAU+FUCSqAAAAyLxsHqOaqoa2P53UX/t0zdc+XZNr9eraNvaSM2amc/fvrjbNG2vECf20U+vIWpfJqe746roJd2yd/DIysVx4YA89dsGgWp177xn7qGmjqonQmYMTb/FulFf33/C464do2s1H13hcJKkeGmMd1spv9R6d87VLQdUxs0GIxDFsn+TXj71p6O6pDqeKUI1RLQm4/zwAAADqh6aNGmjr9rIaj9uzS75GXnWIpORa/1K9aknlbsdBX/VecEAP9e0Yf+meeLly74IWcc+55/R99Om81Vq8bkuN9f/r3IF6fvKPMddZTVSvDvFjiRZJVB8+b6BGjRgV85h2LRrr/jP3iTmxUyJSNYP2l7cco+KSUklS5zbNNO/OE9QwyQlnI2OT0y0ULaqR/PSR8fODDQQAACDH9RwxSr9+dmrQYdSopjU8I9K1xM5DZ1fsyvnJH6suYxOtUZyxqsnGd/vJeyR1fCLMTA+dPaDG11Cdh84eoHl3nqCx/y+5brHxdGvXXDecsLsaVHNH4Ppjd9Vrlx9UY1nP/Wq/8mWG/n5W1TG8iXSXvvHE3XVEkjMKp+Oz165F4wpr5yabpGZS9kaWQrt2zI7mdQAAgFy2eVuJJGnsnOUBR1KzdCxb+LNe1S/Vcv8Z+5QnyNFrtT52waC4XYEjkun2GlE50Rm6VydddFDPpMtJxCkDulT7Gk7ep3OF55VTsJP36ayGeQ2Ul+qm5mpcdWRfDepR80RBh/YtKF9mKJExn4lIZuItp9p1o69NonvW4G665/S9kz4vHUKRqN54Yvr7UAMAAITdFS98Wf741WmLAowkGMftsXO1+08b1FVvXHFwle3H1nCeJDVpmKcx1x2q0dccmlAsJm/SpOgZjzM5Zrhy194/Ht9PM247tvx5uxaxJ5Oq7z7+wxEac92O31FN+Wi6Wuxr62+n7x1zuZ8ghCJRbdggFC8TAAAgUNMK15Y/vv5/XwcYSWol0/o15rpD9cKl+6cljn47t1b/zhXHXFaXfLZp3lj/vXi/tMSSrAYNTK2bNip/vn+Cswv/Ps7MxdmqW7vm6rfzjt9RopNLd27TrOaDEhSZ1TlSd8zlmeqBUGRw2TzjHAAAQK66a/S3QYdQwSF9OqS9jn47t9bBCdazT7c2OnGvmltTE1F5Jt/I8+jr4CYN07+kSESqloe58og+qSkoxWrqohyZFCoymVPXttUnou1aNNb8u07UsKgu0tG/0ppaXm84oV/542aNKs6X+8wl++kXA7vo6Ut+Vm0Z1fnqlmNqfW5thWLWXwAAAGCXghb6ZN6qOpdzycE99d9PC+tczsgrq3YDTqdbh/XPaH311V9O3VMdWzWp9piaGkrbt2isBSs36ZcH99LVR/bVnl3ya6w3Ovl96OwB+nz+akk1J/3vXH2I9uySr7++G/vGUMO8BnrgzKqTQCWjbQBdtUlUAQAAkNNSPQrw1mF7pCRRTafK3ZUb5zVQm+bJJxtf33psypfSSYULD+yR1PHvXnuoWjROLPW54IDkyo7lqN076ovCterStlmNE2UhtnB0/XXRj+kGDAAAkGo9R4zShuKSoMNImehLxgfOHFBlTc3z9u+uLikcV1hX8a5xG+c10OmDuur5Wo6bzW/WSK2ixpYmKtGxmTuOT+6E4Yf1Tur43Tu1Lp+5NxMuO6y3vrzlmIwkqd0q1RGZmfiwXQtSXtflQ3ZJeZnxhKJFtSzqP+7G4pJa/WcDAABA+JhJp+7bRafu20U9R4wq337nz/cKMKodahq7aGa674x9kirztmH99e6sZXUJK+2yve3JzKqd2TiR8BPJ3WfdfpxaNqmY0g3q0VaFdw9NoIbkpKPM6oQiUY2W7N0aAAAA1G9ZntNknYsP7qWLD+5VpzKyPZHMFommJl1qmIwpF4UuUQUAAEA4RTdYDOrRttpjG+c1UHFJGQlXir133WFq3jhzsw+nS+OGDVSyrTRj9Q3o1kbvXnuo1m7epnvGzFWZc5qxqEh5OdwIF4pElb8vAAAAua9lk4ba6I+TfefqQ3TSPz+pcsxFB/bQzvnN1LRR9cnSq5cfpHdnLVWzHEiqssluO7dKSTlB5WeN8hro98ftpqN376jj/j6x1uVElqvZrWPF9+OMQV319tdLNLB7W01a4M/662czu/tL3bx5ZQet27xNc5asz+nPZygS1ehMNXfvOQAAAITb4bsWaNTMpZIUczkQ55xuP2XPhMrabedWKUuq0sllaZNMDjf0pWRt14N26aCRVx6svSp9Tg/btSBqLGj8N7FN88Y6KAPrAgcpHLP+Zul/YAAAgFxQVpb5a63KE8hI8a/5cjhnKsc8LPXPPt3aqEE1a/8c0LudpB0tqWETjkSVPBUAACBt7n1/bubrPH3vjNeZzbJlCcZdClrUfBAScsqALpp689Ea2L368dS5KhSJagA3+QAAAELjvQCWMmG5QU/l5Wn+fd7AgCLxRNabrWkMcMRlSa6HGlm7tnFe8GnMQ2cP0D2npfeGSYeWTdJafjYLxRjVbLnDBAAAUJ89N+lHPfTB95p689EVtnOllT2aNgo2gXvwrAH6onCNurZtntDxN5y4u244cfeEy3/0gkH6bP5q7dS6aW1DTJlTBnQJOoScFvytiBjMbJiZPVZUVJSS8vjjCQAAUHe3vDlLqzYWBx1GXCft3TnoEEKvVdNGOrJfx7SV37ZFYw3du1Paykf2yMpE1Tn3tnNueH5+1dnaalleSsoBAABAVUFca1WeO6hj6yY6ca/YCUykG2qzxqHoTAjkhFD8byVPBQAASL35KzeqW9vm+nHN5qBD0eQbj4677/RBXbVyQ7EuPTS58ZD10a7+upwnD6B1GfVbOBLVqMfM3A0AAFB3KzZs1VH3Twis/lZNd1zGxlqqJlrDvAa6+qi+da7zrasOVrsWjetcTjp1bds8ah1OhM3OWTB2N1XCkai62I8BAABQO+u3bA+0/r27ttF/zh+o/p3yld88MzMA7921TUbqSUaHVl7i3KZ5difQSL+Jvz9C+c1yZzbsUCSqZWSnAAAAKZUNl1fH78mkOsMP7a2OrZrq5/syA23YdW+f2EzL9UXoEtUs+JsKAACANDn/gO6au2xD0GFkTMO8BjptUNegwwBSLhSJ6v692gcdAgAAADLgjlP3CjoEpFjLJg11ycE9gw4DGRaKRHXn/B2Diou3l9Y44B4AAADZi8kxw2XW7ccFHQICkJXrqKbTG18tDjoEAACAei/Tw6n6d2pd/vjGE3bPcO0AMi10TYt3jPpGvzqkl4xbcQAAALVy0xsz9cLknzJa5xtXHqR/fPi9rjqir5o1zsto3QAyL3SJqiQtXrdFXdvm1qxYAAAAmZLpJFWSmjTM0++P65fSMq85so+G9NsppWUCSI1QJqoAAABIzrwVG4MOIeV+e+xuQYcAII7QjVEFAABA8o5+YELQIQAIERJVAAAAAEBWIVEFAAAAAGQVElUAAAAAQFYhUQUAIAeYWTczG2dmc8xstpld62+/zcwWm9l0/+fEqHNuMLN5ZjbXzI4LLnpkO+cyvWpqavTZqWXQIQCopdDM+turQwv9sGpT0GEAAJAuJZJ+55z70sxaSZpmZmP9fQ865+6LPtjM+ks6W9IekjpL+sDMdnXOlWY0atQLZfUzT9UbVxykdZu3Bx0GgFoITYtqk4aheakAgBByzi11zn3pP94g6RtJXao55RRJLzvnip1zP0iaJ2m/9EeK+qi+tqi2atpI3do1DzoMALUQmuzNzGI+BgAg15hZT0n7Sprsb7rKzGaY2VNm1tbf1kXSwqjTFilOYmtmw81sqplNXblyZZqiRjarn2kqgPosNIlqg6jctL7eFQQAoCZm1lLSa5Kuc86tl/SIpF0kDZC0VNL9yZbpnHvMOTfYOTe4oKAgpfEiu81dtkE9R4zS4rVbgg4FQMiEJlGlERUAkOvMrJG8JPUF59zrkuScW+6cK3XOlUl6XDu69y6W1C3q9K7+NqDcCQ9NlCQNuW98sIEACJ3QJKoN6PoLAMhh5n25PSnpG+fcA1HbO0Ud9nNJs/zHb0k628yamFkvSX0lTclUvKgfsmUSpe6MMwVCJzSz/kYnp3T9BQDkoIMlXSBppplN97fdKOkcMxsgb5hhoaTLJMk5N9vMXpE0R96MwVcy4y8kacHKjfro2xW69NDeQYcCIMRCk6g2oBEVAJDDnHOfSIr1bTe6mnPulHRn2oJCvXTaI59p7ebtuuDAHhmtt0PLxlq1cZt6tG+uH1dvzmjdALJPKLv+0qAKAAAQ28bikkDqbdHEaz9hSUEAUogS1ehbzK9MXRj3OAAAAEgWs4E+dTq2blL++JKDe+rOU/dKa30A6pfQJKrRLaqrNhYHGAkAAEC4XXxQT3024qjy57cO20MFrbzElZ5vAKQQJapM9AsAAFDVZc9N1cjpmV2Z6JaT+iuPCUQAVCOkiSp/GAEAACTpvdnLde3L08ufZ6JFs7oklcYFAFKIEtUG/NUDAABI2Lot2wKre/hhLI0DhF1oElW6lwAAAMT31Cc/aGrhmvLn5z8xOZA4nJNuPHH3Ctu4jAPCJzTrqEZzjNIHAADQus07Wk3//M6cCvt+WpMda5ledlhvnfWzbkGHASDDsrJF1cyGmdljRUVFqSyz/PHLX7A8DQAACLfRM5dqwJ/Hxt2/dXtZWur95cG9qt1febTWDSfurt4FLdMSC4DslZWJqnPubefc8Pz8/JSVSZcRAACAHe57b24g9TZqWP1FGR3fAEhZmqimQ+U/iSs2bA0kDgAAgDCzOKsvMO8lgGihGaNaedbfVRu2aadWTQOKBgAAIDO+XrhOqzYW66jdO1bcEVBimGhC+ofjd9PyIhoWgLAKTaJqlf4qvj1jifp3bh1QNAAAAJlxysOfSpIK7x5avm3LtlKVlQXTx3Zwj7Yxt3dv11wdWjbWDSf2kyRdMaRPJsMCkGVC0/X3kD7tKzx/ZPz8gCIBAAAI1u5/GqPC1Zmf1ffLW46p2rLra9ooT1NvPkZH9ou9H0C4hCZRPXGvTkGHAAAAEKh1m7fpTyNnBVZ/uxaNA6sbQP0Smq6/QY3DAAAAyAbPfl6ob5dt0IuTfwo6FACoUWgS1XgzzAEAAITBn0bO1lmDuwUdRgWvXX6gGjYITQc/AEkIT6JKngoAAJBVBvVoF3QIALJUeBLVoAMAAAAImFMwM/2Ov36INhaXBFI3gPopPIkqTaoAACDkXpm6KJB6e3ZoEUi9AOqv0AwKIE0FAADInIJWTYIOAUA9FppEtQEtqgAAABnzxhUHBR0CgHosNIkqTaoAAACZ07Vt86BDAFCPhSZRpUEVAAAAAOqH0CSqAAAASJ9YY1LPP6B7AJEAyAWhSVRdMLOxAwAAhMIXNx2tfju3qrDtL6fsqR/+emJAEQGoz0KTqAIAAITJM58V6sfVmzJaZ+OGFS8tzYwlAgHUSmjWUY21vrVzjj+eAAAg52zZVqpb35qd8XqbNcrLeJ0AchMtqgAAADnGxbpDn0Z3/2IvSdI/z9k3o/UCyF2hSVRbN6vaeMy4VQAAgLo7ez9v0qSdWjcNOBIAuSI0iaqZld/ti3hl6sKAogEAAAAAxBOaRFWSGuZVfLkvf0GiCgAAUBf7dGsTdAgAclCoElUAAIAwMGVusshB3dtmrC4A4RGqRJX5fQEAQBhkcjIlFlAAkA6hSlQBAADCgAkjAdR3oU5Ut24vDToEAACAlMtknkqDKoB0CHWi+u2yDdq8rSToMAAAAFLq/CcmZ6wuuv4CSIdQJaqx/pBOWrBan85blflgAAAA0qCszGn6wnUZq69100YZqwtAeIQqUT1olw5Vtv3y6ak6L4N3HQEAAFJpY3GJeo4YpRcn/6RZi4u0z5/fz1jdfzqpvy47fJeM1QcgPBoGHUAm7ZzfVJcc3FP//bQw6FAAAABSYsX6rZKkBz/4Tis3FGe07l8e0iuj9QEIj1C1qEqZXVcMAAAg3fIaeNc2mU5SASCdQpeoNoiTpzKpEgAAqI8aZOFsRoN6tA06BAD1XOgS1Xh/yz+btzqzgQAAAKRAg3h34dPknP26V7t/0g1H6flf7Z+haADkqhAmqrH/mGfhzUgAAJJiZt3MbJyZzTGz2WZ2rb+9nZmNNbPv/X/b+tvNzP5hZvPMbIaZDQz2FaA2MpynqnGe6cS9dta9p+8dc//O+U3VrHFeZoMCkHNCmKjG3v7oxAV64uMFmQ0GAIDUKpH0O+dcf0kHSLrSzPpLGiHpQ+dcX0kf+s8l6QRJff2f4ZIeyXzIqK2t20v1tzHfanuJy3jd/z5vkM4Y3C3j9QIIj1DN+ivFn0xpyg9rNOWHNbr00N4ZjggAgNRwzi2VtNR/vMHMvpHURdIpkob4hz0jabykP/rbn3XOOUmTzKyNmXXyy0GW63fLGEnSrMVFAUcCAKlHiyoAADnIzHpK2lfSZEkdo5LPZZI6+o+7SFoYddoif1vlsoab2VQzm7py5cq0xYzEfbd8Q/njj79fFWAkAJAe4UtUgw4AAIA0M7OWkl6TdJ1zbn30Pr/1NKm+os65x5xzg51zgwsKClIYKWprub92KgDkqvAlqmSqAIAcZmaN5CWpLzjnXvc3LzezTv7+TpJW+NsXS4oeaNjV34Ysl+l14U8f1DWj9QFA6BLVmlz23NSgQwAAoFbMm9r+SUnfOOceiNr1lqSL/McXSRoZtf1Cf/bfAyQVMT61fsj0jfedWjXRbcP6Z7ZSAKEWukR16F6dq93/3uzl5Y9/XL1Ji9ZuTndIAACkysGSLpB0pJlN939OlHS3pGPM7HtJR/vPJWm0pAWS5kl6XNIVAcSMJDnn9N9PC4MOAwDSKnSz/nZp0yzhYw+/d7wkqfDuoWmKBgCA1HHOfaL40zEcFeN4J+nKtAaFlPpgznLd9vZsLVq7JaP1Zn4BHABhF7pEFQAAoL669FmGKAEIh9B1/XUJ3BPcXlqWgUgAAAAAALGELlFNxJhZy4IOAQAAIHATfj9Eg3u01cUH9Qw6FAAhk5WJqpkNM7PHioqKAql/3eZtgdQLAAAQhHjzcfRo30KvXn6QOrZuquZNvBFjrZo2ymRoAEIqK8eoOufelvT24MGDf536sms+5paRs1NdLQAAQL122sCuWr9lu84/oEfQoQAIgaxMVLMBySoAAAizy4fsUuF5aSzGaQAAIABJREFUXgPTpYf2DigaAGGTlV1/04np1QEAQH3hnFNZWXqvXl67/KAq204f1FXXH7tbWusFgOqELlEFAACoL456YIL6/WlM2sof2L2NBvVoK0l67fIDtVeXfElSz/bNldcg3pK8AJB+JKoAAABZ6Ntl67Vg5SZtK/GWzdtUXJLyOsx2JKODerTTYbt2SHkdAFAbJKoAAABZ6LnPf6zw/Iz/fB5QJACQeaFLVF0i0/4CAABkkWMfnKA5S9envZ5D+hRIkg7o3T7tdQFAdUI36290FxcAAID64LvlG9NSbuWrogN3aa95d56ghnmha8sAkGVC91eobfNGuuqIPkGHAQAAUK3l64vTXkes+/ckqQCyQej+EpmZrj+O6dYBAEB2++Cb5Wmvw6q0qQJAdghdogoAAJCN5i7boJ4jRmnW4qKM1fn747l5DyA7kagCAABkgbFzlkmS3vhqccbqHNi9bcbqAoBkkKgmILJ+GQAAQLo9+ckPenHyTykv94De7VJeJgCkC4lqApxY0gYAAGTOjW/MTHmZTRrmVdnGCFUA2YpENQFHPzAh6BAAAECOS/dS73kNdqSlrNYHINuRqCZg4ZotQYcAAABQJ93bNa+yjT5jALIViSoAAEBAPp+/WqVlXrqY7lbO6PJpUAWQ7UKbqPbu0CKp479dtj5NkQAAgDC6+L9TdM7jk/ToxPmS0t/1V5J279Q6/ZUAQAqENlE982fdkjr++L9/nKZIAABAGI2fu1KSNH/FpozV+fKvD9A7Vx+iQ/oWSKJlFUD2ahh0AEHp2rZZ0ucsWbdFL035Sf/8aJ4K7x6ahqgAAADSJ795I+U3z9ej5w/S0qItatCAVBVAdgptojp0r05qd2ljnfvE5ITPOejuj9IYEQAACKMgZuBt1jhPvQtaZr5iAEhQaLv+mpkO6tMh6DAAAEDIpXNs6h+P75eRegAg1UKbqKbKMQ9M0G+emxZ0GAAAAFVccnDPoEMAgFoJbdffVPl+xUZ9v2Jj0GEAAABU0bRRXvnj3gXJrXgAAEGiRRUAACBg035co/vHfpfSMv90Uv8Kzy84oEdKyweAdKJFFQAAIEBvTl+sxg1T33bwy0N6SZI+HXGk2jZvJAti1iYAqKXQt6iesOfOumLILkmf55iRAAAApEBpmVNZWd2uK/rt3Cruvi5tmql5Y9omANQvoU9UHzl/kP4QNSMeAABAppXUIVGdeduxzOgLIOeEPlGtrdWbtgUdAgAAyBFldcg0WzVtlMJIACA7kKj6Th3QOanjB9/xgaYWrklTNAAAIEzWbeYGOABEY8CCr2eH5Kdsn7N0fRoiAQAAYTNu7so6nT+wRxvNXb5BknTbsP76+b5dUxEWAASGFtU6YO48AACQDW47eY/yx40b5im/Od2BAdRvJKq+Ns2S/4MePZpk5Ybi1AUDAACQhCYN84IOAQBSikTVd34tFsFevG5L+ePf/e/rVIYDAAAAAKFFouprmJf8W/HohAXljyd+t1I9R4zSS1N+SmVYAAAAABA6JKopdsPrM4MOAQAAZJHVG4v1tzHfqrQOa6UCQNiQqEZ59TcHBh0CAADIMTe/OUuPjJ+vCd+tKN/2+peLUl7P2T/rlvIyASAoJKpRBvdsp9m3Hxd0GAAAIIdsKymTJJWV7dj221eY2wIAqkOiWkmLJiwtCwAA6i9j/TwAOYBENQ1KSstqPggAgBQzs6fMbIWZzYradpuZLTaz6f7PiVH7bjCzeWY218zoUpQBn89frRUbtqasvDeuOKj88bVH99XhuxbopL07pax8AAgKzYdp0Oemd1V499CgwwAAhM/Tkv4l6dlK2x90zt0XvcHM+ks6W9IekjpL+sDMdnXOlWYi0DAa/90KPT8pdasD9OrQQvt2b1v+vFN+Mz3zy/1SVj4ABIkW1TRZvn6rlhZtqflAAABSxDk3UdKaBA8/RdLLzrli59wPkuZJIstJo1QmqZI07vohKS0PALIJiWqa7H/Xhzrwrx8FHQYAAJJ0lZnN8LsGR5rgukhaGHXMIn9bFWY23MymmtnUlStXpjvWnMOiNACQPBJVAABy2yOSdpE0QNJSSfcnW4Bz7jHn3GDn3OCCgoJUxwcAQBUkqmnGxEoAgCA555Y750qdc2WSHteO7r2LJUUvvNnV34YUS+UkvFcM2UWSdPFBPVNYKgBkHxLVNBsze1nQIQAAQszMoqeA/bmkyIzAb0k628yamFkvSX0lTcl0fEjOyQM6q/Duobrt5D2CDgUA0opZf2PonN9US4pSM3V8aRkjUwAAmWFmL0kaIqmDmS2SdKukIWY2QN5QyUJJl/3/9u47PKoq/QP496SQkBASIAUIgRAIvRt6L1JF/KmroLtYQde6uroGRcVdFVbXhl3BrmAFqSKCNJESSiihBQiEmhAgJIGElPP7Y+5MZiZT7kxm5k75fp6Hh7l13rmZ5M4755z3AICUcq8Q4jsAWQAqADzIir/u4cpPAsKl7bNERN6LiaqHTPkiAyuzznLaGiIichsp5SQLq+fa2P8lAC+5LyICgI2Hz7nsXAn1w1x2LiIib8ZE1c0enb8T249dwMqss1qHQkRERBooLXdNvYrDL49FcBBbVIkoMHCMqgd8/ucxrUMgIiIiDZy86Lo51ZmkElEgYYuqBVqNKl2ceQptG0ehTUKURhEQERGRK+QVlWLoq2swrH2C1qEQEfkktqhqoM305Zj20+4a6x+etwMj31inQURERETkSmsO5KPkaiUWZ56q1Xmeu66DiyIiIvItTFQtmNizuVvPf7WiCvO2HHfrcxAREZF2XNVJ99oObJElosDERNWCR4a3RvZLYwzLHZrU1zAaIiIiIiKiwMJE1QIhBEKCqy/Nbb1d18K6en919d/vM3Jddl4iIiIiIiJ/wUTVw+7+LMPw+Lmf92oYCREREbnLtmMXtA6BiMinMVElIiIicrH5W9lrioioNpioqiA4bRkREREREZHHMFHVkNRsxlYiIiLyBQn1w7UOgYhIE0xUNVRaXmV3n6oqiY3Z5yAlk1oiIqJAkPncSMPjOiH8qEZEgYl//VRwd46468RFJKcvtbjtiz9zcNuczVix94x7gyAiIiKvEBEWrHUIRESaY6LqgOAg1w9WPXnxCq5/5w+TdXtPFWLJrlMAgJyCywCAUxdLXf7cRERE5FrnisusfvlsT7vGUVj75BCEBvPjGRER/xKqoC+m1DelkcvPffenW2usGzd7Ax76ZofF/UvKKvCXDzYiO6/I5bEQERFR7Txfi6nnbuieiBaNIl0YDRGR72KiasMN3ZqiXeMow7I7qv9eKi23ue2zjTkm6zYeLsDWnAuYtXy/64MhIiIip5WWV2Lp7tNah0FE5BdCtA7Am705sTsA4OvNxwAAwg2Z6ulC6116P/sjx+o21lYiIiLyLq+uOODS8w1oHevS8xER+RImqg5IjAlHswZ1ceLCFc1isJYql1VUYv/pInRNivFoPERERKSTV1TmsnOte3Io4qLCXHY+IiJfw66/DhHY8NQwrYMAAMMMrH8eLsC+05fwwuIsTHj3DxwrKNE0LiIiIqq95o0iULeOrvrvL/8YiPdv76FxREREnsUWVS+2YMdJw2N9r2P9//p5VSd9vAkA0KVZNADgwuVytFBqPp0pLEX9uiGIqMMfMxERkbu5a87zdo3ro13j+m45NxGRt2KLqhc7eq66dVR/7zMkqmb7WuoS3GfmKtzy4Z9uiY2IiIh0MnMvui1JJSIKVExUVWgdVw8A0C0pWuNIAGF1lKple05eMjyurJJITl+KrzYdc3VYREREAWnZ7tOY8O4fWLDjJJbvOaN1OEREfoOJqgq9UxphzRNDcEtakmYxmBcctvbFra1vdK+UVwIAXl62z1VhERERBbQlu04BABbuPIXKKudaVRNj6gKwXjCRiCgQcfCiSsmx1RNw1wsLQXFZhUef/4XFWejYNBqLM3U3RPNbYeaJQrvn4A2QiIjIdRZlnsKy3bpW1HUH850+j7VhPUREgYwtqk5Y9FB/vPx/nT3+vLd8+Cd+UgosFV6+il8sdDFSc5PjMBoiIqLa+z4j1+ljnx/fwfDYDdO0ExH5PCaqTkiJq4fbejfXNIbME4W4/6ttVrdn5JyvsY43QiIiIu9wV/+WaBodDgDoksg50ImIzDFRdYGv7+2tdQgmMnMv4uYPrFf7lexcREREVGtBtfwGeOO04ciZNQ6JDeq6KCIiIv/BMaou0L91rNYhGEgJ5BeXWdymrxhcWl7lyZCIiIj8UuaJiy49Hzs+ERFVY4uqH1LzBe+CHSfcHwgREZEfu3i5XOsQiIj8lscSVSFEihBirhDiB089ZyAqVaagsScz136VYCIiInK97c9ea3E9B+YQEVVTlagKIT4RQuQJIfaYrR8thDgghMgWQqTbOoeU8oiU8p7aBEv2PbtwD577ea/FbZM+3uThaIiIiMhcw8g6WodAROT11LaofgZgtPEKIUQwgHcBjAHQAcAkIUQHIURnIcQSs3/xLo3aS8yf2geT+7bQOgwTR86V4OTFKxa37cytHktzrrgMzy7cg/JKjlclIiJy1I/bXD+EhmNUiYiqqSqmJKVcJ4RINlvdC0C2lPIIAAgh5gOYIKWcCeA6ZwMSQkwFMBUAmjfXdgoYe/qkNEKflEZah+GUJbtOAwDqhATh2es62NmbiIiI9DJyzuOf32dqHQYRkV+rzRjVRADGM12fUNZZJIRoJIT4AEB3IcQ0a/tJKT+SUqZJKdPi4uJqEZ62cmaN0zoEg6xTl1BUarngw9wNR+0en3OuBLnnL7s6LCIiIp/y7dbjyDlXgo/XH3HpeRPq6+ZTja0X5tLzEhH5Mo9NTyOlLABwv6eej6ptPlqAsbOznD5+yP/WAPCu5JuIiMjTnvpxN6LCQ5zqTTV1UAo+Wmc5wb2zXzKaRodjdKfGtQ2RiMhv1KZF9SSAJKPlZso68jIvLFafpG47dh5XKzhulYiIyJKi0gqsO5jv8HEDbMy5HhwkMKZzEwg188sREQWI2iSqWwGkCiFaCiHqAJgIYJFrwiIt7D9zCTe9/ydmLd+vdSguVV5ZhcoqFv0nIiLnSVl9HynjF7pERG6ndnqaeQD+BNBWCHFCCHGPlLICwEMAVgDYB+A7KaXleVHIJxQUXwWgS1j9Seozy3Hje39oHQYREfmw0nLnk9O6ocFgYykRkWPUVv2dZGX9MgDLXBoRedzJi1eQGFNX6zDcKvNEodYhEBGRD3tr1SHnjpvYDV2bxSD3AosSEhE5ojZdf8nInMlpWPPEEK3DcEr/WatV7VdcVmF1264TFzm2lYiI/NYlK9Xz7ZnQLRHJsZEQyiypYSH86EVEpAb/WrrIiA4JSI6N1DoMp1WpGMOZ/uMuFJdV4Eh+scn6o+dKcP07f+DFpc5XFiYiIvJmpVcrXXKetOQGLjkPEZG/Y6LqYu/e1gPfTu2jdRgOS3l6GTYfKQAAlJRVYGP2uRr7nL1Uip4v/oZhr60FUF1Y4sJl3djWXQ50r62orML3GbmqEmQiIiKt7T9T5ND+08e1x6+PDTIsR4XrRls1jfbvoTZERK7CRNXFxnVpgt7K/GrD28VrHI1jZq/OBqAbz3nbnM3IKyo12X7iwhVcKdd9o3ysoAQtpy3D9xm5CFIqRBhXRLTn4/VH8eQPu/DmbwcxY9FeJqxEROSVzhWXAYDDxZDaJEShTUKUYblrUgzeu70HXpjQ0ZXhERH5LSaqbjT3zp4+XaSorLwKyelLDcunC6sT18GvrgEALMo8Bf29u0oCZRXqukadL9Hd+GevzsZnG3OQeeKi1X23HbuA7zNyHQveTSqrpEMJORGRJwkhPhFC5Akh9hitayiEWCmEOKT830BZL4QQs4UQ2UKIXUKIHtpF7p22HbuAtBd/Q3L6Uuw95VhFfEtjUcd2boKIOqrqWBIRBTwmqm6m7+rjiwpKrqraT/8t877Tl9B2+i9ITl+KrTnnMfrNdRj1xjqLx3y8/qjqOG56fyOe/GGX6v3dqdXTy3DXZ1u1DoOIyJrPAIw2W5cOYJWUMhXAKmUZAMYASFX+TQXwvodi9BlZp5yvGN+rZUMXRkJEFHi8MlEVQowXQnxUWOj7U4p0SozWOgSn3fCuurlH9V1/K4y67y7ddRr7zxThwFnHxvT4gjUH8rUOgfxA7vnL+M+SLHZ7J5eSUq4DcN5s9QQAnyuPPwdwg9H6L6TOJgAxQogmnonU+1VWSczb4nxvHsGJU4mIasUrE1Up5WIp5dToaN9N8vRevKET/j6kldZheJzaLsB6vKFToHnom+2Yu+Gow90JiZyQIKU8rTw+AyBBeZwIwDgTO6Gsq0EIMVUIkSGEyMjPD4wv677ZchxZp/n7SUSkFa9MVP1JeGgwbr6mmdZhuM36Q+fw+Hc7a6wvKTNNVPecLER2nv+1rvqrK1cr8frKg5wb140qOdaZNCB1g+wdfvNJKT+SUqZJKdPi4uLcEJn3uaBy+AsREbkHE1UPaBVXD7tmjESKD8+zasvBs8U11i3KPGV4/MbKg7ju7Q0Y8brl8aoAsDLrjFsS2eKyCpefMxC8+3s2Zq86hHlbjmsdit9inkoedFbfpVf5P09ZfxJAktF+zZR1BKCKv6RERJpiouoh9cNDMd/G/KrtGkdZ3ebr3lp1yO4+7/5+2GYi66wP1hyuse58yVUs3MHPYraUKtMQsUXV/djrnTxgEYA7lMd3APjZaP1kpfpvHwCFRl2EA15tho9vf/Za1wVCRBSgmKh6UHz9cGx5ejiWPjKgxrZZN3XRICLvs+5gPo4XXHbZ+Sx1r7z/q234x7c7ceriFZc9D5Gj2FhD7iCEmAfgTwBthRAnhBD3AJgF4FohxCEAI5RlAFgG4AiAbAAfA3hAg5C91okLzt2LXrmpCxpG1nFxNEREgcd3507xUfH1wxFfP7zG+qAAalXJzLU+Z+rkT7YAAHJmjbO6z8bsc8gvLsOEbtU1P+ZuOIr/LMnC7hkjERUealhvKRk4o8wH68rWwr/O2YzEmLr47838woGItCOlnGRl03AL+0oAD7o3It9z6GwR4qLC8NN253re/CXNf+tSEBF5EhNVL1HfKLnyZwt2nMBj32Y6dezag/loHV8Pt83ZDACY0C0Rc9YfwaiOjfGfJVkAgPyiMpNE1VhBcRle+eUAjp/XfUvuygatDdnnAICJKhGRj7v2jXVo1qCu08ezij0RkWswUdVY9+YxeGp0OyT7aaElc44kqVJKkxbROz7Zgsg6wYblguIyvLh0H77cdMywznxMUUVldavpy8v248ftJ0zOT0REpKcfEnLiAoeGEBFpjYmqxhY80F/rELxSaXkl2j37S431JVerp73RJ6UlRpV9H/pmOxY80B9vrjqIwW3isGRXdV0Q6dI2VOsKr5Tj0pVyJDWM8MjzkW/i1yRE3qffrNVOHxskaleAiYiITDFRJa/U6fkVqvc1bhjdf6YIEz/ehMzci/hw7RHE1nO8oMU7qw/hf78exIEXRyMsJNj+AWbGvrUeJy9esTnOlkiPvQSJ/MPaJ4ci18kCTEREVBOr/nqRO/slax2C16hQ8bX0jEV7AQAFZpOyGxdrKrNRMMnaM8zZcBQAkJFzwTBNiyNOspowqcCu50T+JalhBPq1itU6DCIiv8FE1Ys0Yjl7hyzdbX+6v6LSCqvbzPOEOeuPYFHmKQQpTVy3z9mMf37vXOEnX/TZH0ex/fgFrcMIOAJsUiXS2pz1R5CcvtTp4z+7q6cLoyEiIoCJqleZMihF6xACypoDeSbLLy7dh0fm7TBJG7Yfc03iJqXEUz/swg4vTgRnLM7Cje9tNFnnqXG9RERaenXFAaeP/XhyGoa0jXdhNEREBHhpoiqEGC+E+KiwsFDrUDwqPNTx8ZDkvBeX7sPMZftQXFZhUh3YeMxglZT4LiMXH6w9bJh/1RlFZRX4NiMXk+duqU3IRETkBraGidjDbvxERO7hlcWUpJSLASxOS0ubonUs7vLlPb1wtRY3RnKND9cdgQTwxMi2FrefvVSGf/2wCwDww7YTNbYXXinH6v1nDcu1SWa9kS90S/1263E89eNu7HzuWsREsPs8ERERkT/wyhbVQDAwNQ7D2yc4dWx03VAXRxPYzL8wOFd81eJ+54rLaqx74vtMk7lh+8xc5drg7Mi7VIqnftiFsgrHiz75i8826ubRtTfv4fbjF/D15mM299ECq/4S+Zbsl8bgmbHt0TYhCgCnmiIichcmql5uw1NDseyRgfjUqFBD5vMjNYzId9lKZL7c5FwCc/aS7RbU7zNyccLGdAWVVRJ3f7YVW3POq3o+KSV+yzoLKSVW7D2DXi+vwrcZuViZddb+wX5K3+3OXsJ343sb8cyCPR6ISB32FiTyTSHBQZgyKAXNG3GubCIid2Ki6uWaNYhAh6b1tQ7DL2w5ajkZzC8qw3+WZNk93lIeZCs3Ki2vxJM/7MKtH26qsf5YQQleWLwX+05fwur9eXjom+12nx8AvsvIxb1fZGD+1lzc9+U2VccECl/opmwJW1SJfFNosO6Xl7/CRETu4ZVjVMkCpfVlYKpujrYnR7XFqysOIDW+Hg7lFWsYmO9TM82NJYWXy3H5qv0ut8ZdhovKKvDYtzuxfM8ZAMD24xetHWbRqYu6FtyFO046dJyzrLUGH84vRqu4eh6JQS0mfETkbtPHtTc8/veEToiPCsewdqz4S0TkDmxR9RENlTlW9WNi/j64FRY80A8DUjm5uBZSn1mGrv/+1akvCVbtr54Wp6rKfv/PDYfOITl9KfKMuhlvttI6bE15ZRWy84ocOgYAPv/zGDYcOgcAKCgug5QSizNPYfhra/Gbl3Q31nehZaJKRM7YdUL9F4b3DqyeRi62XhhmXN8RIcH8KEVE5A786+plfnt8EP4+pFWN9V2TYvDNvb3xr9HtAABBQQLdmzcwJK7kfkWlFYbH5ZX2E0x9V+OyiipkGI1BNZ7KoFyZFufspTIkpy9FcvpS3Pt5hsl5PtuYAwDYmWv9w5SUwNMLdmPbMcsJ7L8XZ2HE6+twutB2wSFL9p+5hL2nCnHNi7/h+4wTyDp9CQBw4Kzjia+xYwUlqvarqpJ467dDuHjZcpErV8z1mpy+VHX3a1fhHLVE3uFwPnslERF5IyaqXqZ1fBSeUpJRc/1ax6JOiOmP7NaeSXhkWGukj2mH8V2beiLEgFWhovXT2ORPqudMvfuz6uTTuIjO/jM1k73f9lW3VOZdKjUs23r2iqoqfLP5eI3xsHqbjxYA0HUdtlcAylxOQQlW79O1Av9x+JxDx1rzy54zGPzqGlVFoNYdyscbvx3E9IW6QkgvLsnCzOX7DNv1VZvXHsivVUxLdjnXBby27I2traySnKeRyE32nb6ED9ce0ToMIiKygImqjxNC4PGRbXH/4Fb41yjLc4GSd3Ek4d1+/ILhsZTWu7fq85iKKonLVytqbNc/5U3vb0Tvlx2bQuerTcfx2sqDNZ6rNvaeKgSg+5BoT4XSen1FGQ88Z8NRwwfLS6XlyCnQjaOduXx/7QPzILXXsdXTy3D/V+4tnDX6zXV47Nudbn0OIm805q31Fr8wJCIi7TFRJfIRs1cdMnQVtsUXqwHnnr+M1389ACkl8otqzldrzXkrc976EjVja1fsdXw88JWrlcgrUtd6vv9MERZ4qEAXkbew9KUeERF5DyaqfoS9A/2P8c806/QlzNuSa3G/141aPDcdKbBwHtM3x+4Tha4J0AopbXdXNd9035fbMHt1Nr7fdgI9X/oNP+9UlzT58lvelbHnnr+M5PSlWH+ouvtz++d+Qa+XHGs99zbZecVOjasmUuO2jzdrHQIREdnARNWPOFqcxbjMPnkn859oabnl6XBOXDD9MJ/+4y48PG+H1fOMf2eD1bGqe04W4qyVVk0B4IO1h03Wfbj2MHLPV09jU1ZRiZbTlpkkz/aUVeheV9YpXVdgfVXjPScLUWUj4V21z7nKw+sO5iP9x11OHesKJy+6NvnadkzXRfzFJfvs7FnTe2uynXrO0vJKHC+wPH2Rq4x4fS36zlzt1uegwHP0XAm+/DPHZoE6c3+kD8PeF0a5LygiIqqBiaofat4wAi9c39Hufl2TYjwQDTlDbZdNS8orJeZvzcXizFPVKy3kesVllru9Xff2BtNjrRBCV+xp5vL9uMOocJS+uNHcDUdVxyzM+r9KCWzNOY/r3t5g9TyVVRIvLnU8MQN0ha7mb7XcOu1ue04Wov+s1chWpjbKPX/ZZCyyM/SX78DZIotz3/5vxQEkpy81WTd94W7c+uGfeOWXA0495+Pf7cSgV3+3+uUJkbcaN3s9nv15r+r9G9cPR2JMXUSGcep5IiJPYqLqR5o1iMD1XZvivdt7YHLfFtg0bTjentTd6v5t4jm1jbey1mXT2e7d7ugiK2V1kSZ90nvobBE6z/gVAGy2hOrZGp6pb6W1Nmds2+nLa6xLTl9qmPdVDS2SrCPnTKfluefzDNz43kaL+zpT7ffi5fIa6975vWar6Vebjjs0H+/Bs0WYuXyfISZ9lWU146YBoKSsosb1llJi1vL9FpNrIne5fNWx3/vFDw9wUyRERGSLVyaqQojxQoiPCgvdO47O3wQHCcye1B2dEqMhhEDj6HCrU9b8X/dEREeEejhCqi1n5960lPCoqOFT8xizlk99l1O99UZJooDAV5uOYfORAry3JhuXSmsmUDZGsdZYs2p/nsmyterJf527GSuzzuKQinle2z37C774M8fufs6QUhpal52RV1SKWb/UrGR86uIVvLP6kMnPNMjo53Ld2xtQYqW13BGFV8pNWvZv+3gzPlx7BAUlugJW5u8Fezo+vwKDXvndZN3eU5fwwdrDeOibHVaOItJew8g6WodARBSQvLIfi5RyMYDFaWlpU7SOxV+99H+dtA6BVHjg6+0uOY+llG6HGUSqAAAgAElEQVTj4QI0jg5HRB3n/gxk5xXj1RWm3UbNEyT93KcAcDivBK/d0tXmOfW5j5S1Kw425QvdvLU5s8bZ3fc5B7oAGjtXXIaC4qto27hmz4TfD+Thrk+3AgC2TR+BRvXCHD7/P+bvxMbD1YWx8opKTVraR3VsjNQE3XOb54zGXaLbPFOz5VmNvjNX4fLVSsM11LeQ510qQ6zR63Hkx5RnNvZZ/zNW2ypLpAVnvtQjIqLa88oWVQIeG9EGI9onuO38ziYn5Dm9X/6txrrScsc+0D/+3U4s3HESxywUvZm+cA/Sf9zt0Pk2G1UUPl9Sc2qY12wUULLUyqf/AKgfr/mdMm7UWpKakaO+q6qxPScd751RUlZhtRvxze9vRNqLv2HUm+sAAMcLLmPYa2swf8txAMDc9dXjas9YKVplzamLV7Az92KN62VeuKjSRia/w2jM61ULSaBx8StzpeWV+GTD0RrdI/U/77Gz12Nl1lmTD+/Ld5/GnZ9ugTkpJVbtO2u1C7M+wT6cX4x/L85ClQNzDBMREZF/Y7bipR4dkap1CKSxs5fUzydqzU/bT+Kn7danejmUV4zjBZex/8wlVec7VViddJ0yqlybV1SG1fttV+BddygfK/aewaiOjVFiZf7CEiU5WpF1Bt9m1Cx25MiYSmNbnDjuyR8ysWz3Gaz/11AkNYwwrJdSIsOsy/OgV3VdWtN/2o2JvZqbbAtysItsv1m6KrddmkXb3M849xMOtvkMNOuCa+yd1dkWx7QayzSrlvp3Ky3/327NRfpPu/HfmzrbPF9peRU++eMoisvKcd/gVmgVV8/m/kSeMm1MOwQFsU2ViEgLbFENcDd2T9Q6BNLQiQuXMeKNtZj65TaHjz2ktILq/WgjIQZ0BUzu+3Ib+s9ajU//yAGg6zZqqSXOUkEgZySnL8Wy3adV7/+70TjYg2d1r++zjTkmLYLmjYMHLYyFNR5LfMO7f7i9aJN54ar9ZyyPz7UUq7nCK/avfZWV65F7XteyvOdkIX7eedLwZcbpQsutyuY5/HcZJzD8tbXYduwCZiyq7pLt7BQ6RLV13+BWWodARBSwmKgGuKYxdQEAnRNtt96QfyoqrahVwR9nGM8huuP4BaxRqseq4WDjJABg6e7TuGKUKNpKGu/6bCvWH9LFo09O5244ikWZp5B16hLWHcyvMSZz5BvrbD5/WUUVJn60yeG4d50w7a48e7VpsmacIL5rpwVUz16slSq73krA0G/7pNEcvgNf+R1H8ktw3dsb8Oj8nbhU6lxRp5ve34jPNuYYlp2dQoeIiIh8FxPVAPfI8FS8clMXLHqov9ahkI9bukt9y6Xe7w4kqQBQ6uC0EgAACZOiT+2e/cXm7n+buwV/GhUxAoBH5+/E2NnrMfmTLThgpbXSlp25F3G68Ird/c4VW+/uve6g6bUybrU9kl9ivrtVtqohz1ymbl5a4yR57Oz1VvfTt7wu331GXXBEXmD5owO1DoGIiMBENeDVCQnCLT2TIITAkocHoE2CbmzYI8Ntj5GdZDYOj8jcFTd0dzVvVXSXA2cuWe1+bCsxA3TdqS3VDuo7czWS05fanBv13s8zVMeY/uNuXFbG+gY58Jf8WhutqosyT2FllulY49LySvy807Rb9wdrD6NIRWupvgDTAbPkWEoJKaXDY2uJaiu/yP7Y//ZN6nsgEiIisofFlAJIcJCw2bWvU2I0xnRqgoNnD6k6X86scXhmwW58vfm4q0Ik8gozFmc5feyA//6Ofq0aWd2+2mw+WGPG3aLt2X2yEN9sPo7Q4CCHq0FbYz59DGC/BdqWJRZa2X/ZcxrP/bwXeUVldossEbnKgTNFWH8o32TqJiIi8m5MVANARJ1gXL5aiUMvjkHq9OVIiY20uq+jYwCDWQ2RvNxSB4opucpGs67Dxi5ZKVZ0JL8Yjv46SbNuzb7g/q+qKwQ/5eD0SETO0k8lRUREvoOJagD4+cH+WHMgH0FBAodeHOOSc+oT2n9e2xa/7j3r8FyRRIEqJNhyP91hr611+FwvqRxTSkSOmTG+AxbuPKV1GEREAY1jVANAakIUpgxKAQAEBQmXzgkXHRGKmTey+x6RWiHshUDktZorcybf2b8lFj7IIoNERFpiokq1NqhNHO5TEmEiss3dc6oSkfO+vre31iEQEZGCiSqZuKlHM8TWq4O/XNPM6j7D28XjiZFtDcvBQQLTxra3uG9MRKjLYyTyZY5OyUNE7tWika4VdWBqLJKUFlUiItIeE1UykdQwAhnTr0VSwwisfGwQkhrWrbHPKzd3QcPIOnbPdWe/ZOx8bqQ7wiQiIlLlq03HbG5//ZauAIDEmJr3OyIi0g6LKZFVqQlR6JIYg9zz6qfMAIBvp/bBT9tP4vGRbdwUGRERkTrTF+6xuf2aFg3x9qTuGNE+wUMRERGRGkxUySaJmvOuWp+JVadlbCT+e3OXWj2vvTlfiYiI7JFS3X1kfNembo6EiIgcxa6/5JDBbeLQyE633/p17Y9LnT7O8phWvT+nDXMoLiIiInO2vu9sEBHKQoBERF7MK1tUhRDjAYxv3bq11qEEvAGt47Bs9xnD8jPj2kMIy9NrfHlPL8RFhSE8NNjueds1ru+yGImIiMxJKfHarwesbt/BGgpERF7NK1tUpZSLpZRTo6OjtQ4l4E3qlYSM6SMMc6U2jg63uu/A1DhVCeg39/bGgNRYDEyNdVmcRERExuZuOIr31hy2uO256zp4OBoiInKUVyaq5D2EEIitF4ZJvZojZ9Y41A93fLqZ/f8ZjV0zqr+57tdal6B++Ldr0K5xFADdXKy19c5t3Wt9DiIi8l2Hzhbho3W65PTFpfus7ndX/2QPRURERM5iokpuFx4abDHBjagTgpt66OZrbRNfT/X51jwxxOL6LokxTsVHRBQIhBA5QojdQoidQogMZV1DIcRKIcQh5f8GWsdZGze8+wdeXrYf5ZVVNvezNoSFiIi8BxNV0pS+qnCNzwxGBTA+v7uX4fGmacORHBtpWI4K98ph1kRE3mqolLKblDJNWU4HsEpKmQpglbLssy6XVwIAHp2/Q+NIiIiotvgpnzSlnznA+Nvtewa0NDyOiwrD4DZxyJk1zuLxu2eMwncZuRjWLh6Xyyot7tMkOhynC0tdFzQRkf+YAGCI8vhzAGsAPKVVMK5iXATQ3KReSR6MhIiInMUWVfKYT+/siSUPDzBZp586oDadsG5JS0JsvTBUGc2XZzz+6O7+LWscM7IDJ3YnooAjAfwqhNgmhJiqrEuQUp5WHp8BYPGPoxBiqhAiQwiRkZ+f74lYnaJm2tROiSzUSETkC5iokscMbRdf4wOCvuuveaYaGaZr7L+uSxPV59cnqsmNItDZ6Hkiw0Kw+KEBeOPWroZ108ex4iMRBZwBUsoeAMYAeFAIMch4o5RSwmTghcm2j6SUaVLKtLi42he/05K9ucCJiMg7sOsvaUr/7XeQUddfKXXJZeZzI1HPgTGo+k9X5kUyJCQ6N4tG52bReOzbTGWfWoVNRORzpJQnlf/zhBALAPQCcFYI0URKeVoI0QRAnqZBesCojo21DoGIiFRgiyppSiqZqqW8MToiFMFB6jPK6vGuput7JTc0PDYe/0pEFCiEEJFCiCj9YwAjAewBsAjAHcpudwD4WZsIa+fK1Ur0euk3u/u9cWtXVvwlIvIRTFRJU5ZaVJNjI5w9m+FcYzs3wS1pzbD1mRFITYgy7PHsdR2QM2ucqnFM5jY8NdTJuIiINJcAYIMQIhPAFgBLpZS/AJgF4FohxCEAI5Rln/Pysn3IKyqzu9//dW/mgWiIiMgV2PWXNNWjhW7Kvp4tG6LH4RhsP34Rf+vTwqlzGRdmCg8Nxis3d7W5v1pdm0VjxvUd0axBBDKfG4mu//7VJeclIvIUKeURADX+KEopCwAM93xErrVgx0mb258f3wG9Wja0uQ8REXkXJqqkqf6tY5H5/EhE1w3F4Da1K9Bhretvbf38UHWl4uiIUJNtPZMbYGvOBdc+oRtE1glGyVXL0/cQEfmyotJyFJdV2NznLgvV34mIyLux6y9pLrpuqP2dVJBGXX/V7muPrdj6pDTE9/f3Uxecxpzo6UxE5BM6z7Ddy2VMJxZPIiLyRUxUyW9UVdXu+Ml9a3Y5rhdWs9PBvCl9cGe/ZHz41zTV524SHW6yHBYShP/c0MnxIJ1U5cygXCIiP/DPkW20DoGIiJzARJX8hj4ZU9Oiqm8pbde4utCSpeNaxdersa5vq0aYcX3HGt2AHfHQ0NZOj8V1hpRAW6OiUkREvkxKiasVVbh4+arN/XJmjUPreP7tIyLyRUxUySe9d3sPfHNvb5N1LRrpqgU/MLSV3eNjIuogY/oILH1koMXt6/81FPFRYfjob9fYPdcdfVvg/dt72Nz+xd29sPbJIYZ1ato3P72zJ7olxajY0z4pgdt6N3fJuYiItPbysn1oM305uv17pdahEBGRmzBRJZ80tnMT9Gsda7IuKjwUObPG4bouTVWdI7ZemMk8rcYNqkkNI7DlmREIDw22e54XJnTCmM5NbG5PNWvNNO8KDAAjOySYLA9pG4drzdaZJ+dqVUmJv/VpgY8n2++uPLFnklPPQUTkCQt3nMTH649qHQYREbkZE1UiDdx8jelcfkIAHxklkR/89RqLk9J3cbKFVQIIChK4tkMCnhzV1ua+nhw7S0TkqH98u1PrEIiIyAOYqFLAa1xf17qpZmyrLQ2MxqzOm9KnxvbmDSMMj82T0GCz5dFKlUrj1QNTTVuQAWDDU0Ox4AH7lYelUTEl81Zac6HBzv9ZsFR8qk8K5y4kIteQKgvDbXlmOFb/c7CboyEiIndiokoBb8U/BmHNE0PwyPBUAMCLTrYoLn90kOGxpelvhBDImTUOObPGGdZ9emdPDG4Th4UP9rd7/unjOpgsr31yCJo1iEDXZvZbWauMwmmjdEO2VFxpycMDaqzTi48Ks/s8xsWp9Aa3icePf/eNaXxq67v7+modApHfklLiy03HVO0bHxWOlLiaxfCIiMh31Gz+IAow0RGhhgq+xkmkoxpbGHdqz9B28RjaLt6wvOCBfoioU/1rKVDdpBocBIQG65bHdWmCFo0iAei69Dpq4YP90aJhBLr/R1eI5Jd/DERllUTHptEAgJ8f7I8J7/5h2P/7+/uiZ3JD3D5nE/7ILrB63nZNopBx7EKN9de0aOBwjL5I//PxNXFRYcgvKtM6DCKb5m3JxXM/79U6DCIi8hCvTFSFEOMBjG/durXWoRA5xTjBdET35qYJnXnLbFhIMDY8NRRxKlo3jd3ZL9lk2byacLvG9U2WuxptN07er2nR0Gaiqu9G7W1u690c32w+7rbzd02KwYh28S6r0uxpW58ZgUfm7cCizFNah0Jk1a9ZZ7QOgYiIPMgru/5KKRdLKadGR0drHQqRQ3q31I3HtNT111WaNYhAWIj9asR608e1x/PjO9jfUYX7BqXY3H7/4Oqpgbo2U//727Fpffs7KZxp9XZ3O2dcvTA8PDzVYgEsX+HNod+S1sz+TuT3rlyttLn9lZu74Pv7++LTO3t6KCIiInInr0xUiXyV8Yf9yX1b1P58RilWg4g6Dh//8eQ03DswxWUJVGRYCOoqU/bUNZu6Z+GD/RGiFGIKDhLo06qRzXPVD6/u0GFtPltXsVUoK2fWOAxqE2fz+KZKt+4BrWMxZ3IaHhhiOlevmsv7zm3d8dvjLO7ijOfGd9Q6BNLYtmMXsPnoeavbP72zJ25JS0LP5IYmwymIiMh3MVElcpN/T+hUqzGvxsZ0aoxG9Wx3960TEoRlZgmfvQq/ztC3Fr81sZvJ+hZKVeMnRrbB4ocGIDZSF6++GvK4LtbnmrUnPDQIN/ZIxHA7H0DHdGqMry3MNWstkbyrfzIA4OPJ19hshdEn3c+N74ARHRLwr9HtTLa3b1LdImypOjMAXNelKVrHqy/uklDfse7dteXOBtUbeyQ6fWzOrHEWq0lTYLnp/Y02t7exUMiNiIh8GxNVIhe6o28ygOrKurVVT2l1tNc1dsNTQ7F52nB0aFofG9OHueS5rdHPDmHeClknRPfn5KFhqejQtD7u6p+M//2lK25JSwIA/H2waSvkXAe65wULgddv6Wb3mMSYuujfumaiaJ6E6bto39RD16U0LCQYMUbTC5mbMjAFG54aavHnuuCBfnhUqRhtzNb5HNXEqFCXK758eHR4Ku4bbLsbtytFqUg01z45xP2BkE+yNyVNSlwkEmPqeigaIiLyFCaqRC40pnMT5Mwah1g7rZ9qTeqZhOeu64Cpg1rZ3K9Zgwg0iNR1DW7qwAe2tglR+Oe1bRyKqU+KrnUx2KjacGy9Oog0S0ZCgoNw8zXNDFWJw0Kq/9x0bFofPZNN51f99bFBGNe5CX59bBDMqZ3jtpXSYtk5MRpdjMbImnd9fv+v1+Ctid3QKbF6n9SEKNQLC8Hnd/cyrGuTUE85XneNLenevIHJtdAz/mz9zm3dVcWvN6RtHD78W5rFbR/97RqHzgUA3ZtXF3la+GB/PHZtG9yqfIHgKc+MbW+ynBIXabLMRIOsaTltmc3tUwd67ksXIiLyHPanIvJiIcFBuHtAS7edf4WFpNCe9//aAycvXEFocHXiqaZLa2pCFCb3bYEv/rQ8D2KbhCi8e3sPi9vM89RnxrbHgNRYjHlrPQBg89PDkXXqEoa01bXyLlbmg01OX2rxfJFhwZjQzbQ7ar2wEOx5YZTl53egY2xqfBTWHzqHBhGhKLxSjrcmdsN1XZqqPn7z08ORYKN6strxxsPaxeOOfsmQUqK4rAIPfbMDQHXF5yon6n1tmjYcfWausrjtvzd1RqfEaIybvcHi9imDUvDSsn2GGBY+2N/qz0dvTKfGjgdJfsXee+S+QSm46RoW2yIi8kdsUSUii25NS7KYKETUCUGqk12b/3KNrhVPbW0n/TjNwW1Nx6ZOGZRiMi40oX44hraLr5HE6cc2DmsXjzSjuVzVttDa6XFoUfqYdvhmSm+T1lpjb0/qjk/vMu3CvP5fQ+2e09ind/XE//7S1eYxn9zZE4PbxGFI23iLrzclNhIjOyTgzVt1Y43VJMAhNuaJvbVnc8M8vOb059Y3PFsaD2z+/O0aR2H2JMdaoimwbH56OKaNbW/ypRkREfkPtqgSkUX/vbmL3X1S4+vhUF6x6oQuPFT3gTI+yvZ8q6v/ORhBQiA5NhLHCkrQONry/m9P6m7zQ+pbE7thyhcZ6N48Bl/e0xunC6+gtLzK7gfb+uEhuFRaYefVWFYnJAj9WsXia2XeVvMEbHzXmq2rSQ0jMKpjAlbsPYtwC1MPTeiWiEfn7zQsD1US9ye+z7QYw9NjTRNbSz+foCCBjyZXdy8e37UJFuw4aeVVqZfcKAI5BZcxpG0c1hzIN9lWNzQYJVcrDWOvjVtozVPgqPAQJiABzt7YVFs9D4iIyPcxUSUip71wfUfcNmez6hbS1IQo/O8vXXFte9sFgVLiqrsSt2gUaXU/S0mfseHtE3BkZnXlZePz2vLrY4NxrKAE0xfuUbW/LdYuTWiwQHll9QfxN2/tjsP5xYg2KsLUr1Uj9G6pGxN8Z79knCsuU/Wc5uOF1czrO6xdAnJmjbPZ1VLNFxKPj2yLR+btMCkApR8ju+DB/liZddaQgBp/AeHN87iS5w18ZTVyz1+xut24RwUREfknJqpE5LQuSTFoGh2OJ0a2VX3MzUbjyWbd2NlqF1ktNY4Or3USNbB1LJbuOm11/O5vjw/G4FfXGJbr1gmucS2+mdLH8HjG9ernEtUnt1q4vmtTXN+1KWYs2gsAmDKwpWE8cJuEKJsVsRc80A87cy/ihcVZHomVvNNrvx6wmaQCwPJH3Tv3MhERaY+JKhE5rV5YCDZOG+708RN7NXdhNK7nxBBVg1t7JuHaDglW57+11VLsrPQx7TBlYIrFKsS1MfPGzth0pACNlMrSxr6d2gfHzl+2emyTaHXVfIUQ6N68geGaR9Th7SlQvb062+q2R4an4qZazMtLRES+g58EiPzQiPbxThc8opqcSfuEEFaTVHe5f7DlaYz0XXZHd3Suim63pBhMsvKlQu+URuid4roW3O5JMXhyVFvc2tOz0+eQd9iYfc7qtr4pjfC4g9NpERGR72KiSuSH5txRs6oquVZIkEBMRM0WRk96ZFhr7Dl1Cf8c2Qbbj1+0up++lTLYRtVetRY+2B9FpeX429wtqva3121aCNOxr0IIPDi0dS0iJF90uvAKMnIu4OF5O6zu8/W9vT0YERERaY2JKhGRE/b9Z7TWIeBxo7HB1qaGAaqrp7qiQ7B+HlZ77FVs1Vv68ED8fiCvNiGRH+g7c7XN7fcNSkGQi7u0ExGRd2OiSkRkha1kyxenTlEzVyoAhIUEoayiyui4Wjynne0dmtZHh6as4BrI1HypMW1sew9EQkRE3oSJKhGRHe6aOuXBoa1w1SghdJeBqXGIiwrDfYNS7O57dOZYrN6fh3s+zzCsqxtac25Xe6YMSsH24xdxfTcWviHbjpwrsbn9rYndPBQJERF5EyaqRERWtGgUicP5JQgLcTxRU+PJUe3ccl5zDSPrYOszI1TtK4QwJOY9kxtg6qBWTlUobtYgAosfHuDwcRR4hr+21uq2zOdGmswtTEREgYOJKhGRFW9O7IY/DxcgqWGE1qFool5YCK7tkKB1GBSg3r+9B5NUIqIA5nuDrIiIPKR+eChGOTmlCxE5r3H9cIzp3ETrMIiISENMVImIiMijLpRcxd5ThSivrDlGu1VcJNY8OcTzQRERkVdhokpEREQe9fSC3Rg3ewM2ZJ+rsW3FPwYh3IkCXkRE5F+YqBIREZFHLd9zBgDwzursGtuCOV8qERGBxZSIiMhM84a6Kr8DUuM0joT83eH8YgDA0LZxeHpse6w9mK96vl8iIvJvTFSJiMhE6/h62PLMcMTVC9M6FPJzFy+XAwC6NItBakIUUhOiNI6IiIi8BRNVIiKqIT4qXOsQKIDc0S9Z6xCIiMjLeOUYVSHEeCHER4WFhVqHQkRERG4WGcbiSUREZMorE1Up5WIp5dTo6GitQyEiIiIXCw02HYcaFsJElYiITHllokpERET+q7xSah0CERF5OSaqRERERERE5FWYqBIREREREZFXYaJKREREHpUYU9fweM8LozSMhIiIvBWnpyEiIoc9Mqw12jepr3UY5KP+SB+GGYv2olVcJOqF8aMIERHVxLsDERE57PGRbbUOgVxECDEawFsAggHMkVLO8sTzzri+oyeehoiIfBS7/hIREQUoIUQwgHcBjAHQAcAkIUQHbaMiIiJiokpERBTIegHIllIekVJeBTAfwASNYyIiImKiSkREFMASAeQaLZ9Q1pkQQkwVQmQIITLy8/M9FhwREQUuJqpERERkk5TyIyllmpQyLS4uTutwiIgoADBRJSIiClwnASQZLTdT1hEREWmKiSoREVHg2gogVQjRUghRB8BEAIs0jomIiIjT0xAREQUqKWWFEOIhACugm57mEynlXo3DIiIiYqJKREQUyKSUywAs0zoOIiIiY+z6S0RERERERF6FiSoRERERERF5FSaqRERERERE5FWYqBIREREREZFXYaJKREREREREXoWJKhEREREREXkVJqpERERERETkVZioEhERERERkVdhokpERERERERehYkqEREREREReRUhpdQ6BquEEPkAjrngVLEAzrngPP6O10kdXif7eI3U4XVSx5XXqYWUMs5F5wpIvDd7HK+TOrxO9vEaqcPrpI6rrpPV+7JXJ6quIoTIkFKmaR2Ht+N1UofXyT5eI3V4ndThdfJP/Lmqw+ukDq+TfbxG6vA6qeOJ68Suv0RERERERORVmKgSERERERGRVwmURPUjrQPwEbxO6vA62cdrpA6vkzq8Tv6JP1d1eJ3U4XWyj9dIHV4nddx+nQJijCoRERERERH5jkBpUSUiIiIiIiIf4feJqhBitBDigBAiWwiRrnU8niSESBJC/C6EyBJC7BVCPKqsbyiEWCmEOKT830BZL4QQs5VrtUsI0cPoXHco+x8SQtyh1WtyJyFEsBBihxBiibLcUgixWbke3woh6ijrw5TlbGV7stE5pinrDwghRmnzStxHCBEjhPhBCLFfCLFPCNGX7ydTQojHlN+3PUKIeUKIcL6XdIQQnwgh8oQQe4zWuez9I4S4RgixWzlmthBCePYVklq8N/PerBbvzfbx3mwf782Wef19WUrpt/8ABAM4DCAFQB0AmQA6aB2XB19/EwA9lMdRAA4C6ADgFQDpyvp0AP9VHo8FsByAANAHwGZlfUMAR5T/GyiPG2j9+txwvR4H8A2AJcrydwAmKo8/APB35fEDAD5QHk8E8K3yuIPyHgsD0FJ57wVr/bpcfI0+B3Cv8rgOgBi+n0yuTyKAowDqGr2H7uR7yXB9BgHoAWCP0TqXvX8AbFH2FcqxY7R+zfxn8X3AezPvzY5cL96b7V8j3pttXx/em61fG6++L/t7i2ovANlSyiNSyqsA5gOYoHFMHiOlPC2l3K48LgKwD7pf1gnQ/VGD8v8NyuMJAL6QOpsAxAghmgAYBWCllPK8lPICgJUARnvwpbidEKIZgHEA5ijLAsAwAD8ou5hfJ/31+wHAcGX/CQDmSynLpJRHAWRD9x70C0KIaOj+oM0FACnlVSnlRfD9ZC4EQF0hRAiACACnwfcSAEBKuQ7AebPVLnn/KNvqSyk3Sd3d8Qujc5F34b2Z92ZVeG+2j/dm1XhvtsDb78v+nqgmAsg1Wj6hrAs4SreF7gA2A0iQUp5WNp0BkKA8tna9AuE6vgngXwCqlOVGAC5KKSuUZePXbLgeyvZCZX9/v04tAeQD+FTphjVHCBEJvp8MpJQnAfwPwHHoboKFALaB7yVbXPX+SVQem68n7xNI72+beG+2i/dm+1TZGHkAAALQSURBVHhvtoP3Zod5zX3Z3xNVAiCEqAfgRwD/kFJeMt6mfMMR0KWfhRDXAciTUm7TOhYvFwJd95D3pZTdAZRA1yXEINDfT8o4jgnQfXBoCiAS/vWNtFsF+vuHAgvvzbbx3qwa78128N7sPK3fO/6eqJ4EkGS03ExZFzCEEKHQ3Qi/llL+pKw+qzTHQ/k/T1lv7Xr5+3XsD+B6IUQOdF3QhgF4C7ouDSHKPsav2XA9lO3RAArg/9fpBIATUsrNyvIP0N0c+X6qNgLAUSllvpSyHMBP0L2/+F6yzlXvn5PKY/P15H0C6f1tEe/NqvDerA7vzfbx3uwYr7kv+3uiuhVAqlLVqw50A6IXaRyTxyj96ecC2CelfN1o0yIA+opcdwD42Wj9ZKWqVx8AhUrT/woAI4UQDZRvpUYq6/yClHKalLKZlDIZuvfIainl7QB+B3Czspv5ddJfv5uV/aWyfqJSLa4lgFToBpH7BSnlGQC5Qoi2yqrhALLA95Ox4wD6CCEilN8//TXie8k6l7x/lG2XhBB9lGs/2ehc5F14b+a92S7em9XhvVkV3psd4z33ZekFFafc+Q+6ClUHoavM9YzW8Xj4tQ+Arrl+F4Cdyr+x0PWzXwXgEIDfADRU9hcA3lWu1W4AaUbnuhu6QePZAO7S+rW58ZoNQXVlwRTo/gBlA/geQJiyPlxZzla2pxgd/4xy/Q7ADyuOAugGIEN5Ty2Errob30+m1+gFAPsB7AHwJXTVAfle0r2medCNDyqHrhXgHle+fwCkKdf9MIB3AAitXzP/WX0v8N7Me7Mj14z3ZtvXh/dm+9eI92bL18Wr78tCOQkRERERERGRV/D3rr9ERERERETkY5ioEhERERERkVdhokpERERERERehYkqEREREREReRUmqkRERERERORVmKgSERERERGRV2GiSkRERERERF6FiSoRERERERF5lf8H/m9AdEiBMcoAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "f = plt.figure(figsize=(16,8))\n", "ax = f.add_subplot(1,2,1)\n", "ax.plot(train_recon_errors)\n", "ax.set_yscale('log')\n", "ax.set_title('NMSE.')\n", "\n", "ax = f.add_subplot(1,2,2)\n", "ax.plot(train_perplexities)\n", "ax.set_title('Average codebook usage (perplexity).')\n" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "Lyj1CCKptCZz" }, "source": [ "# View reconstructions" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 533 }, "colab_type": "code", "id": "rM9zj7ZiPZBG", "outputId": "8ba98679-0ccc-4ba2-a1b4-907a51f607ac" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "WARNING:matplotlib.image:Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n", "WARNING:matplotlib.image:Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n" ] }, { "data": { "text/plain": [ "(-0.5, 255.5, 127.5, -0.5)" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4AAAAHRCAYAAAA7edcsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOy9ebRn11Xf+dl3+o1vfq9eDSqpSrMsW7I821HAMSEx2EAGho4JISSEkKS7k27A6ZBOQhLSAVaT9EoHkqysBSEkIdANGIztjmM8AcZgPEiyrLFKNVe9+b3f/LvT6T/2vuf3JFRlGTwspPtdS6t+er/7u/fcM+1z9v7u7xHnHDVq1KhRo0aNGjVq1KhR48WP4CtdgBo1atSoUaNGjRo1atSo8eVBvQGsUaNGjRo1atSoUaNGjZcI6g1gjRo1atSoUaNGjRo1arxEUG8Aa9SoUaNGjRo1atSoUeMlgnoDWKNGjRo1atSoUaNGjRovEdQbwBo1atSoUaNGjRo1atR4iaDeANb4skJE/q2I/IMv9rV/WIjIORH5k1+OZ32h+ErUmYicEhEnItEf9l41atSoUeOLh9qO1nguROTbReT9X+ly1PijA6nPAazxQiEi54Dvds594Ctdli82vpB3ExEH3OGce/pLXrCvEETkFPAMEDvn8q9saWrUqFHjxYHajvprX9R29Ev5frV9rvHFQB0BrPFFQx0t+uJDRMKvdBlq1KhRo8aXB7Ud/cLxR7HO/iiWucaLC/UGsMYLgoj8LHAz8G4RGYjIOw/RBP+qiFwAPmjX/j8ick1EDkTkoyJy76H7/AcR+WH7/GYRuSQi3ycimyJyVUS+6w947YqIvFtEeiLyCRH5YRH5zRu8z3eIyHkR2RGRv/+c714nIr8tIvv2nH8tIol991G77CGrh28TkSUR+TUR2RKRPft80w2efY+IfNju/6iIfONz3vnfiMh7RWQI/InD9WDXvNPKdUVEvtva4PY/QJ29TUQ+bXV2UUR+6AZl/ssiclZE+iLyjIh8+/WurVGjRo0avx+1Hf2i2tFzIvJ3ReRhYCgikYi8QUQ+Zs98SETefOj6ZRH5abObeyLyrkPf/TUReVpEdkXkV0Xk+KHvnIh8r4g8Zff9CRER++52EfmItdG2iPz8Dd6vqvu/KyLXgJ82u/qs+n2OPW+JyI9bHR+IyG+KSAuo7r9v93/jc+8lIm+yNjywf9906LsPi8g/FZHfMpv+fhFZte+aIvKfrE337bfr12uHGn90UW8Aa7wgOOe+A7gAfINzruuc+7FDX381cA/wp+3/3wfcARwBPgX85xvc+iiwAJwA/irwEyKy9Ae49ieAoV3znfbf80JEXgb8G+A7gOPACnDY0BTA/wKsAm8Evgb4m1YPX2XX3G/18PPoOPpp4BbUuI+Bf32dZ8fAu4H3o/XzPwH/WUTuOnTZO4B/BswBzzUObwX+V+BPArcDb77eexpuVGdD4C8Bi8DbgL8hIn/mecrcAf4V8HXOuTngTcBnPs9za9SoUaPGIdR29ItjRw/hL6C2axFYB94D/DCwDHw/8IsismbX/izQBu5F6/Rf2nu8BfjnwLcCx4DzwH99znPeDrwWuM+uq9ron6K2fMne/f++wfuB1uuyveP3fJ53A/g/gVejNncZeCdQAtX9F+3+v334RyKybHXxr9B2+RfAe0Rk5dBl7wC+y+oiQesLtM0XgJP22+9F26LGiwz1BrDGFwM/5JwbOufGAM65n3LO9Z1zU+CHgPtFZOE6v82Af+Kcy5xz7wUGwF1fyLWiNMk/D/wj59zIOfc54GduUN5vBn7NOfdRK+M/QCdVrPyfdM593DmXO+fOAf8ONc7PC+fcjnPuF+3ZfXTzdr3r3wB0gR9xzqXOuQ8Cv4Yasgq/4pz7Ledc6ZybPOf33wr8tHPuUefcCK3fG+G69euc+7Bz7hF7zsPAz92g3CXwchFpOeeuOuce/TzPrVGjRo0aLxy1HX3hdrTCv3LOXbQ6+4vAe51z7zWb9t+B3wO+XkSOAV8HfK9zbs/e/SN2j28Hfso59yl7j78HvFE0z67Cjzjn9p1zF4APAa88VJe3AMedcxPn3HWjpYYSrd9p1c7Xg4gEwF8B/rZz7rJzrnDOfczK+PnwNuAp59zPWv3/HPA48A2Hrvlp59yTVo5feM47rQC32zM/6ZzrvYBn1vgjhnoDWOOLgYvVBxEJReRHROSMiPSAc/bV6nV+u/OcJOYRukH6Qq5dA6LD5XjO5+fi+OHvnXNDYOfQO9xp9JNr9g7/xw3Kj4i0ReTfGU2jh9IzFuX58/eOAxedc+Whv51HvbFfcNk/z7Vwg/oVkdeLyIeMcnOAevp+33ta/XybfX9VRN4jInd/nufWqFGjRo0XjtqOvnA7+nzluwX4FqMt7ovIPvAgGtU7Cew65/au8x7nD73HwN7jsE2+dujz4bp9JyDA74qmc/yVG5QVYOt5nLrXwyrQBM68wOsP41nvZHjuOuN67/SzwH8D/qvRZX/MmEs1XmSoN4A1vhBcTzL28N/fAXwTSlFcAE7Z3+VLVyy2gJxn009O3uD6q4e/F5E26vGq8G9Qb9kdzrl54Ae5cfm/D/W2vt6ur+gZz/ebK8BJ8+5VuBm4fOj/byTNe5UX/p6fD/8F+FXgpHNuAfi3XOc9nXP/zTn3tagxfRz493+I59aoUaPGSxW1HX1+fCF2tMLhOrsI/KxzbvHQfx3n3I/Yd8sisvg897iCbh6r9+jYe1x+nmuf/XDnrjnn/ppz7jjw14GfrPL3XkB5Qem27UPPPnrou21gAtz2Au7zXDzrnQzPXWc8fwE1OvqPnXMvQ6mnb0dTRWq8yFBvAGt8IdgAbv0818wBU9SD1ka9fl9SOOcK4JeAHzIv4t3ceML6f4G3i8iDoknp/4Rnj4U5oAcM7F5/4zm/f249zKEc+X3j3v+jGzz7d1Bv2ztFJBZNUv8Gfn/OwfXwC8B3iQrJtFHazR8Uc6hXdCIir0MXHb8PIrIuIt9khnGKUobK57u2Ro0aNWrcELUdVfxh7Ojz4T8B3yAif9oiqE1R4ZWbnHNX0ZzKnxQVm4lFpNpg/hxqU18pIg20rn/HaKs3hIh8i8yEavbQjVllG19IOz8E3GvPbnIopcNYQj8F/AsROW7v9EYr45Y953r3fy9wp4i8Q1Qc59uAl6HpJp/vnf6EiLzCIq89lBJa2/sXIeoNYI0vBP8c+N+NXvH917nmP6JUg8vA54CPf5nK9j+intJrKIXh51AD+vtg+Wt/C42AXUUn7kuHLvl+dDPURyNdP/+cW/wQ8DNWD98K/F9AC/XYfRz4/65XSOdcim74vs6u/0ngLznnHn8hL+mcex+a2P0h4Glm9ftC8gKei78J/BMR6QP/EN1cPh8CVHjmCrCL5mU815jXqFGjRo3Pj9qOKn6IP6AdvU55LqJR0x9EN0gXgR9gts79DnQz8ziwCfwd+90HUEfqL9p73Ab8Dy/wsa8FfkdEBiib5m87585e5/2er8xPohvnDwBP8RzRN7QOHwE+gdreHwUCy///Z8Bv2f3f8Jz77qCRu+9DnQjvBN7unNt+Ae90FN3c94DHgI+gfaHGiwz1QfA1XpQQkR8Fjjrnrqti9mKAiNwDfBZouPpA2Bo1atSo8UXCS8WO1qjxUkQdAazxooCI3C0i94nidai89S9/pcv1pYCI/FkRaYhKd/8o8O5681ejRo0aNf4weCnZ0Ro1XuqoN4A1XiyYQ/MXhijV5MeBX/mKluhLh7+OUljOoGct1XTMGjVq1Kjxh8VLyY7WqPGSRk0BrVGjRo0aNWrUqFGjRo2XCOoIYI0aNWrUqFGjRo0aNWq8RFBvAGvUqFGjRo0aNWrUqFHjJYLoRl/uj/WwSSkP7RQDCA5vG+2IzrLQ/wAO+nt87pFPAbC1tQHAdDoiivRxCwtrFE6PFWl3O5w8pceoZJMhZ556DIBmu+uv37hyBYA8L8kyfUinO8/Cop7puX9wwN7eDgCtdshg2Adgc/s8//7Hf0aLbYXO85yK9hoEAaH9XRCCQF8miiJ/ymZpn+JGQhyEAIQIjSgGoNvpQKnXZNOMsiz9PcJQry+dllmCkiDU55VlSSD6vDAM/bXhocoVmZ1/+qdu2tPyO8gK/ftgVFBGeoboExd3ePTyvtZ/CkmgZVpohLzi1lUA7ru5oXUXTHCBlj8sHWWuZZZQiOPMHg79sZalNyxpNJta7qgFwM7BFLFKWuo2aNoxMZFkaPoAhBRELraKDLnyMj12J46a9t4Jg+HI6iNneHVTy1c4kjkta9xNuGNNz0btrui/n7pwkccvq9LywtIcgRX5YHePRlPLFwTCZKL37jQS7rvvfi3rktbFmafPce+9LwPgmXNPcvPNetbqieO38IFffx8AaT5mkunNnUyYTFQN+6aTbwXgs7/xPjot7aP72Ryvs2fML7QY2LXjacY407q5ePEyn/jEJ7WwpqydDwY4GwurR9b4qj+p9+62YnY2HgJgd29Kb2DlYJdb73kQgLe+9S8C8Nu/+ats7eoY+bq3fxe3nNKjgZqNJnGSaDnyKUmsdRq4kH/4vX8LgFGu5WgnDV71hlfr53aTSW8AwJGTJ4gTsd+lDCf6908/rON7OB0QN7XvpumU8WSiZd7ZYX9P+8HBwQCc9qUo6XDTnQ/ouxSqWzPZ3aToa9/dvHyJ1Oo8bjW5+dQprafRFlsXtwB42ze9jSPHjwDw5u/6e1q2MIAotM8hVGNHBLFJyjk3G9dFgdh8hc0HpfMfcc5R2Lgt3ewIJEdA4cT+Ljgb+1GYAtCKSiTX360srNAO9fvldkSQjwEo0pSh9Y8sL/hbP/j1+tu5rr5rWZJlWneT0ZBWqwNAGDQQG3RzCzGx3bsV6u9Go5SRaD1KkBCIjrMsc+Tos7N0gtj8EUuCoOOz3dJ7uUmTcqpjqLvQpDW3BMDZs0+RNLQNA3GEhc15ZUrm9N1/6Wce+VIeUl3jEK7u9BxAgCA2l8eR4Mz+IAXpVMdiGIArte2ubm7z6Yd1Djp/URX7F+cXWF7W87tbrSZhQ+cMcUIY6P0mkxHDYU/vXZYE9vci0zFcuNSP28AVRGJ2NcTblyzLSEK9d5g0+YG/83ft3noPkQAbWgRBQODttFAtNERCxGxkZDZTgghsrIYCSWjjPS8YW5lSCkK7RysOiRs2plp6jzgUSqn6t/jFjAQBcWLPAX+PIBAk1Ln/Lfdo3XXcxK8L+v0BT11Sm31+d8hgrO/SG08pSr3fUlM4cWQZgJuPq10KCWnHeu18u01s4/2gNyBC7Vm3GdFo6LO3D8Zc3LV5dqLvOiWnbfb6+ELMckfn/VaYU+Y6VsM0o2FzZNlo8/jRl2s7irVrUTCd6rXjcQ42p8WhMD83B8DqyhE6bZ0fqnlzf+eAi5fVFo2HI2Jr7yCM/HomDANia7v23BwnTuj67+jxda3HdpOAzMoxJrH2bsQNBge6tvvsY49xZVPXC0VZgo2BoKH3SKeOwtqw2eywsKD1vLi8QsNsopQF+/vaRucunufyFT0jfePSRQAGwz6x1fPNp05z//33aplbLQ729HcXL56nt6/j4uj6Kvff9woATt1yk5Utx7ojy6srrB+9GYCFhaXZOroMiBu2bokTfvDv/4DWqfXH5eVljh8/BsDc3CIdG5/NJKJr7SwCG1taH48/oWvo7Z1tKKd2L8d0pJ/7/QE7+1qPvcGQstA2n1vscuyYPWdB27h30GNzU+3ueJzhprl/XsPsfpIknD6t66c7Tt/JwoKOhz//1/9nADLnyK2DSADVIBdxz7KlVTlcEODs77nNZ0UOhavmGijM7orT+U0REPgxHCBBZZur+QJC64MRJTht2yAQstTGRVBiQ4BJb4cf/bFv1nKYfZ9kOYWNhSwTXK5larViunPaFo0k9nOUoyp/RmHvl2fOz9OBAK6aXwSxPQhFSYn+vWV9Y26uQyPRsZzlJZOBzu9pOkVKLYcrS0qzzWmakedajl/4+U8+r22uI4A1atSoUaNGjRo1atSo8RLBDSOAUzv+s8wLxHznIgWh7ahFHEVRudEFEd2x7u/vsr9/AMBwoF6rMBIGg6Fdm9Dq6K7WuYzHHv8MALvbF9jf2tXrg4SmeTeGFo3o9Yd+d9tqzzNnnqjd3T02Nq/pnZtCUepuPS+G5LZDr6KJQRD4KJ3DkZt3v3SO0qKLkgU+Ylj9TvIMh36fBCHY3/N85uGJYiEzb2deTpmYt8RHI8LwUFRPwOo0DB1hoNfGofhnh2HovZythu78c4EoN69JWTAxD/4oLckrr4kDMQ/JwSBnv6/e/6SlXrA4iEhRL1LU6CLWhuKmRDKwzzmheTNDCRgVGmX4zFPqZXri6g6RPe+utUUeuF29Pu25wHtjo0MRQOdC8oHee2eoHqXeeEKrM6/VURSMhtr2++MR5Z7e+8j6CZ7cexqA1ZO5veuULfPcucQxF2hfStOMiXXapeUVlpfVqzoe7HP+/AUAAvMURlHE3Jw+e3VlnQ9/+IMAvP71b2RlRevpmXPnyEr1ROblxEfTlqzvJlEIoUZgg0zodNWrvLC4SL6v7yJBSBBquZtxQFy1c6RtPGAWke40G8y1mv7ztK39u98XBoPKy7jEfMue39R7vP51D/LIZz4KwHSwz9j6XVpOiK3/b21vsGQR8/XVNdKpjstpbtE2Uc8rqGdxf6RtNRr0aM9pnZX5CDHP/9yCjc3NPfZ3d6z+Uz9emkkElfcuzXwEwuVjgonOA9Xkc+XSJcb7eo8kiVlaX7PyF6TZxNpojqb5qx78469n9aj2t+NLNo/gmFg0onQpU9G2KiSshhka/zMPvkBkoeOyCjs4IbNIZeECH90LRBDzLDrnfGTClYWvj1bT+lUoBOYZzQjojbX8QZnRKPXzuL9Hah7Hg96+sgiAwuYXgpw4sroLhIhqHEJpYcvpRAhiffepfT8ZpaTm9Wy0CsT6WuBCIptT2t05nEXMgzIkjvXZrYa25+nbX0XTWAWPPvEbbG3q2FteSmg0dQ6igGyo75JIRlb7Eb/sCC2yQlEiNs7KPPChGIcjrDztLiAw29xudwgtcjUe6zgss4wosqiZzNMS7RPOlTiL7u7vbrKxeVX/XpYkceW1rqLnJVmW2veZHyNlGTCyMTAaj2kZC2FtdYV0UkURjYES4T3gaZr5aFsooffm50VJw55drUOC0JFPzZY2YppNfb84TliI9HlhHFNa+dLpiNTmiqrnhkFEYGMrnRaUNl6SIPDMpjwviKzukihA7B2TsUZTQgdVszRFaMd6bStOGNh4KUshswjCKBVlKwDzFuUfjjMCMaZUd4Gw0N8N+wOYVlGUksDqg6KkIihMrT5704Kh2dpwEtNZt4hiK9LwCRCGMV0fQS1JC10jlBaxKEtl5QCULqMobE2VlaTG8piMJnQ7B/aOOn+4wiG2/kLcjD1RFD6i5VzuIziu59hK9GzybkfXGJ1GQmjlmIynnoVFXNDv96yexrN7SEBoYZuW2cYwgiLX79utNt15tfWdVofYbFRWpBRVHUiAk6ovW9SsmJCmts6bDOmabW53m6TTjl0bMLT+PZ7kxG1bIywt6H1LKOxdFuYXiS2CM80KMrO9EgTM21zeCiNGE62/KnK+0J33NiCflgwytd00Gz7inEQxDWuDyCyry2f2zrmS0OYJQWjEti4rhdFYnxdHYyajid1D62K4P2Q00O+n46kf11EYkqYWTQtj5m2tsrS4TLttayIbK0FZ+DkKOcR8k4CyivLjfNQvDoSs6qdUbTyL3qVF4ft/WQSemVc4RxVACwLBD5HU6jkSnI34SGJK6zNlgY9EJhJBrnUw2NuDSD/nVNFJ8RRIEU/+IwpDwmo2cfgI34yIFPh1HuL8F2EwK5M4RxhU83BMLNpGy4tat4tLy35+7w9HpBVjMRDE5owgLHFhFQ0MKY3hdT3ccANY+gLPNiWC8xuozc0Nen2dABYWFlmxBXeWZTRsw9JqaWdIkoipdWxwRLHeY3vvHJeu6iKjt7dJlBllbZRTWueNI/3boD/yxisvCg5s0TidTpiYMUvTmDC00H+r4xdaWUXlc87TFQig9A0knh5Gga/oipKZTkMi62hjp7QZgAcffJCbjLoQhroZBdjZ3WHPPj915hkARtPCd5gsTQ9RIhzYBJuEs00ncGgwmbFDiMwAxnHItDSKZRgRxtUCraSKY4ukVKOi+t6JMLCJ7dTdr6Vh7/jME5/2gywh9RvNIhP6I33OmQ2dgC/0M1o2eQaXd7nzJp1gj693ZxTbMiMsq3cJ2eprewU2UcWtiEK0s46zfYIFoxd0ISj0eduDfSa2obQ9Ia1jN9GMtU9MBiPm5ixEbpO8vqQjtkmuu3aEvV2lxp09q9RRIWJjQzezp0/fzpNPPa5tND7wt+j3R+S2AJJwTMMWydV7zy2fYN6oU3PTnEZXDViQdHCBGtQgTInNQ9BIEua6bSuevutAQqxb0Wg0vNMjaTUJE90IFeLIbCPdbh7n1E2nATiyqu/bPn6U206f0N+15zl/WfvdVAoaba2b0bBPahuv9SOLlLaqaTT0vp1Om3ZL+0cUQNc2g51WQtc2vK3WHCPbOFZ995abbyaxhY5zzpf/2tUNFtpq2PtHRvT6+uze/oCtizoeIrtHI3CEVs7llSUtAHCweY2kpe38lrd8PaJzMUkSc/Wa0ozeMG/UZDfbABauZGIbzkkp5LbgCsPQ07xd4bxhmJgDxJWBX3yVLiAqqwU0uIqCVuaI9YnA5TNjVtE78pLSFtsTUhq2yRxMphSl9ol0su2dJI985qN+3kkzW4QFGWLz1VxzkUC0X8URZKLjL3dTnJu3ercy5yChbRYpiW1zKkFEYOOl1WoisdXNcEwUVnN8ZWxuJrK5ozsXYz4xWq0mUUVvOxgTifaVRqOgDG5oRmp8CVBWNN4owFn/TdMJuS0aXZkjVNT+0C+uSudYWVL69InjOidm6YTQVkvpZEBhNME0HdM/0Dn7ypXz7O3qeI6ihO6czmMVvQpRWrVi5hiepjkj2/xMJhPaNocSuGov8iy7W81LrvRMMVzgyPJqweeI7e/VRiTKE0/bWlha4labC++66w4WlnQO7XS6lEbB3tjc4ewzageefPoJAEaTIY2GrVWiAPObUWQ56bQqskBUpTpEBLZprt51UjpP84rimHbH5vJejrOxn+fFzOFE4CldcUXlm2Q4mz/iTpdOok67cV4w2DHHniuhsqtR5OemoVFAt3tjKrJ7Pk5p2XzaPdH1DsY4ECKjzEdxNNvseec4fo4KxFFUGwmBqVVItptzsKub3+UFpYIeW1tjwfpGwAhnDrUwCGnaWiZpNgjM6I1GU/aMTjnX1TXV6so83TndYI2GPXYtvccVJYOR2p8gDPzaKM0KAusAc2arkJgo0e/bnQ4to/y2mm0Ka9zhOPdpHYWEJNXm0Zx58TT2m7vBaESSVBsUyKuFtUBoa932fIe5ea2HTrXh7HSIw8o+BkxTa8NpxmCs7xKHMc4cFZLEvkxV6kIQhrTNHueFY2ybtICAVkvbqJW0WD+i6+/ovlcCcHL/JlwxtTpKKcx252VJZgvRSxe2OPOMrr+zyYTIbGXa12dkoynVfr4RJ8RBtZYJadh6cn5+nm5H2y6JE+/UKKq+5Eo/FwWHTx0Q5zeGztn4QucufFBCLw3dLBVD8txPGqGEfk8QlMycts4Rz/ZjWp6sILe0k2mYU9rnoiiJbFKZjAv2N5UKfPbJx3RRDxRpNa+WfpPZaTfpLOn4bDZCT2HNi9LPCX6oI34ODiQkjKtNsHjaqpTi9x1FGdBta59dWdb5em6uy8ACapRT/CwioXeARIfo8kUB00nIjVC7bmvUqFGjRo0aNWrUqFHjJYIbum4nFmoWhKjy5Ag+uXxza5M01c9FXnhKXafbYt6SSFOj5I1GQx/Zyosp17aUTnJ163H2DlQoJh+XhFP1dExGOWFSeQv1GdNpRtO8dJODXX/vIAw8PRMRmu2Kdlj6CJ/3doXRjDoSCGVZUVVKv4NHmImymEe9LHLEonR33H6av/rd3w3A69/0JjrmiSpKoT9QT994NOHCmTMA/Of/qEI0n3rks7OY8CFBikYj8UnRo7wgroohAdPe0D6rNyBAPJUiCJTOCZqIG0YWyo9LQvOWhJL5JNkquhcnTcyxxcmXvZqO0U/OXrkKhYnUZAc+mb5VzGingVFciKC073MpEXtIu93ynkOKkMiiE7iQvBIAsL50ZPEIuztKlewVe8wvGWWm1WDzsnoWhyPH0a72q8BcOvfedhtp5YErUuZNqEKC0NOQxuMx60fU+0uZ0e8ZTcfq+U1vfJDVFfWsbGxtsGbes/7ggKEJrgRBzLx5cSVskFd0SYuE3H//61g+qveYjAck5gnECaF5cQuZ0YlXVld4hSWJb21onz/Y2PLeoqiReLGdRqPD0rImjDe7R1g7ahG+MATzUG5a0vfy4jILS0pbzdMSZ1Hv0uVk1t263TmSSrApz7xrLTNvf6fT8hFAl6c0LTIYhwGRjYGj6ydYWlVP3yvuV8GbpBGTVFGnMPTdezqdMhlXnv9MhWCAq1c32NnRcj/95JMAPPKZh+jtHVjVlSyZV+2ul9+N2LjtduYobMRcu7rN1p7SiNOBtqtEIUlFTY9CWl40IqCwUENR5IeEmWBslN7AKDDltPSUTkG8t1OpODZ24pAjFukNi5TSfGjOXrw3nnBgwkYFJXmo5bu0+QyXnlRRn9HuNY6sauT4kYd/w0c9xLx4SRIQWoTt+NIpXvPA1wDwyU9/jI2Dz2o5ktk8PJ1WAkEF3abOu61W5ue8uLtMWSklucLPea2kRZhUNB6LWo5zevsaXe33d7xwUBI1OTiwKO7uiAYWIY4DLwRW48uHaWb9vkwYW/tPp0OGQ4s0pamn7TXb87NoSZnTtCjJ6poKPkzHQ6ap9tmD/V12d030YdTn4EDn52F/33vg2502Y30Mic3JRVl4VoBzzgslpNNpxQSnkUREQSWEMPYRQ999ynL2P24mEBEGM+o+Za78PqDyXxeTggUTOjl98iR/7PV/DIBXvOIePyc3uw3ixKitCI89rnOPvP89AM6JlRYAACAASURBVHzmU5+gv69zVJQkpFb+siwpLfooYeDHgwQhjXAmvgMQloFPBykDIQ5mKRS53aPIxUdGyjCiaZGpTkfHLUXAvM1/p+54GSurOq+vb2xx7ulHtY22r1IagyRyBXGs80AV/SgL8QIS41TomwBN7kK/nklCoZFoPUYt5+lrkf1bFsVsPUSGq0KsOKXBAdMsp7S+t7qkNnhpecGL3kVlQWRruDBueIpnd2Gewp6ztbnL1o5GACcTtQFRGLKwoHYmnUzZuKa2cm9/z6+ZkjjxkdSyHPvPN990Ur9vtIlMPCOKm54O6JhRQwtgkk2tfAlzS1q+Yqrz3PnBk542nEQBC7a+bbQSFi1F4pZTJ1i1dcaJE+skVqdTY3CstTuENg/v7o18ykUZRUwsWj+eDrGAM2le+DSnKlgWBjGNpo7fhoSeqho1E5KWlmlucZG5rtb1+vFb9Pdl4EWECsY+xUkjUGr7hqN9zj6t69SNS5fJjRVzdUfngCcee4JhTwf7fLvD/LL2zeNHj3ihoTCMWbC2LYsMM0sEofWZ0iGV0KBoRFnfz3mKZyDi1yQis2VyJeTj3IyeLBQ+tObEIoIApfi6joIAcVWKh7H5nJsx8CYjhj2d23Z2rtGwPjsa7HHeWGJXL14AY9sFNmazNKdtUfvjR9ZYXdW2L9OSg56ycw76A1JL9wiqqKUEfj0RN0JatvaIotBPeVlWMBlV+xiNVgN0Wlq3roxwtg6PJfYR2MIJgatEmgJKS1uKKGdr0uvghhvA//K+X7TCRCSV0lgCrYpeMBxTGu0qbDa42FOKSJAO6G1rTl67ZRSG6ZQk0UGflUM29pR6cWXzCaZ9C1mOY5LIOkQYIKYEldkkk6a5pyiIOD8JujLyn0VyH2aHwNNMKr6+UM5iwi7wE4McWsXoxBdUt9B3TTNefqdS7/63H/g7rNtE88zF87TtvZZX1pg3es36kRWOLesE1Aj+AgAL7/5v/PoHP2Tvkno6WlnkZFWuoQgTG4SdTtfPAhX9SkQQo7eFkRBRqYcKVTJAHEYEtiGOioCgyjW0d0mSmBM3a/nj+SO0rMz3P/inOLiki9T+pcd8rlYYFxibkq6pYrWDmZJXEM42Z0kSA5XKlsxUQF3IyoJRGi3nYzidMHGmZNTq42yC7WeOvrV9vNJhZ6yfG9b5ExcRW72sHF2h0TC+vYTs72sfTLOSpKVlGo8K2guWC2KU0+Wlee65+x4AHj/zKOcvqSJeq9UmDHViCyLxE+JgcECzpc9fv0lVNp0rCK3tR4PYOPeqUFXRT4syp7RN8Fx3gfC4cflTa9ck9qqMjWaDKDED3UxolVqOxbUj3HaHKpaOswmXt1Wl7PInNHf2NQ+8ltCM3dOPPsrAqEBLa6vsGc11YXmJa5tqSAcH23QtXzEfqyFDnKd+tZLIK5R1Wg3uvvtuAI6fPEnb6Lb4jdKMuqFt4FdwHP7UN0fGq14VEJvyXkX9+dV3vZuf+en/AMDWtUu88Y+9Xq993eu4cFnpGOfPX2bnsi4Q9gf75BWF1vKKozj26oVh7GhWk201wEE5/2ZQsrIkNUdERSlrBpBWk2cIYgugIIz8Bm+xFbEY6TXNRCjNqVHlLLSTDnOWmzlOh1y9fA6A3/3NX+XR3/tNABKXccQWDefOPo0ct829LZZaSUIYVQ6OBkGhFPO7bruf3Yef0j/jvNJs5WwIohQxw5IkLeKwyuls0DA6FK5gPDKKH6VfhK8tHdc6yidMi77do0NslNhsXGAMOkKatKz/7I8uMW90rRpfPnzkQ/8dgG637XN/ptnYOzvCIGAy0rGRO2g3dRERxQlTo4UVtvgVEa8u2+8PuHzpPADjQd872hyFV54sypw0Nbqyq/K6ylnKSFn6clA6YrMNcRR5ClieT3E2vqocL0eJmCJeCZ7GGDvx93BB5HPWJ6m+31yjyb133wXAW/74g9xxpyoSdhoJB5Yzlk6bLK4Zper4cV5hTkUrPmUx5WMf+xgAW9d6iKWdJGFIZlQxgoBGYlQwmS1I57s6ll16KMXDzfKrR+mULKuolaVXfSZUBxrgKX6tdoubblH7cvL2O+iYzVw+cYrFNd1kPfW5T7Jx6Zy+VzH1qR0+Zz+a1a0Q0Gjo3NVdmKPTtVzQOGSuqe/owpJAhzziaXPi66agpPRblBJbciCx8+uquQW1CytLXVJTMI6CLm2jdQYu9rlTcRh4Zc+FbpvS+mErqvIrQ6/mPTc/T8ucF7s7e56qGiazvkRe+HKfOqVO0zCMPf00L0uc2di8zLElK8vxErHZ23GakebqlBvYGuLalQuUfX3ZheVllteWrd0ci5mua1uLS95pMT/XYcdo0lumSTHNUo6uq6Nl72CPzCqvMz9PZGuE1MFwbKrgZY/IUiqqdARXQjqtnDkd2m0dy6uLixwzB3S322GuUou2vkvQQMKZdkdpNEaCgNA2qkdYZW1N7U82GpGYDa3Gzfve834+9jG1W+1mg5U1dZTffPIkLbt2/2DA3q7a5p3dHpGpReeVKnAQkFigQsLSr7U1Z88+MxuLQunXxoXXtXBVt7N8dnMkuUIpo0AcN4iroJE7pOxpvwsbsU+XcGnGxQ3VhXji936T/rY5lNOMvqW17fV7RLdasMVSraR0xLZtWuwssLqg9Z9NHDYlMigy4mKmSgy66fXU0UaDBUshi6OZc2g8moC1cxAHdBPbVJsXP8smpOasDoqARmCKoKGA5QqLc97xtLTcZSm09dp1UFNAa9SoUaNGjRo1atSoUeMlghtGAD/50G8BMMpKgupSER44qeex3b6yxJPnNVx6eZzOoljjA7q2777tlJ6dsrKyzv6+UT1lm629c3rvSY/pUD0J+TglWqgETmK/s8c83Ekj8jQHCdwhCol40ZMkCcktkjEaVaIzs0TtsihmSecOokoEhmdHMSr1wbYlUN9x52m++Vv+LAB33nkXlzbU0/P4E2f57JMaQl8/eYun6h1bX+WBl90BwGveohSu47feTcsSO9/1rnd5gZowLP3n4JDyZ7/fo2MKgbF5hTTiYp6yvPQe2HYS0LSXiZqJPxeEifgwexUBjJOE+x54lbbP3S+nNadexmMnjrH5tHq2+heWOPfwpwGYjje96pv5V2jljqgK5Uvpzz1JGg0fuQ2eFQGM2NrTyFXDvDSj8Yh+qp4mSWBkSf3jXklkXs1W0mJkXpGlZfWAvvxld4Gol7GfHtBpG82n2SAvVdhgmuUc9Daszhosr+p7SaFl+8hH38O1a1qer37zVzO/qPf+9Q98iNe/XumNays7/Pp/fy8At9xyCnMa01lUT+BkMuLqlvb/hAZJYDRdKTxNNC8KrwhVIIglv8+cwJHvx4ELfcJ4GIhP9j7YH7J2RMfc4tIiH3lUk7bbc3am4cJ51oxSmI2nnloyN7+MhFo3K2sr7F5T2vXe9hZFRe8uRvYuA1W9AorFeS71NTJ4zIWsLOv7TsUxNkXewquHBrNk9SB4luJmWXmQs9wr+S7Mz3vVyk5Hy/+Wr/l6LlzWsr3nV36BsYlGbJ6/xpNPnwPg8oWLjKx8jWZIq62NEVpkK8dROeobceS9iZNs6pWA8zz3EcogjIiMBtOuVAVJ8SzwABpGiR2MMi+kFKRDLp3Rc5bCckzDImftrvYvJ0IrMtGi3fNcPfMIAGcf+RQ7l9TjuLa8wNbVa/bM0KvZJUF1TmOD6UjdiQfs8+ijOg7vuOsUp07Y2ZVXHyYwVZwFoy6FMcSmWtZuNWlU55e5Kc6ZwE/UJKwoa0lJNzQKUUP7z97+BmmhYzJpRri08O3ZMHXFlZU1uh193xU5xtE1/W2NLx8+8TuqWhyFTVpGxYobMas2D7TabXaMIr6xuUNmrJhAAh8pWjSq4dr6CaK2tm2jmXhF23TiqFhcMDv/NwA/gfkzZAPnJfGKovAKwCp8UNnW0guETdOUrFKc5NBDfKRJ0x1AzwBLKspU3CQ0dsu8zcP33X0nb33rWwC4586XMTXViouXN3jicRX3muQZJ2+7E4BXRQ3WTFjhTW/Q3y12l2gZteu97/8wByZaFbRaPupU5HghhzAKvALqXDITZDGiNZPS0bH5Y6GVsGhiVa2s4dMlVuZadJvVEszSLebmOHpc55Tu8oqP5oSNBkvrurZY3LjElcvKWOmPM79OsoAMc8mhNu6EHFlTeunCcpuOpce0o5CoUskgYLJtgjpGh0jznKmnqoI0zNbn4mltWe483bNp73rkxFGaRkucjke0TNxr2J/4M/fKbErLaG3ry4u+DirGRZmOKIxHmETCkRWj2U0nDG2NEEQxbbNBlLNz5hKLqgmxFwkaT0b+rExVma0UbEsfIQsoNDUCfFSw2e7QNOXjVrvB1CiU0/GE1PpsY37e980wCrn2jAqcXTDV8ajZ8nWw3xv4OpVGg8Qoya1u16diTKcpLWPNJWEl/BJy5dI1/4yKhtxN2l6dGheQ2jl/UZXrEUBe2WbSQ9RSx7QSKclGxGW1hmwRxtpXOnNaR+vrx1g3mmNWzgTOBoMJ/T1dO1y8fJkLVzVtYDLNmVvs+nbRZzsCq+dQAj8biJupa4aEnvIqiKccy+G5o4rCkXmhmRaCVOqnYUg2GVs9Dr24YUVDjuPIn1RwMBhycFnXf5tPP87Vy9peIpGn7w7SlJXTJmhUiawkM5ZTNs09nTgMG3Qa2u/36TMxoaTqjOROu0mna+3WbtOyz1EcUHXjdpwx1zSafbvlhYsq1uNkMsIIbEQixHYmdxKIv0ccRayvqA1YOXacsrzhFu/GG8DFwpQ1Bz2f30dR+Dy9bmuFTts6T54zsZwD51JSMzjXruoCuSwmjI3jXcbbDIe6UB+NxkRSDc6crOIZSYZgHNe8OhYhogpa5nlOZJuzspzJFKfZTBI1TQczmqXfOM7MkfKRK3XTZyeyvO61rwXgm7/tzwHwqgde7uX/J5OUZrOSyo6ZFtpJLm+lPPK0Ltbe8NoHuOceXax1LHHu+PETfM/3fI/+rdPhl375lwHo93pUpVLJ+Srkm9HvV7lr61Wp/QY2jpw3hvffeZyjNx23a0MiM4JFltEx6mJzzhbyYUzY0EHabLaYGpXszNlzDA50ULc6i8RzOhFx5Srthnbkk8f1HlGj4Y+aCMKQqKUTR0FMs5ImxvnDtgNCClM5TC0fqYhHxB2j0zWXvHGdliOcJW/m04zMBlOnq/c9cXIZiZWWmHRjvzAdj4eEsT5DojFrdoB8Ng28Wt2li7p5cjLkwiVdnN9y623cf6/m5j32yOM+D+Ibv+EbPY98PB5zy2kdWL6c0ylnnvptAJa6x7jlhKpvuQAkqagDiaerissITX212lAk3Q7GOiRvCxupKduOx0ysbefaHVzbnBrFnleyqo5feOjRz3C02gCmKecvKG1ynAmBURSvbV9j56puso4fWWO+o/dYWbV+0Io42Ne6e+SZAx6/pvfe7l/gdx97t9XZTMq4OgA6L3Nya+QgmH3vKCC0g6gPKXzFgRDbLBYGFU224OJVLfM0WODjD+vE/NRTWwzSgX/eyMaC62eMR2pwmpY7k5WlV6pLS52c9XfgXMVZCn35wjJEpjPVQoAgz730/Fy7wdRyGGNm8uILccTvferjAJw/8zne+FY9KLZjcuiLi/MYE4etC49z4XO6WSz6Y1q2Gb92ccsf2QINVtZsUWnTcRLEuFAdEu1mSIbWR6+3xJ23Kj22P9xhvqF9YsGOwlhZafp8iEar5elokzRna1fHdVO6rK7rPLG9s8X64ikAxj39XZYfEMSWSyxNSqs752YU1bXVo5w+rYvpV77iq9i8doEaX1489qjmgwUS0DHbPL84T26Oy4X5LoMD7ZPj3rY/kqjI8upMb3Ysz2dvf4+5eZ0HxqMDXKUQnDQObX5yKu9IGeRU9soZxyki8HQih6OoNohF6e9XFAEuP3TMwMwV+7yovs/ISExBfGGxyStf/gAAr32tzrf3vvzl3HyT0uxc6ehtmnNwMGBrv6KmbRNY/z1962nmzJt37ITaiD/+1Q/StmOl4tYSH/jA+7VuDnY8VS9wod9sZVnGtNIXqJyzzhHa2qMZhQS2eQsabdaPVTnX4pUBk2imtrzX0/VQ2zlft2HU8GsZIUOqo6zihKRZLdQnXgUxMbt724mQ0JwCy/NdTh/TeefokTZxULVFqccYWZ31B/r86nDvtMhIzc5ECSTVUTMu9SkchSvJy+oafdf1Iye46Zg+uyidT7HZ29nm8sXq2AlHt6vlT5IWw4FOmFctDePKhXN+M7iwuMTSil7bbJ4iO0SRyyw942BlyefBilENiyxja0sdwLt7O75t5xeWEbOJJXoUBMD+YJc9O7ppYly+ZrfJkuVgSiRc2bUARjn1Lot2IyEPqk2MY2IugJGtv3YGu/5IpVKE3T2t551ej4atmaJGg9DWoSHCnAUJWkYBbMQheztqjw/290jtvS+fPcuZx475+ggqp7Ktp52EZLZBDyn9ei3LcvojtaXFJKdp6+hG0vTKttWG9NKli1y+orZ2mua+r19sXfZBi52dXfqWLlS4gJRKddto24Q+cKN5foeCLYfUiX3OYxgiVHRzC/KUggtmtiiq5p0gJB2qY+GZM0/y9JOaWtbv9zhhaVonbldb1W4vUZrj+8r5p9kzNfJ82CO13OnhdEpW5fshgDn1K42LsElo77K/PyCJ1Vm6MLfKXKfKjwwYWZu3zAk7P9+m1a5yJqFaGLuiJLZx2J4PiANzFDUa/t3HlnCdhw3voClL55VERWKKwnKJWx1OnFAa9NGjx+lNbnwMRE0BrVGjRo0aNWrUqFGjRo2XCG4YAdw6qxGS3mhMuGQemzDn0Wsa1RuPBhTmAVpKAu/x6q4d9+ekXNtUGsoHfv23vNLiyrEmuQl6uCLy6mJJknhV0TwL/YGuM0WgjMJ7EEuvDDaZDjwFNHINhpbAOjt4Hh8JDIJZhE0Q73UPA0dp9MD77r+f73/n9wHwyteolzGfDLlm5/kNdw9o2nkvr7nv5dx5u1I9J3nJsKIrLnZZCIyCajv48dSxYLSbd3z7O7wH5Vfe9S7SKmE2Cr1ftN3teK9IRQENBZyVOcschUXK2u02lRSDuJzIkp5pNIkt/D60ZNL+MKX4nHpK7n3DkLRUD8TFixdZWTW1uEFM7hP8ITZPxr33q3DKrXmEsyjQYJSDRbwm0qSVVMn7Ac4iaKqqaKF48zj1D3aIjT6zNH+EhXmlGuy19jn/jEY99vb3aVoi6/oxrfPSDVhaMoWkpQXaXb3f9nbJ+lHtY0kjpt2qFNYWiMzrd/YpVX+bjKae2nj2zKMsL2uE9a1/6qt46CFVWhz0dvnOv/yXAXji8cfJCjt8vlKMmvTYNi/6lYsXWV66CYBW94iqiKCCJAfmld/KdxiifXKwYoIrr15U/iuwtTziY/1P6O8KWDKakiwkbDW1T08GB8zdpfVRhf13tq9w0SLtLz99O4+dUU/qB977YX824e23rTJv9J8kTrjvXo0CRSYEUJQJ+32LjjXXmJiH7ePnegSRiQFFkRclsuHLpEyZWtS+KHJPtRZCXKp1kI9Tyuow1qIksoNWK2rvOBvhSvVINlyT2EQELhbXWJqb0XWqczUlyLy41KRSAo5mVNRS5BBVPPTKYAQBE6u0HDxdNa7oy1nA5lMqspIcWeFzn9VovhPY2jynZepPOfNbH9DnjAvufoNRlUxj5drmVXa2lRL+gff/ItOhUcmCJqORPmdjK/NRhVY7oGGRwUpoL3Qh83ZeZBw7cqfe373eOY4ffR0A3/Dm72F3Tz3W01T/RfqUdk5WFMds7Gvf3N3dYnlR2/v40bsRI3IfX7kVybVuzpgoV6tb0LQznYp8jenAeGUuZmlRx0hROja3tL2eeuwx9ra1v73pQaW61/jSYzDW+cCVUJg3OUrgoGdnmZF5waNWM56dI1Y6JqZSu9czz/neJkUVuS8ckbE9kjgkKGdnaIqXUwCK6qwp82SHpbfTZelmHv+yJLdoTlaWXh1UJaIqYTb8v5V0lDsUHQwI/OHvp08e4+1v/9MAfO1bvhaARivgwCIkG5sb/l3W1te5x6Ihw0GfozcphXJhvsPIIl7jfR1nrW6L07epeMyf+8Y/y8Ts8Qc/+EEvJiEiDMd2jtg488JsFWtAwJ/HlRbizzBrxDGtpol+lTNKVyMKcXam6MDOXSucI7coUrMVgZ0P51IhM7sZNhLmF02RMnEMTbRieVXtY9zqMLCIacMJgVENk0aTsDrwvchmOndR5M/7DGwtkBWZV1afX1z0Co2DK1PP7nIitE15sjlnNLxOiyVLKSlLIbf5KMuGHFmvqOLOC1fFUYjkVRtpn75yZYOxnet8y6lbvCp0p9UErzwZepXj48eOkVtUMrV+MxkP2Lim89Lu7hbLudrSJGnQ6hhzzBWMJhqp2dvfZr+vfaKw+l9cXTZRO5BY2NpXGmYRiD+wOw0zGpXaaBISWY7MXNfEwaZDhhaFDptdRkN9x0uXr+KM4nlkZYVlS/FYWegQVAfYd3VttLy8wGikn0eDA3ZsTT3c32LzijJnhqMxo/2eVW/VPjNaZSAQu0ptOsBH8IvZebxZnjG1tUolBhPHiaeLHoyGpEYjjRsNL+yCOFq21gqixIv2NCvlQPBzkRw6UzxgJlwYIH6eKEqIjAoe2v4iKTJcVKlyQz5Uu7+zs80TD/8OAJ/8xO+yaVRUFwQEr/9qABaWdE24cXWDbROV23j6MfZsHzPYO2BiYnjDSeoPiCdu+vNRw0ptPwwIjIY5Ho3ZsVSwdmOR4xZxPHniNJmlcOSmKJtmI6a2txmXuX9vEUfH5uZ2YyYMBJAeWJR2pP2xFcYklbp8Ufp9UxwJYrTfJGl58aP9Xp8DY/RdDzc+BsIGQqMdEpiyT6O9zKRvIegr50js71E4O+hzeelWurEurl1Q0TcbhNVB6tk+LrNJKZrl9YVBPMuXKjzjxB+aPR1PyLJK7TOk8PzW3KuEjUcTelZxBwczCqjf9In4v+FmoeY4KFha1Inmz/+Zt3P33bqp8weqpjkHG2owty9eob2ozzh6282cPq6/W1xdp2G5QGEQ4IzyuLWhnbIIYgJrwKXlJb7mLZp/8LHf+A3OG6efKPRGtRiPvZqkVIe5BwEzPaSc0ur08naPTzypFD/KgpWOKQGtLBLbAuGgyueJmtw8r5NxnDRpxjqBvfKV9zOZaIe+snuZZls3/U4Cxvbbp4xGeG1n6ukd+/0hHdvIDUcrPHCHGqLmfNNTQEMJiXK938QmwbhoIRahXuis0Yi1Twx6l8mN0rGysEjXFOwaTa2DK9eemR0MvL1BYhTXdJpysF8dOD7m2hVdFCwstjmyqoOz39P77mwccOdtupnNsgEPP6STwb333scdt+ti+dKlp4ls4XH8xHG27SDe/ao9Dy6ybNSASViSm3y+G/eIrXxTiXlmpPe+kF9kbJNAsGjjZqnLClr/WSLkpubUlIDI6ArDyS6jvrbLtJgwiuxAV9F36a62Obau4+3k6VMsfErpYY999jPcfKeq473hW9/KkRVd2PcO9jhvxm5i77S8ejOhqVveemyJC6aKNR7s0jRHSzsTMAfMoHK4ULBSHcBeFJ5iiQv8BhAcElQGALJQ54mBLW7yTIirowxGB16ae5KOSBa1zMeOn2TBPscJtNuV8uvsANSKdqbmxCbYsvB9kBJaVWKiK5Cp9sOtC6p6mJ6/SHZWPz9VjNm4oO22v7vH1Q1Vx31m7Q6yfa27bvMovYEutFaNqjXMJvzu76lB2tm6RsdyFfd6Yzb3tG3LsKkJe0BWFCSValtZ0dEDgqq+DnE0esOzXNvQ+eWWI29j31QQO4nWxeOPPUY51MVGY65BWZ2iIwWLDTWC2V7MypIuNrqNgMfOq5JsUaoBjOOUvFqQDVPC0DYRjZiBzQ17+yO/4Lpy7mF2t5UC+o7v+pvU+PLg9GlTcQ5jumZ3W60mieWxBeJ80vd8t0tiB7AnzaY/KiIKdYz3egPGU0u9CAISy40NwgIXVAqTs0UjZTmzi5WNzoqZuqVznv4lQUBhVMIsneXMSHgoF4gZjbSicReHcgFDHF1zXt15x6285lVK1z9xUh0Su7t7XLusi+InnniK9qLamZO33catd2mqQBTF/miBbDLm7NlzAOzYJm6JZcZGY1s/cYJXveqNADz0yCNctvkhCaOZVGEgM6VTqY5OyL3a5yBP2elbLuLugO0DmyNxNGxTsbrcoi22Mc+r+U98vnQQRl5tdzoOiCaW695qsriobb6XDzlvCowHIztkXPa5bGuV8aDH6eM69l/5sqMcXdK2bUTQNFpn1BCfJz0XqC0qg5DM6HmtRuKplY5dMivrfLfDyaNKQVyw8mxtXyK1HG4cTE2Jdjjoe20GofTOP5eX/hib4VDfO52OGfb0nTauXiH3TvV5WpaTF8cRrZYd9N7uerX3szu2yB6PPIWOoGRquWHDQY/S1qTDdMxV2yQe9Ha80mlFg42TrqfDT4sRWwe69gmShMQc4mkwommpSo2gQWC0/PZcddD9Hju2cTx67JR3ZKSjEYEp3p88ts6d9+haJA4jPpq+T9sg1fbpNldYWarSe8Sr+457A+aN2rq0tMh0ruvvDXpwulfSDSCwwdpsNmnaBk+cEBrNf3+wz5allWRmV+c7LbJqU51PSE2RvcgLYtv8J0nb5/sTxT5dojoipiyd1/FAZmvxssjt4PJn6weURc6or+W4ckYdk4Orm8ThzD7m5sQ6+/RZnnpMHfa7ezs0LCVkdfUIt9+ma/iTN58C4NrVy1wxuujWlac4sKPoxsMDUusfLg8oKzpuXHpKbLVJbiUxluLPJC/93FbkGZHt/rtzC6S2tjnY1fravLbBxob2gzzL6VrdLS12/NFNQRASUvWbMTt2PEpu+bDNRuKPsYqiWeAMZnooo+GQ0QW1x4PBkG1zDKl+hAAAIABJREFUCnzLd/K8qCmgNWrUqFGjRo0aNWrUqPESwQ0jgF/3Nj2/LgpDrxyHRD40Jy7DWRgodwVlpYHlIDVqQ9cOjjx69A4ii3SUOH9GzqDfp2fJ6sPRiKlRuiaTCblRy8ZjS2af9okOnRMYm2JO3AhJTdBhPJz4sP3CcttHJCoRGHCHooHOOzULl3Pv3ar+eP/L7iYwCkhu0cTdy5fZsPDyQw99iifOKs3rT7z1a3nzV79Zn9Eck+ZapQ8/9DDXLmkE4dbbTgGwesstRBYpS+KE06f1XME77riDs+ZlzCcTf56SK52n1VTQM5uqqGpAboqgw3TC2Svb9veQxSW9d3NhnaA075e9U5Y7ShOmGQxG/myghfl5HHptGDW9CEzUaDEZaLs88YwmQj9xfodKdrEonFdM7I33WVlWz8v68jpJYgIhROSZqYCZp22xu0Rq4e3zZ84R2z2ydML8gno+51ptTh/ViMWop16hD33kMqVRkJqdOXKjgrTiecSiTrtb2wyM5tOda7DZsUM67eD5IoftbfU4ZmS+H+wNBiwsq7dt52CTD39E6X733f8qfxjr5OInAWj0DrinZWGWTptyVynC+8MDcqOtXswX2M+1XVorIVNzlfmzsZKIhtE7gn5Bz6LGvWLMwA5Pn2sPWO2ot7vbXKKVqMfoykC9lzuX9vn43qf0d72SsY2tbLjJoKde76M3HWNtTak50yfHPHVGvV+lHTjuAsf8gp1vQ8rSgUYHGsWYk4G+91En9E3J7VI1JvOcZdF3FWaH1zpyysCoL4BXnnDC1CK9fes/83HOMVOv3CuHPJVpW51jQAP1ambTjH1TJi3LKdOJevy9Z9rNIgqBCJXuSykQ2/8kLvJnLsbZhK2Htc4+8yu/pPVy9hy3WgL+cNLDZdpGnTzk/nn1dP+57/xefvzf/ksA7nzNK3ntG16t18xpHW1vAzYm773n1WxZJLXfP0/fxtDNN636ijo46Pvk8CpUGTcyYqO/l2GBM9GcubhBt61z3v7BNfZNtGB3X9uyKEPO2LmDw8mQk7dqMvjC8grpyDyL0wFxpL/73JNn2NjXeezICS3/cDj0tO1Oew5x1h9724wnJvgQdXDm0b26+wTT1A4Rq/Flw+236hw1N9f19KQ4TJhJ784OUi/KfHZ4dBjSMQ/96dN63tziyjpLc8rakCj0h8KPBj32Ldo9Scf0B9rO49GIqUWfR+aJH2djynwm/lApTlOU1ZFdJMQUUXUumaPyPweeQum1xikJCe2HnaTNzTdpX77j9jtxxvy5eEnt8cXzF/jwb+gZfk987nMsmHDHA5OUB177GgCOHV2ju6B/375yjU3z/o+Mjk4snp2zsrTGG1/9BgAe+vRnuGYe9d5BD6kEXyTyQiAVhbwgYmxRriwr/XjpDXI29yy6VZYsGl2y04opi9k5iwD/P3tv0mRZkp2Hfe5+xzdHvBgzI4eaq2tsFNDVmEgCICCQkCiJBpjEFSUSRph+gLDWL5AWNJPMqGEDk0kLmUlYAKIRBDgAEIDu6q6ha8qsIYeKjDne/N6d3V2Lc9xfAmZVK6k3/XyTYZEv3r3Xr/vxM3zn+2oNPP6K7PqzZ0/wTJsQBJEKvHD7YLuPOidIY54XWDDBz5MLJjHRCo/YF1jOMo9SiNIYSjFDYC+CK5ulMvSs1I7NU2mBGRNINFZ46K62AhFDhPd3t7F/SNVF9xz3PvkIplmjlRyaztQ1QiaxsRrIvD5j49tbHFNtp5v6Sl+RrXDN+nVNUSJmhto0TdHrkm2qOzkkV3LrgglejEafiY3iSPjK8iqfYcR28/zqEo+ZfTpuBehtM9Eh7ycjNETNbQxljiW38gQSqAJ3zxpwPk7Q+JKKQyUtx3OcntH73Brue6ZZXWlE7Ds/f/c5vPzc8/T51RyS59LBErPlCl2uXt+8cQvjS9qTx8sHngBob3cHGYuxL6/pbMyyzCNhgnCtn6ktkfwA4POQdTW1hXGwZkeeKIRnqRZSIXCk7hBYcKVRlYXXnayFRI/RfYGv+lkYR3oiFODROSG0uw9YSO3YuguM+L289yf/DgBw7913sBXTGbV/sA3LLSUPHp9gPKP3srV7gKNnyabdePZ5HHHlb2dIazQUNbJLspvVyS6OC5rH8zyD5RdnhFqj7azw7PfCVyqBkFERYVcAhllCpfIkdcoqX3l1NriqaiyXjADSjbd5SRBixVXjpskhJb37+WKJ2ZzmN3KxV6X9Pouj0GtGG7PW862rGquciW5OrzCafvPZ/I0BYFaxw24aaMuGShjPDFbVJQyX2a1t/M/GKEjL9OIpTfjBswd45cXXAAC7e7seLnJ9fY37LKNwdTVFyVCU1WqB0ZixzswYGoYt5BndU9puwOcYIpVAO2r5LYMW989AAX/2xwSHc4EU0cA6OlyDiie012vhZ96ifr9OFCJnyvkVB4BffPoJLplGHwrYP2QBzlbiX/w8aqPH1KwPHzzAJx/8AACww9j8gyBcU3ZLhYN9cup/67d+ywtef3z/njeUxhgPB3VspVIqmKdo952BBYDaOLbPHpIeBW/TxYr6QUAGAQAaFeIH7xKkraxDHNwg5/anv/tTCLmMPdjaQ8HlchUlKEt6xtsctH77b/8G/uUf/CEA4OL8EoNt2mS6JTFhuGgdhgiFE+yUYLSqNy4SyuPWVTvyAVvT1Oj36f5ffuEl7DCcdco9YBDrPsnr0Qg5i6qPTz7DK3eYeVWleP/j79P87w9x9xnatNtbW3yNBhmLryb9NgRTaB2fnuD4CQWaTWVwfUW9iLUG3n6bMOVjDsxGixJXY9rUsQjA5yIuRis6DADcPz3DMfenvPHd59G/S3M9E/Tcl9MnKGdM35wbmB69+yANEDsWuUQhZYiN0NJvcNcDsbyaY95wMNsYDHboGi+99ib6BwxP7rSxmtM7XM7mSPv0HnOGLI1WCltDZh9DiDZDPbaNwCE/V9fUSLgvoWFo8mhZI3kKXr1mAVUwkg9UY2A9pMt61tCQ+1EiXSMxNI/7doEiIYM41jUs9xJfXF5gNCGnRpsSihMYTlbDQHqhVWktKjaapZRoOOirtUXI99rUGp98SgH7n/w/FNCPPr2H79y9Q8+63cKnE7I/3bSNFzio+3Z/gDfeIgmVOy+9gF7PMd7RM9375D4k99h99fAEp2ePaE7jGAn3yVrdeDhUY6zvOQhC7ocIG4QsEF1Li5QTD7d6t5GyA3188ZeYzNnJYBsfhAMsC7rP/uAWlKW1PjmvcTwlKM1gax/3H34CAHj85BidPr3nm0ev8Tz2cPyE1vz2oECXA9t2lKLFQuBSRji/pDUdxh10ehsZiB/74DVTFrmHVUYyQsjwzVDG3mnPyxK1C86Cat2GsEM9cbeeeR47bL+lEJ61c7FYYDpjKGGeY86wvPF4jPn0km+EpWMM0LAjGSgg5B5iYRqELCGAREG5YMloBI6O3wWnUjn1CITW+PaM4XAXr/8Unc237t5eCzVzcHpyfu57e3vDHmLuaz45eYyC20fmizmeZTbA8XKCZcHOa06frcoSKTN7x2mAl1+5CwD4j//Br+PhI9ov33/nhwidQxiE/h04Snqr1myBISLE/NxGrlA67gJY36NY1gYl2zcHJQuCBvc++8xfw7G3Hu4feLmMMEwQONHvtOUhpf0B2Z1hOsSIk01nkwzNhAzE/tUSL9ylJKBstQgXCEALgYLtveCEYKsdo2ImS9NYNK6ftJ2gwzT/u/u7iLgv0bXdZMsCTcGcDkGEmM8oKQII/o4iK7Fa0nlljUSn46j2aV46aQ9tfr6irrHkzxZFCesCGinQYXmFJE2QdClAOl2yhEm+QuWkjmBQ8h4ZjSc45R660WiCOUtF7B8doL9LZ6WTS2iMgfWBi0HNMFnoGsKxIwt4KQxtS0AzSz3nO1erEidLSlTc2L8LxX1/i/kcVUEfqqrcO/YwFeKIpTMyJ2dyioaDjq3hHhT3HBZawhFZG5kA3DaQcV93ZUPPGa+NhOZ7q8oa4CSf1kDl7UTtWZ9dj2ZTW9SNk7EKoHlfN6aB4bVprfE9bZW2sOxLuaSC1vCs7pDa/2y0Abu3EE0DwwkAlRWwC7Irk2NKvpx/+jmKLq9zeQcSDGWua9xgcfpXvvPTuPvyGwCAoNPySf35jM6q6dkV5ucEwyyWFbgbB0Veo+IkQ60NpOPbQLDe29o9i4FxbRFJG+24z581ePL4Ec+ZRsx+fouDlEDG6LH4e1XUPv6ZzkrM5muWVZdBUEGIhG1NxPtjlVWYcN9/qCT6PW6jCxWU82u0QcPPYjV8P+DXjQ0EdDM2YzM2YzM2YzM2YzM2YzM24ydkfGMF8PE5iagKYXwPp4b0cEpr4UVBQwkPzQhkglhS9nl3QHDAVrKDyyuK6i+vRx72OZ6MMeUMVRi2PLSvqjRY/g1l6XRKdhAFBH9Tao4QrHmUdpF2HQcmPBRVBRLib0TAxlrPaCUgfBPqT732Cp65QyyO+WKGiqtDVyeUMViMR9gaUobop3/xux5O9+TxMf7P3/99uk8j8Y/+8T8BAPzjf/ZPcX5CrHiTMcHAkiRBFDPkzlifrb199w5ef4Ma2x+dHK/hFlojcNo/0jFyWp+VCIIQilG3kbIedtPqdL12SLGcocXQkQX/Lur08MH7PwIAnD85w81bBK85PNrHMwxXlSoBmEFNqtA39b/xBlUKfuMf/Q5aLOb6P/2Lf4GIm9VXZokpv7gmCqDVmtQn56b4lBtgb914BqPHtA5euvWCzxydnJ3i1h26j+fvPI+HX9K9ak7DhEHom9ajOEKfdbBaqxBpyZlWodHm+T04OEDDqaa9ParcGmOQc/ZPwCLPaQ2uVgsEAWVceu0t7LAI7dXVGDmvifcfUkYpLzXmU/rdKht74o7lIvOZ1Kqp0eOM3vTRFZbXVEUOD+gadZEjGjA0uiOIJg6AimLPqjtullhdUQUnFCEaxURIOV2vmZV470uqNlurELG+0NV4hu09WrPz60tsbVGlRhQFdg8pG/7h5/S9X1w9AJho4cl0iY+YECHSFp+e0c/GNp6wqfDaXiESxyhmjdees0JAM7upqStHTAYYDcuwq4qz36YpIBh2prRByXmpeRPCcnV9b6+Lmwd8/7KB1ZxZdmLSFp6UKLQaStj1//N/SGm9btXj0Tlirhb/57/73wAAfu+/+29RMBHUm7/8XYQR7f3P3vsBWrxv+9t9fOe7pMUn231cX1CG8l/9EQlz3//0PpTTUPryCS7OqbK/WC3RsOZUka+8xtlw2MOyoGfcSilDaITwSIhAKrRD+n0+EpgWdE8//PD7OH5E8/fMrTf5ewvcOnIaQHtYMGT2+NFDXI/oGt3+FBGTKqXBNpBRJXp+Su/k9PwCP3yPmuplaHCXCZFefvE1P9fZaoExZ/xVEKOo/wZOfTP+fx8ZE22sbIacYfSxDNDiarGQEYrciUvXMFxti4IIEVcQGGSButS4vmaW4bJCzTD01WLlYV5Ga8wdgmBRoMjdnueKQZCAjyVYYyC4GiiVcIUmREmElM+i2tRPkbHx2SbW5DG2Cb2u3c2jI7zxJlUA+8N9LJiVs8icZrDF2z9PUM/hYBuNoX3x+Zef4cGXxOqbLRaoucrS39rHjUNa1wNGhAz6fQiGXauImDEB4NU3XvJ6vh9+dB8VQ91iFXmCMM8wHkh0mdFXGYsVn82BUJ4AT0gJxYzUtYEnQMkY0tZOJXBJSIcs/z7ykub/F3/x53GwT2eRStvQwpFFZBBMSPLiy3Sfb7zxU9i99QgAMPr9P8TVNWu6jpaYrsj+HYkIgn2K2gIFnyUVV3v6/Z63UUkr9ARsvVaCNsMYu+0EJVcOsyVXEJv1fARyzcxclwUqZiYtiwoNV3sEFGqGmpb8O2NLrJjlMS9L5MykLNSahTqJFBgMAW0sLJ/1D5mpvcgySLiqauArmBfjMSYMiyurxldPAylQs+7rkue80pXXF9RWImL4qVISkfPBlIJyuo5CgklUPZx1PluiXDjf4xj9Hmsx17lnq3/85X3cOKTfh0EMFbsqIc3B5WyCIqDNatN9NIrOgzLoYVbTc6e59SzT05reD4IYisnGrLWowPDkQMN6SK9GyYdzExpfoXe2voaGFvx3CtDuzLYKiolkdF1DcByQhsJr8wpH+NRouFJfEgeQnhnQes09NBlKZsxs5tfo8jt/tkX7cDlMEfL7HNoagn1B2QrR2acWoZeffwE7DKWdzgrUzCR7ckVn5rv//t/g43eJoG12dY3KwbVhETFpi5bGk0/WtfaVtZrXWq0BxXOXKPjWuLLIccqw1dHVFKEjlxqQfQnCyOs3Nkb7dhUFgYopeOeLJRquPrdbbWz1Pa8//f98iRO+RmCBvR1aE1u9FkLec9paLLOS7x9eY/DrxqYCuBmbsRmbsRmbsRmbsRmbsRmb8RMyvrEC6HQ5rLWwJePZrfb6e2naRTuirHyxMCgLpvPNgW6bIurxNWVTitVDT9me5xVWK0cPvPJ9bp1OGzVTwJ+enWAycw3o9Ls0ihAzPrfXHcAoythc5HMkEV2nlaRIUgfYtRCMN3bYfIh1JUlCYIulDvaHu6iYbnW5nCFN3L2yjlcS4kXWBLz10os4e8C9YbWAsvyslxeYjSh7t8oW2LpJWcb2kLI7SlvqIgVgpKU+AgCddhdVRtcOhYTkikWv3fKkGq4JtWka3/QPGI/xllb7LEwSBWhqpy80R+6kMLhvqx2G6AqnYWSws80ZpbJEwFWnMEqhItb4abfRZqrx/dtEP95qt/Ff/vY/BQD8+Z//GT55RJnWtGVQaPq+VaN9xi5UIU6PCXv/8itUfXr9W9/G9UPCNB/t72GwQ++iv9WB4Ezx54/ue4reHmvaTadLr+u3u3eANODejZnA/DHN//nkEq+/SZWRoBN7Mg7Xe9A0DWruu5D5DO0uXe/O3QOcs7bL9egUN/aoV67TbWHFWffr8dy9FAjugUiSFlwTWk+2kPFeCGODDu8FWODRB9TfER1zJjzR6BxRlqiz04EIXVZce451YRVqvxBqgKuqZsm9lvMGF+dEzvOn//aPMWC5hKae49P3aH7/+X//z/G7v/tf09+ZGgecpXvIX3s6ukKQEvX5pU6RH1AztVYBCt4DtV5XAF2vX6MNDOeRmqbxvxdSonYEAFUJcLM3TA00rreE/61LaN73sqgguVJm1QJGc3VsaxuHB1whsyWMrwDS3ykZQbEdidFAXNPf9SpNXfsAEIewXI6ophe4wYI+Afd+fLw3QMiN97vIUXEV7nw8RcSkBJ++8w6m3N+53engf/+9/xkAMONG9GFvgE9+SD2Fn33yOXohPfdBJ8XhXdIZO7k6w5T7OM+vRjiKaK5LXo+nT64wHNJ+2+0NETF51lfHT/DwS1rHy8U+hj0mJWLirKqe4MFHlCGcX4xQ8DXyvEDMZAG6LsEtOji/HOPZ21RVOD2mv3t8+gQxE+HUKHE5JhvcOn6C0CM8QoxHrh9z3Z+8GT++UbBNrCsNNvVQaQTFgpRx0oJlmZjGZrAOWWOF34oLtlEX1zMIS/tlOp944pfZdIGMUSPaGuS8N2ytUXMVf03iUPust7HrLZdEgc9OBzLwa0VJSQxNAHzpUKx1AIOApCIAoNPtIOE1aYTAjCsnUyb0SkKJ3iGt497WwFf6trd2cNWm/VI1lc+ubw8H2NolmxtztUUFobdBdV54veGyKiC40he0Umg+p4WM4DLzjlROKukrh9paqBXLHijAN8BLAcOfLxrtUTlOt08L5aWzmrnFYj7n+VBo9xjp0kkxn4x46iSGjFJ58U2qgv7Cr/x9PPc2vSvVH+IPfv//AgDM51e45J77SbGDmHuIoCz1hQFg7j50kzYSPoO7gzb6rLtbZCvUvPaaokK5oJ+rjKsiQYyUK8xJEMHwPC6yJVYLruRBeI04AYmCG9kcR8HSSlTuuLDCE4wEAAJeWFGcIOX5aPc66PCZl0SEMmslEQxvjGy1QMU+pDUaymmwhrGXStF1g/MzOkOXLFNllYHl9RrECgET5ARGQTLRjVDWcaggsIHvDdQeUdRgPqFrf/rpQ2z16P6qvPGSCh98+ClaHfZDD/eR9qmnTS25l69eIavo2ssqwErSZ028iykjoeqpQFPQ50vQ2lahgmQUjkCDxlX16gqG0UpW1tDsb1qhYR0pIu/vuil9H5mW8BIaQhoEXPpst4EB97p1+21ssd53xZJEUinfe5+YCvWc7Es1XyBnnolY1wga7tmcTLGX07v4zj75fAfP3/bEQaFukHHVuFhVEB3yr830DHZFCJg0kljxdR7epx7ezz78HkZMzGiMRcB7v91pg1lDMC9KLBg5UTUNGq5i1txXZxqLWrv+SYEiY46LEljxXq7qxlf1ypr2qRDC92EDwsuIdNI2JNsGqQJY7mUuygKzOX33aknzMp3PMJvSz6EQSCJ6t4EQnl/A2PVe1tp4kqCvG9/4v65p1zQKccBQic4BWglttjTtIwroBV1V13hwSkHAB+//CC+8SHCETkKbtKpKbwMBIIjcxpK+VNzoCgE7aGkSI2cdFBVy8FkbjJgIpG40Oh1adGUVA07DJSyxt+cmF9DK6Wm5jlS71gUTAmPWVPnLDz/B8BaRZ7zZ6qHDpBSu9Lt36xYOb1LwY7WA4EDpxp1n8B/9JsEKF9kKd559hu858IFtzJvD1nrd7G0swI5bKgJsd2ieAisQhxwwNAaVE7rmQzKUCoLL+kI1qN3BDoXENa9Ki1aPjESZL5A5OJly7KECnTYt+XyVQ7FQfBxJKFe63jvAakLMpNvPPY9zJxPl2HR0gVdepXf8O//sv8D/+nv/CwCg3QHuMPvS9SJEwZCYrXYbunSBLTncraQLzRO8KmYQDLGsRYaY4aU7O9voBlTiPzumZlkDi4rncb5a4mJJjkD2ZIL6nMlVZmNEVzSndgoc3SJ474yhx6IE+tu0pg/vHOLiip5172AXhi36tVhgm8Xi9w/uouSD6tYter5IBRAJzemonAOsw3T9ZIZVTQ5yLkpfZ99Ot2AZoppPWXemJzF+QJ8trpfo3GSB31sDL0JqGkdeBDRofKDTTFkfM6sRdlinJi1xuEvrYPfGC8jn9Nn79+7h/n2C9n3x2X38g7/1qwCAm3eJiGhZFDgD62qdayCjPZ5ZjSrhe4X1UGTn9Nhao/kbQSHApBCOFLiuIBwG1DSwzgPltR1UNRI+yPRqAdR0eJriEkxCiVarhW+/QeutbnJk7FyZnAxsVlrssCbSvfe+j6t/91cAgO44g2DbEIYCAb8vlS1RPSa40GJF6+eXdA2dk31Z/t+foeGD4OCswOCIvvvx+++hZiIFk8Q4YEHmHd6/P3r/I8yZZKAXCDzPsPFeP0HJ0DVUGpVbB2UDlhD0GpEGAsYJ8Xa6+PhjOsBOz55gMKB1HFeB1y2VDBsuRIkh38fo6tK/q7defQ2DQ/q79z/5GOdXtI/iKMNgSBdvsT2Y2Q768YDvJ0OvTbZrq9eH4tuPZQgV8drUEQqGUm/Gj2+s2FbaBr7NodIGgu1td7CDVpfFgcczjJi4KlsVKGunI0pOVtkIhEyAUlUNjOXE5GALEbNQlmUGF/As9RwNE0o4mGld53Deo7UGMTNitWOBhrWCYQRqPo+1aAB2xJ1Tr1ToyWOsNj55u1oVODmnQE4jRsPnmXNWB9vb6DDpmZAK0tLe2Tk8QMTrN0q7OLpFCZik0/a6cc55r5vSw+UMBIqKrvHw4SOcsEavDBMEsWP3a2AY7+cYKAMYNMIlpuDP6VYcocXw+gYCioPExlrkTKThyF7itgX4HRphfUCZtrro8jPWTYX+Duuj3r2F4SHZ7dsvkZZcsr2Hu0Mis/rN30y8bt/7f/GvsGAbPS1rdNgGpa0IwiXu2JYnQYqAz2ad1TAJ3V8/7UCwsPwCS7BkKCqQzRZaImGIaCttwzAh3Hg0xnxOvlYax+hzQrkVx77VxHi4XIgWr8cgWsMYVRB4aHGgBDrMnhy1Eu/v7OyQbY7jCLMpMzBnGQzvESUSJIl7F+skQ11qLDnBkefkQxAJDLNGx4HrHIIwEpLffWAEKj6Pl5WBZdbQ5YxZchcVisKJls88sVEUpT4Zcn1xhXufUGK4KAr83C/+IgDgVTarV1dzTCtH1BJhwVBJHQ4QhjQHIo2hmESs1WL/QCi/SQJrodlOW1vBGj6DZQ3BiSLVlF6H2DG5NjVpBQKA1tIRv0NYC8vvbbi9h7e+TWvvzt1b6LQYnjyls7kVxig52Xt58RUefUQJ0uL8DEVGQVpkgV7KwvdBg3DBBR1ONtySDRb8AuarJfIlz8GyQi2oDePJl59C8Xkcbw2xYKbfk88+pLkbXSBiXxcy9KSVRdXA4Xi11qhd3AML4Yiq4ApgFpbZ9Gd5hQWz5qNW0PyOVBxAOq1u3ldSCKS87iQkQj7fkyREwPBlCwkpaf6DQHjCSBcPKCUpbgARc7oEjRDrFrFASlTVusDl/PmvG5vU7WZsxmZsxmZsxmZsxmZsxmZsxk/I+MYKYM00tbboIbdUkUmqFjjJgfPiElJQtlvb2pf+Dw4O1oQBTDKgdYmI6ZnTJPGK9k0tPSxLCOErZFEcIGUqdFc90NYi5srhYjn1OljGWA+bsGEJ0eaIWoUuwecbmrvbQw9h1RYIOSpfJSneeUiZ8QIBvs108P0tyrod3rnrJRKW8xyOEbjT72H7iOi0gyj02T1tDRSn9i03uloJKM5qWmMwY/72H977CPe/otK0hkHN5ePVaoWUqY5ljzL4SgWejKSoll76oTvs4q2fYohCu4+U5ykWGrMVZbaW3DQfxzG2uLKVJAVCpnePI+VJZ6L+EL0dgrDu7w2xe4fgr5mhOeh0er7B+9f/3q8hNZTJmV6fImOK7SYzsKzdMi81NGtUjTm78/0PP8CMIX6PrnPoEd3fG2/+NI6YBGabVwEAAAAgAElEQVQynaGYMlyYqz57+4coeA6yIvcwjcMXbuKYocVqXgPc7B3vbaHb4qqqpn97hwdYcqP5y8++BMEZqpu7N72GHMIAAROu2KbAc89SBer8MX1vMZ7ivKIs4xfqAjdZd+jwuR3MjllGZFRiGNBcBwLYZyKT8wnDTJYa1pHHiBLFp/R9Vlts36XKJ8RaZkAKoGEoQTXlpnpbI7xN62T7zhAdJjZQcYo0ps+2Jx384R/8Af3dqsRv/1e/AwA4eutlAASt+h/+tz+jZ/nyBDPOAAZxhK6rTkeBl99wujOt2sCynaD7YorzpkLJmTJTG48mgNE+U2ZzJgIoKyjOUjdN7eHLwjT+OnGk8OLztCcbXWLFNOdfPaSq5io3UAy5fvTpOxAravxeXV7izpDuf/zRh4gZ3pOaCq2Q74N1MkWVoce2SOoaq4p+vlppaIY/vvvBOzhqkfZf0upA8x6eMJHV8uoS82u6dmQsEoaVTx7OYPq0dwJIuOlopSESzvg3jL/qtWJPHKSLApA0N7/wt99Gp0OZ/x/94AEefvkeAGB7i2zDy8+/iJt9+ruT4zMPFU/abV/16+y08fwhQU5H1xdod2mx95ja/bXtV3B6RWswy1vYYi3QWEp0t+j+rdYYMZRs0O/B6nXldzN+PKMoWUakstCMArGzHONrsn9bWxPECa2FxWyFEdub1apA7QgZUrLNlxcTDIdURUrbHQQpy88ATxF31P6a+bJCtqTvc7DQsshQ1Y4WHl4vLA6ANlcA22mEVrxG/rS5Yh8yFKvf7yJgyFJTlNCMA1ysMnz88acAgOl0ij2uqh/uky3dPzpEq0/rV6gIgvdqN9nC9gE9Vxh1EXFG3cgAwmkQMspCIfRyLI2VyHhf33/4AF+dUSVDyxBWOgmsCpL9EusIJEwDRoqh0kykBqA36OPAMPGLNnCI6aIoEXAVJeKKehQEaDHcVQqLIKafwyQFHPLHNOgO6Ex59uXXfJUi6hISKVsV0HyOD3YG+O7PkqbhanqB5ZxIqYoKnrwptMLT/rcTeidpnGLBCI3x1QTlimzks7cO0edKXS8doBfTPF0KshlNVSGwDq0UwIbu58gTo+hSI3Lz3zVIWTOvzzDIbreHiNeEtcrLLyhlEHHVw0B4v0paA+no85ngqjENpnxGZHkDxe+ov9VCYhzZjIDl9ZaVNSxD1EqGopblat3q0FgIMElJtZY4UDrxshd5k/uK+NUFzYtuLCJGiwmhvARLu9vzxCjQDc7PqFoVJQF+7df+E5qzgM6t+VLjh/cIofQX736KyYKrdDZFGHPFK21BsZ6zg92uCg3jzmBdeYI2oUIo6apEFbR2pDcNbL2eX4CkxZwEQqhCL1VUFzlyRs+1WilefJHaeu7eOYTlFo7xKa+1QKBhSOzpg/v46j5Js6lqDsuV79V4BsEImT4qtBn2u8N2pKUDgFtRRF4gZB+5LYFlwbbt9CGme7Q2k3wPcyZflAWhviIpkLPvNC80Mv6OXCw9UVxtgdpBFcMAEcMsHRohEFhDNg0Qst6zMhKMIEdd1agrx4bHWptR7PVXozD1FUdrxfq9COE1/+IwRIs1pltcLRemi1lKE7JcLL02RaAEYq66R3HkSbWq0kC7hfo14xsDwIgX4NlFiQ8//lMAwMHugdftuby6gIOFWCs8PLCua+8o/vDdd2iClPJ9eO1W2wdWq+XS9xVJKTxbY57nyFlktuGgME0SDz1rmtoLpjdNg4D/7oXdCOFtcpKWtYJwTJsMj3jjZ3/Bi3E/evwQITtdjQWuWbj1+HKMlw4JDjrkQKndG3hDW2QlAi7Lxp0uBJdzESh/mBV55tlSXbAbBiEChmmWKHHCunc/PH2IRcqLrtPG5IoggbU1sHzIXOaMUpYKWUFzMF3WXgi7N2jjhc4B/51AkzEGfXSBxZwFtPldGWN9T0KUJH4xNqaBcX2ESRvbQw46bYn9Z2g+tGQ4q4p9v0a328eNA/r/fHyJDz9mHSMNfOu5l+i2hUCb9Uzce+sPB/iH/9k/BAB8/vl7+BELcz/49DFCzRDVvPRaKgd7dD+L5dJj7Z+5/Rw6HVpL5SzD+QMyOlVVo5kwjDe7xgePycBuc7/jyeohxmOao/GDU1hmoRx98CkGjDvfPdjBeEbvYjErsM26WS5AeTK7xMeaAvfTaIbh7VcBAD9z53XssN7f2eMn2GEj/dWTr3Bwk55h/wa9t08+vocZ93nIIEIZMP78RxdAycHqM0O/U7VZ9wY2M1obuc5herQG8w7wyROCYm9VXXznWeqDXA4bdJktNe6HCNn5cmyqZVmhFdPPiR4hZzhrq44RVZzIkFgrvTuqrMZCstejtPHvNmwaSIZRmaqi3j+Q8+Lu32VRmipDzT0aoS6gNBl6q1doDPfgiho3b5AzVzc5Cg4e/+gP/g8AQFGFmL5G899OA7RepHc1iSzOuRdgqhe40ea9s2hwwodxyVCyYRyj5P4hM5DQHbITZWYw4UD6s4vHuPevad38k99+AdsMw7QF7bGbexHCBR/cF2MkfLCU2uLqktbYvBNhyextvTTFDgd1dcOBqDRou/7bOMbdX+CAM2mjZif7rZ97EQd3yAFWgvbK+PIEzQU5jMPdG4i4H6wslyhZO237xhDjkn7ePTrybHbZip5vuHOAMWu+jq4vYRYsPJsIZG1amzvDLSyWDMGenyLh/sjN+PENp2Fa1cLlXFCuGoyY8Q72ke/Jq4oGGQdv2hhY64TXnTiuQqtDjlOatqG1S64Vvi9eQQH+/NCe8diJk0shkHHAUDeVF6BOkxCls12dGGHfOUEh7j5PkMw2O+3bO0NIPovmkwmWfG7ppsEpO8hhoHBjj9a98yHiVts771o30BwYCBki4KS0iiJI5VokxFNQdgd9BBrXkygMGrZj1+Mlas1BWtyhvnYAtspgOMiq4HwSYMG9cKtaY8EQyzBK0B+wrWm0ZwMsqgYB9wK1LO3bVpp4gWdhLWrjRMkjaIa2amHR6tNZ3+nuQfAe1pI1SVUMsI8TmwR7rEPc6rQwuaZ7WmYlMn7eFAG2+2SzDndobre3txHw3GSLGeac4DqVEhE7vd1uC9sMSw35gCry0kMGKZDlFo/pHJeczJ6Nppiy813Ml56x2SWfwyjyyeKirLy+62Crh5s3KchNkgQhawUiFAj4LDFs01dZhumM7O0qr3B0RO07+4c30Lgkpa486+lqucL1mIN0l8DW2vujURT4/aQMYFzSM9BeY1o3BhX7Zi751m6lPjCTIvaJPyGlh/Zp0/j7zrIcCTNO1xy8xYkCGLZ9eXyMCff7C7QQOFH1ukQQOMZPukaZrdk5hZVOfx2RWbdrCAVYV82oa8+3YFwLlC7g074KXhS+rmpI1vFtxwLDbS4QRQIVsxJ/+QW1LnSFRDfhwLcV4rmXiKlTWo0V95K/e/0uTp9wEFxadJh59JCf6YW9LoaJS0wJ1BHdX1JLFJyAXI4uMPqKEkVDUaHHhZ7be3RvcrGHE6cjuSwQsd8iy8Kz+xohfYLMKoVOi317PuOUCBFJBydWiPkaoUphOOhbLUuUbJ9dwJ/G6VOagKHXAs3LCiXHHVEYAR22GUIhcpqikdMlVLDsQ9dVgSJz+t7aJ5LCMPIBKmB9DPJ1YwMB3YzN2IzN2IzN2IzN2IzN2IzN+AkZ31gBnE4o6/DFFyeIEtZgi4Evv/gSAFCVhc8kFGXlq3PuX+DpLOP6e6UQEI+ccr32GUwAvgIVqACtNjcCcxZgvpii5Cxjw3okAEFD2/x3vcEWAsmMYYHAm3/n7wEAXvouZdFtEEIwBLG/2MVyQVC9oFzhBWYD/M1f/WXscPl1zs3z+bJAHLBeW90g4iydtAZKOO0UhYapD62RvuIWcRUyjlpwshxWScwYgjiaV9h/hnQA094Rnjyi+Z1enSFfUrb+5IqqRHVjfNP5KjO+aiDDxsNBy6oCaoYJaOurBgGXsxtjfAXQCOtZyfKy9BkDKWN0BlTFasoVjOrydWgOICQsZ1PipI3eNjE/ZXWDjMv6aIDKOJ2oEM/dperX4W363qMbu2gzEUAv7OG1u28AAE5PL/CVPvbXATNjxpyFubq6Qr/PJCWLApMRNem3VIrtHaoSnbS/wiXDniLVxhFfc9Civ3t07yPP5llWGvWMYD4v3trDDmdtbt0ZYmuXnvuj+1/g8y8IbpgzHOZLXODC0HckCAGGsKxEjbsHVIHaTwfQnOG5vF6gZGhop0/PvfvqEbKP6H2LmcGrb9IctFWM44c0B9ezJxh8i+GgHQlGLIKTdNje7cBwhs1c52i+oMzQ7W/dQcjVu0hGOLpBEMl6VaDmtefwB7asccnrrp6cIiw4N2QtCsc0ZozHvlgGV9daw6naBFIhdGlGNFBcvRZ6zQKq68ZnygxDcRpdeY28QDeeBdSgguBqlIBGypX2KAIiJpMYPaZqc9o6Qs0ZsSIJEUvO2HUOcLGke3pnAs8sfDxZYMp98Hu8pP/+dg9vt5jZrJzh6CbZkcQAiwnPLwQmLPBVqRQhZ9+fa9F9do4q/BxX3oKXQpyV9OXnD2bostbWXy0zVJyxu7qukTJBRFvy88Whh6QEUQTNVZFcN6g469qoAndfojWtGGoWpgKHEZFQHe4+h5irH1IaNExEMF7N8cFDyszWskHpbC9n4a8mS9/cn7S7MGzbGgk8Yg2ixkpscVVhdHWNmt/XZvz4Rs6aZVUJpAyzb6UprizrOi6WnvnT1BUKhlcZs875Ok2vVVbC2K/4twqAe58azkWIwgSSqxpCWs9At7Xd4/8PURZrFjxOkiMQEpJtjK6VryrFkcLrb9KZF4r1ui/43FICiBOnM6axzYyPN+/cwA4zfoZcKSnyEg1XlBqtPcQy6XQ8CUIYKISBg30Kr3fm/RIJKC7PNBXgyjrtbh+9HTrbRMcgnNBemwgLV1KZ5E7/tcaUbUOhDUp+7rLWKB2raKNRMAS+NgYInE0jO5K0UsTsexRFjTnD/ZarwhO+xXEbUjq9R+OJPjS4QhIGUIzUqdsprlh3t1gt/bopyjYKrjAVJsGtI0Y8MTvqoNfFgKsN7TTFV8e0Pk5OT7GaM+PxoOcrZDUT+UVhgjYTSrUAbx/7vS72tsm/qmYLaPZJtIQnyyu52lzPJQqukmZFBcMIsCNd48Y+vYs4jBDwdwurPUosZL+sKCosmCik1BYthszu7B/6dqBFnmHK2paz+QoVk964Vp920vb+aLsTI+D1aA3BWAEgs6XXX7Z2zQgaMlw3ascIuBXINAY1E30tpjMY1k5upylCrn4arf3zukq9FRbFklqtxmefIb+mMyyO+pjMaA7GApAMc025pUBri4D9qzBOva6mEjWTOtEZGzMroggrrJgkb8UoEWsNUoZty0ahYaRUXWeImTxmZ6uFPW4PiFWDGUMu790jjeGWsrh7g9ZXf7sFzbBmnef4akY26MPjK5yd8TVricQyeza3P1QSePsWrZ/+UQ9lzv75IodZMjS9XGA5o7awYXkDCftYW0wWqfc72GozissITJkF+Xy+wiX7guer3JO/1cYgZeZX946ViCBdFdda7z8JZXw7WUdKJI4xFvS7OIw9nFQKRZqWANpG+pYvozUs74WiKGF4D1SlQ0oVWC1p75XFWqOyrmoYrmBqq58Op3yc8nXjGwPAhw8I3nZ+PsPf/fW/BQC4e/c5ZCvaNGdPjhHyQ6lw3T9mjAVc35uDYwgB8RRduHWBnpZwEaC11lPOKqUw6HPQwZO1WgmEjCkvS+lL5NIaL0j7+aLCDWY+3EpivPUrv0ITwX1AZV35gLI7mUIz/O7V51/A3/0OOd+vv/wCCqZ7XnGpfzFd+EB1lWUIHT63KpG0qcQcqPVkJ9x/AcCz/EipYJ0ERaCwt0twht3hHkYr6tFR29t4hpmMlH0V+XzCf0vflRcrGOsCvdozC9baIOC+IVoQ9Ps4bWHIAuBLNrBBGKPLB2pWlAjZgbBCecYtIRTCmOZfhSksyy5Y4ViR1gYqjFN09zm4CBPsMEum1AI1H9IySvDyPjmn10z9+8nH7+HNbzE097XX0OL7+Nd//G+wZHrp5XKJdz/4AABRJANAt9tFyVCbi7MROgwr2ur0EDLL4/6dfXz24Rf8AgIE7JxcnNLavapLBCzPMalzXDDsNowCDI/oOpNVgWdeJVjhk8sz7O7TnE1PGB7ZbWM3o+847Paxm9DhNIfF/JKZ42YW1YrW+tV8hZJFiq+YoSq+2cPR63S9XTPETp+gKl89OYFg6ufywRKnFxSc7b95A5oNAres4MbhPkSbnnt2tcCgzdIsmcEPTu/RfQiLL+7TZ4a9wVOHjJNtEJAM7Tl5eImAA6hAGUhnJmwAsOyCATtqoYRk8emtXopff+s1ul7SwLJjV1UrLyPywfv38L0fUA9A42QuzFNMhlAw0q1BC0gnqNr3grvHX514+nnn+IVBjPNLCuJ3DrYRMCb+3ScnmLP0x0fTFb54xDItQqLk5zrm3tLl5ArJGzR3P5e00GfYans7wAX5EtjRDX7+Z78DALiZBige0WHcv2Kq5ss59iOyB0VQ4jCmP7z9rIJmuZKdC4M/+4wOquMaEI6Sm73RqrJQTjC7LDx9fRK3YDmhsloEyJb0DuKY92HQRsR7aDydI8/pWW8dHaDLcjdn55dIGdJ2eXmBCs5Os3FrGoh6vTYyF2jUBba2mGpdhtg/oHW/1W7j9Ixs12b8+EbG5xMaINmid9FJ2jg7o76XuqigGycpU/teWttYD5G0bECUAgQneSy0T1IKqfy+lFJ7Z9hqDdv89SSO0YDmPmrdNB6W2NSl7xHNI4GMewq7Jsad56hvyBHzFWUOM6H9FCQFYg7ChoNtvPQCwcaeuX0DnZaDQdGeO7u6fAouFXpG25ZYt6UIIb1jJGTg2R+lfz67bi+xtecU6G9vo8fMkili7PG6vxj1UbAfNGH7Plk2WHKyrzREww4Q10DNc6C1heZzU8oQzj9LOLjo9PvoMSRcY44V96NN5isoDhaDpAPJ705KkDo1gNB5qUnbM2sqNEg4aWut9c9tA4GabUnRGLRdj7x7h1WNG4fkLB/deA57O3Qufe97/x7Ta4Ig1uXKO6QL7rdTIkSP5SWG21to8/kvrEG3zT5THOOae7FhhYfh1pw8X+T5ut+0qgBO1AVqips3yZbv7O2izZBSqHAN5RXcO6giJNwaEpu1b7SsGjTsWE/nC4zGdGZcX40x475mdzaGQeTZ6qMwWjvneeElKyyAwMmESAkHqnPPksYRQk7wSam8HMVyufDQ0W6njcSxPgchPIDP8VpYg2pB+wKrCZSTG6gqFAxj1LpGyAEguIdUqBS24XUVWewz63ksYr8PAwEfSM+WGotrhnEzDNYq6TGlVWlQ8Zpvqhy9AX33/nYfis+R1WSCGa+P4ysKWjuBQIeDXRtFrkUNi0WGR3xmXxcGY16zFYQXW5+yHytGM/RZJuxnXhhil/cCihqCky7j2QJd9nVVFEBwv+6Cz//ryQLCcQq0Wog5dmmnFmmHfp/aACtuOzG68T10zududAnD+01a6XtBZVn5PkHRKEgO/LRwNsB45nolA6TMct9OWrDpWkZrPiG/4PpqjMkVzY2Tc2rqGs6KBWrdt2uVgnLyKFHiWzI6na5nSv66sYGAbsZmbMZmbMZmbMZmbMZmbMZm/ISMb6wAph0uU3YSPPssVWrSNPGVn8V8iSh2DHYWpUvVwEI9lXmjfwFXH5dS+t8rpdaaYnZdGdTawHCmsr/FJWNdo+ISehTFaLUoijbGeKaiubWIR0wo0QXaDI3LuAJkjYHlaLnb72BoqAr3S2+/jVdfpAqVNhqGK15dzq4WiwySU6NWPFU5eQru2jSNzyoEYegbzV1Tpl3rwAOWGpkBElW3nAaVUeirL0q0MWRRzWBFEIy4FSHn7A0Ci4hhOd04QciVxrwo0RjO+gmJpOOy/0QQ0G53Mdyh59bjESRryYRR28MfgLUgLUTgs1XyqQKzz6iKCK0uw2TCDmKuHMYqhOJKjIxjWE3ZDZcJ/Pbrr+Dbr1HFqJW0MOUM3O3nb3k2rfF4hrMLyijNWVsnEBEUa1WVTYajO8xoGguMWCC3O0yxfUgVuyeP5njwOcELqpohszKCMZzlshoJN9DPZit89uAR3dPhHoZz+rskCdDtMtMSP9PP9F6G5orjLBvhmol3pqMFJh9Q9TxpQiw42zkWczS7rN/FVRi9zJFwhXuZVXj4GRHhrLIlHMQokAp2Tuv+InvoiYbsgt7AZHyBuMcQDJGiYGxodjLGbSad2dluo8Pw13ans87mWycCaxEzG6xuMliGgZnQQHFmDiaA5Qqg5Qpg2uoh4uzx/s4Q/+l/+BsAgCYscXrF2lCywUsvERlQbX4f7/yI9Pdc9rvMFr4CpbEmkpHW+ob2fr+D5YqyoJPJBS4uqYJ6NaE1cTV7jPlHRH7zS7/6y5j0yRbdm40wvqR193CxYrUqoLEGhrVBG668fSIs/iXDSW7KXWxlLNorBTKGf+/u98CkdajOzxCcE+nQ5x/QM/3oao4PFGftwxqBcpnxNRNdvirQYpKX7RsVmsaJetP3ChEicEyosKhYDHFcXiOUZPMWc42MWdgc3CRSCfoHVIm/PBvh7IwysDIQuH2Xsvmj0SUsk2sctru4f0J2RTE5lcmX2GVNrQBAwrZmeLiHGzfoO7rdnv98nbZwxJnlzfjxDem088IQ20PW5U06UMyYkWcrCIaeKQVPpFGhhubD0p1dUSBh1bp9Q7lSu7Uw/rMSocdLCqRtOtMGfVp7oUpQsT5fnuWwDKe0VkG6cx8NQt5zYSzwSp8qNL5SphSCjNZbmCyRcIXn1t27eJYrgP12DM17o+DrLWZjD3XqD7oQgls1TA1j2I7ZyLejWEGaWwAhHwDySaSzB9Z6qKpUwgvBx0kLSUR7o703wNzB5aoRz63yxDAweq1ZB+lJ46ywCJ32nyoQSn5eZjxvJ2302ecQQQrNlcOs0jBwbJLSIyMsjIeTQai//i9fvcMIpb3tIUxNNnSr20HC9j6UAWomrRiPyFZuDQbeh+v3+/7Muzj7ClehgwUbnNfkU0xHNBfaNsh5/WRxCMOVvFYSodul+xhsDTBiUpnFKvdViIrf6zwrqfIHICvWrNDTPENvSLbm1p1b2GLsfi2kJ9UQvC/SVgdHjESqqgoL1iC8urjGYkH2r9ANKr5Onq08+6dx8FSjPbPtfL7yMPq6rn0VHVJ44iJrrGfgdJrBaRphwGdRlLSgHYmQNahYl08Yg5arRsUJGq4SOpJQYwDLJDB1kcEKbt/QIQJe37apIHktSd5P7TRCh/fpy8/v41vfItKlGztbACORwmBNGvPpZ59jPiI0x5jPQQ0By5DlBg2a2rVgVQgZhhwpi2JGZ+Xs+hLLEaEQnpzQd7XiGNqx1V/O/Xu+urjA8aNHAIDz0Rx55WDSBoKfoWBW0s8mOcLHhGiZRwl2mXytVhEullwBXJQ47NDDtBrg9BH5CB9+8CMAwP37JygcqVWUwLKfXcOi4PeWCYmCfaMojVBVrk2F/SRjPLoxlIGTH4duanhpRSMRcluGI4FRooL05FsKZeVaaSLE7LdXeYmrC5q7s9MzFJnzHdgPFNKTSLaSyMOrhRDodOldbG/vIGWyGQGgYJTE141vDAD7OwylGCx9CTeJYg/dqrVFxMFNEih/PNin2JOEhxit6Ui1XRt9Y9YLUFiJhiEl7W4HB9yz9O03iclwfD3CF5+Tk/fhRx+u6VMhPA5eQGNHU3CjgsTDL1v8LxqN0kEfrEWrw0yL/bY3RDpvIPjFtkOaAx2UXlB1azjEFrNlaVjMGUaapG0Pg4XWEE70nYNJGUj/rE+LZhd54eEF/cHA/19VNRB8cAg2wMNeDyUvyrwsYTkgHrQS5Px7IQXyxgnWCnRZuPWc2aO6vR72meV0vFisRVJV4iUygDUkzQrrAz/x1wLAdfAb8TyJWkEwlj6QZNzceOf7RFt/0/VwvPE6Msaia60xZtre/YMhJlwKD6Mu3n6boLnf/x714HW7A7z5FsGH8maE+YoPnyLE0bO0ZhbjOfYZKjE/K1Bw32fCENd2v4Pa9b/lFjnv3k6/BcMwmcGwjyBgAdxeCy0OvEJ+J51QoWLR0/fP7+GUA0p5vEDC2PZFN8Jqiw12ojA/Z+jWhJ2NZQ1tacNOpzOUDAdMholn4CvmJV567gUAwGx6hsePKOhwbGtxksBwfwhaBpZsI0yh0UkoqXHr8CYk95glaep7Pa2nCRbotZmWXUZe8BXa+v4wPPUuwUCVXquLl1+l4E4Z+KyAkAolH3bdXorIMWDFCZ5l9r/plN7bk8dzfx8WxjOXwTQI2UG7vr7An/7pnwAAVvkSM8b6n5yz/EKxwskZ/e7+oxPvA62WOZYM8ZwuVojZWdpud5AzS+mqdjet8TFLfPyP15Vni1vCosuJg8WhwPCceujubO/i4h6t6X/7kA7Av7xeYckztOR5BSjIjAS9z2faMdIb5MicrhbISgc5YShnGCPiw1VC+R4Bawy0dQmMGmG47o8ESMy6kcxmO7/Co2MKSp994RaiS07aLaeYMESn1U5wg6HggpNbab+LNkthlKvM95VsJ20ohrcV9RzSQZbCCJH45j6Dzfj/fjjYeycdYMB9eKbSkOGaUVsEbk8J3/cEZX0yz3dkWAHlaPkBz6hpTOOhTFEQ+h5sITX29gm2/torlBgOkxQffUDQ7ul8iYrbM7Sx1KcGQJRAq3AMo7XvL3WQsCRN0Ru4tgPl+wx3dg98wFs1FWJOliYsV6F1y0Mou70uUoaAylChYT9DmtonDR1cEAA88afFX2s/caZusVzBuADWlggSmutOsuWlChwkL5VAKj1+1ve2GeNbCqGN8U7oTEhkrofIyQOkbQw5AIyTAisOFq2KoPlLjIoQemdFeBZvfzQ/5Un0pekAACAASURBVFtAAGmb1sre4T4EW6VOL0HESa0gjLDkRGfAgfuw30X4wprBu+F7TsIQ+8z4vT1oIWB/wTEZagjPnN7rd1Dx+za19s5wf6uHLjusMJlnlpTGyYQJn7yo6wxFvY6EKraRSWuA3V2CpVaiQcFzrfl8lQBiTrLVZYlzDkZGo5GH4Hb7Pd97V0DCemefmeFNg5od9aqqoXiCpZDeXiq5Tok3uoZm6KeDkdYFsAqcfbeetT0QAgk781u9Dvb2GWachOvgkh9bGQ3p+uwBL4UGIdDw94VK+zMviuizW93YS6XcvX2Al54h3+jwcAeK+/6SIPSyUvPVxLe53FhSUmmVF+t+zcYCvIesrtFwG8bVxRlGJ7Rvl/Mxck7kH59QcsBUFY6/Yp8F0rdkzOYLaE5iam09e4AK1JplnPfqdWnwo3M6Yx9OV2um3EBhnjmpC4PvcttJZ2+Ij+/R+feX71HMMFloF6/BooYIaN9qqzwcvg4s2HVGO+h6iKdLxMBKn0yz1ocdMHYNrTcWCHh+pWuRg/RQ+LquMWeZGSFCdBiym61yzLjIYY3FgFsunBweGuODZ6sNGs/YH/m1Hkjl4xgh4GGiXzc2ENDN2IzN2IzN2IzN2IzN2IzN2IyfkPHNENAuRZ7PvbKL4zPK7gkRYLyiaD4eCFguO2el8c2RsGJd7uQsXxCEiDljJpVdiynKANZBQHSNkkvM29t9PMOVAsXVg639PaQMB4yTCKsFA7qE8EmvJI0gOUNSFjU6rkHXQRhDhRazEG298hI0a4RNljmyFmWPQqN9JbLgf2U78ZBNGYVe5Lp5inwlUIGvykBbSM4uueyN0BqWYbISBjVDWFbzmSfJSNMUU4aWkB6O06HhCmyauh5fpEUGVTPkQWmfwayiGCVDBqSwEL58Tc8y3N5FmlLGJolavnlV143XHQQCX/gReBrqus4Z+IovgJgbzZ+5fYSkoUxNXeQwPP9CRV4j54MfUgVFCOA/EL8OANjpdrCaMdNpqTE5GfE9WwwZKrTLpC1F1WA8JXjby6/fxA9+QGsiq3IYJ8q6zBE6/cBbuzj9kqBuIadOtw+6AL+fq4cjBI6ttBVDuCba/hbaDJvcbgRef4WYZN/98z+i+coz3JtRZvGr1dhDhHMUqLgK1xm2cbdFmTexqPHgfdKpWV2xLtvWAP0eXS+rK7Ru0vXCYQ8la1TaL659mnz/4BCPHxM7aMyEHy+98C0sV5RVm6s52n3HAKexZDhJuzMgkVAQ256ruDkIjBACPacFFvVRcBZUCb2Gc0sFC9fQzhpSKkWi1nqcV9z4ne620OLrCV3j8ZfMbHt96eEnpqE1Goiay4c0PPuZhV9ulxdnOOu5RuzKr9OP79H8WxlixoQE5+M5eItgK4ihmeBCBAGOmIm2Xze4YkKggm3UnpV4g9fMX80XuHIsikJhwNdTX13gN16ledqVNc55T73PVYDxVoqMbYMtgW7AGkSxhSnoelndYHJGa3ZSGFjHrshZQy1K5A1ndoMWrGMODhVKZklMeyG6ihcZXMXGYloR+dDWjS5eVFT9NbbE8RcP3OQiZ/bH8ek1BizqPF/Q3rtYLaDZLoVBjIfHdJ8He1fYZ1HtTrcLxbaw2+1i4ezwZvzYRofbM1otiaKmd7ecFWh4DYUdhXVOHZ60RQYCgplAXVXNNhrGuGqV8IQC0MKR0UIF8IQYcRThzh2qvrz46isASNz45IQQHOXH95HnDtpooZzEbaI8iZsMhT/rHTRTCCBmcoQ4ST3sMIiiNelNYCFjZhLl9o4k6q/19NqpZw9VgYLi6qOC9ozC1qyrZu7MNNb4Kp2S0kOnxpMRLNspEQUQrlLQ1B6pM2BGamMk4JlSJRpGDJVV7RFR2mpHwI26blDkrgpBv0vSCF0mUYFSiBL6j2637aHxWhcIBRN+COsF7B3CA0IAbJcglH+u5qmqgRShh46qIELGcDPN+qqn6THGz9PeT5MOrhlyX2Qr7AzJ7mxtdTFhHbeIGYdh18yJ1misuNqzmmfQTL6SxBF29xgFJCeYMCRdMWPlIBl4LTZ1NcZAOGbSDnZdS0wcQXILUDcJkTqo8hd0NhZ5gSXbtNlkhMmEtZXrEv0t+o4bN/c94snUFSZ8VuZMgAIhYFiHMVABIr6/KIwQqDWE2FXCSlUgkKybGq7PScX22Wq99qmCEAMm+zk8uIn9Id2TtQ3K0jH20mejKPDrO4gEZM4QxEB5Ntsyb3y1W3P1UVsNzf7mPJvhyQWdDU1TIJBr5y7LaA0+fPDQ6xR2mYFfKeE17crqKeSartEwYdD5+RnOhvz+mxwzfucVs1ROrieYhXRGqCBGzqivqtEIrJun2qN9Yik8vNHB0VVtULLvscprsFQpRAiUvHbTNET3gNB/w4ND2A+JAC9jCKtWEgLurBWQPKfaSlg44jONxjpSpdBDxZV3fyWka1+z8NBjAiE+hZJjExqzvQtVQHYHBMN3FfPVYgbbrJk/HVtzVTUoC4pNZo6fra6RF+tqZ8z33++2YdiANJVGyv6rDAQq90K/ZnxjACiYan3noIXJkhy4rGxwhx2gpulCMxxUF0DGTFD5ovT4VVdCr+sSJR8KEgaKjU8YxlA+GITHSghIRK5PZkjGoswL3Dy6AwB4/913YZ38ghSeyag9aHta+sYA5YpKqhV/bxCuBT2jOEKySyXyq1WJFjMS7XVayCpaxE6sMxAhmFQItjYouG9HBQEaPliW1kLxYRAGoS/X1vx8KlQw7NRXpsaYoYuj5RTMJI3JkxUcliOOYi8Snh4xtbyCX6whGkje4FbXkFz+DgKF2PX06BIZ0yG7g7HT6frNmUSJh59qXXvHmthK19h2y5AM5/hj/apgrfVssFs7WwhrkizQVY6LGdM5NwathIxK//az9KyjHH/xJ+8AAH7urZcxuyYIX10r3Ny5CwA4OTnHAw54zk/pEBJK4vEDej/dXoAEZHzm+RIZOxWBjLG9xwLx4wohi7G6fpkbzxwiTGlOF6MVdo9ojT3z4l18+YikBU7Ox9i/w/0f0Q7arV2e34Dv5wRmQU7Ps2mKR0sKfBtd426PHKTbrSMMU3YQ6gyfpbR3PsgJHhHs7aydryT00JJkUUEe0t/l3QKja7rOcKeFmzcpoFTKMYcFMGxE6kRAs4ORJD10erS+eztD73yFYegDP8eka4zBckXrPwpL1Cy3Ym0Jzc6EsMo7a4ajkvOTGc4uCGIhRIir+0T93CSNTwjpJkPBbJKXV/8ve2/yK1uS3of9Yjrn5HTn++aq92qu6upmDyS7SdEkZYqwKZOSDNiALckbrQwY/iu88kAYAuSNvbIhGyQMkBBhSl5YAyVK3exuktXd7K6uuV69+c438+ZwzonJi/jiiyySXdrI7UVnbF4iX97MOHHiRHzxfb9hhjOSEc+KuCo65J3HQzDsQ4QWhwcJkjIabAHEgRkOBjglGOPOdgoCnhxdoiZYiIbEmDbDNw5u4e0jgp8MKtzZI6uO7/8Qdwljs6Kx+0KzhZ/fShvxP+sXcMQRQPCItKluS4mXxgT2P/kYy4vM1aNETNsDtOHo0ODGkALM5ZQ3kRu3xrj3ZkpuzfsVKj2he5D+v+97tmYJ3vL6qFWNnjh5tvdQxD/Nv22MZkXRZghcXKY5c3UxxfM38pwxaEgR9PjoiOXWewo6ZhfnGJGcdT0YY0g8LF0Z1Pl1XXFAe3x8zHNp0358zXqyB1g6LFZkZ7O00E2aRIc3x2v8PYE+2yQ4vwZnSv/0vYXLyq82KWICdKBhIpJlpe3RcIJbN9Iaf7ib1kSPwAmwFLzT79me92bTKFRkrGwqhY4C7gz3kwJ8yAGSmjUAhNkUFKNiXAtMGrJoyObSdQXFfBiHQGMjhecADsEh+IpeOggyVRcMPRcMD1t2Pc4vUiJrenmKngJW1DW6Lu07XWvRUaJkdDet0xAKgX4vimKmAe+QCTLWS0TaPxqjULbTbH9Vxtx2PUIgGX+h2VpAxAaOklNSAMi8MpVpNxqBgk1ZObQUbDrbwdpsbzHBZDftj7pSHA9kjYWHD57hBz9ISf9V2+OMDnoKEXt7aU8J3mNGZuszSgKNRg0MZdsDYoHWRYvFKsUhg7rBjcOUzNUIWNE8sLRx7e7lxFYyvZ+QxdSN6/uw1M/7jx+jPkxz75V7z2HckO4AxT1d1+OUkpHz+YwPxMPhENukLj8ZTfjAPhluQZP+geun9OsCipIeRlVcAGjqGoaSEyF4tAThFxGoSb05B/VYo8wIIZkS0xiNfbrO3e0daFK19EHCZvgj/WN9Wu8BIDiLnpIGQgg4Olx6EbGiezcnvYvp9AqPn3wCAPjOW9/EmLj6o7oulhAmcg8vz09xRRZXbY4nvGPqU9c7pm4Zo1BRIiZEj/lVGrPQrzBfkM0a6RIcPbOY0yGyMj16l5MyQEUJ+UnTYIv2nUaC9T9EPsA0AR2tB4tWYpHXsz5AUP8GwwbjnZxAqiDXDulAMmDPEseykhA5ExM8VkQDUUrjOVL9ffGlF1gFeU3qg4tb6+8LGVnfQ0ZAyUyxoVghCuRToUBEpKTByrqirxE1q8HOIXBF49j3Ze3OCQSpFc+Zqq5Y5XmxWLAdTxSAl2X+/WVtAwHdtE3btE3btE3btE3btE3btE37CWmfWQHMWd7l8oyJojAa2wekYOjZ5h2DqkYk3yHbeqzoxJ+z2raPaImsaduIljLZXWeLAagXTAierWYIdLI/3E5Vk65a4T06Fa/6FoYEXIyRqEjRqtmWkFX284lwBP/KGcLgAzIV9PhqhhOCBuzVDexOyjQdLxaI9Nt7DcEMdIDM5f16hZqImVIJLHP/ncdWFlUQBV6X/QGHVcN5xsvFDB8QMXn2+BgPyP8ITQ1jMlldcL8rkwnbER1VZ2TsILPIThRckpeqYRjJfFagKiMu6xcPxbpp4EkARcp1cZq1TKy1AEGLMox3vcUYWBVw1fc4v0yVsM+/+hJGO6lPHz06w9MHqSKxdy1lEA/3D/DJu6mqN4HmCs61wzv40hd/DgDg7Vt4n1Q5r0g57OXXX8JLrz2frskMsHsr3bd2r2evJEjDvjinD8+wT0akmrKT877Hm2+8DgAIreAM5n/yd/42fvv//G0AwB99+y0c3kiVmldef4VFOnIFUMQK7f0MQ16hNykTPx6McceneaBPe5yJVIFqncVz9+6lMaUsvBoYXF0STHN7gjp7uyxaRDI71RBMMHbOs/ptJtV3XctzbSxrXLuZxubq8TluENzFVLoo7CLw85yFCoQAjk8STPDs4h0Iyiz23Rx3bqVs/8HBdTwj6GKuTC9XLZvwWgscvZ+Eejp0UCHDjSxn9oWo0YzI51JlqPaikOaj4IydiD2efy5BzG7feQ4XF+m3u36Jo6NUFf67//nfAgB8/Rtv4b2P0lx6fHKBGWV8j+dTGBqn67sTmGl63n/t7iHujtJ9/O13U6a4shZaZVgqsEUKa057jGihs70D2Q7hUlZ40FMGM6a59qZ0mA7StS6G++gISnvLDCAoO/wQEr/+tb+S/i5EPJWpT1mRQsaaM4fWW0SdxWEMpCjICUNzPVeCnbNoO8rE1xUGVH3sriKLucTgYMijVFYVFoTUMLR+QikIlf3GGrz0ehJbquuajbfPLs7ZL9QYw9naTfvxNUvPy8p1XNGTAdjapYrG9hhZGlsJxdA/az1aEhbpyeO2twq+p4y1Q1Ey7C1CnykNAoKU0QYjYNBQpcWm6sfVfI62S2uCMhEZ5yiEQEWV6nqooalqF2RkI2Mhs+KmYkSRcw5dFlGTABUHsD3UaAcEte9ofjcGFe3HTbdiyGmIJT4xUkNl5RRRcepb0jpXKQ1P4dBstWDF7NaukEU2Q7ToqYrVdhaWZXtTfKIgWSTNOQ9W0gqRlSWdc8jKErXRDJO32R84RIasB+cYnje7OOf3o3dl/5CSgyyuDiCyaqERgvfE2owxpur/9u4htnazem/guCurMV9ezPC97ySqxsnRORusP//8HVw7JL/b2RTTy1QRzQreVdNgQL6wk60xtmitP6vOuCrWNBU0CUc1zYI9W9lEvW54rzKmxjaJYVy/dQuf3E8wxu+/8y6jpraGA9y6nfb3jEQSAFpS2YzBwegcR2koisEWs0tEFGTTLlFNcmV3uWqLQpEQLAhibUDINIbgETK8GmuoHEPoKUTea2MsVfLxeIwtUmdXUrNvoNQSXe439a0SmlVRTy+nWKxojZcC463U52Y44ArlbD6j65thPk97S9+1XO02MjC8WkoJTRPcup7VwbPAYoyeY8XeOniqvA1HFbYItrq7NYTNQjIXM8xJ2OVnv/rlNHRR4s/eTqiqWdsxikgjoiIo886gwc198sZTyf8WSOsRkKCedKRAOJ/japb26c6zvBWullc4epyu92Nt8IQg6bmIWNVVEZuLESrD0UUNqeicYBS+8vkvAABe/8KbOPYf00DQjyCypHKMZa0M3jMNTQmwEJOl+xZlZKSOdY6fZW8BQvRi3EwwIQpKOxygI9RUYDqOgKF7VVcG24Q2nIxqvre9c+js2n78mSe8TQVw0zZt0zZt0zZt0zZt0zZt0zbtJ6Z9NgeQsjQBPUvMxgg+vZqqgiYAqxKArokvZQQMkZeFLDjcnKkUwnCFoet69lppe4+eSMiu9Xg2S1mDZ9OEF788v8B7D/4YAHDn1W3sXUsnYKULb0HICD0kDktVzrfsdxZjIWRbhxEJuKhmgI+pChSsw4Cy3VtZah8LVJRKGEQHQ9k/t1ywXH+MsdhAQLAPYMbqGlVDUiVt2S3xyUniXC0XCwzJI8c1BsPs4yEkVoT3ZkJ/DJxZ1CgWE33fY0XZSScjBlQlikjZewDoY/EjzOPRNDULe4QQuIqotWYxjouLc5a8zt+1bmMRgufMEYTE5SX54V1e4vrNxIWbrTy0Sn/77Ii4cp2AoWzFd7/7EQ6I4H3txivIrgb71w+gidyehQy6dsn4+KMnp2iISHx+cYkZZUwn23toaAyWfYQh+e6cZuycxPd/mLhrd/avYbZKmcx33n8Pgiqw87bDt7+d5ttwPMBo8mnJ/IP9PVzdTdW2x88e4vmKeJqx4mplcFOEXKUVwJiqsK+8nHytLmdLTE8STl8IiThI47xsBDRVjGAj49WlajAhSxPmj4TAGdr56QKzH6Ys2NjUGFLGfXp6zJLX0mjYbOVi2e8Bkd5bXJ5hQN8XuiXu3U6ZvHsvvATXpjnWLRPmfyUdVF8yVZnzoWNgfyQhi6BQjAEI2UOQ/D0tmKysEIq4UAx4QlXyV158AQfX01i/98GfYUWZyAPyoPuZn/4yPvg4VQA1BFb06L+/OMeI1prrJmJE9/lrL97GaJmq7m9Stfxhu8DVdurb8NijIp7QqVGwNHdn3uG33kq/8/sfHmFKQgOHxEP9ueEAx8N0Bd8wER3ZVPzMwU1cUSb1bXuFc5KBNqsKvUlzVqvUDyE0+j77LTksXB6vIWz2AA0RDYlZZSEL7wFPY9v2F9jZS/PxbBUQc2UZHnSbMZrsousz75Z4i1WDhix/rpYrgGTcd3Z2AarY9t5hTPzs1WrFKItN+/E1RXYPqdBNFSAVWUijMhVzi9PDl/dCA2uzHDkhAqwDsjBMlFzRWCxWaJd0b13gypXUHkdnqQLvyefy8uoKS5pLt25vQ9HapaViPsxw3GA4oupyBXjkoIJ4si4WxFAQ6CheCNaxrVEQZa24WFDWHhY1zc20NNNnEaGzToAQ8JRJ751n3k1FlXGja66S9tbjhIRJWushsuBKFOh9ruS1bMuQqyJShGKdFHr4Nq+LHp4QRcF6iNxXaTAcNtTX1JSS/B3jQc2VuZPjE1zN0pqhjUmLJoCQhBPSH+d4QxYbi+AUi3iNJ0NczdJnqnrAwnK277BLInQdifp1bYvTk7Q+Xp7PUZF+wGg4QL/M4nVzXBJy6Wqa+lZXNc4v0t4go0RPlSHnA8d5mC7TNQBYdsUOpKU1b9m1HDfKSrPgCmSEoMpa21t8/HESdnv++ZvsA5krW4cH+1jOk2jVanHJlRipNEvjr67mLJwSgsDOQYozHe1hz5485eqX956tzLwPxbfWh09VFxuKk4ZNEcLJcVnXeUZ0TcYTmOyl2veZ3o4oJKPn8nUHv0KXxUFsy6IzIXZo6DnbPThknmNzmv7u2LdYEqfPVJFtCIRSsMzrX7O98IXjlwnrIkRG50Rv4ah62vcRmsTftFKsDRER+Tn6xa8l+66mqnB6lmK+D5885Sp5TF+afk8I1DQ2B5MBNHkuZ32KIANWjiwolMQsW2Qt18XjNN7/IFXsTo+fMCJIETpge3uUKZ9p7aH7FoTEjGzChpMh7txOQjLXD3dxMv+49DV9W7G6EsXXWyCuofH+IpIueFsq9FGxOEznPCxV+joYnjdb4xHmdB7JegxKK54TRgn2KhUxIRsB8qjM67RSBRbwI9pnHgA7emBDjAwVizkyB6CEZHVIHwKa7FcRAgL9sGV1sYBMNzWVYCUsYQQMKd4No+EJGCPgkRaXf/Pdf5LeCxHXXqYHXFY8iEJEVuCx1q/5iWg2pM2bYe8cK2tGH6Bo8Tm7uGTfMtMMMCdPmCmIAB4d6gylBPgAiN5BZbVA59GTAlLwgQ9Fggi3zmj2FPIImGa1oedu4nCcDheyqnD9VpqAewf7ePY0QVFWZwmeN18t2Y9mZIbIDlxdmMP7tNAs25ZhPqtli5p+n/0Fu45hpM2gwWicFq3F4grTaRpzpRQTj0/PTrBHROyOgn0lJSu8JkNeWhCd5wDj4cMj6EGCKGzv7CCfZ/YP0sJ88/pz6K7SAvXowX2cPP0AAFCrEfa2E0z0zu07uL6XDKgblcboyYNznJx+EwBw49Z1XDwmhbLFEoo2gp29Ofb30zheXV6yEM4k+1lGg0CLx5mdQtEK/M7b77Dn0eH+Lh48TOP+//zTJd5+LwmcHOwlfyQI4Np1gljWhhXPlou2+FzGwM+IlqrEYSSa1bkASQtpu1hg0CQC8mT/EFikflwse+wTYX9rso0JwWo0QZqs7THYThv45eyShYvu3b3L/li26xgmIGPAbErzlESJlBKoKVEhvUjMcwBVlEzEfvz4AYuv9OShl4SDcmJHIObNOpSgRgrJzz4EWPzBXpKAEcCCTkB5bkxV4YwCjK9/44/xS7+YYJPXr7+AKfVfk1DLzZt34EJecywEbWAtIjwlXbYWM3iCvS2uZmi7HBTQ/blxA/u/lDxHf2b8Af7V15MIwmwBzFgFTOOKAoF4uoKkded2nZ633eu7mFFS5vHDU1R0OJpsj9kU+fZQItBcWS0V5pMLGqcsyCJRkcKXqao14aWQFnUk1NeUNghJibq6NtD0rBstIAZpTKfK4fI8BedNUyNPwuFoDEf3+fgorTO2s/weRFIeA4DL6QWWZJRc1w1shqMBHMxt2o+x5USLEQwtShkhEm8IvkCp13A+AkUoo8qqkr0ocRgUxwz1EHBdhiaWICOEiOOLBGu/JKU6H3o0o/SHd+7tMEVCG80YLCUEq3HXlUbMQke5c1EhEPS58w4zUv7s+p7VPDvnMG4+LeCiguP/N1oxPFnKWAKxWCCbfbdkQ2v2D5YVJ7atL4fMLihoOhztXr/GSauuWyH4DOlKz5aPgsXthARn6BQkpMsBcihK4Yh8eMiK1FLLNNgAJjvbMHRA7Pslri5SwngwqNgsXArJShQV+SKqasjiN9Y5Xk+t93hGQmvbxxcwJE5me4s9EkZZaRISCYGhiMvVEqfnaeyaQY17z92i3oOVMbMR9WrZ4v5HKTnwpDphv1lve/Y6rLWBoSRm27XoKCPV02ddSEIr6feAlpQPZ5dzaFqrh6MGjvag+x+/j4YgyT6mWGEyHuE20T5mVwMsSaSkt4Gjdu8dwzOlENiihHEualxdXuLsnNRnA9CTAXvdKChKgClVhF92JyPs7KS9eUBQ1FW/wOwyw/Y7Wn+Bne0tpgZ576CycpsI6HkekpiREvAkeylloTXZzsHTwdD3LRypO69onfZtz+JlAmAv0BAiFxRElCwW5UMoHoT0j5RlvLTWrGgqhcAVxVRPj8+hs/mwrlHTOjAapOs72NnCLs0v8+QpF38UJECUhtmix3Gd+l2pgF3628xMiLHEO1vjmuMas3Jw9Dz11uHhkxSfPH3qcUqHusP9dE92dyescLzoI3pa6Drn+TA4GQ1YUOVieoE1vZc0XlIUJde4ppIMxZ6i60m0nONSEiwoAyH4ebfwmNEhfTXvsb+b4l5jKqb45HsiINDTveqsZ+Vg6yyv8dY5TiCpKrIDwY9qGwjopm3apm3apm3apm3apm3apm3aT0j7zApgJkwieD4p6tpwGVsqkeAjALyz8HOCD0rFJ+ci/lA8SwQkVtnXJ0qGMabvpOyMiAWmkb0EjWa56hA9RBawgITImXHKWOf+T8nnLwsVJCEFyhg4h/PLlBmaL1cYkE2BWlnUlOGxulQZO/qNvq4xHFPGYxghqVroup4hA23blqop9WexXJXyrFGwyBk7zVnc6/v7ePVzqcK0vb+N3RupIjT7vVSK9j7J+AKpApiLA8G1IPQmnI9YUXZGC4UFZcpWLMV/DENk8AhgOksViN73mE7Ta6UUerqu07MTznwuyG+urmuuApydnmJEsNXp+QUiZUaPj6dYuGQfsn/rDt54OQlKZGhsZQQuVymr+dKLNzAgQnwIFg8+TrDI/e19/Oov/TIA4OU7ycPtBz98G5eX6b72fQ+xlyp9u6/s8n0ej0fYnqQM5/ziCntEmP3iF1OFp2kabNH/P7z/CF//o38NAHjltVfwhS8k4ZGjkxO8807q/2DU8P3M99g5y8RfqQxqqsxJVfNzYe2nK2Q8D9OfoW40KoJOLxZz4JwknqPC7GGqypggsH9A8NiDPYwG6V6YrIwgAZCQD3wPQxXA4XiLBYVqrVFRVjKggumZSAAAIABJREFUiDDlCqAxCrvbCQLTmAae/OZkAN5/L0FlnQxYkK1KzkL6EEqJQRuImAUpAo8NRBFjEIL50bw2xBgZxRRFWXekqbC1S/DvqwW++e3vAgBee+0lvPxagqVOCW4MM8ABiRM8PT2DpPSYB3BwL2Wsh8LjhCqYP5wtIcjj6ZTmYz/exWovzdEbdyVW306ejbA+mfoBGI40+wAtloAhWHX2/HrUGLxP1hCzhUVFlZW3FwtMqQLxua/9FGLMGcDAIg3tIgvHFxGEujbQ4wzzXsFQtjk4FB81kWFWnu2/5BAMEQl+hof3Eyx4Mt5hmXVtKtQ0J4Y0Z6aLOTqCeAU4hi8bqRBouwgyYkbWA00z4Hm1aT++5rBGeaA9ykjB4iDRR85IJwhieq0UQMAZ5J3JhgK7FjGsef9FVAS5k1BcLWs7j56qEz6LiRmF0VbWRNdQIgu7eHjL6iSILj07TkVcnKe9N0cXSguGqS37HlckJmGD59ihdw4tVahzlV/HwBYrVSWTQRfSnp/ha1IWgTlvBXIBMAt+BL+Eo6y90jUqoobESmJIokmvvPY5HN5MglghApYqZN0f/D6NeUCt6fd0EdeQKrKk+5SgpUASqWsJbaKoIpCg/On13qFmK51Fe4Jz2qfHO7vscVdpwWJQK4K3i1iqPc53WJCvsHMWc0KVzC7PsSD4/HK5wHb2viXkwc644epS2/a4ytV/OLTz9PruC3fxa//hrwIAzgj2eXZxjhn5+S7mlmGfznaoqPqiTM0iMC44HKRCHVfVJru7XOk7Pj7C2Tl936rFmPbsm1Jjtchz0GNKYn6odqnPK96XjDbsUxjF2p5tBEPqYpRMe5AyVTNHwyHOzlJ80lvH4kImBIb+KaWwPUmx1PVrB9jNIoD0EDWdQZNjyb7HgCgeOzvbbMvlvE2oGwBaahbwiQTNjCIJLgJAbSqGGvrQ44LitavlnOlCK4rR2m5RYm5Z4tAgIkLMz3VkSKaQKJ6c+T1IjjGlNgCtByoCMxJxeu/hU8xWaU7sb4/YnuX8OPXjajZn2kwlNf+2ripIqurNY8TDC4LKtha3dqnqTnQuU9Vw2WOzGmA4IkuOZZ9dl4AocJUpEj7kIhscVfPnqwWLKs3bDl22nzEGNX14NBqiJcju1dkp5D7Njzx6vrwWUrCIFCTWBzgDNACfq3fqU9W4HD91fcfImugFGqqMV6oq1bn8095xRb13ga/LR892Gj6Az0LKywKf/hHtszmAVG6sjUFlslqSYjyyd7Yc9BAZC+/g2EwxP4RVVSPKDJWMCCFLdQEiq9xVihdCFwKWNKHzBqekYpP3iFD8b2JkyEYKsPMG4HB0kpSAMqfGmAo1HTSWixX732ip0ZL3n5QGlrgN2SzXVIrL9zICHfXNOgdtCi8ul2v1cMCB/+KKHsgQ2T9msDXGNqk2DcYj9v+4eX0fhwQjaIyBIujfOwu6F80A1GWcX1o0NNOurnqsyAT92ekRQ0C2Jzvoyacmm70+fPgQLfE/hBBYtal/r7zxKu4+f4/vVzYAvTw/wxbx3/KEmguBP/ve9wAkFco3XkmctgcffQy3zHJNAVfHZGZpI155Pn3mw4/epTEVeP1lCs7HCgOC2oxHB1jR9V5OL9CO0gYw3EmLweuvvYzjRyfUf4WPpumwuHRLDs6lqQukxDtWZ83B6mQyYY+zo6PHDG09OjrmwMLHpNYFpDlafIDS3y3nC+YvDOoBDBlFWqHgTD4gRYYmIgRWdMpzo1u1DBc5n05x+kE6cGptchyDV154GbdupMPNzvYYQzoA6jXeQ/Zujq7nDU6bCpr+Q0KwWag0aZFN/SO1Setw7WaCzzz3wkv46INkoqqUwuk5KdRqyRsKYl4PUMiIa9j3GGM5ACIWTocQ/H5eSIP3ZVFdS/xIrdnTsxmNcJw347ctPve5N1L/MpQieLz0QuIIvv3Oe+jpvvgoMSAYsm3nOKdA+JtnLWo6vD2kxfrR+Xv41idJZS60LXsrroMoYtdC6Oz15Njr6wEtd//oO/dxRq+XErA0XP/iw48wJNXiz3ct6qzIGzwOt5Iv5rlN1+e958C6X1gIgjdVqkIkJcalX615maXvvThfwVGSpx302JmQmrDWWBE/6+rZDEKnDo5GY4boR5rzg+EANeFuelvgy1IVA1zvHUNN+77bcAD/f2iCDlhKSWidXyvmaYY+sFJo1wWmPSgnAUNKmz7z1QIHJ0YJ3rO1TAlEAFBSINAzX6sIk0+XuR8SIERb4qSKzJeKiIrWmqgRFcH9vMT5J5lfQ3tsUzOtoLOOlQ9VpaFpjez6BEkESqBjlGQqh1ACktZFIQUrZEqFQpUThv2x+uzL6yzDtgaVxJAUGkdK8+vdw1vYJoiW0hKBMq5PaP4L79FRBkb6wl3zPmA2TXvw46cnnIA2jWGuntLk7/nM4umTtA7sHR4jkFG5UgY9JeVau8Ljh2mdqmuN555LJ6hIHn/Pnj3AJR3IvCvcIyUNRrQGOdvh8jKtCfOrOSYTUhmnQ2ulJfN8Y4wc5/X9CpoUMF3fYmc/Jegm4+ytN8JRk/bmxXCFjqJz23fwxF1L9J40No0aoSY/3uEo7bXNqMGCAnlZ12uKlBFbW3RfJkNc0cFWRI85BfZ1nmveIsOhY0xxKwA0jSzftzbvQyzheVZgroxm/Ya+b7HIibEAjgV2trewt5fGYGdrgjrHiCJrLBhsUQwRQmSl89Go4T5FEZnrBiXRk9+ddxkyGDEcpT3s2s07uKLYE1GgJU6+X3j268v+upXRPNegJPv2SaEY+ipEUiGlAWFNgAwjlKLACLWSiGzQDuQHamF7HBEH1MaIna00ZmfnSV37cnaJISnDHhwcYEYH1CglJGWjogemdF1t63BF8clI5+e6HFpdFLhYEt2mdfxca61gaV0aVAqNJC0NGsbZ/IqLGtFHaHILqLTEtf00r24fbmFIv+n6Vdn7Cb4ZQyxjGsuB2UTF9Iwoik4K642EANBaExywWqbX06sVVm1W+dRcpLGi6GrkmCslyrNvaeQCjAsahjl1kZO2InjmuP6otoGAbtqmbdqmbdqmbdqmbdqmbdqm/YS0z6wAZrGRSmnUlN6LiOizV0jwxZ/Me2g6ATtr4ZA9wNIJ2TrHZXPvU0YRAAaVQU3lS10pFnOJwcHk9/VfFBmIoUBfgg/sG2KdY3+xtrc4P0kqglfkW6aN4SqQ94EzPOlSKaOoa8hcbs7eQUaywqdSBc7aNA0MmYRFqYpCKgoExNO/wVQAqUSpwQiTg5RNHE8m2N5K79+9eYAbOwQvDQ4gYQhL2axKFHW0i9kVvM1k8MCQ0vv3H7Pi4+AFw68pOYXxaITzZ2lcVosrVhQbD0awP5/GYGu8hR+8/6cAgMX5BWZUnROUlZ1NZ/i93/o/AADXDnewm6EvsBjtbNM9Ai5IrKOSBjVlXO48n6p++/s7uEmKS5eXz/CQsppSNAg+9fmTBx9h6nO2J2WI9ib7OG8vaIyAKXkQXc2nGFMmchavYHTKFPddxz55GcJ4fn6Oo2dJPOaD997BzVup+uWsw/e/nyqUo8kE40mq1FxeXbC/yt5u8g+smhqSIJax8mgpgyW1AqjirCBhCGqgZCLDA8CK4E3er3DtII0Xao2zc/L4GQ7x6qsJjvjqSy9jSKRopQUT6DP0p64bVnrzzjDGUinFcNveWs44VrUqimwoFTvQc/b6l76ILlcG20WBefvAcASu3oXAcMVUfS8CB1wNRGlpPSifAVJ2LH9WysCiJ1JKViqsRgLNKFdKL/DOB0kw6Kfu/jQA4OGDT/AhVU+HjYFri3jFhz9M93OsVbaawjfOl9jKAlBZJEErNNkPq+tZbKKqFJPOh8MRXnwhKds+OT7FijKKkn5PRYHnR+lB+8rz12FBz1brMSa1UWyNWfjCIeLiCWWv6/SMLe0UFa0/dmkhCEI3HDZwIUNcIjSpE/Ztgdx4qo6s6hr9iu53VeHwVoJGnR6f4nyWMvTSKPR0D5bZQ1SA74+EZO+0lHFM16IC1pQ7CiR60358rW4ybFLzfmeMQAwEK4sdupBFVjxXcKLwjFrI2eQQAZGzxkIz/E6vVX2lUryXK19xNj6LSUQRuLIilIAgkHsUFjFkNcPISuDeezx7kpAbGSEhpeS1zQbPVSJdGTS0/0ArFrfJ64rWuqAGlEJF1RlTV6y4KZWGIqi9lgogqGMuvEjZw9CXDMZbGJKAWF0PMCZo/2g44TXXxYBI8UKmbwR4REJBRWsRaL3tnUdLaBrbexa8CtJD07VkSGSlBY6OE5zx7GwKSdSK6zfu8NhoSNy//y7133H1MMcvH91/Dx99mBQym6bG3TupQigrhcODBGGtGg1HolSrxZxpJQ1VPwbSYDyggEGI4n3ma1ZCPXr8GI+fpD10SRDX44s5eoKidtZBUuXEQaCjtWnZ9lCZpjAcYUgQyjMSoFt2LebsI13UZ30oVa7KaBiqKnnbYUVVs3GugBvN1Q2JgN5kQY1YfEuj4L2m6x1cFvajSTGoG0xG2XNXI1CcNBmOcP0gUXNu37mJHYo5jCqQu9pkcSHB6qERgoULm6bmZytKwWIoUgn0LdGcfIb1BYy2Uj9efuVl9BTzXVyccsVOiEIFKHplhn9bSgVDc16bGoEpHB4xlFhAV4G/L70V+LUIYHVQrSULGOm16+5DxIKQaBfnKUZrFwts09598+YeBhdZsCeiyh62weGKgEardoWrFVVBM/JQeFaDH4wb3CAEzcGuZ5EaJRWGJD411DI/4qjpxXBQM/qvGQ4gmBIDVHQtA6OAmOk0AiJTKvIYiKKqi7VqoBKSq4ERATYjvEiM0bbFQzF6MNwcLqKhfiih+fzQ9h26Lj/XGU4vkGt2Wski6hMDzyUhE3oSSFXw/A0/qn3mAdDkyaxNgViIyHDFfrlgXpSPETKXLLVGTw9TftgCAoLL8rYaI4LRVUoyVtfajo0yhYxJQQxA3xXYHPOYYuCyrJTrhukVcnQiEXFBUsGfUq3TBVccaPKECDYON2bAUM2MVI2qwNi01iw37HsP47MKm+bAuKmGDNXLnKxV17FKpR4My29LYEw2EIf7+wznS9dYSvXUE77xffCY08JbacljUFcDKNrcR4MhPIW9B5MUBH7+i1/Bt7/9LQDA8aMzmDy7fcAuQVyGgwEePUhj9523vovzo3RAukUwwZOTcyxpwZ4ph6ePH9K1KoxJ0XQxX6IhzPuoqXB6lQ6dmuZPB+DRswRDMcKgMungO6gnMARfa1ctRj3NNzq8BbXCzZvps847bI9foGsd44Qgv0pJhm/2VFYHwJj+4XDIBuZbWyPcJiPZvYMbCA9SP88uzvkA+KUvfQk/+GGyJcmHvsFwyMbFVnZl4fUGmtTbrC0cOSUVLxLZq3Pn8Dpu3b0HAHjD1MwNG44ajAiiKqOGZNU/yYe6/Gz1bgUVedLwPAgh8Oumrvm1EgKWDi6ZqxCjx6pNc+n287cQ8SUAwMmzZ5iepQN219oCEcn04OgAUaScy+Zajn0xFj6gUqrg6fMBMfhi0CrKXDdKcqLIKKCmgGTQ1BxkzIgLs7uzg3/v534WAPDFL7yJVb4vnYUmGJJREn/6TjI3XjqLMa0D12icb9+7jbu30/xWtigLKiOZSyllSQRJJYrSL/FGIgyreSIusaJrNB2KuphvIfuswOgxO0kBX0PcicHAFMWwIFDn4MBK1Jqk24NCTwqNsxlBk/qOAzlthlhlftEgYPca2d1U+zg7TWP2yeNPUNMadUUQ6C2lOWgAYuLSINnN1KQy6F3EgjjGUko2XN60H1/TFEgqLQvMW4j0PAJAXDegLpBRIeWnoNnpzQg+9EsJlSkXKq1Z6btViSxVks0HwIe+AMG/ESP4ufARzEWM+DTvd3p2xJ/P/ciJ1955tlnQdYUmQ6ariq0usjWOUCiwVWNQWYotXI2K9p/GDBAzH8pomPycZB5Wb1l5dzTZw5BMwY0ZYJQtlYxAoLUuBrcGVc90Fo+sQe59WOOBO77IqqrRkgK37TqYrAq5k35jOGxwSZy3i7MpLKkF37h1g5NQnV3h/Q/SAfDk5BQdKUTukrH7O2+/gx++8wG9twVD82M4HmAwIqsiCU5e+RA4Jsq2AloYTspZ6zgRNGhGGBFE3AvJPM1s8l6pgJoORAe1YWP0AMEK49PpHBUdbJvxGJqgpld0rdPZDFNScJxMxhjTZ0MMWBEM00zGGNFBH02FjmLLKivDxxohH1CMhDGU0HRhDRZXCOlCOliRD2/p+sajMe7cuZeuL6ZDCgBMhmPs7qexHg+HvK5717PlieL0RGQFeiHAhyYlBBSNudSqwEGlgKexzCbjvbOo6fB//fodtESxOd3awoK0NHrbkxo7GPYMGcvhOYILNFVTMcfPdh3bDKTnL0M/S7XGZuhu8AynNEqjrjMEe82eQIIPZDlGGA2GqG+m5/fwYB9dmxVNNYaU9Fz1K5yQLVq3WGGLnttdokMNmwqjMcFIr13DYDt9X985rLKi9qplO7haSmg6vcmcENeS1w6lBfKxd76y6GkM+lWLnuCn0gOCChGZY6y0YmuZgMiKvgKSYx9nAyztzR0d5vvOMQVHRMHFoXpSVLmjC2yD11nLLgw+9z8WeH4Ugs9evXVsvaGU5Ptvg+fD4I9qGwjopm3apm3apm3apm3apm3apm3aT0j7bAgonVIhJUOnYgiceatNA0+E4OBdqQoAXC3Lhn/eC65+NHWNqs7mqsCKIESdbRE7OsE3TTGuzidkRCjKhvvgYbPKZiwml1U9ZLN1wGNGlakMOVCmQkenaKsNZPbtURpR57KzhaeslGR/HsHQGSccInlsiegRqdAqTYCmjJ5QGi7LRtF11zbCx5y1rGCoEjYaSeyPUparFhptTFmdFSwuSTQkE3G996y0aIyEIgGdEBxneF6692LybwGwvbOLORFm776QxCb29/cZPrs1HsFw9VSzUflyucS77yQhkPsffwi/TOX82UnK2s4XHfZ3CboogVPyFzo5PkXflcxn9v5ZXF3CUnZjkKFwweP2nVRx0VFge0RE3Nu3MaAKmgDw6OMEDb3cTlm3yc6YM7HjrQn2r5MvUUjZTwA4eXaCAZHLPSR7PG5T5fbJ+TkekMfiwWQbS1JgnM0+wd5+gneMJyOcXKWK4qvN6xgOUzYqixkJpSHoyZBOcXVaCM1mslUUDBWSQrKowpCqk4jPc2UuwhaycXTIGXoRFMAQreLJmSEpIgQImb2Gim9cjCgqjtqwYAqUKgJL6ypR9H2jWuPW9QQVqtUAw0G699OLKRaUfQzsaxURUZ5D9gX1jisCQGRxB6UkZ7Syj5cIDpF+2wnJmXUdLSTl6SolGYYuTISibGCGg1VaYb9K83F3ewuKUnaNBKqC+MEbP5WegU4IDGjMYoZVRgvS04DBECGPo+BbgRAjV8hUdNAy9Ztyr+iCgifjdmkX8JagnE5CUOa5QmCoGIKAJlGWrCQqUHFmVAgBH3IWsWOhB281ZgR9zjD269cP0C8IQjweQw8JQiQXECS+UQ00+4yeHF3hkhT9sqokpOTMtFICc/LpvLic4rmtPfruAUPjV8vVpyC+m/bjaTXtn0IKfl4CPKhoABciF+yMUWU/XhNqkmvv6Qyj0oIrijEKFAHPwLSH4AvMKHJ2HbzWBwC+L5XIor4iGa6qZITLAhbZ5B2CVcDtWgXQW8OCKzAVFK3huVIplUCkOENXhmGCIZSqnzQNlKK9VEuIQGtJLmlIMOJImgrg9VSjHuWKY0QgIZbOR8AVn+T8bx6PEAJCzuADGBGS5CBGzFc5poiYEPLneaJFbI2HeHae9mv/9JLX6VrVGJEQSLAOZySA8uzZEaZnae8dUIyzXMyL6I/S6LJpdrvkir6A52v3LhTlwEjoqHrACDClAo/NaLKFLepzUApjqsjt7/Y0Rgp1NaRb1bAvbPQe5+RF+uzkGdNYohRoqYwcFnS/AzCgeOj6tesYUUw1nU8xJcRHCA41VQZFFGgJcZZjRgQBz8Zsguem1hE2VwBjLJSGOkIQzDjHUbqq0JAyfIL5pjFNvqckMNf1BTqqwJ/Ja70Pjo3npRK8P0YUepRUmivfUpfnISM8bPCIVGmqmgEOKN4Zbe+gzZD/rseiT6+zOq3t2iLyhuJ5XWvNatK9XasAhsD7ZqZiCR9YWT+EwGgfLVJMARD0PKuHxshqqTU9p3KswW8GwbFsJRUMxUwxerxAKKxu1aGivbfJqqNaQ9GaZ+qaUWt6UHO1VQiw7/eg0gxt9VQZXfU9PCkoChUZAZbgmNR/oVioCihVzIwBFRAMawbimt+zLZ6XLrA6aJ5L0JHjCSlUgqEjrbv5PGU7z/BfKQo82TGUv6z18s95sOeYUGnBgm6IYs2Q/i9vn60CKsu/+bVzlhdYRAedJZeF4tJ1HlgACKzRqjAgHpMUEh3diBAjrMuS9GvmtX3hDkgOUiU/yELIYuiOsvkglMkLIbBNm8SUOICiaqBpYtqwQiTIgNQakiCgumqAzHchnDliw3zHIARDPaACoChIlZI3Wh8DepvVnGg8vGdOQlXVbJ2wPRpg3OTJsURLh4eZ9Zh3n96s06OWg3rJJW8lJbbIquHazg6qOisACt5InyeooVQKA4JPXDvc50C8MoX/cXJ8hPsfJU6Vtz0bz+ZNW4aAQA9YEIqVjJbLFo6McRGL6qmMClWTVb7SmM4Xl7hPRut2aflhe3Zywmpftrd4/OgpXS+ZvC+3MSWp6WbYYPd2gtA553B+lA4rzx4/4zk2a1cY0KZ1TOpore1hJmlD/dov//vY2UqHh7fffhuBkgLW9/jkUTp8Pvu//gkm4wSPfekVWogqiZ6gc1F5PoQJhHK4EYIfcF4MUFTJtKoYQoJoEbNyqS0WK0IK5s6EGNlAVlX59xxv5lKKctiSa5YtUvImHqRCpEhxQZuF1poVq5RUDHnc3tniYLOuazaQ77tyfXmTSeqQ9BshcHJCa/Up2GqGjeV1SgTL6qjpcEjBgesRaHwVwCauQgTUTeHjAmlNYfgpAoTN0HSHPitnSUASj6CWBoL6bQg2p+AKL0OsGWyHEthJgK1cgrewPm0unaeNOAAqJ3yEwJYhXl/wcHTIrYJgE9cuOlRkkls1WbFQIFICx/UC0zmpo7kV87aGzRi3bqdAIMOQF4sFhqOkFjscDhkO6MKwcLJwhRWpAutK8Zg2BAUNTqLj/S+yXYz3AacX6bk+vHYdZpCeJy805u2/jWmwaf+uW04kBVEgQs5H9Bk+5kOBiQrJh3RrPdj8PMOalGCop9aG/8P7sMbPE8inS+cDc+fyM94olXO9UFHAcqArmCcoZeRDW1yjjARkCF3koD39LAVAfVG2g/dQRIvQxHmXUXBiVXpbLDK0BPIa6nrUpJbbBw+XpbSJ7xNcj5gV3V2PIfXJ1CWxJ5WCy2yJ6PmAndVWEQIfBlNcSMGrURgTfHow0NilPkllGPp58/p1AAmet7edEo1KKkjOE0U2+LbWMjdFQPO1dC0pf8bAXEWlNVrau5eLJR8GYyxUAQhgOycs8/dKjUj7TAhAFqW30w5Xy2JGbQl6ycbiUkITJ1ErxQmE4C2uiMJxej7FFe07rfOcUG07spoY1Lh5M61tL957kWkAnzx8hI8/TJZEx8cnbN4tokSg9fKVl3+Ork/ksxiEX0tOBECxND94rmuteH/Oh4uoBCRxM71z5bDoO4aReutYLX/9+wTHa2sJVik4FyJQXkuAD15SCRCdnG1LnPPocxwtFKulaqMZrmraHiDOt80G803DMZqDZzXYQV1jSHSKiHLAs7bnz+c5rWLk8VBKssJosC3jvwMc28/IkBIbADj2M5JJH3A+MPxUSZFvIZQ0qDMPr65h1g6gQILftqv028vVFcSMxlkrBPqMtWuHMF+Uj4PNHEzJhQ8hFau9ahNKMkRF1Ez4i8gGOZJjgaLH4WxgxdbgAl+jFoIPeNn2I2iPfMODKyhd5wIEn2MK7a2qPGpL93GNMpOpIUqtcZBl0WSAiJ+Cdf7bDoAbCOimbdqmbdqmbdqmbdqmbdqmbdpPSPvMCmBPHsvRFHhSb3tWVAzBc9UsQEHQ1wnnWP0mm4JXtYEg0qi3jonhQoDrrG3nOCvlnQPxL7mq0NQD+Jh9NWIpO6/BxlQUiGz0HtDQid+RabaLnpWCrPeYLqbcEUkQCm9qBFLrNFl1ahihZVY2i5CkxBQ7BZvNco2EygaPbYecd2U1xBCgBamOSsGVglHVYDLMAjQSjqp+7cLj+KMEYZVrJfY8XkZLbFP1oKkqDETOYCkuWbfOYmsnQbcOCdZ3cnLKVacYPJvlGl2U3i7OzljsJ8YJKoJnZtGIEJZcbtfNEFmBTEqNlkm+guASQNdZ/On7SYDjy1/5CoBEWj8hpajHHz5m80ylFFd9z0/OMFBpbPZ2EgT04mLJ1cCTs0scX6UK4M0bNzgbNB6NYXvyDxzvMBRoRSX2G3eew20ylv9rv/4bePfdBB19/PVvoCNIhDYajlT1Tp+cYWeb7jNl0K117NeWqt50n4WCpj4nX7v0vlaCK2EMnVKhiCc4Dx/zc9PD0hzL5X0gEcYznJk9dKCLV4+QTOjnqjgSL9vTXPbKQYh0P3PVLCk5ZiEIzZkrKQV7gFaVwWSSK7N5bgeCq5JKaBZwMYZVfBM0NL3fdT2bIu+QslmjBcM1lIycZVwuF7AkANUuV4jZ5wiexWsYDgvHYyARIUWpdjqCdislUVHVQMcKjkjiOhOog4SLOYtalypu9Gvj5ArywMdSWSMPSGOACPLaQoNtgm1tX6vZuFX2HguCpVws55Ak+htB/mZCYE7eoYuZxWyesuVBWYSYjaMjtklIC6TAG7sVhCGV4VoAtPYKq5G1QZSSXD0dNCMWZMJeuu5nHz9lLzOBCMiM2oiY0n1rlgvcvJWUULvTE8zbDIB7QUJ9AAAgAElEQVTdtB9Xkx3tVVXHa4lzQIwZvlbTTAcAAcuelhHFS2pdwS4/q2yvC2sjVzqkLIIvMQRWssxaiyEKiFC+I1cb1kQcoaRkryxhJSslOleUB3NcoFSBHYU1RWEtArJ8Q8g+hsIwFF8IyXubtUCVjcidg+yosukCJNo8NOn64FncwS6WEBOqDgCQtB71y64oIkJCUfWrzeiAEJBz6kJKRjOpSrH6oDGDgiCAxohQOxke1jsLRQqv470hdJchcB5np8/oujwmJIIxnw4ZgTFfELxTalQkGCOlZMVnG2L2pYaALKgSeJwv0mfmsyX9nWFajfe+GFBbhxUjvWzWqCOFwoRWybQZ5wpUGGtCfQG+VAy1ZEGYISGDtvf38MLd5wEAu7vbXAnbu3aAs1mKF5aPOxaEyesxAMSshFlYE4Bch1sKRiMKUfZjuSZkohjFFXlvC7DwLoulJNoFkKCLrGIkBMd8QZSqd5k0cY0aVegZPhaBEAmJnkR98nPY9bbsHVoyfNN7wd+njISx2VMvQ5aBSPugRmT1yrpSqGl/V1JyRc7ZjmOOrEApRUSt8hw0Cf8NILgaluDQ1lqeg0IXBF7vs3hJ4HFWWjKFQmjBMCBpgOwtXIlUeUz9o0OAUAi5Ch08Ak88zXGo9Zb7FyNYfMc52vNl8tEGEposrykCPVfk+t6xgGXrHGxcg7+CngVeHz1sV9BPHKPXFc8lzfdbl2coeK60+hh5/EOIyCGbrjQM3ceK5onN+H4kRF2ex1IJPhdZXyrLEUWA8Ec18Vklwn/4O/9DXvILdjmGJFsJQAb9Kf4H+yNG/Dlp+BzwCnxWS1K2Rc7UZw3WDBmDgMzw7RARCH4a1drfCf44ZBT4B3//dwCADTp/49d/A6+//goA4A//1T/HmLhm//Hf/Bv4hOAFi+kCf/UXfg0A8OF7SQlTIuKPv/1NAMB4NMFXvvKz3O/sFBEriY5mR2Mq/NM/+OcAgD/4+r9OfRgM8OBxghQqJRAzfzJY/NzXfhEA8As//yv46S+mA9JoNMKSYB9/5+/9F+k76gpj2jS0kgyf/fiT+3jyNEElpQDfLwgUCXAa/2Fdoadx/Nmb11Ct0iH4svd49Us/AwC4sfwIh7/wnwIA/sqv/Bq2iRv4m//jfw8A+Mf/+Hdwk3hYk9EIL778hfTZv/qrmNOD+s677+AP/zCNwYvG4p8dkzXCmp1GVWWLj8hzVRuBqi7KmdHTxpGRtLEDoXIw3mqws5X6sTUZsZKi7TzOzxPk5NmzGc6npJS4Nt/LnI7QGbqBT+OmWZ1yjTvz1//aLwMA7t57AXWdDkRCVfj6t/4RAKCpBricJhPU2fQcgg7HIQLdqkBlAaAZGIzHBN3dv4dnpx8DAPq2xXiSIKe3bj6H555L1hMv3H2N7Siysa4Pa6braxw165ZQKs3vVTvFiOB8Ahr/8Df/G3o/9WdnMsAWwZQWiyVWdIgPskJWLG6EBwiqucxwEq0xzLDtIBnyY4xIECykDSHLHjvvcI2MYvf30jw4Oo0s+wygwDiih6BnvDIVHHEiVp1jTvJDRbwNrRm2Gte4OECClwDAatUXvmWMrNKbzVedKxBWocTavS9jKkThcSSV1Wyim97ruhKQb+/usBpirQzOT1OiIrjI0EulgT/9l38/fTerTxdbDISiOiqk4Gg6JjIX9SnDfwvHKsTAa68QRQ4coiQcpBDMgfaUBIoxQuvCIWC5/1AO295HhufHEFk98YU3f/2zF/hN+3fW/pf/9b9LIEldDmNCyBL0CiDnd0OUPFekBEOH8n4QY1l7AwSrTMcYs698fid9JkQYWQ5t6b8Cm1kLgGHv64EYQjGVDhb43//n/w0AcLiXEnuvvfY57O4Szw0Rd+6+lP5OA8+OkkLgay+9iOeIymCIJ2uXFk+ePgYA9L3FzkGCD+5u7XAyySjNieHl5RW+/c1/AwD49re+DgA4OTlmiGLbWjgKaAejId548/MAgP/or/8H+MLnk+3MaDKGpETbf/Vf/r00trrBiBKlVWXQE/Ty5PQER6RwOJteMbQ/qXxn4YR8qIoYUvJ5ZCK26CB358YtvHQ3cXd3dyZ48e4LAIAXX34FHxOX/fd+//cBAH/y1g/gKDF58/YBvvzltKe/+fk3WRnx+2+/i3fe/T4A4Gq1xIcnxK0TRV2WDaghywHcCFb2Vkb/BfP0EMDaBoPREANKvNemWHw5F3BFKsLtsue9wfZ5X3O8n8VQMg/iz8Wq5dwVOFn9n/3G30z9rGpImv+XVzM8fpTiuLPzYywX6b6s2hYtBfBt1zLcOa+3w2HDB6WqbmAJ7md7C0Uwy+3tPTx/N8WTd567h93DBOUdj1O8OZ5MMCAa0co6PHqYEvrz2RQTgmEeHX+CjoL7a9cO8Fv/02+m8c2JfqmZ9uN9ZNhyDCVx6rxg7YusMClFgMsWLCh722BokBMVy1WLZdZs8J73ysy901JznNH1Be6qlWTId+/AEFAfgSxjPxO79Hceee2QUrId3HqcmqgXmTcZynrFtJnI9BggMoUGiHx4jrG8L0RJQuW5myxA8t5WfiNCrKmbB6Z6GaPxR7//36bxkC33jXl4UiAXPpQAJ50hRFpzkeYmkOIJrrcJxQlsIfVaUQAc5DrryjqccyiwXOBIuif5jNXz2iYE4KhyZpRGR5zeN3/x7/6le/MGArppm7Zpm7Zpm7Zpm7Zpm7Zpm/YT0j4TArquEsYnblHU7CB8qTTFyNUVIQRCrhLmbPlaxhpC/KW1wBBiyQ5AFYWufIqOEZnZKxEQKGUupODMRYRMnkVI5OBc0r68TNCB3/3d38HP/8JXAQBnZ0d4SCpKz711Ey152rx871WMiIidoQpKF1iiEEWtTECyIiWkYJjM1s4Oqy4pOmcrITM7HCGuwSbEGlxEFQGOqqo4i58J4EDgUv7NwwMM6pQxvX39Bh5SdfGt732HBT0AMCSmIwjdwBhEymb94OgYhnz7Vi7gv/7bKZt5/3f/AW688DIAYGd7wp6FSxpPIyWGBA2cdxY7o5Qpc9bBUtbm4uIcCxpfG0uVqoh1AH0WYIyBM0pCaEiCwWxvjbBFPkzZ0FMoi+2dlD3b2hpgkIVmgkZLoiEPH51gNiPBEuuxQ34yOaMUQ6r4AEQezopuIZQSckzzCUiE5azKdDVP2dy3vvOIjVO12S1G5IMKbUfQ4ig4c9W2HTzB8mgYAZSM0qw6x/Tikr5P4ew0VRH7bsVqUz4GHBwk6GrO9K3nRVerKR4/oUr28hzXDhOU5tGj+7h+PcH27t39HIbZJRU521kM5KXRqOj1qo9MnK6ExwGponVUMeqFww5lX0+XgaFMO5MRXiYYz7PzKX74IGXoV9ZCTNO17E2S8M5IdjCUcawqDckqhJHV/ZSQWJGX09PQFTwaNedcqegGwZVl5xzDI/J3Ahlt8OlVKIRSNZNCFEEqIVhoY91b8VPft/Y9ZUkskJ/1n4qxKLlCAP0q3eeYM+DCM6xFQLKghpSS14yU7cxfmn9QckVSa8UTY11QQ2uD4Eq21jJMhtZVKSBI4dh7u/asFl855wJ8VoENQNcNsWk/3iZ1dijWwBrUO6e9VRSfgsaxuJFHUe3N8CrIsoiEwHD5GCNDo4SIKAt0USXMUKbkAUovAximmYzqaa5Asm+WMQHLRVqrT32qPm1NnuGK9rm29RjtJAREMxxiTtW52oywR+JdWR1vZVc4PU77uw8Re0kHCQqa0RIagtVyR4MBqgzJlFmUTaHqaM0wHoLUe5ULAMHhhFdcaRqYERvLX5C3bIDBapD6ubW7zaij8XCA87xWB88erBCBM/45TmrqChNSyd7SAUNWQo+IpC4MV2FA/Ti8cQ0kDIyaoOfe2iJA13nojMKyLaaLOfd5saCqnY+wIVd88pqninekENBE9WmGBsMxefgNGxYkk1k5URSEz2AwwGhEwnqyQANt70o10DqevkWVVDL6wruAaPN8DIy0iAGsbKykhqb943t/8l0AwKLtGTq6aFuO/6Lvi6J9TJA/gKqO9H5PsZ3rOwyzIf2gZ3hy3/eQpBLftQEtQVEfPTvGKe3fOc4YTWpcv5aqgleLc3z3j/8EALC8muP5u2kfPz49R0dqryK8xsI/WZip1oqrsavOAi4LqsVSaUJARXSErIRqRGD0jlIGgo3bWc8JOoJtfGUQRdU8ZF9IWfwzveCqqq4Vw4l757iCLaRi8Z1V1rbzocS6CCxaJpThex5l4IoWsLb35n3X+zU4Y4GRxnXYNSQYq+BDEYzMqp2xqK8IUdSuIwTvZwGe1W+F89CC4u6Q1zAwNcTbtS4pBZnFroRmmDcrhovikxqCh8z3RaB4CYbAczZ2ASuiiTgag8YoVHQ2CKLMfx89nzW8jywI411Ercoe8Je1TQVw0zZt0zZt0zZt0zZt0zZt0zbtJ6R9ZgWQOTBBcMYPUa6lDApvBwKFkyIVe/7l6kdcw/UmNmAhDedMvIgop1d4+DV+AQBoFVli1jvLfRLRQJBc71o3EILHgnyxsnDNYqHw0ccfAABc6HHzWkoXHp8ewREOeOfwGlTmpoksuFKyrEIIrr6EGBEp27N7uI+a7BX6+bKQQmkMjJTsuRcROMPTB8dZmz/PMcpV2EePP+HryBm2n/7CT+GLb3wOQKpQ3vvqV2lMgW9861t0jQEHZHHwKz+V+AsHu3sYVul7Ly4v8X9/P4mzVMJj+SRx0OqhgaFrD0rAksdMO0/Z2m1TcRVpvvAwJA4Sg4ClTOXF5QXmlB1zRkAJqtRlsQ4hUVNWVlcGw2Gu6jWYjNJ3b08GGJMNQVNTZmukuOoqo0Bvc0bJJRUOAM14iMNk54fdXSSyFYDlMs2D1aoHJVgghYTNdiafmuuKJ5PRwIg4cvc/eQ8AMB6PcT5L/Iu63oKi8bqazrhiV9WGieRKNWjJbzBXAuezAGezt94Jz//gA3/Hz3/1F/HD938AAHjnnbfw6qupf/t7ieuipOb5eXr+CG9951+ka1wucedO6t+TJw9xfpEqTXfuvMCCKqNBGpeDyRie7lXTNGiXJG4TVpy9M95y/7aJuL87UtB03xaXHoYqW1ujBkPir1wurrDq8xgoeMrInc3SPDkcaxZkcUKwvPeytwikiR2CRZ8lqAEous+gJSihDkrmy/tiTZCrgVLGgmpAqd7lppQqHMGsSJQ/l3kEvgjCrH9X4SCv2XAoxaiIdQ6pWPP4ESJyOThb5kQREMibUETBGVgoydlMKQRXdfLvBS/hsv0FSjVQqiI73a48+0utVitMpyljnTkQh9cOsH9IXFEZgeyRCL8mirPG5QLg3GduI5v2/0ErHBLw86Khke3yZAxFKAESMeZnoJBjMrrB+8BVdyEll6ud95DZa0/KbFEK7wULWJT1CnBEANaiWBXBe1TZg81UHA/43qKlDT7bGDx4doQB+byt2oDrz98FANy8VUGLtK+Ot7cxGJMY2HlCWTx5eoqPP0kcr/FkgpczLyeiiFZ4BYssCFOQSzFb1ZgKdV2q8mZEFbsY0dS5sgIWuvMABAnuTOepqhaCZN8+qyJuDZLo2nM372BrlBAo7ymF05PEBV62HYbDtI6+/kri9D135w4m5Ed38fQhLs+SD22lgBWVVNrGsZ2GVoYrm9lLbmgGjNqABx4/TLHD1ewSc7LDOjk7R6SYqBpWzAXOW5+WxQdaaY2GED6j8QBjqgDWTQNNnPs8RnVjeG1TyjCf+FPrJmSxFmqKVJE3Za3M4hrOeti87jtVkE0icrWqqiUmk9Sn+w8SCupq2cJm3rnwHG9KhOJxF4tAwrpdQ6AKVev6IkK2ajnuiiFgXBFv/+YtvPq5N6gfQ7Ybm16kqvDR7ASnR2nfPT56gLe/+73UD9HDr1JserVaYUlCWpOthqvBVRY/NAYqV49sRM+80Yhc8dJQrEXBHnpaMXLPaMX+cL13XLmP3kOxUM/afpj3nxAZ6aVR+PlCKViU2DhzbV0UvE4IRscIRq19qoobi2hcDKVyJUS5G3Ed6cMvA/cjCcwRcgkRBexTEIlSlH4Ue3LF8bfzBaEj16w6IsC2NOxdLMs5xgcHyf1w/y97b9ZzW3Kehz01rGnPe3/D+c489OnTA7vZbA6iZCtSFAWRZTiMEiNxZMCIEScGcpWbXAbJPzASXTkI4gCChziIo0CwYNEaoYEiKbJFsufu02eevnHPe6+pqnJRb71rH6rJ6CZ9w119cb7+hrVr1apV7/S8zwNBElPSOC4NBg1fWytIGUjsNCzNyfcDE4FLXbGm5GQyZ985uClZJ8GACB2zdsbxGYRsZCWqkv2M0hq48sdXAP9KAaB1msu6zgFCEXOVHmz+NkNOTLmhNRTED6XkQ8tfp3koHPTZ5kWFkMxgFIyddSUiCgCTTCBR/qW3lcNiNaXfreFEwBVaFAQJbIJPi5OxP4DjRCJKvPjkdDFFTYFE7QQzZ4U1KPI1s2VuBmmtLAMRRcIpgQcPvSFSleVgLwR3Skhm/nRSItT6pRRIqPHbN7huOKm0wd78wpsAPKzy4SMPp9NKsdaT1oodtF/4mZ/G2Zk/gJarHH/3538RAPCLN3ywOD8+YdjF++IJ/g18cKFgMSXjlK8LVLkP+uxqibe/828BAA/v++B5bh326OXo7ezhiASjv/d7X8cdWoPx6Skiciwu9Vvo0J7IyOi1sgw7uz4At8awAHEnSzEgYpQ6zyGDUxNYuqARpJdqK+BC8CYrlERqYlAhIRhpbAVWQavMNexWMa1dkkScbLDGNux4SrMGUTuLMRr4OX343tv+d63AbBywnMeQUQhADGJ6R8pCYU2EL5HM0G/5FziwlR0+PsP81M/pRMyoQduzerUJ+voLP//L+C/+3j8EAEzOxlgQLCWnl/v4dILp2DsKdz5+H8+IFCBrZbh338NSBRxOjv1z+Yu3/pDf1YSg2uVihSQwPkQKuQgBm2Dj085aIb5GK2oCnifEINdOE+wRFMdA489v3wMAPDuZoEv3tTPKMD7xDtOUiF96nRjtnv/544kB+YAoiwK9pIG1lFUwEAZRYA+zDUw9vCsCioNn/542+MtN2Cez824EZg15UgNDccaxhRCiIQlq1I2eH+EaSilmXg1z5C/ZkApYgvQEFkUhLdtiZx07J1VlQCgfyEjD0fvHjfImwumRD+gePniC0KCephFy2v+LeY7TU/874/EZpjN/bgaI0Qsv3MS/90s/BwDYPxghL/3vGlsEYmHEkWKob1UZdjy34zMcBO2xUjXi6iqDDdpQkFBBM8oIVCEAjBwihiP6S9XGgtBQ0Eo1gaFzCLwGEoLJEerYQuoAbQpsdxGECOReFo4IUPLlArCBWVJyUFqjhGm8PwDAYl1gQQ5QVQLLwPJYO4S97ESEipJnJdmT2aJAQWyZu6MOspgEwiOJiLhQhVRQKrBaC7RbPiDLIv+7c0wRBZ3UumTyhjiKkdLvqqTDsFkHheBtBnhkVdbcfiKNQEp2f2e0g6sUzO7vn2MikPl8hvPnfZD4ZbLv+7s7KGZeb/N76xnOTg4BeCKNekVw0b6EJdzn+GSGu7f99Q4n3uYbIWFIz286X8I9euLn//iICXmyOEKPEnfDvQE+JH3XwHAtpeTgX8caKfknnU4HKf2dlGCR86BfppTiQEKoRr/RuJqDN8+YTeQ8sWxIMPxlAQsYYiq2lUWZULLBGGbG1FJw0nM46qBNxGLvBv9L6Y0EiC85AP7ZhxYmqTQa3g7NwVJOQV9Z5By0FnnFZ6QUGv2+//rmjev46lc8IWDaasMYb+unUx/c3b//APcf+AD8WeWY+TZfAU/ITs9mU6yIebzb7YNy88xg7/+lJKwyMAE2CQtNQYeSQBI0XYNfE0lEQWdPRexfFUZgShDhWCoo0qAWSrPfG9EzjJVCSs9WOoEyBDyyIQjT0sJSEkI4izokJuuwlxQHg27D33cb3TZCCrYjQqBp8eB2igam7hpz7H21EJBtft+hSUKhIUmD3JhTeJ4brRpVZcDBjgQM+ejM+mrDSQTWfQ7zDNB6C8f7xnL8UGAy9XvjbDLDct0wvZZlA9Feh5a18QwlJTBCYWR3f4jPv+Z9wvZwwGReplijpIS3UzHDTk1VwtkmEf5pYwsB3Y7t2I7t2I7t2I7t2I7t2I7t+AkZP74CuEm/S5G1Ugl6O572VqTnOBtubQNZM6b2+idoqjaeojxE+E2Tb/hbwFfbGPUJAcuwzgAdqKAp85jFCh3SAljPx5jff5fmXAJNmy/aBE0MlMx5UUCR1g00cHTqm4MH3TYqysKMT09RXaJKEsH3pAVU5KEZSdpBQhTUcZygojKvNhqdmCQJ+inDREMlUGvFkAgnwekKKTTSiDI5QiAmeISQkiEZL1AGMXvxFsov+HmOhgNu2u512gwZaCUJ/sv//O/4+UUaByNfdbr/zGcTnx3fw5Ke/J3pKfKQPXA1HpLOTq8uMX7mZSWOHt7F7/zhWwCA20/OaGUNfnDqs1xqXuP9xx52OOj1cOn8RQDAl1/7PCSV6q6sP0FGsI6MYLIOwHzuKxDtNArFORS1gUy7tO4RCDkKQbAc2VdoE/y0LmuUnLErkec+yyIgeL/lRY2zU9+cnxIlc5Yp2ITgE0ohCvppCpwpy1oJNFWa0lQizfx+/Ivv+P2zWlZc4YkSjYIqWlmWwFKqMpIaF3Z8lredtVBQc/V4TLTC9QnTSwMNGUeSpui0/J6YzmacnXzjzS+gIPKE4ql/VusXNQ6n/rkU6zEEZffu3L+LVekzu5FuaPzffe8tZLRX4lBJLUpY2ptlXqCg11NpxxAuFUfoEgw3ZPgPaweZBYhIhQFV7MbrgiuUnSTCoOsfYhxHmNCZUNLzGVcSgt7llSyxpjVNZIQOyY/kZYU1wRvbiWaIU0TrWZZFU/Wj/wDK9H1K1U8p9Zc0GZ8nhRFceROeDYi/y7B2Zxuo6KcQw1hrmYp6U44FQjRkLmgkMoRtYJWhmuLQEEsVZc3QEqk1Z0ktQdHGZyv80R99BwDw4Qd3MJv4PS+kQLvToWlIJj6oygItWt+EnuvDR2eYLfx1r/cOsD4hCLctPSQaPhsb5mzt82f5dnw2Y0XSJRYVYoKmq1aEiKCGcGlTDRQJYiJJU8JwlSeQIEAIKBm0HxukjhOSERPWNvq6EAJKBDRMqBKliKlKpKWDIRmZ9WKC+YSqWPnZc+9aIIMIlYS8rKFcIDlQfr8DmC8KFHS91apgCYEAO+u0MpyjVo7R7h50QNMIMMrDwkHpUNWIGJ4eqly1saiC1nFdg7FiSiMle9xudxieB1sxHPuF69dobRMMen79250W2vRutdIEXdK423npFm5d97a8KnO0SApg0PFtGqZc4Wzsz/WzyQIFQfFhK6wCFE9HKAlq+v777+Kb3/NyDkcklVELh4D8MhKoZ37tYpkjITj8oJ1gjzSEDw5GUHce05IFFIJ4TttPEeIjijTiNOLvB1sZ/DwvyUVnmzAMnatthaIMmq45yoBVFg4u6LEGuLFQXEnVUiIOxFZCMwxTa8F6eK00hSIU0HdzT4BmarAP6qGPDbJM0Fmtooh12hIt+TmvqDfk9KTELF/Ss6og6R2KoggV3UuqNVfLpHJokc5cRvZMC4Ne2//dfr+H3b7/+Q/efgdPnh43n0fn+ng6Qz+QrtC5WleGp28dIELF3yqugnoiRLJtDOSRUFrw18H3tGjkgrQSaKehDUeDuPCYrDBSFprIRgqUDSTSCmzWVwOE3FrZkKGEn5vmf/zeaBB/UgQkgW5QNmjQNWyRnWP7jg1dQU9UFX63IUbz6NiwfwMUNOLPcM4xOaK1bkPiqGL7rjT4OYugwygBGaqdUnAF3NdjXXO9gCYkf+7+w0N88zu+deiju09RkIaiM4K1MlOtSC7Cy3M5unaHYNYv3LiKm7c83BgqgaTzx7qigTCrBsIKLZjw7UeNHxsABkiSgoFVhGUVbRjpDzMrm3Kj0AIiaIeggRTxgkPAMKuoYHZNvxkY78T9CVIISLfBbga/xDX93cpJlASdKs2E5ydNzth85zRKOtRXFBhEUYSqCFBVi4xgdlopxGF+dc2GIwQRcdrGYN+zNnX7Q1RkXOdnc+SE3x70u6zvtVwseIMFZzRNIiZSM8awaKgSGgkdLu004ZdzE4YW4CTdLMXOBQ9b1VHT+xVHChEd7lEcIaL5Z3GMgHPQB74p7uj0GcYEUWgfHOCn3wg7xqJHBsycaBZ4/uZ3vovbdz2M5KVbnhn07p37DK8tlksM+r4vo98fcFB058E92KUP8N58IcV87p/RkpjIlBDod0kMHCnSJNyLhCFR7N5+G8mQDE6Pnlu8Rl5Tr4UzqKuwx2Kkqb/eolggLwlCBIPukHoO6GCOdAxNJfwkFkjjEKBoPjRl5Fg0O4olw2aDcLGpyw0B9ohFwTPZRtr385BCoyAh0sWyDUiCT8eBhusetAmHpwVr4dQlJy3ufPIJuqS5OBr24agvbvGR792cz2tMb7wBAPjlr/0D/P2/9w8AAL/2a/8I/+Jf/br/HKegyfDVVc1Ms0mA/CiJdWCElKoR6q0t0oD7lAqnBCVkgkGl0Qu9otbhk9MA91LYJS1HmcZ8mJmiwh4x7L74unfalDS4+7F3ejq9BML5+3aFZr3KdVUFzVhURQVXhwM29OooZhsUQm5o4Dl+n0LfpZ/fX+4FfO57poFxmI1+TGEbVl9nHTN4bY7NnmbWIkIDd9n4GA/j1EG4OojDNgYujjf6eZRg1jEDw0Yy7MH7Dx/hnfc/AADMJjkW1GNpnYSOvbM3HPWRkekuipKd1HCvtW20o5J2B5jSc9vo4TFGsrC4qQ0zAW7HZzfq2r9nBjE0JdlqSCjtn6dUPSTaO5vOak5QCGFBvjL3wEApyNCnDMvOvu+qCe+2ZQiwEAKhNz6QgGoRQzpyjISDooRmnFgI7WHEZV3DUJJVmAprcpIkJZQ/ekkAACAASURBVHakkM275RwWY39+r0YFYtpjxlrUZXDc/N+3Oy10ez7J2Yrb3AfkrEAResnKGiAtr/VyjiePfcBTBb3TskAe9ECNRcQEBIYTq51WmxOFgATJj+HWi16jtd/qo0P99rESqANEVAhmBG2lGZJBn9bAsGD4nPoIl9MZzk58Mk8JgXPnfPJQa4090rvd3e1jRtDtT56c4M4ndwD4ABrwvcC0tCiMhaSHlGmJhOCxURyBO3KqGoL8BRkajpQIbfPQkYKkPSFjwYzNUik+4FjvDI2DX1vL8F4Lyw5yZWv2g6K4OXcjsrtKSsQhwRfH6FL/YZbEiOIGysl9WdbCkDZe0LU1tWVnWkcRi9NrqTbg/E2yI40TZlYNftlMAiUldcuqYJ096QBJEFtXNsmJ3qCPhGxl6DcdDPsMn714/iKuXfXBv3ECj5/9Mc3VISIY5mq5xk5w5undqmzDUFobw86+1GBYdlXVqAO8kRI7ytScVCxEzesshEAVSISVQEw+zqgb+0AM/hwAfHtSYJTVkEAR+ssa/9yiZjg5pO/xBBp752AbThC30UcIC2sbLg1s2GHBAZ7lv+NEo/VsnYBn/W2Ss03gaG3jd9uQNPXUsbQGsoG01zVDNcnA+88UDjX5sobWQMAhJt8tSdowQTPXNnqEcI77T5dUbLpz/xm+87Zvx3l2NGFRe6UluoFVd9iFpkRsoipO/GpKdKiohSSjeEW3UZeh/UhyAsHPMPR0KrgfiqF+eGwhoNuxHduxHduxHduxHduxHduxHT8h46/Wve8axiUHx5ANt8GohOe5Flg3DhzJ44egQg2MKnwtRAPdsrb5zGYabkPrCzCcUWqgWA5mowIoUIaMI10j0ppZGZeLKU6PfWXrysEO5quCrucQaYKzBMIb2ULvgq8AFmWJeycEtygrZr381rtv4faHHop6fm/IJeiUSE+klFx18jpdhu8lZDPjKIKlaoPSymesAPR6lMEf9NEiCGWWpYgoa6WVZkiE1oqhDVEUMaPpMPJVugt75/DOh75S8PTsBK+8cgsAsD/aRZ8+553TD3Drtc8DAP7Xf/Z/4IwyjreuvwAAOOmdwc0pA9tqM1znZHzK2n8SAjuUscuijDNyUdQ0K4em6FYSoT/wGc7OQCPpUMZX1lhQuVySblFsHVeDhNTMCFlVGrMlwTOLJRRBPPfOd9HpEikLZfSKlUBEukkXz3eZ3Q1CoLaUTaxWqIIOjGwYpKoqVKnBkIlEDyAIojOZVyin9F5skA/EqYTStMcIZmJMwZk5rXWjTVhWmE89G8pqtcb9+/f8Mxz0cO3adT/VWy/5e7n9DPef+IruR0+W+OnX9wEAv/IffQ3/+uu/4e+7XvH7p1SEvfOkTUQMZVkWocr9nPvDIebEZlZXDVvtbD7HmrJ0CWX4L7ccaw1NVjW09feVtNoMfR7EGnnu72taCLx402exvvZznpTo7oMPcfnAP8OVEXhy1+/BH/xgjLN5gA0naFO2drEosOYKn+C1CzAO5+xf0vgLvys2ICA/zAK6SfBijHkOysnQUSEYjrt5noXrbmp3Sil5TkKIJqtpLTY/2tJ5FM4wKww3lEudIg7Vg04X67WHJOXFkiFwhmAqs9kCKcEBl6uctRqjKOUs4minh6zl5z8ZT5CQnlWrHaDyhq+7Ws6YYQ1SMoGEE5YzsKUp8ZwO3XZ8NoMqHlJKVNafKdpWiAPrX5xCke2qjWBGPCUMMyLXXMkGVLDHUrEWlXVozlmhYDey+CE9HSrjVjgYF/TyHGuE5ZVFEZhpC9NU3EyFmq4hQ2K/NoALeqwCc0KKiBoQdMavV0vkBcH8RYDwx2zv1usc84m3VUWcYEaMUtOzE8yn3mbPp2Nm18wy2v+tNpaEEoIF22nrGk3OLFEMMTN1xU5FklFrSCtD1vLXS+KIzwctFWuLpWmCiDVYBV+70VtUEIQiunDpALsDT7QXJQpJqHrka7z3gw8BAJ/cvY/JgtZDt2g910gIHlabir0o65rzqK4Mcvq7mZoy0oUZ17VERJDIVjtBj2xzu5OwnyFEg5JgYg/h/4/ubsM/FAwTjKOIYYpKRoy2ilVzZjfslRqdFlVFMs0Iq7quuc2iMBVXtZl1EYKrX1XtoNnHdMyaXBQOlio8rnJcyWMNVieYtE9AoiZkSlt1MSTW0eVihtMTT+bS62Xo7O7Q+lG7imhz9ShuJ+j2vO17+ZXP4Vvf+R4AYLo+Y8b1Yj2HzEIJiv5xgqvNpnZciZdCcrvEMi+QkP8RE6xYWMkauL6LK1REBVwo/1YVM+sPBxnOjfx97dA8pZJMWHI2XuIRtdKcziusCEknaglUQVNyAxrqmr3BxGnGoCkXCq4gW+sYqolNOx0uJhodQCeet5/M/LJh84VsnrPgQwwb9qypTjtYz+4JQEVq4zIWdRXYaAOZykbFUShEoHYmJ1gcwRobgFxY05myXK6bip6KmGk+TWNcOCC24MsXkJA/b+pGLzSQGe6f22f4aVkbCATItObnXNU1bDhvTc3+5I8aPx4CimbhGFKlBDM+IYr5IT/ncImmX4B/DsswE6Bx4LWWzLpYbwohOvGpHHvhAJNScf+b28AHb5Y0BbBRkqdAJG4OsLpeYzb1xmK5qjFZ+WuMpwXufOwhItb4vy+cgI1D34DGjKAGSaSRdQk2+URhRrDDg2SP2cgC0+XZ+IQlIbRWvscCHqry7ND3SRwePsOXvhRYthT3PfWohydRCik5hFmrxXTKcRw31NBaNr0KADNZhs2ftFN86cteEmK1WqFFTmOWtTEZk2xGLZG1vANx7+FD7q2bkCDvpYuX8ewwYKElliRm/cLeeSStK/5+JxPsWP/7tXPo9v26Dwb+XgadDmTo42gr9HdIqLedsEjwZLxkls9WnwzFXgJJwZ2QggV5YwVmsNNxirTlv97d7SKjHow1BRQnT1coSZIhzRKYqun5DH2mnVaEgg4AJyrej+1WgwE/O/LP++6jFeo6QKokO+1SVRDkrFWVZrptxlBuBP/GGOzt+eDt2pWruHrjBgDgr/3MX0PMbHwSn9z2fQ7hnrKXd/FlYtxa5gK9zK9d1jlAr+MdiOPpko1nXuSoW2Q8A8V2VXDC4uTpIbN2KaUxDYKkVYWE4CV7NJ+rl3fw3gPPHDtZFchCH4fS6I78/llOZtBkZK690MEbb1DP2tRTdi/KOV58xc/5+F4XBcF1+70SKcFn+/0eJiS43MoypF1/7588pQAdjTPqnGM5Bynlc4FcCLattbyvNkfTj/zp2HljN4JLryZP60SyJb0eU8Gz2Dv8+Rh+x1rH76dDk1BQdBwLCFgQS5gzDBeROkNMsNrZYsYGc72iYCx3uHLpEs3nCeYzf972Bj2cv+AD/gvn9zCk9X0kG+rqcwe+b7c2gp2pyckxjAzndwzY4CBvQHqs4/ltx2c3ggmunYUjp7c2FmQu0NIdcPO0rbkdAVZxk5BWzR5s+jstt3XAgeniBSRS6vWtTSPCHpLBqCuGWonaJ1EBoMpXsHS2OoD74p2tNyBawY4bBDpmYSXLQwjn2NaPj06w0/d7uRVsYtLC7mif5mw4eBPS2zQAmLsTnFHSdr2cISXbllA/W5EXLDaf27yBhVuDihKaVVl7GBkAqxy3uSQRiZ1rjSjA73TE1PeRlo3sS6QA4jbwkHX/t11qQej0B4ikb/Hopgn6PYLxVhVAUkyHD57g+MyfuXfvPcWK7jdKCEqmNCKWtLAoCdqaCIErBz4JfG6UQlT+epEz6I0CXBh0LxEHs51uC22yNWkaMa2+tY1vEfoF1Qa3gRCyaWGRspEAyWJuE4mVQ5YGlnS/B5erNScs4ihClIQ1FV65HHhOKiCSCpJsbPB3hZTsCyhI1KFoYSQHD1IJZhOvKsN45iDnVNYCwgUeB+DKrt9jn7v1Im6R/NbeoIVy4X3Io4cP/XMCkHV8Qs3UFeogtxHFvGcvX76InV1vmx8+fcALb1zNSf+wf6wDghC53EhS1rBcYDEWfI8t2tutVrPORWUZ5thtxdwPuF4toEmeYNBv4eZV3ya0OwytAQazpbdFSaIwJ/j1InewgaW/qhliLqs6oEQ3mP7RnD8wDAH1MkahH1l5EXl4CDC/f6o5D5jhUzT201r7nARTaPdwaALAsEe11hxECiE2pFQkCpp0VTWyEVLrv9QeIrAp6G65mCREzIGmskBBbLArYoB3NXAw8nFCDA8BBoB2r4+rl7ztPdjb4/aj2hrkFHgHqZh+v4v1sumjDYkTA6CmDEFV1qiKwC9RscTYjxpby70d27Ed27Ed27Ed27Ed27Ed2/ETMn58BXCDzKAps25kx38IRtUwfG00BVOEHAmHhHTStATyha80jU9POXPRG+4iIZbGGhK1C7C8hkm0+Tz3XLaehRyFZFIEY0XoVWfWrBUEM4NqqRETcYcxKS5f8xWXu/fGKE6+CwD46a98FQDQ2h1ALcb8d21ie2olMR4+uOfnYSxeeu1zAIDp4hTDoc9U3iIWzm/92Z+iZsZHAxMm5yROT3xG7+jwqGEONJZ1lhj2mSTIaI2SOGWogZ9XII6IN4hkmucYMiHWWmjK3gy6vUa/MS/we//0HwMAbr72Jv7k255R8OTsBAVBae/c8zqAN65cR58qn8cnx0gJ5vir//Hfxk2qXK3XBT76zu/6+/rwG7h0zWc2BwQv6LQy5CufhWx1FTL/WBAJwNE6dZM20o6/dtIN2ZEFkxAIJRHkWOI4QocybBApNMFPo0TBEqxTx6HCXCNf+71ZrCpOHepYQlEZ3kYWVWDB838FACykLSRYi806xVBbCMDQ9UzdkCdY5yAD2x7DIBoGquFwiP/+f/wf/Hq0MiyIGODo8Cl6tGd/7w//BBllrQcDv/7DQQ8j2mPDfh/Lsd+bTyZTzGckUlxL6JD51xEKqvS2qALk0BA7QEnWxCzKEpYyW0kcYRDIiHo+Q7j7wucgj78FANiRCpcu+Szpl964gKtX/fP+rf/nm8gLv78//5UL6Hfofb9AWdusi/nUV8mfHs/Q3fHvqowlMoI13bhyFd+f+zkvVgtEVSBOoSrBRoUNEM81vPM5tpE9lRsZ6c2xWS381CEla+AJCGYRDEMpxSy3edFk37xQ/cYuCtBVpcDKuRtVmCAUW9c1yjIQ76zQagUW2y4zjC0WpNVnHPq0J4QW6PZ9ZnH/3C4unb8AAGinMTK/PbCcz6FIS3Vnh6BLUcbZ2rPTM2Q9P8/SGib8KPIaeU4w6XWBFsGhtuOzG4HAwMI0sCa4hpBHAE42xAWBbQ8GjT5agDnWhiuK3SRiEozFfIU1iVUnSQSdNMRWYJhcsPNcpIAxljVzbVXC0lkoIaGjwPLpUBIsPA5MkoiZCMLVJVZUkbPWMDxwdnyGo+QeAODClcsAgE67h2yPWDTrGkkc4HcCJc1zJh3i8O7HMfo9X32RpPO1mExwRO9y6RwTStnaYBo0dRczhr7GsUREiKAekbO00xZSOpvTKGYCOaka26vU85WOgMpQyp8ZURIhVQHR4ja0zCx/HUeKCaPmqxVmRNrU7vi5RVECRdqESaQQkZt34/wefvnnfgoA0G9Z3H3ft6tMZqcYtQmxQn6BlBJxHHREY8RJIMVDw4IM25ClqobIhVsaIsX+SRJHvDelNUxgoYVDTC0hofWlWhfI6WyLs5iZS4V0rDHohIAOhB7WQhAbdBCQhwJXUyAEV51MjaYKZAPA0+sKsq9INr+2FmsiWzo36uHf+dkvAwBuXb/J6BsVaUxPPfrpwZ3bKKmC3SEyvVa3A03V3b39ixiO/J5erldIyVfpdlosOG/KGtggUwJAWtmBMdeipPNZCcEoMmPBSK7dXV/lbbdiJjGpNwTHD/YH6BEb6dGzQ+RzD4feG6RoE9xWENrDVAbW+GsoBdbRq61jSLiKIqS6IS4M0Fz2dR0gRGiF2EDqCMFoKyEkV+zgGmh0GA6bJDCWmVA3kYdCNEgcCEEVxuZ3pFZcORQQfC9KS/aBnTNsj7WSjIoKVVnIjWqgsAz7FMqxJnFtDaqCWnnywBarcP2St8EXz+0zE23W6WM08lVXHStmIhbW8P0y0itroVh7f269jOBoX1lrGKlhTAVDDLtVVaDcYJj/tPFjA0CGfeIvB3k/PBzAFL3OGqDyhkNSf1xdrVAVhB8+fIwHn3gY2+RsjDVBoLqjA9x85TUAwN6l63CZdwprfjhgKEVVVcxqCNcEg8/VNJ1rehXIYVyZFYqcSuHtDO3Evyy7u5ehUv8gVs+mvDkiYufMZwusj/yLnkYRlnSvs6LAh9/7AQDg4w/eQUpQCiMqzCNvOJK+NzajYR9PwsYFmEnPSTA0MGu1YILxtI77DEZDP880SZntM2u1GI4bRbo5hCmAAfx6cemcrquVYpiPrR335EVpD3/rV/8+AGDhNH7jn/w6rXXJQcxiEfrSlgzd7Xa7KAo/p4ML56ECA6looUWl8IeFwpUv+XsIn13MV8jJkPWyhOE4al2jLIgV8vwAmpjNTRQMD2BDj0CUICV5jkwmiOjw1lqwQ7IuauRE52zyQI9moUsKrk0baUwwOpEjN9Snl1jkMfUFSMN9C4s59ZVKh7KkOUXO49v9Dxie56UCaP3rujk0Zej9aNgVhzsj3LnnGd3efe89dvBX8zl+6qd+GgDw1vffxmxK8g8ETVqtl8zqeeHgALde9H2a169cxeXL3kl654N3sKYe1+Gogwb9SAdiDXZ6onaGZycTmqdmVi/rapR04nUuedx6MtR484v+Aa3rLtycLjxfYm/HG7jXX+2iv++f15WXXsETevdn1JOzm93AR3d9QHm8ehsvnfN/N56PoeGN0+ETBR2kIqyBpWCQX5CN/mBvTD496GNjIOWnCrmHw/35ZNPGoR8pdmKVUhswJP8eVlWFJQXucZry34dA1V+rCUqVaqDggh0/CYMmuRXYYGvZMJp22kPMpj5Qno59D0qcxBzEX969ilfaPhhvd1MvZg/AFDULx8dxhmAfwvnT7naxJube5TJneQ4jLNPeO6uQU0KtyGuooCWyHZ/dEAGuJpk9sZ1l6La9/dG66eUr65xZC5M04b24nvkz8eTwMQzB2OLYYZ37c2U6nrDNaPd7GI68Pe4Pd9kpCU69lcwcD6Udgj9oN6j7683eV+MQEXRSmgBrqqBDm0Kaoh0YKxEhCfTz5QqzQ7/fR5QMaRuHeu2vsRyf4ZgSL6YyHHxOx6eoKclqhYNV4Uwjx3RdcDJMZhkUTXReF5id+bNwPJ4g0FfqqAVJ/evMZB2l3BsdxWqjD6zpf4uU5l5gJRVUEFAPgZSOmh4k5TYSvDVmlLSbzBccjHcGPSzn/nnNAtTTAFmbbHCUYEjQ0Dfe+Bw+/6b3r9ziEPc+9CzS1gDtIfUPhojONGyYSjf2zFrDgaiSoulHQwhK1EZPX9TIKCUR2i2fxMx0xAlXJQwnvBfUd/4kr1CQA93pdrkHXUiLYLiss2yvpI1gaQ9VIdngJFyQ0pEOVWgREhaCfDoJwa0OUrGrCBNYZo1DREHm3t4e9ne8A//s9AwnH3pK/36vxf7Yk6eHeEYw4yDjYyC4v7O1M8LOcEA/FxgfH9M6xagQ2EarJjlsA0Tb8TytNc1ElQDluNFOY5zb8f7VwYFPBisYWOvfsboqOQm+t9fFuT3/u5kssSbZn0tX+mhRdrCixPx8VWFBfoOtLcOeratRlGRjlURwvIWQnEwqialdYDMBwhYbEM4LvIcRzgYJZjrlZBWaljSxwRIqrGOYsRKy2b9ScTDY/LzphfYfJvhrGfas0M3yRhvSNwHeqTTvf2eBkoLjSAOSkqmQCkJQ4M0BYoyDi/659AcjtNt+/bVu4KCT6RiOeC5K41CsSMJLeV+r3+nAUItZvlxBhZyeNahDAGgNJ9+qqkJVbiGg27Ed27Ed27Ed27Ed27Ed27Ed24H/LxbQgPR8TsXYIgDipBBNNWe1wodETrG7M8Sw7zMPp0eeZfPw8X0sznzGo8oXWJ75Kka5zpk98eHdD3Hvrq+AnLt4Axdv+mzVtRuepVIICUepsrpYI6XKmnVug5SgSSRAAjfO+2pIet1XGPq9IVwdCBv6yDo+S9+TLdy7/T4AIH6wwhtf/SUAQPvMZ9Xe+8Z38PiOz5gp0eikRNKhRc2a1+sCNZGXdNYVWh/4jBZ6BAHAEq8QqUwlBR6S/tdYRABlUrN+D4JYoFxdMzwwpsxikiaIueqnkBCLmYoaOIOUkitCPlNC0BDKUISSvl8vx03Wt+/dwa//038GADg6PgYoI7o73MGFcx7O+i5lvubzOXapKbrb63K2YjgYcoWkyEvYwq9BHvUwouooN9tPK+xkHrqj1xJzKm9DlOhe8RnO9qUWHGnkBOFg5DF06T8jNgkLlccJsJd6KNvB4ACGKnxnkykyn0RBQU25Hx4/xH3r92Mtlyh6xAymKuQEnbItAcX6U4a1hkISKY5iIPFzWtfgqrGnvG2gCc+RkDDxT8giGYZz3bx1C4ckDvvxh3f4vbh4/oCZTrN2hpxIWUJ2rK5Kbhj++PZdfEDPqJ21sLPn3xGtIszGfn3LosbFEe0nEhvqWSAnCNR4vebsWRwn/D4ZqbEmUdILFz2889Gd9yGtv+7+tRTfe+ozsWfjJXYu3PPf3yvQP0eQmM4RCuHnv3jmCUt2rw7xyX1PCFMJiQ8/8mu0Gq/w0g6xrZ0e4YR0JCPjkBBBgS4CgU69UenbJARzG9B08dz3Jb8XgVAIsATjkEqytI5SCprIHWorWPRYwbPUAUCbWDQnkzNUlIGLlUVVUzW5rkHEwrBmIyuZCM5Oh4Z9aAsnAkOwYQRGkgBrqrz1ursMnxbwz3K0M0JByIvzly7j/IUb9NkrPHvi0QtVbtAm0WmpUxT0LtZ0zqzzNda0vzr9Fp8/VjhIBBIbwXBVISKs1z8+y7gd/z+MAJVEQ0SQxEASh/3rIAmOdjKdYj7x9nb/3DmkRIziqCJwevgYJ/e9RlWdT7CkSlJRlkgD2qTbxYPEV4mGO+dw6aIn+uqNCHoJhZIIB7qdFuvhZe0W6sJv/LUBs9LZyuCLb77uP4ds5mI+53dy0Bvg0oFHL7SkZoKOLFZIA2nc2FfmZvMVZmNi+3xyhPXSf11WlgWcnc3h6H0xUjC8NFTaMZmCTACyJIMIZ4JooFhVvggIRJ9lJz8iI5IG5VRDhqIkdBTEzBXDEZVSDTQUzfe5w0aC9fJ0ksFQCeGDD+7hm3/8JwCAh48PcTzxWoGlFIgJ+leTaHxlK8R07tRVjA6xDN56+UWc2/dViI+PHuCECPAW8zWyzNuJcNbXdQ0Zii/SsN6pkpKRAJCyYRsnMFYsJWJCQ8RRQ9SXyght0lDsZe3A5QJpLZwLFWB6JlWNqgzQRcNIIygJF8oedgMeq8zzGpXhPgKkszQsti2lhCK/y9ayqeYIwcSEhu5PiQhZJxDW9XA2PaZn8RFOqXq3u7+PK1d9ZTAv1lgRC3lOPlZpgYLuRRw/xUd0hqokYvis3Whh8tqG4d0OzPuyAQI7x/Yg1htC8LDotAj2G7SmawNNGzZKI67Qw9bIV1P6pRx7pLN8+dyI20SOCUZYTivkK6oubQjYVpVhJJ1B4+5YSK6MM/GRcFwJ9t+nf5VkrUlnNxQFoFgnMjC5KjQkMGKjaiil42qg1prfLU/yQl8HsivrABsqcw0ZjYCAigjtGGteayUco7BgQ4WzqVibuoYK2sml2SDVigCCYCva81Hcwi5pel66+gJadAZLJ5CXft9H0mFNSIFTc4YzarNAgIU6iyJolWYxbBLIvyqGgFZlxbDfqq5Yw/FHjW0FcDu2Yzu2Yzu2Yzu2Yzu2Yzu24ydk/PgKYNB3cnKj0dywPp9yYGzyk0f38I3f/k0AwHDYw5d+9ucBAAcHPjsiRIKzxGcFHz+8j6cTn6UrVgL7Bz6zeHCxj8XCZ67yxRIP3/0+AGD59CnNR6OmbGdl1njpjS8CAEa7HW5edlYxuYoTBv/tf/PfAQB6fY+57Xb6iAxlDCBwSlXJP/vGn6C68xAA8HP7r+FFynCP338HALD4wVtIiXpZGgcECtwoQkpZk4NuDEWyDNXJMVpEMuKe+OxrS5QYEVnKvK+RU7Z2WtWoqLrV6fWRtH3WQBQlUdQA3QFhtrMUIbUlpOAsi6sNIpIeiKOIew6klJzWC/1DUkkY6r2LswgFZTF+7R//Lzg59OtxdHaKgvDDX3j1DVy64J/jydhnjpxxCCIts3mBV25cAwAMhjsocqpQxQ6Gvu5duIFlSpUwyhAmVQvtZUrztNAZZVmyBIXymYv5okZUUEYxYPeN4Ibbcr1i2u/9nV28tudJeIatPZxQlXlVrDDo+4zLs7HfS7KWyAZ+vSatMSxVWK1sKjLlpAB9JFxSQ0T+foOmU7kuUZahfwvcU+azkJxHQtOn1mR3xYY+T2hMvnzpMr73FmkDjc9w7pLPgBu3UdWGYLx3aQLu23JWqt1OERObjrMSayJaWC/X3GTf6/WhKEM5IsKEVqpxQn11q8oiTYlKOkrgqBpYCosrV/w7rJSvKNmqwIUrPrN1YSSA16hnbFGhWHjCoLpr8fG3/V56qQDe+KI/G+qnfg537t3D7jnaj53r+J3f9uQEiZBwxs9vbSo+rGILDKnJXlHlWUjBVREpN3oB0GQivRbpJo24oTWjZmpolEHU0Dj+O61Uo+HnbKPVk+do0TsX+k0LU0JRldTKCpIqxK1WBEH07wKa56e0QEHEBaupzx5bZdFu+7tN45S3z2q5QKH8nJN4iDX1PR0c+Oq8NTlOzvzn9fu7uP7CTQDAvft38PY7vnqwnq/wsz/zFQBAWTkU1O8StMxKU2FNWUbnMs6oW2tQUYVnucyZmGa9WkMTKdV2fHYj9D/VtkETVMbC1KGHrtHFPHpyHx++/RcAgJ2dXdy65SnsM+oNE2YJJSjjycu6sAAAIABJREFUXxdw1LefSIeYMtjCOsynvuq0XkxR5P6s6J16u1Ssc4ypythuD/Ha54kmf2+P+0/r2qIkbSvUNf7uf/ar/hpdf6ZYGUFRdl+6EtMj/3mLsxPUpAXbaiVo07xB+/T47CkK8ifMaoGaeuJQGa6QRFEjQ1UWBjYPPcR+DKRFFgd0jkUdyHRgUIc1EPGGNnFz5iZhHa1jXgIdRdwbplVDcKGUbMgpFAttcf+vUo0OIJTG46e+3/G3f/cP8PXf/T0AwGxZIev7M34wHKDf8e9fnFFvZ1miJBKv2hqMhv6svHjhAhT5HKsyh6BqyWB/H+MkvOcBFRFBBQ4IJzeILzSkC+SAAlIELTIVloX7TaWRiAk5EUuNhMo6qU4QqLPMeok86AZTpezcoIsu+SdpK2JSqtqW3J/ltVIDIsdCUGUnyJKoDfocQLDumnOikYGQYMIPWzfyJ6Fy6wqJlMh5hoMBCuqFm45nWNeBhA7IMl8lXOQV9xeGiQppkJIunxMSBZG91FWFjPw8OPBecs6y2x32jBaS3yEtgTr4EMoiJSKQWDm000AuRP3e0nI/YRxFcOTTFuUap0fejzP5DOfItxwN27we6xWhdKYx5iRJtM5rtjm1sSxeaaxDHeRRHKDJaw3asxKSK8FCNjZYCvlcnx4/L9f0BoYeWGMNZECdWMs20VrbkCppjYjWQ0jJZ2TzkjkIWkdRCbig/acF3EasE9bPCsMVtCrIdJgIEUvZONTUe2oqANT3B5Gyjniwq612hSH1f4729qCJ6Gc9O8aTB/cBAIePn2JEfs2g3cG0TX2+CfUJ2wol9Wc714ekM8rVFe9ph0aX2VgHYxrP8dPGX0kIXqCBvbmNErWAZaegrituaH70+Ame/eZvAQD2971zspxPMZ96yEZd5ijLELgAOPWbrdXbxdnE/44QFbPVffs7vw/AszxmaWCYqqFJ72S080X2kIUQ7Ihba/Hqq2/SwoR7ESyYOei1GSL33W/9Kc4TpPTLn38dmHkDsTzxsNCHt7+JiPS2OoMdHBE88oOzNVoZvfSFDNw3UFWFNTXBHlCA8oX2EILIaFxRYdTxN9hTBjFB/JI4ZgMRRRF/LQPBDsBlbh0pZuTSWjHsEz/EjBQgaw0rkmI9JqEk7n7g4T+mLPG3/rp3Dv/57/4R5gTJENKhS1CIDhmbs7MJpsTkquIEV69epfk5hkKWVY7lwm/YK2++jO9W3gkRdDDEXY05wXLOdYdsMBflGuMz0ihKLJQNmkH+7zr9FhvJJGtjhxILu9kFPLnv5/SgnmJGMIfDyVN8+20f2LYpcL9w+QJevuKhxZ+U9/CIyG2KwoJQkbBWwCaBJQOQ1HFdcZO4hbEB5mCZXQwbe9C/O83uC99vQB2NZl2v32e2VXjxTb++UjAUCA4s9MnDodFNShUkwYpM1TgkSkWwdBiUZYlu2691IJCYlxUHWyrWzR6qSgg6JvrDLq5c9vvg3DkfeE1PJE6PaB9UbVy47K/7zBzhdOEX8salAfK1P5hPDi/iq199EQDw/tu/AwA4OznG1Qv+4Hv/0RpPj73jt59EzAYnrEMnaA3FwHo6o/kTcUPcEB8JNHpjm9p/ADbE4h0/gSBYL5RCFLBJZQMrt84xa11lS27w3z+3ywQts8KfW72dFqIoGDUDoRqdVN4HQjWQddSYE2xoTe+CUw5x7N+zThahpmD9+OQInbYn4kjjFZMHBRbCs9MVjAkC1jGWpBn0+7//Z/gX//w3AACtJGFGyMnpBC0yMkH0OV/OsaJ5LFcxIiIVEUJgQdA6a8DshVKJ59Z3Oz6bUdXNWRNIBIrVGrZDSUdrUFki6lnP8eieb8/46MPv4+4D32ax0/VwwOOnz3D47B4AoFzOQZxgSJMMrYH/nVavD0tBf13kWIazn6BMxWqJBYmudwZDtKkFRMOirtY8b0nvgLEW125cBwAMuh623+q20KLEE6oaT8kx+vC9dzEluGJvZ4SYbOHZQx8cPf7kNgwRiKTWsbMcSYWk7a/XzVqefARAUecoSNssEOJoqWAoCKpkhJreByiHnIKAfq/dJFaV5XcgsDLDCijy5KNIM4umlGqDiVuxPRZCbJC/BMZQwYRMVV3gwUOflL796CHW5ISqdhsqJbKZVoyMyHSYKRIJckuwymKFEYm4nzs3YMZsZx2GpEs2GI5w23ndY7YXWjMRDoyAsMFBVpzbdA681o7siIsiCHJupdOIJEHgRMyi2aYE5nP/PCfHh1jOfGK9Q60y1y6eQ03zXFQ1jgLpWZkj8Kw5ZbgY4FwNFeDpzLoRbQQAG+e/c2BVddUQt1lTQzNc2P88XyzRGvjzb3dnBxUlRqzSyDp+Xx1cvISLlz3b8qq2EJQYDSLeQgiGyRrX2FshNTJKLCyLGSpiFnfGPefMAz5QZT08JT1TJbyDH5hAsjRGlobA3D97JQzAhD0KgQUUtUFRePst65Khi6Y0cMRSHlgvkzRm32I6mWMya2C6YG1Fx/teSs1BVqBIE0px8UFpuaENLrgVAkJsBJSG/SMVdDWV2iCJadqaNkXjpdLsQyqlG72+MDclmNTHmIrbLFSs2BcQcAx3FgIcAFqCVRpTQ1DSV4jmXoxzsJTIna9X6LR8LBHmI7XglERkLRZz7+N840//DL/9dc+UPz48whc/7wsYN1+4RlElENFnrJc5xqf+XRiNBlDCB+5yI9lhneV2jqoyG0mQTx9by70d27Ed27Ed27Ed27Ed27Ed2/ETMn5sBTBA07yOR0OrHrJWUSThjI+oDy5exAuvvAIA+OD993F87DM8R4c+0lVwDLmCMaAiCla5wfHEZx2WpUNBBARVWeLqNV9V6hG97ag/wM7IZ7PS1OHNL/rqXpKmWGzOFc1cQZnSQGOslISlu16LCunIZ8DrVGH/xWsAgHNvvgx332fEdp75f01co5r7zEq/n2JO5Cvz9h5ufcmT1dx68Qo6owCRA9776EMAwEe/5SP83vASZh/767WVxIhgHEm5Rrfvs3EsbYHnaeuDrpxMYmgdMoviOchJIJ9otVoNvbEQjd5ZgBYAEKFBVgr8X7/hobt3Hj7G9ZHPXPTaHUwI0nPvwX2uQAUIQ6vdQkY03NPZFL1el+5bMFyuXs1RULXq1pXrmN//Y3+/9NmpyJB2fbawNBaLE5+VLQuBg6F/9klqMSG5gFAdi/dSxFT5qWY1Tue0Z8bHOHzq91uR51yehyjRIujLSzd91vnmjctIhj67l8wlJiv/rAqz4Eyq0DWgmsycKUKWKzyTDSphIRlp4P8g/LOhlSnQQFFCo7dzyAgm0Ov2UFCWXSkBZxpNGLuhG2hDppKeiRQbjdVSMlnOelVAE5RQq4bMZbUskI581r2gLKqtDVJa07kzPOe8ruGIcj6zMaT0mcO9gd/n389nUDHBxzCCSv0+ttEh75X16RIvveQrVxeuXMJf/Jmv/N1+6OGuVTlEWnqkwJ9/6w+QBf2bSHNjuAAAQhvEKkJN6yRk0MKxiKkKqKXiqpQxhqt+vkmcYDBFyZl2x1zmjkkZZBTxe2OMQUSf05YR2kRAEycxSiJbaA/9Z6tIwFEJ2aKB8/g9EyClUaNfJGWT2FQBOuqQk35QnpScyfRkNP5zfvD9tzEkaZgk9fe3XK0wJ4mSd975CHfuebjz7/7bP8bRoT+PO60IH33kq0Hr5Qo3b77kP5PW/MHDB/xs0yRrmuC1YPKpeb7EeuXnF0UJ6zptx2c3AkmJUhuyJFLyeQVYxImvUly4cQW3vvB5AMDJ40M4HajhSc4mFvy789kcOWWyc0RQBOdqJx1ERJ5RmQp5SdUQgu0lrTZees1/xosvvIrzl3xVpJVZ1Lnf67EusZzT2VloSEF6ZyqQnoFpy6WQSPt+H3Z3D+Ao697eGSEl1MWz23f9v4dPAapIdqSEIhheFKdoky5m2u8iCxJSRY4FOTHLU19FhDHQ5BgIWLTapKfZ66KOCW45GqEK0gi1gyabktC5I4yACpW+SLB8gVKK320tG2p54Rzb8gCXE1JxVe3R/bv4/lt/DgBYrxbY3SdZLFiGeK6LJarSV+wDGkvLjNt3pKnRo3Oi2x1AUWUwSiL0R/77l69fg334ADRZmoeACrZKSJggd2QtI0kUYobcKYK8aSmwJmKuIi9ZJmklS0xP/TzLvMKU2mmK+Rgtag0ZDnybSb+TQZBer1ks2f5UoiHFs86gdoHMxUBLguIxbNXyuQohm9YLIWHC2WstVwxtUQJBN7AKcD+HUd/byd29Azx85JFS1tSsV91qdxBTFTxOEwQNBEltK865DekBCR3QXXGEhJAiyw0ooZctaepR/p4cVzGFFAx5DGsBAL1uim4r6AeSXwDLdlxJw3qKkbNYm2B/LFYEwb13/xl0FEgWySdfFpgvvH91dLLAakZnhhVIA1Ig09yGI2PFsm0yDtVHDRX2uW70cL1/G2TRHBNEOZhGE5ArfZL9LiWayqGzjkleoihGmhBcW22I1wXfPxKQOhDsRLBkp1UEWNHECbYOer0WZYCvV81eCy1AWZzCUQ3NCWCa+7U5PhpDnPfvRafj37F1nuP+nXsAgMNnE4xJUu5b3/gW3n3Ly8hp63CHfItMCGhCEKzmfg9iXeD02P/dhYM9iP3gB1ZcnS7WK8yplWe9zjcg658+fnwAKJuHYza827C0zlZcmu4M+vj3/8bfBAC8+vqbmE19qXg09BCSYrVETcGdMRUWhDFel5YXN4o0rPM3cnLyBJo2Skz9ZX/9Z38efYItVGaNhBZ3uRw3ml4OvCEkNmGRoWxeQjvSLsoLgFhA007GmkdvnzzGLYI6rkiXo5IxKoJ6HD87wYocwt2XdtElyMPARBgUAUIh8XLiD+yHU3/dj8/uIClCaTvGLRKNf/NnvoJnT0k/ZkOcelPkMiVNMa01sz8646DJmOtIszCtUg0jqLOGnV5eC+ca0V5rsZwFNrU5PnjoA9TdQQfTWZe+P8M3vvvt567xxutfwBe+8AUAwG/83/8KN675ZxTFEfcW2OUEHQoSh6M9dA799WIynMIBNb2Ei/UURRCCLRzyx6SlpjQzH1qCO8yykqF6k2czLE78XnMlUJNDvpjNmdHq4rk9vP6aD/yCztThk6eQZwR5TBUi6uVTsUUd0WGsS2ZMrQsHZYJjQTBkFaFeNcmGT9OV23yGnzaE9NBmwAcUAbZydnoETfc47Le4N8M6ywf/mqCDEo4PzPW6YmdeR5oPW6UjTgrAgRneDGlwFs5iQTAHuwEbrpzDCzc84+fFgw4unPPz2xkRBOYKsLvn1781UDg+8X938+UXcPe2N5if3C3x5qt+TT/+3jfwdOYPsd3rBMW2fbz11gcAgOnRGAeE/baiRkWA9jRO0CFYlpUJ1iHBsaZ5VlUDk1Vgnakf7gfkRBYaRElgWI1bMVQS4CSNGK2pDe/7SLUQkzh9IlqIyOCcOc927EQFrRvoS4CMNXpeFIyHgF4ISG4ACY5TjhYF0mVlkRLc68LFa6grv+7vvP0H+KmvvOH/DB1a/z6+/XWfZDk5OsOrpKk6O55w0mXY76EmWEuvH6Gq/bOr1vQO1SvoyJ9blW16sorVDKvcr8d8UWJOUCCIHJBbIfjPegjK5jkodl6iNGJHsSgqduYvXn4BI+rzKZYlKnrn4yD5VlcoyAkcn51iRnBKYwUyYvNstTqoaY88uX8XJ8feToTM0yuvvYGvfPVnAPi+/yACvZg+xpzOZK1ipMQkWkOyQxpexLqyfMZHUNzjMtjbRdr1tqPbSuAmPplRUNC3yteQFDgKJaGJ0U9VEsvg4OsMCbFliiSBJn3acuLXqC5XsNSyUQiB8xc9Q/H1q1fhWqH/vg0TIKOxZahjnIReHMHs4Eo3545nAQ2agA1cH9Y0cEoKtKXS3BP38Ucf4xM6Q6VUaBMcezyfcwIA1sM8AbB+cCuNoYixOm31sDvq0pwTCPqddSn4ve0PdhA/C3qEwcl23A/mjOUkT5HXKGYkEl4CCcFjpQ6s44oTk6YuEYUEdlWhLAIzqUFEn9NpRdxTvSIW0+lkApn5n6+KFSc1hHbc+qGVgQmwSVXCESdAmL+A4/O0qioYE3xCwRpzvpuJmBSl4baHoM8MOLRo30ECK9rTTkgYuvZkOsUJaf+VZQFN7QEhUWfqGpZsmNCW+8faOkYcEh/W+b42eH8sMJYGz8FKB0tzLuuaiwHYYJftdRN0OgQLpjUyykJSsNjvttElkXeFFAvSVF6tHMbUez6bP+SkaELzX64Mnh57X2y+zDkhGG3oPWqlkYTnEkfITQjI6F1QAlo3668YztrAoWEsB3WmlhsJ0gCdbmxoEkd8XwKOReGjKOGWEM/aHvYNzSdVUHFgLm3WVEWCYwYIucFU7OBo34e2GxEp9tHryjDM1doaJwTPfPzoCP2R95lG+x1a2xofv+VboNaLAq24SdDcIAixqw0uDCn5qiMUFCPVxKOhM4GSOFImx2MU50PiJ8dy6c/C6WyGJfnLZW05PvtRYwsB3Y7t2I7t2I7t2I7t2I7t2I7t+AkZfyUSmB8eofk5iiW09RG3hzUQm+HNm8x4F7HQRw1b+SqAFA4msOc4ADZkj4Da+Czjh+99F8unPqt+547/dzyd4MJNz2w3yxeomYGqgVsIiA34ncWSmo01N2+DIRjWAIqqDV/7G/8h7t33Ted3PriDWvtM+8ePDgEA92WMmhrs1XKNiqoly9vv4w8fevjgH7sKwoXmX8tQVFAl4ZmsEYWIPJf4KSJb+NrNmyiLQFzTrLNzjjMhja6fQETZtiRJEBHkAKKB7PosY2CpkpAyEMhwXZCJL/63f/K/44tf+hIA3+g/OfPZrNV8hptXvNbTejrDycpnhXXk18sZh3/5L/9PAMB/8Av/Ll5/1bO+zaenSFo+61GMD9Hb9ZmQTq+NjiZCidBsDwtDzElRAkRd0lorBBYTv2bvvnsP6+OQcSQY6nc/8uyIAHpJD53YP6tIS1i63ni5gKL90W13PEwIwDMRYEwWkmA3uxd3sXaU9c7WMKnfm5UumHBAGwlRh+xjU/ZnqKGQvP4OjvcYYJpSk5MMRwj7xFqgRUQFaZbgP/lP/zYA4Lf+9W/i9NjDZNZ5xVpfSkrs7Hj9xZrgCcvphLO/7aSPKe35RCqYAFkSgAnsoWvLUKY5sYQe2xIVVcaVjrDOA8RScIZ7tVhBVD5Dtcx9her06DEzCO6cvwFLTFfVOkca+8z53I5x97F/j/7i7RXaBKs5f9mjA548WuMe6QDupTFaBBdZGQFLmWxVCwiqhO3s7uLGqy8DAL79b77ln0nUMGuauuYKWxzHjR6QaMisPNyTsvKUyZRaeGpjePhvYAPTABytXRIlUCCCCBtB0u9nVJ0UkWoIUmQDVbGugQJbV3Fl1qHmDLiizKmoBLMal6VBRiRT1gq8995tehZr9LukUUXVHV0KfHLXVwwe3TvE/sjvk5duXcYbn78GAHj51Rt4/XW/dlW9wO/9zjf8PKjS/dqrL+DjT3x15+jkGJev+Wrg3rkOCqo0RFEKB/9OTidzxNk2j/iZDzrTLTTDNK1MUHBxQIG4fiBlglZ3DwDQblnIgAoIcEX4fQsAdW4YAmptDaeDqJhATpnlOE7RJch/QtDMV177Inb2iQyjqFARO+9yvsJiSsxotmIYl1QSgdeXSSOixt7VcLAEL+yPRuiSvXWrCR4+9u/A0TMPAbX1CjURX5SVg7JBHyvHmHyOx7MZ+n3/vmjtUJV+fjmhX6qqYFZlRBrnCZZ/8coLEEO/dnHabpi0pYYMLIIBSg7HFQFfAQznyoaGGQQzM1opSPcNCCJtVlisC/9uHR4e45SqnVVtYAjOVeQlJBG/yDSCIzbUsHajnT6u0Jly+dINvPySh3k7ZTCj643nK5RBdjTdQRTE41yoLEuulGkpua2jshXma/88T56coSQoeEnzV9CIyK72s5S16erKYknai9Y4DHve5ulIYUKwdfPIt28M1hqDPSKMSQUyYtGstIGNwqaOAKpixVZCBG3jgFqTYgP+KGAcU2sypF5oMLOkgmK0jwzwGAmGDCqtMaQ2nf29EXJiKV2tcxwfk89UFKwbrBT5uhAQcahOSoZYplHENkpLwVU7QKOi/Ruq/JWpef5lbdiGZbHCoEckNcMuWunzxG5CavZV+t0EXbIjzhpUhV/T+XyFxZrmoSWyFvmbhHArXYklodbKWiAh31PCcdVPwvD+jqUJwIMgbQ0J8Lvi9XWbfRVgjg4ygHbgjGgq0UyOpBATg34Sx3wNOLDvHyUxI6GKvIQpQ1WVbLeSiFLS0JYKhva3jBzzAjlrmeXWGiCJ/DkQUTW8lUSsF2mLisu067XB0aH316aLFZKu94MG+54hfffgIr73nm+9GB8/g+j767545TLe/Jy3x1mri2ukr1yUJW5/4FFRbYJX7wz7mBx436mqCqwL0vB1lquSaZwwE67NV0x086PGXykAdBvOiy+VBqdGMJxBCNl8LRvB5cBgB2e5tO6saQgT/XcQ/gnCiecuXMNTEqC88aq/6dG58yiqAJ2K2YA5uQF53CglCzQwrpr+LpGJp7AF0Gm1IGkJRv09PJU+SEgiiftT3xvw9Ts+uPukmqEkVjUray6dymoMVzYU8TI4+NLyYcqLoetGeLSUqN/zwvK/OF/iEvVMyA2Ry030YEQyBVpLJPSib8LbhGwomYVoXgqxEZg3jnAjZv3SK5/DP/qf/mcAftMFYzaezWCop+Dv/MqvsLDpO9/3shhf+5u/hI9u3wAA/Nf/1T9ETIH0MN7HisTWp0enUB3vhEqlsajJgSz9M3ZCITRkWgk4WrvKOITi9G53F1MKPk/X/qA9m86gpd/86X4MTTBTZytmFV2sCzaIJ5MJHj6mazuiNDYVBPXHLXeXMJf8/WUdgZoOGm0cYmLWciVgJs1eB4Da1gj+kdamCd7/X/berNeSJTsP+1ZEZOaezjxU1a2689Qzm02Kg5ruNgVSMlu2AUMeHm096j/4wfAP0IMNPxm2BQM2BBg0Tci0bNiGbIliU2KTVA+3u+881Vx15j1lZkQsP8SKFXlu3+6mAfu+9I6XOrXPPjtzR8awYq1vIFsWFDLgLA+c3im/KBzAyTQrq56iGaV+/L1v/VvYEy7W1eUVLkVh77lnbuoBbzJOF3/8kLASeeCDgy0cH+7p5+XD/wtf+iXce5h4HgSDK4HjXMhcWAbGVFROPQPnizTWD/ZHeqjev+HwzJ20QD1+mt77xS9MsJzLofC8x+FBmjcP3z3D8jw9I2NXeCwS2rvPbeGLryZ+Jy/Td316+kDNzBvb67g3XBQmr/oeW6Iit7w4x0ow9LpZmqgwE2dL8FVVpHhXskUxrJ41MONsylvw/5mLYxsDlgjakcWIRGq9q7Fapf5oe0YnEXclUOxqO+rh31oq3IKB2S9HRlRMvcNY5o5n2eyaohJauxF++MOkRPz++x/inTfT996bTVEL/LQSWN3rLz+P3/vW3wQA/Lf/1T/E5VWCpPw7f+dfxxe/lObqeOJQyXNezRscCa+oE/jb6y+8gjOB1v3Jn/w5xtN0/88+dwP7BwKXH7W6Hs1mW6DscL9pn1nrfN4HA5DnbccIwp3ylS+Kj8HoPuhQ+INZEc+HCEhA23lflKUrpxCiYB3G8v6DG7cxFYpEI4mP8WyKVZv5QUETXW3vsZDDIPs16mwSTYSlHFyi7Hm1q7J/MqxN6tIAYMYVgtg8/PjtH+P730lJi5NHKVHR9y1WAptcBcAHgUb5iM4LPO/9D9Wsejoy2MoqmsiqqUU9b7I9w5eFcrH7zE3YqfD6R1YD9boitZjIwSrIaFxgrSnwtnSF9BYa2DEZM1grJBCmoAmk45s3cEfUtZ8+PcGVzMumGeFYIKovvfQint5PybOL86R0/Vu/+St4+cVEebh96zZeevlZ6Q/Gw/upPx48eqr2EZPZTI2348CeKAfZtXWIkoAOwaGWw2XLEZey7/SSwObAOmb8bBsY8NWuRJU4hmLc3kePi4ucrEvP4maYwe3JQakZ67jz3iBI37AJaOSQEnwLiKp8UQF1msAjQzDqH886vomocNCovC5oUkQmXEryggzjhRfTs7BNhbMr2dsMwNkWqoslYS9jg53T5GAzmuhBKPQ9VkKJcs6iyYl1XqOVuZi9ITof1fqp9QGNXG97a4wbstdvTScwEuzn5GJlrRYInDWqyH55NceVxBPzVavPaDSdYkc4s1m99XxxhlYg4yFCYauOoia8vY/CXUx9RpkcmPlnxioX3tVQyKZzBKem8RYuDgoz+Z4G1Ccn3F5LbqBAWlS+a2f1cNwR1Bw9CO+VMeBLu4FdnIlFDZZZD3XOOmwJBJj7bLdhEMWS5mq90DV0uVjh7DKtc10kjISetneYeK2vfKHH3YepkPUXJ48QpQD2yqu38cUvJSrHZDLD8WHaj8+fnGN5nuZqLUmDvf19vCRJqo/u38XFWUpe7ezvYbYl2ii+xmidVZc7+Mz1+iltk7rdtE3btE3btE3btE3btE3btE37BWl/NR/AgdcGGKp846OB97lsjmLwaFiVQgssDup1F2Mxx04FqiLMkK9zdPwiZlupfDqRCo+1VqFuDsnsE0geHCZXXBhDixBtQ5PuTipbIZbMdVU5fHQ3QUD/8nvfwVdfTV5lV09TlrG7PMF6kL3M1zMRWjI2RHpPydwzKw4VNcGsAmWsw1e+kkQann32ec3mcwiq8kimeIdkoRDniqfKMINlrVOlMePK87rWv6b8XZT7/L1vfQu/+7d+FwCwXLWYS+Xn4vxcYbMvPncH//F/8p8CAF6SzOJ/8O/9HYwnKWu1XK6VuAyqAUr9+90f/Cu0o1QK/6oB4kyUJUO6BvuSQY6GEVcyVhYGbp6+y/rkXL1PrtYpwzJqajU1P5mfY9FmMRRSAu9yvdZMWNd1uLhM8JJcNWsWN7ojAAAgAElEQVRmI2wLpIP3aoSZPMNxg4mQ+xGDmg2DAF9lgQWvn+tDGpvjZoSYoSMgVNmXCEtAYNIxFsNwLuVChfQul0ucCezz8cMzrJCyRN/4+m/i4f2UPfrH/+iPUEv2OovB+L7XZ+z7iMODBK0cO4cg4+PeyVOMs1qdNeikAtVT9oMkrUB1fYsdgX0+d7SHV19Iz+gLr03xxl+kOdJKhvMbv36M738vZUM//NEJ4lHqmx/86BHunaXn/OwdC+/T+7/8K7ewXIpwgGQZz9oep+fp2T5TFfI7MyvZfsEBnVQBYs14+43vpze5PC8IlcBXXF0V0rlDMda1pFBq5xw4e/RVGdrr4CRjGm2AoE9R9Q1mRuBjlcVSPAiX8zl69X4VddzKIOjaFpEXI2NIq+uIsVzbFNJ5vjdOXx5AWjMPD1MVve8YD++mvv7N3/gN5Pzdo8cpU/jS576Mv/m3/gYA4M//xXfQydi1FZCLdJE6rLNy42gL+6IEeDkXYQ2/xKuvpYrBt7/9L3H3bkJFfPkrr8P3GRbMGIugxmIxV9TDpn12LftkERnUVTY/dirWxjAq6AAuQmAWjKILIOOULWL2VCMuqAbr1NOyC2l9BYCt7W1MRJAkZ9+trVU9DxTRy57oyShVI0YPFzNsLKo4Rr8Woa1mBMoLpLWKaLGmxsU8VbnufvQhVuIbV0nlnh1hLnvmsg+KErLWAAL58p3HSq6zXEbMJQM/kn7pu14VkZ/Z3YbJMcekwUjMmaumwUggr85ZXXuK/nNBHlgz8CcDg4cUlfwVjUHu7Fz1I1hUO+nnr3/zG3j9C0lZ/XK+xFwqgIu2xY1baY7u7ezhf/nD/xkA8OEHCTL2q3/tV/HFVwVWNp2gEize+mqBB0/S2nVxucK+UAlsPVFFxJARSKGsvT0ZkAighT4iZnheVevz1/GDAPbp98urBShkYY8OC0GB1DBgUQddLDqtzDor1USzxtGzqf+nxzVghf6AAK5y4FUqjZGcomzyfRAqLW8YgipjWqr1deagVSLuO6Vl5MIVwysE9+nTczx7J/XX/sExqlErfQAsRKzj5PQSrcaWAnkEVBXT+lYRcaHzGuc1tlJBpBh6tGp2KOqnkdCKr+C6jxiLcOHhwR5u3UwVo0lD4OzJHbNCMKk4z8nFAvc+Smv56cWFwkW3tmZK8XBNgyjKvEuJcU4vV1iIOM+6i0DWbSJGkA1yufZ6TVp2qMcSU0hsVzmLRszMm8YBg/g1x7KuMgjZaxJQ+s5olMWJaliBNFAk6KOKUN5U1/Zaxe3aVs8KRk85UQUsyXI5mxClqqL0eYbIswNqoS1lZGJlgeVcquQgWPleZumw6vMccCCZc17m0OHRPr4iUM8Hb78JFkGtm3eex81nb8l3dkoHWi7moCpDoWTd3drCXKp7q8Ual+KPeXTzAFbEmHwIyIKX095lYMdPbX91DuBgj1cuSwyFy8IYGDwCMXeuxrn0ib/PH1iIOTz4bCLCVMzH8+99CAMST4FSDJsxpB0WQXpQyLCFEIKav1pr4EMx7BzLgaAzjF6kVH/nC0na+o//xR/j0aBMPBa1IWsILk84Krh5ZtbNLN9xFyjh6wBsPXOIz30uLe5PHp2pseje9lSx32BWSwJbFduG3Jd2oCxobZG+TxBPeY+x1w6/2kfS2vVSJ8KkdpjWqQ9uHuwU64HIaIX/MZtM5T4clrLwJc5VViY1ypv7lV//6ziXsnhlLNayaJqsioUyTmpMcHEqcsNvn2H5ND2jR4+eYi4wk6zq5JgQZPE8nV+oaLKzFi6bj4aQERRoOWCeMfZNNqGv4O7I5n+DIetewlJnHk1woCC/sB40zqpieVaRKlMtu1bhlogMY/K42gYbgUaZHkb6NMMLiQiLZVoM/uAP/if8G7/26+n3ly36WVpEnp6c4HOiGDueTrAU9c9sUjq/vMLlZRqvXdtivU6BP0dSqE24PIGVxbRddYWbRnl+jHAuz3jLMMYyfth5vPJs6oOXn38JH99Nh41tl+7h23+yxvMvZ7W+G/j2t9PBjEcG28IHvLE3wkjgtv3yCP/8LxJM9Fe/lKwfnr+zh7fekAMpE4xCdJz2U20YZwIxuwqEF4TPMN1Nm8xoVOnGE6uo6mjWQhW+rHOD5ElZkjKPz1BQKBSbiJiGLtaP1ugE9kmmx2KV7iNGoxt9tpIgN4aRQzfHFgZFCj7LxXOI6MTGYtjy/GzqBsHn3zNu3kz9VNkRvvMnb6T+qGpcXaVnMBEJ/74LqvC5v7eLhYzBEBpcXMia5zjBYgGYkVGblo8+Tgf7z3/5WeyIlc3nP/85PLj/VL5rxFo4B2Qc6iYbBTvYrOa4aZ9Zs3mMMXQPY8vKgeoZ2SMalqzuzQxSRWGFQKHsmWSN0jPYFLVotoDJ9j3VtEC+ZB2OZNFlxU14dHmNr8a6d4VVVIPsgDhQv5X9vevQ91kdLyKfXzsOuJLEZIwRlcDlXC28wLpGK6eqy5VHK/0xG49UbdLWFk22J2JGp3C5dO1ViAq9uz2aIsjB7OziCjeFP2mNKxY8gRUrmJV+Kbpix2OpaAAwwQxPI9IIw0NiXo8L//fmzdu485zADm2B8c7na41rLq4Kz+fgIM3bO7eOMRPoWgwpMAaAtuvRyLo53p4Vhcu6UmhiHEAN25A50EuIbziuLgOW87S/uKqFraQAUMnBZ+URQ/quy3WH1TIrf5YDz8jViBJLdaFSnp0RCOjVao6lJIk9W4VkVg3B1pl+FDTwr5oAtvmgUBIM2Ui790ETdbYKqi6LSECUMYS22PDEslZ28vO7H36Ic4FNBmJE+Yxbx0fY2Un7XBs8SCZdVqQ21ins0FmrGeAQep1DvlspF7Rt18hnkRw4WhBYDmS+D5kuCmctZmJHNhkBa+HysSRADNViVQU8PVngA7HIWqw7PHMrHbB3xzMdk+vOIkoyL9t3BG7U5mLVLzWxg2j0GS66gK4rmga1zL+to5RcrCqDRmKP2jm9BjOXWNYUNWwDCyf961QFdGANEQy8JDe7dY+YuYMg1UWIoU80LEBjAecMMhaYCdfwj4UXasoU9VEtN0rOjEEuq17XqAWm7upS3OnBWAvfdXV1Lt8Vqubd1CP0AiNd+hUuzlPsVpmIhezpdz/4EA8eJbrHjljVrfq2KKVyr5Z5zB5kSjLKSdJuNG5guw0EdNM2bdM2bdM2bdM2bdM2bdM2bdPwcyqAWmwbiMAMmzFGoSWpWX1dz8yfcgAdeqPRJyqDQ8+unM0emtAP7yP/zFwc2IZwVQJh3WaieZHXjMhQkJVWA0Pv8fprqURbOYN//A/+QXpdTMgX6HEllSgiQi8Zegugkqs7sjByEidySkheS7Y/ErAnkKsv/tLXML9Ir//LP30DUyFxHh9tYe8wZRx3drcw22qu9Z21RfjFGKNZRmttEeEBa7kcpHnIQRG2vGbN8AEVoYq+KwpHzjp88+t/PX0HyfoYY1TIgsgq3DVygJUMxG/9+/8ROqkwrNuA8VQgATHDgwLmJymLMaEGLJlDOzWAQItvHu2CqpRhy6me/mqJ04cpw3J20mG5kqxl9AiiZFkbFKiTgZbTxTYOB8fA/uvy3PZajHJlCD04Z+b6WLJwxip8IHetMQ4LqRQzRbD4w5GpEbMAELcwMWegxloxUiVRYxT6N5o08NnPjwkxiB9eZBUJ+sqXvoD//D/7L+QZdXpv26IIOT4+Ujhx3TQ6F44P9/E//I9JtfXtd99Dm/0mRZF1NnGwshw4M8KFVBk/f3MfR4dJeffx+TYOb6e/25frfWhPsL2bMls/eucpHkvm6/M3DrA7SxnpynZYSyaKxju4ndBLeOaGeN7c3sOPfpyqglcf3lWfxXo2QRSfRfYBI6kknM9XuJCM4/QZ8biqDKzAg7gi2GoAw5SMY4RX02lDcVARz8+TB1lDg0uBnOKpgwm5erCGl5pzF6KKIx1M0pxlDAxoqZDZM/wdSFWTfO3IUUVgslaQZ49e4JYBjE7WsPff/xCtVB+vzi/gMw5mmubH229/gHfEHDsyoapS5vbddx7g/kMRj9mb6poyHV/i3fc+BpCgnACwWK1hxKx7Z2cHDx88le/l9bmk75DhgNOisLdpn1nLxss9kxpzR1j0Mk6b2qjAmTUBlNUOiTO+BCbvpVQ8sZI5daY0GK2KEJNWHohYYdM2V8HKnwEolbC6qTFqBC7aVFpxiV2ncyJXvMgM4GvEWuHx0aMW0aFmso0LERlbSCWqJYdlro75Fp2oRXvLsFk9FKxoE4XGAljEXInyqATeuXN0gF7QJvfv3kPXpT44vnUTLL6IzazBKMO/s/iagVb/DQ2NqE1Rp7SmaN4N9m/OcFdrtOrADDV8R+u1OocYEGTdccbjtdeSwFOFRM842DtU0ZDgg8YCk+0pXvvi5+VeHZxA/0xjUOeihs/7UlSYHTNgpE/rsYc7Sp93cOMQ/nZCDGXV6MuTOa5OUtXj9EmvnoZtF2Ay8oANWOgGNQgjGUvNbCz/WrCofXqKRUGVIgxlE3FW0TgOUSs7hQvk1cMNbFU0JHqGzZ/hLFhoG42pCoopV30MYSQxS+dbXMwv5eeA8TiNg/Fsitui3Hjz2WfwRCgcGaVDzMW03BfqR991Gs8wR0DG/XJ+BXJpjR8LsmN3e4aL8xQn3X/0VMVh2vVKFS4no21MpBrlVWEaaMWcPIRSOZ/YBlWdYpUAi+hzpZpRyXPZ3krXtm6KVtQ02/AYrVTeAnPCSAIgV7y8QwiI6ZLYOZBKvbteGc9zPPoAzigAy4NqOEAq3JYF2gA3iOdaqXp3fYe+K3D4vAgl5FyOjUX5s3FarTdE+pwjQUVghjFwBBefzoykCk7Xj+l0pkJPRL2iDD0ZPD1JlJ3qg/T787ML3LubYOyPT0/Qr1Oc9P2/fEMN4pmtxgtXp6e4ezftzc90aXw9OXmq68hkMtZYJvoOISO2OKrKcG0IpvrZNb6fAwEdHPp+SiUxP7RPHsxyuf/ap33KIXL4esIEF/XKT5poMxfOwVCZNEa+Bg0d3n8rsLEcUHGM6MWYllAUGLuuRyVQphHV+KFAok7XqWw+fe0ZnL6XJNZD7+G7NCENs/L0nHNobOaETXXUnJ9nY0YPPxEZ160RLgS2V/sxWgnml1cXuHc/LaDT2Qivvva89Ad+oo+SUhekDyJMJm8QK0TSGnutT0vPlP4qB+xiwGlsgfT6EPG3//a35M3pH99HLZsbUwJCMqSmsSEwetnAYvCYrUefuHaAIGlhrMfecwJ9eeUY9SjB3iJ5dCKp3AvmnNc9Hv84Baw//PZD3P8gLcxmZDGdiepYZ+BFRbaLHrQlgf+2qEPuW4jXNprKw+fxYxwoY83JKN/PgHURy+OkXRfbj359Cqb03GzVwFTpgGSbHeWFBLZgyobAOXkRsSNBxa//1m9gKWX95pCwLxGGNRFLOViNqgo3jtJh4/Q0jc31usA+T0/PyrN3Fr0cZl//3OuYivR213fYE3niabaP2N9Bu0if8fD+PX32V+c9/tf/O6mHen6Mb/52Ogx+6cuJD/HSKwt8fFcO+Rd/im/+ZvrcxRnhfJ4w6k8vIkZ1+o7L+/cxFjjI+UnaLG8fb2N3N93b03c6sCihcmX1uzhrMJHBHicj3ZSqaV7kB6anxuhBDhyQgQ7W0OB4xspl5iypT2WzIBNV0n3VrWFlbpElhfEQAbXIWI/FhJeN0U3LoiiXpX6XhBYXqJIzVpNQcS33XLHCO2JosZynZ/TuO+/rWLl79y7caZpPP/wwcTv+yT/5U5zJxnP+9AIXT9PnPXzyAKZOc+jWM0fY20tj04Dw+EHiuDz/fFIre/LkFCGm34fAClVdrS8VU9h3rHBnY67tEpv2GTUvfLvIATYnQn1ENSq8M6K8dhmF/RO8Jsby5h8jClfDQJN5AhhNL5MpOV1TII0ZPp0WR5kj0SunLHJQvqIx0MFClhKPDtADj7VWA/zIpPYyzhG298QIfn8LHwkn/PHjZC0zm21hLeHGygdk8HTfBg0kI0gPWbWLhYcka7oHYVdUMbd3dtAKnO7uxw/x6FHaX85OznHrZlrfZntb2BGD9RwlkhmEU1SU0JmhQbsxpIdEEOn1h/SNvK9y7IuyNAMskG42jKqWA1k9xW/8erJxGkuCZjqdYb2SXgheaSTG1mqF8eLLL6EVGJo1BqNRes8om3RHVygszOB9sRhAgy2xYLLWoZXDeIZ9zs9W+EjWozffeIAnj1Pf8aocsBvLCL3QR1yLWqwMbjyTNuS9PQLyARBrVKLKDUTtR1BUu4GAgJjVxPOBn2zhCCIOeH2xHAxNpWOdCLo3yLkGzjhMxILi+PgYR8eJW79se1Wu3dqaFkuUvkEn+3AnPOvlfImVvLZazpX3aokxkf7Y39/F9nZK4lXOohXI7u2bqT8+//rLeHyS+nHRtmqx8vj0FA8eJuXXg50pDsQOKBufh0hq3RNjwHQmUNUYEbKN2Xyp5u7OGThRVq1FjpdD1INxjBEhJ5u4oJnZkKqNMpsCk84Qc4pqR8WRlcrDNDzgMcrhrUBDyZWErMsBmC+WT53v9f6tddeSJ/ntuSBR15VSPDBILKQwvdxfsWIz8FLoyfDOwB2iWM4wMZYCcX58coaLK0mCtBHf/35yD3jzx+kc8fDxA5xJYuTux/fVoifGtzGdiLq2MxiN0hpkYoe1HKp7WZuX6xUmwr0fTyrI10K7XoPyegsu9BEOn1Ai/sm2gYBu2qZt2qZt2qZt2qZt2qZt2qb9grT/F0bwPwnbJGMGGZTye0OkxoTDlrOQP6UQeM34/FPvgK7DQfW9hJIxuFa0JBwcpIpbNsc0Azv0rZ1tVJLNWrU9aiEYP370MU4F7mmmKWN24+bzeO/9VMINjCKiEUslbM0o1a/VUvsgyu+rukLXpWzQP/vn/xfu3HkPAPCFz38VhwepzBtGM0xCzqQGfPBeuuaNvQKzUoNTa1XAZdhvdpCJTJn66/3O4JJ5AQY1kfKUyVpAoUUoGZyQy+3DKi9rVpOMgc9KWKEYXhMBU8nGGhHrcEe1qlQCAUSSkXQRjXjJrPsW3A9IwwDIRmyLken4LYvdVeqb25/fxpe/njK07/zgDB/9MGVcwrzH0aviW3aQsnWj7QpnIjrD47Wq51k7ghV1yhgsViJ0YwzU4JZj6fOcxY7Bg7NpOUFJytwvwFkwqNqGcQJZzOpzxuHhwwR/fHDvHp59IcF5dmY7mpEMIeq4aru1ev6NJDu5s7sL5zL0+HpFJldwjg8OUEkm+ObuDAtRoVpLRvLf/Xt/Dx+/mbJWf/QP/ztsC2H5rbfu4odvpgrgl3/5izg4eAEAMJ+L9yIOcPdeghe++vKr+NJfS/PlctHjne//JQCg+XCNyUF69n/yZ3fxwnNfBgB89G7ywdxunsPFk5TN971XNdLLsws4qdabjhUCitpgvkivu+zhR7EIvIAyogbMRTDDGFs8oEC6EpR5MciIxaD9OzcrLMTDMsSAkZhEIxCmexO5D3meRKWYYm1Rou37osRoSbOWxISVjJVesolu5PS59S3DS3b90aMnaMQL8d5H76ORbP79s5QRjp7gsphDDywu0hfr0eD1LyQkwUsvvqBY0/sfP9Fs5/5hym77EHB5lT5vPvdYyjiZLxYKq+26CJcN25gAt8kjftatkuqGg9G9bUQEEfdDbaxmrTlCoXHERtRpoRVwUAQ4z50CjY6MMl8MdK9JmfX8zGWPMEYRLwxS6BQQ1XeQLA3WNEaVFQClMldZCwern6pQVUewAjnfv/0ijMDvzlZpb2TbYiXzKYDVe7OLBSXknNOM/zqyIjBqm+GHM8xEzGPhI+6L+FEXCWMZ68u1x6JN8/JodYgu+yWqag4hlxh4uJsStNRojB2ocbP6AOabYy57LINxDZ0zAFU5Eg+8yNiR6lFGL3jPSp8xxGoMHbpekR117WClskZk4UTMxQosN9FI5LtEC5cLvZYxcp1cp1VYXlb+HFuHo5D2uLOrS9hRuumqqhRtNV/2OHmUkAcxrHF4I1W6nn1xKn3RA8iKloQ6i7b4XpFGYAa7jC5K6qSpBe3OrCJrOCQ4n/ydqrOHAhM0RovCCoeOvcdC1sLl1QJbL6Q19NatHYUT7+5sq1CJcQ5jQa/MJrkOTZhtZUTFPnxGpLWtwgpnW7uYzM6kgynBn5HUHQHg2edfRB9SJck5i0tB9Tx4eo43301zYGsywViqR1k8kQxrXLC3fwMHt4RqUFd48CDt2Zdvv4OVVIsnY6MCd3OhtpyfLzC/EkEeHxVuzgNf2xihwoXWGRWegcAPoxl44HJZd8gVIUdrGIrCJNL1KqvjWmc1ZopgBPVILKr51prBmWAIN5CudbYgrwxUPMmYIuYXg0e+kQjA5+qnIKVi8OCY1bAJa0EW3n34CB8/TM9w2XpciediHhvd6koF12qKWMrfXV6cYzZJSq7j2Q5G+ZxCNUaT9H23BR2F2KOVyrJxRiHh86tzeBaYPUHVWXngafjT2mbn3rRN27RN27RN27RN27RN27RN+wVpP7sCKByCaAAfsxyqUV4AIyrO1xh7zXvO5IxiUWfRw2jkwtn7SZ5f5vXFa5UMIJ1WlTTKQC9/2hlGMNl3JSq/hhmFuyP32bctJjmDz0UG2liLXojV+/t7+NbvJc7bd3+cZO2vTi5x5ygJcVzN58otXLdrvSdPJWNn2aCWitHOXsqub29t4fhG4ra13Rrvf5AEGxaLFl/4fLKbeObWc5jOUgZttr2FDH8vghXmWj8PeZW5h2OMhSeIQmD9REfnD9aXYoyFr+Y79ZWhqlEJ3qhZZQ9ni7VGroh671XUJN1nSVvaw5RJcs2AZ2XFA4o7rXJ5JnSt+E+tgcVZlL6WZzxhMAknYTbBNFHisHdzip2jlC0Zb60wElK5GTXYvSM2G7fSvz4adMtchWnQikeUs4S6zmOzBzmp5MGBZbpkf5kYAzrJ5MTIg8yugcky436B6Zjk/h8h+oTlt7PMBSQlB//+73+Er/1y4nP89u/8tpKUe9/jSkQ69na38dprr+nzyv/mcZDuLWfii1fg9s42WAQYpmOHk3mq7Hzj9/5tAMAzN47xp//bPwKQbAiyaJLxDBK+0fz0At/7TrIh6M8T//DVL38Jc5EuPj5ssL5M97G9ewS/SJnpOj5EHdOz/9rn9/FIuH/TkfDZHp2gE/5hNR5hJd+7XS4xHQkp+nIJK2uQJ6hsdrE0sSpYwVksAakCnscucyzEBRpIhpssDEOaze84oBG/JYwtKpF+3tk1qJq0fpy/t0S9Ldy/7CVIBn3IBHaHIBxRawu3pw9esfnMAX32Q8sV5ECafbcjp/dJhnB5karaqB0mIhz1uaOUQTw42MNU/Mveeet9dG363L39PbzyynMAgC9++VW0YudQ0Rj37iaf02efTZ+xtWOxEv5hF+Zaoeh7m4tEYCK95xgZQdbCTfvsWpZJD4HAffY+61BnOXMb1JIIxhTLE5P+BoD6vIFJ94MhcCfEIphR5BnS+zO/Rqvo17A1w8qWKfMvBrWsiCEoesJkEaEYFV0BQ8UT0BuQZOhf+dxX8Gvf+J30FlEusdxhJuvRBx89UHRD23aKTHFVBSvVECbSdaMWg8y9vW3MdtN6dHHVYn6e5oV1NY6Ec73nAzhXN/qAmL0FRvl7m4JEMqXXwMWvN1278P0UfTMQrlPLjsiKHmIOuq7F6PX9PkT9jlE4gl0Iuu5XFvpQ2cdkpfXJZ9H1UA5U5o2SAbn83HqtIjI6zDsRxOo8evl5JTY5/Yq1uuG5RTUR1MnBNvZ2U1xzerZAt76SZ2Swd5Ce7aHs3ReXTxXhMV4zjMQLPnaIgo4yCOA27389vBeBuYHYVq6kxRhgBzinbDNkKJTqEAEk+zvL/a/aFZbLdJ9/1i6Uj/bVr/4S9nbTmKDg8VSEX/pujU7iCBKfydm4RiXiLNbWMHIfy+Uac6ku+p61subI4fAwVf6O9tO/IXrce5yucXp5pR6+zhj1dfzR2x9hLGJLxzfl2btKBZFMXWFspY/MGE7GfV1N0bsc7xi0QqadX6Q94OnpHBdSAQyBNP7rfbFk856V2xiZdG9WXuDAJ5bDwIuPCEaIbByCniVApAInZT4V7qAPQCf81Ui9VgldY4qdCnjA883rXARLPBoiEKlYumhh2UDHvY2s/HzEEvMph9oArVRj56sWygW2AVNZE24cpur2zuwYRvr84uQUJ6dpH9+ZbuHlFxL67/jgQH0Dk0hQ6qeDvcTt3NoZYyloIcrPAMBi2arIoSFGJhgaa/R889Paz4GAygRDROTsy2E0OKQ4FP8YKFpxUfMq57Vy6BveUgqKyoFGYSTZvXl4N0TgfJrhRDhN1xu6PRdQIyOq6lJWzLu8vELtsjF3WRhCCLrwHh4e4+/+h38XAPD0PKmAXi4ucXGZJtt3v/s9vPFGCoSfPHmCtcgedRw0EK8jqcrZTPwFJ5OxwjSa0RTTaXqwMTA++igp/ty5/Rx++ZeTQfxkNsaWLJofCTzPDsnzZLSDIxf9QTuQZCNAv9eAAatCLAwuvGoqZWXfrcpE7Xu4bDasBHanB19jSTcTEF0zOS8iQcC5EMZ5mQ+LAY0okVUNyncJUQdu7AAKohgmizzPHSCCGRysTvSqrnGWuhH+CsiIQdSkwgAsRPmwMOjncgAIe+AuHWJW66CJhdGkQi2+iDCmGJHKfU6ntS6UBFKSsq0sbopq55PHT3SB4nUHJ397fpbL9OV7s2e883YSGvr6b30do0n6vqfn5/iv//7fBwDs33oGLIvjWgJv33mdkxyhzxZEmEzFIHQ1x9OLR/K8Gnzha78BALjz0ssAgP/+v/xv8M6bPwAAjOtK1b4qshjLM7p8co43hNS85dLzfrk/w84s3efJJcM+TNCe9XuP8Nbbae48/8ouzi7TvbaLCwSBQf/g43Sfo9orRGr3+BmRDfUAACAASURBVAaWAp8YEcOLulzLQQPdYA0w2OjT4zEK+YFjTUAxl4O5scMk1UAtOG88gAY6BKsB7dovcPP5lLg5em4LZ/LM3WOH0Y4kGUSB1IdWxQnIGFjZZNqu1QO7IafrZggeIfuyCZSl73udZ6PxWM24v/Slz2F5mfqmGjW4dScJt0zEQ3R7e6KKizdvHGqgP53OsCVCCzdvHeD8PF3vmdsH2N9PB8aj43RYJ+tRNRk2FPHsc+kao9EEts4w5E4VgNP33KiAftatskUdWiGDPiJI0q6uehDnoAAqXMSUAp70n3KIV1PkgUcvlS0bHAf0DKLB/jzEdmVAkYFT701CyGtuVeDRHlEFG/UYxAUiasio0Ey6BxE4mU3xtV/7LQDAngRGI2dwtUoB1b0Hj3B+ln5++vgpzs/SwXDVrRUqFiLp98pB8/7BLmaTtNeG4BHlcDHdmmjS9sbNm7j9TArWDvZ2MBbxj1V3pc8iJ0LJlMPg8BAMY8phj8xQMzh9V3BJUqH0B8eo6xFpCj79Rcz/8+WwrgdODJTTqaihRyRhHACI3uvPWT/Cughnigp7HjJ9F9TMPPYGndAz1nLQ6Lq1qlBGF4picuzBIiJVTwhb4t+6YwyOjgXi5oq6smjwYLQCgs1ibgw2Wf2UC/w0WBU1yZ6CMRQDdo5xkAAxSQQIyVtZldMNqaBb7v7KOawE8vvw5BHefestAMDN40Pt/dViifc+SHvifDHXz4MK6AW0bYYnR1XvHR7+D3b2dL9lAm7cuikPIc2xN958D2/88MfpessrzKYpJtmeNBqDPT6Z4/2P0/5OEt9ub2+DcwIkWPTz/Dw91nJgr+sxaJqTCR0W4rc5v2rlO7XIoV1Sms+xG2sywceATtag6IPShWZycGewqp+Doq5LpSQknZ5pRAwVLspqtvApBZ/uk7U4NZpYNFKImEwahe+2rddrRpPHYDtQto1KfyDDKgI4hAWDGSGfgWQsGUQ94HKExm7Hx7uaH2tGExyJQNQN8fDbmoxBso+HPmK1TPt4DYujo5R4mowm6AUm6r3XZFgtwi/W1mo8HzhiLkWBujEqeHRNZAoRP1lGu942ENBN27RN27RN27RN27RN27RN27RfkPazfQDzD8QgZSD3WgEhYzVLl07zhRiuUtLqTVdgjKlamH9dTqipipWaHXoJfoplQZJZlhM1HGwUQjM7zQ6AYhFhECleZsZoPPqJazOzQsisJYj+A24eiAfQwZFCu155/hV8869/AwDwwQcf4oO7kgFqV3p/znORYJV/67pWIqtzFcZyH9tb2xgLLPXG8Q1YyeSZPgLid1ZEPoZlXYbLWrBUoGxExSsQg17UxIb0X245K+jIqG1ADG1JYHKEz5lNW8RGokKBizUFAK0Apv4t2c7TpymrlG0UrDOlKsxGJbGT7LBURXyAJLURsiccCK1kqMLao5ZxV8Pg43eSLPLFRYvRVurfrZ0GI4HwUUjZxvnJJc4epizM3q0ddL3AC5cGrXjWdY2Dk8+AY7hGspxt7rugFWkajEcwcH6aCMGHR4eYZH8YAJ348vTyjCtrdHQ7a7U/njx+jO3dlOn70Xd/gC+dJnn/9Z0DtNspA/ndP/5XAICTk9OSHSarMBPnKvzS1xK0eNwu8cLt1O9+Msbt514EAPzhHyZvwPP372JbxqNxTiGsNYCRPPvQe3z3eykL+sydBE/53d0DbB2mDNYPfnQPLad7e+MHfwEIxHPrcBerLmXlb+ztoDpJVcJ33k1E9MPbR2ARWVmuDbzIYI/JKMQiGqMZ3XUfUWVYRxahiFGhuaAwyPgORRcGVSsizeqRTlOjQj7GOCyWCUq2dTDBwc00boJdwUrCut52MHWeA7I+klfIZh+CVtTjQJLZACqnbZ0bzIEMqfFaNfbBa+X5C198PeFfkLLXtYgqZK/T0cihEijI7TtHRV7fOK0Ex7hGLd6Pz714qJWQepThsDUaEZi4ebPC4UGCOk1mI6zahICo6wpdV/q/qbKJ2KZ9Zo3SmlJXFpFljeo7QOC7plrD2CzfbhQOGg2Bcuo7V4N88VQjUJF3ZxrsE6bQL2A1863iXxwBsTxhePXwg/FY53kZvcYOdV2lMg2K8IgxRv1RrTPqt8oEBEHZtGvCsUAyd2ZfBQDMtqdwUg6KfVDf3acnZ3j0MK2bjx8+Uq/LJK8u6AOBc2/NJhhN0sSubYVKvlczspgIJWN7a4YdEYqZbU3QyJxad2lekBnYOlij685gG7yGPEgiDVkkrcjXR90/iyAPU9lXh5YF0uH5EelnWZcFy1hl/C0BVkOHWMRQwOo3qN56RFqBYoSMgEPXdehEBINDpRXd7MNnnAHnAWShlZXTy8tSqUREJZWM6aTR6vN8nsbuqmOspPoxX5CKZ3j26uNWWUJV5diT1ZZB6UnM6lnsOSo00CBqnOec0YpdZGgs63QbtyAR3Qpdh4V4+92791Dv9eT0DPfvP5DPCAP6S/qM5XqNC0FttP1aoduVNfrMFweX2BI4fzOqMZ2mcfhAvFvfevsDXJyleGLSjLEjtgHjxqrQYNv2ePAowUS3sifwdAuNvDeAcCUItnW7Vn9AEKERH1qsGQufvlfed0FW94vOB7QSjwYulbAuRKyzCJNn3QvHebwOq9Ao1bSEt8zVwGLPEQZDO1OIYjDoc6zFBs1U1gxrUMlYV2oXgDhfakWcZWy2fV+Qb8ylAsnFOxGxzEULo7DTPE8DMyjfE4DRLN3HneeP8fwLqXI7Hm2hkX3Yynv7bo0gtjajaY3JOD2jmh1Gdd57gToj4mqnfZPpIBGELfFnrEcO24v0fT13Oi8Sti8jHTxitjz5Ke1nQ0AHRrEmbxDGw2Atf7xOvlhI61suuXKMMNnrY8Dpy7835voBJTczeD+MuWb0rvcz4MLl4Dusz0HB61ty2Zgt6QRvuzTgm6ZRxR8eBISGjPIaiEiDrqx8F6JHZdNkmo22sL+dAuDXXnpNoVut77TLqlB4ENlU1hqjCzMxdFGyzuq1Y2CFV9keOH9wcq2PhhuIHULaBobuCfJZYCRQeGA+xNlrqkykhI4Im2EhltC32SeIwZzVwQrUVg1OAxc+2mDAWVs8YUIIysfM3n9kvKo9db0ffAYQY+aBeXRi+Cuwehjn9DCOxqNyco1+jrPTFMC3fa3+L5j3WCzSvV5diifPZUBYpWdxdm+NqcAYw7zH+kIgLmgh6FPYicV4VyaqLAYHhxPFg1tr1BOmW3sshRc3v7jCzRvH8v5jhZlcXojx9tWVjleC0XH6f/7v/weObx5LRzl8eymQ0Tffh5mmMZHhAgnSLAFe9BoPWGvwwx8kpc29owmefVZgvMHjje8nbuu9HyVoyYtHN2BlPq3WHWqBKjtiTCWhYsFYCkbo4n46aP/hH/0FopzQ/7Vv/jLOTxMM5eOP7uPl1xNX8ezE4sdvJijWKy89g2qUFsobBxlfH3E5T5/x4PElap/nKiHoODbKWeoj9JCbA7Xr60nZcD7JMR6+L28GGkwNINWWGFl2b7Y7gRnlANqpj2G3xWBZ1P3gPJrXOUtGvcFqVw/uxelcDDHq/GsaGYNdr+tI3/ewkkCYThtVRnS1Q5vXPAE4+8CA+AGRDaizWbdlRF7r9Zysac6VICRDxgyVtWE8YTXpZi7Bbe0qhJA92pzC2jfts2tlfDv1oqwQ4WJaV0zfoCIJ7NgUyGAo4z0HyKHrlDMGMiXgYELeR5hI6RlkTTE8V2SjLVFv7BC8cHT8Jdr+Ut4UlRdPYBi5ZqZhuNqgkUOkcQXSnS6b1ZYZM/Hra7Jv78jodleNGVuyFh7u7eP5514AAPTdEl2bYfdRDzd5P/Z90MOpc7XCrjmUA0EfGE2d97AGVfZKU+rLAMZGtngWpxeknwacyIFZdT4ERQIolERnDlKNsWpWzTwQWUXhF+Z/rSFdUyJFUChrYY7XDEe4vGdwRNvJgV7477GPRQ1zAHvrfMQ6J3+4mMwXjqAFMnTUVQhyWFwsIlo5KDsi3d99R+hCSgg2oqS86HtUcuiY9UZVBNaelddniGHFK9BayuhR7O7U+iyy72rsAjrx8/Wx12szs6peghh15oyY0i8Z3e5rg7kkBN997z2Mpykp0HVrrOR1Y23hWMqG0PlOKTijAYQPMeg+sV6t0AjMrxpZXF6kz3uSlTpPzrAjCp+zranuPyZGTd53IeDkPPXjx/fSgbSaTHBwJNcjwtl5OkQulkuNtSrrdK8H2bLfmsLfWwsNYz5fKtSXEVQzo+36wetFYVeVuFESssRFMZ6p7ONEpHBVMuUz8vwIoSQsrDGomjw2oWM62KDzhWoL4/IBUJKpFBUKT4YGc5J1rYkD2liEUbXXwtFFSfqjxM7T2QiVLfDeHAfnYk0H1nifmbRY45wFMp/bWE0eWyrrCih/bolfGjvSfuyD07glIKoKf0qKbCCgm7Zpm7Zpm7Zpm7Zpm7Zpm7Zpm4afBwHVrE7UU3ZEh4/eT2IRkd/TahoRoRa1o/Vqicp9Irs+gFgSykm29x6LxULewvA+q+ZZLYVfSumamTVT6b1X/ymLgENRfGwc0Of7B+DkVJ5VOydNg1YIw33wmKjqJStkylirikTqI2TrARyGtHLlrFXYCqhkBChE9JKh7zPUJsaiADbI7hEV2Ic1mqQAIlCb9L06J35oQ//DQbsuajH4BbPC0PS9hooHFDhXt5NvX85gwcFmr6+IAWF3CF/JVWEUYYFBxmHo90hEcGP5bCF1994j5MyRh3oX+d4jgQ+Bfm1UIctmM6LKYy0Ztt579PP0GffudljOCySiXUlFbtWqopzdSr8fT0c4OkxQooPJHYwl43ixPMOyE+GAxQrVaFT6bJ367Gu/moQxtradZsCdszg9F3GT7z3AxWmphp+epopd9+QJ9vcThOhZEYl59OA+ngpc1Ayq3j54vPvWOwCAxjmEJinJxssO/mlSqMtj27oaRiCuk4lBJSXW+dVcK/d7hzfA63QdNj3ee+dHAIA7u+l+9scjtKJ+ZmDgcuarC1rNBwgmq28K5OPH3/u+Qrieub2LxTxVIHaPj/DkVJTL5hZ2mqp+bz9idG2qHq7OknjCGVnMpQLYr1eqPOiD0YxqCLGMTbLgjDy4Juoid2msZl37vtd1p3auqONyvFYpTZ8xEEngNawrkJRlm9njGaKcbI5iECGeXjK4roH3ZQVqKoGbR6i6QF5V03WCXj8LWZErMBnnLLxUXZkZJM+863t0Ib+H9L25D0LXwUimeDypyvpii4gXUywVT5nrzjr1PCLyim4wplbxqRADbN4P4oDgv2mfWVMaQPTos8CLPceFVM/nF6cK1+cARcIYsgovztWPNMdk7BmLKD+HEEA5+2wNYsyQxlD2IFXmi+hFSCv0axiBUtcVUIlgVFO5EgMwAVJByOrEIELMyh6oADPYtzIFhS3qjEbKhYuBeAyYVfjAWIeZwDRpq0HQMj0VxWDKVZhWVagJRpEwsFwqfBRhKPepRS9wOMqibCjia9cz74NK6gAOOnxXVCylLXBRHqAUyJTK1ADCmtZklD9I31wvQvJtgLSmZPRdZGjFBUxJdRBAnauultFnKBl7tLI3912LLovNcA8D6Q9B53TLgCACOutli058y2JLiEt5niHoWnd2EWCepP3RTtJnjSc1dkREo48RVnxQOUBRDzFERTbVDcHK5z13O+1nTVOVqmrLCsN8fLrAal0Ea/J3MUQIGaGUoaDGoFfYrcNaFD4fPHiM6ZaIcVVO4c5d6xGRK9/lc/cEtnd4uIc6C7udnOFKBFecA6QwjsqNcPI09UcrtKXt2Vi9EOuqQpT9pWtbfZ4hBnSCCPrwfkLhzDuPo+NUFTw42CuQ2MhKQfDeo0Puj6DV3QzxX617PJV9+vxyoetO01Qaq3R9qaRacjB1XifyfGJFjxAG+zRKzE1c/LSH81PRa2AVhrFkdI6TY8R8/9yBMhS4CrCCWsviQzAEqHac0fssckgZ6VKoGirqM6xOqhBlHKyDXmGkCbdUoNtyozCydxtbabXQOKvVVjIOMfsfGjdYJqSCORB1iREDiHnxPQ6xzGumsn78tPZzOIDDoD4HtB7LqzSoumVXYIAxYns7cX4uLy8wbgqfDkhQoZEE0wzoYkYhokHhyWTMrCMH06eD4dTJIhKjYtSjjQiivFcNTSIHMAweLKD5oOqcK+pAYA1wQogqK8zR6yKsg3jgME9EyjciMGzmxTFr3TuAtW+qvAAPFEqDD9ckt/NkMsbq4dJ71hGUB7khVzaNGItc7k+0DDHr9T6MK8qJNBjYJNczbFROFtHqQS0O4JlWJZSLyXUxhE/fTg/BkQeQO8ZiIXho6bvgfeFMoiizMQguLwyVQ5UhoxJUrJZrnN9Pi+PVg4B1Go64elosGjjSYHFkRJt2qIOsmHizws5xuuDWAaG/lP71FhOXcdajYjIaLKpODOfFSqLre3hZ6EMAtnfS4v5LX72Nd99Jh5/5ZcRY5MofPnmCpyeiKnuRbrqunPaXq6oSnDNjZyfPpzOYfqA0l8v9MUP/gL/xu/8mAOBXvnpbVbb+4Pf/QCGlLzz3Cs4uv5Ou09QFZinQk1W31MXWWCqqan2HhYy4q8Bgmdf5YGbjEoejxPt788cP4IRTM57t4tEDMWo2PSajtDFXzRS9zLNWDjar4GBkflqCcgEiWzWH5VDmQpJqvj7umQsMOfS9HlZ4oI7brhcKQ65cpRtiDn6tLbyA0EdwTjhQxHqdDrYcCXWd+my04xAk6cXZrNgYDRINLIwssX0oyY7EUSwQkQwHiQOoVuaS1HWtSa++82gzlClEGIHA5U2BnVEYKcADtWBOnCuk5SnPCyLS9Y8lbWat1TFf1UbHt+97lfWOXdR1MUbA2J+9yWza//fNDPYFlsBvtWjzmQp9xxp8d30LK2O8qsaqGEcZQlmPUI8SJyUyIwyUtq3CHBk+w/tDSWSyQNoDAkh/3yuXNhIQM1ffVco/jAxN0LHAPslYIM8RHwaQKtZ9juGVhmAH+gPZsD7aSiH6oAT5S59RjODZMEIGFsqh1jmHZEAuyWV2eg3KCbAYFCJniEAxw2DzvkrXgi8NUs3A7gGFk6fKoMBg7wslZCNT9BYGyWXYkhQ1RGBb+iFfWz/QDPqGSflBDNIYIMZipN4O7KGokv0FpJyxPjBCHmRsE/cTQOjTPXTrHv0q3UfbR3QxJ+yLHkPfBpDJcqNRE9ATWeemTY3xND9vLgdi49SiwQdS9eTIhErWoJ3d9HcjV/ikjhoc7KVxPJnWuC+G3U/O12i7/CygSYZaknagYWAdtB8DdyDhAzaNUxis972aoFuJuUaTEY4O0t73yisvYkusRk7PT/H0aUo0Bx8wzboUBrh8nO4v7zOVKfM99F6LGb73On4Dir3Clcyr1b1HOJHD2+1bN3Hz5qF+bublLpYrLER1e7FYo5d4IQ/5vvfwchj0vrA46xjLYQtl7JEBRrIHmbKMKFeUwUV5FVTibEKxgBmwRZUDiwKJTAjpUH7WuQNkNKsztcI2o+pXQG1VQKxJ5BDK3huZs2QHADsogpX5piEt2QE9LaKTJAqiVxh9XietcwozNdbBSFDrnFV4tecejjP3vy/0uRy/G6vY78BRFXZBxTomQVtFNRQEjj8b5LmBgG7apm3apm3apm3apm3apm3apv2CtJ9ZAVQoITuF+BkwDsSbjvapnL4HeL/xZP9a6TZ9ViFdDj1QiIpfHlEzgDdygV821U+8Bgwy34BWCsCDoicDCzG8zhms+rDW03LXdUrk7rri4Wc7GqhrZuGaJJqQfjaatfchZN0FXDNmZy5lXs0aFhN3kFfICVAgoN4HRPXVMgrV5EG1TSuOA+/FdMkBoVb7qZTZC+QjauYOoGI2i1IBSUa5mQgcBiIRueJbKjKpL0rV9bpBefrZOadE7EwejjEWk2tSXn7KnmiBktTwMmeiViuH1Tpd2xsgSiU4mIBsj9WMgNFE1KGmFazoIWzfSm+otwBjU+br/vwtUJuqesu2xVoEDJx16CSDbD0QvIzDVZe7TqvNIXg1XXVNxIuvpUxfHQ7wxo8/Sp/hjJqS9hmGvPKlssJBIRbz+RxzeULMDO7Fb5CuK2rl9uaP/hwAcPH4TSwETvnRR3fxwovPpf6oPSZVVg4EaCfN4fVVUuekUKFpCqF/LQ/AssUqZwt7jyYL7sg4uLhYYTQWov/9x7BNymR+45uvK7Txj//pP0PXfpDuoxlhf0fUSCWb27atehtaaxDlea+9QS/rTqrwyT1VUJGG7D9pyGjVzDnS1FZFFTrxHYyxB2UIly1zMX/ucP72bVSISF1ViiAw1iqMlIjgSJTHshkzs3r8VcaBxScrBo8gFc+qsiVrDI8wgL2nzy1VQWZW0+qqtpifpzF7cTXH9l727pP3RkIjvmZVZXTtSl5U4p/lnK4Hvg2InBXDct869EIib2qCdSVLqivKALY6GjVgUcrbtM+uZXEhhwILHjtXcJFsyv7CUVUSS/oauvYyrFr4hUhwg7R8Nr+OETCy0TFB/dbyfuFcpcgDsAP7rELZD9LMPXqpkBCTwsVZFP3gK6Aq3m26nxlSoRhG2V/y+HbWqfJk5FhULwdzHOwRss9bLNUELT8hqqAXG9L4JYSgt0/EpeIZGU7wZFarCkbfm0RxSiwwhKqr7ysXfzoVcsEQUl0EMzAQ17DGqOcsDXUetOhXxKxiLKqKZiDclr52fnZAK353xueqSURoB2gmWceYirIggcEZnilrdt92WK/Et893qqKYqi+yP44jGqFcjKYNmlkaKzs7mcZQoRnnPf0KiGPpL6PoKJhQYgcTiwBOleFynVY/rGM047TmHR2O4KpEpzDNFR49SRWy1TpgJRBPn0X4bFXgv2BVwWUG2j6refeobIbGs8ZrtcSKIUacnqdrvPPe+9iWCuB41GAi6ps+eC2XBS4LrarIM5QatW47jYeSgJ/E2YEH/no5JomYX6X49z4eawV5b39HBeSenpzjyUkWh+m0clzbjJSxJR4lp/PCB1J6CQiwMpAb5zAeVeV7QUBxWbyHuaCqrCtznIGhtFGutupcoQFMc8A4IBpStwxcncUbrQqcZSEXWEKkbOgeCzWLGSEH8dZqbGMLoE/7JTAn2CZEODLfXwiKXOr7CCfPJcd2ztQwsmYbsvq5fYx6RmIfEDKWGrGokWYovxvs9YZ0eQkcNYY3tlL0XAwOxfv709tfUb6NYIbFwoF0blSM9ADYbgg8MGIGZEwPOvMnFD4hD/NT+G1QHajyEemuPvVWBz+SlsszzzCEMCj9ls8d/mxtVeAZqhRUrm6MgRXIQQherROIrt9flMNDVu8iUyRryRBowD3Skrv3kD0yBWu5H2O+N2UnXOP95f/rN9cfTTFq5gL/0AN4DAVaQk5x1hyCKjTFGAsMI/+hGXzZIZb1WiPFmjOzQmlqmxXKWDdg68r36D2rkpGhoJtM26V/LxcEloPG6EaH0Q2BrE0sMsq1GVtMJ2IQOhuhlg0g2HSQCqbX217wGl5UEv2EEUVdrA9rDZBpxOhl09ItOZIG59YajG2G/0Tl0C1Plji8JYFCta0GqysxYg29L1yFEBQe4X3heVxTrrw2XzJcF/jgre8CAD5g4M6LCc515+UK+8fpPSeLu3j8UJIgU4sXZmkjWuWFj4EoB/N11+NinSXkCTWnvpvWpjxrGdsjY7Ajh8n7Dx/h6iQpl337T/8p6rwwo9fDXvAR7LMhaurzuqpBAqWEJcjjxrrvFZ3PhuDl2o4MbD64DPoic+9cRZqwiig4fedqHbN912swzJ+SRBmNRvqqNUaVLskW6fAQClwrL+4+lMAkhB5Zlty5oPBeMq78DMJylQ51Wd3NWavwn5QUyLDgAJJDOrUWfU5O5DjBBniB3oXoMeToLoSf5XqrCbdVXyD8tQSzoxiLgXJXYGeeE58y33+22vHRg2xZOzfts2m6j3BZG4xxCj921g0SG4wgc9iSQZ+frwZzGMCoYoEPwhZVaA7ImU7rHCpT1rr0uQNOLQExqycSMEwO5gsxB/Q54BaOTuUYvckJHUKfVTkrA1ZlTNY9OZtqh0FCFighQKQCNwOgyoxs+CeUM+HLYczygDfHQfvJcglRCaxc2XxFYtKgnchoPzIVvj8z6etEpuyn2s/l3gT3Vl7P0FdjFF4Xr/GQ9KsWyF1EdmhIl818UeZyPiUghKyCKJ8bSE2wmUOJ52JRS+fACPnzJGipnUWU9WA2G0jcRwtv0utNDWxvpXG6uzcG8r6afT9CB1Pnz43oRHE4BFJOal0V6Ksjq+f4vB8T1SU4t4SsgzudEpom7Y+TSYMtsXm6vOpxeZkSEivh+gfvwZqIKwcrRkTnMyy4R2dKjKa8LFlPadWiFSj2yeU5xtN06Nvb3cGO2DXYqtAR2tjC9ZmilBOUQfe2vuv12Zuq0hjNw5cDg6zvI2d0z2z7Dg8eJ+79ct0px+7s4lIPqKuu19imkUBq0jSgTPcgQqZLcCxj2rJV7pozFZzcR5dhq8ErJDYVGcq+m20sElUsG8cXddAc+xg2GnoQWMcKGVKenTElYQmysvaUM0pVVSAjSujGw0hCFs4AMV/bwgxoXMqNln6xREkPJN+b7LcwAZXLNjissY/avhmrEFBCOThGH1EohREs9xQ4wOp1clKP9WB8DVYeuaxLg4OhtdUg0v/0toGAbtqmbdqmbdqmbdqmbdqmbdqm/YK0n6MCWqo310zT87+xkDh/2t+WzOIn4JsD5bhCKL/+nk+2axUvxnV46eA6w3Z1lbIbOXsNMLo2Z5RKdt05p5k352rN4ucydz1y5bRPpfpirUXMZq7GFFgNF9Jz0BN5yWrGyCVzQQSqsgobYCV7FCIhY3Py58YYtWJEQ/Ln4Cw/LKKmzKG8ntOCZEsV1/cgV0j4GU4Uyap6qKtHWiEt0JlybYD1msaYYuFekQAAIABJREFUgeKgu+ZnlhEDrsreYlYrtMwWWTxxuQhYrrLPWK8QWrElxGLBoCZ92P6hw9Z2yuI140LKtdZolstYAmWNDspKXyU7zDGCKFcRqRB4rdWqaVMRnFSOL9Hmr60wk9Z3qDNM0ED7d2s/4GAkCp7P9ji/St/r8eNUkXn06BJLEaDhvihS8rUq+aACPhjeCmEMpUK/czTBS5+7kV6ngGWfxGjWpwFmKuPRWBV5yXDLPjLWq9TBPUOVP6u6wkwyUbWr9F4yJNlWDi/c2JFrPEYtxol8doYLGfiVtTo6O98BnEVSUrMO4IEiIQ3GuooqoFSOa2e1AlgQ1zwYo6W6YW0ZE8ZQySgaoxV4DOayriOxVDRCCCCp8puqLJnee71+RgSAo2aNa0tF8GEwRzhGrSgGBOREZJuvOAJ6qXiMRzO0MvB7HxCzr+BkBiv3UsS1CG2fIS5lJWzGDaqsDeODQlQJVqGrlNX8okNVz7RPc4fMlxcqCDOdTIvfVWBU9qdsApv2/1vL1S8eQPnsgBAQ46D6FctzDKZADHMFKzCp2IGPPknJIa2VIQu4xKhqh2bgZxuVouB17gSKqmSdtmbSnwsdoaCTsodv7DqQhCSeSWGmYAOyBU6WRcvynmkiFaieKd81wUilumRNoUAEr+bQRisT0EqHIdIqC8CK4PGMAeTLqFwLVYOKncIEB/HSgP6SvoD0hy2qz3GAotE+QlRoNlAQQ4ghP4pURdE+FdTGwEmbyBTooveKyAGx9g2IS5XQ5ts0IJthsBaxyz6igBPKRQyksLfcbEUYjYV6UU/RCJKhqVyB7DqG2NphNHZgk+GN8u/aK9TNR4ZT/0kLYzLiDFr1bWytsNROBe9Iqy8j61APRPxiNttuKkxFebQPNbyout59mGLGJ6cXWKylwtb5Aj2MxVg+is5q6v84QFNB7j8gCFLDmQrrdUakLeEFcVPZWsW4evTYydXFvG+hVGudNVp9r+oidGcHVbHJNC32s8lM0RwXiw5rialCvIKTMbvuer129MWbMIvOdEP/TyoKmAYMherEqOPHSv0ufXnlZelYs5bAyBDKAl8nisXLmlnVXEjV+EnXH1hX5iEX8cBoLfo8n0JQ0SovrzUGsKMsUOMULV9xKBwyJvW+Rey0qpfVbuvK6msAgzPUIQJSeIW1UFhwhnBXdtCPYL0cMWusEmLQ70JkdZ1QoN2Q8DWgusXBISzErOGMhNL7FL/1YfsrQUA/CTvjcpef+p7h/z/tQHdtsRt83qfDGX9KI5Ry6ADb/smKZz4AziYpSLLWYSWQq6qplDeUJF1zcDjEkWYI1CAg77sC/XMliGKwbnwMgLOssCw4MUa0fmCBEEqQqn0TygGk70OBoijukPUBB+/LIceUw29CbMriYY0OPP1KA/Qmk9UBH+MQDEcY6GyrdPiQ6zT4xKLIFcthNKLwKsE0CMRlUkcDisLBXEesV2lWLBcR65UcHhqCE4hIMxEo4rZTk8/pzGE6zZtMpYsqxyI9DAOsQ4IA9105yOYNwtaMWOf+IlQuP1ujylQUASPKcJWSFeMARlPUwELoUYma6qgeKTaf0eBou0CxAGB7u8HZaYKl3vtojtWycEwUGjoc1IOci6rMVaXPF+s1/vLP7qfXLWHvOEFO9o9GMGsJFCqHxyeJ+9fINVoGRAAM/w97b9JsSZKdh33uHuOd35zDy6Hmoaurmk2KIpoA2ow0oilQErSRtKLMqOkniEbtYKalVlppobVEyCAtsAAhygwGAd2sAtjd6Lmquqoys3J47+Wb7xQ3JnfXwo8fj5edlWhRQG36nraud/PeuHEjPHw4fs53vi/NU/R7NHHFOdKM2jdJUHsoNT3vPEtBRKKIjMCY8P8KBnOqO2vrBjE5L2XTOkFWhMkRrWX2LgGJqMPiG8RkLVKCBSUSnXqjMM+kqaeLDfT1rs9RYEcFBl23EaRgAbNlBgFX3Qm0KBUEWhtjAqNwHDFmikWajWXKbtEZQzKSiAnfbxACYJGKsLV5m68bAOIkiKsnWQ5BLMhQJWKaPzYmMXxPyGhRczW39CyisIDIRCKjgAlaze3rmIppU82i2hmfQ+uWnSxhR1gSbbkSMXo5zyBQVHe4ti/RWg/36oqFS7DzJYI0gtW2I9QN7hf8ngls2DCGx4AVwTmxIpRfwRpYE4It/hyW6WXDJgy2S/sumXnSGgWRMObS3ZKuefNiAcRMf17zOmhhQo0T/bYLLwV9A8/wKRCYCI0W4GJ9KTqOpfdlTIhICcOBPyE1fHGgsp2goWhQUV2cr5MUIkbr4XsqjC1X9hccVj9/O9ikfx98bKh4MaGBBXhjjk5gBsCVWk++J9/mBlzWASGYGdHR6tPGtjHIC7cja0Z+A2DQ0m4wshqtr/O1Grb2d6B5I+SncmUNEoKp5zJC5teOWHCQFUKzswzhgr8AGKYZR2kITEUCEc2hWliWIhHWsmRFJGJE2p1wrlw9mxItrzlUCECvVWA+1uCgbqwEZM+1QUu/pxKJ4xMHC10Kg9Y/b6M4CGGsYtivNTYEALwvJgyz47ZNwwHeVVWhJl9FipIZS60AjGdip2CeVIoDl4nKkdLncRzxmGvbliMqOTGKZsOMa38bLGEogN4a7Uog4MacoPEXJRJeMcwHDeqOT2ggGPINofgYjRDgsJAM/UypZKeMbGccBmUAKWL2nwyiTnS7I5ruobROR8m9FoIDT6GlARig9ptSa3jj510FXQuumYRSiHhOURDEqG6NZebduIkQE2Q0TqgmUsUMM3USSJSoAELNoemoDvg5wBre92jTdnhBgiSKkBKRJAZPgfC+3wxH6rkJN5WmzCOgLdhpstpwsPeLbA0BXdva1ra2ta1tbWtb29rWtrZfEXthBrBZOijZYrEI2iNti52dXQBXIZRJkqAk4UqlFGfw/PeEENjY2HA/GkXManTz5k1sbW0BAA4PD3Hv3j0ALoqV5zm/9r/99a9/HYCLdp6cuKLWuq75e/P5HJeXLruxvb2Nf/Yv/sNfujEkh81Cxi2QdtquXOQvfc6/Lrvxn/0uAGDx+cdIGheV+gfv3cCru6RrlykUhdOVefTwPjYp2vYfvLqLP//eJwCAf/mvfwAAqLVFS/e3cXMHr77xCgDg4acP8fgBZY8gWY3Flf/7bJQzIyyT1TQAbu7fAgC8+d5rUB6ao2KMxhP+3m//xj935/YQyyiCGP0iKY4UqsOKpp6fDeZH9exnX5yV7toVuLENx2qESI0xhlnMtNEMl/wXf/Rf0fWnHPmMYotytaTvBbifQUhj5WkGTVHakqKvJhEMnxBIoIjFqhenAeKMQDbzvK4nBLjY21oB4vuAhMAxQU6ffr4KEAQjcX5xcaX9RCfrrYzlonorAeOJQGBQ0wWU9L0cEnkHBaRF+OtRkVGHV6JWlouzZfdZdV74122HWKKD4HHkq/Tv/+G//98BAJPxGL2+GwtKSIaDLosZIF20c2dvE7s3brhrsj1Ij/qg0KixLUfuvAYjAFzOZtw3i2KFonAN3Bv00RuM3HdpHqyqikmhqkaz+KvqRHGNBYrC/eaqaPDP/8xpQ3o8jxChsN2obhs9wyb4/2k+el4k0HaCrva5hwaIbR6SOt3vAQwH/vjv/RKXsba/Fnt8QrBfLRCxwHBnRAkwOiSJEmawBgRHi312L4kT9Cn7AVjklLUZ9vtoKtfXLy5OAmysrLAgfTEvNj8c7WA0prGgFGYzNxeenh5jvnDHVtWKGQx1q/Ff/o/fcsfTRadpYLWuWw2SKu2SA16x1A13ZIlASVC9ur46V3RIuf+djaHbIbEGgUBc9hu//Z8CAKrKMrHVoJ/jlddeBgDc2JtAr1wbnB48xGzm1unlchE02KiNimKFQeae28aoh5bYGj99cIizS88LLYP+MiTPK37NNEoyiqjfy7C96dbg7e0x+uRT9bIEeY8Yg5MMf/fX/+srNyulYqIhpWQgsBABRu+eVbdFCI313Gmpk/mEuPpcPOKA3zMB+WEFZ3K0DoyJ2gYkmdYGXnfy//if3X00bdBh7rLOWx3m6raDmOiyLfs1AAioJSsT2ITTVVDUKRMRssy6o3XIHIJSwMpwvppQMcWqZEg9IHh89gc5fvDgkH7HZ99VIPoyhonFjAk/JIVgJtxu43IWugMFFDa0gbUI8GknxkdXFNrFwzeN0Wi9P9R5FsZaxIxWCsP1f/lf/xWdJGJtxSxPYWiM6Lbm7FZ/soX+wJXKuP5GCB6f3RYGaeyzoAli5cukgIqgu3UT/KDapcIAAK3X6xSSIZHGtiGhLgXDRa01qBqPIlvgf/rREQBgbt2c2DQNowdgBZO5SRlBcGkFGHrLqEIbYOq6k82UNviKGobHX4SA4msoLetKzHzmEPwsgJDJM9bwODJWItdufvmH/zGea+sM4NrWtra1rW1ta1vb2ta2trX9itgLM4A+ih53dLBWqxXOzlzEOk1TpCkVVUrJmbw8zzkbyLpaTYMeRRkHgwG/H8cxZ+8+++wz9CmKP5lMONrtCVxu3LjBn3/00Uc4PT3l6/S1fnEc8zH+7y/fGG4/nAqD8R33erjjomRP7leYHXvaZ4UQ3vqC8ORfs/lfboWCoihqog1y1gQU6I1c1uLmrTv4/LOfAQD+/OACMx10AwFgY2eCndsuu7t9+xqGI9dOx0eHoWbU2CvRgaC56OshBIdab97awatvXXfXZ+Y4fOyyS1vb1zDZcFGdy4tzV5eETgZQqSsRKh+xk1IynvqXygByuOrqcTakLH7BrpCsIPRTaQW0J+oRnbIQ6+o66CAAQJqlXFOhUSDqUy2lNlzXqZuWC6ejRAUNH4r4WWWha0+lblk+Aqx8hSv3b4Tl32QCA3u1PbhWx4RMjRU61ASJ7r1TlrFTm2LQIUGwIZOnIINWFn0sEbJ+RoTyFYPwOIQEZ0EFqO/gmcfSrRn257OhpvZZ47e9/IIFFOkcxkkGSbIeWRQo00eTGAnpjAltuPakrn1NastzVK/XY/SCUlHIfBuLhBhVTGtRrXyawh9g0SeGg94gRu2jidYyIULVtIiori/LJLqs7+6v5bByt+t3s93PacEvrJvukrk8a6FPPPeLV19foZt/fsZwbV+O9TI3Z+vGMpW6M19jFzIFUJbTK1JKzhp4hIewFoKizEYbiCTUVM/PXdbp4vgE12465E8vjWGtIwryMgBKSc4UCGMhPJmEsIjpfCpSrAOom5ZJv3z/kZlEHHkCKO1LYNDaqzVyG1tuzG9u9OiaG5xduPpU04Seqe3VLvzXYTzviYhlJaZLouhvamgimOvlGQZ910aj0RAl3PXFsYYg0pNIuf8D4Hq1LFHIidxp3M+xKr3mrITxhBOd+Vm3hin4WQJBhnrNWCpuU2ssSro+C0BF3ndruH6MJQ0R1kQnb+BboKtDG64jIDgsTyZSiCs6arxuWRG8JmM7PgXdkw1rWKj4pBo0f3021LgKBEK6luAljh+CzhsFsh0hVSDvES1nEaWMWLPNz6Nam6BHJ0NGScNwhkoJwfWAMA2Ulw3wGcBI8jU7fUY6d93AUv2YEQaxDNICceR13DoSFFzyJllOC8IEbbmOs2b4XgW/L0Sop5cC8LIeAsGDdfKXkl8Dru4y1ONZxDxYg/xIq03wRYxBRJ2obd09RYicBBOcHIjvLHVVQnv5kLSHlqBEkemx1p73KCQCGVAiY0jri0gNlJeSsK59AJ9lCz4HAAgb6gwFIu4HFoD09bC2BTEJoNY55rJPx/j7E8FnEoLlOwAVnrlw1bHuXfIDJdBJ9YVsvQhZPGEFH2NEG8YUEetBCiK5cr6WsEGiwnBNswSVJKMyEivvnH2BvXADePPmTQDA8fExb/peffVVhl5Op9OgDSUEhkOnB7axscGbuq5TUteuwxdFgWvXrvH7h4cu5X3v3j289tprAICdnR2GifpzFEWBR48eAQBmsxkP6tVqhc1Nt9Ho9Xp8TUzw8gILeweBhDBhb72Z4K1/6DZ+VeLeG+/28MM/Jp2Y5RWP+0tBhFq/UYozaC7eb9Gjgl8IDZl6oUlgvOXa9/sPPkdSuwt8/d9/DwDQ39mEIsIMJQWWxL6lkjQUIxtzZWvrhyNvqqTE3o1tAMCd129A5b6wOsJw5Ba+za0JWu0WPmtrZiFl7SprGXLg/i06f7ueb/j1cKz/THQ852dgcc/b+HHRffdzGwpnr+hgBcfZJfDJefFkBxKsE5gPMobEGmN4IzE7n6Gn3DOKkwS69cRA7lkV5ZJFPK0NrG9KREhoQ9O0LcPsECmGkfoC316vh51d55wN8iEXhj/47B4zkNlIhMXOdBbEK6y6oY38Yi0sAvsWBE8Y/SsEUPRpN2IgwgSshWBCFdnZaOrOBvwKJsgPrQ6Wy+Jq9/DnGE1c0EkpiThxY1YlBsXKQTf64znKhZuvlvMLJPEdAMB4dBdI3OTekGMlTMywcyklB7cmkwglQXfqqoUiiFZV1ahJz7Grh+qFoQfDHnJC1tV1HWDGqxIJ6R4KYYMXqMMz8c+luzETUr5wr/Ui4qwXwaG/6FhjOmOhCyHqEHdZgAku1vblWd5z/aetGmgPJes8LysDFE9JE/TppOX+6TX+hDDQREhhdItI0BhAg+XSrXmL5QwCOwCA/miIOHNzzKrycFKg8kyRFqjbUA7iSSmMsYhpkjSJD2kijHeDDjmCZKcmlsEhvbar8M5XXgIAjCducB0dnUOXTwEA9bxG7aHdv3xzvtCe171dkIbuRbt7rasKETEE7l7fw+72Ft2LgSVfRgmDyLNhSovU6+QNCCKXSOQd6KUnIUvjGAmxMddty3BxwPIxHtMeiQgZlRL0+xn3FSMFag91q1tERASCOEBv/dZMdDZ3Tp/Zb5Su7EbC93iuDhuzK69FJ3wlAqtlF57JGxcgQFwteBN2NZBlA7EOLK8lnp1WGh1oX6QAOhu2QFAoeYzESY6E5uGayFcarUmP1/XLri6y3/DEUjgGXAAxQmBVRp5ELeaAt4oS7uvzxbID5dQhEBBHYbn1d2pthxhIXAlGeuIwa1QHmhjaX5uw0QtztrgCt7WdDTsHXzt+End/IZk4SHaeMkQoFzIIZFCeOC1PUiTe34wAy2QuNXTr/MP5pWCyxNFGj6Hlnq08jgWTxwgh4WtUBAT3exmpzm+D4dOe7CpSLZdkSASSQKEMBJ070sR86j6AQBN+B27+FH7zKYFYeTZmGyLhAvAoYu71trOmK/AYsjoCs7xLGXTbTWCgl36OUJYJuozVkF4LVlhErGNo0TL0tQVYWP75toaArm1ta1vb2ta2trWtbW1rW9uviL0wA+hhn0mSsNZUXdeYzWb8uY+Ya61xfu6Km8/OzjCdTvl4gFLbnYych5fu7+8jIYjIYDDAnTsUoR+POevosylOq8/toieTCf9G27YM15rP5/w7/ppfaB344HDL7Yff+fsjbL3iojpP5y4Cev2lHIc33Xk//3kJ3jt/ScFvmbh7kXGMu7f3AQDvffVVJI0rto+iCDFF+uZWY4MyI4+jExQEOdm55iK4q6rCgorOhbasnVJZCUt08G1bhQiaknyfPfqN3Tvb2KIMYD7egMpcP1guCng2/ixLoH10LFZXIorub0dyoyPl4fJ4PuJ4FQb3rHU/6+STnrHwXic/2IlTgrNwokPg4oKdPhIpmK3AR5HSPGEK8CxXKEjGIkoj7ntN1mLSd9npfq+HXuP6aTJ07TWfz6CML26OURcu4tRL+9jddTDdZbEMmke9jLPBnoykrmuOkvYGA6b5z/JjDIeU5RIlR6NiGV+RHnmRtUZDMRw3RG6VDi3JVC1WQHnYtq07+FnFWVxlO5HDrtwKR8rCS4VAGNMiRCW78NJtaiNrNOLUtbmwSywvXdYvsTOgdK+PLx5DeEKY3T0I6eaVgiCgZmXRUvR3uVwGiIuRrJ8VxylTede1hqZ5p6sp6Snrm6ZBb+iy4Uopfl6mk31WMqQ3PNhJigCH7toVHdTn2IshoFfe+cJzWHeBdB0i3NczkGkfphYAZ4vX9uXZHhF7NHWLuvbaYiVqKr2QkeL1W0ZhmXe044HwwL0w0JV7Xesa5YpQG8PcHQ83yj15TBRHjEJJmKAtkEJIIRDRepVZwRmV1rgotju+CtBP331Me1WaggLjsQI2HbgI7729j7/9npNNSSjLkogWR0/cGL+ImD0fEn+DBRoWjpYdgPW/Z4FbhJp676vvYjh2c/zq4hCWkDBpHGM0cO9HElCURdSUsbNNHCjzjUTrSfaimMl5VAMYosdPheWsV5YTIdx4iMnQzW15P0ea05oO2Vn0JI/tptEQPsXUeSaSX3ekPMTVNZfX8isrKz9Q/CJJG3/zSlvSJbm/NsynppP1u6IpCcsyBAKs1IGY2hFWhXlMBX1mCBl0EgWQU9ap3+8jJt+nJNhtVTVM1BJJGTQNI4GEfMwoDggkCcP6qIwIiYLPKyDR0JjL0hhl7nypRhs+d5RG3QQr//XQPyUFly5AiAD9My1LI8V+aZcd2RIdTqhNV18ywBFd04bMq/+GlxGx4WM6lLJmogMUQSB1yomlKU0zxLGXranRkJxGVRRYXj5x35udsbyF3L4GmXhdWv/sDaOZagSJrLIJ8xKiiP0a1bSo6HhP9qJkwtrWSqYggBWsiCC8nAYUoxBUFPONeXSXI2jz81xoUykFOyVCiqCDytlVFdBnsCzPAalZT9EKA219pjqMI6+TaaDgHTCJCAykg4BUHvagEPkSJRVDdyCmz7MXs4ASlPKtt97i995//33eAI7HY6xWHtseM2RTa80sen5zV9c1f962LZ/j8vKS3z86OuKN4cnJCW8edwne5n8HALa2tsLDVorrAcuy5GNGo1HnbroTUZi0gm9kMblOmh+7FgVNEoo6Yj7U2Lzmfu/hvTDp/81auOaEaj7aOMF//k9+EwDwT//Je/i//rffAwBUxRxN5C4qSWKG425vjfH4kJiMLt2GOoKFosHRSoEFac/NmgpL0nGrrcbepnuGW6Mh/FK6teeYXEU/Qj5yTsh4soOi8BCiJXZ2nFMeqwSKKC7byATMfmeR8XdIXZ0/532hDdDLK44rf/7sWx1ooujMYs+Ym9jCgAwwtqvQkudBQBPSPZtsjFDrOX3NIE8pUKFCfUieDTAgdqs8jmEpVd+u3Oc9m7Hjfv3aPsMxbGvRpxqSPO+zduVkvImNrc3O9QFPj57ik08/BQDUjeXFrtcfIiaoodHAiGpEt8abv9ggHRMisFtBqYA+segwwNFbQsD4DYwBhF8wdcM6X0bgSo1JJ+bynB8PLyVCLYgAULOeVYCKZRSAEgKsnSd1gky4sb+8eIxy6vr9crFA3Kf+3byBybbry9T9UbUNLmeuflVrgYRqQrQVENILwcZQ9FqYih0cf51V3SKOqVak0bAEIYqkQkaQ0lYbrsWRNuqIB/v+L8Nm7hlt1BdttcSz3pl/v7MBtABDS0QH7sw1DlBQwmsbtVhJYpu0YA0oayw7XMYKvHiJWdvfhG3T3Ky1RUM7nouLGU5PSFvMWvQJZq7iBI32UE0dNFy9c6M166Maq9ES7Gq5WKBcug1luVjg5OmxO94Aka/ro3rYNE6Y0U/KGBl5XSsVoSDx61VVOfgiAAjBzIYMe4JgqHtrLTtuaQzcuO7G+d39TQwSd33zgoSt2xVD7mQioDzMtEtJiDC1SCEZRn/VupuYF5uB4fqegYdvxiP8xt91LOV/5723cHzsnNvp0wUD95IsgbEhaO6hi96JbSXQ0txQ1Ro1OctWKER+kygC1DFLYmS0kdjYcOvx7t4WhhR4grWoaQ6qbUdR1gRUtwW43ZlluBPsC/yFV4Oz7tiwCXN/w3tXNwwd5k8ESKmrRetCPN13eF6F7UBDgzarpf+5YwzPnan36k0IcMioMy9agcizfcYSOWmo9rKUWU8jH2yNI4bXZnGEIW2qB72MN4DCGmbbjSOFFQUEVyXVpGqBinzapmkh6XW2vcmbDq1DcK3WTfB3vNY0gqOkROjSLqbqtewIngtAKm500r90tXyatf9abg8pFeJOvXl4Rr4/2E6b48qz8NBWA8sBUnRqBn3gIYkj9gPTRDlFdgAXxQVOP//YXVNrIQhK/dKdVzEiXorCdDbUvk65rVHR5LAsKkiqZe1nI74Xo4HYtykxGRdNjYyyE1kv5ppPKUMtcysBrQNzJ5cfGR8AjoMagBUcTJMi5iiEEKFu1XItoGCYqdEWHqBsG8nfa42FZUIAE/QGbWfce/4M0dFklAKSax8Fb1atEpDRi1fnNQR0bWtb29rWtra1rW1ta1vb2n5F7IUZwOvXHbPjnTt3OBu3v7/PWbrd3V2GZzZNw9nAxWLBmQJ/rBCCYXG9Xo9ZQr/73e/i/fffB+CYPT3xwmw2YwjLN77xDQDAG2+8gcHARbbquubzTadTzvbFccy/XRRFJ6bnuX0kIFb82kfB+kOLwQ23o76wK/SWJhwPIM0yDG8SW2MK+EAmUSq5e7Td2GGIhHUDixyf7zBm2GeikR26Bf6XinxWc4g33rgLANjYu47htoN16qPKFRkDWLUakiIyd67vIc9cO5YUlWrKJUeX+oMcRbWEv4FtyvCdnp8DY9e+bQ5kFKUo4KIpqulhu+eyjEkaYTZ3Gd3hcICNDQc/bZoapqGMUBSFzJ+/u05grot462bb3OuQ+XnWriY8xBUois+mddOEnfroKxAL8Zy3u6cWnX979GSSCCRUENyUFbNKDfIUBWXsIiUwoLbLkgiJctHfi4WDLzeoISiK1zY1IsoiCljMKBtlhcViQcXSizkuKZPrYc9VWWFAmbDBcMgF17tbG3wvRbkKGjJ/VdjHhnZqYTCne8l6PVdYDKD2BExNg5LmhhRAXFI2/y+/i6+845ADvWvXQtahY/IX37r6OQL5kHsu3WyY+1uTvlbeHyAjOE+iUly/eQcA8OjhR1gPJoqsAAAgAElEQVQROcVqtcTpE0cidbr/CYZbLosy3HRzzqDXYH7u5qvpqUJbkZbgcobMMwtv70ESO1tkY2iClBaVez4ePgcAo/GIsxFRKqAoIyNUHzHNc00dIsiBGKGTAezYvysE9Fkznfjks2ywsJI1nXJY1Ax1FxAEJxESILABrNVMjrC2L888w2RrA9lFWze4vPDkAiukGa2JKmYMU102AeFA646yhuHVUZKhIcKjn/3sQ/zoh98HADw5eIyUUCjbW9u4edORjO3ccHDMnd1tTGjej9MILc37bRNBgEhSygLLlZsrBAwCr1WH55H8hX4EJATgGW0A1zeJ4VcUWM7cF+dLmv+qCkp5xnKJUnpCsue3nXlu9g/4ZTJ/3oQQiGhen2y6NfP27g7efONtAMC1zU1Mp0d0HZoJLJJEoSVehiyNeBJcLgmC1mrWJGtawxlCqRSShLI9EohpEk/zFAmhlHhdUoJZRSEkFGVrsw4TodaWycwsbIB0+4yHAIQKfs2VtfA508zz1kzZyR0KBOicYzskJIkK5GRCm3AW9gs6LKC2szp312wRyGbynuujWRxDeYhw1FnwhOAsShwpJF6zWIAJNjLfzkgCg7cQnMWSumGykbqpOeuntcaC+rehjHucpOy7CSkRUVYnSyOknoRMGywp016tGm5F301F57l1eFpgrISt3H2fFyX76JYInbS1gCCoaiRgfQawaZiVs99LIXPPXPpsrhcuo8dsUoCgchVrLSPwWmM7ZQyax5dnfVexRE7IskEsMczIPx+O8ID2AdOzp4yWef31r+DWdTf40x6Nj2LOqJlyVaMonM96MauwvePmoFG2h5JIqSQEJD0vQ4KiwgrAM70aICOiGSkFtCLiH+nYX90xKpAfKs/qEjSZhZWs8ejIn6h/GMFlEb6vaegO86pAQq+bWCK4twIRvW9lEx4BjfUYFoGUJrDnRlKwXrUxFhFDhKO/stTnhRtAD9Ocz+e8Gbtx48aV+j4Pz8yyjKUYJpMJ17t4J7UsS96YZVmG73/fLSzvv/8+M3sKIfCTn/wEAPDuu+/y7z996hi+BoMBw0XjOOYN4GQywXK55NceuloUxXPwZhqjXdco1/dz9Pru/eGGwGBCzu28RkMLmPEkOlog67nfe+m1Hh79nKAxZQer+ws7FJ8q7kKw/HtAYDKSsOg+KM+P3aGypU63uTHBrX3X5pVVkOQInM/nUKV7RqeX58hJtHQ0znE2dZ1jXly6W6mbALtpW56Yo0jizbdeBgDMFjMcnbrjjQ5Ofr7lFrvNyTZ6BJ+IpURK8JQ8G3Jq3VrDAQAgLBzMKAZzVQy8i3rjTZvhmjeeBK1kbLVAkAqQ1kB2YCT+vkyHycr6Bc6KDnNZh+XMYeT4+pk5s8NOmdAkvjkZICWaYNmBgty6tc/HNkWFhFgmW6u5bmRZuc3z8fSCiZomoz56A7cpqaoK06nbTGxMxjDbDt5zfHKK48OHAIDTU1f3srOzgwnBcUepAMgZklnMzkSMFJdTB5P+0c/u4Ret03dt6MmrusZffuQkRRppMSvduJ7Scy10w5ClgYrx7p1X3PUvFji69zkAYH84RjxMOuenP3/VfsWGYEEswPCIbrBgeUn1yBYYEdw17wOjWy8BAJL413GPYDCt/hgFBSpWs1Msz1w7DkauT2exQjrxWE6BJw/c9y4vCgxpod0ZXWPsvRRh09XSnKO17gSENNdjJIlkqCyUg5gCQGMrrink7wnhKMOBK4/FOToB3vOs1/XLsIA6Bk/3nhYRU4N7oG9kWgxJcHcQZSiKAMnz0iWtCbIYwmpE5sVMY2v767eEAq9Kl6jI6dHWAFQvIiCDEHyeOmwT3GbLww2lF1BuS6QEa7cixief3QcAvP9v/hSffvqRe99ajCkIYrRhxsQ4JTkIa1GQMzcebyCnmvAsiyEi58zlcYwVOaFlWeJZX1NBYJMcvu0dYHNEgZlBgs0BSRboOZa08ZvNqd9ZIM/d/JKnBqvS9eXqmW75V23vOnHaK+/KTikBh4Uj4YklsUWs19d2byDL3evKSETwrNwiqJxrzVC9JE1YvHtOAb7amCBKDcFM0BsTiZ6vGWtqWDqfm4sISkjPtakr6Dal38jYEZdxkFQy1kLTpq/ROgQHPfRShCBst1RDdmv8AgIutFp3h2KfadPuIZ4F0XbmJt/indpjISQ8yFyKjsi1sWGHBMsQ1h4FQkf9Hob9ANn0G/AoihgmGscRB0GqcomaxlHsx00SISJ/wbQttJdwQsvBhbJwCQ8AODm7wNxD+6lzjIbAiNa+Xp5DkO+glGBW9KaqsSKYdN2GDRTDAQ1YOsSIiJ32s9MLfPjpAwDA5wdHuJx5ToCCrjPi9tjb2cN4TJvjKOGN746dBPmLOOaHxM/CdjbukMFnsjKwV9ogN+FYWylQSGufkhFSXzOZSGTE1n1j/zaOb95ybbBcYDl1HCIPPv5LXNtxc8Zkx/m6sSlhLQVZTYHV3AXHl9MltjcdF0UkQyDDastlOEok1HaW2XOlNdwfIwUo2jRFwqChu9emYb/bs30ahKyFhUBE9yWVcBIXcH3Wt0fglpCuhg+AjGL+bakjCOVhopaht62WzNwcRd5fbmFqH9zSiGOfHNIwJT0v40qQACCKAVu/eAO4hoCubW1rW9va1ra2ta1tbWtb26+IvTAD6O3x48ecVTs4OODMnJQSBwcHAIBr164xfHM0GrEun4d6FkXB5CzHx8f49re/DcBlGTc2XAR+Op3y9772ta+xlqDPHD569AhHR0d8XR5++tWvfpUzkavV6ko2MITzKLrTM3jlPRcRuHa7RZL76BmgaorKzzUo8M1F2GpRwZCe3t7tCBFB+T77pEA964YyQ1QqWMA++hrlJFFoKEJZlxaCWL0sFIyXfe9G4agNblzbw862i47MZzUo2In7jx8jJ9Gx4SRDTBnAx4ePMJ27TJ5nyIqUwoqgCnVhmMFTJnMMqaD91vVNjDOXIdkbDzGl6MzB3D3PKLIw2p3j6eEFBkOXuer3B5xCj+OYM4CqfQ5rTocN8WruokPKIrqgTp9CDyL1TiTVZwbtM1Fcf45Q/C7pmxYd6K3VnWhniCwadIhrYBjq5rPheZZjTIXOw36KATFubo7GyAhioYSAIcxPazQigut4FihdSwgvhpokMJRl+ezhAwwIDvraK7dQLlyE8PLkMRYXLiM+oojw/t4W+jQWkjjie72YnjHpgtUCpnTjdpSp0OIdGKyP/gpjGSd6sazw08fu96Z6hcZ6PasAa/HR1VZqHFQuGvrKr/19XMxcxG7cKmxpHyJsOYNmnpPl6l7Ts7Ajz6fmAmJ0rTQmq+kCj04cNFb2NUYUFdzb/ApeeoVIWcwUDz9188f54RNsDlyb9SkFK3sZ2oVDIyzPpljNXRYxxh4ySdH3UkPR2FKxhaTMiuKoccIQXGM0jGeLEjFH/o02aH0krxVQBLUKjGdgTaduVk90Mo6uYP+ZXMUzGUDmPbAhai+sxpAacqmB2pMcgIhf7AojgtD1hcaG7+v9HE/OCRGghjCKskEWTAawti/PfPZmernE0yM3Ps/PTlloPYpihg8KlaBHRAkm1Zx69+LvotWIaGF6+PgQP/3RDwAAn37yCWqCsvX7Pbz8yqsAgHfffhvbVHoAWtNnqwJPDxxJTDEreU2fbG5gRMgZNRygpWs6P5/y+KYAN7JYYv+au86vvDzCZOLJGFo0FM3X1RKldXOh1/kSVmKQe1RGwpqGQgIrdyjDLp81Txo5HKYcOa/qBlXTGYt0pW2rUbcetWN43ekTEintDRGTVl+apADBw+qqxnLm1k+rq5AhERoNOQEejqmNRaNDRiAhWHscR5jQ2uyZKwGgKEpU1A4Z6S32kiisVUIwu6WKFWf8DSxnmmQbiE9kh2TCoxCklIFkB12N0kBs0Z2zeR6SHcZNawNyQwhuA2O7unw+SxogydYgoIEMeG2+oiFsDPdlD1eMpcKw7/yafi9n4pcsTtDrE2lPlrCvUpc5E1tlBFfMkwSxh28ag6Zxvs9yUaBuAuQuo+xzVdco/ft0q1mcYI9I20ajAavdG2OwWhHss2lZ/F3IlEtJFHsxhmGAEkBJJ//s0UP89Oc/BwDMViusKAPl+782LeYe2psViHuuPYywMLQWlcYyI6i1FoJ8T4+6ktZ2tIINp4ysseiArRgpZQ2YOEcTO3GxqmFXbiysVIuESJyk0Lh522UAy2KGowOHUDo/OMCTex/Rub2/KSFpEMdNDUvM3k1ZQ7ekVVos0Pjx3tacgRz0U2pHyWgbAQNhvIavYl/Q6pr7bCQsIlrXjd8pWcWah8IIeCyMRMR+kBsMdAz37ZA5dNlf934rAUsImli4bCkAaKu5vyXkn+RxjIyYUlsDKEtIQaGxMpT9FQpGec3FBFr+/2AB9ULw3/nOd/AHf/AHAICXX36Z6/Amkwmzby6XS7fhgnOG/Hf9BlBKyZIRXdinfw9wkNJXXnEQsr29PWxtuZoCDyMtioLPN51OcXnpHJLBYIDtbefwNU3DG1H/PSDADl95o4e7r7lGGe0ZjDfcAL88b7E4NXTNQE2lPLahBi8sP2wTF9h5iSAzqofPfuo2nM0KkLTBiyKBOHW/2ScK6zyX6PVdhxoOhzg9cIPt/qdTxm1LIWARIJQevOgngPE4Z0ai2WyGmia+p4s5YsJFvz65gQUxH51eXgSaX/K8Z/MZzs4dXDfLB0iJcWlnZ8J1BndubqGvXftuDzKcDVz7nn3sNuXlcgpLk2eSDpDTTcZRDGsqehYtT/pu0/QM5ucK7FB0YJ0BRiKsYOH4wLhor9Qdtd5xRoTnAX3cdtEvROSQG8s4/avXEuCgXQFcIS1PEt7Bt61GP3Wb8SwJVMK2biBYMkIx7FRYC0HQP0Mbs3qlMcg8XMeilm6wz9pzGHLKT6ePUS9osOsCt/bcsyhpgh3mEV55yUmDJFHEC/vFRcZQmiTNMF+4hej09DwwpImO886QQgFDk9XTiynSvtvcb8RDRCRKmlrXBkkSB9YxtAyJqEZDDCfOSVxUNcZMdKkDZzdvQrt4xmAaQWi4CxUWsOzEct3rYoqDhw8AAI2cI8rcHLWzuY+XX3dj9c7tO1g8/QwAcH58jCc0T/jVenenj9mpg6AvDh8j9sLWwqAuqDbwMsNgg/p6Khk+kzMbaXCc2tagrjybag1Li3+9Aira0LcteFGynU3fc2sAZag/cF+4+rmSV7/jF2hjQ98dpgqvbhOb8fEKFfXHcUw1VNpgp+fuqacExNydZDiQeGvH9YPjuca9Uzd/VEjQag9ZX9uXZZ9+6ubh7/3bf4v791yf7vX6eOn11wEAW5tbWFFwBKslUnJk0zjBkKCVufJU9gkOj90m8qcff4RPPnbMfKuicgyKAPJ0gPfeew8A8Pe+8ZuIaEMwpWCwfvIE5ysHyzqfXmJJULiLxZSZiONYsmNcFBXXAPp6wXyo8Oq+m9tevpHCtK6PNU0LawPDOEOmyImN4xaDPjnWMkJEELN+2aCgmmTypdwxACiejdTDOMcJC9YvliUWS8/caND4cVbrwCyNwLqYZB5+GkN5B62tsCSY2vHpKWYXzmGNI94zQ7caNdVbckC20dwuISwMJLHEztgFrPa2N7neebFYYFVQDRp9T0OEwJPVSLxgdBzxWLXGcsAJQoQNCMPsBTu/bn6nTYmUHXh6qCH2QaCrJR0dNkQDrosythN8NbYzjwWYundMTbccRFoXnITbFBl2vjVD5/yGrmkabpBYKSS000/iCJl/HUUAyTilkeS6So/IV0I4qCAc8+Ri5vrj0dNjfm55nsPSWp7KGBmNM1/eMMpTbI0Jzrg55hp6bQ0nMPq9ATZoM7iqK8fSCcAo7+8EqK2RCQqCUR9PV2joGfXH24g9K6eHlrYtMoJ258M+ROpl3RRDvtM04kBGpIIovPFM0VLwumJ0gA1r2y3fCYysEKGmd0GlYsv5EvNLFwBZLi4QUwDn5f1t3NxybXPnzh1Uta+DLTE/d/sKvefWnDjPAPJp9aqAXTrfVFUtmpXbADarOQzXv1kIRf4aPRPXb6meLrKcJFGR5vrCpi3hsd1xLJCSr6d9T1bKo0hhjWUYr+wEOJSUHBD2gZ0IFo2Hn4oGSew32BUktfUkiTEeUSA5zVC3nlvBPe9MSWyOetRGDaz2khYWhuDLZ0WDi5J4KZQIG5IvsDUEdG1rW9va1ra2ta1tbWtb29p+ReyFGUCf3fv93/99/OEf/iEA4Hd+53fw9tuO6Wp/fx8/+5kjiLh//z5n54QQDMm8fdsx9Gxvb+PDDz8EAPzRH/0RZ+fqumbI5ttvv40333wTgMsY3rrl0sP+2A8//JB/I89z3mXfv38fP/7xjwG4TKTXHnR/KatEW91bL6d45TW3W7790g4U4U+mmzMcTdx1nD6s8fKUom0UvTmNF/j4wkU7l9YiSt3u/OW3c2xQwerZcQVNu/IkTtAbUgQqJXiCEhCCoKgoMC7db++/FGF+QVm9acFMUW0LLjb2CYE8USiWJChtVhiQqGxZL1FR/nt6JiDjELGrqcC4aj2cNEKPYCvbO1soSnffvVThFrG79ROBa6+49l+cn2BGUZRb19zn89KgWBLELxsz4UBdlVyQHSnJ0RJXiN5h+eK/HdVTDkiGbCBESKOzqCYMk1Zoqxm2IjrRWdFhIJMQMP5CKO1vbSgcF6oDieyk7yEshAwwExALYknRuovzc9wiQeZitkBOkMyoP2E4kbFAS9fdasFRVy94XJkahiJ6lxcXOCeY5s8OH6BYuojjwckI49hlnVblikVj48g9w+X8EjNiBt3d2eQI3O7GkDPHGhb93MGy+mmK8zMXkfMMZXneQyS9cLTAgrKL9z5/iCkdO1vOUBSke6g9WQA4QptlETa3XHvM5w1GubvmvK6x3HHZ/Ls3r3NUNWKGuPBf/wQAB3PjSJo1YRB0MsQ56Y3ClkhSiv5VEnbpooJT/RFyYiPd29zDyYYbqw8PTjCjrN7dgctUTnb6UCsPlVthYVwm5LwGKsJaN+cCiljM4nyElASvRQcu5dtcqT5069p0MW+wpHFWVRoNkcpoLRAxYiq0hL/VjnIXpLCQTBzV0QT0Q6UjHtsIBUrkYTPS2Mrceb7+1g2MlJu7Hg1bKIr8+yijrTQizw4Ji63Ys5oW2N90c+I5lhDHbm04whCnchtr+3Lt6NRl7H72ySd48NnnAIBrezvY3L0BABiON7E8IxK3z04wnbmIuWgN69aNh24s9EY9PH7oSjne//P3cX7islVVXQK0RunBGDGN52w45vW0R8QGu20Lq9zYmS+XTIp0fHzuzgMQyyDDDPhePEowkgK7E/cbPaVRrQiOhgh9CmT3sgzSQ+5orZ2vWqR0bD4AhhN3HWUTYekzgG3DDIZta7EgRIUfe5O+RS93F5LGEilpmFaNRVm78VBUBm3itbcipBmxHFKGLU4FoN28uZyVOHrq4OaX8wtUxBKsopyZAa1oYQj+5Ykqh70ETeXJXDQEaTImUYIRZf22hj3HIApgGAss6P1i5Y6dlTVK0oas6xKqIhbQuM9rojWGs2Wt0Yzm8WgOYQFLGRRrVCBwkwGeDis6MEx6q8NYabVgOKA7lLJ32jDMzpgOFNVnn0TQ6utq/BlhIbwgujWAF12HCZB5JvCwaFqvdxq0caUQrCfXNoazhMa2UJQR93Djpq2gqR2b1QrnF24MHR+fo6Vz572MC1LaVgedNs9AmkTsl7Rtxe0YSYmEIJnDQR9+Bl2uCjReg5r8NSuM04sDYKXGU4LiP3l6isMzIvazCprgm14XVMoIw4EbT7paoSICt3GeYkQEOREaSJ+ZUgNEVNrhs3uuVMa/Dhlf99wDeojLRyLJkGQQW7ZuDC4v3W/PLs8QWff6+tYYUeJ8ku2ta5jP3fq4ms+R+FIp6bOrLSrSCZTVHBm9P8osZO3muXJ2DBsR3C6KkCiCZvvB1WHeN3WFmvw5a1ssydfSBsiISVZGOZO4eRNSwQqCDxjLJEGQIjApy8DIG9OcqFXDGslpJIDG/V5SXqBH/sT+YAODjNOLsLROx4SAW5yf4OQxkWpWBTIq3UrjCLXXXa0MQNq9aTaCil+c4/ulagC3t7fxxhtvAAA++OADfP65W3CePHmCE1os8jzHBx98AOBqvZ8XJH/33XeZGfTs7IzPbYzhTpUkCUNDp9Mpjo9dTYF3AqfTKdfCWWu5FqtLdVpVFdeduWMD7AAA4lRjZ9c9wFgWMDSwhv0lZEKp2MsULx+69+/SJuFyc4zeroOVfe/hCaIe4f97K4wm7l6v30lA6ywW8wY51SUkqZfEECjI6azbCnHPnfvmnQzNnjvH06MVJluuA56f13h64BYOhiI0JdcA5IMNfE4blJd2NyGI/WlnmMLQhLiatchJFPbN2y/R9eRYUXHEYDjC9//CPbftUYqdgfvt04N72LzpnGshLePVe1SXVpsWJaWoF7MF4sS9D20Cq5QI+H6lOgMnKMx2zDJEFdaEwWqDnIOHniihkahASe/hRlESwzAcTfLiI4RCQxua1dJNFv3BgIXDrbAdanwTsHPCdCQ6DLNe+bS+AHBx6vryZDzgCceKCCUN9uliCkWOQlkUKOZuY5IQLKpqWjS0yBfNCg9ODwEA9w6OeKHqyx76A6rJu5xha+ImTQ89quoKP//M1QIMx19DRrOnaRtm49O64UlpPBrijBhEPbX7o0cHvPDVaHBOQZk/+/Z30MDXn+7iWt/99nLhPjdGc9ADRQWTun5y/ycfgtZIyKrEkz0XENr+x7+FEUFfOvv1Z2BA9LwjiSUFfJRSQXwXoev0qc4jxgbkHcdge/r0AIulc5DjrIGhyXZ5foRTmlO2r+/jrfecaPPd19y4ENU9hhZvbUyQRu4e9fxzVNptYEu7y7ISI9sn5jR3BQAwiBPE9NygC6xmBM2YXuLkwvWVJOtjY+jaUVoBmkrQmgC7YXH4zgawK8orYTt07GEz7uMmsdDA1N3r2dGHQOKu+c9+2mJBwYLL8zO05Jh6GNNquUSxcO1VrFbMLGisQUs1vwoxjA+63HoTd3/zv8Davlzza8p4PMCAxlOxqvDxzz8FABwfnWA2dxCsh58/wumx2+At5gsWo45pbuj1coYvzRcF1/tbY9iZfHL0GB988Of0vsIujeeIHIyL2YxZQOvaoiGG0apt2RFXIuI12/rAGrrIc4Eh1WftbFhMaUw1xsAKt4bJWCIjRtCW1pOytTiZubE6LWsUhNmsdAKKraGqNV+HbhSGeahpA4A00oiULwkw7NsmrURKPkLfAC3J/kBFfOEezq+gGIY5XS5xRhvwWCXojxz8vpcpjAYxXUeNpkfwWCq96OU9+Iu+uFyiJXinEgYDknOyTcVrb1tXqJehNg0AVnWL2vvgdYPWb8iUYmZBbTTD+YwxWNCYT1NyHpMIoOCVVZYDlrqteTNlrOkIyPsAmGJIIaDCxkBIBNFsESC9rUZL/ZEhqbZF7df6SELSOu0KObo1+b5fBQhigNNbtMb7G2DJDgWLhhIKxpoQGLZAqUkwnOC4um3QEtSzqSpcENx5tlxxbX2lNTO413XD8hs51b1meQJD19HWincGUUKbWwCJUpAUiDM64X5a03NbLissCOa4qlb4/JELLBwdH6IqqY5NKNTkR3g5hdas2O9ZzmJk0vkWsQRvsEeDPnY2XND26++8g1v7N6g5FD8fUCDA6MBSCStgfdAQEl4lQRuLhtbHAa1xSdxHFLk5ark9gSY2eogEs3M3boVs0adNaT+LsTl2+4Zh7uvZDG9wU7QYEnxTComYZN2EXkIkvhQjMHEKgnQarVHTJrharrj+tm1WLNuV9YaIJfkZSYaGJCQU+XZxbKDrEIj2dX8WGsqX/ViNiH5TUtlHbGMkviYx0bg8cfPx6f0PYRrXBieZgKK+UlY1s6t73pDl5QxLKnubL+fMBJ1nuROiBzAvK8TEx7G7/wp2SA4L+A08z9YQ0LWtbW1rW9va1ra2ta1tbWv7FbEXZgD7pPvz27/927h79y4AV1z72Weu6PyDDz7gLJ0xhjX6tNac7fPagG3b8mutddCjMYazhR999NEVchiv8+f/Sin5dRRFDPWM4xjGeAKXkrNOjlCGts9e72UZIaHo33RZcFq2P1xhSMf2LxVGD91OfGfDQZ32iwzROw6eenm3xNPCRVdHWQIrqcJclhwRlUIDxMrkGbZipRiiMJ9VrCOkVMuQzf1+hOHYXdP2jQQlRXiU1wepS46Snj59hI9+8kMAwDCJGIKT9XqoSfS9F0fYIFjeV7/6rru/0QYSgq5JIVETQ1nSzrBHBBcXhxZHT92zvbyc4uTcRSMyIgTR7QqTkXv95OiEYRzD4ZihbHVdc8ZWpSkODlzmeEIZLFiBjJhLe3kfV2W/GdfGzJ2S2ZcqyCjAO1rKfE5GG1hS1DvN+pgtvBal5oyKoSif0RFg3QNoITv6gJajglpr7ktGB03DCWW1b+/twVBEKU0ySLr+ptaY14EEISIYUjFb4vLctbUv2rUygaT+v7G1id2+yzR9bi9YyPTOzi2MSEvr09VnMCN3HZuk/ZckMQZE5JNEEae7nYYURaikCqx5EBgRNMRnAJdlhU/uuezB45MDvPau6yv/6Fv/GAlldzMhUFw6YoOjE9c3VqslNggG2+ulDEk+P19gNnNRyw9+8GPkRPRwcXyJ3p4bZ7V/ngKoKevgxHjd9TewOKZMZRLHzCy4WCwYatP34u9ihMnYtd3O9X0cHDu4+eGTT3Bx6q5Z1p9xtHmys4eMoHDFzEVGy6MfQxAEV6kEEcEfB+oB0saBdOLh67ApRbXblsWSM3ovbuc4f/BdAMD0/ACrHmUlz0pm7N27/RaU9EyhgKooU+AzurDcXyPHu0ftFCDJLhPYhS2DGAvd5z1R4uCRmxvuff9PER1CgNoAACAASURBVFO0NlExUjp3EismRNAEO5JRjMHYPc/BaMwseJACrUe/tUBLUcsSBbLT+1jbl2vbEzf3/q1338WY2IcvTudYUnnA4dFTFFO33h4enmBOepltU3rtcRivpzedYUTn6CUJWpobiuUClc8Mr2r85PuuXz95+DkGNNa8DL0xDSLKHvV7A1jj4Wg1Wp8lkgkyIpHqDTLWovU0BQoRLL05yjP0pPuNYlXgghAry0XDmaTBmH5vqKBjgvVdGrSlp/xs4TmRYyNRE+y6rQ2S3I8ZgnEiMFNGUiNmwjGJhKCj2kawnhhFSqxoPq0o81nVNSiRg4vVHPOZW4viSCGncdZWBdrKwywNBkREtbPp2vzmzT3s0rM1VqImaOt0usTlmfNFLi8usST/KrKWySAiGtfDLOO0wUIb1ISKWVUVE7tYsGQajAUefOoQJBn5BaPJkMn+0rwXSF5k8GeEQIdIzc9REZOzGAskvq/JkCUEZFhXTWBENJTRq6vSQf7hShOEv2YbyjOk7ACIDBhSmnYJXjpM1/6alQT3TdM0vB4sZkscn1Cpw3RB92KhfCYHhjOsi7Jgtt1VWXPbaKuQUgfYIwLDvWtbSCg9piLFcOEoVszS2zQtDGWjmqZGRc+roTTuwdOneHTg/OKirLEiSPKNG/vokw8WJTkq7bVcPftm09Gw1KhXbj2rljOURBh4eXaMKaGOdiZjXKPMfszsrgbWZ2i1YSQdrOGMrbYWmginjs/PcX6yoPt1/SfPNVLKVmm9gTn9np6e4PSUSlFEg5ygmpPJNmtrxh4y22rUBK+WTQHpodNWIiZ9QGkKSJ/FlzGXW3mCxdX8DGfeb5meoig9W6qBItK4bDCBVZ55V2J+4a4vo88jDDsZbs2wZYgIhiClEgLSw3ipy0tIEBEzoqrA9NCtmQ8++SFQuvbSbcPZ2yiPMSFfqk9IjUhIJJQR3Ux7HS1NjYoy1Wgr6BntdWbHMOTPf5GtM4BrW9va1ra2ta1tbWtb29rW9itiL8wA+oxHmqacAdze3sZv/dZvAXCagH/xF38BAPiTP/kTJnl5HoX5xcUFyzO89dZbePLkCQAnZeAp1FerFS4uXLT+9PSUs0c+06e15mvqZhHjOGZyGNMhjrh9+3an6Nx976MfFfjGEydRsX3rKRN6AGMYiuqI4xonsYuE/PzYRU7/6b83wjWihn7l2hYun7jIgJAR8h7V/4wkFEUddG2gfTGx1xJUFnlOuHodo6ZIYNsYziTEsYQG4Y5jiYyO96QnCgqW8P/f++4H+OwTR1ShZ+dceHw8n6MiCuqbe1sczfnLHzminJde/Qre+9rfAgC8/NJLyKWLHpzc/zF2b7j7butXsbp0EcdFJdDfdPfiM7ANWuzddMcaYTCbU7atXHFthFIKDUWQWwDf+Td/AgC4du063bfGtWsOc379+k3EROmfZRlndJVS3I6S9VlqRIqIWC7PYQn/XMwPAeOiQZubu1hSJC/LB9jecuQ1cs/VYhRlg7NLirxAwYPYjTGcLe7WpxqjOTMYUy1gTwEVFeMX9Qojqp+0dYMVRQvzpIfl1NVc3fvJA1xMXX+KKIp3WVdoSQZibzRG7KnNL2rI2B1z9ugAj+aP3bkhkVBY6a2X7gIAJpMx6/oUVRmKwJWC9oX1RqGmTKSG5vq7FWWKN7e2MN502baf3vs5P6MkmwSplrLEgiJ5Xlvq7OwU3/jGr7nzmhqLubu/5rJGb+AySedPT9HcdtcKo5isoFGk1yUEP29rLb+eL5fIKQo9m85w9MknAIA//uM/ZgKib371v3HXmWQ8X0jbRybds2iKBuekHWrLRxhSzY3UDSxlHWdTolEvWtjCja1E1rBeGy8voRuniZmpY1SGamP1EGPS3LGVm8/uf///xPyRq5Wayxwn0Rnd1wTDTdfXtzZGSKj9VqsSP/ju/wMAeOUNhzDY3LvO5bDShvlUCMmlsUKEehhOlkNwLdfq4hjlmbvmnV6ggs/yEXyQ1JgWGbVHkrlIoRQWK59FOjzgubTfHzDfg9bgaGylWxRE8b22L8/8XLN/cw9bG25OE9KChj7qcoUTIur59rf/HN/5juuTy2WJjPqeJ8+KVIxBRvU3g5zr2KxtQaV8aITmGuGjg0MU91zf8pT0umkA38d6Pa6/sW0L3ymNCIQMWxsbSHxmhzIdq9rVvQNAdaOHfur6pIXAydy9f3KxgqGat32axDbHYyTUj6OqgjR+vTVM8qYgEZOkU1U3nIXz8rRCG9jWz5UtTCv5c+MJSRRCzY8N+oULqocuygIV1Vydnxzh0UOHlEqkRp27e51PZ7j0ZC6jFClJO2TUYJmwyKhdJlvbyHpuDq3KEkdP3Dx275NP8PSAKOKVwISyJaD100YpGpoUDi+XOCUkRl010OTxCSlhOvwIP/2J8w1i+u2trQl2dx051mi8gf7AzXl5L0HrM4CwqOn5V0Q019YNWpqXEhVzRsxAw3JdVMR+RK/fRxRT3VbiuR4EakLymLYFKLsrhEVEOQsjQ/ZZIkgPdDMamrIpVV2h0a5/KBmFLGgkmdvg0cEh7t93c/ic1hYpJEsIJJHi+jejNWdfWmMYPaQN0KOMy8svO1mm7e1NLKieXhvN86lUMtRdVw1qQgw1xmBG+rk1aZdML2e8rmpI7O66c9+4scc1/gYSC6pjq4hMpe1XyDJPiKhQU+3jalHghGRfjh6XKMhXKRcLLBeu3XPKeEmpWY/QaHA2H7BY0fkupivce+TQXT/96Q/Rrly7X3/pH7k2h0QS0/iMIlRElPF4eo7y3LV5ggrXtx3CZ2s8hPJkPoX3z1uYisikdIvYBr9Sl4QyVCdIiaMi7UeIKQteLN36dPLkIxwfuuyjbVZoqT+qeIiN63sAgGxjBEX+vFTAyYHzuwZDNw7TSKJHZFhtl4rNWs8RCCEttCcuoueaKMnyV21Vcl8ajrcQEfpCtxUjcfb2trFJfcmv0W1lsLhwc3pZzbn+1tgIMbVTamIoGk+RbFGSXMYX2Qs3gL5gezAYYEXp47OzM95svfrqq6wNdPfuXfzu7/4uAEfE4mGd3imr6xrf+ta3ADh9wd/7vd8DALz55pvc+R8+fBicOCmvEIj46+m+d5XtE/y9/X03QN555x0AXhTWNdaTBwX+799zg+m//e/2kE8otV4ZGD/xVgp/+tBNmg+pIPSf3d5Ffuoac0f1sUdsZY0pUNOmdNzrYcNl/qFbg+WCGPlIC8wzR7l7kSRy42BvfiLNMomY4KB13cJ7d170MbEKJQ2Eez//FIckAhwbzXAdGVm8edcVf/7a3/kK7rzsmFjf/6HbLH7n/ffx4aduAf/mr38Df/trXwUA7N+6hump6/BbGxvQBJEz+QBPP3HvZwM3SL/+2huIEtdxL2cFiplzdKvKMLOgkBla7R37Bg8euEnikBxyKSUePn7g2nR7hzVX4jhlYdE8z3FKAt8rcs6FbZET5K6XRrh715F4WNMDZcjx+MF9Jrq58/IreOMNd0zecwvZ/ftP8OSx2/AMxtswHiIKDQ8zsTKwXllp0NII97CXy9klKupXfRjEZ+7Y7Y0d9GnT1B/1EdOGJ7URzk/cxnxCzk3Vxjgjh2Z+vELqIX6xQuyaHw/uf44lBR+2d7aREmzWa+AZY6E97FZKIPWbdImWnLYnB4cMj7UiwAp9Xzp6cgBFY9YaA6q3xvnpOUYj98xtVTHLiCeKSNIUJS0E/V7Cm5FlscTtuw7++K3/5D9CTAu+zmOoIU1oBMGdzWeYk8bS7s42BxBSgNnpposlw5Yv50scUb8vSdOz10sYYr6aLnH40E30i4s5HhHEMxWH2CK4alQvUNPxIMhbkuyg9kQW5hJx7Pp3HEcoCc6l5w+h4dhx5/YCUrq+efKJc7A//OBfY69Hml7ZNprEtU2Wxpj0yaEajsOGFzGePHLwq8XcLVSvv/EOXnrZ6blFeY9JhwDLjvVVAiX/aSCGMbC4c9fNg299/TZ6zJwYo6/8OFMYb7nxsCCG0tWqREHQwNOzM9Z2vXH9OiSRDNy9+woeXbj2/5f/6k/Q33v7Fy9mbX+jluUESSpaxC2JJScxtsghHA370G/Q+lI3+P73nLj7mW1QVK4/JbT8Z0mChIJQaZawHpeUEevkWRHB98LGBj5nH6kQKkJLzktVNRhQmUI+SJET22E06mOj56570B/iHukXVnTm4+kKH993EK3X93t45yWClo9SnM7dPRarGYoZEUNRcDTdHCCi171hgpqY9ErdQHoMmVAoPTPKykIQiZj0G8E6QEA1DBNfWKu5BCFKFBS93+igZRdTYFtCoCUo3+nRU1w+dc7aZCNDLIj0qy6xTVD2N19/BTf23D3mEcESqxqHh26NaLXC3nUi6sn6uHbdBeWUMRgRsVi1XEEQVWFt3Li+rDQH2epWsDaaheUAOkyXJEPh4NCtyZ6S9fzsDMfEYjoaTdAfjOheI95YtVWF6dTNG0vafFhtMSJm5p3tTexsunsdjPrIad0SnSDmcJAj67s28GuEtQqLqZsLy7pESoQqxn2ZXgcSISEEBxk8aZUQgoPPy1XJuokxMiiiflVKIvFtt6pxfjGn33TPUCqJlNbYWAWtRHfpnsXZMNlM2xrcvubm+Bv7bo0YDWIu76nKhllYjbWw2kMoK2ZxbI3GybkbA9rDqI3FZMudt98bY2fPbVb6eY6Sxn7bWNiWngWVJ6WJQkblG1EkIGi9tWmM4XhA7XQDqGldmoyZwdNYmlNkWHOsAfsTrTGYU9D//uExfvLhPQDAg89PkdJzvOsJMpOIAzGZCIRHbVNhvnC+uKyXSMm/unltC03j2r2kwHyeKGSkNW2kQubJWVY1Sus1IAsk1j3DftSHJVb8s4dufX3w4XdREpnOoDeCzEm3PJkgJ+KUNOvzHkQogYuDB65NU2Ibb1sOjMRpDknM70JEiJTHOwsoStJ4hGjbGghKMKFc4vYN1z9ev30dG1S+E1tHSOPOF5JCnmSqLJbMGCqwgyGV6ElIrCjIUBkNSSSMJ5cL9u++yNYQ0LWtbW1rW9va1ra2ta1tbWv7FbEXZgBZ8wMhG9g0DS6JinQymWBnx+2Gd3Z2GOJZFEWge6Zz3L17F9/85jcB/L/svUmMZEl6JvaZ2dt9d489M3LPrL2X6uouVjd75QyH4DShkTggQAwEiIfRWXMWeBIg6CDoQPAyJwkgNDMgMaJmBlKLzWEPp9Vkk0V219JZa2blGnuEe/j2/K1mpoP9Zh4JsQs81aXdLhHw8HB/z54tv/3f938f8Id/+IcO9XvjjTfw05/+FACwu7uLdttkmprN5jL7vGOoU3Ecu+/O89xR0xxFDQZV6vcNJcaIRhCCY9lSOsSb/6/J2n/ulS1893duAQCOFx+iygk+5h6KkclGfO11k83Y0CkOKKvmC4V+j8RcamA2tuoIFdo9c9/pQoGRr1pVWCi6dChgVdZOItkP4PxtOK/gUeV0LQHPijQQMiErD0OSvN47PMOIMqO+KjAYmPu+truJQTegz9OQJMv769/4KgBAqTfx/R+Zgv779z7AyXf/CwDAf/Xd7zikMRsNwZl5FjJs4eNDEr0hKuXD/XN8+JH5jPl4iE6DMnCQDv1lTDvpfsWYE7QpSN42jHxUlKXJ8hkEweae8J23jid8zIiicHA4pL4tnBx+Evm48cSgEV9+9WUnlZ34IZj156m0o2w0qLC2LkscEQ35ud66Qy2lZEt5b6WcuI2CdM9L0WAaZXNMSpOxubYdQtP1F2XuaMF5mgFE97x5ZRePqJjbJxsIndYAvzBXCAEMOIew6KMQ4B5JU0uFNgkwhJQR8wLPyRVLuUTdi6JyohCMMSeiEiUN/NG/+TcAgDNCwUowfOfXfg0A8Bu//l00QusvWWA0MhnpydkZGFEvzonWenC4j7ORQVJ/8598FxGtAfMqc0JJr/zy6zgbm9+HdYmmts+ZKD/TGVIqSl/s5WiTyM5bP3kLTcpyjScTjMg+5oUXX0Svb0RZrJ+e1woc3YXVEoLEJrKixGhoMqpbvRztyKwfo4NHaPYNFbzXNFTmoH0Vvb7Jrtb5KWRh7jsMOMKcPAGHp5hODKI4nHyAeWqu6a9/+H/BXOgUATdropcM0GoaFM7zNtFumwy+74cuS2ssNMyaMCGbiLtv/QQj8mDdvXnDSe5HYbws/GYamtYM649pivLN35uDbbz6nLmX//rbn4OmovnheILJ0Hx3ulig0zeZzTQz/R+GEQpa56RU6FA2f3t7C4yyzY1WF8EDg+aPxt+DaJus6qp9di0iGlISVTh8aiTFT0+OENG6cu3KJQz6REfxBaKY6ATgbh2zlgCdVgtXaI9NmqGzcLi03sd8bp7tOF0gbiwpSe3ErCWeRQg5c3TRMArRIfrSWr+HhFA/FgSOYliUFf7kT8ycsVFGpoH3Pzb38vx2iBu7tI9HDXQS83kb/QVOU6LR0TqS1xIeoTB+ADRItyjgETj5hdUS4FYUQglnq2RZMxASmtg+HlcILXVUeM4uKAy5o1bqekkBbdC1+WHgMu7j6TlSotd3Vej8O9txgutXDDvn9s076FDoMp+aNfR0du7Ee+aTCerc/N/2zmVHc13f7EPQPjE+OcJjYjs8OjRr29PhAhMSj0EQOAGORjOGkpaOK8FhKZlGdh6A88XzLtBdq1pjNLa2P7WjGuaLHCnRJQuiUtZlBY9Qs2YSYUACdNvb6+iQuFQjbqBHdL9a1rhikV6iISfRDILWOV8IZysllXJ2TVxrSIvCqSWyae9DSOlKOXzBEVhqRFWjQaif7zEIom8Gvo+EvsciwcaQiBBFCOeTq7VGZYVkwDHPbPkRkBAd11oxVVI5Cj8XnmN+QGlH1/e5Zy3zAFT4yd8YW67hiNCsQQ/P3zKMkP56BwHB8nmWISWa6CLPMCTBkjHRBJuNNiJifgTaQ0Esj3k2d6UE21ubbk424g4WuUWgSJStklDWb1FrUy4DoK5KnJEv4unw3D2X7Z0raNHcDwhpDTwPHu3HgcfRI5pcECWO3ZfXCqckCHN6coYBMez6PbP/dLoNJwYUFxk0xbTh/ByL1FzrvFLwSrLLGD7FeGj27ycfvGv688l953/tBSE8mpNJHDgxQsE9V+pQVxLToYktmTB7s5IZZGr2z+76GgYbJh72o8DBfVoz56dJIQlkJc2YBLA2aODqtmGkba23XMyKWiKn/i/zuUPnfH9pfecTcyEOYiSEPkqlUdO6o8BwTsjsX/7t2/jZex/g09qnHgDtwA3D0B3u3n33XTx69AiAUQG1AcLh4aEb3IwxNyHtAfDVV1/FRx+ZIGp/fx+/9Vu/BQD4zd/8TVdTmCSJO8w1m03HE7ffURQFPv7YwLkPHjxwlC/P85zqKPDsgXDJGbB+HUufsT/6gwfYunYbAPD6P7qGeW4CxeqmwksfmcH2a30yrN87xSdTs1DxboFeRJz389rRCDhKcE4qgz2OMdWK+URDCUIfBRnrlkXtXufcQzqzymVAbA9TUjvvthfWTP/vtCPsk1rj06dPXQ1XFHP0embx2drso0EUyfFkhsXUTIQFTYjXn7+CgODl9x7s4T99//sAgJvbXXzxORMUtxptp+aZ8wgDCriPyffx+3/2Y8xIEfLVl2/j9g0DaX/w6NDxxCN/uXlWSi8VQWly+J4Pz7eH5BprWybQTZIm9vdNIJBnC8Q09hoU3KSzAhlREaazFJOpUVR68vQQfaKt9tsdZ8we/+xjfPjAUFhfe9XQXaXi2Dsyh7HBpS3ExMOuawGPjDRlXbs6K62XG0NOqlIl1yjoQFRCwqPvZp5ARlD+8dEZxNyMFQ8VWvQ9TFo/pgUUbTLCF2jQgu2Bo6R6Eu4HzvydM6BnaTVEZ9SyAp+b8dMJYwhblxZo9BrmEK+2tlDSBgAu8Mm7RiHyb982tR/dzW1c7pvF7OYXPocWJXYSHeDHP/gRAOD61auGogngk8dmHh6fHkNq6/kloWhD5Q0fE1a6Z2if8/DxPRQfme+0npjbW9vo2k055jgmqvDBwQGu3TDzc//gGD975116hl/CrRtmAZ18Yu6p003Qp4C3aqXobn7ZXPPtS3jnL813j57uoaDFO5AMJR06pS2MiXto9MzaIdQ1ZKSO24wK1MeGQsflx9jaMZtxJaZIn5h+bFSPzL2KEudUiLXdfAGaahiVBgT5fkqUrp5UGFk6AABjtmZljocPPwQAHJ3tYZuC86u717G5aX73PAZJyROrZBhGS7W+UgK9JgV+cQxBdNbOoIO7hXl24+Ex2sKMpQbRYKMogefZz2Vu3QfnYLQeK8/DfGbG99njp5CF5aP+c6zaZ9NCGyzHC4xHJuD78Y9+CEb77mBtGz2aDw+fPEFGtT0MEoIOOm3y0Nzod3D5kpn7W5e20aL6wq3tTTTJFHlWKEfdiuIQSWx9sczPupZISX1ztsgwo1rnqsohfGt87jnpyYRpdyiyZD4OgeOpuf6/+dkRruyYBMYXnt9wQeNNxRGeEEUyMOswZxLCs6q5QMMm8zztTLVlUUJ4lFwLa6c8qaWtr2Xg2qodGjqt+XDPKTcKwTEnGqlQCpzooH1KrMVxAzWdgtN0jtC3gXWC7XU6MPse1ihAzoqF85lT9H9xGKOOzWuj0cgpKafzCa5fv0x9HmBgFaAriQ8+MPvf0RGVSpQcOaku+9pD2Db3EvqB6+xKw3nPcaHBuS0OpKSSF8IPzdolfN9R8WWtYO31tDa1wQAQUb3V+WiKlOho48kIR2eGRvp0fw9N8nmLwhABlSmsbfTx8iumjMiotgNpVuKc6oqb7TZqKjGutYJnFRrDwKmYVnW1rG+jeIhjuT4GgqFJkXhd+yhpv83TCjkdJJiq0CdV2SCj5GFeOHNv3+NOZbWSFaTtD2hHDWWCYUAHW0uXLrIcZWUPi0uvVyWX+gJcCPiU9NeVxpt/Y/bbJ+T3t7F9Feu0N3cHG8jyc+qnFBMyMJ/PM8wpBl4szNwrqxKR9ZdMkqUKaJk5nzrGI3Aa32VRurp4+9piMceExmCW56hKazheYUa+iKPJFIvCxjABmnRIDKi/fHDEVN4TJQEiz8zlq9dvwq/NNe8/yjAmhc69g2NcuWKSpbb2tLPWBxO2bj5DTiVHQTYHfAJBNENZWp+8AnMqNapOKemeT+HRHuuLCm3aH5Ne/4K3ZeWo7LwCWE0+uTTmJwclyqlV4x2gpETuoNtHQElzLTh8ulZNRyyhtDGAB9BtdLC9ZubveicCahvn+YhCcjPwanjC+rWS7oLHTekYiEZNSuJMK8iGVVllKKT1cKycEvHPaysK6Kqt2qqt2qqt2qqt2qqt2qqt2i9I+3uJwDQaDeztGQTle9/7Hg5JSacoCkfl1HqJ8Hie94yYC2DQQpvxuHr1Kn6N6Ga3b99GQhQFAM5LUGvt4GGLRM5mM/daEATPqI1aIRghhHuP+WnPuBZjLwFCv85nNf7X3zNoWjO+ia/8qnl9+PVTXDmg4skjc23/cZLhybbJXFzeqdAiBcNSCVASCVJqp8DYaEaoSNglW9jCZM8JWUBLgNQ+fV8gtB5geQU/sNRQZsUp8YUdkwnZbEY4PyKhlkhgjTIlzVigR/SDRiPB2pqhzwRcYz40z+vw0Pzftsrx5TsGSeg1Y/zVXxt0492/+k+43v8Vc1955mDxOAzRowzDg49NtnGt4eHOJSMu89or1zEcUeFxGDl1MSE4GGVVBRPkSwdY0s/Ozq5DGcfnQ5SUXUrTEVISooDmGHRMxvSOSYKhGo8x1SbDIv3IZW/m8wL0dZilSyqqZkN8/InJAv35n/7A3PfGNgoaG/ru+1DcKnz56LUNEhlGoRtjjUbDjaRZQejHZAxFGdOTyQhrPZPVYVwgpqzNeJGipuLytYZwWeGCMoi+x6zIGVpJhEZkrrksa9jqa6aYU4YLBXe+jiWpVGbZbOmRyDnArVKuWiJCZeGEW2pVOmDc+tT4UBCSaKaQ+Pff/3/M9Y/HTm2qepTj7scGvbOoX3uzC1Bm6wd/+UP4NGB5oBEQDbZiymUi2xHDyX1TMP7hQ0OvaHc7iIiam5YZCnqIpfbx9ls/AQDs7+1hj5TGUJV4uGbmw+cuXwMALMoFWpQRa7QDhLEZ31ubu0iI+vWn/+dPMZsZJdFeWKAmuoilL83TABGJYfTaPURt8wyHpz/F06fm/3QucG3HXN/peYonj94CAFzbMff94HiGwDMZQlY8QlibOVJFt1B65vtqPodP3xP4HhgJVVkEgsEDc76lEzwhlHNydoZLNOe2uj1ElLUv6Pl4SYIBqbc2QmBnw6C/43SEU8rWHp4c4m/fMvN9b28PzXfMmmHXrXRRoFbL9dPSdBln8GmNbTRbOCXPrEU5RrlHc3XVPrNmSxriMHZ+bZ4X4vDQrHMPHx44xsLx2SkmVDagoZ1YgbX6StMUjx+b/6ukxsuvvAAAeO65O9i9ZFCnSinU2opWKCgrBkHZ60Uuwabk/ZtmGJHfabqYokO+dlGj6bzDGNPwaH22XmzgIRbkYfbeXgb/h4a+lNc1Xnne7L1Rs4UOCa1AmHWVBUs6dBgEkMIyV2pom7rnDNYAkQm495M+CjyfOdqhZgyCWAGM+c6nVWoNYSVS3S4HdImuFvoC2cJck5I1WiSckkQhekR/7PZaSAg1K4sKVjqwRTTZqN1ze/fB0z0cEb336b37SGjfXNtYR2hFe2IPMe0pLaKKNRoNdKzWiwicAJTgHCUJmDFlFMXNPXLH1LFlE54fI4xJZIJz5JKoadII4ABmvG2tb7j7BYCyeuKE3xSDK4uQmkNqqwjKMLMo4b0x9vbNPrA+MOySdqeLiGKZVrsHTdS6UlZOFbrX7yMippeSChVRIWczs8ZqBcfM8hhHy5YLBb5DXefTKVISIckWqSshsAJ0VVUhI2S2gnToUS0lahLEUsyDoDKd0PeREKpuN9ZSU+yIcAAAIABJREFUVqiULfuRTiVWK+YUujXglES5x5CRINqM6NfxfI7KirWdHmJBv2dZjqy0ivEBNH1nSNcgucKY0MBa1QDNrVAICG6ft9OxRFFlOB2O6Dma550vFo4pM88ypFQesCgVUksPn8wxI5ZTGIUORbxD4mDc1w5FDwIOj5v5cvPOHWwQOp0vJjjaJyrzwQGunhLrJTPMsrhsI6ZzAguUQwOLSmJB5SUKDAGJYMWqRiTNdSe0v85VCU0MPFbmiEicpxV7CHyLANZOSCgHg6Bxbc8lhc6gyHdwkc2hC9MHi866o1p7UYSQEO4FxUBRFKLfXlJjPYdCz1DTAaJSFQ6ePgIA7D15gJrQUUWU6nyRoyCWhQZDRPMT0ijPAwCEhxmh4O+++z5OyFf957VPPQBetF+wVKBvf/vbmFIwkaapUwSdTCZ4+20TWOR5/ozROwCcnJy4w2Capvj93/99AMDv/u7v4vZtQ/Oaz+fuezjnjka6cND2srbw4vUxxpZqjRfqFi8ah7pKA7Y0huCCY2/PdNzv/U8f4r8fmAH7lV+JMII5LH3wn0mhrPJw9VsE5YdjaIp52l0fGcn2znNDAwCASEjkpLZWFTTI8wjcKvAFpTNwbcYeWgRH7x+fuzqfMPQcbSUjGeCUa6A0A3utESKfGMg7EQ3EtICVdYUFceHbG2uIabOQCwObc5Tgufm/V65uYCMxdLmfvfO3uP+eeYadTgfcW3KaS6J7JrSI/PZvfBtdmrz373/oFMMmWYgwIdphWSCyNYDcB7P3TtLKRaldfSfjMaZEVY2iLvoDkrZmAteumaD3kjITvT47xlNiM2baA7O0FRZA2YXN8yAt955zWAHW6TnVW6WnuPGKMTuPWpuQdPjhzHdy2l6YuIAlK6X7fUHUEj7T8O145AoPD42y6mZnHdvr5vV5leOYJJdFP0FJwUtZEvVICnCaIz4LrccxpFqa13O9XKbjOERNz+BoaPq8znNHky2qZd2fEL7RywbAFIeizUKC4ZRqvhgtUGHLx8HxIwBAY34J48r0dcZzbO2Yk/fh4SFatIg1qT438DzERM2NW7GTk2+FfWiq35v6EQIbPE4VfvjWjwEA71Fwk7TbaBGdtYKCsPM6k/AtncQTGJ4+oc5J8dEH5vq+8y/+BQCg048RkNR6GNBzhJFxD2y9VLKN2ZFRHozatZPZthuu4BoLouI04gBTslKZzmcoalpstcLCJgBGM+SV6euOpSm3N+DRpqvHHyBpm3qfRu9L4MoEdkJ79rGACYBLKztuqWbK/Q7NoCnYyCcKx1ZxbtTBVs/QdDlteueTCXKSY79z8zJOaNz9+dN3kNGqN59PcUjjZpiOsU+mvHadLkqFwkngV06dWCl1wR4AzipCqtxRulfts2u2bgdhiBt3ngMAfG2R4WjP1DUfHBzi6Z7Z/I+PjwG710Shk9K3++BoNsOMSisORiM0Omasf27yolNuFH6EwBa0QDiFwpysZRZpBknjVKvK7cNMCKdKp6GdKp5SNTxKClmhzlpyCLKFH5Y13qQ57vH7iBvm9SvbLWz0ySS6SUF2UCOjNTEPNBbaJssYGNUecVZBUV1TrUtIZVX6qBf8wFFSfcUQclL344FbS7Kygs+WyVyfEm0NCvaYqrFYkPVQEGAw6NEj8l3tf1koZxtRMKAVWUsIS+ltwBd0sNksUFKslU1mmJERvM9qdMn6w+M+2hQIrlE9b5C0Udn66kKion1rnmfm0AlAMbHU0VQMPqlF2kNw1GgjtCqDXKCy6usSaFEit91uY5cU1ydUv4UHT1ywzLmA52wzFEpLWedseeD0NaZTW1Noxm5rmuLqLUPxv9TuOGpdXtdgNo/PGAoKnHVdoygtvZGeq1KoKnMvceBhSvEVF2a/BIB0NkFq7RWq2mkQSErEKaUAF0Mwl+ytlIYVdBScuSS24B5yCtCnZMVQ5BUoV4IsL1wsq5m3BDiED2ZnI+fIqf88SnILeC7OPjo4xjFRvjNZOp2JpNlwn+dbNUouIdUyLo592x8CPtkyiDB0B+yz0wP89V/8JQDg+KlJ9PueQNPWkDZiCLJZKLSPnPa52fgcw3Mz7hutNra3TVJA0B7msQChdUEHoKyNVnMAf5vimvZdgNYJBY05KefvH5rvmBcCO5fNezvdPjxS2pZhG2lFpVv5HG2bS5IlGMV0dp4GQjh7hno2gyL6bMwqp7xfcIaMvpsrCaXNZ9TWWoszMDK912WBiY1rshR+aOZL0ogRUklFnRGwEwVgmVnPJo0QFan/erJyce9iPsG9901JyYMHHxo1Ypj6agCYL3JkCyo9kxoeN/OiqCUWRNOFF7iEfTqfuRjn57UVBXTVVm3VVm3VVm3VVm3VVm3VVu0XpH0qAmhVOJVSTkHw9ddfd1mM6XTqsg6LxQKPHxuK1t7enssAWoTnhRdecFmMJElw544RUuj1eu7zzs7OcOmSESG5KOxyRMWcANxn1HX9DO3TtqqqnqGiWj8fyxY1Zq70ZskcFWTvIMfv/Y+GDvrf1TfwpW8ZdVD/tskSbKspmE+Z8+MMDWZO336jhSEVhlcAUFHRbeChik0GJyUFRxUsHGIQVho932TsWl4fZWAyWPAFZGXFY4RTWcspw5XXEdqkOrZ16TKOKvI+y2aoT8y9no2HCImWMNpax3O3zb0M1kxRvS8zBKEVIckRMpNV6IU+Dj8xVDe1u2PvCAU8gLK7g8h8x0abYf/YoKTTrIYMzfhAUbtCbakUFhVlgnmFzUsGDbGmoGEYwSN/lbWtDtZ3CC1Jmi5jKjV3wi7RqcmgNDzuVNpqpcFcyTWD5FZwgLmHrsGcsahnKYpRC701Q5drdTeMyykMHcaCyPUFU/K6rl2mjFlBISVcdrWoKwypcN1TDJqoI9PxFDMqUE9r36iBAC4TVcp6qVDGNTJCvWuloSl1qDwNRfSOcZHjbRJRuX7zGgAgDhuobGqUS0xTK/gg4FEmu9vsAcpm8BkenBjkx6NxUKGAZGaM3f3obbxPKpTQGveemDGxu7uLpw8NqlCQul4raWCja7LR690+NkisgaUzdLcMcnhSKUfAPv/ofTzYNxTQIal6Rt0ILLaGqkCDzJGDsIbKzNx5cP8jdDrm+i9v9/Ct3/on5r66JusWRktfK/PTqvtpx3Xzkg4yZdDpk1GBLglOhB1CRosCjOZnFHDkNK7i/jX0rn7N3HeWYkzoreY+tGdoSxNS3grYADnRl1UwAzKDOG74YzRKwzAICoaFInGHMoSuTLaYk6CG0BqcMsKCe2DUe1qVkJR1HVWV86tqd83cS5IWxmemT+9JjYE0a1QnHEPbPkDlxBims8DRBK0oDUMNjzKPhZIAoSaVrh3tT0qgS95FeZpimZZftc+qOVFDVSEioZaXXvk8XnjRiFxVVYmnT8x+/IPBn+Htt4zS9iJLETBCcLpmLjTabbTbpLjZb2OL9uxaAYsF+X1GDIrWdc4USosq0do2zyrkhIwDGk2idPsVHD1dlqVTx/OFACNBA58+C1paoBIAQC6dePfBGIPeIwCAemUD12+SCFbDrl0LaEliHsJ3FD6EIXxazANegxFtR5YCxA6EBdoVGLgghUDGwOUSAfRoXkZB7UzrS1GjICZLTfMwXywcBVT4PpotUj+FdJ5peVEh8C1LiEOTcNsarWORX7u9KvI52iS2o2YpMvJxHekKEZWSxEkbmlA7yxgJosCxbCbZ1BmLj/PSKRzywHcIg9A+woi8yOhz2/0emiS+5wkPjZbp86os3DPstNvod8zr54QAFmXlGDnCY0uBPMYgCUUpVI6QFE19LwCPlt5yAJBXhdtrk2bDKTNXWqG0bIiiQEVMqKos3OvK0k+VhKXTpHmBc0sTlNL5qhV5hay0kt8As6UONH4UE+BWnd0TF9Y5CcGsWbhwfOw8z3FKnsXdQ8O+qFA5FLSq1fL7RA3f/qqZo0gyAIoolNKJKiq3PmfzBc4tAlhVkHRNangGRWUZHl1zFHouble9HipigqlCwCcBpSDPUZI33tOH97H32OzNE/IibCZN+K4MJkBMKB1XwJz6vFpMwUkYb2eticuEAC7jLw+VVb/PAVnR65JDkj9ms9VHQoJ1Hi9hWeFHYzN298+nOBmbufX8S59Dt23ijN42gxXyHR08RUFlHWU6w4Jo74pi0NgLUNG4qucp5kcmBsrX9qEFMXxaHAFdUy0VpPXzI+aNYL57bmAckGZeF7mClraUo4TVkJJ0TsjKOUpSZ9X5HM3QXPTOegc5rRlFNoVU1l9ZQAvr3Wv9vQUSOlPkFRy6W2QFUnqGmgm3fnBmlGc/rX3qX23wa1WdAGP3YGurTk5O3CGt1+s59U3P83DjhjGB/spXvgLAmL9bSuerr76K3V0yymy3ndT0ZDJx9YBBELiDnVUDzbIMZxSYKrWU12WMPWMKv1SbFAgbSxqg+clc3ZNmChVBvLoMcf8jc33/y//wAP/s1AR23/oNMzBagzFKS9VLNRq0kMaCYZbZjUw6lTNd1BC0iXDiC/A6Q0TXsd5uoB6aA/abb2e4/rrpx43LMU6fmv4QWiOMzftzgqtPaomJ5fSHPaxfN/Ua5Wjf8Yp1mOCQuNz3Pv4EojBBY+v1NwAA/Y3LKFMDO6u6hKCFst9t4HxEVgujE0REP+Xr29i+fg0AMNozC8TsfIzCFlA01pENDaXUDyQKCgoM9ZIOcnWNS1e/aJ4RHQC58BDTxiM8D9wquQnfHcI0BEDy88dDcx+TRYmsJlqO0u76A6FAHwERem5DTIIQ622zmW2QrHAVxJCWh10VjinMwJwFhdbKwemmroEsB2jDrUvp5ohChYgm2/lkjvScaiYKiQbRT5gQzhajtmOXl4isHjQkKruoSuY2ayWl4fAD2L40QJtMS7VdVFmFRtiG7bEeBXMaDILZWrMAta2lqBWOx2aBt0qRG5du4ld/3dR//tsf/QUqOgglcYIWBTKBzzCbmecsQ3sIrqFpkazL3NGoeDrHgubWx8cnzlB3JwyxQ0qWDTrY9Dpt7NB6kNfK2RqAL+CTamo7fgE7ZIA76PWxTqq03FI6qhqZtuPAA+OWPgH0NsyGdO25VzGdGBXb4fF9jEamrzvrlOiQJYqZWV+KYopkxyQINnau4NaXvgsAmKdTfHJo3lPqIZLLzwMAppSkaq3dRtQ0m9N4/y7Ozg21Lp78CLogs/V0DWhuUj8OYA+risa5KnMkNrnlcVQ0roKwhYJqMxQU9IICFqrDqrs1gsg8q7OTCe55Juh58aaAb5VGwYwaIIAoCLGgmhkbYMiqcLXLqqoBS6kqClc7U1YKPgUenvAwpLm/ap9hsyqymqNBQbvf9hytk3kCrZZZExaLhZOA33+6jz7VKl+9bvboy7uXnMl4Z7CG7S1D6+s0G9CUUGOe78y3mWZL6xH6xqJKsSCVQSkVmh2qf1MNt4aa/7fBiQa/MEcBwPeUO1ywIESemXE9K4D3H5r9StcKJZVWfP7zZCDe4AhBtjoaKMnwOgxiZ6800zXO5/Z0GaDXsQbO5pVFIV3iVQsP6Zio/0WBdsP0weWtvqvRrvcnQGprsC3dL3PBHGMaAUWBXHIoSiYVeQ5NFGsR+I5aZrUUqsqHT/2cCIEW0W5n0JjPbPJbYzGgZHq7h6YN8unQMV1k0KRRMFtkSK2FVK3d84RSqOl+JWfoUnI4IauPVruNBtWSmT3LWs5IMNqvgsB3lZDW2knLGhHdtwh9p14dBMGS5siE21Oa7TZC20+UYIgbEda3duhZcSzo+hkXz5T6WLVGCe0CZ6e/ecEcXmugpL0yy0tnASKY52wBqrJEUdqkM41BIdz+UimgpD24rKWjw3PFENNhO2xELnbIqE5PMgUR2bKfEIz2Qa0VLAFPcOZq9bVWbu3v0Bza3dnBgEovjocFKpoXZV1B2qSnx5yl1pTmehgHCEOKk5oJtDR/P8/mLj/qw4M9bckyd+qb6pIpL2jGITyqrQ/jCAFRTuvZDA1KZOxs9dGkBMELr3we124YEGcyobVDKiyodpAJZgX54TEBScmo5voG2pSQr6cjaEX6AXS6G0+nODwmRcsgwpe/+CoAoLe5A0H00jBKMHxkktXp6BSZNbW3ZTBRA56lro/nOH9iaK4eBNSBiYfa115BZ9usi34Uu/ILbmuJpXLj1PcCcE5/B3e2MJ7mQE709IoOZgAk7auz4TEYvf7cc1fQb9raQO2sUmbZwFGmFT1Pj2kULlmmoe1zq5ZJF1kVbk3m4M5M/ue1FQV01VZt1VZt1VZt1VZt1VZt1VbtF6R9KgJoi3kBoEXmzL7vO4RnbW3NZa7Oz8+xQZn29fV1vPbaawCW6F1d1w7RS5LEoYiLxcL93uv13HcqpZy6mf3u4XDo0EKt9QXDcfYMDdQijVVVIaJCcSusAsAZ4UpfOipImAUopUFDHj2e4Ef/9hEAoHFi0JQXvhwh+YJ5c7MboyRUrdZHoKQ7wlo5cZiyAmqi1MXWGLW1g+0t891HJ2N8/89N1uHBvRwvfdNk4O58nuEda7Q60whCEh6ZkKqaSJHR9c8Wc6zHJhW10+iAl1TUKhe4sWOeRckEZlTwfpqS11C7Rk5ms1xVIBYgOr0WTk6MMMd0eIaQsrFxZw1NMh/PS5uRmaMITIbq3pMDjOemzzuNhhMDKMtyicYGvlMtskaxgnvwfEu14e69ml3MSzDnLzWmrEoetOAT2txgDOSLim4jQUjeRX4Su0zloNPBzUsmu7S7ZZCjs0WOR+emn32fQxCayTmDkiRcoysAlgLgQ1IGTVE2kUu2RKByhZIEREQcI6SsGecazTXzeUE+hSIRGEWZ38jzXBZalQVqon/kOnDelUKW6PbNHHjt859Du2NRU0KevQAB0QUqxRFbSk2tXParKEtn8lvXEr/92/8UAPDLb/wSAKDXTtCnjOOd27cRTaiou9FAg1D509Mz3NwxgjxWrQ/QaJBQRLfdcdTd2XSCioqbRRThuZsmq3YpaiM4IUGjgNTwGhHUxMyVo6eHGJ+Z/7t2awf/+J/+lwCAJIwQ0Byv8tJlzk8OCXHkALT1lBQQPtHOGAP3zH3dfP6XMCQTdD9oO4R1TIa2cTNGmZMy6JNztCg/1l27imjdqnkWGL1v0L7R3hDXds19dTuG7pps3cH0qcniPfnkLura3Ou9uz/AoGXQQC+5jd7NfwQACDcG8KlYndGzb3KOK0R75kLhSUYIW1WCpjKUFyCOTf9pQgDHskTSGlB/9fBonzyKBi1s90lYomZLmlerjRExBey6K5VCTR2zKEqH+tWFRlEszbMzoppKcFfUv2qfXbM+doEQ8Bp2j61QWFSn1uDkg/rinTuYk4DI+qCPdsMqKZqx0mp34AWW5hg5DzwvCBD6VpiIufVUa+b8zCTRgo3oBiHplYRHAmKhH8KiR0rrJYIDtUSErB8aNJhvafY+GoS8NbnGojBz8dHh2K25ScNcw+7VAJrMlFnAkJAoSzNquX2/WqRYTKxaIEeP1lMrOpdlJeZEN08r6fzkTk7nUC3zfXd2r6C7beb2+UQiJS+yGQmJFNkSaa3r2gl3Ma6hrIgKh/NEDUMf1pG7WFCJh9AIIyuOoxyCw4XAmLwV54sMra65l2Z3EwmV6lgVwtPhDNJ+blW7PvADDmkZj2qJijHG0Fsz8UJMgllB4EHYceBH4KSACa0dswZMobJCLLSJBXEEaGIYxDFa1l80jiAI5fIC36HTa5sDdAg9SoiNEkYxmPVy1NwhyJ7wwYUtXVGoCI2qNHOMIRvbaTBYtNlUg9B3e8KxUeJQgBEiNB5N3fh1mInQkNb7Ny/dH2pZQVLfBfDR2zLz6Lnnnsf1m9epH4hGWFduDoFzR9uTSrm5I6WE0pa6yvDKS0bUicgeuHX7eexuGUTu/GyI+cTsjxU44p6JBZqtBGVJqJOlA5aZY0QFHnNMHV1XYBT6c64R0fV5QReS+qAkJFvrCiXRwBeL1MH1cZLgtS8ZRld/0EOTqMDdwbpTGD+bEGoMDSXt79KJ4khPQ5AwTX/nCi7fMmy2D376JvZOzL4k2pY95WNRmvH/yYOH2Nw0/XHruTvwSQytLxhG5FM9TEvMKW6UFqH1I1TkVyjlDCnRlrn+BNmRWR9HC4WbkRmz3e2r8IjFBKLX9poBkoZ5tlWt3PgQXMLz7I2VqBe2dILmvRDw6LBR1xIHe4aeH3oV/BvG/3Kw1nVzOI5bKAgxrEurSqtRU+eVRe1KTRbzFAWxL6qqprFvvpM5dtnf3f5eFFAppQvO7WuAOQzaQ1oURfid3/kdAEZ1zL7PUkSllFhbM8F3o9Fwh0gppfuMTqfjzN3runZBiT1kHh0d/Z3m9Iyx/9912dd5YKezXeyW96c9OOpfU6RIWmYx2+1ovECI79H/YQbU7IceotfM9135xgAbL5pJL9UcHWEGjAhSTDzzGWc6QUm1NmvrRKkp20B6l+5vhMFlWuRkAD0z773Ua2B8zXzG408WbuMICIIuigJRkwzpm21kczNw99M5uqSUlmUz6LEZYOtXr0I1zCH2CU0qDyUiOiyKOocgPn57bR0dChTGh0dYLEy/93QDuTKbakj0w0IqPHxiAuGz8ynWB+TRoLWTzeacu+cshLCMjOXGD700wYbzoDUcZnpQUlUoqLZkDpJYbg6cYTo4nAz2oN1b1jKEMeyKvdHv4PJlQylZp/qyTJ0jIR58K4qhpTXY1K5ItKwyKG1lkXMoou54tJF1Gk2UVHtSyhqt2DyXS5u7aFJWQNY1QuJb1BNgOjbBfE4UkZeefw7zuXkWk9kYs9Qs3vNcunrByBd46SVDNbx1/ZozvZXuQFe56/CDGCWZGwdeCE6mz1rVYPR5oWZ4/Y0vma5skfG57+OMkgKPPnmMn7xnqBS+5zkaj+8HiGhj05ZqIzhEhyZRo4vtXUP/qIoc84kZS+0ohiaLjz/4V/8Ko3feBwB85ap5JncPn+IeHcJGkxRfeMGos17Z3kUrsVLHGiVRX+qicgFoWVgFWIaazI/rrIAgb41aMHDatLToorth6mGHJyeYTE0NQCbNz972GnhhroNVM5SC5u+lM4xjCtoa6xidmiDv8bv30E3N2JwRRbunByhSCoq4BgmJojhIUTaNyl17u4lLL9IaJXzEtHh3Wmb8fPOXv4avv2qeTzpJ8eY9I4d/cnCK4Zm5PtFO3Dg9p8CwLOcoSfpaRiNk9PeHDxgSqmMK/czR5dvtpltD7TqtKgVJyo5VVmKeWYPlEhnReKpKOqpTkDTBw6WNz6p9Nu3itm4D/CAMQbEOqqpETevV+uYWXnjZmG33OwNHJy9tMFFJzElFzg9jBLQHy3JJqfd8D9y31Hjz+QCQ0bipqtrV3nGuXcDNOQO3tEOwpSa3rl3NrvZs/al/4dCk0KX9c6OXYKtFtCu5AJSZL8MjMxeacQMxmXizUIM3aXz7HbRJiTMIZnh6QibRmUIckpox0fO4zIGaAuFYY0zXlM8rzFRO9+KhFxO11Q+c5dA5BeRlmUFbpUtVw75BaOl+V0w5yiPzPKcavqA13RMSggJ5Fi77rt3rIKUEzNnwHE+Ozb6UbGbwKcneJLuk/ZNzTDOihzPf2Qn5nufKF0p10cgCiBJL86c6SN9zdYTCD1wtHLReJnw0c8lenxRde90WJAX7caOBBh30kiRBRGrFvu+jQTWi7XYbDYr/7E/wpUqyqhRKe3BhAoIOhlVVYU4WDnk+Q02G3XZtg9buUJj4AZqJTXo10KR4p5VE7gCopHRxJqdAvhu1oYgOenp27sAHKOYSHJEnsLVhDiDPv3AHAwJBclrTs3LhaHZSL2NUwRns5NIMTq0bXOHODbNHWerlzuYu2u1lGUZB951KCY80LrTPwQqqR6MyDQ1AX6rosziKeklxTUgjohFE8KjG7Hx4iocPTYnP+THtjVmKmg5NRcWQ0Fr/9W+/gZc/Z+qNO4OOKxuoKwVFAAxXZu9mGi7O41yAu1icA3SP/bUNXKN9/+Enj3EwMt/fODX3OlgboNMycWyelnjwwCiCJ50O2r016rsE5wvz2Y+P5ohoPWpQzBqECXhgnvHiyRFOpjSv4xJ1TWVuaeEUkz3OkUTmufRoHD9/8zYGVE50PjnHhNRea2WU7gFzXrHq2SVZTOXQzmaEQWBOJTZPH9XoUuK93W2BkwJwGMaoKPm6oBhDVhI1rQF5ukBq1XPT3MWEClSvClASyFKj/+62ooCu2qqt2qqt2qqt2qqt2qqt2qr9grRPRQAtlTKKoqURYlGg3W67v1tfvn6/79Ce6XSKjDIg/gU0yH7e3bt33eddRBGVUi7L4nlLBSP7fXVdu4x1GIbLbA/gvu8iHZQxBu5bisqF+7KJLxHAoyzBtXWGr++Yz3s5LtCcm7PxSWk9/BTwI/MhP3nnBOtfM7DtL//qb+DKTYPODMVTPFiYQuwz2cCJMrQxfYnEN4r/GyE3mY312xzfuWE+++nbAd78E4OWbF8qodvm1N7tNhCGZFJNCmBgyolJhHEPG2QMPT4/xf7QUEpVDmy2TSZhU0v0SbHvlCgup+NzbHPzezGfgDOT0Yg7LfQ2DFVyMlqgJgWj8XnqTEY3dgzCk6scJ/fNvcbNFiRlhD3hPYMWc6sEpXGhgHtpKGZ/1+oC9YJzh9SyC5QTn6hLrf4aaisgwBSapJTWbLfheySIECVO4TAII6PWBbjCcQXuPHS0ulBQqziYMH2dl8eoaoPazOdH8LnpmzWiTt28dRUFZXKUUlgjtbKNwQaigAroAwFJ/i8n02Nn4nrtuqGKvPbGVzCbmr/P5lOcEQL71rvvo0U+TJ6WWB+Y7FeRZ9CULSwpGyQ84UQ88ixzfRp4HkBCKzUkNCGGZVnhrTeNF58ttu8ONhFTlusffutXcfdto1755JNHF6hfGoIejGdFOwVHQEbIgZ+4gux5UYKR2Eij0cC9Dw398e6r/7NCAAAgAElEQVRP30P52PTpTaL8zBc5zsmLL4hamKdm/E/HMyhLaStrJ3qjpbLMKUwtlVkwJJTlZZ6GoIx06AlEtGYILvAyIWuL+Snef8fMl3xinqGXzNAfLLOWjz7+CIAZ312i3a61NhAQBeTD+4+wQWO6Ii+/9tYx8pFBxkstMZwRtS5+DlWXxs/zX0PYJUopOGJaoz73wjUAwI3r25iRweysyhH3TP8OtEJAyLcScEIgPiORrLxEVho0Iq9GkL55no8+WaBHIN3u1Qg1eQm1mh239p6cGDpblUsURC3J0gIz8vubpguUmVXa086IVwSJU/1dtc+uLd1s4TLWWi2pcMBSwZiBIbYIyGCAnAQ7OIlrKLUUSUvnc0cTPFIS2gpt6Bqa1lCPM4diWVVlxvRSJEZwh1xxIdzvzJGTjOefF5AinxU2UDWsUSAHEJPQ1NXNHm5vmz3KrzOgMmO8lxBlb1FiRjTqWvjQU+oP0USbxJZ2tq7h5tSM6+OTEWISRyit0XeuEJAkaBB76JLPWCx8SFL9LVKJnEoymPTgEeV8dGrmu9YSPiFlgjFooo1BKmhCzZhWTtU3CiNYa8WKpAXniwoVCZrkAZxISdJfx4DQgdGiwsGZWfeSg2PsXjVrSa9LKt/RENnUoEAlDJUXMMqrFvVjWK7lmgGeZU2JpaKoNTj3Lwq4aOmekVS1Q3QDWmPbva7zGkyaTbSpfCeIYoQh+ev6PoLQCqP4EPSdTsFT1i5Gq+rSIYpSFpC5ec/p2SHOx9TvdQFuy0BINMz3GBgxP5IkQYdioG67jVZC6GMjRkXq6lFw7vzuQnoog/UB2m0Tg07nMzx4ZFTPz0cj54MZBgEGAzM2G0niREOq0gqjEYUQxmDeiuKBc0d9Vow5ZpsnOBpE7eeK9u6ywmxu1uGyLCGpfyejIdJzgwSfHvqoCX0ek3dk1EhQLUiMpqpRESUyTzNoKqGQixQ5CYHtP3qC/T0TnxZEI1VSIpubz5imOeKI9rlOB2vrJg5SnKGk2DLPF87rWEoqUdAcjMThPFY703uhuUOgvKiJazcNonj5xif44G3DvDsdkRdonLhSMAHgZGTG/ycP93GDNW0H4/Epvb53hh7VNm1RCch6pwPFiKFXephpMw507zJapGTe3b6MRkJK5kqjQcouW+Sx+cKNy7h6xajVD09HODwxY3A6nzvmVZ4XmFA8MyUKaZoVKHPrJQhU1h+9THGwb8bY5uYGIoplgyAGHKJIwjVZgcWM2HDz1LE20ryArpbiR35oRWq4YxD+vLZCAFdt1VZt1VZt1VZt1VZt1VZt1X5B2qcigBbRS5LEZSjOzs6cf5fWGlvk9fX48WN8/LHJ8iulnPiLRf2EEO7ziqL4O2sKq6pyPGyllPP8G1PdlOd5z1hNXKwBtN9jf9r3MN96aFE9o9Lgkl6THB7JuDaCAFfXzSl/S03RJp+zK3R/WS0Rktx5MQT+9b/8EABw96M2/vl/ayTsO801BDVJMrMOYm5O6/34PXOd4c8ATcW1FUfQJLuEpAk8Nr8/+venuPEN09cb19ZQErI5kSar4wuFDmUJ8mmBPUIetBCYUn/NphOk2ZLbvguqo6JnknsCgorg/ThETSISWVEiGRiUYqoPcfjAZGHWpgCj2oCAbBQYEqRUnOo1Aof0uvQx9bl7/UKWa1nB8myBKmNLpOmi3LMdKwllgLzAcx5GjGnEhIq0kpaT6PX8CNJyuf3AFdROrQxwKZ0JVFFVzgNISY00Nfzys9FH0NqgnEIwVDWhi1Q03YgT54cWCM9lHFVRYE6Zt2argYLqw85OT5zfz+VrJovkhT48uv711iYiEv84OdrHOiHgs+kMnmdlhUtwboVzKPvOFXzPjnOGWtq5Vbv5BKatLRKUUrjznCm4TqhIH9xDSM9qmi0AymQ3O7F7TEpLV/NhX/R8gZ1tUwPx3/yz38af/Mc/BQAc7z+2CSwMwgg+Xcc//M7X8cM//ncAgLhl7uOXbn0Z7CPjwXnv8QkqEuRpNBso8iXC6tADVTlBAT+wggoCCdXAhj53ogW+EAitsIQABmTR8MbXv4HAN+/56K7xSBMsxEtf+AIAIEpKfPjTt801/eyv0OxRDV2S4MXPGcT/P3S6+PG7Zs17+YXbAIC9kyFGBwbFnSw2cfV5I4Z1/blfweDyNQBA0O6h5BE9L4WQalyTzIyTcn8fJ8ygk2NZ4YiysXUJjGiOL7I5tgh5bRIimUQ+hiPzWXWeQtjs46LAkwemHzcHL8KLLNpdOa/XLDM1n/N5iimJZYzHM8xnyxovmzU2kBHJgQcBOn3yAF21z6zVlZXdVw7Z10yjtNLn2jAbAIOi1BdKQXzrN0lsCVmXUNzujwppavYapZTzYyuVdAifx7nzoWsk1iogWrJ9POHqpLlgTpgDWDItOAAiSThcUEkNWVhrkxpVYX3eKpcxb3CNNULjb2zRHPJynJI/5mS8wISG6XAClKV577UbHF0Sd1A9hYLEniSJvQRiWYPOATQJNeu3Giiormh0NnF1xmUq4RETYzY2850zjkayrO9zXrZl5XrAY0DOCVGZCmtR6qwTgiBESv+XTgv0PYNKtTtd9KmWf21eI31Ke9TwDC3yXvVI/CZpd6CHVntBISH/tyRJUFihk6p2dfYKytWRcmJ7CI8/IyBhrbOUZm5M6BrOviAi9KzT76EgpCmOEzSJveX7PgShxZ4QSzQwEM7+yYkEQEGRp5pSEg4g1jXmxJY5Hx1ikZp4LAyYsw/pUk176PvwqIYxjmM0qJa8GUVoEBoOMGR0rVlaOO/kzXVTU7a+MUCbkKEdsYGE6hYfPniMRWoF5Dx0LDLFgaq0IobLCWfDHq21+13KZc0sGHMILOfMCbpJbS1WJApC6dI0RZZTLXy+QJ3buSMcWs+ptjcOOOLAfSEq+r88S8Gp3lVBICUkaTadOjGakITdhFZg2lxbUSj4xOpJogQTYuvJskIllwxBy8aryqVAitVolApLawIJ5y2LUGCwZsbKa1/9KgKKnUdDI0pY1UuPys4gcT7KeV45tE1rDk3Xuig05oSEysDErMmG75Bl1u5h+7KptXzujW8galkhpQ4a9DwhBGbEqjuiS34yeAKfcLPz2RTDYxMfjqZzp8kALhBaZqTboxOkFsUtFlDEWCirGvv7Rq/j8s4mLu0avZAo8JGQnoXto/FwhJTGQZZlyHMSuyzLpfBiVYEvqCY8D13d4c9rf68D4Hw+dwe6yWSC0cjAzhsbG46yaU3ggaWyFrA8kF0M5C96+IVh6D7b9/3lgeFCu0hFtX/3fd9RQC+KwFRV5T7b932nKmZVHrWswZVd+CqAir0Pxhn+5G2zkH/gA//4i+bBXd6iRSbiyI/NKPjKqIlhYbru333/xzh+ZOhyN69eQkwmuurmJXz+q3Rw1X9l+jPjyAsrGaoxnJj7Prqb4SttMoYcB+hNzELz3scphgdmoHS2zODRPodKSMVKKEzGZoIsag1OlJPr2xHWqGB/PJrh4RMzSPu75toylOgSvbAtItBbkWkf25fuAAC+9Ks38Mf/2x+Y1x/tYe2aOeRuku/g8cf3IStatEqJiHx9VK2c6arWGoKeeZTEz3gxmueGZ5qjg2p24TUNZwtnPYWYOZABRk0uILof9zxw6+zLhaOAMs4BK0RAE1MLBU3XIbVyJsGlPIPSZuGAGEGTqMag8yrylOiBMId/VeYoSNykZgwxLVCLsnCbT1UoDE/NM8qyAp02eVdRYiItZs6PsCorp8B3aXMNG+R1d8CXnVVVCrQGu8W6KiokJJhQyQJ5Rb5JEMjJM0t4HgIyrw2DCH5k3l/aoB61Cyr/+D98H3M6jLQ7Lfec/MB3dFy3OZWl8za8994H+PDtd00fYAZl/e3aG5BUyFwVC8R0yJU08E5PTqAX1o+zRq9rFuxms4HTs2Pqr8DReAENUVt/T/OKUdiyFNcaWlq1L88djn0uXFDcH2zgq7/8LQDAnVtmI3j73Z/i7nvGG+jW7R5u3L5mPiM9QXX6CABwcr+F3iVzeP+lf/AP8K//538JABhX5hmvFRF21w2998vffB3Xbn0VABC0tsHIQ2xRLZyaV1VpdCkwGn5s1pHqydLDSLZjiE0jLMBYiPOnhj4rwgCqYekiZkDUamni3hQcfkBCObKCT/Sg/DxHc9t6LJVoEq3abl7D0QTDc0Oxy6apE9BBIY2LL2BoYETjqSYc56m17F61z6qV1ndNwdHOalU7ilktlzTRMstQlrR/YKlCaYP+WnCU0nqLSTBL9QxiJ/ziK7U8Iakldd+OGxFxl5zlgi/FXNiS+q9UDW4VQQEE9NmMqJR1rZBZgRHFkdFB6HB4DtD19XiJ1hWzDm+urbvrrIfmcyepgqKk0cF0hL1jk8S5+8EDbG2b/+ttN9yaFZNPsNISJYk3zWYFClqvAggoSoYNj86dN2Gele6+rA+gJ4QLln1vWfKia+UM0eH7juqYZrlTEXRmiAigJO1RtUao6VDnNRBTnLR2uXKep0We4ZxUSO2azsMIytJuNUNCh/Sk2YCak9roBdEezc2zMf1gn5u/TBhKicqZ0i7p95optyZHVpW73YUfmj4KwgAhXbPnCbd3CMHdAdAXnhunNR36wDUUs96R0vmc5vkC44nZD9L0FNoqNDLf9Z+4+B20l0Zh6ITMPF84cbUaEoWl11Ul2tYTd9Psu91e120w3POdWMfaoIM5Pc8wDN2411K5x8jtgVowp4DJmWN9QmvtVEcB5q5bMeaehT1P1GWOojR9cHp25ii2wvMQEN2v1W1D0HfmNNfb/SY8StTNx+dOgK7MMwh6LiJallKFYQTPo3IIujLBBGRAJSWBhxaVFikojIcGIKhk6RKyZSWhrbgUrUtRwF3JABfMbh1ggYZHezPjlZsLt29dRst/HQBw/54RpRmdHqCiv1caiCi+yuYpTo/MIe3y7i7u3DF7+f2fXcZ7PzPAy9NjE9f3t69ic8uUML36zevYoYTs5tXrKGxiJ68g6V6UkjglVdExmdCns3Pc+8jEf1lROR/dQipIq9ouuFNOb/fNmInilhO/gardGUV4PmYkJPPw0SP4JJTUaMZO5G1OB8ezsxGKeUr9XDpRQqklShofyHMomk/5PEDdssmOv7utKKCrtmqrtmqrtmqrtmqrtmqrtmq/IO1TEUB7Sp1OpzgmqLOua/T7Bhb3fd+hgRc937TWTojlIk3Ttrqun6F+XhRtsa8LIZxQjBWa8X3/Aoq0PLte/L+LlhVSSggLN1u0nXMIoofVF3yHpqXGB6n5jEOP4yp1zbVblGEZlHi6Z+7Fe2eIL10iCfY6xJuPTcb86MkU4bahhH3zznVEvjmt5+ThhzxGQJTTieL4y++ZU/v6xxVukddQFDMMT8x1vLU3R51RVsdm3xVwPiWKReBBU9YBdQliMGA7YLgqzMn/sJkgD01WYf7EZDPSmGFKBfEvXN/FzpopHudRA6OcvJd2dnH1NeMRd/eHf4bN3WsAgKxhMhofPjmEfYRCa1hKYLvbRU50y8lkDEXPtuF7LmPq6snxLNXTfp7W8hk6qOMg2sy1li5jppUEpywXF0vRE8a5o6cAHDWl0zLK+JWKQVL+Q0sJRlLfs8VjFOQ3I3UKSVnJ82GBGXGLfOLD9NpNVCRRXRYlmhekrUOi49TVAqe1LQTnDjnpDQwSWMkcIVF6lYSjVsmNbQR0v93BmpsDSdJ0/k0WYSvK0mVgNeDQx7KoXJZR6qUgU1XVz9hzAGaO3b9vaJh//Rd/gZIK0NfW1nGLRI7mswVqonXMU0vLFpiRVPMf/e9/6BCBGWbQlMXNtgosJmYuZNMFPJKSPqds+mg0wXxh/r613kYnoayyL8As/Ks0lBVV0Nplr232PQiBilBBWS0Fb6SqXZZRg4HRmhJ4DO2eGfeDdfNT8xA/+NH3AAB/+5P7uHHJIAatJIGYG1RbnhzgjDKtX3zjdfzsDSMUoxam/z//xnfw0i1DB720fRsiNGjmNC8ghRljgi0l2AUYHlJW70lFyHN15OaI5AL+fYNKMs/DjOguwhfIRwYB2blsaCOz2RwzshS5fOs6fFp7T9IDPH5ikMPJYoHnv2gQyihuIIlpPpCdxvHeKcqUJLGrGprmkC5KOBNCKFhbHYkMpVyu5av22TTnJ1aUqGl9qevKiXJIpR3SsUhT1OTXJjhbIjj2wzhznqOcL73WfN+H51t0WUJj6flnhWdsE4K79Ypz7pZszi748irmPpsrBs1pDFG5hfa4Y3ZIn8GOttN54WhvUZGioQ0755UXjfjJc+s9XCZEqfRjSCrVqCcSJ2cmQ//w4RPkqdm7+oMbGPTN+itoDS0riTNa30fDCSbnZKdSa4esLLIKmqhpealQLizdz1xpJTlAdk0VW3Yw18xJ+jOfO/q0BsdkZu7LWtIcnM2cN26rEcFrE52rEk74LGy0sU6lKWejMRaEtCgqO0jzErlldnAGKez+yCHpoipVO4SJg1+g41hmDRySwLRyJRKAdtCUVnq5NQfmmqOk4WygPCGcJD0XS8sLweGgBwXpRM04t6wNIzoEmJ8l7cez6SkWiyG9d7G0lWJLISHLABLMh7BxwcUxz4DaxhyMuXW40hqSfreoWlVLV2riYUnTbLe78OgG/NB3Yjp5njkGz1LUxYMkVFtzBkVIE4da2qBQT5gfEoL2TWd1VBYYkWfddDQCp8+LwwAJWVp0mjE0UWn5nMpjFMPx0RH1gYeSBOgCMNT0rdoXlhSFOA5dDGODMY8pN/79wMMWicY1k2QpdKMlKqI0anAILO/RPBPm9mDBuUMAORdg9Lw4uLOSYEyj2TPje3uX+oUBs5npg8l4hiHZK4zPT7HZNSUIa60EW9vm+l559cuoSTRJEvNpcOkqtq6bvXnz8nUkDaKEa+58OH3m4eJg0VjGUgBwNBxjSuwApZlDyZnnLSWutEZKpRrz2gqu5a78RyrpvkJ4vqNaD89n2Dsw60Cn28aIBO7SufXVrpFZn9Gigp1EWjEjoEVPYDmWKtTlpx7xPv0A6Ey8hXAefupCXVer1cLJiYFffd93wbwQwgWW9rWLdXoAHGwuhHCThXP+zMHRBr3uYj3vGQqo/b+LB8qLhvOMMeeLZP/OAQhveRCRluEE5nw8pGL43k/MYalBweo3Xmvj2ksmUFzcnGP8jnlQr9yvwenAxpIEV142fPzNV04g56SMKYmz580hqPbngx8WGP/EDKSvNNsoEwpoOwpvf2ge/P6ILxdH+0yYdnQABeUCAa0lKjKQPz/QGJA3odfdQJ8m06O3jZ/Yk3qOkqD8ac3wIjOw+aBSaBbmMMvPMkgamNs3X4Y/MJvtm2/+BADw4b0PsbVGakmydpPX832EVHNQj5Z1BvN0gWbfbiLLhWFJD1JuvLEL9YJK/X/svVmvZUd6JbYiYg9nvPOQcyaZJJPzWGNXsaokVqksqd0qS20Lht2CG23YsB/aMmz4xbDhv+AHGwba8EtbgCU32pBbrZYglVSkiqyBQ7E4JJNTztO9eecz7r1j8EN88UXcZJKS3G36oU4Q4L157jn77B07dsQX31rfWo4fEBnom86iCIGptYlymeKeckLEB1kIXiDCfrkx3mQ2NOFCIDDCYOzpfHluE1+hbeRUQxKoU9YafqgX2h10aExba9lI2EzHTFdYWFjAkDZQQQ20qac8ucz155HR2O30utC0Set1I2Wp1+mgCv6YlX9vkWU8PgqhoGjzOTZASRO6huNFSyd07LBBtNbyM/7VL30RO9t+DFZTg44ghbeFRWxRwufDy+9Rx8XFrt1us9eT6ViceeAMdTbYtNTUFmfI56ge+7H20JNP4CukrLqysoI2UZaWlxaQs/Kg4gW4qiqu+QgTYs8WkPSsGO3Yo8uqOHkLIaBIvlS4ONGHpMDxk2fx7W//FgDg3fdexZXLb/vj7dxCp/ZB5dL1Xax7/1ssPfosXvjVf9v36dAvQieeeBbtMKdYgRb5NBVthwk9IxY+APO/S5RnH0XalMo4kIBQXEOlINCbBid4zSa6A6rpsoslFmmBbh1ZZxW/02trsOSTZU2NA1JZvXz1Cqy5BgDYIX/BtgJKUiU1MLC0uRA5YMO4FzIudsKwN9esfX4tbMa0MTBhbZMCOc1HUoiY9Bor5DSPNVqjpvkjqPEZYyMdUFgIhHk4qnYqIZmiZW1M3IX1B3A816tExdnT1IO3n2AJbucARclXGRJ/2nLCNssMBC3OjWkwpJqlaWPxzg0/Vs9+7GOP/uJJtIg2fvK+eXRJnfDYyOL2hk/cXLt1EyYkWHQN8o2HtWQmrg3u7Pg5anNzgPE4xCoF1zMKOEwGRN+sagz2/fvLfqR2NeHxhGAFPgkJIYIRNqJHIgqEkOj2HT8XDsabsPT3I2tLEIVPQrmsxDzVruVCw1Jwm7UqHBB1dYdMR29u72FIa0delKjopKY6gxEUKArH1MvGmKjWTVetk0Q6EGsAndPMBHaJQregTWZRllBZso6HXZoUCNGyc5Humdlkpxx0HJxhz7ymqTCdEiV9ugdjR/R9mqm0dVOxUmhYI7I8h+KEhEq4lxLORYVaFTau7S5UFpId9NZGs6SBQwZBn+u028joujMlWcW0ria8cZXBQzHLeMOjjUmSLuDSBJ+ajHFcSNY0tInX2sJo/9piv4sjq+SHVw14MzvYP8CUkiQjSgIaGO7/smyzomyv14UIA08blFnwiRSY73fouvyfMyk4Cdvu9HDypI8D11cXoGgsNabhRLl1SWIpbHyN47lI5AJZ6BArYChhopXySRMAVhqA6iDXjpC+xtwcNCm2bm7cxBVSFd+6cwebN/0atrjUxxNP+8X53COP4IFz/ncRvKHbc3FjnrdhXJj/DCtZC1mgJt/D6XSCI2d8slTR34sygwxJaSE46ZWXbd4oa60hKEEqaQ1uFy3WK/DXGDxTFZdEFUUbe5R83RmMsE/+vlXwb253YOi5dmbCdH8nHUDXAiVRUOyQt8p4Iz+lzSigszZrszZrszZrszZrszZrszZrvyDtMxHAgKS1Wi1GB5qmYRXQ1dVV9pHq9/s4INWdFJ0LCEOWZYcQnvTYAdVLaaRKqUgZTM4noIJSykMZqvB6lmWHfhdEM0kErThDIWygL8Kz8zJSIDUSN0hh5y/fIE+evRonSFnrkb/3DE496H0Aj966hQf3fX8caAeU/j1TtQcWzUNQcOzhrVd9pvLHf7aFZ9Z9Fq+31MEPLvljDMdj7JCy00SWkIXf8WdUdNztKExIFbCpDKNRzXTC2fqrtcM2ZaUev28V9S3KDJJYw5vb27Af+YzS1Ys3cOWa/+4zZ07i7ILPni6KHM1m8Ddq8Nor3jfujde9YuJkcoB5KjDNhMSEIPnJaMyZ4LxsYUoF+ePpBEdOHlZ98SylkBkVnNGTjkUoSRGU89D+vUIxeiNkzKhap+BCoW1TcebKuYIRQOOIlts4WOPPX0rJVFohMjgETxsLR0jNcP99FPIMAE+1ArwiV8g4IvESdDYWx2dW87UsLi/Djcf8HsAXwYcBWU0rlOSjJ5Vk+kw3L2I2fzRCnSjrAj4jaYgWKoRAHjLrLQECvyCcQcQJHcZ0HkFpVynFz+HXvvwlNFN/Uu9fuIh/9cd/QR8rUVDGNPjzNU3N51EPp9i1fiwtn1pDCZ/Fu37pOjTRk9pZzkpXEJ7CePLEcayt+KxmmSvMz/k5RbUKTAgp1VrzfFBXFX/nhMaX1l1khIQJKaBEQnsKRf860oWN8wXfANhDsYDD+rrP+B05egoXL/hs55/+8ysYbJJC5+4YrTWPqOerx3DmtM9QDrfJG1VnsPSsNlpBVIFnlTGd2yD6QWmtceqsVx51QY0pVyzEoYSCCtQkSKjwLGQWTt1NvXQ85jMrUBC0kucaBWW3ldQwREspO0uYkrdf0fHXdOK+RWjyIKynNVOBnLZwjK5LZgA6pRhJmrXPrwWPUFuUjOTIDOy519QGFbEQJp0S7YrEAKYNNIlnBCEnlQn2nBJOISf0oshznsesi35UxplYUkFwkLOO5zSka7d1jL4IIFXBQNElnzNBCH1tObsulGK0MJMOMiCNqo3tqY8zfvyez/w30uDEST+XrK6vYImez/seXMDjdLyrVzexs+/Xs/XVLhRRMg/2/Hpw+do+PrrsWT8bd8YQ5IGrBFiwSTcGI5ozRuMGgxEphS8Htkqc150N//PKqtNpnIOC8FanDUAFWpj/+8FoghGtpcPxmOfb2lisLZCwRKeAobX+9uY+LpEw1O0dv84fDCac+Z9vFezLNqkVz39FWQLESJg2NbNoWBnUGASJEucQKZtW83U5l1wvGy0L/g4AkDIgW5pF2ayzgCKkzFgW6AvralVPmXqpFCCIGppnse9MU6PRQbnboCFUWykSwpEyCrEIwSIlwoHRNgEwYt5pd9Ei5lJG6JOUIlitQiZidJnK4PKAVEvww2ANdE3xK4IITFRTlU6SNy9gnOA+s84miLpBQ2teWGMFwMqlp04eR0Hw9bVr13HjZqD2DzCmkorgyeysBUQQwsvQpVKTg04PgyBS0u1iaY6Ehlo5lhf9uA9idXP9Lvo9KoWYn0ev75kuvX6XSzKkBQvqwdaoSXAsoLxKGshQoqUjMp5nURVVasAE1iAUj7HAk23NraBLYn9HT5zEGimkazfBhZ/5mPTy1YsoaF5cmF/EqTNe8KW9ROKHtcOQRFusjmis1nV0EbCISsqNxv0Pe9HDFjEBpRKJs0GOFq2reTuK1GltWZAxUNqFFNEP1QGG0HDjNM+J0go01GdNNYWjuXV+0ccW3W6GhlT4q9GIBf6EMIxQ5hDsMwohME7UaO/V/kY1gMvLy7ypSg3YlVKs4Dk/P891RSndM1XkDDRMa+2hWr9U+ZOpmlIeopQCfjOQ2kqktM/0eHxxWQZ5D4wzUCykBZQOxwMQIG0JTEGLn2MAACAASURBVGlifpcCpK339nBwwdPfHn6jwgvf8YHfM19ZwJE1P+jU/i72yfS75VYxmfiF6tZNfxJvvubw5hu+lnK/anCVFtrhnQoXjA9k1UhhQhz0aQm0CUafp/W7sBOeEI2LC4eVDhOaBIUyGBGV5q2LFzB/x/fNbaLhjfJVaOcD9W6W4e0PvILrsHLI+v78u90+8qBw5DQu3/ay0xdJrj8rDFpt30fHT5zmes2tnW2UROFrdduYUrB5iJrMHHzLdSoqeeadMbxwAJHfH6gbvnIjbiZZqdBmsJY2nKNrKEnmvNtuoyZltcHUU3d392sUbT8xCNuGI7pRlrUB2rgc7O+jpofszu0Rum0yJeUEQ0x0CCH4dZtsAFE1/LxMRaTMBMXQdpmjbsJEmaOhZ8hYy5TXLMvZBH00HCBjQ12ysZiOecOjpIKlZ6QoSjZHH01GmNImDIgKv+HZU0qxZcTg4ACDPd+Pk/EQRaiT3d9HReq3gjYfSoAVN6WUTP/Z2byDV37wVwCA5fl5PPbwOQDA4lwfi2RBsrLmF5n5uT667bD4AnlJm7dMoUMLel1VTH2WUnJiKZR5jsZTmHAPlUJRRLqrpUBSWAlBK7ppwDYhYe6onUNGaoPtdo77yMLhW9/9B/jj3/+f/PcMbqB5x1NDL16+ioef+ToAoDvn6ySl7UJmR+h4YINcpZS3HoGn9gQGjjUOGZcOUGBiHdNnlIucmsYBzPS3AiZw50ITgutR2kbABBqK0ahFqKhq4uJjDW8CShonulUAZOCrjEEW6PkQ8TmD5GeysYIpU7P2+bU5MrzWjY5KkiIGV9rWTMtTKoe1IQkZ1SszlvxXTCO1Diio7i/PCw6SjDWwWaA4xRqtsEmzpokBvpGsGJqUe8G6tJQhJjVzUse1TvO4FwqQXDFiWVZfSAlHlPRb+359evnda2hf9vP6XLeNc4/4wO+LX3gYp4iy9sCjD0FbqpUdj3D5ilcXfPMdv/Z9dH0T23dovRs5ZPRQOqNYhr2uDAddVeNigBvYV0rypkQbw2uiaVykxUFBkBl1mVcoqWyA+98YVDxPDPD+x/789oYTrMz7eXOuV0I3fn6+emsDN277mCJQ2TMlMbfgg0ZnHUZEKZRaoUXlAdIpmBCoO8O1oyJs7iB5bYbWsBRPOGMgXAiWY4LUcU2o4fElhYPRYcMfaYJaV8hNoNXWbCJf06Z2MD6Apbmr22ujpDij1iWmpDxeD4dsjm6M5iRx13ao/x0KihGkyvlZgHDRqLyqeEOpsoyVSRXFZULlKFSoS3QwdD8braHD5kw4trEQ1gfj/neiyxvJx/ObQXomoTiW1abhDay1DobWuYa+w4kY45S5gAq1vYN97IfawMmQgYYs1NkC0DqUvmhMLN3vxrJBfDMaoHR+M3V0+QjOnPbPyNoRrzzd6XRRBMXWvERGiSIHzaqWuQJvYoyOVlEDsoMQWRvKUt86C0vJ9syJ6AwAoNahNCgmlljPw2pOfi52+3jg3GMAgEndYED6Att7m/jhD18GAPRbfTz2tI+/H3iKKK6tOVR0bs44WBfiNY3GxPMP8QKkhaNE0ZjOzTWaExyZ8MrLAFDUCkLEemJHlN0Q75V5xjFhrgAT6PIipciHu+YVWcP9DOU/pmkjVK9B60gxFxIlzaFOW1YwHk9q5E2M+e7VZhTQWZu1WZu1WZu1WZu1WZu1WZu1X5D2mQjg/LyHG1NUbTgcHlLcDOjG3bTPFO0L7w0Zsel0eggtSd8bMvvOuU8oFaaoX0oBreuaj536DWqtE5poQGrAHkUAWJnKOccCIRZgBcMh/ayM8UoIAF597RLOv+sRsae/voYXvuNFHI6eOol26bPnG7em+OlP/E78jTd9ZnHjhvbiJABUx+Di0F/fDedRR8B7lemA4Lqai5BXukG9xGFMmaOD8QRNQNWgUIcMW1Uhz0m9EjdREE3X1oGTugy5cMafR2YwJiPbDy9uQCz6c70hppCEROqijQsbnj4zIWLImdVFVnaaTicQhATX1QQyFMZmGRSND4ea721Q53RSeS9GgEQIApXmsCBMFLoJgkISOhStW4eMjYQrOEeGqePbGFEWrtfrw0mfGRxNvIDAsNpALvw1lfkqSkFea66AJKEeY/YxHJKy2rBCJvz9VM5fU6vV4ixwOh6FEFyQbaZjNETFskWJmsZ0MHYVtkRJKq15VnDhbz2dsO8T4KDJ96tolWhTxjRk2qrJhAvYKwimPQuhkCn/+3Q05Ex2ZTm1zs9TVVX8/E6mU9zZ9mNiY+sW+kQLEZlFQwPO0bNgjOFjOKdZHMTUGoY8Eg9sA0VCQydPrGNt3dM9e/1QlN5i3xyP/Ptza5qGxQlypWCD0IDKMKF+DJ5JdVXD6FDwnsOQMl+W5TzXiMSDKBNhzCEaNhuDnDLgUgGOqDaPPfs8NGX0fvj9f4ZLVz+k79zCSPtrOPekz86fWTzLY6JxkdaklIKmLHldJx5cDtCKxhDdE+EkBKPagqlHRgA2oRMJBn4iCh04NZXTjPRp5xgZdEZBUpYccLABBaXrFrDIAo1auGiOLQTPiQaO6WG1szxHzdrn1wJyb62NtEphWJVOOcOUejgbKceNhuZxE2hZCpoQPWcl04gFIpJnnGM6MJyDCHNxAHishaWxbiBYGMPR94fPBR9OB0Az2kTHUg6SkHupHUcnQltes50z0C3/nhHRnmwt4II4wu0D3CZj643tHZx9wCODSytHkFGJx+b+Ht5+z3tufvCRp9Dd2RvDNcQ8cAJCkbLn1EHXgQZtUOnQN9G/zrF4STQCrxvDisnGOFbchNN8u6rKIA8+stQvtW74WTQW2N4mg/vhFDdaQVgECFyA4XjKSFEQhVJlCxlRRxs4WFo75DhDRuuxVYoVQTVimQWrigvLt9tIBxXGDCLV11rLqC/fY2u4X6wEK0LCxdhuPBoxNXE8ydAn4ZGgenkw3GP2ghMNoxsWgAjCgIJiMniqZKBTthjxEpC0RimpOOaojGYT9+m0giYarDEWLVJfHU993y7MdQHqL2McmorW92kdPW6t4OfJwjJbQ9pAs44sM4cohKOUjEwoh/hcOBdRcvpprGPPQ2EbFNSnHZVhnlRAndHslxgERpRQjEg6A0bspJVwpNrZynOsLfv48KGzZ3CaShpCTCKUZIVXKRSXHDkXRfQcBKN6SgKCvn84CsrDGjlBV0JGj0QNCUl0RSui4KGS8XWXB49kB0vIaJm1MN/xDIizDz2Kg5G/lvNv/BgfXvC+n9cG1zAlRlDZ87Hd+olTCMQFDQXrCAW14LlSO8tKrcY61HRdQV9ZCYHAVzTG8fg2soaghVAbA0PPpFSB9mwYDcyVCsxvOCGj6KhV4K1JOg5YQbVgVoQsSi6DUhIIZgdGaID6TBkRS9w+pX3mBjAETuPxmOv7dnd3OeA4duwYB+U7OzuYEORrreXaokArm0wmvEmrqortIbIs44mhqip+f1EUfLzU+iE1kL9baTQ0zXC6ZVn6dCMo6KYJIWCZfxXnMGWBPFHwAgCZCUiqVSjWC4xJNv3Flzdw/rzvm3OPzKMz7+/QxxcP8OFlf/4VQcqqVKxWJeoYzNUQTLVyHQNFVhEwjqkZ07GnH+qqYtPK8XiKJlAoixZvhOoaaKiOoNvKID3TDr1AKbi2CdP1D/rmlkFDQez0QOMGyczbDjBeJyPK/cuop55m8sBRH7yvr81jQPVNGxtbWD/iawf7/XmuNRMASqITTcZjnvCCgtYnnOCTFu5havHBC6NxEIHWBxEVtMwQRm/ReweYkKLS9m6JRenPW1u/ATS4hm2qmcjFKtbmngLgN4jBuDXPM0iaQLPMYjL1dIuyWaF+bljZKVOx9vTg4CCePwTGtACvLK9hTJPVhBah2jXI6FmRMj4Lw+EQe2Tw22l3MaHJdF46HFRjOnagrUpWUNPaco2AUo4nYTOZslWEdY5VeJmmZAxTUtrdLlbJfLzslFilezscDbG1tUXnFzaZkqmUw+GQa1YyKdElY91WobAw7yfstfVFLC/7zVKbArmyVbDy56SqmK4jbKQIW2NhmjgmQhS1s+XrdpaWHHpdqoetLWoKrFqtVkIlFzAhfhBRyTD0edM0TDPOcxnKVDAVCo8+9wIAYP3YGfzoB38OAHj91Vdwe9t/z1nQpkrlTDGykMnm2CFEB01T8wZQSgUXFNTC7G9FtDCRBW+qpQArgloTa1LYYkeA6wYcDMsr+ssMm0TpAxU6p5CQcmwArZCpQIFJbFruStCEZ9k6GxVBZ+3za4lUeXj+TF3zfakbHRM+U811rk5EmmITxqlrOACyTiDLSC6+yKOsvZScLNBGc6BrOIi1SXmG4/OASxK87vCYDecXgn0oiSLYQCjrN4EAZCZDdQZgNYrMzxXtNik+ImNrBZsXGJNS7uvvXMf5S14ptDO3iFbpjz1uauwSfXBv7F/Tro0sqWcMqupaG968GWvgTBIviPA8U79Yx3O91uC6bNi4wRIuVbi2qAIN04Y1Mc4TcOC5pNY16jrGIhz7wCFvURK+9K+1u21OvDbOoqaNqMxqKIoLhJCY0Eanahpme7p0Dkof67CpOETqFXAiJN5ZJjSevrCsVqvgmOY4mkxQNxTbjQTGpF4OKivYHw2YWjqZDNEmerrIXEyiwfFmSmsDa+JGDvDjJ9wfC/B3a2MwpU36uGrQVCzbij2QSuwN348HgzHXo0MqjMmq6M7WLtfXykwyhbXTbaNNdXY5PTcGVXK/EROQmeLaQGsNb361sVwrloe1ykRbIwhgbc0DM932Q3jwrKdsbu3v4uNLHpTYO/CxonUOzkUrjGDBkskcnZb/jpPHj+DBB88AAE6fPoHFhR6dR7j50cJByozXXV+lECmIGZWJNFYho0WWN1VNzbX8QrZ48yOM4gySEg5h2RFOMrUyUG2rasrUWIiMa+SzcgFnH3rCv+5icvvG1StMs9ze9THf3OIa14JqGK7ttSLqBBhrue5TG4OMKNPhscjg+L4AitdP3xEUK8qME2ChtlYKx/cV0sGxcr3lTbWE4OfQ2Ib1LPJAMXYFTKjfNpYfQ6MtJ+d0rVn1vG4i7frT2owCOmuzNmuzNmuzNmuzNmuzNmuz9gvSPhMBDKhfiugNh0NW/hyPx9jf99mG3d3dQ55iQWUwZAWdc4cQwjTr3iH6l9aaEcBWq3WI7gl46mjIpqQ00kMKgXVS5J5lTGMIx0o37CkV1CX/llKwcqPizHmkQGlrUVBhcqtbIAj9vfXeFpvaTqaCIfIsS7LoIZEjBdNdFOL3wUV/HpWBKXC3NjzyYrTFwZhUKhvNGY0yb1jwo79UQJJHXF52QEJn2Lzl79WcFtjXHv2yWQeiTRRRleHO2GdZBtUIJfkczXenOH7av0fUlEGxGpaymoPRAItTKuRuDCpSSkOmgDmP9uR5wUXUDNiqjH3ZAM2ZECDJkhqDJhjdkmdM1Vg26MykgxSBvraPuvaIUFXtYjTy43fjjoYsPYpVW4/iVfUudgkBrKZ7TKcrihaECL58bRbEqMc1hgNSOiV657jS6NI5V5MBbhNNNs9zdIiiUE8rQJV03g5Tyrpev+WpSb2VOc7OS6n4Wbi+sQlN2d/5hT52tvy1nMgcWp1AGSX/t6lmOqh0CnIk+TyC+mOn1WHDbpnnnEFmWm6CtPbnumyGu7TSR1OtUp9OMCTT8snU8udTRdF2m1C/Voljx7zIztrKMpaXfaF5t9NCpx28l4L4A6KK7HTKlJNCCEYpjI0IQ13X/CDvEgJoaw275Pu2bLeRBfqyjsgs0ufdARVlf8Oc4xVnI92dfUtzBW3JcHftYXzzV70f6ENPfhXTys9p88tH/XmIaOwK6xhhBSKV3VjLVLEsc5xJjcNfRGE5z6HzL1uZUETACmk8qSW+mv7aA3/ARkpK8rHUl/Xw/BjnKz6eAIsLORfnZGEPz6Oz9vm0KbEA6lpjf8dnuKeTMZcxGG0xpfE9nUyZ3QJIOBHXacBTnawLAi6WjeUbnSEj0Q1IESlJUiDPQ1lGVFoMNDxtdUIjBU/4TriE3mhQCpq/6MAyi7ojykS6thKCKU7K5cgCFZw8R3MhoQKsYB17izW1xpDm21E1RIsYB1oJNJaEOei7y1yzqqFxEqYJ6ACioI1UCDCFM4JLBWp6fqfTKSsIauuYoSFFRNO8ImWY9wQamvuD+p+zGkoGZDSyPCDidcHmvG5mqkBJaE7w3y1aJc8ljbUsqDGa1BxbQAJT9hSzEAFxIcSIFV3hmUpZUPMU8Wl3CbobFbAdM6ycMYxMWKETH8lYDmQbwyhFiK+qeoKK6IzDkYMaEFOkyJi9UFcNx0xSOCjn+7TskjK5lEy5s856lASEithEoDAgzs5iSkjuxo4XyNva22UBDue8WioAjEdjNITcqjxj1G9ubgFzc/4YLXqtLKK3NWQ0DrfOspetSJAfa8Hnp5jNkSiQ5hI5oVKtssDquo9r1iZLaHf9Wn9r06+J4+mU48cyz1DSORW5QIeYWaurC1gm30yRqegtl3goByN7a2WyLlmOW4QUyIlu2yiDRvhjt1sUk+QWmsTVKuNQBIXuDKjp4c+M5HIgIyRsoCc3QRBnCk3r9MFwiHbbX1cnz9EmGuz9585BUkxx4vp1GGLp5RSTDKdjtMs2f0dt43oWGQscosOgRpkwbvxJJ8injJTf6Ifq7xvToGncCSXZicA5BxvKTpTkZ0s7G1kWOlFVZhYOEAIAB8tKwEZbsPh9U6HR8Vn86yig4m76ZNr+h3/8OxR5JHQAFzm8SskIXwqRbJbSupQAX0azS5HswsRd/+YbAZds1gKHGmya6FykfhoTgywgBrICwL/8mZeKfueddwD4zWIICOfn57G+7oO54TDS26qqitSnxKj8Xob1qV1FqgRZFMWh9wBkS5Fca6jj6Pf7h6wrwvtTGuz/+E//2J+PtpCBdgFA08NkhUAWFNQAL6V7d3OHfnzu7Xd+wytBBqPKg4MBLwRKZihbHf69moZ6KRM3CkHS1inmnMPF2rtGV7ww5rmEIZ67tQ7tlt/IHT3qA/UHHnwAt2+TzYV1WKMNyvXrV2Cc7/Msc6xqqW2LaXn/+D//7wD4ADqtv0qpyqlFiU0W0/D+cI9TtdvD9XRpIK8Pff7u96R1tM65e6rt3v2eh77+q77P6jBZyBiJ+U/475Mu2TepQ+PXvyEq3EkR+f8Cjo3ZY/BJ5xfkrxMT6UObiFBb4JINCASrYhkBSPrWX/s7pwAAeZZjYcHfwxsXP8AKffccJE6dvR8AcOLLT2Iw8BTnx558HN/99V+hPvX34sJbP8Uf/OFfAgDWVtfQ7vmF8fLVKzh+ztf5NgfbePD0GQDAv/Wr38LN878PAOgWfqzNlS0MtB+vP33H4uzCfQCAP3ztx/jjP/0zAMD9KytYpY3+ptU4eYo42hSYrx9dwsqiv666rnH6tP/9+Imz+OKXvgoA2NsboqLN5+JioGWfwOf/dEdq6Gdyumft32j7b3/Xr82Z00wLlkn3Swj+tzcbTuprwuuhnsfG5y9YlAOhNo/ei0hVggNT+0ItIKQ8RAsO84dDWmOrOcBxWY4fvEf2BRs+YJ1MK65pF9JBJfHCiGqmh4MxbDBzDlS5XEV6FRSr7ak8Q5naRgUKXJ6hoN+DhkG7VXJsoa3hjYbfvHHUzjQ1xzMQ8L//wR9RXwAOwX5GMXVUO8tzl3NZtAVI7YcQ45qw53NW4FB4FiiNFocec6Z92zivMhUSUekXItlgp0b1EvhP/v7XAPiyBrrUQxZNivogk+DzT2Mwtlw4FCADOUn3T8Y19siovtYaGdXQLfeXcfacjwuWTviyAyVa6BPIUBQFtrb8Or21vQUV6rmdZXN6IcFlA//hP/gv4jkHs/ZMcaIcIt43ax2vQUKApzF5KHYNQVNct5yNFGeX0OH9e8LrUSk1UCGNc0kSLVo/uKRUSRuNr/9yWJtp8+MAQXGNtFGrwhjB1NZqUsFRAiOMNaEkMrKVynOZJPMEc1ENZKT/OsFroQsm7hC8fhtjOGHinEzooBbKhVpQB0kf+O//s98EAFy9eh17gxH3h6LNYlEI9Ai0WF2dw/KyXwdPnTyFZ577IgDgxNHTAICXfvgifvyTnwAA+t021smWzmiBnDazx0+dxvKqj+e9FYI/j7wgS5esgLGh5h0YUYnNR++/j7fOe2XvejrhzXu338Xpk15R+OiKP26710av60GNo+trrEIuswwra+vUBTnPR/Nznq7bn19gqrCSMUksRTSCV1Lysy8zhbhjBN2AWEtsmpqzV4110W4Clg3pY9UnUJbqnmvzjAI6a7M2a7M2a7M2a7M2a7M2a7P2C9I+2weQ3dMTxC5N1Lv4b+uiFIASUcnv8Ptj2iogCQ6HEZLDJxARjvAZNnI2JnqBJeIwqYKkc47pXYGm1jTNIfQl0FmbpjkkUpNSVz9xWkJ8AskJv4d/BwVTICJ9KbqX5/khsZr0nFIUcURCJqEfnYgoqFCCFfgcBCzR15S7C1FBeA+d/yf+8vm00JchI+xcRJesiz47IpMJbU9FGqAJBqfyEAIYhFjyPNL9rLM4RtmjL37xK3j0ES/yEsRPBoMBln/ZZ5Em4yFef/WHAIDhcISyDJkVwwbs+/ubbPp+r3Gcon53K4LeS6zoXghgqnKbInnpZ1OV2xR5DueR0vrS99iEQmmtjZlxG0VAQuZQWw1tqMhd1Zw9LWSkFgWBg1arC4tgbhsL7yEAZSJNkK/cxez0IQSQVV2BwHdxCf0RTnB23QrD3nmrq95zbzqdYjAc8uc4i4uYSdV1g4bmg3pa8XNgWbxCoqAKbyEircM4hykhh7pqMCXqxRsv/Tk2rtJ3LpIf6to6jPSZvs2dj7FElGOv/uqPN3EWlgrrYRxGhPI/8sjjAIDFxUUcWffF/evrx7Gz44Usrlz9CC0SfGiVcyyIkGWBzhO8/v41G88pnxznwCfHP49fuu5Z+/++lYTISBxG9MIj5RFAx7+HZ0o6ydyQgNrYpARBuLh+WiCqwCJFvwQCVugQxK4S6pRLaIIeLqTvU7BM13cY0jO1Q4rDlTawLs6hi5Q9b3d6qMdElawPmJXB834WEU6pctQVKQfaGkGiUcpIqTIAGhJKyAPbpszRLiN6FyjwRRnVzY02HH+oLGP0M6PXmomGJnGTrCwAei6LLGc1QW0kI1PSSZi7VuXYs4BOF2rn1Un5n+KTrxumnEY6JZxktBYiUkqVEJCMAIrog2YZgk3obZrPMi8zBiaM80qToc8Ar0g4DcrCAFRNc7myKHvem3B5fhFnj58BAHzxqSfwyGN+bV5Z80yGiZbI6bh7gz38+KcvAQB2dndgiRpqdMVKyVmuIBV5UId+dDiMkibjOESqft2Mz4i9O2h1gJKBIhrXFIvkHqUidNGCG5op0BFalM7FeM0xO5b+Hs8pI0SroXFsdIOK1i09mcISAyxX4LXXGs0Kr22iAquiZDEXBb8++y8XkISYSghUIR4TgmmzgkWhXJwnhOTxD2uTyoOclaOt1fxcLJJZ+2D/AJMqKMBWqNicXMBSOQicY/EVZwQW58ibcN0ztroLc+zfDTgWYxqOpgB5aHe6c7wG1cYxxbZT+r7Li4xFEwGLwZ6nzm9sbWFn09N+nQLTT7NCsv/yyqqPFZeWj2C55+el3uI8Dvb82jwYDyMPJpPsAxkM2rOsQKZadC8EP5OZyhh19dKqQYlSJKa//kfdOFiixFZ1jZzmF+vi/sEKCZWH9ViBZU8/pc0QwFmbtVmbtVmbtVmbtVmbtVmbtV+Q9pkIYMj8O0Q0UHLFwV21Ts4x91Ql9U9cI+ViUfTd2eRo/RCzbV5NOEjwR7GXUBPX7/cPIWjBViL8O/xMEZq7zzmtMcqyDKdO+Vqbvb093CKRjhR5WVhY4OOmgjehpUhkq9XiXXmKFqaoTXq+zKWXUTreGMN9E2tyBRdNb9+8hCXiPOfFQlL3d+99/b0qdP6aGtF/Y8dwLgqO8DUJgTYV8guhoKkAZDKZQEmf3ciygvnqGYmeaO04y2usRYvu5/LyIpTyn/vCc1/Eb//2vw8AeOihhxn5m0z8OWxtbaGgTNtodIBdqjN478J5CBHEAKbsn7ayOH+oVs9fkztUu5nWA6b3M47pe3dU+rkwppumif51ac1scux7He9uYY8UUTyE3HB9TSwMbcbe6uPDD3+G25uX/XmYAWd3+3MLKMqAXPqfK8vHMLfgx+Da+n2w1t8jZzNGICCjb5iF4+8MtTXpZXiHhxQZDH9Q/EblGhaFeOGXvD3Dxxc/xls/fzP2l4sHrEkWfjKe8Nir6ypBM4O9BDAaeURPCoW5zHP9TVOjpteVnUJt+765fH3EtScXp6vU6avoEEpnhxfQ9IPFQ6RL1BCoaazneYGlFZ8Zf/DcQwCAxx/5AsYTj448fO4pfPTxBQDAT376Horc1yLO97u4c8dnLTskCsTX/q/RnHMsQiCl5DXAumQuFRnPzU3ToMhnyN/n3RRn8JEggCI4f3i2TKhpklms8ZNZkrmneVUiQkrOJiiR5dfVoe8BC4Clc0pAjGVaUAWBLIu/h/o9KyyGY7+WH5AlgzZgeXclMxS5H9fHjx1Bt+1FtbTW2N/3mfsgEd/udDDf889qkZXYIVGcnf09NMGTVikWroGUkBmxOQK65zRqE0TZFFwdxBtMtGRxguvbrNDRp436wkpgREyj0fYmOiSAtrx6jJGAxhpmWkAg8d3jbmQk1QFxKU8QIx+P0e8O0WYxHEBGpI9N6oDEAMvXcJogcAEHF5hMNljwRAsqa0TCBnMwJrCtYlwVLqDSBlWwm9ICDc0T3YUFPPeMl+t//pvfxpNP+N9PnjzGnoV17YXAqqphq6vedBVz770LwItdFFlEeoMoiKkttDyMYCohIoPBOraY8O5q92Ci+YLG2NmIH/Uv2fiqSNestL4/yW4HVAAAIABJREFUfR7SOv1QK+e4ti6tPwQiOqeNgaSxokryUDYGW3d8nezH77+HvZ3rvp9GQx6/vcV5rK/5erWFRb8WLa4eRX/Bry3I2hAmIHlR/AhCILMkCOccI1PhdiuuxPXzRcaxMxC8UpRSXBFba8cobY/G/9rqMvsfX59UbNnhnIqexUrCku3LZGrgCMnrLfqYe2VxESUhnEKoWEtpIkvrYDCEyMjKAw6S4qew1rbKklkzxhpmDE0mB7CMBCOKWTWaGUML875PT993H9cAloXCzq5nL+wfDLBI81hvPgdI6MnJaKHB+ghSQiLGdhzbQyAWAMexF0BXPZri9nVfN101FVaXfM1s0WqhDqKazkGqEF9laOiZWj22hnu1z9wAhiDPJTCwEpIHgYDgoEYIIA+LUnpRjKZHHpiEwHRK1CnboMj95F7krcNBbaC5UMe18hwH+55SNdzbgspJaaksDwU+kuHQSI1LKXl3b74ATw38lV/xohBVVeH3fu/3AETq6AMPPICnn36av2972wdfH3/8Me7Qw1kUBcPeZ8+eZYGZ9Bx2d30h9LVr17C56eHjVEzk7kD97iBfyEih++C1P2fxkoe/8utopH/Yg0n336T9/yHbkG5wVTC5FAoFTXhAwwqNUmbImdYZEgKR0qQbx1Sg8XiCb37zKwCA/+gf/iOsr3oVyqYxEIKUO+l+5rlig9ludwWnz5wBAJRFiSF59GV53GwVRc4JjnQDm1I5Uzrwvbwr03uZ9kFohzwPk+SFV7O1hz4X3n/3cdP33L0BTBsHh/SzMRqv/ej7AIA//5P/A+9/6IuitRlABXW5+R7anRAM+et76MEncPz0WQDAL3/j7+HBM1/w11UDJlC4nY3F9nBxbkjYT+lGlNWvRLo5jpGOsjUrpF2/cRUAcOzEcWhScbvwdgPQcwYBnsQnkzEMqY3WVZWEqEHgIEdJCmv70wb98HpesOLWMTVEnzY8zdxx7G36xMHXHvUUkXG7xIXN4N01RSuICDXRpNtaixHRmkUGXL12GQDw0ks/AAA8+sizeOZZ34/TSYWtbX+NW1s3ceWKN7DuPrKCNgnJzPcX6Dok+41BRNGtQ1yoQ8rHjv/tkve++tqrAIBLly7h+ee/AQA4ceIUF9BfuvQh3nnnLQDAjRvX8Ryd61e/8i3M2ufTMk4uAkiFREKQISSXCkgheRMjZR7p0czWjoGHhOXo1jnBc54UGQQMv599a2keMKaBbWKSJ3zO62jEcwobCSM0uxezB5pWUIGKiAqa5oHV1aN44P4H/O/Li/joww8AAMOhT5IsLa7jwft98nZpYZHF3N6/eJW9YLMMWF31FMO1IyfQ7/sgrk/JEyssJiQKsb0zwJ2dIEwz5CSgVBnT1EwDkPAkr1sSDs76+eXWjSvItkl1sSjRX/BU9SxX0NRPTsU4ibcN1sEE7mcSD4rIpP3Emm3vet0Ch0Q8JD/jiDRe6+L9lxKZCjFD2Dg2cX5UMRHX6CguBLgoIEIq1NoArglCGwJT7Y93pOzhy1/8EgDgW89/A3Mk2FXVI+xu3OTPAoCTApKSvZ1WH/NE8xcoYGp/j5QUrJJttAGoPCPM0yExB3jRsNCkjBTQuyIrfsWlqmXJ/GijWkq6hQTfARfVH6MujOX1LI2BAcff45xjlVLdaDCWQUnwaTXEDvXRxY8+xM2bFwEAd25dYRp1u9PCyooP8heW/Fr0wGPP4smnnwPgkxAh2a6k4YSsseC5IXciLg8h9rc+AeDPuGEKqBAC0oZkjUMT1Jv0FA0pFE9IuMnBl+cAQKeVI4g6KiHRpcT8fG8ebUoEOKWiOBP9zPMeJA0QLaL3prWWz1VkJZJCtMSH0/8QUvHYbSaGkwWmsTyfqTxDi0QKtXG4du0KAOASrbsnTh9Hf82Lw1nXYDL1/b+1dRtLy/71bn8eWRBPpqSTUNHbVyhw/GRhIIInsEr2JsJCOt4Z+nOeDnD90nkAwMFoAHOfTwYfO36Mr/razeu4dMmf6/beDoJt/T/6T/9j3KvNKKCzNmuzNmuzNmuzNmuzNmuzNmu/IO0zEcAspYCGHTIEe/ykUvdeKph2uBJgGXkXs/khOS2FREkeeMPhGN3uEh0v46yItS4WpIqI/CjKcG5tbWKdRD6KPGPqnHOO5aqVVLiXwEwqjBF+z/Oc0Zhnn32WKaA3b/rMy/e+9z22EOj1eizy8uGHH2Jnx6OSrVYLS0v+WhYXF9HtErJJWQylooz+zs4OfvaznwEAXnzxRRZ7SVGku4VA/GsWJVHMzp1cwc9f9qjN3OI6Tjz2TQDA1GU4xCn5jPa3RQD/tpTR9HtMKBQmnpLWGmT9AqUMcpIsNvQfAIymIxbmMARzK5WhIKQGqHEw8P6G9z/4AH7jN3/Dv6cQ2B36LK4UimlBb7/l7UDOnLkfzzzzLACgVWYwgrKJcoSMioY7rQ5nYerJFBP6/nDvUzqmMYYpm6kYUHjf3S1F5lKRmJQinPpg3stWIv15L0uIz0IAsyDlzVwPiZMnfJb9y1/+Lk6f8RSdvb07OBj5LNf+aA+jqUe+n33GZxZ/7df+PiwBtx++fx4rRNXa2zpAf9XbL/QXVyP5yCElkdO5JZLYEZRCQuIBYKFEoBYNUVOm9+WXXwQAtNodLC95xH15bQ1DomM4F1HWZjo99FSELOKAMpV5maNpyI+wVhge+HF19fKH+M4XngQAPLS+Aj1HGWmRYTLyNLY7xqNx+4OGv08Yh/HU/300mSDABw+ePYVHH/TS5y+98mNsb/mxuX/gqU7//A9/H78p/j0AwMbGFbz40r8CAMz1urhD6IZ1ii1PQrLbwRLFyfdzHHYp6ieSMSaT/vWf27yzhdfe8M/Iiy/9BV5/09Nq/+v/6r/B7Q2PRP6ff/BPcZGyjLdv38YPX/kBgBkC+Hm2MNc4Z5mOaV1EGJSKgmlSxvkoU7HMIizIQkimcImECmedZfQrU5KzyR5dimgIAFihYJroK5eTiIpSipkbUgjIIBoCYIHoXUp4SlttDZwK7CKJASFyB4Mh7j91BgCwuLiCEyf8OlwRm+PEiZM494CnTx87soLx2D/Dl69dx+0Nz7Kp6xGWKUN/7MRx9istEkuGcLw7e3t4/wOPsrz97pvY3/XPp5IKE56nLMJzFeYRpXJ0uvPUdxIXP/yI/u7w6DNebK4138O0obm3sShzEjIJ6F0joAPTQcWZEi4pA+H/gZR6cKh5kmOYRA2zK5SUkdbpL8J/d4I8KCojMc5A0DqdZVEQxsEyImSd815o4QQBQDqWxte15jHW67SwtOwZSrqZ4s6mj6vG1RT7e56tMSJ2Rq+/iF6/R31gsULj5OiRNWzeJrqwrZnCLAQ45gt+1WVegoagR+/4AlTSX+m8mJCWmQ6NRGwnIqYOcT2VIqKgntYZvQ6pk3D4WUloqUnsYG2IX6O4igoxNAR6HY9Ynz59Bv25FvVpB7skoNQ0Y7RIxKhNwipSOlQTYjOZGjWt49PGYa6/RN+nUNAYhFI8biKqLOEo5m6EYjZeDqCk77MWAD0Dw/E+DoZ+3bx40aNn2miO3fIsYx9PIRWKgnwby5wZQ8JpRsuCB2EmHccqptGoaZ2GEFhe8mjykdVVZMQKPBgMWOgJZYyjwlw5rSqMKUZotGZErlu0UFAcOqlrnL/kn+GKhOScrfDlLzwXugtXL3s2wu0bV9Anamhvvo/FeR+LhPklzwWkDDZmWbSPkJGt561DDF8X8sP43OBgDxc+fA8AcOPaDWze8fuOc+cegdW+P1780Sv4yRuewbN1ZwctKnP6NATwsymgSWAaNmNKyGjAmdDbaJfoXxcRSo6QZvy7gmBFnLqKkHCWF0wpcWkkyE2jS7QnsbLqjdXpPPJEoZEhfvfJwDcNsp1zHKxprfEmBTvHjx/Hd7/7XQBg37AzZ87wotvpdHij1+/3D23SQi2i1pqvPdSfpSqgq6urvEEcDAb40Y9+dM/zS+uhAE+v0aQwdPzB53H1XT8gzr/8R+gve8pj99jjbOjpb4VMjoB79Gvom/j73XuWz9r4fdom8u6+D4pUUgb1Igdi50GbGjIE+E5yvYNSEoWjDXRQW2t0pLhYgRYZ2b/w7e9gecVztWtdw1FAsrd3gP1dH+S/c95D6NdvbuD4CZ9AWF5awqVLPlDoddfh2n7BcbXl6+72e2yQnNI2042ZYTVJdc/awHu1e/n6hWOlyp73oo+mSY90HKeb0rAgpueslGI+eniY8zzDw495752HHn42miJb4z1nAGg9wZs//zEAYHHR9/lzT30b+5WfiK5/fBU//P4fAgAmgz188fnf8u9dWkVNz6qCZKXQcA7+T0GJzMIShVlYBablCIOcJtOrVz/CeOATNOtrnu4yGAywu+PpmPW0gmlRgKxyzLX8XNM0mhUEjdGoyYx9MvHjJMsK9EmpbvPqNq5pn0D40v1reGieaom68xB9H0gOb9/ElTt+XP3sDf/dx4+uoU/GzEOrYWnc6UnFVKTFhXmcfchvjn/02huod8O49+fxVz/8S+zu+nrMXreN7S0fCC8tHsP+wC9am5u3URC9ZDzyAWpZlIkfWqQjiURbMKXtAz6QAoB33/PPxfe//wP85Cev+z49GOCddzwV+H/93/5nXL3maxFv3bqEpvJjsK4aXLp8EbP2+Taus3eSfd5sklpJ1xopC66tEyL6d4UdhUg3GkZz8UmagBECUESN8hvBMMcEc/IJaqJcCWfQJgpfURZx7nEOoHlbQWJ+wQdMrQ7VYk9GEIEaLQVGNNYvXbmKOaq/P37kJM6d88mYtXVSClxeZXpbWSqQgCDWjx3FmDYVTT1lVcBWq+A610BdlFBorXoK3cnTEsdJpbJTlnjphz8AAGztDCCymOQ2d/mcSqXQmfPzx8LyMs5f8M/U+Xd+jh75iJ3pPBHYitC15M/aUIInwOqW1nIVDJSMypPuruUkvH6IuCjCxjJLfABFVHUVMvL/hUCLEqoVQlBs4EKdsoyUvKpuICh532112Ysv0HGNrVEUPkbrdBWWV/xm/ZHHH8E83ZiDwYi97Kpmgp09v2HY2fbrSLc7wZFj/nN5YUBl51heWYYzQ7powXWtsBaaNlCc7IPmZyQtCXcyAgRSyBgKOXB/hD6ycNz/h0j0TsTSTBe70SVfxGu6tYfiuUAjNdbclagNCVzDfRoGysJcH48/6ROyjz35BH+u2huiqWh8Y8J1hJkKlOUcZdc/F7Wp8PY7HnAY7A7x6FOetn90/T50yH/ZKoemppg6xBjKcpwtCoc8JCcAtEOyJstwQOvB7vZN3Nrwa/Pm1j6fj6R11wkHlQfqqIImynczrTjesbaGIy9mrmluanQppjK6QkO1fO1WFyeO+7j3xIlTvJbv7+ygoRrjgiix8915DMYeaKkGY0xpQ+wmNQRdb5mXmKN4va6mrND93vn3/XkYg20ChZbXVzCktbcxU2zc8fV5C5vLyITv09UVArfgoEICTQqmhgKI8VVjuMZVCMlm8Qe7PrZ49+ev4YMLPt6/dXsDG5u+n69efJ8BpLfOv487B6RY3oyxd/dkcVebUUBnbdZmbdZmbdZmbdZmbdZmbdZ+Qdpn+wCmHmcqeNeIeyKAUkR9UCESX5iQ2ULMvAgbi16NMTA2qAIV0eNOCIZrAzwrRYZpyH5UFbpEE0hRGABJluWTsFVK20vVPOu6xrVr1wAAb7zxBn7rtzx6cfLkSf6OIP7RNA1/Z7vdPtRP4femadAi5CEgfdZaRhSrqmIU8Tvf+Q4ODnyW/+23374n2pOeP1MOFk7iyFNeBfH8H/0B/uT7LwMAvvUbp7BEfWOtTSh1oZj33lmBz6KDMtD7t6CA3o18/fIv/TqASHOdTMacXaqqGloHQZ4eWqXPoORZyaJDeVLYXbZa/PsiUUu+9cLz4HyoA/ZICMRah7U1n909ftwrZZ05fT8r6VVVg7P3ewrR8vI8clKmgnYoC58NalQFrYN/jW/p/fYelZ9UuQUOU47v7pvUnw+InpFlWTJ6l1KB7zU20r+nxzPG8O91XR9SJpWkihYynBmcp8cAgIxiAaUqIIXvg1yu4BzRpM5f8GOtGu1B2JBlH2JElJT9zQ28++M/BwAsrZ3EkBgbiwtLqLU/hqGc9URbbG94FKmeDnDmtM/w94oVTnc7WaEIdInpAFvXX6fv9Pfk7H2nsTjvx8Hu/gA1zTVlXuLBk95TLy8k+/mVZYsVLgNVrixa0ESleOLxhzDZ8qjw40cX0Fv2x7Dzy8ha/tna2tzA/k0/Z/zOAz7bed9SH27Tj7vf397BFgk6GFhG5/YPxrh05bLvfyEgaBoOYjWj4QE++NjTMB+8/wz3wbSy0NZnwD/44E0sz/uM+ny/Sz+P4voNT7vJW0C35+/biy++iM1Njyg+8cSTWFryGdNXXvkR9ghJ/fiyR/fefecCtu74PqrrBgvzC3SMv8T+wPeHgEY98eNqMtFRyXDWPr8W5hrrWRKAF4YJa2yRK2SkRCBlHj36jGXIKrr1CdgIY/A87KzmCd9BoU3oV5ZnjCgGr6p6NGYxl0JFRo7K8rjeuKhkYoxBnzLtRUDVnGHmR64UVOb/Mdi7g/M/90j07vYOlr/xdQDA0SOeij3fbvE6XlUa4UtUnqNDjCFX5jDEalBZjlIG8TGaKzWYUqhUhtOnj9J5PoOtbf/svPTKT1FVhExkGSNkLGLjBATNJSurx7Cy5p+zt9/5OV5/zc9XWXsRJ8J8lAMk0ndIrokfJ5fQEm0CVkkcQq5YcAThzeBmnYsUXKc4dhCIn7PW4dFH/PrXIWpsLhz2DjyqsLO1hTFN4L1eF2cf8qJfc3NLrKC8SSIlewf7LBh44tgpPPm0R67OnDmJ+TnPnDDOeLoygEoDTpCATGDs7O+h3/NMKpHtYW/PrylFLrG+5pHUrMwYkc6Ew2TizzUIrlihGaGVFlGR2kZxJCei4q1IOiQoax6KdRKGm5fBDS9HlpZ1joVF0rU5+u9GL8rD79UJYuj4GWA/ujzDAqGqWZkjaMTYxUWQWC2MM9Darw1jQr50NUFFiua3Ni/jrZ97ltntWxuYkFDLF54WWCJRkyzvsbJnFeICJVh5v56M0aN4aGFhDhmhWLkCdu/4NW/r9kVsXPdrShBHUkphfpEEy+bmkQdhF5WhbBEqnwkWGWuVJQQpyU5G/riTyRCdkvo/j3PKYreN4+se4Tu6vorNG36sTPYOUFP81CPkcK0/h4ZQv2qwCx0QQNMwAqgA5IE6DzAyHvY8GxubeN14JHXt2DpWV+apn3Ls73oE+8rliyiE/9z6il+j+90OBkRJraZTZk7cubOB6zc8wtfohkWpjHXYuuPX2w/e9+jjhxfexwYJTk6qKRyJQbbaH7KH5WA8gQoepkXx1yJ8fyMVUIHE8BNgdTmZBL1Sygid2ziRp8dIzckDxzfLW/zwKiXR6frgyjkVjVbpwE7XMMT9zfMceZBLSmkt4csASHlvFdA0IA/1XFmW8YP65ptv4vnnnwcAVv6s65ppEMYYpt8VRcEbPSDWh/V6Pd74he9rmuaQ6mj43NzcHL73ve8BAG7dusX1h3me8/vTjUb4fSwtls4944/35mW8+YGv0ZEvv4K/++1foj79f6cIevdG729DCf20zz34wIMAIk3D3wv/t/F4jMmElBaPncCjZIq9vn4Uw33/oH74vudbX7x0Eftk4vncc1/AL73glQrbvZJluA/299Hp9Pi7p0QFCvft0uXL6JN6otZT/NXL/wIAcOfOFa4dUFBol/4YGhJTurfffuG3P+VaieKsFAdAqa1Heu/upUQrhGDqaFmWfK6pHYgx5hMJj/R46XFVoqaVXjsA5Mz1J9oIHEshe06L4F/ZLFd4CwYAuHPjQwDAGy//C1RE1xiObuHsfV6N789+9jZTQeTR02iUXwB68/O4SrVkoU5I1wYbRC+8/vF5/Lu/8R8AAL72lb8LY8IUJTDa8fWH0+E2Nq57KsQL3/bKvStLC+hRbYQ1AkeO+iDLwWFt3W/+y0Jhl9R7m6qCDosc1Zi+/trrLPO+vjSHdumvtTO/jNaKTxyUy0vY3PAT9s6dDbgdn7h5gmotjuzsoqFAxsLibe3ntrxVIKNA/dLly7h16yZdleJkWKCnWmsxnYQ+3YWwvu9ef/1VqIKk83c20SF+//DAn894NMA/+8P/CwDQ7hVYpgTTW2+/hUuXLgEAjp84BSn98W5cuwFIOt7YBw+jQYPhINTaCkyn/rrarR4QKNrGsOJq+gzP2ufXgp27g+M1Vqmc54YsK1g92QnFiTaIBsGuOlqsSF7TDQQH5845Ngs3xnDdjbUSisZeTeNAa82BXbvV4g2iT8JGqqFwof7HQamwUaNrUjE5mGUCrS5tRvIMu/s+8NHNGIOnvPpdSdcn8xKcs7OW6xkNoi2GzHIoQ30mI7XVUrBnpIE2fo51ukZJNPRTZ07i+W98DQBw9eY23n3/PH1QJ7W3tGGwzm+aAcwtr+CZZ32t0O7+BNcu+prZvOwhz78FADh+dAVhem5CeZCMtX5ezjPco8SSwNy1VoefJs7fXIsGxCSac4c2PMwMdUCP6sbCBmuh28LucET9LFAo//ennnkGTz3nSwWKrIPJyM9/t2/6Os6Ll97jMfHUM8/iicd9/NTqdjEZ+OON6wo1URfLMkef1ukdSarodgwNP4fWwxq7u37OrqcTNESXs/WU522pAGuoLp8BBMQaO2tif2WSOZ7+z2HtlZ9YQyUfyPdh3HO7qNbfWJ4LjbPJXJi8m+PNpKQnLe9JS5Vskp4PehhaM4VSNMA4jBUDTkjUqDA98Jufq5f9hmE03UNDIgu3dq/hYN8nK7WZ4tIln2AcDgcoKaHZ6y9iruOTqD0qhTBVhavXfB3cwfYBHrj/PgDA1772VSyt+9+dBmxN68juHQii6X79+a/6Y80t4DhReo8cWeF5wid8Avij0ZDy+2TaIM99L3z0sU8Mf/zRR0zn7nXa6NFGaW2pjwVKgHZkhim5BOzfvuWzOgBU7dcwNRnA7fsNpRkcABTPdfIMFY3/XEk2vshVhm6HqOw0TyjpMKHj7WxvQtE4zZRCNfXP3O7WHhqiZLraJ4Y/Ot/CpSs+7rl5/Sb2SLdie3cbW9v+GWoVOVoUExkIDCju3SVqdF1NQPtiVI1BTXNNkedsxSSUgw4qqwowiRbFvdqMAjprszZrszZrszZrszZrszZrs/YL0j4TAQyeUl5CINAwo52oFJLhfu9B5D+nZNx1sv9QUqBunWb1m05nDjB+p9vKaigzooP04ChTMKXCzYPtTSwseMhV5R3UIRPiYtGzP69PXsunKTEG6ohzjhE5IQQre4bPBUQH8EheKsARUBsgolu9Xo+zscF7TmvNr9V1fcjoO5jQP/nkk9jY2ODzY9pgTKJyc06j0/f98fjTX8JHN/zn3vzpj3B82b/+5S99mY0y0z66V/usTP5fhwh+2vvSf/+Tf/K/0If9D6WiKbEUCs8//00AwDNP/xoeIpXEbrfPxbDHj3qBl/X1Jbz1tqcEPfbYQ1he9kiHFY7HVV1plEQjPTjYw+XLlwGAPRs3N7fx0IMPA/BiAjcJkbl58xJ6HX8MW1sIokXmmUJNSlDi20lG9R5IXkrDPCSUlLQwDlLBGCEEj0EpJaN3WutDx4t96z7xe6oYaq099D2pUExAAUK23AkBEfhcxnAmXkkLYmhB6DE2r/vs4sdv/QQAsPfx68jmfH+N2go9ol2szS9iG/65ff/aB2gISW1uOQyJAqooS51rgQ9+7ilSF372Mko6j6ee+iL6fY/kTQ40bl32qOOtq+chrJ8zfpNUXwUsWqSO65zAPPlMCeFYpc85h/k+eWUag4qyya/+1H/3n/7f/xKL5A/2xot/gQdPewpXp91Ha+gR53yugy3Kdl++eh23Rn5MvEjiQr1eB82Cv9ZJU0Pt+TGxIASeOO6vpdfvoUWF92VZ4E82PToQ5thOXiJoa4ynDsL677h98wYcqdXaaYXFtr/eZuwRwCvX38W7F3wfjaZAv0NqcMqy+MY7755n4YkyK9DQ3DCg6/C0pJghn5KiYjVpkAeqnhDI8khhtpHBPGufU4vaYCLxPEto6DJ1gXSMjCgoVuYOU4mUkUGjsgw5KfM5YyEoiy6sBihbP5pWUKzYHMXJgthZu10wPcmYJppKS8UiKnAJelgHnzQJEUpNoFgUzJga4YHIRAkVBKPowFlRsnietFVkACWIqFeKDN8TUetgteWshQgehFZjMvHn1u228DCtRU8//Siu3PQU6929Ic/3irLvzlmYJmTfBY4c9eUjjz75BG5teIGIn73xOnKi0am/802sr5I6KJ1HrR2XvkjpASvqYCQAHwuFOJMoNoawS0TxEgkLKyODg+3dRRQSklLilVc8PTAwXpZ6xaHSka99zbNsvvrlZ3DkhEd+jDbQdY8+R6c53cPOvp8r+90eC1VZbXkMZlJhSNzXaVWhJuPwm8QYgVNYPbLMFxtQolo3OCBl5tF0zMKEeSFYyTIPwkdCsuebFCLGK86yyI5niNHLCUWf/XWjqC68Zk5ce4OhOKxlU3JrDVPxYgwqIv3aGTCzBlGQxzkX1fLhmIYZVUAtbBBLSRQylbCA9WusMBq3tjxD6sev/ql/73gTtuWP0eQZuvP+uxfWlpEFpcvqDnb2fdzY2u5gmUzkgzDM9Ysf4fWXfwoAuHb1Op77skd/H3/4AcgzhABKjYZQMV2P/XkB+Hd+07Pa+r1FZCXdn9TTWNdoqORCKcn+1rqpoKhPXyXq9M/eeguGUOOlhT7WVjx7a3lpHi2aJyYH29i6fhkAcOf6JThilF3p+WtpTQaoiNacVyO0qX9dkUETm7BwgCC1tlaeJ0KcNFcZi4qEcjA00DQvWevQkABNUW6gmXpUb7ztx7SwBh9f8QjsjY1tTIgO6oxBRddd5Dngi6gfAAAgAElEQVQyLgEqWOgwI5aAkAVAn2usZdrqtK6YtaEyhUyEuRcQxWdu8f66GkD/Ux7aODh8at1TMH62UdpeinTjGJpkuXJZlFhf81z+4c5tvH/BB5hN1gM6fhOzR6asD5w6hlWafDY2t1mpUMB96gbv7tqp9H1ZlrFZ+7lz59jmYW5uDisrnlcc1HXKsmR65921XGk9V/i9KAr+zkDfC58H/KYw/HtxcZGD/RdeeAHvvuvl4K9du/YJJc1Ur7iwEoKir7PnTuGBC56m9vY77+Kll35Ix17AQw96fv+9aiI/z9ZpBTVU4u7nOZrK38NvfuNb+N3f/S8BAEePHOOJFBAQhf+91yW1p5PH8fQzvrag1+shD9Q5ADYn5bJWgX2SEd8b7qCqPfS/u090uWqC21Tbsby+jl/79X8IANjZuQSV+fulbAFhwyMyZeuS0FI6cfh3aKkSZ7oJ+wTNJLVSST6fqoBm/w97b9Ys23Feia3M3FPNdeZzz7nzAFwAlwBIAZxFUhDF7hbbbkkRCtsvjrA7wm+KsCMc4eEf+MV+9oMdckS7Wx3RVrTbTUmUmqQogg0OIObp4s7zmU/NVXvK9MP35bfrgiAo2Wq+sPIFB+fW2ZU75/zW+tYKAlGXnVcEnTeTn88j/GXFWospL6a29IeYANMhtc3eg7tYZ/PdWsNgh9U1p4M+Xn2FLAn279OlZflcB66gfh0dptgZUn92l1cxyZiCphOUfDrJXYGQFQITnhd3r36Aa6+SsXgzinH1Gl2mXn39FXzjJbJDGIyGSDRd+tY6QDGmZ/g+SeI66kwLIQquXzUh1Ms8y+WSldkSAa9NNz+kjdNBod0hOsna2S2sMx2qs7qOnT06wO32jnDrOq1RO/vHSDk48b2H1EYuMKgLjbeODucQmFqErVVqU5OEUD5oZAJE3Lc+UJRnOZpNqv/uzhFGY+qrycwKTXSc5mhGND6GUzoUxXmMOv/7wWiAKdM6g0Jj5QxdiIvIondIfdTPrOQi+LXBlhU1KU8tFGecKCgxsNbaimw50Zo+eZNZlL//ImbcqsrJByAnVmdLuUgoYyGWK2UhB+rSy/yroFIqtEQjAoDIGBim/tkyQDqh+TcY9BHHPNf48gPnEMR+vgdiCWGtrfIFUR24jTIYD+gzXnlQOY1Q6KwaSUDfsbZ2Aqd5b75w/jSeefJJfgbP6zQTVU8UFir0l8jKwFwbLSrSSim5TEmwI7DIfWpCANiCc9OnpeTFnTlzGhus5jmb5ZgyPVp2KhdKUnXhKjriyfPncekpCja+/P1X8MOXf0B1iuv40heIXrrUpvkezeX4G1M9uwTmtBfmv7S6JBae6ol5GqOWw6F1lXIm5R5X+2ajTe+4vUntvL68AsW5ea3GKr70m3QBPHn6IrTinGUHRDwOVrldoisaPc6zXltbh2FZfm0C4bammGLGl//xaIDDI6Yu3r3D761xYpue16w3UefAwuryEhyPXTWnLNmoh0DggxrVpW9eiV5On07NNR43oP+80GOr27UPo6g5aycLQIsCvRHT71CH8j0+NzJNU7m5a1S6FgXwuH2E9J2Dtn1+Nq+rkYLj5+XZFNN8xD/P0OM1vjfu4913/gYAcPMGBWfDqEBjucZv2UTAfRVECjrgdtQOMe+PxuaY5HRO6u2z8fx77+DRfeqXXv8ID+5QKsHdW+/g6SfP8PNqiHgfrscRFIgOurlJqRdGGWnxwAQSdLFFiDD3/G8Lx8F23Sgx6hHt94jz8GfTMWocmKrFCRoNr/9hsPuQLln9vT5uvkepIeVwhJKD0TfefZP64vgQkQd58lQsbhI4YM62psbroooCsW7y6TFF4WB4jGW5w4zzgNNZCst7abNew5hVcfd2ea1NM7kgtuJINBacjdDkcZUXnrxJVymvZaL4ImjzrHI+UBUFvkCVameCKpjnUKm5/qKyoIAuyqIsyqIsyqIsyqIsyqIsyqL8mpRfIgLjIyEVovFYdEk9blat5iMu/LcFR8gDYypqqLKiwAilURqKure6qwhCQhbuHfeQ9znpk+lGS0tLGI0o4jGPMqo5n8J57zxjzM8hf/OiHKdPn8Zv/RaJpVy5cgVbW4QwzdM3/d8VRSFKndZadDod+Q5f5qmhH4fUKKXQYopZrVYTdDEMQ0F42u12FWmaQzArRsEcjmoDEfEIWzU8/yLB8/ce7uOQxS7++nt/g+Ulokh6j7zHEvM/UuYCi3/v5Z/+F/8VgEqYJs+sGAN/9atfQ7POKEVmRXV2vk7eP87ZUvxekqQ2l6hdjYkwjjA99DSTAnlKv793l6KNpTP44b9/DQDwox+/hZUuRWCPe9cRBOw1VCiUuaeFODRbFEH72lcIhdZaV1TJuQjh/M8fRaY/+v/z4ySKIkGC8zwXFdB5pHEeddcfM+bnBYrIYHbea6hCwx/doqhYnbk7tXYdD2+TutWN997FPUbfo8Tiw+tvc1sHODygpOyIw56nL57ElLuq3D9AO6B6HE73UAZEt4yDAIVXNHOl+Cneu0bCL+/97BWonOZCZ72DGQun/MV3/hRPXSDhoNWghUdHhNSp/BCRY5qu933KMlhWhdMKQu1Wc3zwIDCisOecw3hINO/VDkW/rzmHhJ+RlznusxiTNYFE2X/26k/x6AGhgZO0wClGQDocNc6gEPKXBKiEJcpcoWR/Hp0GMNzPSb2OdEafiTnCmdkCE69kmGbImFY2mWRo899lxxMUHn1OPdKT4VKDkZKlLqxhBOXJp9CcEtr9k6PXcI19OA+zHF4iKJNlTIkXqwkLhAGNpThOZOyOhhMxoW/VG5jNHlfHXZT/8MX4lISyMqJWzglKFEAh9CbjYYiA997pbII8955XTAtVCqU3qQwdUY0AJFEsdGGUGXIWOspnFgGrCAeRF1axwqd0DnCe+WOq6DSUFmdB55wkhdQYrc/DDAHPv+WlZXzhcyQi8fkvfBFPsvLk8lIbTRZmGAyoPsdHRyLEYcsMcUJzsl5PKsVBFVf7h5qn5bPgQxCKn16kDRTP4TxNkc0I0Qg00Oa1on4cyzrm298BsMIGKkXYpZa0cOESiaHcvLWPt9+gfef1n72KU9uU+tF5ht5PKYi4zDzbZ05AFbAVffdxTs/PM3zcnPn4Y/sIMLdnODxxkfa0Z54iZs2p7W00anRWWd3axJkzLOCGQARmtNJwvA4kLB7SaNTRZbEgrUPhpeowgeHBGTuI8s90mmLG4yrldu71BnjdvEr11AqGhUyCoMSAzbtLVyBiyn8Q1xB5RoVvf6eEIqfnRM3o3xltU3PeidZBe/Ejr8JZFnIOCbWGYdEZoyDokQmM0ImLOQ9NkXLJCuR8Bk6zmSh72jlktrRWGE+2dNi5SwyTIqd3HU8myFgAZTYZoc/KmMNxH/0htdl4doy7d4n+P53Q2a/VbkOFLKySD2C9efpkQoqfAOphF5HhlIXpFPuPaG97dJP++/DeDnzTnj65iiSiefbO6z/E+jKb05+9BM1+so16AH5dWJ4fhbXiChBFNYSsaIqwRGp8f1lYnnPKlphOvNE7/d1SdxWxH/NFjgELpBwdHGHAqtX7D/dwuMPnOwtJrRkP6Dx3J79F50UAtSSCZpVSZzQCnv3GVikvsSllPnusLAwVrPdZVEZUXYvSyTMiFUDxOCiY4RZDY22J5og2IdLcC7jECPkMdnDYxz4LHtmCBGno83wHUU68kIMogNPe61hDaY9govLEtHP7xC8oCwRwURZlURZlURZlURZlURZlURbl16R8MgKofETPwkdNlNYofZKsMpKDFihAifyohuafffQg1EoiLEqVcCxs4JxDvkuc53K6j7MtjoSkBT73j34XABDVKELx1k9+iJUVQolqUSg8fQASWZzHV+Zld+cRQJ+s/vzzz+OFF14AQP5wXoBjHsHxSMLh4aEIw8RxLIhdYIwgNXmeS9QhTVNB+PzvlpaWxJfIGCO5gUCVJ/jgwQMci3+dnePy+/esuO1OVZEmW1pc4tyIFz/zEN//d98BANy7cQvf+Uv6+Q//0z+ktjClSEMDgRgP5bmD5Xy7RAOG2zdTGpa/xwNzap5bPIdqqTmKvftIRPLb3/4L+p6Mxka/P8IXOAfi69/4BsYs9hPXIkmi/vleJaECHxk67g1Qb7Tky32eR6PeFBuIsjjAbEyfP9xnuft0LNL3k9ExttcoUlKrjVGrUV9olMhT76OnMJ3Q93xc3t9Hkb3HkfG5vIs5qwiA0F+f++Wce8w+YmmJxnoQBOLFM5vNZEz6/16/fl1+Xl9ffywndf6zlR+RxZ23KFG826LxOC6muH2XpKHT0QS9YxrT3eUYaU7jvkgdtKbnRTWaK2NrcczyzUEJpCmN3d10goDn8nD/Fnoc0dPGYTCjiN2De5TrapIhOjXqNxunknvy1vsv41vfIjGA37z8PG5eo9yGBkaIDfWFj/yWNoO1XjTHoWRZ8DCKEHBIPTAGKUclgyDA3du3+L3Y+++pi2iySMxhv4d8NuL2vYGU2//Ro0eS03Q8GOFMg/pof0rsAA0FzdFtPTOwHgkJggrZV04sbIqyQFl6xgL1W73WgOPk+EY/RYsRvs3c4LSmv1s/1qgdU3/uz2je9NZCnLrIuTjxMnr8rp0gxroiFsBLW58D9ll0ZilHxJHgBieL375zXRCIODSIGOGp12o07wDs7x2hzj5dzUYDaVqtY4vyqyki1JIXKBghLrMcBfedVgrGMUKVZ9A1GpMtYykfC0ASVn5/Ke/XSgfISxrrmM7EE7B0QDbjORVGqPOeXGcJ9kmWVescNAJG9bTCY/7AFUJSQDEiFPk0IA20m/S8zzz/PP7hN/4RAOCpT30KbW/xkuWCLBwcUn7Qzu4hCl7ntDFot+izpV1GvUbPK1GtFUpXPgqegWKUQtKguV9rNKD5s9PxCMeHpEEQ1+pYWybxiYcP9zAyzEby+dfOibCKgkLm/QFViWaXnr115gxu3SCEZ+/+Q1z/kHKWzp0hgSjvfyftxVojrsghX6MqcYfSzkGDjHAqrcW6R2nn0+OgtH5sR5YcIwXcvE3MjqNjWps31zfw4me/CAC48MxzMIbbtMhhfI6U8mfDau1yMDCMIBPSSt9lIgXjEWJVQ73Oa7XLUfJLKs4j64+PMb5KZ8JZnkJzblWjGUkeYRAa6JDq3+slaHdq3AX+xOAgea/WibGtNtUeDKsq+xNbiliez/tK01zQPROFCDn3MUpCRCyiktQi2QudpTwuoNKOKPJSLFPSyRApzyEThI9ZpTjJwbZ4743v0Xvx+O4dH2HE+aaRURixPcd4OkHq6HmTdILxmMZjreFZUBoFD5rhdIIs8xBaBBOwLVbioEA/9/eGONijsT7q0X4WxUCNWWvtRoSE4bGbV99CyQjlpz/zGTh+x72dO8h43Umz6vzrvZy1cnI3UIFCxKiZdQqZ5NNlOO4PuJ14v9tcg+J5n5cWu3tsCTVLMWAbhcP9Q0w5Xz6ayxHVPA7yooRjlLl0BWoegYVCyfMlMAaa6+9ChfHscZQ/DAJE4P3dAJrbo9msYZWF5yLtUPIZt+Amb9QSdHhtW1pegal5lkITtSatKbfv3cfkDerPwXCMmNkLXrdAlVpyl4PAoGCrGqcVAkYzAw2M2ONxPJ2KvsAvKp94AQwqOS35nVJKDjVKAzE3Vv/eNYQFdUQjiRGwdE0QchKq1sjGNKnT4REePSLRhPEkQ6PBZtvFGCEvbONhgftniabWOUfG0O2lDWimmIUqQuEV8Wwx53Ok5gxzqkvKPD3Pm4KfO3dOxF6iKBLqXDa3mfX7VOerV6+i4IPO2tqaCEgYpYTmmhe5XPCm06lcDDfZsLbVaskCFcex+AQWRYGdHWqPpaUlvMhUzu9///ty8J/L+UdFYagU+7TTsFz/06e2sNShSXvz9j28/y5tMn/z/e8DAL7621+pLpG2gt6LLEfGPm+1JIbjzTErnSz0iikf5hfayUv1HpcABfAmK0f6JHKtA8zSZ+hPjJNoQTH3d845Sez2d87prIDhA9CDBw9EiGBjfU2oFEYHiJlqoA3Q6tLvL16myXZv9xC1Fk22zopDZOldO51ldFp8GSmnIiYyywKhMcxfpOSV56iZ85RM//8AXcL8Rd+Pq+PjY/k5yzK5sJVliWX2cTt79qzQkzudjgQUPM335MmTP3cp/GiZp4YWRQGb0UK/v0f1meUTHO2QumVS0+j1aDOZDuuoNTnpHDP0B/Qux2xGdP3RPvp1GudBMUODqZlHrRSlZU/GG7dx+yqtDe1uC8d80d86R2174vQyJmOq2+HeAVBwO6cpXv4RBS/ifB+djOuUTxD4cWgqqrbxsmrKQWufQG2F06NURQnNZikGnGDuu2r79Ckc8sZ3994umg0Wq4mVtJ1WBg/26ULcVgaRT+T3hu9KVd6KxohYh5oT6XEKyL1Zr7UYDVmshfs1mYVoa054V2uI2L/s+Uub+OZvkz/p1Zs38X/8q38FABjx+62uG+gVujD3e2NM/b1s2MP5F1i97cmn8Gf/y/9IbXB4gOKYN8Q2rUVba+sYFofcLiUy3shggE5Mh9gTG13xn+oPephOKnGrRfnVlEA8I40IOeW2xGTivexCjNmba9g/hGXVxXZo0Ej8Gs60svEUwzH93aSwoppb5iVyDgJaEwEs3tTpdrC6QutozOIJTim5UGg1p3KsDfR8YoHxaRFajLD95ubgkCQ0Ds9dfBJbZ0hF07oc+49IWbLIc/TZe+727Xv8fn1Z92u1mnjxaVQcvzoAsEiNMYEIbOTw5tMR6k06hzSaNUkpMdoCjg52G6srWFtjlcT6PQTH/rIHaQMvAhIaIOeLUp4XiLidzp87jVsnKUhz4/odfHiVgm4nTtB3PPnUFayzuXSoqxSaWTabo42F0Lqiefn90Qe6tHYSsLVOiYCFNnPCI3CVOqsy2GOvsUcPaE28c/M61pbpXX/jhc+ibPt9pTprOaUrGqlIASsJaKXZGAV7qYZ1A6Vibn8gYcGuer2BmPfYdpPX24TEigCgXQ9Rr9NZZrlZR8yXG4QGZeGVFK2kJIhoC2wVeICSCzEeEyWyculLp2NM2I/aG8SmeSn0zdLmmPHcyvIcAQf5Ot0mNlk4Z2lpHS70F00vBBZCaVaeVhAQocgzuRQBTkS1yrJAmwXAVMmXzKSLNKe2yWcFnGb/ulYTM4743Hk4QMp7VMAXhv5kigmnIAwmOQrnvewyUeocHU4x6POFd5jBeu88Dg41khCtGrVXHGrkfLnYPZphMO5xnadY26Iz9WhyjNJfWPgsaVCd+eCsBHgVAlHmNc4JBbvXG+DR3i73AX02TmLYnNp2Npkh53OUBVDwQM6Vxth6lUzI7SZU/kyoUPiUmDwTIEsphcyrumog4GfrRKM/ZLE8EWYK0GF76cA45Py8rRPb+OpvfhYAcPDgAO+8+iOq/4jT105vo86+prVaDMNU1GajixUWUMqcwdvvUyBmuHMga0nMQkqNZoKs8KqjledkYEK02Q1AK4vU96EusdRdxieVBQV0URZlURZlURZlURZlURZlURbl16T8rURg5v3ODICIoxWhLjEakFzseLwLd0BysYN0BM1J5WOOULhAA5YiktmkwIyj2tMiwKM9iugliUarRdGPtbUOLl0kW4OUpdRncQLHUUujdSWd6x63o7AS8VePUe0Aujm32xSR2djYEDqoUkqoeHt7eyI241GW27dvC9Vj7+AAJ04QqredbT+GDJ08SVQOY8xjnn8AoT6eZhrHcSWsM+cl2Gg08Ju/SVH+N998U+ok5Rfot2itkHPk6s79+wg4EX5pdUWiWN/9DiGAURziS1/6MgCgsA4FRxXKfIQJy9d2lzdQcptN8wzGex4pH0VVj1E//zblynNnqa7Ky3EbqIDerzc8QswUnEmaQnmTRzgoRiK9F850OhX8sdnu4to1ovIFOsTKOkWms8KJmEVZWGxvE4L2la9+HgDws7dnWNrgMeFS5GMWRghi8ecJVV1EWdSsRF566ftqfHk0cJ7eCTw+d7yv4+3btwVJ8uOx0+ng0iVKsA+CQKgjBwcHeMQiJB988IE8t9vtotuld/QWJi+99JJYmLz88suCJjvnBBGcF4Gx1uLypz4HACjYg8kYYPsMSZUfjXeRG0qm7u9P0W5RdO/46BjH7KMYJYScP3xUQp1nupQpkBTsw6hCmIDaphHPULM0b3sPMzS67M+4T/N+ZbuLRpOR/zzBeJcjWC7BvYdUj1un7uHFTfb2G5U4GNym72H6ZhJpiQiHWiMKfTS8oicZo+A4Urxz+AiH+3vcBtQuvcEU9+/T+zXbzUrKO7cyt47GU1jv4xVksJ4SLZLq1VqZFwW8krfWWryelKrQD6uBpU5bPkP1yZEyev0g2IBhGtuzn34G7X/8O/TeP30Nb//bfwsAONWlCPnFSxdwYotQk9LtQGmOsp84gX1GGW+9+ipGKa1B6TQShLLO0+3yyQt44wH7XSYGAefrF2WO6ZT6c1lrREzBzVGiYKGYRfnVFTsncOH9LwOtMGPGSL1ZR6QpVG1nA9y5dxcAcDTrIfb+YpyGYbMMUx7f06wSw1DQKJgdMnMGIYuCNOoxQi+Vz/8eR5Hsu67EY3YD80XsK6CRThnB4bGXZhaW1/rO6ipUSIPvuD/D8QGtob3eAAOmh/UHhKbkWY6cbSeC3hD9I2ZINGKsMCK+1OmizZT6ZruDMPKpE8zOQIViKxcAZYWmeI85CwflDTrLKXLvp8h1VjAi2ACt4PUXcqcQcXssrSxjfYPOCEcHQ3mHt14lYZhyVqD9Bdr/m8uaFmawbxx/tzMGZeERYCc07cpxzgjiFSgNr+8DpaE8RdJVfWMV4PgZTaaSxVEMy+yuweAI9YTWKBNo1BhJVaC9lV6XKW/WIWd/1fFkCsWUO6s0+BiHohwJ7T6OAzSZGnfm9Cq33RYmbDkSxlqk++tJgJgNEwunkDI13hYObDcoUvwaRrJcHCDrrVJG2CFZXiLjug5GY1j2+W01aZyvrHURcaWLMsPBIaU3HB0dYsAChfv7fezt0e9NfB1dZu34c167VUfCfsTNdlsomf1+H55xCrg54RmHk+dJGCid0XidjVNMU1q/D/b20RvR99XbHSwxE+Zo1MMRnx1mbNcz6U2RMyI2yywzxgAXGoTG01wteofEQBodTyS1qcWCd2WZwTI7qnApZjxZp2WBQZ/q1Nh5hNoyp0QlqhoTmsWCEIhojlZOzlcapYirEeLOyKCyKPj8572XsyIHu75hlpfI+d/DMIBlhmKOEpn3EnQlwLYSDW4jbUso44VmtFiKGEDOyIBG7qhvQxi0mEkXGS8Yo8DsZRhlkDAS/NQzz+O3f4tS1n7y4x/jle//kNqDX7a7uYX1E3SfiYIIfUZPD/o9xG1eo5a2cOr80wCAwQyoBKr4jhIaaC/wl+fwrmRxLUGXRSknkyH0gOZct9PEk088gU8qf2sDJzGXVnN5LZHD5S9+HQCwsbmG93/wZwCAH/2bf4Em0x9S/ux0bNGNaaJDjaANG63WA2Qz9hrKS3SatGmttGIcsBpS1vT0w5oYWCtrhW/vD/q+nk6MECva1bxnmj9Ar66uykR1zsnB+fXXX3/ssAwQtc57p83yDL0hTZpHe7tyUDS6yuu7dOmSUDwPDg6kHk1+vyzL5LvDMJQLwXQ6lQP/ZDL5eX/Duf993HeOFIUA4OlnnsZ93vAPjgfirTQZ08D4f/71t7C9SYPxiScvocfUh0ZNYTJkpbdeD+EqLVyzLINnEpeB540EVVU+xv/u48qpczRIXckXr1wDzGG/cf09QNHMqsUN8XmxrkTpPceESlnIItNdWkbJm/XNu7dwOKC+jWoJDo9YUalUiOq0MDcbNNkClSBynFcCwMbUn8pYebZ1QMbUPhsGCGuPU0DnL33zFNB5Zc8kSeRydvr0afnZU4/jT+Boewrx/v4+7t8neuaDBw/E1P7NN4kiffHiRZw9exYAsLKyIuM4yzKp60cVQbtL3jTd31AsGiukRNcYnMONXfIUevbpc1iJaCO68+593LpNB5V2gwIgd+7dwi574K0+08Sjm3SgOX40xdopWjz/s9//KrYb9Jl//a0foc0bSv8R9dve6hgrVH0sr7WRDWiDK0YFHC9RP3vrLk636XlnV06hyZ6A/gIY6hANzsWIomAuHwkVNdSVkiMaBCHOPUH049o9atvd3R2s8EL6zgcfYMB+VltbGzhixcHBZIbNDeq7270eJrkPhnHbwlUycloLtQdzwShnK2PuUgO/93t/AKDq77WVNezs0prxkzfeRj1iWtyFM3j1NTLG3d85Qo0veJfOnwEAfPN3v4H2xnkAwN2330XvDim5rccJFJvTn7/0PJrr9Pmw3pQD8GyXgiihG2LrClHup2mInL1Km606Iq7/Wz/4M9y6Q8G+ztZ5/P4f/j4W5VdbvFeZCedy2+IYEa/P3eUlGacXLmxje4PWv3d/8gqGexS0jTw1ymnEPpVAO4RzvqWeJDQrnHis1gMDn5zmgxqBDlC66lJS5eS7Kt9cuWr/UnPK4t7DSmtETO1v1tviFzvoj7C/RwemnZ2HyDPvH+j9dWcYceCyLEsJ+NTiELs8j7qdDjb4AHb+0gW0A9onfD3TWY6Jp2JrBc17aWlLjEf0fTu7B3i0R88bT1K5KM971laH34oOWgs1Iu+nWGvgxBatnUf7BxiwUfTdG0T9Otw9EIX055//NDr+IK6ANKV9Os1LNFtNeV+fp2S4vXRYnY2M1rI/K+WqrgDkAlK6Au0G/W2Dg03dThcBX7Z2dh9C8wkgikPUOVAbBRFK389exdIVmDL3vChncEzbs70enBpwWw8w4Xy1sszBaV7oLNM7bZZdjIasoohCfONKnWJW+Do75D4hSllo1jOW/Q5OTO8DrWD5gK+DEIrztmJjEDMA0G51EHlfwTZTcOKvQb4AACAASURBVKNYLp9BEOICz5eycHIZyfMCh0c0Nnf3dlF6j82yCoo55YMkiegVlKXCgMdsNqvOM0VuxQxcBaxtECQIUvbanVjMclp7g0KjU/e0wmUkMe1RGe85s1kuuW1Z4YTHaJxDa4X6eb3bQD5hOujBAKOhzwvmOiuHgGmtrSSUFCyEFkVKP48nfVGebLc7qDFt3HuBpnkuqrW1WiQqmwoQTz1Km/GKnx2cOkkB+9SDMjszzLgPi9KJimzpNKzP9dSB5HqWOTArvU84j0GtZNArrVHzCrWBgfYe5mUB5Pw8Y3HpAu2nXo8hirWoAg+HUxQsn31y8zQ6yxQorzU7QExjudWldn7iU5/BFoNGo/4Axzdor995cIRcUSD63OXn8JnPk9/mpSsvzonb+3VGiyL4uN/HcFrlIGcZtdPk3h1oPkdvndzGpz/9G/iksqCALsqiLMqiLMqiLMqiLMqiLMqi/JqUvzUCKOpK2iLgSGCeTaE5srV25hlJQr539R0cffgWfQHD7TqwKFiOKjF1ZD2GdmdAremfDSiOZq011/CQ1bIaXtluqYGy8KIntqKczIluuDnDnHmK3jxtL5GoTiAUuTRN8fAhRUZHo5FQMv1/6406Sv6+oigeUwet8ost3n6bPNPCMBRqn69bu90Whc/BYCBiNPV6HUNGG7797W/jBz/4AYAKEaB6f0ynqCq6Mf89m1tb+Po//AYA4Dt/9de4c5MiRvUaRYv2D/bxL/75nwAAfvd3/wHW1gkVq0UKk32KcNZMihFTvvI4Ft8yTwdwrhLUwJy/0GPV+0ilOb8fxYwjGiqA5ijd3s4ddFhFqXc4weYmRYCWlltIc6/+5FXLDBxz75a6S9g+yWjU3Ske7RKaMx6PJTG92WhjnLJvTI+oREmogZQjhEEC42mwRVFJFhgnDR84DcPTxQvz0N/6ZO+4QsmtFero9va2UI6n06kgxO+8Q8n/h4eHMjbTNMX2NkWpz507J2M3SRJ8/vOflzb1Y88jff45AI0ljy4Oh0OhMmdZ9pgP4HhKSF2FkjsUPH8fPXTYY8pVgUPc7NO7TA8nKBXN24c71+mvshmePX8RALDnDpD1mab0qISh18ZT50/i8il6r5++9T6OH9Ic6LSpvw/uTdFZo8FRbwAtppMczmYIFUU++6MUj0Y0b7/wuc+hxshsymPCFoVETo02ouRmTFCtDZWYMZbXN7DGKPjtD0mNdPzdb2ODBRiajWfR5uTsW7fvI2lQnTWsoGYIDXosktL064urInbKKlHw0sbAetECOBEiKMoSx8d96S8AyGYWQ/Z3KnKHggUf2p0lvPBZQufeffdDMDMNW5ssFnT5DFodmjd49yqKGn2g9+4NTHeo/uuXn8d5RgyPjgZYY3GhSYfa9ta1N7GxSRQkXcaYMVWl3Wljb5fWkf7BAQKmcf/+7/8n+NxXX8Ki/GqL+GqZAAGvw0U+RcZ7xmAwQLdDc+rk6hLazzwFAHh49X0cPSRkOPTsF+tknBpXVqh1UQqSVFMaIYsidRp1xCzuFvL65zQQMLxkAyVrqIMSkRoFmZawzol6rCiNFlZEIYp8itSra6cZ+izOdHx4JGiC9zws0xmQe8TFilJe5hzyjH4/GkwwYdrYytoa2ozyeFVJ63Kh32sTwDCtuXQldnktvHr1Gu7ep7bL8lzaxiOACk4obdZZGC+Wh4p22GhEuHSO5l+sLXbvUDu+/wF5nO48uItX/ppUIMe9I5w/d5YeaBQOdpnab2Kcv0DULhMbWY88IhlqQCnviapgPB3XKFEKtSUEvQOAtS328YsIuVheWoZm47Xj4wMUvN9GUYhmg9DTbruDwNPh5xRIUxYfsq6EV+EZTXJhIPUOH8IyghzXI8yYxeEFLhSc+EsGRguqUzgFxVRT4zTi2AuLkBAMUKXbkPckQ025hjbery2Atex9G9dQr9MciaOY/ZEBxyqVw9EIOa/NgEXAwh3tZhMRo3PNZh0tVnjdOLGMCfsT95mmXGaFII5lXkAx2u1U5d2mjRMhPmtTFJzmpFjgLEoiBN6vd1igl9Lvb33wCPWG95A7Qr/HY5bZKAh05RXnHHI+d+mywFKb9oznnn4Gly/SuH+18zbefouYYzNW7baqRGPKa81KXbwcZ5MMOU/m3mBMvG8AS0st5F6Uis8pYRCJjqTRTs58zpF4IVWqOi8uL7XxzNOUjtJkhPZqEGGPUzbiQImIk3IK2ZjavBZFUJxOMRoOAKbCehEkN+cJHBqFmHnDoVIibFTAwi8geV6iy89bY+WXuJFgwneUUA+gQfXrrq2iYNGeWmMJnRU6229s0tl6/eQ5tPhZ0CE6KyymczzGOOVzQRBga5vWBmOCirnHauSTicWU2Q+jQR+r3L6T0Qj37t7kz6QiULS9eRanzvw9UECJ6sY/OwXNnX346Da+z2bOZy9eQHOZLxL1FgK+GKZMIaobhRlPjhkSFKwIZW2J8YSVMV0Gl1Dlw/YGTp+gg+UkWebPFpKD5gzAawG0noP+SwvFG9FHaW/+XXxu1Y0bN+TQNRwOsbe3J5/1v/c5gnEUifKndZCfQxPKRC1tKZvItWvXBDb2B/LpdPoY5bRS+FRymJ+vk1JK2u8XEyt/vmRFgTVWHv3tr7+E73yblBTv3CTVNAWNmzdvAwD+5E/+JS5eOgcAeOLCWTxkxdAVG2OF+cPbn/98tVj9HfL+fs4EnRdkb84bJQ3kU+qXWXqM0ZA22nffeQ+3btICe/7iGcR8c4xi+l0S12B5Q8pmEyjeTMosrSTC93bFpHNcP8Q4pWfvs5l4mKQwEec16hA1Q5taaYs52l6JwOdg2EDMgf0mEwSBUHfngw1pmmIwoA1gNBpJPw8GA8npnP87Tw3s9Xro9byyVon336e+eOONN/AHf/AH0o7vvfeefB6gMdph6mKaplK/OI4l5zDPc/nZWovCsWGqBE6AomDK8k4ftqT2/fGP3sbePRrTy61lWKY8+PydbDTG5S7ltDxKK3WxxAAJbxahy3HqNI3Hb/72Z/HP//e/AgB0ajQOJtMC+7foeeevGHRXac6NBynyEefiNEL87Bpdml948RKunKbL23CX3rXb7qCUpIocCVNrtVaSd1EUOUo+DLXaHXQ6fJDhxf2v/vzbeOo3iBb6T7/xO3jnXWrnk2cf4f33+ZI4GePObdok11eWMXxE7dhh2oWCkslqtBKqXumc0C3CeVVlAI+YQuvHwYnNGuI6bSDR7QLW0po3mlrknPsTBgEu87w9sUlqfYNshAkHOGYG2P4iGWnvvHsVBW+GBw8OxWC2d3iAjGlljjf8IG9ixrmZyhbQTCncu3OAf/fd/xsA0D/ewVPPkX3Ol7/yBYQ1bye/KL+qIsrIDsj4YjOZTnHUZ5XB3T0MD2ifm2yfwCqrvDpT2TjJlcmW0DxHnCvnVEALGA74aBOhGdGc6nSXkTQpmOhzx6xVFe1T2ccVoOfWGD85rAOmLInvFU077aZcYvZ2D7DDAdnJaIIjtmI4Pu5VFE+uj0Zlv0D/VBko+2pkWYbeEQVBDvZ3sMy5gV4m3ZUlCl43J+MRNCvvpUWOW3dp33zr/ffxYNcbTVtZb5QY3TuZ78oWfAGii5A/YMbaYmOVvrvbbeCAc9Zn3srlcB9Xr1Lw/Li/j52HRMtvdjq48SEFxBElWGU7rK3z55AyLczvVXEYwKhK6VIUDOcYuFClKChrGCytMO2Rqa9xI4bmgH1aDBHyJWA0spK3OOx30OJ9J2GV0ySO50zNrYzTJAgwYOXg2zeueQYfNk+eQDql807K6tDWWn8vhw4NDF+gAmi5gJAXRpXk59c3byZvtIHjA35ZZLK2aqUx4ovmBGMcGtpDZ9MUU7ZaCLTXBgBSthOCrayznDVEqQSpfMacGlIUCjN/QOfLrtYKLaZ9TqdjHPf63BdKlELDQItabV6UyDhYrUWmNYbiQKgNIowy+v3713dhFCvXFyksB1KiOvdhXQkN1mYWBVtaTTOHiPerU1sn8ORZtgJyDRztUR+8+94Nbi8gTX3UxqDB1lrDUYHc0cWkP55hZ4/GxPrmBmocKPIU7cgEFa0cDuDcY7jAu3lBqQABJ7XVTIwmU1tbfMlfbsQ44iB3luWIOTfTlg6v/fR1AMDVIsOAP5/nM+TW29ZQfwbaytjUitY9ACiVQ8HUXTgnwQ7kSnRGSv6vUQYJU61do4W25HzW0OeAla41sclaINt8AUwadWTe6sOEWGFNipPOIbdev8TAcj2KNMPOMbXpzi7dSybjQoIUCg4J63xkWS4B4+k4x3KXvvP0uQvoduhs8IvKggK6KIuyKIuyKIuyKIuyKIuyKIvya1J+CQI4H7lT8rNXBmvUW7jxYxKF+Gf/8/+EL/wOCcLce3CA8TFFEmqJ9zRRMBylGeYFpin770QhnKdxhSGidboZ58tnoTgKbkXOyUokpyzLyqtHaYn0Ofe4AqOnf817tnka5ltvvSX0POecoDP9fl/QEk/ls6VFzOGKIIjgCo8oWqmHVtV9+uDgQFA97w04Go1E9GNtbU1Qov39fRF+8QgiQGjOvJANuBs+TnxlvmgYQb/WN1bwta+R4udf25cBANdvXpN22d07wPIKRTGiKw3c2qUoy83hFF89TwjD9koXE0+99d/xUW2aj6nLRw3TQ6YNGU7YDlSAOkcOo5B89wAgDIEsoyjYrZsfCPq5ukoCKlGUwLK0WWhyNDnCtraygowFbXoHe8iY/pO0SkwdRXFNSFGrZqjEzD4KIwSokNmQI1FpOkXBiGKZlSL2M+8X6WkO+/v74vEzHo8FuX3qqaek/6MoEj+/06eJZnf27Fn593nEOgxDGcdvvvkmvvWtb1G9m03cvElwvxcXunz5Ms6cOSP1+O53v0vtGwR4/vnnAdAY9HXSWosyWZ550YII44zacZTvodUl9P2ov4cRJ1+PDo+BlKPMPBfWVrv44OFtAMCwlgr6leopXEDRQuMiTI+onZ67eBnvX6HI/k0WKmo2YhzfZWW/9RCtLZoj3fU69tk0Pgxr2BvQ+37/9b/BpSeI4vzOOxTBfem3vibtBUBQP2QZClaGK6yC9X5EkynYogejEc37C08/jS9+meZKEIV48fPknXf5yhQhR6F//MqPsLVOUbXCOYxqTBEaMp5irZhIhyaE4ohdNDc/lIOgB5ExeO7TnwYADIbUBpfOnkEdnpJ1AmOOWK+vraDHnpHDyRQnmQam60RV3d0dIWB1w2JlHYGjeda4eAEhR2Pzg0MRjRiPJ7j94W0AwBGbCzfqTbSXaBx0uw1ETHta6izhzBNUzztxDReeI8+jKIoRmY9fhxblP3zJigKYW5c8CjTNh3j/HVonHty9g8sXaX0YTFMUbNQtsoy6ELq8URZedSPNShG7QFFg2XoPtgglUwzhvLl65UtV2oomrZQTNMfZih7trMPEK3owghY1aigYffzw6rsiQGQzi31m58xm40pRMPPUr1Dml4KRzakonag/Omcwm9E8unf/AWq8Z2ysETMhqcco/BkntSjZ72w4HuDeXUJSj48HKDMv9lRWypLcRMpCfA4VKjVyOGGaAjAVtU+10GivyecBoMynKJnytb+fSRet59s4ZJQlSBLPisRSrYExY7km8mIvlfBLWZSw3ImunGNyKS0U1QJAgyng4tVXN0hZDXGSDwE2ER8PxxixWXzpgDqfZ7xAVLezhO1TNNZa7S6SOq1jSdJEFNL6Np2MUVhax+oNhzSntcfv+dZl0l6BBkIWIXEwACtL2lKJh1+elbLveDXYWeHEEzDLM0HJy8KJYE292UGbfdLS3IkSZGepw+/SRcf7C2qFgBt9NsswOqLxMbU5+j2qf7/fF0/XMauYmriBZovQ972dHRzusChRt4P1DTrfNlttzFLvMZiJmbyIm6CUsblzNEFa0N6twjUxY7d2CGkEnr/Z1MnZaTa1mPGeGCrgiM3THz44wGaXfl8LI6wtUXs0EtqbB9OZeFEaHaDGwoZRLRWhpLRwuHWfmCeNZg0nN+m9TnBql1FaaImAFjS8LHI4/2xj4dVhjLKiqt1kiu7582dx/vxZ/juFmvcOzxx2+cx648Pr0MYLWGlY48/rfFbUoYjzaKhK4BFWmk4rI59XyiFiNDgw7FEZtZHUePw0gI1tUt2Oa3U5b2odo9khhL7FaFxWahGZymYTWOeFZuoY9OkM894778CEhLymWYkpn2U9uh3VKkXZIKqUfo022NogRlS72cDFs4Q+PvsbL2KT01h+UfklF0C/tDn5WanKZLTRXccWS9Hf+tF3MevTIt1ZO42jG3Shib1cdb0BF7FinwtR8kGyFscop8z1NyHcElV+FK9Cc56P588Dc5eKx3LOKun1eWXGIAgqA2c+HNbrdfn54cOHcukLgkByDvqDAUJWFPSHfmU1DOf7GDiUXlWqgEg/FxoC0eocePtNonJ41ca1tTXJGRuNRrh6lSgdR0dHQtubzWaPSfdL7mWVbve3KJUCGZxFgw1uPdUmz1K5dBRFgRkbOf/olZ/hfVYO3D5xAp2T2/Kd4UdvfPQ19BXzv/oFiqBKKUQ+X4DVnqIgltyTei1BxItEoAwyPggUaY77jzy/mRbPRqMtuSRJkuHCeVZzTE7gJCus5dkU0zH1Z2spwaSkyRTGvPikFin3Z55PAePzAQ00H3S0sbDeYF3lXqRU+mc4HIo6597enrxvlmVyuX/uuedw8eJF+TsfwJi/RHoj+Lt37wpdVGstNg9/9Ed/hAcPqF+m0ymuXLkinwdo47lxgxaO8Xgsz+73+5IbuL6+Lt89Go0kD8LTl2azHAcj6ouJStDaoPzVi89s4qD/76lO6haWElrYJgdU526zhg/2+RJkNRrMc3cnHSKmgvzge28jmtGzh5MMn3uBLhKvvUl1LiZjdDs0Nqa7QIe6EJ1ODaMOtUc6sDCKqNQ/ff0+vvxZOgwltZP831pl2aGUBCiK2QyaczpGoxz1NqvLljnysR8LNP7/o3/yTypamarMiLUCeqz0dnLrBF57jdRXD3t9nNwgCldrgzaqtTDEdJc+OzqcwAWVDU3pKlNhUSbLS1y7/h73AedX3ruJ505SDsSnLryAgul7ZTHDEasQtpM6fusrX6Nn8EG6HFtMe5wvMHWY3qYAVDqeYMIXvLu9HjIfyIoSdJi2HzAddvPsKZw5TW26tNQRlTYAWD9Fm11hUzz3LI1B44BQjJUX5VdVCq8yWFg4pqklcYgWBwOatRruM1X5qL+DqMaKmmmBMR8+6qy4GUbVBaQea8xqnHseJJhN2LqpsLBsEeCiWKjjfnRoVJc7ospZ+RcR4VUW8CkZZYnc51zxISsylaT7nTv30D+idSVSoVzqnCsR+/SLmOdWFsLIBUtLpYrSCu1KRQkcnyfu3HmEw2OaJ1sbtNgsrXSrYLBSGHAg9vj4CB+8R1T80XAo+3tpS0mHKOW9DawPlKrqQgw3txdqA82yl6EOhL454TNQFITIjL+IDjFh0/uiXEN3mepXOAUfx1IBhKbt7ZLmmgDaKFivZI0CyvkDckVvQ2mRMI3biqJ6hoLfdZYZSXmZzQY4PiY6bjbLRePB5+A1G3WUBY2Z7e2zCDdYnTOpo16ni1Wt2cD+LlN6D3aRsUXQjP8O2lWHcF1ZOEA7ad+yLDFjECHNCgTaU/840JWWGPElbGdnT1SV+8d9OYP9xnOfwTPPkOz+6vpJjKd8mdJ+fYzg44hpYUXSP2ooNNqcJ1ZaHB7QHjsYp4DjAL7zVH2F2KuOwkFbn+dYivG5QrXXzCYpMr5s+zy31AKHI067GhVYPkHKlE/WNpGzYvZwtIsJXyT8xWE2GSOzPsXGoiwrdcvbNyioofMfY4NNxI0xMBws8GkF4fGxnO/G0wwdznesJTXEHBxM0wKDEfXhjdsPRbPh2Se4HcOoGptQEmzSChKwL8oCQTy3f/O5MGEqtgkMEq8MbxWiyKvSVgDRYDyU/g+iuMo95nNvHAcoWeGzyFJ4tQdlIRTy2ASi2upUiT3ebycTb22SobNM7bWyso7VNTojJ7U2ypyDTVBo8VpSr1Gdp7MSPT4TjkdTlAyiDYZT9Ae0ztlcA4bHTxDKM9ZPEGiwvLQqZ5y8KKC5/iZM0GZV4EYzRIeVfKN6DTr+5PSMBQV0URZlURZlURZlURZlURZlURbl16R8IgJYSmJ4hbCFWqPke2Me1tDhSMHKscasR1GdU89+AZtPUvLy/jWKbndPXMGTlynakuclFNOa7l99G4eMkI1LIPPqPoGB9Ya1EklzEld0toLyfR3n/kd+9EhXq8UCIkkinz0+Pha0JAgCpB6JLAsxOx0PmIKWKjj2yIvCUJSCLBw00yfQrSHnxGMDhwEjOw93KNpy7do1aUdjjHy3F/AAiCZYGUL/fzdY9uB/FISiBPnGG4RctOt1iXZe/fBD7D6k+p0+cx4MbqG7tokT5wkFmmQlguD/f5Q/9LKF3HZFOSNaB4AszbDPUbrJeCJCLLN0glaHxVoSas9SpRKlu3VviFyoowHOnSVa3LOfeg7vvU+iIWnewzSn6IsgmbYyX1XKQhsfOcyRp1ae50VNgghCL/AoXRiG4ut4fHwsaNvx8bHQM99++21R8AyCQGgH3iMyjmOhZu7t7eHdd0lsJM9zfOELJOLx7LPP4vx5ivq9/PLLeOWVV+T7AWB5eVmet7W1hXPniLo7m81kDm9ubgrKfO3aNdw+vE2fySlClDRjfMgKsL1ZHd0aUUHiTo71lae4zWLoPkWodh/eonrWG5j1WEBkNYBp0fPWltbgptSHf/PX7+HCGtFCyiIDDM3JL3+JhERee+MNfPE89VusZzi+R+N10NFor9K83R33EcYUcZwUBv/mOz8FAPyXX6RnhHPzJggMMvbIUZqMdAHg7p2HuPwM0YjDJMLe9Wv07glTcEMjputQAXkCAbBZjt4xtU1iDJ6/QnXd2d3DiNeHXZ5jrtNG8yShavXlBsojaq9xf4wooHVCu8oMt4DCgwekrinGzHBoMcVupbECO6E2mMQJEm6DosgEhcn4WeNCIeJ/N90EYYsRHhOizd/d7A/R4DWx3mygzuM38QbQoUHsKe1unvmh0PZraGNZ0A+jtYg0LMqvrnhBLU28QwCEADaYGpWEkShdTnZ3MWMFQKcDgJkYYZP9cDtt1L1wRBggZUr04OgI+49oTTPTGWpMlwviBM7TITyKZyA8R22qPAU3x2KBc0L/crYUBVGP3oi6LoDZdIIBoz21WiKIlrUFCu/flYq0IIlLgKiIEacVmNLCiQp4Ac2Cbvk0xyMW49hnoZmkEYuKsINCnnvF7xx9FmUzSiPgPSwoA0GHfLqFdqpCIh2EouWUgRJA1Mi6HWotoiXOs6pabYxYFGXSO0KjxbSxYV/2DqMMSqYMOlulnohmiLaC6BZQoqJpUYlSAW4OsbWIE+qD6p0cSnD6AzJkpVfqTOHYsDuMgVrglU55PbA5dlkteDwcYcSm5U9cfh4tZl+cOf8EmGCFyfQAE6ZQ5pa+r17XIoDiHIRtRWqO7MELjZBFVBAZQUMk5QdWaJXD0RjjMY3p8XgkqNPd1XX4gbrUbUt6TuH964ocU2aFpVkBJkoBuvJxBSoVSl0qmJDq5BGqWhiiy2tsfOoUWiyiUmvW0eZzX7vVxYSfEZoQSjFzgynOw6nBQY/aeZQm6CzTmTtpKGQd3msOYtxnar8/7xWZE1piUToRJZoWDnfu81llv4+tNaa8NppCc91cYupupDHOWZXUucqzM4rFHD1uhPADvNcfoyZpPSygGCcw/FkoBe3ZL0WGvR1KzekPx2gzuhg6g9jvR0zRVqaGnNcDqAAzz8xzpawNk7RCuKMkQsloYL1B+93achsFt2l/0Jefs7SUlAytnMzh3Frcvc/Ku5YYWFGtjvU1avPLl69g+xSdtYKVEI7R09xmqLFXpuH+zqFQOhYXwgwJ/77R7mLjBDFuTBAiYM9fa0LELBjZYKQ1iRMRsSnyXESEOq066owA1qNI1h0oJWzNX1Q+8QI4L3PsDyfOWSjP+3fA8ZCoWFnSxuUXyMSwXgtEXU4vEXy59eQXUWNp1LorUWP1sEmmsXOHGjnUQGeNKCxxEFVytvw+1lo5eDjlHuPg+463RTFHuHdyAfSw//xFcZ5uaYyB9WqBZYkhL8yKDR3DTGN1wlz7OdNvBYW8Rwt2kC1Bs8FmGgE17hTDB+/9/X35uziO5fKQJIlcKpIkkQ0iTdMqn3GuX34pDXQuD3I6HeEh0wezjCbyP/6Pv4kZv98Pf/Sy8NaXVjbQYe7yzdt3cZflwj/9qcsomaoyn2dV1Uc9zsid+/18CSP60ITpAiiAkodglqUImIZpAoPxgBaoqD6BFzJyAUvjFxk8H3M0zjG6TvU/PDiCY/rmF7/0ZZiQ6vyTt74Nx7M6441d5U4milYWhs3pXVld0uACybUoy5nQRKc51SMIArl4LS8vi21HmqbSt2+99Rbu3aNF7sUXX/w5FVBrLdbW6AVfeuklnGKaXb/fFxuRMAxl3D7xxBOyUa2seDW5rjwPwGP18N/38OFDCSicO3cOU0vBmg85v8XFGvvemsWeQ+qVAJ1Gd4kun9rFePt1UvB8eIP+Pm2mmHAjKWfFXkIXFY3DzAxOn6V+7q4keOcuBYW+9iVaL1YSjTovwP39I6HdujHQ3WYLiqUEIz4oNFo1HI/oe5ZXiBZqTFgpw6Y5cs7hiaMQh336+d69HZw/S3/XezTE7nt02T7/9HP0LuMpphFTruIYlg8bRV6ixYfp40EfNc4RXes00eGDc7NOl6PVtQ1kvLGM+4fIWGG0djTB7m06bEIbmc8FgAFfIn0J4gg37hN9b9qb4vQZCqadPH8BCRvLJu0NaLZ1qfOBvh5EsjhYZ6EKb4pMsuMAYNZX5aDodEm0PADGS3NbIPc5RcqI6rLRWtYdY4zQbB1vpQAAIABJREFUiUMT/J2UgRfl76f4A3IQhhBfBFQWMU4B9SaNyc50hmXOa1LLHRkj7TZdAE+ePYNWhz6bTieY8aUjbtQqxeNRKKrWSdKQXDfPIbK2usQoVeVfAU4CubCuSmXQRi6Ans1aFqnkGymnkRlOucgLlBwQzNMUEy/TzsHWzGixRGlGIYxjejuUXJTSvECd87kajZrkEfpc4XSWyby1rqLG12t1ecfReICc983cFRLEERVQKFHc1NoB1p+ZCrkQK1dKjldmS7m1nXuCqYhr6zAf0Lr01muHkj6SzXLZ20wQ4eqHpGZ94uR5Ofz5iKYtnOzHFkrmZ6B1pdA5d5bSxsB5xVJeh8vcIeBUDetK5Hzpy7JM3sUYDR16NXT6rLIWU85hPO4d4gEf8MfjCZ7+FJlSr6xvoNVky7D7H+L+Pu1BWeoD7IkoFReuUm5UhhUzQUqdzqueAqIj4fMCSeHRpxklqHGQL0tSOM4H3N/Zwc2blIawvrqOiAME4OB/kkSIeR93zsjYtbY6OzgYnGBV1zw/Lab1OY+lJApEsRJFpRKvgsoeJQ5jmQNHRz3Mprep/fhSuD/IsNejfXCcdRFE9C4mDFFjJciBUZhw4GY44DGaZkgLf7FRQr/OrUXpA0LZDB2+jbfrsaj3x4mnwbaBoU+pCqF5PyjzKr+w3WyIuXs6HaDks0PkgyVaIWArI+2UpEwdD8e4eece/90EeUHB2XQ0RIOpjm2uWxAGKHJPTS/l8pnOckmtCLSG5nzRWhCjyZfxTVbMPbW9howtR/r7NYz53HvUH6LHdh9ZmYudXWEL2MAHrPj9JlMc7tPZ5333ngRfLlw6wjIrgubOCcDS5qBHGBvUOZXDhAFqfF5oNpoImOaqTAAF/90O8Mr7vEZopUVTAFCIOOc3jBTAaQCFpYukr7R1nwwiLSigi7Ioi7Ioi7Ioi7Ioi7Ioi7IovyblExFANUfzqX6n2WQTcNkUh4dEjVo/+RRam4QUIOuJYfvJCyTQ0V1eFTTFBAYeZz19+QlMdyjJ2uUzNBgGtoWCi/z3VzSkxwzfPU0ASmifSlV+RB/3LvPPmFddLMtK5XE0mQiKUjIkG5o6tmOqW2AUtPKGnkDO0Y/pZIppzN/XqAls5+PjrVZLELQoiiShsygK+b55o/p5w/G/S4xdqSpqMJ6MsbRMkd7//n/47wAAGyfW8MYbbwAALj39BNqcBDzoTWGZhjEbZfjWn/5fAID17n+OEye8mlAVgZiTNqsseeaa/qMIoDflNUz5ycpcoq7tRoWKxXEsyFrSMHAFm5IKPUUjnTE6k2lBah7tPMBrr/8MAHDixElc+RRFVW/cfRM37lOEp8FiCDrIYQuKmJWw0IwohlEgUfQ4qSFmOsN4OoDihPfphBXFylKQkFarJTTjeeru3t4evve97wEAXnjhBXlHLxKzt7cnfR/HsTyj3W5LxH1eDGh9fR3r6+uPtWtZlvKMecN3Y4z4WLbbbfnuixcv4rU7P6G6chft7B9jxONtqb2N4YgEnYIwRo1R8PHRLh5cp2hth6mGT5w7iwOmsN54dBuF96QKAhRerKEZIFun+h+2jsXAeTgh5bAT9RAP73MUOAixxL6f4707UMyOXtpowdSov7rNDk4wvbHNqmVFaRF4yrgqUTDF/Nbtu/jf/s8/BQBcufwsrr/3NrVpmmPIQlXr6zS2iyIXXzCUJTSHNUfjCQxHBXProBlVaCQJ1leZKsso3KXLl0XB7GBvF6NppfzV2iS08u0fvyNqZHAKKRvBStEBDlkR7/qtu2i/Q+vjU5sncfYCCQptbJ1DhxW+UkYFh3GCglERGxiZZ8oAOvDiD7UKMTJOkJCY6V4mNDDMzojCCJE39zZKaOAKFcXcucpnbFF+dcWLYTijJOKbuxwFr7Euz1Bn8a8sa6DZYnpVpDCaMF3Zq8glCSJWxSycRsKR50a9jQNW2JukTujF0NU6LwrepRV1SygN58eNq1A/B0goXQNyjrCqSjWxXkRNO0FzprMZipL3pckMjhGGlMd0Fhu0GGWpGYVsxt+tCNkDgClykpcGoIwStNKJKXW1VxmtEHKbxlEE68UWdIV8W6uF+innE6tlj4DTlXid1SIUU5a22sdcic4SoUdXPk0LnS1KnDh9kdu/jlvvkVBclufQfB7qDY/w6o+JAr+0voGvfP6LVFdmI2TTmZy/zJzXWqnmEEDoitWlyHcMgCCwRebgGO2phSVibrt6LQGYSQQUiLjPlfbpOgohoyZ5rrC/R6yHDwB02A959cQ51NhMfml1HUtMLT5kZUoLK4wu7SC0YaI2iXFcRXWzShAaW3q1z1LojHGSCApdFqX4Xw5HPdy+RakMp7ZOo7NK+202J23k13W4UnKRrCsQsJJuEBihfdZbLdR5/1bcyZEK5awLZZHxXlmWJcbeexVKFFnrtQA7B7TPjSZEn93fzXH3ESOwxQSra/TdteayvFc6TTFgRMszrLQtRfhPawt4U/isgIno5zgCEjaZDwItTLmkzqhlWiIa03Pb9RbWmJI5njo4R2eEehyizojhWFdnIi8EVeYp0ikz9GyGCadS3bl/FzdvEdPF6BhBRHNguL8LHkLY3iJ6ZFjriH2gqTkEjGztP9hD/4juIGGsRXxnqVODZhG6FVa1XlnuikhMt9kUWvbh0TFu8Tg43D9E4ee1dQh4vHnqtNYVcrv36CH6h9RHH7z7AbZOUtuc2NzEGiuFl+yXms0yaKaAhpGBtyweDKdISxr3+awQWqfWIYwX6dLV2dSj9korTPkMPJulqPHcrxclWowu6sDIWeQXlQUCuCiLsiiLsiiLsiiLsiiLsiiL8mtSPtkGwkeIUCFQpYPwUBtxiEaTIoud1ROS86NrXWycI+GInPnWKGdQQZ2fW+XixXGIs2eJ+/vo/h2UHOmz1klipo9Uqp/DwebzEj8e9atyFyvk8DEU0b/XnK+gNlryAfsj9kvBFCc2Oak1jFCwR06hgQFLFo9dDjdlf688R85oQlLnpPswrLxH8vwxv0KfqzXv/WeMmUMu8dh/f1nxkdSlpSUsc66HT8S1gcPGKcrN/K//2/8GWywz+1d/8R389BWKLKKY4db7hDz8sz/+Y3zz974JAPjUlU9xnVGJ9CiDjwFdf6542wXFuVpFWaF6aZpKdKxeryFhFGhW7KK0HAFh+WynFRAw0qSsIBO1pA3NuSnvvvcW2h2KuHTaS1hp0DuGCSf3uwEGuc+9yiAvUBL3GwBUEUNHzOEOwypfhOup5nJB50V9Hjx4IOjtcDjEX/7lXwKgPvfIpkfsiqKQPJujoyN5RqvVEgRweXlZkMFOpyPP8OPk3Llz4i8YhqH8Pk1TicbVajUZS3Ecw4Sco8v/jZIcSUbt9fDWbYkorW6cRjOhd7x19xbsmJDU55+m5Ocym6DJOR/b7QB2nSLaejlAwJHxKK5hWCckIS3HSHjNGLKvX9xXKDly++LXv47Llz4DAPiXf/y/4nqf8gVbl1axtsGiFkd9nGZE2ukq2piyLPif/9mf4/MvktXEq6+/hw+vU57Bl1/4Et76GeXX3Lh+DZ/7f9l782jdjqs+8LerzvQNd3z3zaOkp9mSZRvLsy2DRwy4ge7EiRlMJ1nQadJTenU3IYNJk+4kK6QbWEmTZDUYTGwCxDSY2YCN8SjLsuZZb57vfXf65u+cquo/9q465z69J8lGspBf/dZ6up/Od75z6lTVqWHv3/7tbWJ1l7jQdHYm5A468ujDOCt5PF2rjaNPP8X3HgyRdKQ/5jk6Esw90xVvrVMhXjZvzyATUZne+hrUfq7TuSNnMDnPlkOTNnOY+rxWJXIREZiZX8C6xD5+7sij+NpR9lrubs9j5wy/1+3ZGSnDHDqFBIPnnSDTjSKFybxnMIdKuPxZlkL7WFRvEc7zEGOqWx2QeH2yNMWMxBrddOMekMRdVNYiDW0Q8c1CKR5uOAWbeEGYFFUlsSCVhc7E4tzqoPLzbdJCIf1iLN4xUxFIrMx5YaAyP88pdOZ4HO4NKs4XAY4nnYoH24thEICmK9h7iIkIsN5LVAtvOWvCOd7DlugENnjNHJyMwwb1mDscjYJYjI/5JSTBUj9KU1iSeMHKYCTewjEIVjwZlhDEEYJvSSnoxEv3aygvTjYeh7FaWVsLTek65tqG9QSCwIVWLlwDikU4uMxVsP4rIszKWOgt/4kG9h3gOPBX3HwDzp3nMejcmdN4WmID773vC3hK2AvuDy0SyVX7ilcx22qmPR/yFaqUYCUW0ZALTcReS18HLuRf9OseZ+q4YWcd2sKcKfQsMlmjTSej4PX1MZ+mMsglhq5VFEGcJSsyDCXNzWQ0wARjubbBwjyPw0qx16ScjmDFE+l8BXLlhZQW08qEZzEOcPKMNNmUX2jWgwAwHk3Q7/EYemF5DcO+5DEsS/zpH39a6ncFuXjCViSVzsrKaqiPTBPIU9jIoiP56Rbn5kN6grTbwr59hwAA20RLYX6mi9kF8XYuziORd0iTQiXzPq9nhJmkCeTZIcI0Gm/0sXx6Tcq2gv4Gt+2BAzrERJbTCil8ei3/t4B3xplyjKIljJA5Yo8ggJlcY0kYJCklwduqJc+yqoCWiLns3b0PN17HugR26nBOxMs0gELqJuu0QNKXSd7fyaCPjYscN3f69Pkwr56/uIHlZX6uudk5zG/jPnHh4gZKeedulblvdpuF887f0mF9nX93/MjTWBMWYpHqsAfp5HlI+dWWXOStLAvsI1VotCUudNvidrTE2/a0egIboukxmvSR+zHPj3fgfgMA02mJwab0q3Pn8MSj/H62uzPYsShrN9Fp6MwthLQOs3OLyETkxsJhKmO5KRHYf1mrHebvQhhYeZZyX4HsbWRMSTIKHsddO7cjl/rPkcKJ+NuV8OwbwAatMuw7qA7v1gS0u17tZi647UukWDrIHeXc01wpo8Em2hLsaJ2D9i9TaZFoWRSPbMgXorSCIU+T8vduqDw5dwnV85k7I6UoLKj9xkspFRbIzQ2iUioM7mmSBHqpVxtaJ8JRIwIetkLlfaeJrvP2UAKaSADuqN7IefpHc8OglAoBwc65QDvs9/vhc5LUdMRac6qGu8ymt/Gt/NEI6X78RGWA3bs4f8nuXfvCIPe97/9uzIlozid+8xOYlaDVjfV1/PZv/Q4/l1BFXvWqV0NpTz1ygVXWpNIQHMj/vwMmI6FWSqcs8jbGEthblQaF0BUX5mexts6DhBmmSKRP+DUPEavVAawF46l6WZZiRpKubmysBQUnUlPslnxtgzFvYDZHBvB0OWswlYWTIkIiimiTaoTxhih+UiuIDp0+XeeWbOac7Ila3PLyMpYlUNgYE8756Ec/Gl7woPhobaBdTKfTsNDRWofjWust1GG/QPf0zp07d4ZE8Hv27AlCMt1uN5yzY8eO8NlaiyLjdtzJ6zsM+qs4dYLL/5XPncbOvSwAdMttt2O4zmU+9vBXcP1emdhmeQB74snjmJOFQuZ6aO/i7+lACSV0YmdKDEVFzlUZ+hdlAJWkvtOJRdHldmtvW8SNr2FlzzuffBJf+z2mKreGFTwvJKsmuH4/bwB94mvnLMaSM+vo00dx731svJhdWAi0uE6RY99+pqmfOHYSY6EebW5ITrCZOWQSVH/tLTejlK77lXu+hk2ZcLrtNopcqGemhBa6nJPF6KA/DgmnK2cC1bN0Gu2Cx8quLqBl8mGMwzMAgDEV8hYvinbt3IveGo87w14v5LY6MRrhqYuc90nx/IccCVqQhVqikcumr8iKkLC5m7ZRiCEly7KwIFGyWUiSJFDTs3Yr5EPVSoe6aU/eiD077uIyp6oWTYr4piHMaxWCgFWSpggJ4rRDITmorEMQFIDK0G7zWKiUjL22EtEMFlPyRjRK0qCK67IsKC87BcDIaiwIobjGNE3wqzXn3Ja4gMtRQxMvDq3rRY2Fq3P4Evc/gJWZKxk3/GRYEmEsC9eBqQKt3VqH0lOVFQItfDwehc2N31zoVIdNhDO6NkxORxhJLsTRaIiy9AJJzxSWI9cQVmmSq0hDwSeQV37aEYO2V5n2m3gV8votLW3Hzl1s2Dt04BCGExG+OjqLFJLj7tiT+JNPyUZ5yuPBDbe8Bt1ZHod1koT8uk65kHdXKRU2wWSrME/759aUIPPzD4CObAKSXMOKsdqaSaBhmpC3dwyt+NxWK8eihPQU3SUUMmePxj2MxWg4GW0gE4PUnNBC18sq5Ed0RHBhcaG8gLgYoGVjawgk9XguzPkqUPXW1nuNJNxjjEd1uMRjj7GYzokTp2CEVrguIiqmskGYrkgVlIyLnVY7PEsrzzHxTgaVYZuocs7PdeTcArOyGVzcuQvbZLO7bcc2LIqa555duzAzz0Y8ax1aIp6yIIbeXfMWTzo2oG6cW8F4lcs52hgEJdH++iZaMsZ3djLVdt/2XSAp/8kTJzG2vN5cmJ/BwqIIiBUZUmnDsj9Fvy9Ce6KE6qxDVza4N95wI66T+fP0yVOBtkoAZub4ehkhUIcTabeynGJjlefPxx97DI8/zbTg/rBOep+pdsj5Nzu3DQMtG3kR4Una3ZCjkojQk7y2F9f7ENsxtMqCev9suwghZ+22749tkM81PS1RCG1cWYOOrPmKNEU/rJ0BPz6kXpBFaygJCypyAwtvJLEYyPp2NBjgWE/WIsePAQDGpUVX1Ejnl+oNp1UuqH3meR7WE6QVVKDiS16/vNUw4CgkYjSYmeni+ht4v7W0OB+Mx2maoEga49BlECmgERERERERERERERERVwmePQ+gt+41PGVECNasROsgk0ukYUOAt0ImlKhtuw8AAC5cOI2DYknQWYGqqlM4mErkfFuLwTtkXFXnGLINaXPlLVUW5KWHUVvPna0dhqoh+NKEt1inabqFhuk9dkmSoi3Wl4mWQN1WGxuZzzvkAKFM6SxBKtezkxIk1lOXtjAR792mUE8ABOu6MWaLt8eXaTqdbimTR/AANvm4jXOe6f/0dAX9jC/JATboTtQ5g7rtdvDMXnPdNbj55psAAEeOHsfGBlscP/lbvwsA2FhZw+vfzMHnrU43eBFJaThvFQFtubcWy2bREg+DLtAVT4jWeaBYZEUrWP1KW0H5sO1AHa2CYI+xJng2O12gXOT2arVS9MQKk6ZTaO2pt97CZZCALSUGSRBISZIMbbF6V5UJeZrIViDiSvM5/vjaYrFpeHcPHz4cKJsbGxtYX2fL7HQ6DXRPrWsLeVM0Zn6eraD79+8P6SGIKNxnMBiEvJH+79mzZ0P+wHvvvTfIER8+fBg33HBDuN+SBOETEVJ5p3aLxzTfvRPJqrwXt+/FZz/7NQDAyUcfC7TIwlm87rV3SP1xPX/fBz+Em6+9EQDwbz/ys7jQ46DubqUwFXEe61QI6sbA4czDbM28Zp94Za1DR8Rc1tY3sHyBxWGWlnajW4nH9EyJccF1d8t1+7BrkS3L8DST8QRjec++93vei//9X/48AODRIyewvM79YOXiKg6L4MsbX/tafNt3cBqKrni5KM+C4FXa6uKm2zk9RG9c4sIKB7zv27OEi1I+Y3KkYu1cFCGlTmcGg4Lrpj8aYSq5yowDHniIBWj6F9eQNJUxJBrdOyW0puBdaLc7aIv4lNmxI4hhTHqDkKO0L0yC0XiEkQS5b9gJxKkNNRgg8dZ+rQPVPUmSYC30FskiL9ARS2V3Zg5zQgGcn5/Hjjlur87sbJDwt1qHdzHimwdPR1eOQn4vALVIACnk4vEnRUGKPklTdISK5JkEcA5jyZOWaBUEj1LNVnMAaLda9VjuHHQQtqhFT2pCjkOdr62mg3IaCC9ioBtSGzKnw4V50DVyUGqlg4eMQCiF8kh1zgVM5HO/tKj8+kQhpAqAVjDeU1NZeM+bn6vImkZuNwXrJD+fM5jImD1ssHqgKNDrQmqqptOP6lQM5AwS5YVkKLAFrDOBOqlCokALJTlAHenArLFw2JB5pNOdw62vYLrn+fPncfocewbvf5Tp8tSawcGDPCYv7MgwK+8zNDCZepaBC/WfKBVSVoiKPoosQ+5FQ0yFhLwsfxf9VFg05Vqog6mMAcPhOIR62BLIc7435x7li1fDAXqb/CyTSQ+afK5GflhjE5RVHbrj7+FIhRQ2SiW1MEeC0PfWJb+jMRZG+sRkWgYv47b5ecy0JbXQeIx+TzxdZKCkAbs+F6wG2h1ui4N7D2CbzMfTykJPh1J7hFL60Ghikfl1odSdsRarwh5ZXlsLY/zs3AIOHGDWzvjWW7CnYk9vkjpk0hYtmTT3bJ/DzQe4D5461sPZc8wu2lztIRO69my3i2sOcijVNQd5zX39tYeQyiv++c99EXd/7Uv8rMZiJhOhEE0YiSjU6vomen2+T1cYQu2iwKElvu7B/QfRljyjvf4Ym2s8/2QguB08N3QXO9CeVeJTBSUJuuKFa3e6GE+4vc+cX8dYaMFa5bhD6vHWV94GK2kNduzk+TorUkymNWWgEFGrXbt34dxZ9vqura4gl3VeJ0/R8qkkhF3U6XZQCLV43B+EVGirK8s4LukoNtY3MBXvs7EOpafpyjjXztpYnOe9TZZmaMmeptXuhny8vc1NrElO6MGI+8lGb4RS1kyrG2tBEJHgkIo3WSVZne/U1SFpReY9hK3AeEqTBIW0/dziLGYlZKe88bpAC04TF2jtV8JzbAA9l6JWldTaQfl8MzYszUX1iCsuhYMpuTaKOVbGyTaHGPd44Ty/bR8mnr+qHKZeuSlth7waUzOGEppJ4NKjVnsip8LE4ZxrxAVQPYlY+wwlSudcWFwppS4bD6iIMCMbwLbwcNMkCROIolq1CZULA41zhFToiJWtMBaVJz9pOOdCks6tm+q6jNbaLZSqWlHpmWhe4+uBa9BvAAqbvrXlFXzu818AABw8sB/vf//3AAAeuP8h3HvPvQCAM/KyffkLd2NjjQe2N775jdi7j2mHxtaxgaA63xmokXdP+YHe+i6DUbkJt8YD2+zsXiyv8EBeYYpM+YTcfK4xDlXp46UMUmmjXLcC1S3RLmwI1qebQaXMb8BAAJFQAFAhaUv/hkIpA5SzGu2MB7ZWlmAkio5FUSvHNvM6+k1au93Gzp1eNRUhl9ODDz4YNmeejtnv90N7v/Wtb8W73/1uALwB9MYCa22gAi8vL4ccM74vbWxshH7y8MMP47TkfUzTNBxfXV0NlGOtNfZc5zeuXMY9iwvY/noe2HYsDvG1r/CAeOHcICyi9u5dCBuXZIapLgt7DuGVb/kuAMCdjzyM//ypX+J7FAkGkrdqUjpYoQ6jN8Z0VWjNe7gMo8kYs0LNuOuNd2H3Ek+Gd5/5DNRINua9NKhozee7ANlY2VxiOssJUslh5FKN973zbQCAn/v3v4LhgAf6+x54AIfezsdBGhPJhZSLQcKNpyF2NjdpeMcP33wT1mTT99u/80kksmCZmckxENqpmZM22RgH1drJaILHT/DG8dHHH8dUFBUPdmfrHGJAiGGt1VtVmBSqqoJPh9rpdJFJf3QLNij3lZKhuDfpY3PMk/JkMoUZCf10WqIqPS3VovTUdGsxEqrPWIxRqt+DXpdJJjuLXKihnW4H25d4AXTwxgO4URbhqcqQJ88eSRDxwiOouyneYgEc8xZyPJoybMxVoqE971AnSITC7BOfExyG3hhJLqj+FkUS4gWLTjfEB5mqBMSYF/Z/jVzB1ln4QDeeg+Uci0bS9Fo9s/I51YwL31tXK9QqqtU3syKHlnHFeH0BuLABJFPByPWyNAlzPSU60D4rYzD1MW9N4mojjCEohdsSlfX5gW2gs7qGOqWnmrEyeK1MqRpTs2sG8fv7QNWbYz/PWwWS93M8ncJU3Ebnzi1j+RzPt4t79+Mt7+KY/I21Vdx/Hxv/1jZ4/nzsoYdw+jSrKr/qla/F4mtuBwB0coSYvekYQcHVwUH5dpG1VpEkUFLP01EZnqWVF1BSB+WgRDIjFFqqN+ieDjocDkNsKRyFCbw0FSrJxzYd1crjk1J0EEYlnKvzC/qwIAcXjNWJUrUSuyK/nw8q1FVpQrsRCHPiqNg5Px+ERFdX13DqHI/Pg8EgGKC9TuikqjDbZcXtN73tLXjV7RxXrrMcCep+4FWcl9d7Qd3bq5tPylGgHm8O+hgJddHCYG2DN8Enjh/DxuZGqKY7Dkm5ZRPcSTUO7JANz3wbK2f52v3NHtZG3OatfftwcAeX75ZrmQ54w7WH0Jrj93dzrcTDD7Oi7MbqKh4TjYLhsERfyjwaTdCSOMidXo2/NICPaU8IAzEUbaytB32P9d4QKxe5/EpZzAud1ee0I6owJ8bD6645hFOn+bnPn13DshgxT586g7OnmBr6ihuvw4zkLVVCcxxuDlFKn9BJhnaLr7d3n8baed5XnH7iCQwHbAwZaAU1K7HO8mJfPHMhqICur2+gLxvzC8srWL7A1yhHVXDipNRYuyvfNxDUVJMsh5J5Oi/a6EiIRHemG6i5Q6GP90aD8Hk4GmMw8GEfk+AMq+BgKp9v24b3bCJzbWVNMLwmaRrONXaCk8eO8LNcuBUHJPwub7VB6bNvACMFNCIiIiIiIiIiIiIi4irBs5pum94x7+kAiLPUAyiNC4pyxtZUzapyIZ+Mt4jt3L0PGxtsaUCvj1wCG601gCjXobCw3mpgVaCUIlgWXYOF4rZ41oI4TMODZhsewKbYS/P7QKG8xAu3RRAmPEnjXFOXzVNRdePao9GoFoG55C/AXhhv1WyKiTS9f82yPh9c7lkuh+b3TW9gfzgMQixvfMtb0OqyJed1r389Dl93HQDgq/cyNfD++x/AV+9/CABwamUN734Xe65uvvkwnLQ9ORNyjlmyMMY/o3gYVCt4XMrJCJVYla85dBinT7MH6tjJU1CiXqW9DJS1WBCLUlF0gps9z1qYEWrAyZMnMezz8fF0hK7kY+m2+JnKsgrW4SxfCiyeAAAgAElEQVTXULkXBlIYjUSJ1lTBaqyUQuoVvBrtFvrJJXRi325EFKicMzMzoe69R6+q6nLccsstuP56UdmyNojKDAaDQBM9c+YMjhxha4+nlvZ6vUAdVUptoXp676O1NvS/sizxwUNbqdHOVVjb4Pr6iy/c37DAZqHfd/IUA1FZ27efRYSG0wESERXZs30vNk+ydfL0qSm8089MS1ihas50M8xIjqHJhD2YM0mOC2dO8bOubOLM6DgA4MEHvoplUWQ78fAm8g7fp7fcw2SDg9Hf9SbvAXSY+OD+cYWbb+QckG96/etwz5c5N+SRx5/A0zcyrblFCscl98+1GdOpSFvYNS707GwneMaryTTkyiSdYl2oL+PpFDvafPzkU08CAFaWlzESEZiN1R6GA66PJVKYETpl1RzHrAueB99/iiJDISphZTlFJdbkVqsd3s/SWZRilczl3MK1saNiL+5kOsFIvDplOQn50KZlBWO9MJaB8XmkjO+vNRU1g0Yq71yRanh3/WDUC/kxnXWBjhjxzYNnUzBdM/SmIDpkygqlWO7TtKblEyUg5en6wsSopihlzLZEEEM2C5bIPD0zO4dU1PScc3CegupF22pHJJy1de42Rw1qaDMHb4VEvFFe2ZFZpOEiIO+qUc25l4VNAAQPobE2qJcYpSWjJXtnSq9qMgWc0Cyrqgoeb59r0KGmSymqx3UHF0TcLmmBIArmQn44V3tErQl0cucacnVU5yCEs8ED6YtprYETMZJxNYU17DWYuAlmFtkbdWDfThw6fIuUldCaY7bJ45Lj9NEHHsCDjzAd9NjxY1gbsXfmla9+A2a6dW7Qcuqp51VgdnhPXqoSkKznyspCS7ux0qIcn46RTXk8UtKWeZrBq84plWAqFL/V1YsYjibyrAbDCXvCppMy0NpLU+fI8wyZIstDu7CgkG8vqllMBHh5R0/HLfIU1gmLwrjgRczzFjIR/BiVFpnkcduwNvQFP55ZYzhnNZjCmshaN01zpPK8pRmh9Gqd0z6GY36ugfw1ZZ1VMCVCmXphFIten9vl6LEJcPwoPO7YL88rA7G2Di3h5s6289pZbAxGItoyHQ6RSP/WzoeZOCwtMVPnusOHsLiN1yGPP3oCK8eZJTQYjoOwSJ4laMkcm0jOyWk5xLnzPDc/8chDUKIKvXZxBalXia1KXFjhtYhxBsbvFcRDSMYF1ejt83O47iCzAk+fOhs8pv1eHw/IevLaA3tw4w08v5cihFJZC5OKWmlSoCNzXkaEjlDWzaCHtdPsEa1We+i1PO2dveFKpRhLH5xUU6GCs5gbWb+GN9AS1qEyB3Jb8wCSSjD1bMFpiRI+H/gYbdm7GGPhpNxZm79fbM9ibk5EYsoSpeTSLMsxxhM/NxtMhZFjbRXYBp6Sr4iCWi8Lx3qGhMHaBns+z54/F9ozzQsQnn1ufl7cnSY90oG56QAwKQ0k3yjWNnqYneUOpl2t9uULD9IYltzY1XCClizUiSyoI+7eiUIpPPCcVJAQboiAbqVsuvr7OnZgK+2zGWsFbN0gNjdYSWOj19yQNTeOzXgvFxqnVmgsyzJQDJv3acZ4+fI0Y8aIaMtmtnn82Sigl+IboYPyvfnzth1L+LEf/1EAwNLSrlrq2xGWdvPk89bvuAsAsGPvbjwgk8xTR0/i//vEJwEA/be+Bne+lhfUOu2gko2EhYYjHvAGI5+sOAMqiTdJZlDK8eloFFJXPLA6QL5N6HXend4bYDLv22WE0URoCf2jWF/jF2g0HGGzLzET7QRWJjsvn15VBi1JJr40v4TSTcL3U+epIzbI/45Gw5BU1cduOudCn2gaJNbX10MaiPF4HFJ89Pv9Le3s4dv74x//OD71qU+F4/6cLMtCvF/TsOApolVVhXIURXHZftBUIyUi0EQ2I0J31aQwB54sxmenGK36uNU0vMvGUNjczEoKgu9+33sgYULorQ8x7stiLneYzXw8XYZMYkG2be/iwEG+9x5RR1t5+gwunOe+8esf/6WQZPe+Bz+PXYe47W98zU24T6hOwBgLi9x2I0l6W04mWJP4j3arg401Lr+uACNT8GA0xR/8/h8BAF55yw1YvsD0n5UTPBlu378Hc5JSYTqagdJ1DFJXjCG79+3FqRM8IdIqcGGdJ24nCx1bloE6upS04SQZratqSpJVFBYZlTNotSWeUdq7OzuHXCguzmokomyWtGcwv20HPC5I+cfSv7ROgFwmsjRFqxD1sMqg7fxGeQpnfR+s1R/L0vdHGxbKzpgt0v5DiXs5cvRpWBnnZmYXAm0/4psHL5OvnQL5kAadBIqcsy7Q1LRSIYG2Ig2SRS2F+ZNqaia5kNagrCpkQgHtzM8FAxhvJmUsCSWqNzZEGqDa6OnC3FzB+axQpt60+fhxU5XhXCL/H76j8ec6BLqfh9aqlhJtzM2Vc2Hct7YMGzZrbaCAhg0igNT5uUiHuHKCC4Zo3tTWYSdBnFLmCOcQ3n2lsCU2sw6PdCH5PD9fraTor1uGCIo0zDl79u7Cq17D6XHmugWs8SkEMuzby5T5xXluq+0LczjyNKtbHjl6Ap/5kz8EAJw+fRa3v4LpoPt370MuYS6kCOklc4YpK1hZwJeTCr1VUaHMcowGbACbTqaBGpq3RSURKsRSlsZiInL+o8l5JHpN6kaHsJ/K2BAD7cfbNMmRakmCrTOksoa0zsHK5tgYG/opGQrXsGJYhVJ16ixQiPcaD6eB97a52cNgVKuQ+4gnRV59Ow3xal/68r348pfYkFhV0xADS/KcAFP0VDBa1O9hJjTBJFFwru7HpqpXtraxwdDEGyQv51/BgaQfzMzkaLf4Hv0ekKVSB8qhkhRNfUlzMZ70wju5ubmK/iZvjqAd5hf4XV5cKtBt+zjfFDsXeZ3hY8I3V/tYWee2/+xffB4km46Vc+ewMMP9rTs/g5YYdWdmWsgkbGBVwiZgKhjZbK1v9jASums71di5TSiqfYfVZT7/sUcfR65rYyhXJJCJSuzszCySkVipyimGEopkhlP4xAHlZIq+vOM+TZGpbFjTWge0pJwFJVCZjBkKtfaI0uhI/L1XIW61uijkvUnTHFr6qWrPgmSNo50DSZ/wxtbSOLjEr9ssiq6k4DEGLZlLx+MJjKil68b76Kmg1lUhdQ9Zg0zosd0iRV4IVbY/xLT0BoAkbLyvhDhzR0REREREREREREREXCV4Vg+gz9FBVAurwITwcxg47NjBVLDxRKE0Pj9ardAUkhwrYNeeffy5KdRCFolYwHe15oPLFbYKZrGm56J2nLgtlErvRamqBjUAdcJtT7lrXs9au4XO10RNAZX8SdQUadDB+0JEtbdnMqmD2C8Rc/G/a5ahLnN1WfrmZT16V/DyXXr4WVMEXgFF0UKr5XM11m5vB8JILBo+B8rNt9+KvYe47U8cOYIj4r5/6u4/RrL+AADg8GvfjdklPoeQhOfxSVlRWfYCAlBuE+WELUO/9rH/F1ZMxWY6xngolrDSt7fFQJSrtEphxL5alQ4zXbZg3XTjrbjnq1/k+22uQDfaztfPWLHXZv3iJtLC00UMqrHketJ1RVZwMEIT8H0qCMqALbfeM7i2thaoDb5vACwU4+ug2Xf9sebvmmWdn58PHj6lVPBW+2OdTmdL//XlsNaGMk6n0y1e7myT30UlFvLpuI95SXr/gXe/C5Pe7wMAjly8GHIyTcpVDCdc1uNH2WuW2ALnTx8DANxzz9cwlOd9zesP4Tu/kz3BqS6RCPWlaGWYnWNv2voF9ij92aknMHDc9o8+/uWQPyidM7jzLqaCvP7tt+KNbz4g13NYnON+c/fvsbLZ5uoaKgmwT4Yl1s5yQLldvogbhfY7MzOLeQku7+StkNi812MLYrHRwuwCnwsFLJ9hi2R/dQ0rR9nr5546h0NOkrRnKaZCKQn1n2QovaKfq+B8n1UE5ylmqIUjXOWwsMjviJXxs92dx+w2Vl5LdIE8Y4tjkaWgxOf4ydDuejoU9xnnDKzI+2oNZPKuaq2DB6XfWwuiC1rXXua2jLtJomC8RLBzwXXR7F9KaSyLEm7R6oR8RM+TUBLxAsBTgWBrCzdpCh6ILEsDs0OBgleDOb5yXHsqpQpJ150xmIx9wtVxED9IsiLQ+S9lr/APa8+bauQKdg3qOdk6d68jwki85tPgpWvkEiRqhH5QsMo3HNJBGk5pVTNrFIUQBGtt8N6Vps7J6RzV4SVhkVF76TjUpBajqVlHJsyr1qngNVPhWrbxvujwrMaaus6Vrq8BW+dS8xmqXe1xTBIgET5ua34GXWECTMcjlF6dGhMo8bZuF4Gx7a+/E69+zW0AgKeeOoIH7uOwjeUTT+JBod2Pb7gVB689BACYne0GVkAlnrnpeAIr3q/JoMJTT3LYwfmz57G6znSzjX4fpWWvR+lzmDoXwjqMsagk7KPdzgOLomi1sOkFzHobQcDPi2FplQQaqTEITDCQqxXhTd3HoBDOXxW1ci6DeLItMByKkut4HMZZawxGQr+zpIKCpx/PHFRQl3/qqacxFlonUZ27Lc0KdEQZMy8KZJesJx1ZWOEFT6tpyElrbRXEP8oG0ULpBGnKgnojGYedtfCOsJ072rh2H89bnbTCRREw2zZH6I/YU3fyOIcjtFsawlrF3V/5LI4e5ePX37AD1x5mJsnOpTnMdrxacAkra/4NCQdZaVssr/J7ujk8HxQy09ThuuvY83zgwAF0Z+qco0bml6/dx5TkcjzGWNZUG/0elld4vuoNBmj58IUsgTNcZ72LqzhzkufbTkeE8IxDd4bbVm/fAZKcgcO1VWyc5bVITgozonhrpiV0cLQ33jG5BzkbaNdK28CGyIjgRATGKBVyNfp9QFbMoNWtPYAkFVy0ZpCFPMsEZ3z4heyJYCDNDW2mYbxNXAnKhCqbjFBVMr8nqvYCyrnVdIo0lbUpTAjD6BZ5WLdr7WCFqWOqMdLnmJKf9Wu/p7DWBhpjTgm08m5sB53wS91JCJ6bUTkTqCj+r3IUKhmwCHsjVyeZ13AgGcwcXEj82KR8NKlzlXvmMQMAIT6Btiy0ga3Kn02VxEuTtPuJrbkxu1xi7slkEjaXigilfaZCpP/bjPtL03QLFdXj0k1fmGgvUd2uKwTPwDey+ePf1XUjHJxGOfxEX6tfzUiy2dtf9Wq84rprAAAXjz+Mxx/kRfmX/+wPcOdbOTZwdvdhaMsvU0uoRMMRqyYCQJ6WIelqrzeClpcwz+oFqY8zmJ3roMj5JUySOvnxzp17sWc3L5wXFxawIHFx/Y2L4WVPA11KBSne0XAC57neSiMRNUmtHbwEoyMbJqpa6Wsa6ms4HAaVzWafAFgVFODE7JcunJpGgUtpoV1RLut0OsHg0NzEXZqIGNiqGFpV1RZKcrMvtyouk98wn++fwNo6D6RLO7v4mx96MwDgyfOnsCpqZYkxWL/AC4/7v/ZlAMA//cl/gknJk/k9934JBw/zePBtr9uG624opFBpaENjJ6hkc99Z4AHsDe94Bfad4EUFGYWFWX7u+b2zWNzP5ZxOzqLblQ2NUpgIzeH+T9zD9WUNihCXYWClfltpAZljoI2FlpjCkjTWlzelDYS7f3YNGw89BYAX1YMVLtNwo4dSNpfalNghcRDDqqrf8aAJ75DIwGQqExaxSutgILNwIY5GO8L27YcAIFBn2p0utCReVkhhKq/aWISYa5WkUKm8O20/pmiQp286FxYYzjk46RNJblBWo7ocXjFRNSZJ2VuoxhiQJDrQYJIkCe/AYDAMfdMryEW8+PDxfTAELWOUJRfGaUVJUDXkSUIW1K7ueyHzAKiRusei9Fy4qpYiJ6rHHdcI8WhuzJoxWX5PZC3qud64ml4FVnNuQmmNJOxTKbyXWiVho2ScDdOSP5YnKZLUx+LUSclNVfnsBoC1YYFJikKcT5Nq5TcgSlFN4TMu0BwJGlWgs1KgAWro8L2P+1OgkKgclNRjr6k34Anpen0kNyFtgwHMKRvSzGg4zEpYhGu3ochrL1SBSquEHpl3ulgq+Nzt23bj2n1sYDp+9ElsiIHOmRLra2x0KzId5iPfylNThXbLigROFvX9QQWfeTsrskCb9HFM1KSDu3qeS5KklszvdDCU87XSDWN7I8beb5KtDeu85nhErpniqw5A7Q+88roLca3TqgwbvclkGuiIzlV1Yu00gRYlap8ygrQKVGAzrUJcdtbOUbT5WfKsQC6rbJ0mQb3Zx+8R1VoWpqoV0itDgcI6mTRo0sog8dTEkufPgevBx1/v2VmgeDWvcdbW5wNFdb49H/Qzjp54DACwunkGpeW5e+3ck9izk8v0ba++Bodv4LjRdqumihMlqGRe3TbPfXrHthTrPS7H2rBCOZGNi2rhwH4ux7alRSS5VywHKkm19Rd/zuUYDkZBKXU8rTDx6r2K4LPJ5EkKLUb44XiI82c5bs+v14y1KDIOeVg7ewqJ5TqabA6wJvTkVpqjyiWMSGn4vCzeVJAaFeITS7JByyRxDhBDRUkEV3nFZIPu/DYpn6w3szoOn5IMJGNUkregde0UspLKJfW7UFvB+c1gkodNnUMJUlznSidw8m4pF0aPYIxKdAoT9hplSDXSStsovNGishhJGha4JKTEuRIiBTQiIiIiIiIiIiIiIuIqwbM7CD31ErjEc1ELsXgPm1IqWAubqlfU8DoEp98lHgufLY6AkC+Hz2laeJ6Jy6l5GmNq6xFtpY/6v5f7rLXe4l251HOYJHVOIWNMEPYoy7IOOq+qLeqPTc+f/9vM7RYoM5d4AJv148+vUedbahh20dDE+Uug4fVzW482ldwAQIPgDbhT65DPsDfwwCtehz2HOAntmaNPNIptkJMEyYq7OoNB5pNW6gnmxPOjdAsrFyUvixmFe3slr6oCNsbsgWBKJNdRnpf4ypc/G4rp1RF3794VRDeyQKXUGIp6aFZ00G6zxRTKoZPysxiUmEpOt3G/Xyf89YpoRbGl33lqqC8Xl7nuY1VVPcOTl6bpM1RZga3eYk6G66ldWeibtuFtbooV+XtPp9NAB/Xn+euVGXu0LgqtBe0pio4I9uSbmJvh3926owudct1kWuPpx5haubF6HADw1ON/iraID7zhbXtxw21MLdl/OMVk4nNLFahE7MdUFaDFGyWB10v7Z7B0gOvcTS0yMQu6AiCh6dpqitKKSpzSQWggFxO/Ix3y28EieAQIFPIYkSY48YgOzpxDFgL2+X7jPEW/IW4RlAyJ4LxAQYVgOSyg0NBOkXI0cpXq2vLPVD2hcJnaO6wSjW5Hnl2OaVUgk3PH4wqlZJhN0hwz/h1RGmnuhVgWpW6nQSShLMuQ+3I4HCAv2DOrdY6JiCbxY/s6q1XJnK+XRr5TpQIRC71eDytCAd279wDoCuNzxIuHeo6oc+c56+rhmwgK3jvdoFNaCu+O925wd/aUtSRYzMvKBiVcBdRiPw41E8TTmhqJz3n+994vFcRQ+Lj3ADq0JTl08MI0hDvg0PDI1OEDTGWWd1HGxEQnQUCkMhYmrcVlUmEuJVkZPIME1Enfff+m+hgRBY9+olXo3dZWwatkKhPK6svpbCMvb4M+TVTn+7PWhnUSC3/44744KrSbddPwO+UoKFKSzoN3Q1kd6OQk7z6cQyW0WpUo7BYP4PbtS9jwoifDKfp9UUKdDMNC0AtmadLQOX8udIbE9w8DJH4sNLV3zgZVdNXwKlDIA5imeRCcGo1GwWNorAu5k3WTYeWZWajZWKAGLY4su9TA/d6/DX5NYkCopENOjQ3UxdLYQIffmt9ah/sHanSiWNUUQNKh4J2GVkhUY57w6z9j4ekTpLxwjQkviXW2dpOT8tMSrKtYFAlMTfROo0p5cZwEmbA9ZloO7Z3MtNi/s4tMxvWMWjh/gc85feIYAGB9+SxWV3ie3rd3Fnv3sQDQK+/YjyST3LG9zXoubWcoZD1WCN14YaGDXYa9X4OxxXTsx4k0KLXmqUVV+VAXDSV05vVNZtvYyjGNGyyskqR+XUBBzdiBUMq8uLk5gPEeZS8oVBqQXKNINFLvza+qMIAouECztFBhDPIU3IQUvDgV91Jf5xoWPqzHBbajSw0SyVCgNdeBzlogEcjRaVarIKsEfjultGqIAEnoESk48eZbVHBWznW15zm1aaDgkq1Q75ik/6QZpqIe6krywreYmkmoj6kpMRzxmm44GqIIo9fl2TnPKw1EcxHrGhx1/50/3lTarF9aP3C7LZvB8HvUCx/XkEe/UnnCZAegIa68lQbaoCnecMMNAJhGB2yV629SOZufm5u/ZtJ4v+CeTCZbFtOhwzi35dkupYD6mK1Lz7007m/LpjrU7zOfs7lJe2E2gF8ffAckBI82xtBQBS9o997y2saArAAtvGhZyCetBBmJaudgA21RYWsnbXREWrIydb2nEv9krUEig/TC4gKs8ZufeqEwGg0xbdBMLo31VEqhK5SU6WSCAYTmmGsM+ptSzjFKUWWajEeY6dZ9yMO3fZqmyCUJvTFmSxt6Cmie5+GcZr/yx5rla6L5uyZl1N87y7ItfdCjuYls/rYsS2x0Ob5tOpLNWMtCJZ5COYUpeRDJYcOmOU0Jh29mSsQ2SRRrplO05oSqOj/LEsBgSW8/yStVJ/CFVvAZYqyfAXUJnXhjCGEgNJNxv0Qn4+culPbMREkSLWX1KldEQS0OZEF+c2yrQNFKoMJknGU5lF9Q+TEABDOp6SmQTWRVVqGvW61CzCkpFeSyff2bqh47TCMJNhqLc+3q990Qwfg4TeuVakcoJO5vPCZoqf+pqalMed4Om7qpTxRrgFxoKHmWYgR+tzozdQyprRyyzKfgqUL/nvqdrCOQp7Q1M1mThRMa72QyblBA+xiL5PXOXdsR8c2BlXfZEgLN0aGCRW342NJ+AucqQOjufmtjgaBMSQ0VPBCFFAiOLSLheODUNy2QnhVqbfidbaQ6cFRHc0wdsG1JjG6yZdDNlEvWhbgurXTYlFYNVVEdlE0bcf/GhHRBDg6pnJsbG+6tmkbUsEypN2OqEX+Ypmkw/BlThuTXzrpAByVRkAYV4Xui+t13KgmbDueYrs71YcMGVDfUQH1bOGdCbB6UDrRPpVSdZN6GYgfKLFEjxhiAcX5R34JOePFaFSPkkl5pPC1hSz+ncfnTxsZXK6rrAEAimyJK6gWku0yMJhxBJz4OWaEUiXuUtfqwUjqk+6BQBxo1OU0FujGhEffXMFrA2ZC2SGUyZo9NUKuHokCHVrahiKtq01WSJVDeOJz5+S4JcYFFnofN4rSqN/E60SAZ/0ip+p6BCmtrxVBrQ8oQRzqk1tDGIBjiCBj3t6r0qqmD8mkFMoRKt1ahKGRtSQqzXf68ezevv9b6AyhRntyzdwGZGHJzGmMgm4RxOQFJGyXTCRKJKwtGWOfC5h9aoZA6Ki3V6uC2gpFULmVVwZSeZul7RwUXqNEID6BUbVRyQNikj6Z9TEqZs30aCEdQspgca4XMjxlE0MaPBwnIl5sQUtH5tcKEKlTKG8LqzbixBlO59qQywcA7hcZ22Qh7irxR9XirCLDar88tjGxgSdWKyCHNCyH0QbIq8O8daseGckmIBSaqU5f4FzzVaXgDpq4EQgoKDat8epccRkJspmYEVfoeLkrjlyBSQCMiIiIiIiIiIiIiIq4SkPtmu40iIiIiIiIiIiIiIiIiXhJED2BERERERERERERERMRVgrgBjIiIiIiIiIiIiIiIuEoQN4ARERERERERERERERFXCeIGMCIiIiIiIiIiIiIi4ipB3ABGRERERERERERERERcJYgbwIiIiIiIiIiIiIiIiKsEcQMYERERERERERERERFxlSBuACMiIiIiIiIiIiIiIq4SxA1gRERERERERERERETEVYK4AYyIiIiIiIiIiIiIiLhKEDeAERERERERERERERERVwniBjAiIiIiIiIiIiIiIuIqQdwARkREREREREREREREXCWIG8CIiIiIiIiIiIiIiIirBHEDGBERERERERERERERcZUgbgAjIiIiIiIiIiIiIiKuEsQNYERERERERERERERExFWCuAGMiIiIiIiIiIiIiIi4ShA3gBEREREREREREREREVcJ4gYwIiIiIiIiIiIiIiLiKkHcAEZERERERERERERERFwliBvAiIiIiIiIiIiIiIiIqwRxAxjxgoGI7iKiU43/f5iI7no+534D9/oFIvpH3+jvv477/KXK+WKDiPpEdO0Lfe5zXOfDRPSrf9nrRERERERsRZxHI15oENEfENEPv9TliPirheSlLkDEty6cc7e+ENchog8B+NvOuTc3rv1jL8S1X0hcrpwvNpxz3Rfj3IiIiIiIlx5xHv3Wwov9fET0YQCHnXM/4I855977Ytwr4uWN6AGMiHgZgoii8SYiIiIiIuJZ8HKcK4lIv9RliPjWR9wARmwBEf2vRPSblxz7WSL6Ofn8I0T0KBH1iOgIEf3os1zrGBG9Qz63iOgjRLRGRI8AeO0l5/5vRPS0XPcRIvpeOX4zgF8A8AahMK7L8Y8Q0U83fv93iOgpIlolot8hoj2N7xwR/RgRPUlE60T0b4iIrlDmF7qc7yOirxHRJhGdFOvcs9X/cz3Hf0tETwJ4snHssHzeRkSflHt9hYh+mog+d8nv/bkfkXr4PXmWLxPRdY1zf1bKu0lEXyWit1yhvAUR/SoRXZS6/QoR7Xy2Z4yIiIj4VkacR1+6eZSEbiptcA7ALxGRatzzIhH9OhEtNn7zZiL6gjzXSWIvHYhojoh+hYiWieg4Ef1DIlLy3YeI6HNE9K/kOY8S0Xsb1/yQtG1Pvvvgc7TD/0NEv09EAwBvJ6LPENHfvuR6zfn8ViL6lLTVeSL6B0T0HgD/AMBfl+vfL+eGa0ld/EN5ngvyfHPy3SFp5x8mohNEtEJEP9m4551EdI+0w3ki+tdXaoeIl2+nvPUAACAASURBVAGcc/Ff/Bf+ATgIYAhgRv5fAzgL4PXy/+8DcB0AAvA2OffV8t1dAE41rnUMwDvk8z8H8BcAFgHsB/DQJef+VwD2gI0Sfx3AAMBu+e5DAD53STk/AuCn5fO3A1gB8GoAOYCfB/DZxrkOwO8CmAdwAMAygPdc4flf6HLeBeA2Of92AOcB/BdXuPfzeY5PSdlajWOH5fOvyb82gFsAnGyW55JzPwLgIoA7wVTw/wjg1xrn/gCAbfLd3wdwDkAh330YwK/K5x8F8Em5pwbwGgCzL3U/jv/iv/gv/nup/iHOoy/lPHoXgArAv5DnaAH47wF8CcA+OfbvAHy80VY9AH8DQCrz3h3y3a8A+G0AMwAOAXgCwN9qlLME8Hekff8bAGekTTsANgHcKOfuBnDrc7TDBoA3yTMWAD4Dpori0t9Jec6C5+ZC/v918t2HIfNz47fhWgD+awBPAbgWQBfAJwB8VL47JO38H6TeXglgAuBm+f6LAH5QPnch/Tn+e3n+ix7AiC1wzh0HcC+A75VD3w5g6Jz7knz/e865px3jzwH8MYDLeocuwV8D8M+cc6vOuZMAfu6S+/6Gc+6Mc8465/4T2MN15/Ms9gcB/KJz7l7n3ATAT4AtbIca5/xz59y6c+4EgE8DuOObUU7n3Geccw/K+Q8A+Dh4wv9Gn+P/lLKNmj8kpox8P4B/4pwbOuceAfDLVyqX4Lecc3c75yrwBjDUiXPuV51zF51zlXPuZ8CT5o2XuUYJnjAPO+eMc+6rzrnN57hvRERExLcs4jz6ks6jAGDBc+FE5sofA/CTzrlT8mwfBvBfEtND/yaAP3HOfdw5V8q8d5/MqR8A8BPOuZ5z7hiAnwHwg437HHfO/QfnnAHPt7sB7GyU4RVE1HLOnXXOPfws5QWA33bOfV6ecfwc534XgHPOuZ9xzo2lfF9+jt94fBDAv3bOHXHO9cHt/AHaSpX9KefcyDl3P4D7wRtBgOf7w0S05Jzr+/4c8fJE3ABGXA4fA1vDAB4cP+a/IKL3EtGXhHawDuA7ASw9j2vuAXukPI43vySiHyKi+4SCsQ7gFc/zuv7a4XoyqF0EsLdxzrnG5yHYevWil5OIXkdEnxYKyQZ4IrrS+c/nOU5e+iPBdrC37uTzONfjinVCRP8zMUVpQ55z7grl/iiAPwLwa0R0hoj+JRGlz3HfiIiIiG91xHn0BSrn1zmPAsDyJZuogwB+q3G/RwEY8GZtP4CnL3ONJbBHsFn247hCfTjnhvKx65wbgD2bPwbgLHGoxU3PUl7guefrJq5U5ueDLe0snxPUG1fgyu38twDcAOAx4nCP7/oGyxDxVwBxAxhxOfwGgLuIaB/YgvkxACCiHMB/BvCvAOx0zs0D+H0w5eG5cBY8aHkc8B+I6CCYcvDjALbJdR9qXNc9x7XPgAd4f70O2Ct1+nmU68Uu58cA/A6A/c65OTD//0r19Xye40p1sQymvexrHNt/hXOfFcTxfv8L2Iq7IM+5cblyi8X0p5xztwB4I9gy+UPfyH0jIiIivoUQ59EXrpxfzzx6uWucBPBe59x841/hnDst3133zEtgBezxOtg4dgDPsz6cc3/knHsn2Cv4GPiZL1e2K5V5AA6t8Nh1yfNcKaXT19XO4GeqwLTaZ4Vz7knn3N8AsANMsf1N6ScRL0PEDWDEM+CcWwZzxn8JwFHn3KPyVQamAi4DqCTg+V3P87K/DuAniGhBJsS/1/iuAx60lgEOkAdbBD3OA9hHRNkVrv1xAD9CRHfI5Pp/APiyUDa+XrzQ5ZwBsOqcGxPRnWBL8JXwDT+HUFA+AeDDRNQWa+M3uhGbAU8IywASIvrHAGYvdyIRvZ2IbhO6zCZ4wrTf4H0jIiIiviUQ59GXbB69HH4BwD+TzSeIaDsRvV+++48A3kFEf42IEmIxtTtkTv11+d2M/PZ/AvCcOXCJaCcRvV82RxMAfdTz4nO1g8d9AL5P5vPDYO+bx+8C2E1E/wMR5VK+1zWuf4hErOYy+DiA/5GIriGiLrid/5OEgjzXc/0AEW13zlkA63I4zvcvU8QNYMSV8DEA70CDtuKc6wH478CD4hp4EP6d53m9nwJTDY6C4x0+2rjuI2Bu/RfBg9dtAD7f+O2fAXgYwDkiWrn0ws65PwHwj8BW1bNga94Hnme5Xuxy/l0A/5SIegD+MbjuLosX4Dl+HEzVPCfl/jh48vl68UcA/hAc8H4cwBhXpqfsAvCb4M3fowD+HI06i4iIiLiKEefRb/I8egX8LLiO/1iu8SUAr5PynABTcP8+gFXwxsvHvP09sCfuCIDPgdvxF5/H/RR4s3hGrvk2sEjMlZ7vcvi/AEzBdfTL4I0qpMw9AO8E8N3g+f5JAG+Xr39D/l4konsvc91fBLfHZ8HtM8bWDfqz4T0AHiaiPrhOP3CpHkHEywfk3HN5iyMiIl6OIKJ/AWCXc+6HX+qyRERERERERERE/NVA9ABGRHyLgIhuIqLbiXEnmDLyWy91uSIiIiIiIiIiIv7qIHnuUyIiIl4mmAHTPveAaSM/A85hFBEREREREREREQEgUkAjIiIiIiIiIiIiIiKuGkQKaERERERERERERERExFWCuAGMiIiIiIiIiIiIiIi4SvCsMYCf/cQfOQD4+Z//v7G2fhEA8MY3vAEdrQEAxkzRXpwBANx882247777AADHjh3DbbfdBgC4996vAQB6vR5MZQAAP/fv/x127OE8lFolcJZpqLYygObcnhdOHcEXP/1JAMCktwwAGA8HGI25bKOJgXXT8Bh51uJzJxNsbGwCANYmFj/4d78bAOCZrlprKMWPTaRBxPez1sKYSj6XsNbItfl7pRRI8ms6KJSWj1OaopNwHfzZFz6Pxx94EABw7tgpvOauNwMA9uzhHKg/9wsfRX+DFXPNcIBxnxX6p+UYDnzvshzh29/7XgDA+977dtx8zXYAwHu//0ekPAqArssmz0XWQCnez2ftDN1tnD9054EF7LmWr7GwcwcA4Prd12LP0m4+d6GDc6uc19RNRpjtLgIA1tf6yFRL7kI4dfoEAOCJx5/g5zu3grMnOW/ohXOrKEecCkYjg5ZyEBk0KcY/8AOcumc04jpI0xRlxW24sbEJOCl/VmD3fu4fM+02to+4blbW1wAAj/ZWYSVlzf5du7C0nZ/vwQcfhJO6abU6kKaFUiqUwxeHqP7snAWntfGfnZxD4bMxBrOznArvPQ89AgAYVG2MpCnak020DPeZ8zMW3du5/5fv/GG4bfwsi8k6lOJz9Jjvl1QO1YT7QWktOvNdvrdWoW9qreGkoRUIZsrPbqoqPJ9/VueAydS/FwDk3XKVQSvltEPWVDi5wqrjWcptfPjwdRiPBwCAs6cH+L1P/ikA4Oabr8c73vVmKd8mKjvke0t5tMrhHIV7l2Nu2wQ5XnnrWwAAy2cHuP/ehwEA/dEQEzOROi353FRBy3s/HG5idY3HmnvufgRf/PxDAIDJ2CDLuLL3792OzU0+598+wP2SpL08HLieE+ugbQoASIfnkH36DwAAT33hM3hwkxW4LfH3mbFIraQ0IgcrbUXIkc7x+6L2bsdCm9+tfdfejvS1b+DytVpyDSD1WZGIYH2RCEh8exKgJHWSIofpo1w34zEPbtZaTCuum1E5hQ1lAnxqJ2UdyNb91P/1Y0CaJo0+Ub+DpAjTMLZx3+LzuQ7yPEea8PhYGYOJ75tlCW8vTJMURVHw82ZZeJHe8dbXP59E1hEvAH7jF/+NA4DP/emnceL4MQBAUeRYmpsHAOzZvoS5Ln/WKkEpY9OR02dw/CSP92fW+gCAqhzjmsP7AADf+30fxNvu4vd2bmHWD8kYT0uMZPy9cOYkTjz9NF/j7BkAwGZ/iOmU+2+iASV91gKh/w7GI1TSr1Ve4J3veVc4HwCUTpDKGKWUBhyX2VYVyL8vCiDLhXJG3gtnw3xtKEGSFXLvBIMhj0f3P/woHn/6KQDAdFqP5Urz/S6ureHoKX6W9bUNmIr7/aSqUMl9hpMhbrv5egDAD37gh7Bv3yEAwPd/33fydaFAWuZBcmFM00mBbs6fDx24Fre/6rUAgPntixj1ecxtZzkA4KYbb8TCErfbeDhCb30VANAbDuAUXzvLc5CMb8NRD4MBr3f6Gz1pnxWcu8Dj4/kzy2E9lKgEpPhdHVuH0YTvDSh8xxs468G45Lmj1eoCuTR+peFT1uVJgVabPxdJDpXwuKH8mKcstCwrLRSsZJYbTgaYDrl/lNZCy7oqzRNkms83crtyYtAf8TwzHo0xmnAbVtaFsYZg/dQG6wiVtP/7tnPbp8tDmKmRuk2Ray5nmQBzuw8DAHbefgfmr72B67fdDeO9TvmvogQbA+6v6/1NIOFrF+1ZpDl/hmuM8Q4wzs8Z0g+0CushBQsnz03WAZavnRJCe5IzeOSBP5D24j7Y7WRI5SWpKoXpmI/3+wOksu7dt+8apC3Oxz4p+fvBaAwj732eF+h2eZ3abc3ASOX1+n0sr/D6ure+irGsHfw7a8wURd6Wzw733M3r+XvvvR/LK9zfRpMJFHxfz1C0uS//8hf4fTJaQ2kl3wPSxVCghJFnSdfP4PwjnwEAnH/8Hgw3OcXgVNY448phKn2snFSYyjMmSqElz72wfT+W9nM6yX03vxq0sIufveD+VRQpoLgfGEWopG6UApTMq6lO67WWqzA4xXPzVNZck2kV6tdUVTjXGAPnhyNn0VyJ+L9hKUAASZ/QSiGROZiUgvJzuVLwe4+wjiVsua4/TiAo5d8LhLYgIjjph9/57ndddm6OHsCIiIiIiIiIiIiIiIirBM/qAeyLdWpubg6nz3Ae6IcffBALHd5xHzy4F3NiSVteXkYlu3UAuHCBvUMXl88BAFKdYHFhAQAwWL2IydwSAKDdnYH1ZiJNcGJBOXnyOJKELQk2YwtEf1xC52x96mRAWbIFs9udxWDAVqKcUsxrLpPtl8HriIaVHLIrJqJgPXfObfES1V6NegfvLY4WQCmH260OMrGYHzt2HJ05fsZ3vOsWqG4u9cgWuFtuuA4XzrEV1ZUG4yFbuUbjUbCerq1dRJGINQIGFMwKqZQjg7cvGFeBEn6+nXvn8JY3vREA8D3vez+Q8e9OrD6JoWIr4uISl+21h9+CxS57Ax87ex9yuffGYA3TKV/7+mtvwrycf3b1CayKt/XmJbYUv3nxJpQjroSnHj6LJx46xZ8fOoHVC2y9ccbW1g0i3HPPPQAQvAqdThfnzrGV6Pz58yDxzKY6g+6wJyzXCW4Y8H1GYuV7qmUhBhvceOha7NrN3pm777479INWqx28Gk1s8RI1vHtE0rbe23IJqqrCzAxb0L7nTa8BAOweDTHsneVrDdbQ6nM/nhlZTC5y/18pS1y03GfLSY6FFt9zvsXPmlUlHn2IvVxprnH7zlfz9VTtAUyS+jUlIhhwWxip2yRJ6r6L2rpERA0PYIVCrOuwBieO83t5+hRb/0xFaHf4+23b9uKOV3I5nnr6CaytcXumxQSbA+6/Ssqk9QSKxGNUOWh5bw4d3O9fM5w9ew5j6WNpqlGJ3aksuT0nk2l471fXLmJ1le9x+vRZjMb8jiikmMr555dXMB6z9dF7vMLz8qfwvhuyyMXDkK5dhJXxaBaAFuvjWIknFQQt9aidAZF8riaYWeV3+OLyA1iT8m9bHqC49XYAQCb9VVsH0vVYE6x0VA+2ighaOnCiAN0qwjmA9Ed5D51W8FZEHpekTCCoMDY1WAoNO5//zN47RpZmSHTeKN/Wcc4YE94BIgrvUJqmgPMe6ST0SWvtFd+ZiBcPCy1uQ1NO0d/gdwSlw2q1wd8XLSx2eLxqJxmmnqmQ6TC/ZNJP+z2L0bqMm4XG4jx7oNJch/nTwsEo8RYnKZx/76RPpDqDTbgfFFmKTHN/qozFcML9j3QFcXSglWdIxG2khLWhQDXLxtrwmeBAfnwz7DUHEKzbzlSBDeG0gbLcN5MiBcRztbDYRet0S343RVrw+9pp87vXn0yC9zHNM1i5R+Y0nIyzmBCclt/NLyCX3xpxASWaUE34AcfaYLbgdVLamsE1B64FAHzHW96KO+68k5+XKhwV7+3Uz4mzXbTEGzhaX8f6Go+FY2OxuJ3XTNu3LSCVultZByphVFDFx/IDKQ4dZNbRoD/BOVlzrG8OsLrM89WFlXOAsD8sDPobfM45uR9UgeGE55nNzWFglVCVBG8aKQ2XShtV0jdUitA1ElV7A63FUNY7FQwK8fq18wJaPqus7geQ+q9KAxu8IgQtfdBVRsZGHo9IxsDrd3Ad0cwMRj2+RpIMUQ743mlVoZWJR3RzhHIkc1A1hM34uZzide/6YIDVVV5jjscVtu3aCeD/Z+9NmiTLrjOx7977Jp89POYpIyOHyqqsCYWxQKBAgACbbU2ym1M3ZVqwrRcyk5kk004L/Qpt1WbaSCa1ZCYthG6S6OaAJkgCNQCFGjIrs3KOzJgj3MNn9zfdq8U59z5PNlDSQlYb+F1Uenk8d3/vDmf8zneAKKjCqELmeT7fk5AOlaFNkZFxqA2jHNJICgMhrL0DaM5GKSkx6pO9Nmb7e3W5gZXFRQCE0DCcgep3LuCV6GytbaRQjGCzSJ7JeIQstWckRymk/YhAujXy/QiBN5N1d/qK71Mbd61UHibTYk9knMqjrD59sOQF8Hk+pqD5VAJg0w4ZgIy/L8l9eKx7RZ5Ad0k3J2eHmDAKiMFfyGUAwTrH1wqeRcxlKfJRj5+3hx7Pb2tjDQs12gse6Lm19gB+VoNCT2uhITkzKHzfIRLTnPJpbm5AaAXnr+QazsgRCkZZ2SWd3W4zhELAJQOlKPKBpH95H0O59w0MhP3tGf1v7V5oMaPri2sBASln7I+ZnOEvGp/pAI44DR9EIfyAJi7PUjx78oRuWGr4bDQGYdltlH6vi7U1cjDGA1qcLE1Qr5MjN7g4w2SR/n5w+BS1BjlsC8tLABsT3e4F1jfI2eicsDHnVzBhqKEnAKFJUIZB4AySJI5REfQ7jaXIJWAtdIAMFt50wjgDMs9zZyjpPJ85CBYiatxaa6ORsGIMShopa7VvffcfIYoaAIC008bbH7wNAGgz1KZaKWNc49/QHkTIsLEsQ5bSPVVbq5hk9JvHZx3sXCJ4o13GOBuh1iLFc+l6C5tX6PeuvnADr1whx+R73/lNrK5uAwCeHj3C9//j/wEAODki+OZJ2Eb98gYAoNfu4JCVUEXUUYpoTrcWr+LuI0p/H40eoD8+4wmhuU21jwY79L/1j38D/+WfvAwAOD8c4i9/8EMAwHvv/RQ///n7AAhOtrS09NycTqdTBwmq1+tQVrEIH3nOBmmW4VFkYQx0b8ulGuxm6w6H6D8gOFJzcQlRmR0N5Tl4m9bPwzqB5x1+wH0dAPOcQWtfZ1mGMkP/Vn+dYAa7NYH4goTW+a2HGP0tQUM3lI/elJ6rnyY4OyXn+NHTO7i6Q3vaMFTr7PFDfPjBzwAA3/r2ryFg4ZlBuvv3lHL7WAoBeIWBDpAD+Nw9+4XTbRWSEUAYWghoAsMwqpMjUjZHh+dYXuXASXaB5WVSds+e7eOjD+m51rYiZIZhirDnIofP0JgkMSgpev3qtTI6bXIce/2Bg4ymaVJArS3ES6cOsjmZjPHJ7TsAyAG00AxjNEJ2RjzfA5JiHd2zzgR5DGPXtBSQbCDJw2eYHOwBAJZKIZYZ2jIY0H3mSiGzcAujIViQTmFQ53uWyQBVViILOkXEQiG2028AbRWmKNZIAVAOAiqgrHMJg9TChvg9KYVTolroGThOTl8KIFCqcO4tZEUIdy3NSwFvNzPOooV4amOe/257xYwXaeWjnIGkGGOQsEFojHHwnvn4/IYLUOaA1MX8x2Pa693uAKstMhqjWgWC8VPVwEe9QnK0MmSYnQ8YYWF9EoKDfaNhAsOGj840Ej4DUamCZmuJf5/3GHLEMX2fMARlA4DxZAqMSP8ZAF6Nvq9aq0CbGdw6AIgM2uLpjHZwKGN0sddN7i63sE9pNABrSHrIc7r/NDGYxnRxrVzF6grZHE+enaDdJSeg26d7OzvvYMIQRQWFgGVaKgwiG3SpeTA5vX98cuGCPkHAtkcmkbMTVqnW8cbrBPX86pvfwtfe+AIA4NVXXkCtTrK/0znENCHH+4KDXr7nI+cz2R+O0ObAkx9FKIfWaa052Gx+dogR2xeahdDlnS289BIFppoLDYxHNB8n5xd4eI9k+V/84Pt4+Gyf7xsA694p21ejSR8XvD+GwxgpO4NKhA43poR0FqktwRF+CmUvMAa+z7JGSuQcNc+yHCnDLKe5hmeN7zEHxfzCbJ0xxWCkhoSF/xqAYW++H0DxhLxyjeyQsqwg4bUfXJzg5PFH9Cz9CfSQnIus14Zm2T8p15BxScLjA0p2PHx0H8mYnntzZwdb7FRXKqGTiwLSVeQYAeR2b1o8q4CD5yljnGMgjIFU1kGVTp96noJi3dXtUJBTZTlqJQrm1KMqMk4GXPQnMAO65/XtKRZ5f4zZ2e1fdKw5jcgvI2C4swacM5JlubO1IBSUtEFzuodhGmN8wY57Dpwc03yNhlMXwBWQLhgjJVDh4JQUNnkhnLyaQfFCekA+pvtPh6fwcvqdppc5BzTnwHEuMvigPeaXFPinkSQavmRHtDdAb49en600UVskG7i8QGVNuVcu7BaRwiDk20ughbUXDGLWhZlOi/Xi5/MUnG0B4znHXQjtNqrRQIEL1nYKHESUggJW10tnZ5ANY9dCun3v7BqYQjfLIsCLGTuWjqS9vgCg/rIx19zzMR/zMR/zMR/zMR/zMR/zMR+/IuMzM4A2EjWZTl2GLYwijDjydnSwj+oSZQ2uXXsB165eBQD89J130OeCccPRuCyNcfj0MQDgow9+htV18s5/+vaPcekF+txqNkaF4Q9+4FHRL4BLO1S0O3p2gFKDogAlaRBIip4JaEiGEaRJgtTCQVAQIWibmv8HcKXZCHiaUDQiy/RMGtXC6YrQhTbGRU0qpRpSJvS4t3+G3pCgf/d/+jZkSNfsXn0RALB3/wFOOKqTywpizn7FcYwpp9al8HDyhMgppuY2Ll+jqFPG0dfNrRX8/h8TSczWbhWXdqnQdbF5BYMuRUs+/vQddPoErWwtruLVG5QZPPsJFf+vtBYATZnZ6XCAiqJI8as7r2OzRZmfmqji2jLBVoQ5RPeMnmu5Ret2eeEmtjfo7xsrO1hsUnQ1eKmEf/Tt3wYAPH72AP/df//fAgD+9E//g8uw2n+fPNlDf8BRLqkghC3KzSEtdEQY9AOGR+T0fGHXOLIgSOmiccYYR4+jSmWXpZjNgFhoxmzUhOIkFpoh3L6HMe61MUW2+OUrm/RdYQq/Rft/dG8CCMqSRp5GmtKdlJHCTyjiKM4e4Z0HRBIkOXJ+9PA+fEFr6/vfgOfZ/ea7DKD9F+CoDsN+LQxLKfUcbM9GRoUwLiSpYaA8hmt5Pq5fJzKDTz5+AgBYXdnA6hpBJS46IySpjQQqHOyfAgDWNq8iChlexf/meezIe8IwwuYiZTirpQb292i/tTsdTFN7JnMXGbRnLM0yZJklG0lwdkayw1M+fL+A5kZMPqA8gR3OYP+iDCAEIPVMVC1l8oFHD1BJCB5TW1jATpP2/f45nbdcJEgsfFNJB2dNPYm2shlpgcheMzpHeE7nTDRo/yeQcLk0KSAtbFIIlwH0pCjmAEX22UaxPQGEkjMyee6yc1leED0pKYk0C4Vs0FrbREgRpcQ/2D+ikIVmJj44mz39Ra/pcYr3iwi4cBHH+fj8htEk28qBgm8zsJl0azsZpgVCwPddZkoJA59LK8qMqBh6PhZrJMc8qXB8SqgGpAI+Q9XjJEWPYaKeMqiV6Xp/hWWvUpiOGHo2HUGwPPLCBEGZsherSiFk8owoUg7O5GRzrpGhyEbYLCJ0kQEEioy4hYgGyoPH8M0s1y6snWvtdCyEhOLnTTKNTpf0jmBUQac3Qs9CaWGQ2PsPQpcdk4HBhKGat27fx5S/+/p1sk/8IERriXTDK6++hre+SQRR12+8ghoTURgkGA0IddHvXWDYZ8ilhW96JQT8LLXGEhZbNOdG5ihXae5qlQoyzoyMhyOkE3qWVpPQQlev7mLnEtkNRggsUsIRGxur2N0gxJCOB4h+TCUZ7UEfpbKFx9Kcj0YJDGfQfCh4DB8UWhSIA+URGgNA5HPWRyoYXgBtNATLYV9IRxijPIUAhSwUFvljy0VQEITluSl+T8NlWXJtHDzZmBQeZ06urZP+waS49mB6jM6Uy2rGU6Qe7dNs1Icc0OvhcISP9gkd8sFtKsnYe/IYS0sk1zcvbaG5QHNXKQfILGqjAIZBm7yAMEubsRacobbkX/bizJHDBEoC0uo2oFat83dTJrJzMcBCl8tqBNA5J9vt/KwPwzDSySSBxxDamAlLut0eanXKfjXqDVT4ez3lOWKd4bDnyixyXayBJQpLNTBhIqXRVCONrXaTDsJKZE30jLVKBa0l0qtSsD2dS3ictSQSJ7jfk4w2MNMJarwPFtaWMAhobvYPyXZq9yawqVbleS57Cg2ENkMvY6QTun68dxf5VVrPYJ3s1NgXYGAWEinh82J4Uj2HBCzIXOCQOsYR8gGw0FapZhA3BTJBQDh0k7FkWEY72QwhMJPoK/StFDP+xqwett8LPEcqY30UU5R+KCFnkBMzl/+S8ZkOoK0litPEQv3R63VdDVe9WirYt9LECexGo4ozViK1CsGsPKHRqJEQOTs+RMqbbtTvo9OhRXtyvI+SnYAkx9OH5DCu8SH0l1eRMoBYT0fImSXp0s4GEnYqsjTF1Nb2jJMZpkfj/nUwQOgC1ql1gdsWAgV8ahbDyxAGbZyQq9UbeMowwLNRhliTkL775BjXX74MAFjapg14/94TnHZJ4MRCYJz07URD8AExRqHS4tq7ePddTQAAIABJREFUqcbjAzJO/+iP/hAA8F/91/8NNrZIuA+GY2xvkUNWqXh4dvQpAODH7/0A9/bfozkLfDxhdrNymQ5mu93DdEAH7wtXvwm9Qb+tOwbyggTReDC1aDPoYQUVswMA+N1f/5cAgGtbLxfQL6GhQfM/Tbo47hC05LB3C7/1e8SGWWsBe/dozfcek2Dr9i4KxlApHI5ZSQUhC2UQOliQPbEepKstnDFklYRn9w8KlsxitQr2RSMkjGDsvoFzPrUxBV+TKLDfWheMpjV2frqxxr337gIAJu/fwrqwDpYHMxzyPHZQYrjU4nIN7/7wLwHAOTkhDEJ2bD755BZuvELQHemHzxnirobVzNyTsQ6zfO7a3NaECRI8dI2Bz85lo17H0hIpBssMdn5+hlKVjIp33nsHd26zEZhXYUspd6+u4coL5HjlgvaP5+fwfWaEVC2sLlM95mQY44yZ6OIkhbasob6EYJkhZhzV0WjA86xx9Qqdlyx5hvGI9r/vKxhDe2xn9zJef4PY237ZsI6XgEAyoHMmj4/RYmdbIMc61+tuL9D6HHXPCqhqnkJzza/SGcb8eiEX2GBjzj+9j/G7BHf2lnbpe+sL7h60MdYX52H38cw7onCErSMohIDHziDBdm3wqoBsaq2RWwXATl+WZYVs00Bu73/GATTGOCyKQVFf+g9rAd08zgRXhLBnUrjvnGXYnY/Pb2h2PqKojDLXok3j3O3ZaZpAc2DAUxTEAIBASrCdiBLL79VWDesrpBvMdIqzR6R3p+MRwIyaZ+0LHLfpjK4uL0I6A4f+rS40nPGipAfJwaZmVHVWb1iKENo6ZOQY5j2+3spYA9hgBhIYdoqMzl0NII3nz4v2fPgMw5SZgbK1fEHoapY7nQt0O2REDycjVNjh3b58GQCwt3eIgwMKdMVp5spEVJwjiMiGUUoh5pqqx0+fQfM8/ov//I8BALvb17DN7NVb25dQZkbtJB7j/IR0cOfsFKMJ3cfJ6RFOjiiwap2+LMuwsMCO3G4Fa1w2cXB0CGPZn5Mpsow5DwJgd5ee4eWbBH+8fv0GgpDkup4MMbFcA0mCMKLf+dLXvoLm6hrfUw8ffPguAGA64jqyyRRJZo1i44xKKeRMwEoiYDllgxBSFgEwkxnA2OCU78Rf4HmOkVpIXThQNjA1w8qoReYYKyEcAp5sJhauOQBhYYwp7VGctWFiunjy6BmydsfeFCLekGGaoXtBNZGPOyPcvUP20+MDYou96FxAscwL/QALCxz08CNkHEjUeQH7TPOitssKfvOcoa5dYETAQNpoHQQ837I3+2gt0vqXIwvpHCFmePJ51nEB2Yv2ADIk5dzvXcAXZBfWq+TMD6s1bG3QGi+vLSNiOZEmOfp9mqez87arPzXCuNImmwwhl5buP0kzwAaPKhUX1EWWwpf03Tu729i5zGUuxjozgPXXhJhBR+Yalt5AKoUgoj80amUs8nn2+Tvy+AADlnk6nkDachAYBDy/URBBcW2mGZ3g/NGHNB/Ll+lHtl+CtlDnzDjOiUAUQQ2da1eG5s/U8Fv/RwoJz9q9BshtPaAuynSEkE5+WAcw18bpY2108b2Qzh4QAJTdK1K510WyorBNjTHuLAjxfHB2lndAmEL3/6Ixh4DOx3zMx3zMx3zMx3zMx3zMx3z8iozPzADa6PALL7zgPNrzZ09dmjtNYzTrFGm6efNF3P6YUuebG2toNChdbhmNlhZbrvfO+vo62qcUxZiOh2g06Dse3H6CBpPN7G5fxrDL0a9jgtbtru9gzD3k4m4fKqPo3vrmalEYjiLA7nsSqWUx0xYuAEhpI+DCpXt1DsSxZcsr4H7PEUs49k0DnwlB/HKEyZAiel45Qjy2jGETSL5mcZOiq41KiAn3QBGRh2nMEVq/jDDiiB0M1q8TJDYfdfCUU+Df+953AAAvv3Idwwlllxq+wP4JFXUfHj/Foz16XakbNLkfzt7+fYQ1+p2tTYpONuplvHr9a/zgBvd/RqQb02c9JBlHrmoZsECTUyrVUa3RjZ+26X5atQvEzMJ61HmIJ4fcH7B9iO6I1qvVLOPmS28AAH73N/4E//QP/gu6nvvOlJVykQsI4aJ+Qs5AzKSEYqiBTcMb6bn3lNTwGS4Q1Uow3PunpAKUlikbU6tVYDhDOR5T5O78YojB1EYQFXwbKDF5EYk0BtK+znMEDGHpHFJ27ORnj9F95ycAgLXBGSoMUTBJAN/Q2o/7XYzrtP6ffngXYUaRvI0aRdLGoxGWGKNzcLCPCfeCa5YXkNt+bcb9B4BwWVPMZKyf6/U288Lu+3I5wuoqRRYb9So+vkdRvxb/9mDYgxdQxPHXv/MV1KuUpf3p209wckxrfnzUx6tvUJQZyvYDFIiZqksphYzJJu7df4D9ZxT1HqcGlooz9BQ8l9Gizw2HfQw4S5fnBgGz/sZxgpSZxpI0Q5kL1L/4lVfx2hevuWf/T4eApRvMBCA9+r76wjr8MUV8dZ5hESRrXmPmOCljDJiVzDcZKhy9zjUw9un10hRY5rhZ5Gv09unsyNsEp4q+9CYGgs6ylypIYdkJjYPraMhikWbglLPZNpvpo0xbkdG1qAwpRHF2eFCWzlbYC9jEn9ba7RXle3B/EEV2sMgc6tkEpSty17robeR5BbQeMEWkcj4+tzGdciZbKoIpAjDJCBbSEvgKgrN+0vNRqdLZWVpZhWG4Xp0zTVGp5HralYMIfc70XbRPEFZIjzzdO8DTE9JzUfgqPM7aWYbrhRwIA8vQmMLjDGBQ8RHy/QUw0Nx7Lss1cssgWWCnnMzTRjuIX54mSC2EVUqHgLAEIyIXMJytylKNEkMlS5UK9BEhCNoXXbQ5CzTu97B9mbJzVzgDOBhMkLI+juMYkjMJOi/6zXmBj1Az5D9OMWZo+dUXvgwA2NnaQeCz3B/EaJ8TEmYyaePshOydztkBUpYrWTpAvcqZWb+AxlbYbgj80DGJPn26h/1nlJnV2TrWVkhuX9+9jDqjrFY2CZ0BJZw87Rwf4sEjgsJd9Nuuf2dzeQOXuTRlfVfg5x8QWVuaWPKeBHleIGEC376WsPi1XM/A+Thzm8Mgt1kHbRAyqVytUkXAvx2FEcoMOYXQSGIL96TfTpLUsUaPhiNAWLIRNQOR0wVBGAoymu5PSR4Pnp5jPKG9djE5g2CCIl96kJKhwJMEHd4Tpxc9nFzQXklismukNA5lMxj00WP0Vr0pXY9BJQ2UpbgUhVx0CEWvyAqKPIXgOTU5XJ/IKPAQRDZr7WOpQeu4UFvha88cBLdcLqPe5DMedTFg+O/J/hnS1+n9RoPsjTRJsdikfV4KIlcycH56hkdPaE8M+j2XfVaBgLRlOI69MkOcWqKiHgzr7CAqQQjuh6c16ozue+nFy1hdJdtf8j5XwnPmi8nd9oGnJITtTQwPw5RRdTlQYoOsxYRV640GPCY5iiHh2fIRCYQ2AygU6mUuUSn7mHaeAACGz94BAFSqIVSL0EVe7rus3iQ1LhOpBLHUApQIrFp960BmomDiBByRFdlfBRNnziRA1pb0jEHGMiw3+fM6lq8h/W7LfmZ6Arr+48VrbYo1gtFwYDzIAlE0Sz36S8ZnOoAHB1TDs7Ozg3fe/jEAoFatuCadUhDmFwBWV5awV2OHZ/EGFhZIQFlAQGuh6VgcwzBClpFCKpdLWGAW0EopQsiHaW111VH03r//BABw78FDXL9K0K9R5xR1ZtPKcl3UxigfacZY/jyD4SbQDkaAWaMGjp0oiXOMuYUARP4cxAmgFLWtM8wBVDmdrnyJlFkGtVLOqPWkwCK3J6jW6dpqJYBRljXVd/THQvjOOMxNAsF0xJ6u4tk+Gd8VQQ04/VKOJGcB5WkYhsG2+89wwY2t60kd1ZTWpT8aYPfKZZondhzb8TnaSyQ4mo06ltbZQXn/Pkybm9NXMiy+RoflrW98Ez+7T1C3jz79AQDA6BGO208BAPsXd/Fwn6CQ7UEPDV77TGzjZk41hQvlq2gu0O/YTU7NfmdrMq3jBWewGqORMwzDHQiTkxENqq169WXaE1d21nDGkBqTa2ww/GGh1XBtNqyDddYZ4qM7xB562u67VLnWcNS5hPotnEFriD/i2gn85B5Wx2QU1UXi6B/zLEaVm8pW9vex8AIxwL19PobPTsfWFYJJvP/hx5iwYDs4PMfpEbOt6gB7zLbbHwzcPmw2GqiUaW0rrPgb9QZCVq5Ga9coGVq7dgOry8tYXWnxPBbwyzI3bVVKo96g79MmxptfpzYQO5tfxP/2v/5f9NwPDnB8yC1FVljQBsIpbiEUOm2CdT24/whtbhTrlavwI9sqomjqbJ27PM9c/duTJ4/x0QePAABLrXWk7FCet48d3XZU9tAfUvBny9mORWN0CIKzAEAGjXKNDN3w2ssYccCklsYIEjoPm+ycDpYWccqBgqovsMHGi1EC2tZPdnoQhwxLrVZREVxfePvv6FnXFhFtvcrT7xfKwmgYWQShtHX2xMx+E1b4FxBLSDkDASmo8SE9Z0BYSIoBYEs0PCkxA0qBHQRHEv/w7Rl2sUJpGAMHiVUCzvgKfK8wvv8/UE3Px///YzjkWvIsoxoiAL6nYGxD40A6uLsAUKqSzPDKZdRX6DxYaGOl2USlQjq4FJRwfsK1zGEJFYanh8pHmaHvS8urAMvw/ojl6miCPLGMoRnK/DlZks7By5OUYKUgiKpZsA3bLSQcjnUv18Y5g2muXSAXyOF5ReAOoLo1ywooYaBt3XOaI+ZyEJOn8Nw21Qg8e3bo2ul0gB47swYCgbtSupYLnq/cTo+iCE2uIR4y6+Kzo1NkqTWgFWK2k4bdC0xjWq8syyGsbgiBUtm2O2KYqcnhs/MZhhLTjMsNOhd4+piM9koUYHebnITaahPgerlTttfibA8xy82n+4e4dZd1c2+IgAPsN1+UeOUm3X+93oTPQQHNukOnOax5SA53Uadk4fVGa6TCMjqzY5Zp5PzewkITL9+gYPbl7U1XT6+hUS6VeP59JInlBuD6w8kIp+dk9zx9doARwx8B6SB1Uooi6C+LMpAzLi/Jn7QRs5OGYIoKO7Ajz0fG0Mve1KDP89QdjzE1tDfzzMpjYMJr+/aPf4Ihf9/mxqo7c0pISN/Ojefgx1avNhsNLLbIIapVq/AtxE9nzrkshz48hjzC9+Cz058blrE6xCK3TVvf3sAmlzkKWcH7P/0AALD/9ADnJ6SbNzkQ0IxqriZ8NOxjNCB99+DBM5xf0LVKFkFAin1aWGrm/j7mGsCLXheK7/PyzhZOT8jWGo3GWGzSM662GljgAIyLUyN3bKSAdHWhKpSghkxA4tcRJ/SBdrePJgeHLJ64Ua0gqPIcRQE81tlJkkMzFFVhghLvPRV4CDlBlB6Qzh9VFxFJboXhLyPl9lyeUq7FW66AzDrpRrs2SBbuLYyGtBwAM8VFxszAf1FAOV1AS4sikJFLV5pjhHBRFIkiwCukcjrZOXqztoLO3T0/V5MvBDxVnFWjP9sBnENA52M+5mM+5mM+5mM+5mM+5mM+fkXGZ2YAj7mZe1TeQrdPkf2NehU+R+sr5ZJr+P7k0QNsbXDGq1pBxGxfIUcMKtWKy3gppZCkXIAZhPA4+yU1kNp+W9JHlSGjGYcPet0OhswaeXxyihXu89Hrj1yPNs/30O3RNQEMAu7bYyMQQgiX+tWieD/NtesFIqSwpFzQmWVoLNLOWgAlbt580TnD0DZaNh4WGXb4a9/7TWxeJpia7V3jl0qIOUOYjMfIXRSgILU0EK6nmBQ+jKJIWbVC37u2soujzl3+nMHG5g26uH2O6gpBzy66YwwY8lBfqqDDvRjXuIh5Z/MGqmWKKGVxguYKZcrquzt48JiyW+WghvV1gog0q1vYWCP46McPfgQAOB0+xOEZRdsGows0uUm6KhmsrdP3pW2FPOEI63iKVf6dCkc9RZJgNtvgSl1Nwfw1m8R2LIrIkPK71194CV//KmWrqmKEOpjdahxjZYGicAsNHyOGKZa491ujWkWjSq/fe/8WnnF/G2Nmi2iLiA414aT/zz6+DwBY7nZglA09+0iZHUp6OUq8gaLjE6xyZu3Sxibe+4DmN7MZXxnggCN34vwC/+Z/+Tf0jJ6P/X2K6CZx7EhgyuXKTOaP1vPGiy/i619/EwCwsrIC3/b70Tki3qcLzbproAxoBAwpCThyNBn1kMQUaQ0iCT+i15s7NXz1zZcAAD/863fx/nuUnfvdf/otAEDncA+1CkWS43aKn90iKFH7vAfJBeMNr2iw7iFyC+r7lAH0AwEzovnqdAb4tbeIPe9f/ME/w//0r2k+/uKvj1FmFrPeMMf9v6Oo3qu/b7OPYoZwRbgMYKQ9BBzRy3e3kd0hWI042HNMc5ZdrBFFSATd89LqKhoRnT2VaxhmIw0WYwx4bbNkghKf57hPa6U/fBe1MmVaO0tbMIYzKJkAFMOkZhALQgj40uKPOTKqiuhfprMCvincNkAuhMvCea7HaQ6dFfAPy4ynlDfTz1S7KLryivdtpluKmcjtDI2YEAI2/+F5nouMSlkUsc/H5zeOjghefd45x4CzasLY/1B2/bxNEfB6vY6ISU8arQa8EiMImqRTgrDsWHOh4Rqcy66PjOW3lIDnWbRA4PaLJTZIpgVFYxREqHNGIKrVoDgKnafUtwsgfWuY+ddKWTUjb7UpMnlGCBcZN4BDf1j9qbV2ByMHMLGQQqMgjCW+asFj6F9YLmGT0SGtBtspSjqmUc9XkJasRhtHUiOldPC1yPdRr5IeG40pszJJM0w5W5VNY0y4x1wSx1hbJrmzvLqIvaeEnMkmU7RYb0ZcBuMHERQT78ArI2Qm8UtbO5iOKUO5tryERYatlyOF8zaVJOztsT6ejlGq0PcN4ykSRhbEQrusmFcK4VVojYKg7FA5HveK9fzAsXMard0ceMpz65VleUH04RBY0mXHrm5v4Euvsw3RqODCNveeJCiFvFciH3luIZSc0ZV11Ooke8fjMSYTsjEzrYt+vRoFKZwxBWHhCZ0Fr68R2N6yZgot6FkmYorzDsOQmxkqzF6+OB1xY+8CIaaMRJLQHr199w4Ojuk+lhZbBULDZMiEheUJRIwaqTIybnt7DS/dIP35ys2b2FzjrGul5D5XCyIknLlKculYej2WsaWwjCaTizVrDYRMDpOlwNEzWvtHTx/jlMuFNpdpb6sowohLpjIYtLl8Z9jvOhIeMsmZXMVIx8Zt+/opZa8BkniKCmc2X7l5A8eHrPeTCRaWbE9uOEI9y2ie5oawnwBghINwS1ksYbm5DLlEmcvR+WPoAcmugHVVIHN4XBpSajZRZQg0VOiYeSe9C8QDekbEY0SC+2OeEtJLNpcgm+SjlBtlJLbnobFN3wEtJAyjwTwY1zPXKjmJAuVHQLUih2b9ChhA8uc8iwrTRVYwkRkMy1sNQLu+fUUZlJwpg7IVFlIYl9EzQkGy/lcCM+zxErMVGRaa/cvGZzqAH31EzTO7/Y5jyMyyFFnGLQukxEfc6NuTBn/4h8RUWa/XXf2grdsiCKXNo0rH/KQ8Hz1m57r9wcfY3dkFAHQ6XTx4SAtXY+F4MZ2izFjpP/qjf46VJYYRIEPOTthoNMYe08+Pxj0gqfNEWMy2hFGFsrDtHOI0R2odL8gZBccGkgDAk+wFCiFDKU6PDzHl+QhFiHVui7H+xk2Umc2pwp8rVXwHj5D5BIJZDSVSR+ttkCJmqKZMU0Rcg1FiVq87n36KoyE1Do+aClPFDVrTBq6u3wQAVLdruPcpXTPo9mByZj4a0EGpXmogZJhBkubQ/B033voC1q+S06rCHM1VEjrD+Bz9HjfMPWIW0/4trLaI8vq7X/wGznokiG49/BkmR/Q7X775FXztC9QM10cFIRvUTcald09O4DHkN9NZUcc2w1QooJC7RDX9fbEWosI1pq+9fA0Bw3jKwji6bd/3EShrDGuUmC0r5bYkURgiCml9Jjd2cX5B9avDcVIA52QBbjMoHMKFU3KoI5NiapnIpOdw2HkgMQLN70B4KDO71SvXN/EO4+mPuNmsVyo5BrX+eIz33mP2Vi94rgm9PU9Stl0bDQupvnPnDt59h3Dub33rW3jz174JAKjVqqgxvLpUKs0Y9rn7joCFyKCfIZ5YiE4KRiEjKAd448vUMuK9dz9EmymoywGtYXt6jDGz031462NENdozX/jyl/D0KcNx8pSKIQBAa/gW98/1pq3FpjMMlRfg5VdpD66sVtBk5xkAyuywK7+EXHO7GEvBBvMcNMOWr/gZIFmxj5aXUL1CZyQ9OkBo2ADlv1e8ECk3eS8FFZS4JYofZ8gMGXaen6FymdjWup/cRoPZjKsVFvQPb8EwNCz6WgmpWOTnFtDaOpyAUIXhVGd4nm1bIqV0UE+STzM0YfZxpXL7zVHIK1pbgJS5hWx6M56jlBKSz1yh3GYawRvtFIhSntuDUil40n6fei6YJ2WhBOfj8xn3H1Eg5vGzA+i0qCWyTlR/OMAh2atoNGvYvkIBvMXlRSiWw9Knfad83xkknlCIWK6ctM8dPDPVQMjN0afDiWvRZK0evxRhZZWckoXFRVdbrDzfMX7n4xFyVrIyGaLHNeR2v+UzzY81ilZLs8G32Tqwoo2JdnsakM9Rs1cr9KyXNleh2dy5lo9RYv1Xr9EzLdarqFdsm4iCNRrGQLPOyDOFhEsIsjR1gd2RdcARY8JF/vF4gmRMgehKpYpr10imNRcWsH9I9snp4Sl81h9L7NCVKiFCrmlXfgbFNYU3X38VzWWa00Y9dGyeWucYDGh+zznwLaREnTkFVlbqyMFQ3/YZBM/BpZ1raHINqMq0cwBtLWKpVEFmIZaZno3Ouno7JZULBEUc6I8CD+urJPNef2kH2xyU1jpDEtCXBEbA41ZGoUpdTbLPEGM/Cpydd37axjGXdSTxTMNuYKbsp6gLNRz4ViKFsbWxnnB77eFYQjTIQdq++gIWX3sFANCrAP4tatGUJ6SjW82ma51wdHqGToeg0WkyAaTlIDCudUJuDAKWqT3LFH7RwdEhfe7w4Agv3bgMALh+/QpWOWEiSgohO9uINSSXOtjTEJYCZw9lSQLfJz2ytrqKF2/Qvto/fIZTbtUVJ6SvA2nQ4SBEVVQdM68fKgQM+c106hj0BQo298zay8qD59tgZIJVXtuvffVVPH1Mc/PgyW00mxGvnXLOuK2PgynWClI9V0NnIdhevQa1QtjW08clxH2qmW0xHNqTAJh5PNMTCEF2RlgvIxDc3sL3XFuJJJugzKVZU+79YI72IBtUIxrnJZRJvSMLF5AZC9k0ELZ1kxTwHTkE7zUYWFWp8xy562ovCo4KDVeLL5xDZyDZsRTSOLZmbYSDgwpBe5nmXbk6a+Gg/GaG60E4e85X0p1DoKipNgByXbz/i8Zcc8/HfMzHfMzHfMzHfMzHfMzHfPyKjM/MAH74MWUAj88O0WlTFCOIK2hxAW/ge6hy9DrXmYvAZ3nqilqla/Lpuf4fxgBDLs4+b19ge4cyCK1qEw2GJXjKx5RTuxtc8Hw5CvHqTYI8rq+uo1zmL0ynzqPOcqDfp4hcIOGKsm1zeCmlS7nmwti2QxjHibvW8wKX4rexwDQXyJglrFmKMJ1SlKjXaWNjg7Ih694qWosUmeheAL5PqfrFBmeOaj4Wm8z+2BvDGCadSQXAaeVMT5F2CN7TKEW4ukPPu7hE87L70hYe/d3fAwBGgxHKTYp+7Na+jBe23gIAbK2vwLRp/t+580PUGfpSYnbOTvldbP82RYRrS6uQPAnZqI80owzPWece9u9RlDmN+3j88AndE0MpF+vrWC5Rgff4ooy6T0QsWxUPGWj+N1dvYDjkrEaeYjqh9Wxw/7XO0YkL/wpRFKxS/zSOYmjt9k3OfWC++9Zb2Nii6OXpYIpel+ACKytlLHPPyIPzPtodgsyMJ1NUGZI04X2Q6qlrjuz5ykE2hqOk6H+Y65myXrgoaJMzvplJYcOkQghHrHIS+dhjeNXay19EmaNIk5UAr3+J4CA5R4YWl5Zx7WXq/fc//+//Jw6Z8bYUld0eHAwGLqrteZ7L1gwYDp2mKbrcKPbTTz/F37/9NgDg29/5Dr74BhGShKFEixl7w8DHwSERCgSc6ms2llAuURQ6x8BBfvIshWAShNZSCSfczyriCPr16zfwNz/8KwBAtVLDN7/9GwCA/aNjZDzXWZYVhCW+tNXmyDWTNZkcnTbdf78/QK3BzafTPoQoIAyr67S29XoN2OeMoT3XQj7XK8zmbmVWMG4lKoK4RBnA6Z2fA2e0P5hTAuUsgeYoXtLrIeZ9Kv0Qks++yTRCzuyXljcwalPUtRoxVDxuo3ObmGGDtSUEGzT/iQkdxEwI4QiDIAXKJYb8KBt5VIg4Ag6TOdk1mSaIU9tfqCDJsOxuJs/gu2wKxXMByoTYTLv01H/C/AmQzAOo4NwxiokCCSGlcBFJTxVRXimEI8eaj89v7O0Rm+3xYRslJoXyZYRShUnGoBBzJmM6mbhsmTEKHiwSp4AFuzIFT2LCcvHp3gnKDJFc295BVLNwd4Uyy43VDSrDWF5sYGWNQuq1Zmtm/ybQMenBqTxFynDJPJ+6LIONuEuRF6x6QjqEQJ5rpyeUFAXrLN+O0JnrH6yERMZQdk/AZQDLpSLr7nsLLmNkG5kvNmtoMaTzrD+AsoRNuXb9FNMkx5g/NxoPYBjqlvKZFMogZfhpHKcwXBLQWlrB+iXOwFZraHFf2P17d5Ey2scD6caSn0FJy56okHH2UXgGETNHXwz7ODjb59+JcfSMZPkhk1M1WwtYYAhEtdHAdonkWLW6jCmjDJq1FVS4HCIXmUNoeFw+oPwAIrewRA2hC/knuQRIKM+VAwWs47Y3V/DqS5SV2tpYguB+hVJo1CKaj0waGJvd1RMoS4zH9yBSwGPZVQ09xxoc7dqPAAAgAElEQVSdZxmMRSF4niO/kp5yJGLKt8gzDT+iz/mVEJ0erct5SWBjh+zJ1rUdNBbJdmu1F7DI+rF8lcjvvvbFN1BdJLvhr/76Hbzz8W0AwDQfuOf2pYQM2O7N4FBHY0a5DEYjHJ3Qujy89wBv/z2d1RcuX8ILr5JueOWV17DEaKs8z10/bcNyeqpzHJ6TjTOOp1heoXO2srSOpRXSiUr6aJ9TKcmUETmBEogn9Ny1hnRwxXSaOHh1nE4x5fPSWqgWzeQ5K2hEBsssXPIMblwh1NerL1zHlPtY16Iy6nV6rlqzDMFZRGlLDYSa6acMN1QOSJYBfuBDMkxaNloYHBOrfJ1LZbxcIGP4shZTXJwR1DMyQIlJ3mRUhs8Z2+n4ApMB29e8B7PuPuJ9QnpFXgWqznq3VEYmWP+lAoYb2Jt8FgJqs94CypaUpBktOui8pLAswik036uFdCrfh7JlaJ6AFjOdCJzZol0G0Pc8Z4PZDCCEcXIaEK5sTHnS2XEwRX9oQBTN53/J+EzNbQ24o+NjeGzcbm1tYneVNuDSYhOrl+kwReXQGctEJEfXDye0uaZxgsVFSh/nWmPAzUnv3X+Ab33r1wEA//JP/hVg6f19D2ub63y9bWgrEQV0SG/duoWbr1Cq24fBBW/++/cf4Ec/IsbSf/Y734VLctoUb6ahLQRUCkfxrDWcIAqjyi+ENWX2IFQqrrmqEgbbawTNqKgKckMCvdpS6DMjYtYl4b6+WMa1yzR3/fM+OqfsBF/04bEyKUcKvaeUps6kwI3fIzjfBTOzXd35Dt44/g4A4KS9h84jMkDXbsRI2VD/9NF9tDn1P8xTLHI9QP2UYJqf/uBf4+SM6ghf/PpvI2bB8N5PfoBJj+rbSlGCRBOEb3E5RdUyMEr6rvH4FWiPhNbhRYxKjf6+vH4VfkD3cXjYRyPktgcrS0VtJkOPBChYAFDK2zUIRdF0XcNzdRcWBlEu+1hmw+Tp/jOkvG6Dcg6foS+58jBmQ6bX6WGZcfVjdtyV9JCwoOz0evBtQ1KROjiAkJ6D6SppIBnKGzMcMBcpFDeb9X0fmpncHtVrOP0q1bH9xvd+CwNuePyjo3u4fIPgg60S3ec33vomrr9KNYx/8/dv49FjUuwjb+QgnlJKxBwM0Vo7+KZ1DIQQKHFQZjQa4cc/oQDBzz54Hy2Gyl67ehkrSwzNee1lXH2Z9mGZocd37zzEndtUmzKJe/jKV6l9x/lZGw8+ZbiqF6HTJsNzPGRmtniKMj/LF770VSyvkTz4t3/254hZsSglHSQiKikY24ZF0FocH5/g6VOCRcEAUjEMRY9x7ToHKqoVLLVov335S687ZjWR2r4q0jklQOHUGeTIbe3otIyMG7bL3atI20/ofZ7HIJm61g/jixRn3Jy33lpGNWZHKI0haiRsG7vX0OaASMwwN9/L4Z3T9+bv/C2iXyfZMGmuQ6QspIVwEFUNgdy3ysLevcZ4NOLvE0h4Hx8en+L4jKnKk9TRPa8sk0G50moikPbcSBe88jxvBkIs4eputXkOGkq/J50SEiicRCGEgyQpIZ5zHsUcAvq5j2lM+74/GrlyhIV6C1vM6FwNAyjWV8165IJnvW4bwwFTzvO59aMQfmChzwFGXaaWPz7BS6sUMHnxxZsYjEkGSVlCyPJmnaH1tVoVioMIWS4dK2TgVzBiqvoHD/Zw8JgMO2ESyO0aP41w/9qdlJu8oE3PipoZ5WlXSmBrtkb9IcZ8VkPlIWA5jXoFlRpDUaMARlsjDmR9ojCuolCjWmP43lTAoPi7Nch1qhEbMnrjyRghw65WWqxfGy1MWR6MRzGGzJsQehWAGQ7jLMUaG+1HK0vIuS1G5+gp31uO0j45dHGcos9lG/1hH0M25vvDC3SZ0dEog2mfWyZMaU90xylSrnnb3JIIOagupAfDMPQ0iRHzHpKQCLnuMOQ1DIRBaunpjXY1YcZowNZIiYJd3UJIVzdWsctOQjksWDSFSJ3MyLPUMX/mmYZSXMMlJ25Zp2xAZ/HEsT+HYeB+z/eLMhzpec5Z0gnXokUGkmtZUQ4xSOnadKEFsX0ZANAzHi64JVW728HlHZKj25uvAwC+/Y2vw+MWQo8f7ONH71FCxKQ5oogTBL6HQFtIco6E6f9jZh2nxAQ9dzfJcHZIz3VxfIZP7j0GAPzbf/cX2LpE53Z5aQl/+Dv/xK0XAAyHY8divm9yrHM5Tq4Vxqxj4yzD0RE5RZ0OwSc3VjcQ2WRNWMZwRHN+eHzmYIc5UpTYzrhyZc050O0O2Yr90RhjZsdVKsfaqm2tpSATWltPpWgtkE23tBCh3+FnZ5ljTA7fOvdCQXEASnkCgl/nWqBapTmoLa5hxC2pJJ895Ws4MGOeQ3MgY9yfIIG1K31ohjB7UdMFLTT7gXm/i9EeOYC+rKHZIOffr68hYl06zlOkPNcCOfwms4BamxDaBZfTaYLRkNYi0QkSbdnNM+SpDXjT58OojDLLysALXJscDVXwXZiiNYjyfcfgLh2cdEbvGstcQJ8pIPBwjqMxBUP7LxtzzT0f8zEf8zEf8zEf8zEf8zEf8/ErMj4zA2gjVVIaDJmB6rVXXoXibMRwOEDQoahDNa06z/jOnbt48SVif+pydOrxkz289da3+PskEi4GPzlt4xHDC7++vo0VTm//h7/893j9DSrQtV505CkY9mjXVlYcEYEvFWKGFybTGHtPKJomvQAxwxhi/lyujYs6GCOKlLT04Hm2f0nkyFr6PXq+/cePXESy8ubXAE2/5ymDLKYQw0SfIuc0sPAVmpbJamKjXTmqkl5PshHKHNJolCTAvxdIgZiZpyrSYHhB2ZcwpChe2a/id7/7nwGgVO+zQ4JpXnTbUPwwyytrODuledzc3kXepSjXioXaZgN8/NH36fnO7uOcIxB7Z3exWmbWTt+HZGKLqQlcv8QzTqt3z06gGtw3KSihxJmThUYAzxbC94Y4qxB0oRKGGE8s5M9GgCQS2zdJBo7BzhjjSIIECnZH29T88PQY1zliVvUMcs7SCWNgQZvjaYLBkO51NBq5fkspR60gFDxmqhXCgFvXYLlVRszpGRmU0OQoYq3egMfsW/F9gkEqKJQFRb6yqIHDMkXH9l94Hd0rBPU8H7Qhx5TVG3c7WFuj6GjI0bHDJ3ddFkbNFPnGcex6Nkkpn2NrtAW/xkWckuJapVDmiJ4fRbjgKPT7P//InZ0//7M/x//wP/4rmj+OkKepxoNPKQv37OAAY84SLC21EPm0l7rtJ+he0Jw+fEhR6oujPaxzg/nr127g0T7Nzf7+AUKOeFVrZTR57gymiErc55LnfzypOMiY53toNCxxU4ok5cxaILHEJAjLS1W0Fmh+LQQDUsClkEUBK9MidZDvIA2QW6jQpSvI71EPJXNC0U5f525OQwCdPrOzBhVEzA4qpxOklpG1VkPrMqEQBkcMl877qPKG7e89xPgBRRzVa1XEks6wkhKS93IuDJSwUVDLamjQ61EGv93p4JAhRAenbQw5yh+EEYbct7TGvRm/+eaXcGmF9qCYaTbred5ziIbZ7J3rrWkzgEoVmUgx0+FPFKQEBUUNmBTiOaD0fHwOwxIiRb4Hn8ssqpUIC1XSOSvNqovs15otB5scjoau35pforPVWl6CUEzwkhp0RhRRH04SLCyQDbC7s+tIRsbjKcqMKom4l2BYjhDbs2gKkoM4z3HOTOF3P/4Ijx5QBnBjo4XlDTrnFialjXFECtoY1yLWQCDLM/d+wBlAWyJyuL+PkCPnrVrJEVjl+QICZrWUQQDDujkxuoBx2XOYTCyvB3ypncyQAsgyC18zyDmDlidDWJTrCsMIr1676aDU6djgnElD2icnmDDUMyo1scpIjJs3X8aEm8VnOZ3hsyefwGQj/lwbF136XCYkvDLJD60ChIyU0kJhwrrXUuulaYIOMz76fohKnVkeU+ngZkmWOGI/McMuqALby9HAOPpB5UoQFITLDCaTFJKhFhZqG4ZlSL63ZDpFnHLWGMYhCzJtMGAbIYlT188sYh0mpMKQ7bnpZOwYDiu1CnyP2YdnEB/KU8jYFow5Y+ojcIQ205aHaURzPq03ccI6++L4CNkhl3NMuri0wutyifZ8WaaOWGUwGTnWZQ3j7iPwhTt/npZOzma8ryYpYLhkREnhyqSay4tYWiK9+eTZIT69T7ZUfzxAGFr9QvMxejjF2Rntg9FogMNThoNOcgz69qz20evSeb5/lzKLi6srWFwjwptGs4ER26HD4RgDJigqRSGqDM1daDRQbdA6CkbhjKcTTCf0e0InyDmddtFt44L1oy8T1CJ67lrJx8C32fOiRMEw0ssLItsOD3qiYRjNVFE5VEC/vbx1GckzQur4HbLlw3QCy8ditHGEaXmcYshySUU+fK6B8Os1eB6XmjDRTDyZQJ+TLkX4EOEm/UZ16zqsK5TkBYGcMGnBwMkaL8tzpFy+1m53cfcewYKfPn0CxbKmVq+jXmXYdY3ObL3RgqcYjeADgd0/QrmeqkIWzd/lDCGM6w1ojLORobUr5VBCQM2wgM4y6/+/qeZ5BnA+5mM+5mM+5mM+5mM+5mM+5uNXZHxmBvDmLhXDjvo9/PSEsgNv/93bWOJeIMZMIYICHz+Zkuf58d0HuPYikYJYgpHeedv10RtPEzx+SJmt9nkPrRZFKUphFVnKBCJrC1AcsfOVLQTVri/L5uYaFNe1ZFmOlRWKprRPDl1WTwsfo5gx+5xxzAygHB5ZQmtbwFsUS+baIOUagX3ut/RXf/HvkU/pfl7cvY7Hjx8AAFbXq46QZGqKTIzSBsaje435OfrDCfIhRVPiYQ+SQ5yNSlj0KYRApWz7vEl4YDw1FvjuFGKOKI2yFLX1ywCAJKjjaMR9ZaIA9ctfAgB8eeM6Tj4hUpC6R9mIq691sfSQMjJ//YNDHBxTxGsilxCscSF0tYlY0jUHZwGQcTS5TveWqgh7T6keTHsVGI7iJskAr79EdVsvXt0CKjTX+/0uNNdrccIDXlhCylkprQriHSlEUdTtBZCKM1qMAq8vbaDGmauy78FwBKVSinDGBEAXZxeYcv1p4IUwdv2ZxhvKQ8A00Z4AQq7QvbG7jZSPxSjOsLJKdahGBq4dhRBFXyjNNQLH9RJuX6bzMvzyb6LLtR3v3v4JFjzK5lytLaPERAONKj1L9+wMf/MXPwAAdM7P4XFkKMtyTPlea7Waw3JTS4jnSTzyXLsILYxBi2tt/+CP/zkUZ8IG3QGODynT9dN3f4zxiNeCyQdWV9fw7JCiWaWwiX/3fSJ2uXr1CkoRRepvf/wYNh37/f/7TwEAL1/bxptffZN/20OHe495nu8yvlGkXPS9XKlieYWiYhc9+u3l5QbWN6n+4sHDZzg+oshi2fcw4FoXIYFGkyKEn3zyHo6Pae997/e/TL+tc1cT50E4wh4tBGRuCVdGLvMmFjeQbtJ6Tc6oZqKiPUjOjgTKIOA9I1KJrEp7JQgqUJxxnkwzlDnCF0woCzA5HLiWRwIx0kdUaxuubCDbJFRErn0oziR4IkM5os/aqGaW56jxPhnHKfojkpWf3HuAUUxfvnnpMmpcw3A+on3y8zuPUArpNzaXGy5qKWVR1yfl8yQw/zADCKNdpb6BcfNoTJEtpDYV9v2CjH8+Pr+xyFmn/fAAGe+l4WiKw1M6O+l0jE2u1a9CIGX0ivAAYetSSrZ3bhmG019GCqcTVegjqNA+DOsN1B1pi4FkOeCFlkhLFq0hIKA4MyR9HxET04yzBNOECdpKa0g582Oj7MYI972QM3WyQroWTBDSPcvxKZ3bBw8focXtHJAvuJq9yXiCGtdAh5Waq/eTOneZxmRaZJokLAmc5yjWcwH3fp5rTDnrPhn1XX3S+Tll21aWu2i1yBYolyNsBjT/yqSIJ9w+KfEc6dMLV7eQrNDc5BPWq3EHkjM1Z1rgOCMZOtYeBNtdpXqEskVuyBLOLuj8nzI6Q3khGHyBdDpA33IoeCXHczCZjB0hTCUsod8nORtPeY2VD7CsVJ6HWsj7IPARM3pF5xIRI2RsNqtULmEaWwKdHPwTAHJXszQapbhgZFgSp2B15ggDI99HznT+cWYQcpa5EpZQsntW+U7naSGQ84L1D57Q90oJr0R6cLLQhFqlfWCqqxhyrZlnpij7tN9WahXUK033jABwcX6KO3cJ6XJ8sI8qP6uQ0rUy8AQQcdY3igqkhdVbidDOPolKAXa2iAPgt37nn+D6jZcBAB9+8D7u3Sc0ijRFn7mAe/jGxuAZk8ONxmPIU8p4XfTGSHOax7PuEO0zRvvcJj2+fW0bX/oi2YHNpSWMR7TXw0oZPW5dogXgcebN9z3Xx7DFKJzzTtm953kST59QxvqT2m30+rTvjaJaVACIcwPDGeDpmPajgISwtaJ5VjT1yhME9n88IAetealcx+oW+QQy4/7MZ30ornfM4KPMGd08kADXY6ZaQ7M2KpUDBKru7g8AvOkUw4Tsk7x/jPyQbHixvovaAtf/ph6ktDX5RdbaibZcwGOCn1KlCo9rZ8+7Hfe8jXoTtQV6vbLC5EmC+qYDQLkUOhsu9ANXPyulhHKtlgrdbFtNUP2tFZba7UEpCjIrKQrbBzAFYdYvGZ/pAPbPaLKgM8fI1b3oYymiw16tV91DRX7kmm1qGcBy99VYgVy9vOtuLMtyLCxwAaYfYX2L+n/UGw3kOf3mC9d3ndFr/zVpDo+bN2ZxjMmYNkyn34OIafN/+P7P3CRNxlPE7KBOmPDDSM8xTMHk0HnhANoCaaM8Bw1hWYbuYIoVVibLy8v427/9IQBgYekqHj4k+Fe92cTGOvd2QQG3sGQ60zh2BB46186Al3CcORCCFxpkrFlFdMHp/Q9u38GAoS8IAkjGoUStBsKaZbpMCVIJYGFxB6uv0evuz/8MAJBqg8tfpnu6iRDjH9M933pvGY+mNI87O8tQkqASnvYg2AmrsaJtNg3G3FsnzkfIeU4Tf4KIGTXH0odhhrR6sIiQ10CzQ7+wsoqrNTJY/WoFZRbupVIJJYYHhlHkiv7DkCZjfbWB2MJXVIiM53SaSZx3SbANJ7FjvfJUgJCbHw+5wagwBiM2nOM4hWeDDJ6PUoUER5gZjNjpn+Q5anWaX18zTMYr44Kbfh/uXsbw298FAAxWLsO/8yEA4Gx4AtGie33zpTeQc6+pcoPu7Sjy8JgbuC4tLcF7SI5NksSuP89kMkXERtvS2iIihhqct8nY61z0kLFzLZXvBNFicwE3XqBADDSQMUPd9StbYJkOEdN9tBZaCCOGzCRd3GAn/vatWzhjhTMdKwS8LrduEVHR+kIdhwfkWG5uX8VobAuoBWJ2YIWsosSwz42NVZSYvXc0ofeGGGDnMgn8n79fwn/8IfU0XPq9f+ykd7NZw/oGw3jiDgKfG9YyA6YSKcSQ1t4M+q4BahyVAYYCGaEgmX3LR4ZghdZu1GQsV3+CaMqNZ5MYVRvUiMfI2alWjUVInuu430PCECePjZQ0BzQr13I9gO7Seg7v/AQV7mk4qm5iynK5kmXP9eMDAKW1ayS7HpRQYhKH1bUNvPvTn9O63P4IlToZLFeu0RnqdHv40ftEVPDFl67i5i4xNFbCoIDZQToYEtWhF6QxADfVtj2nkM8UvwvYsnMpi8bcAs8TwszH5zN87uMFFbj+td3R1DWF7556mKbMnuhFWGLytGalhZChhJUW7R/thQ5eP44NEJK8XVhZQ3ORnZiwDJ8JvbwkKwgKbDBVSyg2J3IJ1ycNuXGQ006vg3PbjDrZgumzgWiDfVLBcHDN8zxH+AFjip50eYrRmIzCvccED9t7sgdskE1SLylHNhYGbaxsMuPj4opzlLO8CBZZso40mTgiiHJU9CqL49wpZ2Ny9z4RXND3dS/ofnrDESp1CmR5vnC9Z1uNBfT7BD3TaYKyJSfxyog87sVYI52jpzG8AdtUkwjemA3TRCOxMk+P4aW0/lpqlNlILfOUm0CjFNj1SZFzj9XRZAjBxvn5aRUVts1EaxWDvmUX5kbmCyuQDKeLgrIrB6rVK46t2PPLqLGurzXo362NFsqW7TOeoDcge6LTPkWP12U0mmIysc6BQYmVUZixLZZrsB+KFApl27C+WkWzwU5aVIaSNpAPTNjT7D4hhy30Q5R4f/fXrwBVuv9qaRnaBumzPhaqNKeXqkAp4qAc77U0jl3pU7fTQzrivpVh5KCGUnpuPTfXlh0r/mO+j+m0DSMssY5CnZ/15ksv4xu/9g0AwOWdy3j7PTpnJ+d7SDgA02jS/bcWW3iyR45XqrUrF9o/aiOe0v31eiOMp3T2P/iIgo4LC3U3d9cyg4sON1eX0vUL9aSAFzJLeTJFGBd9FgGgVgmxynoyjzOcnlFC5PYnBh12AKE8R944yTPXqzGdcvA2A7TtlzzIoSd0XnQ6dtDRGAYmpf1RjnsoWUZ4ae3zHDnrd+mFUDZQKz34JdrHg1QDXGKVywyZZ4mjaL7CyhLA+r3fGyN5SvPUj2oo3WAoamkbQ5ZjQdlzzpkNpCMQCJigptZYdPOrlIdPbpPNd3J6hos+2evtcwpS1atVDK/R87380suos4MdBJFj6AaEY+D2VeEMppbtW2eAseujkVv2b8CRwAhREO5JAUj12eHZOQR0PuZjPuZjPuZjPuZjPuZjPubjV2R8ZgZwzIQDlShErUlRgExIjLgnFiYGt+5SUfdCo4f/h703fZYsue7Dfpl3q3trr7e/Xl5v08tMYzbMho0DggtAEgAtkVSEI6SwJfo/UDActsMR8lfbYYl2OMIKO8KighYpioRImyANDkSQAAaYfQaz9/T2uvvtr+rVXnfP9IdzMqsZMqBv0AdWfukX1e9V3bo38+TJc37Lpz73RQDAU888h4AlpiW3Zy9/ahkDFuU4OuphaYUq6rNshJRJz7N0CGiGZMxmEHz6N10zWTgw5j9SwUJICgVIIwAhPYzG9H4//O538cgXCSKW5gZaAiu7CqUMygFloSGYnaqQWRJvlhlitcLZs+fo84oCBwwNvf7EBYwZPhFV69AwFQtlyyimg5kkiRX5CKPQQr6mk9gKzGjtQLM0//rakq3Wn96iLunzn38BE+7wlIVGzl0WxxP2fgmhrMRzejRCyUIawyn589x//Q4ee5Lu6enaACtfpg7OhfYm3n2bKhoPfvQaVqoz/l4uZIUgqKMRV1scgc3TJGji6wIZ93yLfIpXX3mD/q7Wxlde/DUAwMWzV1Bl4re/QlWy1FnFmQ3qNDm1qoW+hmGECrfZfdeDwxVFI52b6BJDft4v/srfw3BAnbB33v8IA4acFNKxnYlUC+yzj44rDZS1hSQ3fjsRYoYfFInCqTWam0vLa/CMeIn0LaQuf4s7OdEa8sc+S9f8i18GLpFn4+STB/COtgEAq6eaWGfIY2WpjSoLA8Gh5xbUIvh1+ozD7i0Uuala+VAMNZxOEjSWaD09dmEFK+w1eXBIVaRb23s46tOcH88yzAZUFdy9fRdXLhHM0Q881Ln6+Bu/9nU8OCKvuje+/xoA4K033oVmq4xHrp7F+gZ9xle/9kX8/r/6UwDAd//qbQgx95sBgDfefAu3b5J1yOtvv4chix0MhgOssD2B40oEoSH4w3oWmq5DqXO02eLhP/9Hfx+/93t/AgD4oz/8JhohVUHX1lfgetwZdzRaTXpGmuFIAjn0kJ7xwduv4/AmdShLT8PhDrKGC1aLR+hoVLjiWONqdBUKLs/jsFRQKQtZjI8xm9F1yNYyPPZA9DIFMJRJ8L2rNFsY9gmu48YDhDyvJh++hoI7y96n2+hbKxQJlyt9Rspcuo7trEgHqK7RfNtYXcZTj1G373t//V389v/02wCA7XcJPvTUF15EnyuLRZaiyRLgFzZXrbw7CTqYymFpvYSs6JIWFlpVYA6pcTD3ciqBuXCRM4eULsZPbxjJ/0JpK5KRFQraeEaWM+CIhZWqDcg67d/LWw3U29Qtq7CQTJoVSCa07+bFBHFqRGJq8Lmzk6jSwkT9SgiHURAOI0MgXWjHIFoKTBiJkUyHePCAuhe7B0c47NOaGw5GSBi94nPH0a+EFu4vZYnM2MgIhZKr2nEcI+N95ISh25PxGNMxXeeg37f7sa5FMIHKcWFtf8oym9NECiPXnyJihEmUCmuzUeSJRWKUpYJgMYs8jSF4Xw8MnAvOfC1LwHGMx2oVUiZ8myR8ho1JlSDL6J4G3PnMUaIwAhd5ApfjredqZNqIryQYTWjPi1WATPv8Hfj+Kwkt6HmWUiHjvCZJM6SFsWKQcBgl5PohSs5bogbFj2YYIGrSvlStNLG0RLG8VvNRYRSI73qQxsOvQn9fbbcQsQe0LhW0T8+lN8wxYV/K/rREUdLvNKoRfO6gxfz9xpMMWWZEaSLU2OcyrNfRblHcr1R8uC7NG8dzkaZMNzhD+129vgpxhnyK1dIasoDz18IDh3WETgkw5SL3czQ4Rta5u1dWAoSMxCiz2NoauXCskKBSGksMBV5tN7HOsOuSu3HD0Qgx/12JEnuH1LW+8/En+PR1EjncWFvFMwzVfLCzhLffofxpcELPeDTpYes8dbKvRZfggK4zS1Ns36M8NBN3kfO8mfRp/r/8/bdwdMT0kwsX4fEa6k+G8HitRrUK6hwHSlVgxF22hK9flQXObLIojh9i+w6t5YOjQwynpiPnAHyfkiJHUrJllX0PDc0dwMnJEQ5ZGC0b7MHj/XE8SjDoUR7a0jkusQDOCgscNdLc5l86LxDzGhfBBAF3ACtBBM1rDipHyX5+0jVegwoIaA5G7gmSI0bn5G8hMnnXeQ23Qnu9RhWuS/PG2NpAyfne53o4fZaeS7X+JaxvUj788ss/wOuvUH71YEr33wtCnPDzDP0Kmm3uagcVi+p0pLBoA8pd6eeA4Z1lIZCZvbsE9EP2EEY2UUBYGLIUc6RD9i0AACAASURBVDubHzd+4gFwMOY2Zn+EIfOpKpHEUZ/a+rXCx7hLAT3OHLz65lsAgJXNTeQlB0VOvmZ5iVt3icvyycc3LbdqOh1B86FPI7Ucv0Jpy/2TJr/QrlUPffX7P8THt24DAE5GI/zyl8gvzw1r1sdttd20CySxHECB0Cpd5XYjKIv5AUoLhZxvXJ99tzrNpoWV/Zs/+APs7tHkKfIc5y9Q0KnW6sgNzFEp+4CMh1s2B8Qj8AOkzD8Y9PvIDY5HOyhKuu8XL3UsBDRjrHeWx8hZwSnQgkMBUOSOPTBAlQg4QFUCB16Lgrd3+Wv0ed+5i52PKUFOqgobn6f7mJ3LkDd+k957+xoGD/4SAFBrCkjGXFc7BCvrDXbxgOEHzUYbvREl33vDfTRadFjJxAB7XSoQxPnT2GLo0ay9xM+tDzCk1w8iSFZh1dJFoQ2EBVYBLueGdaYA2aT5U9+8hG5CB5CZ24Js0HNrV0Lrh6K0wmzMpt+sdpfLADlDKaqtFTRWCZsfNduotel+BbWGVRITmBu9D0/RQbp57Vl0vvh1AMDd659CajhcyUfQDHOoVNawzLwQ+BIF4/tNq7/aaFiYQbfXnS9eR1pFWV1qCIbltZsh1ju0ISI3/JU2VlaMotgR9vcpMbr9ySf43IsEM2m2aog4UTl9agO/9Vv/B/3ObYJBPP7UdXz5K18GALQ6kVUQdFBBh5PHdrWGX/nKLwAATp2h+/+//8t/iXusoplAocWeWFGjihYXjeBmCJjzOCsHiDOKH+ZA3+x0kMzouW2sN/ALv0Rqwd/4vX+LaUj3dPVUB3XmALoOcXYA2I0/URk2OJGoeRI7D27Q/Z8eQkkzbwRys4l4EgmY28CKXV6zDmF4SSjgGA+i6QBll6G5lQhenX7fdyS0Z7zP6Dqj9jLSAUFjZqOu9VmsT3NMXn2TrqnzCKJzlJwkeq78alT5tJgH8SBw7QGrKApU+VD9M88/h5eZZ/3SX30XAPDne7t49Nnn6Bm+8Hnc2qZrRlFibZX+zg9DRHwYL/PMbjgGEldqCSVNEq6gC6PipyyfQDqOvb6/4U20GD+1sX/EUMpZAsU8vCDzUbC/mlAlnBFzfo5PIEPaxza2Jqi1OUllb8s0y3DMa/hkMMYu843iJGFVOcCXDkz1xJEBVGE8LRlmLyT2+EC2u7ON+ztEi0jjmeX1TWczzPj6Mu3g6D7tzYb37PmBhSLnRWkT7mrkodmgJM/354W4gOOHLnIcsw9w4AKKVbnbnZaFy+d5ioK9OlVZ2CQu40OJLnJwugFXaGhOPKHmfoRSzPnrFVeiEtAfdCL2AYyqCI0PmaPtmpJ+gFqNfkfo3JqIq8K3fCJwvuQ6FUz55/5MIq8whzGoIVUMmZ/lYPsxJKWPJhfDNpcpURdKYDDkZ6hSaC5ip7MY48TQCgSEQ/G0trKOKKDra7f4cFcNUWtxYu1HcB1TRPOQ8SFSlxKCIfUmfekNE6gaXbMrK5a7WW/mWCn54OsdI+eDd7USIuJnFDN8eZLmMHDzSr2KiPefsNZAlakcnuPbQ5jjznWJl557HgCwsnYVHifkMnQQd+lZnex0ccKHHC9QCJn/1mk1scKq5xVnvgfU2Vi8EkTwHIaAOnKuHpvniJkX6nsSGysM5dVUsE+LBIdsWn50dILDHVojP/irb+PKpXMAgGtPPo0WUwyEkPid//OfAQB29+h319pLePwJ8uU9vXEKUc0ceEJcuEzz/s69c3iwTfn18S79e9g9xp3bpAg6GA7RYQpCWItQ4f29UQ2xys0Y3w+g+UCmjHKpEKgz37habWHG54C93UObD3uuA59jg6s1LAmeFdclAM1r8vjBh3hw43UAQNHfh9QmNy4Q84EyUw5Cs0dxMcEJJFpcZHWUguL3VqM+JBdWo5UaBFNs0ryAUvw7rGgqKiE8XqvVNAZO2Cv7YNtSipYqEeqnaV9VpYZ0aE05XCDRSljSsoC29KNWvYEnnqS913dDdNlf8v3Xic4ym/ZwzHDpnZ0dLC/Tfpys5Gg3WVE5DG1s8HzfejWaz4ZUkHxgSYvM0q484dj5LwUsBFQDc27ZjxkLCOhiLMZiLMZiLMZiLMZiLMZiLMbfkvETO4CHLDIhhYOUD6GzQqFkiOU4jzHODTG2hm/+vy8BAFZOnUHGlT6XT/JKAVMmfb//0W3UuDNHYilGAUVZkrLSJQoDzzQFhazErTvUQv8f/9n/ghu3SMUnVwo+n3qvX7mCR64+CgC4dvkS/ut/8S8AwMK2ID1I7pRVPAmtjPiBRIshD1tnlhCZUzl3idyywF3+vN29A2xwWzzLMqQMxynV1Aq+SCkhmIBpyLmTydjCSbKMoBwAUKQZVRbM31kfj9J2OEx3T5UCLYbXhn6A0pBUhQvfN96ELiR73QhPI2OIbYO/33Tlt/Cjb/w+vUf/+3hsQp+9s99HDlJ7vf7L/wV+/3foO9698R7+wd8ngZP2JXrt8KXfw6177wEAntu6goqg79LRY1y5RN2NNC/wg9eoAuKLTawzUT8PGEK5GiA390uX8BjuEigHgtW5HEdDcAekwiIlnuOix4Tyl77/OqYz9kryQnQYUupJoGSIJ6RAxh2j771Ec7QoMgwn1BW8cvUqnnnmaQDUkTF+bGmeYsZQoKgUmLGgUOfnfx0AsPmpZyC3uFoUVgADR9SlVW7qDgY4q0ngpL3UtOpPxt/OT1ILp3QlLMFeugINFvkYdSe2xR9WAjTZtDCeUrVrpRXBDZhULAV6XIn/8KOPcOs2zdmzW6fQrtO8qdVqePM1Egu5dI0gDL/6d38ZD/YMVCvDpfPU5bzxwTt4+y2CGP7MF57HV36BunMZdzKzsrAKsGubmwiZ1D1LE0y4onfmwipWN6jKGEQFlkDPP+F1MxpO7fw+6N7H2ib9/1d++Ut48xX67CD0kHOXLZ5N0WKoUsqv3XqwjZzhalkaQ7FoTgppSfWlV4GRnMtQQmUsFjWkZ1ELfKwxDEwXCh7Xx6J4guyY7k02HcBbpe5nuLQGwd2L0oQwN0DnFM3BkyJFxt36iiow3qFuePHaS6iNaJ31ghrcRwlBYMEazhxmorWAY0QXXN92mZeW2/gn/91/CwB45F/RWv7GN/8cTz1GMOTZZIr3P6RnPx700WiQKEEYBbj8CD3brbUV+Ka6aCAuMkDCBPzjk2MoVm3sVH0EDNV72FNQ67mH2GL89Mb9Herunown0GC1u9HMIlpcKeGwgNVeb4hJRh05N6hhwkIItTZ1spM0xUfvUyy/92APsxmthzxz7LOVngtHGlVID4oFTgzlYTDs47vfJ8TIX3z7W7h1h1AlV8+dxec+RyrBjUYH1Tp36qohbr9JKBRD5VBKIGeIZZLmFk3TakW4dI4QGlcfuYBmm75vj+NZrjSOGOKPskCb0Qbrq01ErJyZxWPEBjVQpCiNqB3D0SfTqe2eloWC4pZWlmYoUvO9BcqEchjZaqJVpThV566D55ZwNIubCM/6ZmZFYlFMgRtYH9ciU5Alw+8K+i7jbIT+hHKgcVGHF9F3DZdWoR3uIiYF3Jj3Ca+CTpP25I0N6njF4yFO3n6V37eAx10zN3fgs39xEDbgcfdLaWn9Ev3SwMdcuAynC5wQAV+z7wRWGdGTkUUoWfRRJqAS9sLTCik/Q1+E2Fwm9NBaZwmzKT2LTCXWv3Uyo2uLE22FSaq1JhoMWXaDilXgFtIFNx9RZEDOYnOtRym2tTauwFmjztWlyEcaUdz8aGcXIxbmCps+AqZ41Foh6pyrypTyiThO5uvJkfAC041yrR8b5Qocn30PDd6DpGCVyjzDGsP97vohHrBw0a1PPsF3vkPrZWntNDYvkCeduxTghBEkkrtqZ89vYYXn/N3tWzg6pO7S0tIKtniffuL6NVy/TPvI/TvU9XvzrbfRH/J6C3y4nFs7mKNvqlEFTVZ2DwIXE+7CGgG3oiisAmYQhVhfp85Vlua2AyUdPadrlYVVGTZ5vSM0VMm0pfEJcqZIyHSMgNvuUeShw4J6nnIgeS2mjEDJXQd+xeVr9lFn5Ni0nCDuk1ejcjVCj9ZC4Psoeb8y1C4XJTRT0pwwQMGIoUZc2DnRv/u+1asNTp+FIymXNUgHUqc3iB0SlAKAUglEDEW9dv1x/DwL5DDYEMfHXZxjV4XO6jp2DihH29ndR4VziLAaYPMU5RZXL11Es93ie82emapAxvN8OBraHKFZb8DjTvvDSqEa827gjxs/8QBYGJy8LqEYhzpOMuS88pQjkLJKX6KAnM07/UmMoy7BvEZ8QOwe9zEb0CR45+0P4LmG36ct13A6S5CwBPJoNEbEHJaYJ+NkqvDehwTteveDD6xCXQ6FVzlJvXjxMjoMhfAqLqZ8WFUMZ3W9CnI+VEldwEANHOmiHtImc3ZzE5tnKFjdBH3ee+lbtl+aFTk+/Wk6MFw8fwlD5jvAiVEwtEErhZxPsN0TuhcHu13EzLVIkxIJT3I4jlX8caQDMP/HdVwUfIiZaTqsvPzWH6MqabJurNbx0V1KJMvSw3rziwCApfZlNJr0d/s7O+h27/P3pWQ07/aQM48w7yrUvssH2KKKD+8Tdvls+AK2LpBM8eu3bqPLhrTX2zSJf/G5r6IOlu5/93V4rBh6pv0z+MVPfxUAqXwOv0Cfk04LvDQj83ncJsjm6WvX0GtR8CkiF65v+BCunZlSAz7zTULDh5Ce5SYVhUYlNDAZbdvmrnQwZKhhnhZY6tDm+MSnCRKpVI7jHi3C02dOA8yd6h8dIeJNpiF8xBw8hNLwmJfQevZLdHH1FgrBkB/lIDDmwlluVfoyOOjzHBxNx8j5kHjINgbJLMVyh+7j+bOncNSlRE3Dt9Ym+aywRYFaLUSzYQ469Nxc10PB90iXGTbW6P3uHffx1psEO/zMZ59DtMbrwvPw3AsEKbl8nQ6AreUKbtylzeL/+ZNvYXjCB3Ml8Uu/SIf/p594Ct0+FW6mMavrTqaoMEdjPJlAMcxnOJmi3mSYTzVAyHYgjqtwwurCJwwlVyVs8JzFMZSiv/vSz30WUz6c3T/YRhwbboxCbhTjGF62NxrCZenw1UYdrceeAAC8/L2RtUl44rlncIlVUSfTEfbvU5J66yM6DPePYvzsRVr3dS1tMtTQEsmYIXfDXWQpW7m4AmCIlgmzaams6qhTW8KMYyKCErLk9XfzZcR3iOcxbGzC/3v/Kf1OafhDcq72/FBBoizVHN7hOwhXaIN48gXij1x4/FFsXqQN697uEB/wAfD//rNvYW+X5lUlcHFuiw6oX/vyL+Bzzz5L92yN5sHxaIqXWUm01x8gYlPiaxc28CirDD8MGxFCmPxnMX6Ko8fxOEWJhOFoRZpA88SpeAHYjxluCkwz2j/K9z/GrrUtICiodjx8+MGHAIDd/a5VVIzCGtLMJLoZYk7QgdiqY8dMb/jo45v41r+jhPa7P/ghhlOKrXk8w9Z5mm9KunA5tpZFgS6v/4zfLCsKaFNYKkoUfHgYDgcWklbxPWysr/2Nz3YAjPn/++Ox5YlFUc3qt+daQXECnJcFBgNawz0+OE6nQ8snLvPYKoKrvETO8LW8kFBc6WnU6wiNAjorbsb9YyRs85TlhaV+lHkBj/e2Vr2KiDnffgCkzGU6PqbY2z04xuSIPm+a+FaJO0xbmLBCZ5oVmEzYzkjkEBWKD1usSCiVQsZ5Q641Tm/Snn3mUgMBQ+A6zWVr8RHWGqhzAcgU3fWsROjRPfDDGRwulqmhhOOxbUCk4UpWOOfYrIYBpoFRgk4wGtL385wADVZ+9+sBRlycmGoHBcNthUk4ZxkctqDylAttCrnJFEbKQQkHpVFtz+aQwOArtFe5a8vWcgmRj3ZG19Fqh8jv0P1FWaLKB81IS2iGHB/s3edncoTjE1orTsVHJWIeu+tDSs6Bixy1Oiun1muWE6n4u7TqNZxep9haD5tIuQixs7uPV18hytTnv/xVXKw+SZ/jZ2gs0TNfWaL9+unnrqOc0j3Y3XmAGwzrPH1mihYfEhpVb65uz/dFCI1qlWkTcyAhykxDRgzNjWqo1rk5ggwpz/XBhPfmvETK3yUKQlRr9H7L6x2UrlGnLJGxmu5gPLG5rLGzQJlDcg4fIkOVufyINRy2OQtVhHqb900RIh7QXjk4oeupC41l5gy3fQ8Rrzk/lZj2eW+eppixCrnbbsFjmKtgqGo8m9j1pgYzFGYOpgVOxjR/h9OPIViV/cz1J/H4F75A72HUsKWwVA2hFIShJRUlMqPErjUunmM9iGco7o7HfVy4SK8F9Rbef5t4kG+9+j0cMGXHERpLS/Q8P/PCC/jcl34WALC2RrnrdJriaJ+g+g92dqwDw4WL51Fv1Plel9b8XUhYCP+PGwsI6GIsxmIsxmIsxmIsxmIsxmIsxt+S8RM7gEpYXJOFepRCwGEoXtSqw2NYSHN5Gd0RnYBv3NrGPYaATJiIWWQayZhO3LNRbIncjXoFR4dUbfC8wJpH9/sD9BiC2u0SgX3/aIyTPp2oCzVXEUzLHG/9iCAsuhDIWVyl+/gSGky8PD6mk/M4GVhD0rJIUBRz89f7DGP4/Z37CLmrYYRaZnFiiamOKy1Z89233rWeNbVWDefOU9UmT1OUXK4aHlKLun94aP1sZRRgZlTJhIA0MFhV2LbycnsVGyt0+r+fU4cqUUNkGXdB/3IbB8f0LOqdNRy2vg0AaB8dQmfbdJ/iHyBU3IU7Ye+g8jxqHaoK3uppyMYjAIClpSb2bv0VAODP/uf/Ho116gCOhgP85cukFnnpGlXxms0m1tnz8MM7BX6ZK29Xtz6DQLK63EBhpUHV2HAtQCuje9bLqSPmDg6xxDDGaRTCEXSfci0x9YwPjIQ3b4fQawLQzr/vQ+Y6DlzjqeZKZMZkvigw2TMeZsafqsAKV8eyfheHXEVCqTHRRm1UIDEmwEVmn/noOYKAZolEyIXYwNGosKjCcnsF55/+DH3veAfHrMpVdfYxYTGaPitCHR30LCRrqd2wXofTPLUVNNeRqNZ8vu8N8gICIPgerK2vIOFq1ng8xmn2vtrpjaxH5aA/gMvdISGAy9fo+Z+7SDCwwfgAZ86zgfxv/BJGbC58/swW2g165u++/SMstwjOenhAFTpHCJzbOgcAOD7uYcKiT1G1amGrhYqRMRKgSGLcvEXXNBrRe6wsryGe0fX3R134LCx4Z/tjFCx+tLKyjC5XyU96hxgtU5y4tkn3YLXdhhgN+ftVUFmi6/SrFfgcU9TRPhQT1AMkaE4JirLDYfBGLPAEV+obSiFP6J4HrgdXc1W7FLh/j65/dnCEhGFSujQiTtrO03Q6Q85dAL/iQJvIM8uQ5vR+02FhO2jWW08K21FUQj9UpRO2oic8D+/eJAGfu0z6/7Vf/3W4XBWs1Xu4w88+iCLssaDAoHuED96nDvzbb7yDz3+O5uk/+Eck/jSYlfj+a4SmSAph4W1hvY6zZ+kZNaUD6cy3DrWAgP70hxXecayCa66EhdPlDix8vSiEhTfuH54AH9HzX+P4Xa9FuH1zGwAwnCYAG2Evd1zbpT8+PMIn/Hd3tu8jYfrChBV0H+zs4MZNmm9xksHUlu/uHeDVt8kfy4XCjA2yx8O+7Y4bxWpVKpS8jkqloHhvTpMSt2+xwMV+D0sMXdUMF42zEh4rawrfw5SFTj785BYOWHXUr4TIjS+sHyBjY3ajtNjvjpGxT6ootTWu1kJb2JXKNQwmMAx8ONzN7Pc4jty5hQn73o2niY0J9UYdEa9LLaUVhwkqEiWL0BwyYiqZnMBnsQs3bEKAunonA4Uuw/lmsxhHDPOfTcY4s0WohdU1inmRK1BwrAmDKpaWqWPa2dyyPoW+9DBjOkruSOTsj1tNWIRCOIiY/uKMhlDmHigBzUqtwnUB7q4Yys/M8xEyWkzIEjEjupwggttgU29fWvpF4VYQcBeuzflBoQDBomDyqI+sx/6B8QzKiHRphZChrb7jo2QxEdd0/YIQOceoOFNW/bQiJdYZTrnajqyC6/BkgO6IcsSDe5S7Hp/0sHdEuWepAZ+hjQp/02zbdGI6zabdsyfsAwel4PLftZfbaHGH5/7uHnqsWl1kGaQx8lYltrYo59s8RYi0RquOk5KuI2yHOHOenvPVR7awtEyfPR53cbRL+dNdhpnGsykcj5FBkMh5TsezBFGdcjTHkyi5e16WCWKGOCeJ8UguMGPZ1NhLEXHu73q+9cDLywwFr4VZGkNxzrTBInUincHh/b/pSSzxHjspM2gjsOQXEKb77CooxkwnfJ+HJTDiZ19TgGs60nkKMeFO3mSIuEf3NPcjeCHPUxb1Gc+mSNiHNB1PbJc2KwuMOVcc6QFy0NpK4OBXOJ80ntK6LO3ZRUFYZWygRMFd0FmcIo5pfte5Y7qy3MLmKVqHrlPFbe5mHh8PsfuAnlcy6yOZ0nr54P0buMHovs989kX6bAHc+Jj2/MPDPdQYCdYdTPAi06pajeockFPM4a8/bvxkCCjjb5WrrJnlLM9IPQ7AaDbFjOFa93fuYZbTTSpLDYdfLxki4DgBBEMYNBwrxz+ZFfjXf0DQwCjy4DNGdzqd4PXXCSY1njB0Svl2UZC1AmOMpYTDEr6t5WX84K//GgAwTB61yZhRF1WltAcG8kHmJDXP0e/T5EmS1CrvGb7L2voaJP/dIxcvIuBN8ht/+Ec47LHEsyfwD3/zNwAAB/sH0Am932eepgAdOl0kvFHlOiR8IwBVlpZroTWswWOe5rjNm6raIJhX9zjD9IQ2nFkvR3VG731ZCpRDMomW6zfg1Ola/c4MrqTn9eev06bXHaf42a/T5zWfrUKcIeWscv0yPrv5OADg7v/62wgK4j2tbZ0yImX4k5cIIvr5n/sc3mAFydWtF3H2Kqmw6kCgx5C7fDpF9oCNWQMPDQ7Iz50lGN4jYQ16RNfUG5xgwlCQozzFDuPw14IaWkaJszDGxsKqi+lSQaWGRwgYXfujcd9Co1bbS/Ckgf8Y5bjMSnrrskDA86AUEjc4WCVK4zJz65wyRcEHxvtchKj7Em1TGEENPidfK50NXArpECYnTYwY1pnlCprnkIEH3717D3sHfPisVNDgwDzu9TEa04ZZlDn8gDe7io8JFyUmnEC5QQTXm3NnTWFEOtJyUpXWVm1PCIGN03TYa3To3iqZQvOGH9YCVALaSIPIx917BDlJ8wyHPdpk7t2nf5996imsnzoHAHiwv4MBW1BIx0XCRrBFEWPGPM3hqI8Zq6YZI9/hcIyyMEpXGrmJL8MBZgw7C2oBpgxbrtZbUBy6aizZ/eTaEkouFPmHx4iHBLE9Wymwss7Kdif3MOtRsHWCApHhQfB7xVEH7hrB1fL9T+Aw4i0tMxQc6A8nM9xnxcG+7mJmFTzpd7UQKE0BISvt8/amHpupA1ACM37+G4+s2ddNfBGOY2Em0IBwjOWCsM/W8wNMeO0Y8mA9iCg5AdCu+igyiptFnuPsWeKY+MKxsL7hOMVf/ZBi7JnHKb6sbW6h2qQD/86dAxwyBNer7aDBsfnzT1+HY3geStni4GL89EbAiViSZyg4pgWOtND4vChQjg23LkFu1LVLhbSgZxoz9WJlbQk9XreTJLHWQ9PRDH/20p8DAG5+8jZefoXU+3b2jlEx8ZcPM9MkwdEJrb9cl4CNiwXu7dCaqzgaHhdg4llqDeK1qZ0oZahVUEWJgg9briOsFHh/MrM2SA0uyFaiEDXm+vm+a7m0d+7t4ROWrR9NU2tQ3Vles3yvnJPfdDy1RuVxVtpkucxLCJv8FZbXFFVcW5gx3OQkHmI6ptiQZQWikGJ5reqSBD2A3YN9jMdm3ZZQMKqc88PwCvPYt9a3UF2iw0BWCJuYZkkKwcXqOB1hnxVc9w/ou25tnEZniegDwvet7Yd0a7ZQOCpTJHzgVa5EyBC4KlN6mtUQdceQ7OZigkUZQ2rmAyKHKOY0HACQRYDAFLKUg9xhJevchzPkgwYKePm8kF+pGSVw+rtlWYUqOPGepHAK1pkQCWRhYqFEjSuF2pGYcaLbNyqy6dQeLkZxhh4fsAOngstblENuLEdoNDgfSGLkPGcFw2TjkwkGDG11fB8Rc04H4wSaP08XgDRKuRXHHipSXm8oNeIx56+sBQEA1UYNLVbXbDSbqIZ8oMkk1rZW+HW23ErHEIYPePkUNmNqMqwvd6D44L23v4e9fcoLTeG10W5hynttkhUoE6MOWVpOqutKTDm/zooYCR9ijBWQBpDxQS5PU5R8f1UpLEXMdSqo8TMESmsA73KcCXQOn2ljQeBgZFT4xXzfRJYjY80A5QtkptBZo7xH+wK6TesJFQcjXi/ZaIpsSHMsHUzQPeJCp9Yo2eakcLiQlOXWWibPFNlXAFDaRckH5UKXKCXzeJnOBRA8E2CqvDA82Tn333UcFBxL8mSKspxb6AFAZ2UFVdYhcT0fbVZL910Xri0I1SAlfa/hcIjX36R8vrNMirLtTgt7rNA87I8w5nkV+RVcv0pc0OVOBzYh0CXEQgV0MRZjMRZjMRZjMRZjMRZjMRZjMYD/QAfw3DnqYhz0R+gzuTLNUlthE9DIDMxLKmRq3pEzcLnIN+RJDzELR0BICxtyHBcvvfQdfl2hEnFpQmjbtTNn2DhLsM/iGUIA7kOdPAMdvf9g1wolZGVpTdhNRcP3Q2iGLQjh2OvQWsFjT55Go4GQBWiaLTaiXl3F8QG1ZM+e3sQnTJpfW1nBc88SufPDGzewv0PVuErUwDZ3yPQVJls7ADcL0R/PEHOnTKjSUnSF1hYmevrUOs6epmdwxC35Jy49hde/R92XZf8NtJpU8SjSs5i6VCnYvX+MOwwHWFoCHn2UIDN7I6rQ7m6/h0dv0fe++PhX0Wiw2Ih00blA1bHPfOHz+OTffQMAkNy9A3/9HABg1qaK5MEQuPZpEkPp9ocYsilrXGlAcyVPLBW2LZBNgwAAIABJREFUW1zkGTw2i70aU7Vo7XgGLjhiww8BniuHZYGNGf3OtWoLG2ya7RrPRt+xIhkoFATDU5xSW5XHj+MCCXdYz7VWUWfiujG8VXkG15hqqhIlk7pTrVFnoROtPXyxRtU25BNMuer+1yyoUXMlOgOaH26+joIhB5ETktcUgEZzFVVWxlxyFQbcmRoOqdq1vLyGQxZF8QIPZ08TLGu/P7AeM44j4XOFyvU87B9TFcgIGMyS3IoFjcZzJdqyKK0/YCUI7OsQQIMJ19U63ZfeaIScy1nCcVBkRsBohIjVuYpyilfZy+7Zp6hjtNRy8ObbtBZ6kxGW2M+o1aqhErKYkScw4ap3PMuwxNXpGVfyDw+78Hn+VKoeVpZozodiCUVBHeSqF+HMGZp7eZkh4E76bELrrROPUJ3Rzw01xMGAug6NpG8NcFVZwDNGsUVhq9YC3E2BQLlE8392coTGjKr5eZqix1Ct3cEII55DU0cgNyRr07lQsF5mTuRZoSqlrXAckiKHw3Pl6WefhuRna9AapKtiYtscbqSh7etlXuL6VYJoXzp9DgAgIS3srFmv4XlWtr19axsnHFQ21tfQH7AIlu+iyiICY+4GVJMEFVY3bHeW0GVUxEc3t7HiUQx68bknLfwN4mE41GL8tMYKr7PheGDFoDzXtcgC6ALSKBWW0iJMlChRsL+oFzJ9oN5CFNJzHiYjZAbOXI7wp9/8MwBAVPFxzN5WlWoNTe5OmG5DEmfIuOrtOy40V9994aHkfTxzJBzu2CdlgZznnEW/qAIG8KzV3B8TkNbPz/UieLxemiGt5UYrhOsYESwNj+O+X/FQZ9PysJ5jwrBVlDkm43lHFADSJMNoTP8fZxkUq4PneQ7Jc92TGg7nHFJpJKzwa2gFQSVEvcEeeHEMh+FjhS4sJPC4e4yUu3B5liJnKHhuBE20wCp3YFfXl7HEMN28EHC5i1imY/QH9B3DEFCs8r37gESfmlUfjRW6jnqjjRorJmvHRaEZKSIEFHfnpPBwlVWa11hoba1Zhf9wrOTOWzqKUSh6FlW/YhUf05nJ1XxUDYZfaswmFDcVXIT8uh8oTIa0p8/yEjVBsb/KdKK+zubQUemhyuiWyHUhfKNA6kJwvqYrGrt8r2/e2qb77/bgsbeh9l3wLUe9GWB1lSGbdReNkNdLodDoUCyM1glieW+aolAEPe4sL8GvMc0i3sUknXfKSm5h56nCSUKdxgHTG4TSSDgXGI7HFjoNrVFnb8hOp2PSFkgotJp0n0pJc2Y0ndm17EcltR0BHPT2cNJlkcW9vs17ty6y6FLuWCXoyaBr11YU+RY+CCEx6BPSKCnHSAvufvH1OHAhOKYU5XxfSrIc/S51ccMowAoLkgm3QMKwcMnKn67OUGEqmC5TCFZAd3Q598BVJaYzFoNMZ1AMMzbQ7qLqw+2w0nk9RHJA+e3+6AgnPRacnJSYcBdumJbIOE4Y/1IltFV1TUvYfMipSEiDaCmBiLuOG5vrxvLP0jQk5HwjF9paHroSVgFdijpqVXq/LKX7Ug2rgDS+5i5WVyinWl3rYHeH5nFS5AirlKtL+8EAuIO5vNLBWRan3PNdxEwnGo0HGPQot9cXTsPnmKfKEs5/oMf3Ew+AVy6fAwB033gHkqEKUs7hAKVWVgJeCMdetBAlGswdqYf0/832Cj6+RQeoVBVQvHFoJYGCzVAFILmvvLKyZPHI5vA2OR5iOKLJ6gng4jma6Pf3duwiE0KiwsHg1s07OOEExloIuBWkLHPtuYGFg8ZxYvljq6uriHkSGyhoWKlY/HOZ5RiyfPTzzzyDSxevAgCuXrmAe3uUsLZXVnFco0T37Xcp6d89yLA7oACQaB8lQ2wD30GzSsGgSDMULNUsRQLwIuoe0vu+88NbmE7ogHJ6K8DREW+67bPY69Lrb31yA6cZJz4OUry+T9eac5S5cPUizl8iae7Hzn0KmWI1tlGMwZhgJF94ropaQsnwvQ/eRU0SLv7kFi28N+ISX/z6V+h9axKjGX1Gs9qCkgZq6MBl2GToAL96iZQZl1nhSTiAY3hRuoDLh/UzykGds+Gap4l7AfAMBDKHzG4BwFHzgyFKjZIX50otRGGCmOMjM7/Dmysy1/qLCF3A47JGoCSeiM4BADztocbqmrHwUOVDSo/VRccFMALPk/snUMw/6NSaEBUOAIVAq83WFIGLtTY9l3qHXktyiTv3twEAqysNnLtEG/5Hd27DAb1HteZhlbkDeVJg9wElYhNWtNK5ttCAQgaomA0fBZZb9HMjqs6TQA3U2fB3zFyMOMmtslyWlAg9tkGpNPHmKyTX/t2/fA1/5++SEfyFs3RIK6c+3nuPuEEn/SE2WMa4EjpoMWRjPBnCZaXWJC4xHtL9qzCstVZtoOTDZ5qUEKDX2611qw7a6x7jTM6mrJ5EyvCM4XsEk2iJDNGI5qYoYmQpBcRKnKBgSFJSavtzoSRmPIdmHCOHZYayTZ9RtFeQDam4AkfjmA+wR3GGCW8sM11a/JpLyxCqhMVVaGiryqhcB6WJNVC4epkgmUvrKxaK4vBJSghh9xho8VAirG3hQwiJlRW6VrX0EG+K57mQDp55ipRedanwf/3u7wIAfvTOu8gZ+hcFLau8m3G8m06GSDkxr1UcPGBuUL3m49rVawCAwK9YLo6jFwfA/xhjjROI+/fuI2Y4oJfnlkMndInC2J/oOScvcF1sMMT3Chf7ljpN9BlGeNiXKG2kFYinFKuhQstRVTrHmA8/RWbgjykK3oPrlQravI4C6SBOKcZkCvBMsvyQwbpRbmYCCb2vfvgAOFe/DcMAESdXNYZxVmsVmEpimaW2COK4LjaYd9OstxCzwuHB0TF6xxSDfD7oiYqLKhes0jRBybHSlS6Ea6BzLiqckGaFRpfvWcGHuMnJMTwD15ZzeFVaFjjqUzFpMJkiikwu4qGYspIzK38Hvosq/TcczOAKus9uWMPaMsXCo0MX7Q5bOIgI0xF9vik4q2KK9XUqltXbyzDDr/qQmnUMCg2dsqq2K/HpSxSPzvgU910ABRdWSzgQJfO2wgTxjFWy6zWEnKjPxobD6KLBuYxEiRSsVhs7WOJiU80tMXONOmKONu9tFU0x9jgNMWEaRlgBqh7vsSqGVqy0CM/yoXMlEDGP8KVDykOkmKHa4hyi2YLk/6/VPEQMywsiwA9Z8r9SQ8CxcHCbcp3twxH6bBp/+sIlnGEIX3c8xvjBiJ+Xj+Vl2vMc10f/gOZEl7loUBr+hHmv09TeU8/xsczxO4iaNt67nmvXn7G3SuIMIV+/5znwQoa2jsd4sE/5Wnd/iKXmMn/fNt8vx+pJJHEJFiZHu1FFi9XX03yGOKF8WHiFtaEKmeKU5yVSA/UMPLT5EKa6E/QH9MynIxfnz9HBpFWtIzVK90xhEcUMOudidf/Y8kVLKQDD0yyBLlNbBrpEzjFNcGG+O3ZQsuZB4bqoCLr+oxw4YQ7gLMkwM+cKT0JyI6IwjSIpDOsK8CQyLnJLSfB5AFBSoLFM83F1fdXGIGn3VTGnakgBwSdA6TlwOO+NAoUsZyP7nOknQWhjitLA6goVXM6fO4s7rIqvtba2JMIRlk+a8XmlEoZYXqW/G6eJbWRBzBVXtS6tHZnWOamSA2CN0H9vLCCgi7EYi7EYi7EYi7EYi7EYi7EYf0vGT+wAQtPJM4kHWOqwMMDhiW0rSuHMIXUK80qwKPAIG1t+6iJVoirVNu7eo2pFXpYQhhysgYzhD64IkLKQzHgaY8yVE2U6NQKWddnuNPGb//A/AwD80Tf+CG+xQlm9XoVgcu3h/UOMuNvQ7jT5vZQ91QsxF4QplbYt4YclDboMt6tVQ3jcDRwPBmhy+z6JU3z7WwSTefaFp7G2yi3ciguvSaf1XSr+4dSlS6izL87ewRGER9WUSiVAypWmwWyG5WWGR8wO8M6PCILQZaN1J1xGWVKl5P0PZ9AFdZTOnooRcLdwzWvhs1de5M8e4O2PqUvy7DOk+Pfsk0/izh0i9B9P3kAlNNXTCtj2C2NnH+ISvd9nn30cPqul/ekfktrqrXsDOK9RFWzr0hb2P6burto9hWrIfnONFXSNB54UePIcqY06DH2FI+GV87qv4OeMEmgbeJ4ordKYZuhDKAGHK6by4Q5EoayXUwhlW9VCaauOB+7oiSxHwbC+UuWQDF8SSmMLRgFOwOHKinRKpKy05LEfUK0WYZlNXn1XwvONSqwA+ySjFlbhcuU8y0sEjK04ZHWxt999F4I7SleuXoHyqQK0trqM+zv0OyKKUGNY3nAwwvGRUSxlcnMYYcJQm34yRaU+9wy8cO48X0eEKZvvtjtNTGa0LroD+ow0KyAdqir7vo96jebmR+9u40fvkIF5mfuo11v8+/QM33n9XTQZ9rS0tIwaewrVGiGaLfoujYaPlL2+8kzj1CmqFhqCuuP4mIxZqTYvIBRVzTbXz8Nx6OfJeAZYz50coyHNvWs9fhbjIwRsoKs9IJvQ85w4EfZ5fhwXmfWSzCAxZmjLIYsLdVUf7975AACwenyI1RGb6JYKXb53qfBQsJ+YFIA0nkvW9FbO8TMPde9KSEwY0tvZXMejDM+Mi9SSyucGPsKiKfTcZxjQeCh2OXPvdm1g7POuiaMB3zf3cR09hu9dvHDeQod3Dg4RsglwEVNcOtpN4fv02qg3xoObBO998olruH71Cuyw3ZmFAMx/jLG2RuiSei1CzMIjcerA5W6y7ztweJ6mWW4VNSuuxtOPkQLwFz5Dgl/j4RTvmU62AISktS+hUDHdqkCiIik+5EWKhOFaRrBJFQoer89Go47HHnsUAO2xH75PnlfxbIyCO0ZSCytYZGeSIJVJgNaWWRdCCgs9y7MMKa8Nn+NqGAIBx9BSutbDT2sBl0v+7WaIVkmxYtzv44SFcwykc2V5GZstyht2949xyEJfeaktNF4rhUrFGMsn6LHolCmjHx/14LBwSr1WRbVq1j6QTGnPrng+zp5ldce6jxEjiQruwjiORKtK1zTsPrC+wk5Yx5AhqqPBLiqskr2+2sAxK2COBoRY+PD9YxxyJyrJCkxYibi1uoWQhUykG9muquOGWF2h7lG7wnnNLEXCaqoQLpRgyKBTIGDBEukHEJwTOTl3ZYsEIQsGVioSE1b2DJwcYcZiKa6DDLQ3tL0ClZznLMN8I7+Gast4/haIGE1TliXyKT97UcDlODVBjpjzCDOnHNeF4O5SjgIuz6vA9+Hz+3mhB7dmkGoCXRY4+4CVFj/ZvgOH87xL164hqtE+d+v+Peyz4MrKShvXHyMEWC2KcOOY3mPKeV5QcTHjLvp4Vtiu5akzm7jIqtyVio/plOdYWSBj5JvxkUzzHB5DFKMogmeUKT1gymq2J+MRJOj+9Yy/ZlKix/DIJE1sd6nerMNnWG2eJ/C4w+35DrRjIOT0T6kFXIb/usJFh9FFDkKonOb6yWSAQW/C9yCA5G7a4IjyfTEbwU0pRpXDETJ+FrH0MWUBv8NJhh4re45ViVQbb2rusBU5jtgTcDjNcLHJeUElwJihl1MlrS954TrQvlGEN/haCRiERKlQGN9dLSwwrFqtYo0hwJTLGMVP2L8T9tTkPCQoKe3+LYRrDeALSzETVizN8SWW2ePxsceu48E20VW2t+/DDQ0a0kPvhGLDAXv/7WzfxO4R3cde98Qi4zqdFlZ4/Xqeg9J4ZGcpksR4t/7/j0UHcDEWYzEWYzEWYzEWYzEWYzEW42/J+IkdQNehClCrFaLQ8y5cg7sDeZpboQoqIDJeVig0ubLy4vNsX9CfIuBj9kDFKFmAY2Olg4Bx54cHfTiSKk3TWQJzPjXS1lqXFnMb+A6mI6pkP/HEdbz6I6oy3rr5CVpc0UhKZU/uBguttUbKsrZKCWjDP5DCcg63792HzxjoKUvTVoIKVltUAfIU0GzSNe/e30bIFaVe7wBRiytoyQyZsaCIqPKydGoDp06x98ikB80dJQgHY+6wFWVhpWqLbIp6gyumBXEZVk5/Dts3qQvX/fgmTjMROt19gDilKuILT38NX/ws2VHc2Xsfm6tMnmVPkoPhNnZiEjJJ+x8jYC7ApBcBU/qcg8ERVEBVriSrImXfNzSYjD9JcOMdEgS5t/0eHn2U+EGqcYDQbfO1XsXKKnWAXVmBE9F1eMwvS0IHfk5zolLKv9GxK3muZI6GZzDjXcaUxwnAhHHySTOVFwHBVgdyLrpPVR/TXZRGzteDMCIlgQ/B1Sw9yxDmRnVDQzMnxW1U4Ef0vIwwUFjxkRh59RzwmIBVq9Usz6AWSLhczVLJFO+/8yoA4PsvfxcAMOyf4NHHrgMArjz+aUwYcP/885/B7e0/pntQlFjfIM6dhrDWCAV/9vr6OlyeB8VkCsWV8V/56tfxpZ//eXpdmTVFfL8HO9RZNjYnfqWKlLlheZnDYTJyu9lAWVJ1b5p2URQ0T32PquW37t3B4RHLNzsSsxn/LNoYcYcv1VN4XI1bXl61su/GJkLAQbtD3fLRSY5Ty9RpmgwV+gOuQqsVvPcGe/ggQWnk5HdfBgB8aXUJ2Yj5jKrEDptzfpQBd/i57CQKBfuQ5dBI+HXFdiBhJPEX3/kmAGBjNMSLy1wd1hoTjlel41ubB1c7cy9SrvjlGpanp5QGNxpQKoWcK4TVpQ7GmRF/yJBx1zrPTbdZPuQvNK8yaq0td1po4CFCIP2dENBWjUbbbsV4PMaYRQn+8T/5x/C5Mv4//NN/ijsPqEqrmR+VTMd49yZZQ6SzDP39bQDA1a//rOWTZlkGh6+PYvOiC/jTHhcfoS764eFFTN/h/aIsUWMBgKDioygMukVgxpLsuihxkYXFLp+lWH/7k7uoBjSPfdeFzxyuwJNWUCUuC9g6eKGtvYkwAimusBzWKAxxepM+Qwngg/dov0qTBC0muAWhb/n1RoiN+IsPEUot10ZaufU4zZDyfhAnBhFQotaga44cDz6LPviOAArDsSvg8p7uOQ4S4w3Ka2htpWX5TXGaYsyIipkobWdKawWP3wNCIGHxnVNn6Vk4jo8hewLuHx4hSwkV4wUBPP6OjZUmHtky3MsGjo6Jtz/sM4c7V9DMf9rbuYXpLRaS8wMMxkb848RqGlRbAWTAMZJ9a7rHffTZ/3AyGqHGnLZWewWrm+cAAOsbF9FYZn5gFCNiJEua0x43CwPkVYOIcqFKymFUKACL2tHIEp5j7GksisDaBmDmAWwD5ocuco4xZVIgZ56bLz2kLOlvvQZdoMVej6IsIZgHmcUZ8jpbKcyEteIK4KPiG6EP7o5JByUjhvI0s8IdYeBZdIUjNVJGf/ROhnj5h4SK+s5ff48+Qwg8+dxzAIDHnnwGkjtvV+/es8iw9U4HLc75hoMeeofUFZbSrIWK9ZJWKNFq0T539emnceVR7pJLbfNoVeZ2b5NsneC6gBTz1zyGFzUbEVpNel5xK0a7RffMzOnBaIwsN9odruX3BRXXktpUkcMJ5rxV0z0yXbxAuJZf5mmgyv7YUehheZm+yweHXfzoPUIJHXc7EC79/hZzFYvRAIr3ep0WmPLefDzO0GNxpF5cYqy40+u4tpNr/tHCwS53wMO9HoqC5mvoSeRNWrdTOcWI1X6kEii16XKanE8RYgxAVgITIwITOFip0/utbaxiaY3iolPxrIik5Q4K1i0BaSxYRI7WD1m5PRQfeZ/UGjZPRTnn+586cxqPXKZ8J04zVPlsVavXcec+6X4Y7+jX3ngXd+6QqE+aZlht03We3lhBg/MnuAEcTrAyKa1+yo8bP/EA+MUXydtt5fQWXn2DgvjBYIYL588BAO7f38FoYNSfJIx9se9IrCzTZKxz8Jz5Kar885Nnz8ML6cF3Oh3s7bCKz8BBpowUHqxJu7LmqxU4vDiqQQXf/otvAQDOnD9nDVr393Zx7RkSGzm/3MKr3JIvlPFDS5Gy+EpeCASBgQP4ljw+m8VIOBlOU5M0Z5ixj0pdOtaDBpHAY099CgDQbDUtLOTWRzfw8YcceJcpWR4M+zjmhOukP0IGTljTHELPDTt7rOykCoEqt4RPtwnS+cxn/xNkY/rsve89wCF7zHWFRtnk77LXxvu3/jW9h/8RZEAH5UPOUn/w3j4OT2gyXpicQsEHlINeD3Gfrm/S7eHsKTowztoZNhp06Fla4sPYeIwpw+xu7R8g5kPwk9eAYEiJ+u2DE2ycpXuzunYOP8OLyWfjc0dqGJml0p2bwkjJJrMAPC3swim4X63EPBGG0hYCJxXswgPmsDxBq48HzykpoBm66G+uQTPUJt05hDDE/EKzXh5QRA40P/IxH3LiZGrNiKOKj9Wa8eqrwLEqWgVmfZqD77z+A7z71mv8t/R5zz//Ap58jnwYG51lNJhI/OKXAnz/FTJQPjk6xDF7z+3d72HAh5hlJtW7XmCVJxUkLl4kqO3XfvXvoMEbjnRc+zv9wRhraywgwrdxOo1t4lSJqlCcONXrAb76tZ8DAHS7XWxsUIBaW6Lk4bFPXcfOX5A3pBRzn7q0TKE5vIzHY2xuUrAqigwjLqokvLaajSamDPd2ZISnn2D1ytv3cOosEew//OgG3nmHYlBztYmAVTQl6D32pIM9Dswf9oZ4L2dlvNkUI1bfypwACcOnHakRMkRSslpQpV7BIUNLp6MR1up0zV5RYJDSXBoWBXpGWVhra7A942Qj0QraMSJILqoMU2rXGjh1hg18r1/FMicNvufZ2sTcUF3bmKcEbWb0fgJmR1Qa1itQ8KYm5UNG8VLZA+Xjjz+O/+2f/3MAJNzwysvfBwD8N//Vf4mb27TJ/Oh9gr56UuLowV0AwPbt2/C5sNAJPYRcXEmTdE5oF2IOXV2Mn9o4w2rBz336cXgcQ3u9oRUqmyYJDg4p7pe5QglTAKWkCQDK1IisFdhYoziRaQ9uSGvcdYGCE+STwRBa0ZoSnrT+fyEfOCu+g4LFg6BLjEdGjEMiZjELKGWT5bWVJZigbEyWFSS08R/T87mslLTqpkrNE2NGnyLPNRJO8lY6LdTqTK2IBByGnJalBte6kOYlRnzAM95/uiywzgntYDizxau4UPBYTMLzJSIunnieZ33+OssUSzdOb1mo9Wuv/BAffUBJcZHH2FwlyO4zG8sEEwegygSqNL5fdG/zokRpPP7iGaYMDU21g70jik3Hwwnqbdqbl7yq9UBssrJzPPGQxez3N+jZQ0A6GGPIonjHR32s8CG9s7SEJ+usHMgiMUJXrBp5rn2kHCPTRFrf3cL14XKMSRXPmaKLgou6oVMgHrFfq5uh4EOaLxSsthASaHPAZkVOBQ+SIaAybMKRbFo+k4hy+t5uXgD8bCsaiHx+XindxywrbGEtrNXRqtD1CalJOQ6AUi5GfcpVbn38Hl55hfaxKRcHnv3MC/jCz1ABdWl5GUZE+8knn8SdT+hgXsYjjNhcfDY4Qcq0kiU+wNYbNUwNDC8psLZJ6/aZZ5/B+Suk4uxKxyr5uo5Grh4q8oHUfY0opFYKkv1rl5eaeOZpKrwnlzRaTdqvqjXamz967yY4ZCMplFVc11JZ78qsdOCaA32hoa3aCe85jkBgFHi1D8X7XLXaxCWGsG4/2MfOHsEUJ5Mh6jyXOqcoH8qzHHnMwjWTGboMSz3sDjEyUHI4KBmqWYkcewBVfBNcuLYR1C+AQxZwaQiJlBf2kSow4v1Y5cCU8yTjJ+o4rr13QguEnENcungej1+nZ7GysgTPNzGjgDI+yrxmCTLOsRSYey4LASHndDhtW0+8R6O0XqZKAtqIFa4s4+lnSKxtOkuhOOO8dPkizpyn5smD+yQi+dEnt3D/3ja9awHMhjSvTm+tY8oFXndjzcJmXUjUox8n/0JjAQFdjMVYjMVYjMVYjMVYjMVYjMX4WzJ+YgdwnYnm9/cPcPkCnUY/unHfEinLQluiudYKgk+9vnTQqVM17WCfpWLjFGe4a/DUs0/D52pPmubwptR12qytYsRVlpPB0LZX11apsnFu6ywOd+k0PB72MWFi9d7+ie0SSaFw9TJ1QB7ZWEGHCb1Hx/R3cZyaAzxV0IWxjxC2A0j/P7eVAIBer4+YvYOunF63Re8oqiCszsUzTAVzMprYjtb+DsE8uicnSMZ0Ut/sLGPGBN4kL+Cwb1JeKqyvU5WoGrUhuZ2bshdeL/Vx6ZlfocuMq/j27/4OAKA1yRBxpTKbzvBgm0Rx2ucnKF2qoL39Q/rsu3dGeP4p+oxHz/g46nEdYBbh44MHfL+OsLxMVgXCjzBjyIbTpOd96mKE2Q7D6ZwIDt/Uk4N9tLnstH4mxGxG17F7f4yDE6oMRSOGf3i+rfSUnoDHlRxXSrg8NV3l2P678ZVDtWFlv1FiDu9Uet7SUupvvm4FNgxWr0RWsvfLtITLkDyZFhAxC8KUGh7/nPZPMGQfQ3GaOjl5kUNzZ9nxIjiGWF2kyJnYDt/F/R3qsvzwlR+iwWIujz9KsM+rTzyN9a1L9HdwrLbGlauP4rkXSLTnj//w32DMUsD5bGrhUG320xOuhxlXGQulcZaFX8Ja3fpcBRUPXkCVrTQvMGUPPiPnX5YKjpFZVilKBHyrawjYE/PilTPwNMM6+fNOnT6NOsOvtABc39SUcgQVFgCQIQquOCrlI4y4istCBtVqFcMBdTi7B1384JW/pNfrdXRW6Z6eSev4/9h7s1jbris7bKxmN6c/t3vv3dfzUexFUaSohlKJpaZKqopslIAU4kRG8pcgP8mvg3zkM0EcFAIDMRDHiVP5CBKkAsR2kEpiWC7JqirJstiIlCjxkXwNX3ff7e9pd7OafKy55t6XRTIBEjAfOgsg3uW55+6zz9qrmWuOMce48Ojjoa+1Qp4T5XUvoOs3jha4R8j5da9wncZHbRV8BCZQIqMM57ULZ/H0I6HfU/KIXEqLu/cC+pUXJfYzer0qIUkoaVFa3CSxgJ2DIyRS+gRiAAAgAElEQVQ03GKGthIChhCKR65ewW9/I1ilrA272D4baFRPb68xTd15yXSXiNA2JpdBnCrqFwnZ+DAFwRd6vZ0dpjkkvIeNzIlOB889FwQ//uRP/gR/+4/+CADwD/7b/w5/83vfAwB8hnxN/97f/wfYI48lbyvUhFD88X/99/DYleAz+rWvvcypQwvPstir9sm1nXsBaZpMp2zPMB4PsD4K+8SDnSO2ZfBw0IiIXYqK0P2D3XCN+ckJRlSmcPVCDpmENSpLU7ZXSjMFRZoni6LAkOhmF88H9KvTyXmfm0wKzCZhneh2NSRlwfNM4xyJFZw5swkVvbeiX6FRkKKie/ZMfVZCMRrivWGKXMyxu7JEfUAUy+4QOa0vg1EKRehFYQ1mhBIeHRc4PAnr6YxoZUIcISP0tLIWJaFc87JGzmI1KdAL/bu1ucUoT0WI13B9hItXw5o8PSnw7nuhP/Ye3EJOqORgNMackNfpbo27d0Jc8s71wBaqrUdK67RMPFSks3YzOKLwlbZCSnQYrxLkg/DsUtof54sZJodEhYOApgVQCYtepE1q1Yj5mRozKg9wZM2RdHuwMZ6oHUpDLAoYeBHR5CVquoYeUbnLMkdhQpxRO4WkS3Y9C8FItZMSqhPW6lmlEO0GHe3NKllgP7KEEgPqDoiexJw8e6RzyIjJ5RdTzAiurDfJRgRFDOEwlD1083D/udbQjCDPsEsMsXffe4f3ys99MdA+v/rVr+E82RqZ0kLQfnDl6jWMiKp3cLIDQ2weV1cQEUCj+8k7ObOS0iTB2ka43qOPPoPz2yGm9hAM9zkvUBPqHmn2WkremwEFQbFRN0sxvBCevTyXIdMkdOio/GTQR7cXfl/Ux9DkwZlqwRTQyhv4MgrrCGiakyxC5j0MxXbWCRweBhTauARb58Le++xzT2Jvj0pDlAzMLoDZfJO6whGJ4hweT/Fwn0RxpiULwnjlkRMC2Bv1kRC6XpHNjBISSewD4eEj22qwBtcNcXS5qLB/N8ynwhoem9E3tJMotndbGw/x4pdDfPXSF57DFp11fG2wIMaC9+A9NK4/ToD3u8B4ivRSD+Ul/x3XuSCy3fQpBLCm/h0MBnjiySAitHN/Dw8eBNr41sYmHrsWENZHr5zw5934VbDk2t3dxTGJBL33q7fw+qtvhL/b3MSoT+t31oHmUpIPbx97APzRD0Od0u17d1CS0XQugDEpI6VpArGgwaM00xkyCLz7y7Cgze81JteffiJ80QtrI3TJhLYoKjzy0lW6XhcVwa87+3tcWXLhYgg80iTBrZshmP7pv/gZukS92Jsu+LDVz3RwrEQYVJubIViPnkP7BvAqctvdKfZS23coliJEWHe5LFDSYWB5Zg0JbSyD/iaGRCV85/p17vDBYIAsCzSYozn5wByXeOxK+K45FHao7s9CAkTD9F7iPL0n667hcC8EpHJMXO5awavwgJ/58m/hPgWsO6/+FJ2z4XOe/eYUyUaYyLfvSNx4Pwz6X94Mg+vwsABsuOcbbx3grXfC6xtXn8Jz18Jn92BQkXme1yOk5OO2rIiGUtyAJg5OJjX27ofXn3niS3j+iUD7XC4mSEnVcm24jR/874FmtiQKYKZTjE3DP48bkhKCAxZtJZugKvo3kRIiSiA6D2+in1/Y5AAAdcWvSwCCFg9BFABXFOx/U3vD30XXlhMB1juUxEufSo+avHhyCgKkt0yVzDodeFrwBCT7YxbGYkhj8JnPfRFntkIy49HHAu97sLkFT1TEurYQtIMMRj3mhm9sbmH7QpgDD27PoMnQR2miVy9LzKmOQiU5NF2vrGpkedi0hNKQdDDXSqCiAKdZ6C3X5OR5B10Kog72pljSuJfJAJZ8mGZE49jfX6JH/eEBpr5muYaj9yaJ4HlmjMWMPKMcBSbWNiq9VWHwxluhFmN9cw0bZymY27rIioPT4yks0Xt+fRA+cL+e45hOY0XagTsKzxYyYaokrMGYanfPba7xeqWoLmk5OUaPaCGDwRBPPBZqNJ7/0pfRpaC4NDX+7n//xwCAe//Hn+LKlbBIf/qp8N43fvVL3KL6ys21EX7riy8CCOuSrcN6MJscYFnNqd8lrlCiInq4SSVbBz3R3L/zpyQTmwNga0OKKqCsORYo9AUFKRcvXsK3/5WQQOqPRlgQ1f1JSkh869u/hx/88M8AAFnqUVPi4d3rb+Pv/p2/E/ppMcNXv/bbdI1h+7y6ap9Q+/U7bwMA7t/fxYISoaNBH5J8VSWAmIspnENGo2Ft1IUnWtP0JOzNvVzhMfLxQtqHpISPkpqD8tmyxC4laaYnc/RHYb5e2A7rGYSAFmHuvHfjNjq0P46GfYxGpBxcFBjTQaib5ujTwSXW5He8gI2UwqKEjfWuWvFY1k5ynWucCta66I2Nsqig6J67vREoJ4faShwekQL3/gkmC/ocCrbT2vC6KnUGoYk+W1osi1ijW7Na5sUL21xzdUxzaL6wOLMVXnvhxZeRkc/pm2+8Agp3kPYGOCbF49n0BHd3iJJJ/sCLZQmlo5pqiq0L4fOyfg+9UZiLen4CSaUhupMjIQqopu89GPcxp5hDWHDd4oWtTTxJiaCLV5/AeJ0ON97hl3//jwEAiQnPfri5hWQYfp4sFaYL2nt1ipzquYX1UDbu07QGmbKpCNaC/dBSVUPQAVYnCeb0OVLWiFxT1SGKuTNYkofccn4I3aVDazeFpQO7ny+h6BnWswVOFGk1PBPKKaRI0SVF6vHGCL0YFLcOvouiRklUws5gjOde+DwA4LPPBzrglavX4KMXovMc7A9HI2zHkoajuxh2Y7JaIyeF8Hhc89ZwEl9IBRc9NoXg/bYyln3mHGwTRxC4kkqFLO75KoGjOW5qAUvU0bosUJEKvCdF1uVsgoTinW63iwHN6+EgR4fq/ox3sNGM3Sd84ImJcuc8oty0tQ4H5P03m5fQFE+c3RphOIhUcI0l0ToP3w+00JPZAntU4nE8L7GkQ18FHctJIaRqahvXxuhQTLGk/vcW0TEa2tbYPB/Wq5c+/xmMkvDZP/3xa/hfjv5XAMDR7gSdQei/jXWaQ0rCU8x9+co1fP13vg4AuHLlEubLEL8uTiahrAGAd2WrpCi+5jhGc0Kwh6mwAi2tUH5/jFO9aJK3wnnEMBXCYUy+rBcvX0JFMU6n28HaRlhjh8OwTpr6RTwgz+hXXn8dB6QIeu/WTfzzH34fANDrZvj0MyFu3NxYQ0KJ8pxKuD7YVhTQVVu1VVu1VVu1VVu1VVu1VVu135D2sQjg7XeDwMiiKrBLnhSuXuKZpwN1almVmM4CnJslGuUsnIDPrm/gIkHn6z1Szkw1PNEtTVVgSgqGZVHB+5htOMQ8UqlMzdc+IsrV/v5etITD0eEeHiGk7Mbd1+EJQeumGauD7u8LbJ0Jp+s1UrmaX77aqFFVJWrKhlprGVWoi4KhZxNfM5YLPpdlGSArAI9cu4KjE0L6jg6xSdmGg4ND1JTZHxBCqPIML37hSwCABzdu4edvU5G4ByQaiP8uecz8u//234Q9CTSSX4avhHPrM1R1yILlMsG/8+/9WwCAyfTbeP2toGB4pH6KtAwUXKWuwpmAEl67HF4bnr+AH98OhcvbA4dPvRAog1meoEfPa+E3ceMO0UEne1gfRtQ3ol99pLNwUy898QxuRfWlvQK//b1vAABGW5uwJvS1tBb/0f/0AwDAIRVZ63yAPIq9JBIqIlReQBIyCAl4UvxUhFYlMokioIC1EJGeCs9YvU88o8LwntVeGTnMBFws9hUpatGhy1nOghnvYEhYpPIVMkrbXKMxMex10CGEM88SpGT+0skSplI4JTHeDqpvX/j6BmIeK+mEfnRJh1U7vTJwRGux3uMpQpX+4LvfxfkN8rA6fIA0DdnOSOM0dcl+TM4DOw/Ji2i5ZDU1qRRTxYSUMKR6askzq9/PMSKBhtpWiIoJ/VEfs2W43tF0HyPyOoz+Qkr1GQE/Oj5iZbbxeMgF6INBDkmI6J0791FRofZFQva1SjEjH8xeP0eWhmeRdRVKM6dnUbCQhTUlPNHA5qSaOsszLCkrXi+WSGPmVgimTkmpMaZM8LCT4+CAvLJIAMi7CiAq8NFiiQld4+LFa+hJYixUS+SEsAKS14T7D+9Tny84Ay5dDVU3KGhEiw8P9iAtIZRQiMZCnmlALQ3blrefEKLlCdj4XEYIzksBQdlVJ9Co6gqw6vKTTz2JK58KqGWn22+yv7Tevfj5F/G3/oO/BQB4cPcmDshP7Aff/wFKWnzfvn4dI/JafekrX258DFftE2tHxyREVRpGjIQAe8RaVzaekMIyLSvvZKiIzcMeUVIj6RP9Ls8YWZsul6x4W3mPCLNZlJichOyzZEEEgZ3dMFaODvaw26fJ79dYRVC4GoLWG1tVOE+lHTPyirVKM93fVhXKIorUeAgfkYCKlXVZqK0yjEIvlwvMZuSpJ86wh5wWGtaF61XLkhkJmvYfCYEyKkELCct0LglD60Cv28Wnnghz5+nPPIvxeqCNlVTS0OkMkJPX2nhtiDPnAkPp/PltvPd2UCkv6jlTc08mS1jys81JrGFRG34+XiYYroV5tnV2G1aHdeekrOETWguhsaT3J5FKK4AOIVE9NcLGONzn1uXzuHYteEBuX7iIDnn+labCT98OpRqavPqyPIehZ7GogRqk/tgdsHCe8A065EjZUXnd0C2RAIRQ5T5Bl5CyY2cwI9EKUwH5iBDnSej/48UJDoswvubGwGSEPiYJShJgU4slLPnCeWfhSQk1ljlokTPLLM86XF4iRaP8LpIE4/WwP35m8AK6Wbi/ra0QJ/naM3NMaLB3dSeTeOzREAOL+QGjbNZULFAU1eMdDGa0T9dWYkbCeSfHE5w7T/PJN3GL8AaSwnJF4mVp2kFGsYoA2NOzqCqU5KlczUp4GkuO1OqWZtnQHzOFLo2JJEmQEmomnYBXnj9PxRiBEEAp+BEjgWI6d1GXqAhlLssCBdERi6XjsOuEhJYmsxIF+XuLJEdvQOMnqWEjBVQI9CJiu76JhOjYlpS9AQlFFxZWo5eHmOOxpz+NR8+Gs4bKOvjhj4i1uLuLVAbUK5ZQGOlQuVgS08WYRPSMt5iTGul8scCCaKLeGI5bWMxFChaT8uGuEJvgMiM0IjrxdxBMHbXeQ+moDipYMfbc9jYUxSrrayN0iEGgcqImv/AMMqLGvvDiCyyKt3PnFiaEzL75+quwdfgul86fxzoxNDbPnMeHtY89APKk8UAR7RC0wNu/eBUAcP/O+1C0qKYyR0Z1Mom02NsJ5ob7FECfnBxzsAlh2XQ9z7tcg3N8PMUBLfrTxQI1BSUdgq6LsuASrzTp4O56eIAnhw8xJurW+rAHT0G0FMB4HDoxwumbTsAR7aMuKt4kvfcwFKyZqkBJlKk5LTjGWCwKUpgqZ3w4PXt+G6+9Efi38B6WFveiLKBIMawgGpWpa/zgR0GBT5R1U/vjGjVMQOCxx4Oy05NPfw5bw68AAHb/PAzsTz2xjmUdJkWqUkgfJlNvkeEvrhNNs3gM2yRr2x1NcPZiWEyf2QjqqJNEYqcMQf39Gw9Qm7BBfPbKF/DG24H+1c272NwKMHvtDRZkMTE7IcnxrIPeM6HWcjFO8NVng93H0qb4sx/9zwCAL37p87yYGidx9HYYsJO9cPi02YAPfTKTACkOSifhqa6ikgJeN7QJAJAiaeoQhGOym5CqqZ/6IC8tcrnpXwvPwa+Fg6GxaYxpFBidAaqG2hKFXy994w/D90aJlAoYtE65dgNKwdDnK+hI/YbIBCpavA1t2qkoIWkx0EpDxIPGosbl7RBA/O7XvoSf/yxIU5dlwavK4UmY9NZaUOyCRVlgZzc824PDQ1y8FJ6hl4IPtkoKlKTYl6SRi96H1lGNL0FBst7HJ3NW66zmC3TpQBnlkY0xmNFcuHP3Dnprl7k/ekRNqquKbQhMrbB9NixGUS28WFRcB+K9YwuEw6MTVuzN+xkfYL3ocmLmTBa+08nkBI9QjfH6hQ4/z+PFHPHinU4HXaqpGXc0Hj4MAWtN97Y1yCAoWJ0UFZa0wdmTKUCWOMo7JFUMdiQOaSzfuh3WO5FpSFrEx90u0livVBZss/Fgfw+gg62SCcuLy/ZBr0VHj4bNQohmk/GeVW7ZfkfKph5QtO1z4vsAnSn0KIDz3vFYivXbg36Kv/adULfoncGC1v0vfeW3cHwQ1pFnn36aExzeC5bWXrVPrkVLF2tEU9ZclTim5OBktmjoXN5womIxW+D27ZCsONgJc6GsSswo4VbbnMdNYSwWtA6UlWXblrquYCiIjnRLpSWmU1KstCXmRZhTOw8HODgMc6SbKE70zOdzfO6zgXZ8uE+UcOEgY/2bcLwf29qyubu1S15DK1rDposKc7ounMWEKJbHJ1OsDYmqB6CgxFPlBK8JgoLKpbHY2SfapxAoSd1ZqEbdb328gceoDvnM1hmWeF9fozKH0YCVlK2pkdF3uXD5Mg4OSAdgd4GSYp9lVSGnmvBrW6SyfbLBypTdfoou1WZ2B32MaM3rDQZYkq1HXRnMJmGNjGUTcBJbVB5zdnMbOalowtQ4ooN7f7CGitYjSIGa6p5S2u9MsURhSaOgcBBk3WSKMRyZhZeLAgsqJVjQ+JEKyOgAkgqgJCn+XgpkZBCvbYmT6E/tZsBe2PM07fMnBjAiPItSKZS0ni5sBTMLsY9eVkjoMJVlHfTp2iMaG0gcl2F4eFYtlsogoYNVL0nRo0NfIgXSmGim9dLYmhVDA22UxgQ0Hn88JM1Tc4Tpbiihmc6XKAnYWNI8PFkuMCcKsVQpZkQ13D94iAtzKm1K81a5hOSDXxIVZ1XKa6yznhVeq6KCodKPsnKgcxwUjW0pRCs55DnxUFcGgg4uqe4yDVOKpu5Qsoy550OhFCkUqZRXpWOlS+sSuGgjVJZRYgNJEg+cHfR64edha3+czmsUZUxkBFVTANg8t4mSEgeHJ+FLZYlGhxLsZlHgZB76sawrZGRNtr61xgeooqxgJ5RY4lpo4IT2+klZYEExvlAO+8dhzh3vH3C5RKIEVKR7xhgTDXUXcEwLFi1KL0STtJWsHiobCzLnkcRYyzlYinGGgxxpQkqueY/XdUv6CYlO8MILYc18/tnP4OtfC+Pu9s13UdE5pdPrYJPioOGgx2vQR7VV6nbVVm3VVm3VVm3VVm3VVm3VVu03pH0sAvg4wdx7h4cYUdH3fOFwh5Rq+hlw7dnwnixLMCSIeXPYhaFsYUH+H1lnA6NRyGYtFhMMybyx1xvg5HhGP+cYEC3laHLC93F+O0C8h4cHKCjrd3Q8xVu/CEbklQcunw+KSk9du4bLpNLY7/XwN/7g98PfHkzoGhMspgTDH01YFchaBx/5pbbGGlEDHiFFxTNnz+DuTlAr/flP/xyLw0CPLMuS7095gX1SVjPGsDl3FHcoTY3dIvw+seCMgRcOkg11M9y7H/r3b/9n/zkepc+3w4Bkvnf9AQpLhphZyoqmNTwsFVzfuVNBpdFf5QidrYDw6Sz0Uc9rHLwXvosqx/jUUyGrOdmf4OAuZfbzLopZ+ByVFvjF9UBh6anQL2fWLuOFrwQ66913f41Xfh7Eeb7z+38dD6kP/un/+Wd4+tOBxnhu+xL+kAzilylloqRERkaylUrgSNTEeKCmDHNRW/YdiqiecRaGEJnaW0bbHAT7tsCZxhMNQdAFAGefpG98AmEdZFRKk/EvQku5yN2jIHpPGemuXqBYhr/LE4OEfImEkPBEC1FWQFKWSKmEfbpihihRCo7QNF9V0Dqg3ZAC3ocxm2OJ40iDPjjEjLJjiuRArRM4IipIkndx4WJ4zhBgxb/EJU3BstfYoCxRloc+6nQzzBdhzi0KiwmZDjsrOHOepJrNTjWNXWMs1seBpnRH38X6WkB8e90h+4Ie7B9gPivp9QGKJdEOLXlLlRW6g9BfvV4HcyoiL2ugIEpSZYH9g5AVPtydokusgAtULP7+YoKnroVxvNHPsXcU0L2Oq3E4jXO/hCVk/ItfexkLera/fCcIVmUDjSvk03T/4R4MCRGUkz3MHBXsawVL6LOVHktCCiSJXkABkaeeKoE80mdUgl2iST08nkKIgt6TsSpdzBCK8JDAjQvRPauLSSmYzixaSlbx5zYF1DeEUnj4Zp3zvoU00t97Dx3FN5REQmpr3/jG1zAjz7Q0STEcDuk+JKPBq/bJtQ2i4A57DopQiqqco4gUOSVwjjLq3c5ZVtjb2Oxhg4RYUhJNWi5rDCPV3TTomHUCFSHiy9KiJsjelCV7pUVEb1mVILYlpJKwEQmZCgyINr427qGbExrlga/+9ssAgJOjsO7U1RKFieuwhDONMEr0V/MerC68RmJ0Ku1hTuvf7u5D7D8Me1ueKxa+KpYldvfI47OoIBTNVzRsj4PoCQjBPrNCSWS0L3UHXSzo+771q3dYKOvslSC6MJ8vUZNfqEoFMqLZKZXjzPkr9LPC/lGQU632dzEah/548ulwDWtr3L0f2AQn0yMUhA4sFkv2Jzu3vYHpMvxc1CWITMJiOnnWwQUSybh49hKzL+YnM5wch89eG4xQUblHnnbQIcVPXUXVHAdBtJLcN8qOqTQQ0VwcQEloYPQ2VN4xYlRKgbmJzBqBnGQ5pXIooqK2Fbz3RqonugJSRcNxzb6Vbl7A01jPezV8N6LQDqB1aj6P6rMJiHwBayxcpNUKBago0NGIajkH1DQHWDDOCab4aSGQEvplhcd4TOsfEi6PWlYeisb3jKiLy2rGscB43EeeEyumLrCchzlUGYlMxZIXgZzGbCajJ6OEIwXbuq5RLcNYL5YVPD0jAQXB8za+17bYURKG4oXptMRwSF5xuUQdO6FyUDIK1tA+owNCCQCZlqjp2vPlkina88UCMxKKK5Y1SiqjeOxKYLKtbdb8d15ZxPjqZL7A0QnRvL3AaC3M5/6wj+n9oOYZFTnzdY2c0KxFUWNZUP8ulrAlUWahUdM4TIRmNdXD4/AZKmu2O+sEJqTIjyrBnM4gk+Nj1BRvai3YYzfKUCklGSUFBBPNhJQtQT3HCGpboK1hYEl2T4ATMMTO6OUdZJoYEKrxVRc07jw8+RACMk9YgO7c2W1GESHD/AcAawy0Pr2/f7B97AFwi4K59fV1LKtoHOnw2c9Gw+gukjQucsCI7BD6mWJbgGVJilajIRYE2+7v7WBjY406SGFOB42iqCFo8C+KJdNEn38+GCW+887bzPvffXiA+zRIRNrB2TOB8rA1GmLUDZPQCIFv/u5Xw32XEf52TJ9ZLpZMT5mcTDAhE+gbb7+LjA5vTz8dDmCPP/4YklE48Hzp5c/hh3/6jwEARzu7OLNNSqOVx2uvBJqjg0ddR3osLa5J0sDpkMh9rCXy6NImORoNMErDQHn/5nXcuREOVi/99W/TeyUsTeRiaRDxe1MneO7JsKGOx+fxz/4yqCG57ASG6AWfeSS89/lPfwWyCPdcTU9w82eB0nu8fw/PvxRUC9e2B3j17VA7ON6SqKtwf/PD8Fy3Nwa4ejVQGPqdLt54LdBg33jnPTz7TFB7PXt+C4oWP2OW2HoY+tfFiZcKDKJiFQBJA11kGTIaB6n1sCRfbPoky5/1YAvaILxHX4R+1tWE68Qq65jKKQCm2MoIvVc1HCUTpHFQkZ7sm79z3kNTwF0Zg/tEx/mndKBQYgDvwz3NZjOmGvT6PaY/eq15YXDOsAIfH/6950OVFBJCxHoTiTktqrdv3cT7ZAYqkj46o3C9nZ1dAEBpLC/Al85cxIjqRpRWqGmzds5xvYvzDh3aqGqqRauqkg/M3nucUI1RojMIQepcwx4cfcdI3bXO4MFuoDdlecr1YL966zoy2uSTVKGiwKgq5lA6bha0Rizn2NgK/TjvJVjQ60VpcHBIqmPLkm1fvEvhbHj99/61YFK/ORpiRAFSqj22z4RaF1tU2KVrHM0rLOMGNp9yQFUSRWexXOKxzwWVvDTtYJ/qAowpYImmbrRCYRtKMtdv0iLujUVO0oNpJ4Ol4EUpj4PDXXpuO+gTJWbcT5CrqMhGm3XbXL3F0fC+MYjXSjONJzYhxClKu+OfXeuc57lWyNnWe+IGLTznP0L9R0M9yigpFuoWaB2TksfVqn1y7SmSDs+SHF3KMsznM0wpcZpIicvnwhq/fW6LbViyTKETa63pkFNZy2qfrgaiz8ysrrEwsRbeA1QLZ5yFR6SyE1V8vsQx1d4vZgUqooSlIoWIQXueYUh1Pp1+Fy++SGUDFOTVVQnvYr0uUNEBcDaZY0IB9fHkGJr2x3Nkg7N55jw8TZTZfIoHt0Nydn//Dtcfvn9/Hw+I4llWju2HdAyujOdErUo1+rTWZ0mCHiXtxuMBjsngflkUWN8MB6hHngrfQ3qBmqivrtYQVLujO10MKZZSaYolBXbufgIQza9DBxgpBXpEqb+/t4d7RC+8t3OE4WaYf9CeLTTgLM/hrBuusTYcszl93h2wSmyW1FAqJpkcPNEUjZRIqH46o4NGWRpoKi9JpOAEKuoSPapF1D2N1JDNAI2lfpoiIeXpZQ1MiELcFzLsDwiUvDkljXJnkNJnO9r7ZrCQsa5/qOEj1baqIauwt3W1gqXDT7FcxjJq7NH66K1nSy5nwXV/1imUdaxJbZQlpVRwdOiJ5SdSeySUZJMq4SSatRZzqlu9t/cQOzshhtQqwcZWKG84nBFQMZ0gp2TDeDTGxmZIvHY7Q671lCg5UaG0QxJpmLFEyHhYihuqqkJNMbCrLZdLaKmZphiVTWvb1Acnqea9eT4vsE+AiNfArIg00pJrHh0lJZMkRY8UPnudPjw9t+ligaqMZVIFiqhGWnsYqkP+1lfDGmWMZwsEAcsqvrOywnxO38vUcI7ql+EwJ2AmloA4OHQprpeuQqcTxh082CKrqoExKfPmwwGUjVRxUvsGmGEH+4wAACAASURBVGKuhUVBdOg0T+AqMo2vC9RlXPMEH+RiXaNQomV7JOFVrJVsbJmUag7j8RDnLJoSJqmaselZbgFZlkDIsumzeL0Yt1nLtPjS1vAxUaEASfWfNRxMZP4LBZl8fHnGigK6aqu2aqu2aqu2aqu2aqu2aqv2G9I+NnXbJ8WZwtTo0Om7P+qxAIB1jWCD1hIZKQ71MgVLdIS1rXX6vYapSWxhcw3dHhUneokBZb+q0kBQRqysSiSUFblwjrJnuMLFoVcvnkf1bICYK9cUzMIY1FTEOS0MZypjcajupsi70Vl0hItXzvH3jUXR9596Endu3gIA3Lj1LgDg7p2b2L4aBC7OXriI7/7hvw4AuPnGT/CTH/8FAODt67dwcBhRA8FKSiPKoAjRZM4H/QEGvQB597o5LpNq6qXzZzGiTN/O3j72d0OmZm2D6CbnxjieE3pkgXLW+CZFStjG8DK2xwE1ffvGW7h6LdBPLq4FitzNN9/Br14LqN+v3ngdd2++S8+ixuuvBkPo/nAElYbvcvZygvEGZRdH4bss9u/gzX8Z/NouXH4Wl66EbM90doCaMmy5Spk2dHBwCEWiMZMd6n8tQYkyPJhPsFiE7KosUy4gNtpFP1dYGfplnufYIaWjfqJwdUzITyGxSQqSqVen1BMZikfzGijj66VAGRH5VkpEAHCE4BRlgUNCiqaEAG6uj5FmUWUOnG221qAic1VTFqz+6FyNhMSK0ogS6YRh/W7WgRLhM967/hZe+fFfAgBu37gFi9AfL3/jO/A0/378L34a+vb4BHfvBYTw2hNPY0iUzCRmiQGkaYqUUGZYiwOiU2rdiA9VVUR7Ms4oSZdi2CclUe9wj1D3XhbG7uicxFPPE5pcbaIi1Pv69R2khACujdcAHz0LTVDRRVOsDngcHFOGUHr2IFrMF0w/cV5Aukg7rViY5jEy07129nyTzfKeRU2EM7CUEjNC4oAyi3/52it4751AtSrIo+i+XeAgqqophSJSc1UGkEpf7S17+KRCo0fjNPpPSgn0CF09OjjA9/8iiD4NuhK2DOPmscuXcHadKHydPmTkKjEK3WgYOdlk8Lz3Dd3SN6q4H9VaxFDOgAOehZBgTUM/ZuEjd2retJE+vqBqkEdrLaOSq/bJtfMXwpzr5Bm6WaSeV+yV2UsTbK0FtGHQTzAl701blbBlRLDJZDxN4IkOvaiXsPSzNxWPTekN7/WdPIWM6xetKRfSFJ7Wv9o6GEJ+IDws3VO1LBg1EGmCixfDnlcuIrJv2V/XwrFwV1WWmBNd7uHBHpazsE9ExeEkUyzGMBoPoXxAYZLUYm8/oO517dgDNuskEPVpZWkpMqholJ2kGA3DHF4br2GDFMSH/SEGVMaSpx10B2Gv2d7eovvowBL6kiSSRcYqZyF1WD+kFhhNg+BLZzBmYasJCVZorUGkDVinsE/0tao+ht4P9M1uL+P+F4lmpdMuIYDD3oAJBM5YpITIaZk26orCQ0SBqqrEvEurJyGAVndgy0i7ba0kGWCTiGwq5LSfZjKMwU4iIElYTDoFSeqP2jpkZFTvoCCjuqYp0SGfP0/iJ9LWKOnesjyBJOSwL3Kmn/Z6OWi4YVEucRwRblLq1FkjTixlI1hSuxrEOIW2oR+AoO4Y6fVatsozqKyj9AWLcc3nU+ySOnJVVUiICpwlCSN552icpx2JnOLoy49ewtlzQeRDJxmj3dZVbJruS0BV1B86MjgMl28ESU7qJwHeE53wrFy/IGGYypTIsohQpWwmv6wMHpKHXGEN04yrGrwfWOp/nUikh6EPup0JDInslOWSmWi1rRvPaOd4f1kj1NuLlqqo9IxKOqhIKkBpDSbHYV4fHB1hPA5zpKDYaTzqY0DzrZcKrNP+WVYWB/R303KJjUuBlbZ1+z6XixWLKLBjkJCfYneQ4IiUM01HsW+mN4bp37WT8KQ+rIiO62qLWkXRtYYi6tAggMILZgQ1bqXNHHLOsSo3IHhcOTgWnFKqjR7SulpXpwQKHQsDmSBYhCDeY+NZJ1UoYvD8EU34D9SAtNs//Cd/7oEAn8cA2rdkyeP/f/B15xzTN/nvnGMY27Xf6xuDTWsNBzi1r7Eg6tad9wK9sDMaY3zmUbqeYcqdc40aj3eOaW9lWWLnj//L0Bk+mr8rrusKZrmkYiYkB0HaAT2C7WOHV94xF1dDII1y/hBIENWmwkQEAOMcB/ZxYTHO8e9rD1gZJ0IT8Fl4LImKsEQjQ/y7/0NQ51RSIsuIOloe4MYbfwIAmNz/ORwNgtotWL1oOp2j3wuTKS4QSZK2lIxCr4T+UKz5q4eP4OqLfyPcU+8MCkkUODqIpOUeZveDsbtOcnTPhcNlKUaAj4tOzX1qrcV3fu8qfR5j1MG6AeFgFgN4Ac8TIREKkcbMRqBwfJCrhGeeu4Lga0jvoRGfuWDaAc/H1oSFAAe6AGCj5LjwkJFbAs0LtjmJdZ4fXfsUx3378Hlq3jA9VZyq4YpNMaHj9N85ODZI5rmHhi4ivOQFWJQ1QHVnpRIAbbQyTTGlTSIq2VlnuU9hHRIZC3okm6EK4K/0Y/j/FnWRv6Bvvnd7cfzAe8I9yBatovWI0H5rQ388OjxkWsd/+N/coTe0ePdKMK+i/VzbLbAso6oo1e24Gk+tk/VMKXBrSbQs0eU6FekFhI2FIzGsoNoShCDG0Sh03iBpSeBXKirQSvSIwtrxJ3j1H/1XAICUakyd11C0ySdpFyAKroPmse5b1iZNvwlYEzayy5tTPEK5LWlrLMguY1k5VBTkFZWBM9Q/1LeTZYW9KX2XZD3mSOCcZYqc8xYGpCDoLSunHh3e+PgT6ar9f9b+8Q9e9UCoL2/UPj2rfXo0FDj41hpyyj6kWQBrG+mdjmuI2vR04xyPN1fNMTsKB6uDvfBvkqY4e56UETsZZkTLWpTHqCmI884189w4/PS/+CMAgNylOh+ZYY3orDkUelTrDqdhSPXSIkGHLBAkBec+sdBES8wV2ChbSAcBomJZB0+KlFZaONqjOOkiJEQa92sF76kspXQo6T6WpYeOB4w0x6wX3v9v/o9Bodk6h5oOlvPpEe7eDsnUg533UC9C8qcqp1gsSZ3Qlay8nKa07yqJNCpjKwFHNXTeC4iELCa2ruDCtZDgHYwvonThgFFGrYFizgqHViXojcIBdTjagiFap6krVJHy6it8/TMhkca2Hg7I6Z7SJGXVQjjPlDXnPCsVRo6c8L5Ra4ZsqJfesYK7a1HVXaukJa6mxjVWTAIt2f2GFRfAh3aiin6hq9C33no+AUrR1EBDtIZ9cOem9wDNW+I4aF4L8ybeh28SBxIcvHnvmiQkm4VLOBtLPATvYalQrO4okEPQIVd1u/jJKz8J749KklohjQrpQnFMCO+RkbWG95JtqBL63u5UbXZTCd5OJHqBU3E7lwK0Eoa8v8Pznon2fu2bWAUQHOP+p/8wWIsUPuUkglKq2Xdhm4y7b2IDg6C02v4YKTwflOAFHMX4ti7h6LAtrYOimC/Jusg7VCNMsas3gCegyDjHdZWmLmDpegVKWPrsTlbhnf/tP6H7CNfI8x7Ttq3xHGfDyxB3IDzn2pJFBp1wlagwoiTLZsdhTCUgqbKctCurGobOJrWpsYh0bIrJF6XDpIhnKHH6PEVxqqkt0/Kta+wmDvcffOjevKKArtqqrdqqrdqqrdqqrdqqrdqq/Ya0j/cBJJqBtfZDUQygQchOoRQfRgmSklGnNm3IetsgFkKxYEbiKty5F9T53nkzUA3PXrmKM9tB3MElHYiIAFrL2U5vDWf9vfdwewGtUZzUESye4LyDjcWpolFjdN5Di4aeFu8/Zg0dBGf8HSRnN7z3nEoS3nEmMv7eeQfL1wuZdCBkYaKvkvOOPGeALOhXAgDOJ6TymCRMmbi79z6qw0BjGyQSM8qm1FIyOlpXNVwnFuF3+DtFumKSJEhJDMPJhIubUz9FdRCQ16rcRnI2qHmyAtL0AIvdn4f3Jhk21gPcoPtdFIJ8GK1mE1GlFHYfBHSlOyC6SD/ljLByNWc+nWuUxJT1SCmD1qCFjuHxVInGQFsI9rCqvWXVK6UabDAiWAItoQ0BtHNYNcH91qOhaVgwyjyQDZ3ho9qHKTO2f44zRLQUGk/9/Ue83v5NTIgFZI5ekwL2IDzDm//kx5DvhQL1clZg/TsvAQAufPPz+LOf/zh815jhNwLH9Hf3b99BjzKSZ7a3MCTDVK01I/vscyMF+10GymlE8xvRmfCVmzliOKtNXogtempbUVIp2WR8rYWzjahP7BumtaJhFdhWRlIKwRnpU/Z69DugQQBTKTm755VmOo/0aZP9tZY7XklARRpxO7saUX6teM1QTiIRpN4qHFJJHqd2DkVomoyUHyEhRBRGEFyAbnxDCxGwrTU3ZtCbjHsiDfokrmCM4ESrFbalRuYD5bbVB0I4+Gju7HyLcuL4ebX9A6uqBGw09Vq1T6rFjK9Ho3bsfM08CuF9A2CIFqfYN3sKAx1esAq1FgIy0uFtw4YQzvL6a1yB+X5Q2jy4ExD4/toazpLoTJYOUFEWuqo1TFR/FA0yD+2hd0gplMbYGB49UqPMjENmo1m1Z+aJ9QK5JRQrUsK9Y3PsWire22QimeqZKM1qzC4BhDlNARXegdnVxnBmXNcGljqqYwQS8mlVTkIsSISE0HVT14yeTmZT7BM6erj3ENoHBBO2hCeVRO8cBNMe47x2LNSSeMkm3aY2MEWg7RcnGWaTQEFM+htISDnaU7wxr2r+bA/F6EWvs8mIXaozSFLwNFbgaB4F0eKzsrCkWGl7GStjatGANs56JFG4SjZxYKOSCJgo6iMaCrlAWMvi/3n2UotxG1h8SkA2iGJr3YFoFBG98+y1q2qKueA57pItxFGhtdcLweu6QsPeUhwfNqwkh0YJ0sMzK8k7wehhbRqmnKP4USkJQV9WuxpzUrzduf8+jiaB0tvf2MLwTKAubl64iCUpi5dkIG+dRUl03Mlkwuh6L8uwSaIyXkp0KY7LCQlUSnNs18lSONHECI5QRK0l2lhfjKNjaOOEb2Im35TQhL6KFEXLLDeg8R60kozMveQ9LDwnxX3Hz0KCvbqF94wAShlFzwS4WsUCkMRYUAIm+ug6y+inhERC7JaESk1qJ1DFuAUZq9wCCl6HtajjFIp4HlkC80no90RHBXXwGlbVDjmV2QiVcAmWqQ1KQhS9iOVQjapu0s3RJfp67g1K2V6P49jU6Ca0DtBr84Vjyqxv/gBCKp77xlp+j5aSy1U+qq0QwFVbtVVbtVVbtVVbtVVbtVVbtd+Q9rEIYMzsf7BGqS0ScDpD3LzOmZBT9QkNMhhROuEFo1VCAIpSVMVkgYd3gjhJhuh7tuC6LiMzKDpxKyk5te9t40vlvUdCHNgkel9JhU6scwNACQZoeDjKHhjhEPE/5k0HTfRwLUikIl5DMDIEAeYBCy+4Fijmkaxv+YbAITWxXkMw4uUhUXKNRpMpm1chc1TNKwxs8KBZLvYgHXn1hVxZuCcpoag+YjgcMvKaUYH9ZHrIz2q8NuDsTFWUKKh2YDyqcHI/CMVU3S2c2wjZ3aQfCuJ9AvjymL6rxvLwRnhdAOk4ZKXcB3jnP/hnwbexOwxZ1LWtAa6cD4XCY7Fgiw9IhTFZGZxdH8MTz9py9g9cfBsyVU0hbo0ogWw50a0hoKNIUMsbsM1j53o7gK0CSsulUbDGcz8NW/V9sX1ULe1HIYHN+P/QP6Pv91fnWRsWbOodm4ypmy2x+4PXws/ffwvuekDA61EHo++GGpJupwchQ93IkKw1To4Nbr4X7Bze/uVd9Mlj6RkJnKPC6ozEJtrfV8rGF0cnurG3cI77VynViIkIwQXQrlUoHa+3LIpmvKYZv6euaxYX8i0GQeNz4xvkXzS+j/Ee479tJgO/n/pZ2Yxl4RU8enTpXBRQOmRdVWJPeWKlhOrFuV5boKJ14tgNULse3V7G6B1EDU3LUSrA/oueaqWgc6RJeD7dfAihCAmBa7ENAKimVgugMhwaH2vDDAOyXKudhiKUwqFVL+0FajTrd+ijNr7q0KBFDkI0ueI4r6WUXIuwav8/NNFaJ2SDaABNXTBazIigbHV6/XK+yfgL2TAMpAYzJ5yVUGQnNJsc4XA3IIBT8icVwqCm2rZ0UzBLxfoOM12MaSGUSsBQ/V2PXhsiwZAQMeEqCBblUEgJeat9DUXMJEM3bbxGRWOzAIC45wsFpWP84XjBlEJDpeGemgy5Z19V0apZ1kIiifU1KbivXekh8vBzQQISzgfmCQCcHB/j5CCsveV8gjSlbL52UBnVOhmEwiSAPevgJYRu5N113Ce0b2TtFyfYf3ALAJD3t9E/d4a+F/nGHUrsk0hJUSyRd8NeemazgkqadTgGI9Y6/PJXvw4/EzoJ4TjG6fc7GJN3ZL+XY50YIVmiWfiPA0kh4/CCTCRvoE6A5ekdfCNgAQHL6HS0anCMSoXxHV/3/LPzAhUFb7W1HK/pJsDimDD4/cZ4FFyfByG5Bt57DxEhYBYGau4/COY0fce4mSeJf/ocFlCi21BWQhPjRZwc4/6bIQZ669Wf4Rb5PV9+6tP4wjd/J/zBpUtYUhy0JGun2XyK2+/fBgDcePddzGZBwOX82XN49plPAwBG62P0slgn2Kzf0f5lWTheA4TSvAcroxoilFK88jPy6Rzvj4EhF5HZZt80xnC9ooOHNGE0aPa1dbAiMu0auwrvwQwUKRJ+FhC++Q5xTgrL1lOJtrx2KFdiSMJ6Sd4I/xlbISGLKx1Zfmne1KRaAUvsIS0TaLq2MTM46n9RVZgX4RoZWVcZV0OqaKcloCiOklLAVRGRrniMReahkgKSFpskFSA9JKTCI+U5CdiFjV3AfSAIyfSmRk3iWl4mjf2SFKjKBoGXPAUEMyo/qn3sAbD54wYo9C1xh/Ce078DwqARreCPfnkq+BKtQdqOaqP4x879OzjYDYqDAyqY7HUSKFows1Qxxc8714gVyOQUBTTSKRNalZTOWMzAS8lKkEYKaLqP1AIZG4CGvy9rw/B3Sv+Fn2VzyIFvKINeoIqLFX3XREimGTgLSOp+6z0LbdS+URnSLTXDGFQWtUd9EjZaIzymZPzbl7JRXsscMlL7mk5nmJMPUOwXrSUPeOctrIlKbx7rozH97GCXRCOpp0iIfpIMPhWeU9pBn9QhTTXH/bvB/3AzH6C3GQrKK9FQ1gCJu7cfUL/TZPIF3urTc5ncw3E0rC0Nvvjl3wIA/OG/+gdwSUPZBQCFZpNJjUeiGjEdXlyU5AWqlrKlphjHoGPKr5TNwmGtxYIuXkPxSu6F4gNgu30YNfrDRF3+37RTAkto0WCif6AHHBngPvznr2PxoyA+oHYWWG6F53n53/8DrH/9OQCATRUsCQJlZEScpwmmx6S0d/8hsstEQbRL2kCBuiiRROoCUUuUknx4MGXJiq2h2Dse0j3PAdvqp0jfVEpx0sYIA01BYKoSMEtDNskVBwVH64BOmjUgUoiEEtDRlBUNrSWwN1qHwRgUxMVaeqZF9XQFlYQ+7ZgpyjkFc8sTeBKkML7kDcy6OO40Ll4KNPWuP4f9ihJa6EHSqqF84/VU1xIliTd5Ui0zi0MURCGayhSKKF6d3ghZPqY+yHGKU0XfT1C/jPs5eh0Sr3ACkT9TWw8XDeSlZOEFZyOdxPI4dw5wURQCvqGJtjwGlZZslrxqn1yL9HPvm+RnoBvH112LsgbI1uGdY1emsdnAZwYAaznwCCrf4ecsFXCk1rk42kdJfoMKJMBgStTzQGkrl+uQpAqZSI2EDNGFTDjJCjh0aWwNKXwfwiGv4sHMMVVcJynTMJVW8NHAnubcUnhYGRUmPfI00rgVOqZRnuRThQJUpINSICygISlJnLTEnbRIed9J4WBprs5N1BcPpt4AYL1kU2dral43M0hktK5oAWR0QDW6MYGOUbjwnvdjJcCHQW8dPAc5U8xJhOfo4fsYjUj1tBvWhlR5Pqyb+QmqZdi7JQx0DOadYapjLhx+9mZIGhaktuptybFHplL0SAV+fWOMJx8Pat5XL13CaEDCVbEMIHWcqPUWnOH0ohEoqh0QuahWCFbrZDqg06wo6+E4Ce6cY3VKCcGxnbNASdcetARjYpLYthJWoh0QywaoCCUh0buNHomUjTqM9a0EXpMYgW3WxUCKpD2ZkpUZaiz3QpJk99ev4fabr4d+Pjhgpcvz2+exsRWS5sNeyor1eSes+5UzrLzslYAiQZjBsI9Bn9T5uxn65LE56g/4PmPJRl3XLLYDNAr1Dp77SbTidrQACS7XgvgI8TrJ/suAbPYGjsUUU8wBD8mnY0DGEgnVup4LqXwgKKSGP2sSAYn2kDbEirY4YpSkdg4uGtKXE5QlfSYlj9LeGmQv9JFP15HkI+qDDqwnv0HhIUjwTSuPuooKxU3pSywN8V5y6UfuZUt1FjxXJf2bpECX6KJneik2SEBKuQSaElqdxEGYEM/PSwtD1zYRVIKHjolm5znZ563jDFIoQyNKbCsp/lFtRQFdtVVbtVVbtVVbtVVbtVVbtVX7DWn/NwjgXz0fftACoi1336aAfsgfNh+qdauwtEEApVSYkyfX7du3sLZBfmaepH2dQ7EIJ//iZMqowXA0QrcXKAoVFPudAEBK0s8sFVF75IgF9B6K6B25a+TuF8pjlzKmNUm3r5/Zwvnt4OEirUA1DVm/2dEE6YTsFWrBUvQOFWRLhhsInjN1zBYlqslQedf4frmQgQSAGh55zKLYKE074mN7OtzEYBAknrE8RppHTxjJ1hSb43UUhAwyglJ59ChbVM7LIOQAQIuEn10FB0VInZQCZkG+cSZ877t37qAkyDtLOugQMrS+PoIU5N2mq5Z0LzAnLyqtiNIra8xm4Xnm9SE0ZXQ/deUqylnIRE6mcwiSo5Yq4X8VZVNkIjhbWzvAipghsdGNImTmIv+kjQDGbJbEKQSwpjHhWrRf69DIL7MSsjg1/nnUC/GRKOAHqaINgat9d2iJL0cEMKJYnjPL/Hvj8OC1X4X++tGbUDcCWnWsJM5/75sAgIvf+jx8N1qeeJYXr5fheY76A3zt5WcAAMPOBLNpoKec3RrDkrffX/7Fn+PMVqAbPfccoYlaQepmPnnKMlpTs28YvGt9b8E2LYsZUcayjJkCzjrmo3nnWzQ2yZlPmTbCAYkO46fNOvCykQ6XQsBHglBrnrUZCey9kxTQKY3R+S5md0OfHu8+gCMPU1dPYf2Sr6E9jfWIUKQ5xkQL7W5YrJMv5aHJYTjD6SFocBp0mAKV0/xUUjKvaLE8gvGBar1YHGFt8yr12SYETlMvnVMAzc9BfxPr5I95UlTIauqD0kNGHywj4UVYb2u650RYKEb9LEQUcfAVfLxn75i6JoBTVOpV+6Ra3OMayo8Uihkm3nvI1ns8C2zYRrDoFMWcSiusZ+8rgUaEwdkSs2lA+E4OHiAhCtZaL4z/LAWmB+SNZjzSfkA3km4fKQlKSa8YKQeAdZqXhFegB82CX5UxjEDVteH5VUoP0ivBNGbf19bQH4VsftLJkNF+oaoSjvZpu7S8BzgAPnJeI8XMCqhISzRgip8TEomK64pkoQoFCQNCDaKfnjW8Tvd6HYyGISY5WewxeqSE4LUuVQrWR6psFJJp6O1l6SCiSwUEEtXQISStycXxQ8yPQ7+vE5LQzSwGRE89qiYojgNaaItjdEkYqqjrFvrpcUB0VU/7QqIkEmLeGGExL8N+LKfA+zuhVMArgbMbAbliywitkZIPqpK68ZlVgp+9dc3Yc9Iy9Y8ZI1JCiMZruo6iYe40EyYKFFnvGFVi6wOvIEWkF7ZsoNCiFwrBwHeAyX3s3fBr6ZmWKtCUa5y2NWqsTTwEJN2TI59id3iIe6/8DADw/i9ewfIo2IFcfuwJPPFyEGU78+Rnka2P6XoeuoX4AMB4OMT2uYDyzmcTVEUY69tntxFZgIeH+4xWie3gr9nrD9FyLGIxHaUbqqfzthWat+w56HtLITieDny06IsnoZnd11BKfVP5wTGah2i9V/LnOefhY8lFa26Z2jFLL9Jya+uhY8lUYliE6vD2r+HozJCIBo0vixOQbg4UWcSMtrYx2AwxvBpfQpZcAgB0B2OmddbeYUq0WVfNeB+ObjFWGBZyrGtAkB1VXVboEmIrVSNml0UmpLfQhLiPco+NHiG6pkH5U6lgloSM1xVm0ZPT0DWEQEq+jrY0qCpCSa1vSpVcQ+bwHvDxxj+i/T8+ALZImh9aj+S8YwNFKSXvT0yz86KhaEkwbG5dwjcsqgXe+2VQ/Nzs5Fh0A72gXNCkWhq8/S+D586iKDCvwjUuPvokPvell+nzHDzXIVkejeTLDS0AGdV6vAk8b4San6gOOvUG7+vQcY+/FPx2Lj/zOKQPQWAv6yMhJTLMK8yvhwX48Bd3ICZhADo4aHpYER0v67qlPChA7DVI2UwgKzQrFdXSQ0eD3nlYOPJuj6H8bjLA+bOBjnH/3s9QWVKNmswxiAffPGXf5jpSKP0INXGNZ8czTOchEL944RKyvFn0DQ08LTN44lMPSI103E9wjxa5RDoMyGsoyzJoMtTVEBxcG2NhaqKuzog+VM/QH9ACYGssyOPk7q1bmCyDh8zt27exWIbP2b50FQBw4fIj6BDN4cLlCxiTUW+/20GvQ3QjoZiK4rxrKBS82ElWKHXeN4qsUqE9sOOY9kKwZxNTJtpUaNFQseBPH96kbDaImGSI+5GE47kQTqp04ITkGhg4x1Q8Zy3XqkSPwt3Xr2P/+0GRNXn3kM2Iz3/3JVz9XqgtSPs5mo8WSNPQf2ZJG+1yCmd2qR8rLKbhPm6++z4mJ2Fc7d69jzOjkJSJW4hSGlWs8/DNmqGEZBVhKVXLg1IzPTkeBI0xPP6TNOHfV2XVJCeU5k1ECMHJjKg4ptDUy8NYLAAAIABJREFULlmVQFHNrAaAPNKaa6YOO4+GOseBh0JG68F8eoIHt0JdjJxM0IlqZNKipqDGKc1zMY5zZ2os56TWN38F6Th87/Obn4Xt96kPMgxUqH3N5Qh/Guc+meka0XglhWAxUpwdJuRBmSdzpESFi+q51mgUy6D6KtWA/UIz09QS+xQwMvSpgQeIYhM3rIU1OCaqn8g0VKzJ8sewhrxApYGKfBcvmX6/ap9c42AZvkVr81wXD69bDGEfyh0AwAn4eKiLwa0C15bCW9REG0ukYOro8e4eHt68DgCYHe2xUqUmeqe3BrsPA8W/3NmHzsKafObyRZyjdTvv9FCZeEC16NK4jglZBSDazUErrhcslYTohoPLzFRYZOGetq49CQC4dO1T6MdksRIoihAQLnfuYyLDodVWU8xpH5FeIaVgP1IilQtJKwBwhWXlPdXxXN8E55k6p6VHj9LKTe97jmWGvRwl0frq6QOAvMqApjZTS8HfN9YAFtZxorEyNauUai1ZmdwpxfuBXZ7AUE2YoQNn4kuMyU9xmiuk5Dkq6hNoSQflcAymr1ViQaUklj67k2um34uiAogee3i8i3v3gvLrm1kHgzz6o4Vk09kz53DhkUCBHw7XkNIeoDSgk/DetJNBy5jM041vXUsvom6BCbWNY7fmMZtI3ToMejbCbtRumwOPFJ7HkreWqXrhc2NZgeBYULNkfOODCS/4WdTWnarZ1DGmq2ok0zDGysNwQHn48zdw77Wfht/PJjhDSrmPPvtZXHvuS6CO4kOFSlP2c7athIWgQ4kpllgQ/frwKIMhymBZlMiy5tAMAJcuSigVE5SSE7LBdzcmhESLEu4bKnKMSWST2FbwfEiTUvChKdQVN/FVfEYdOgBCyEajAJ59EZ3LOB6WWnPsYGXWHLiifoYxGFJcmSqBKZm7Tx4+BOZhz8tTQJE6aOYqJCLuj2GsbXqgF/VG6gI5eXOe37iC8UaIX2vhcUD1wUVp+IQUCd+1NwArHDs+IJq6hCU17DRLWFskzel7FwZROna9s44LmyH+OjmoUHsCWpRGnxRcy1SjKELcFRNXifDIab8urIWheMF6xyUcOtVIYuLDu8YL9iPaigK6aqu2aqu2aqu2aqu2aqu2aqv2G9I+FgGMGffTVhKC0YtTCodt1MS5lhIRZWGcYDgd3rMiUSIROBcA3r/5NvbeD8hPRznMKbM1GAaxkePjY1Sk0NPrr2G8GbJ+5y9ehiaFwqSuG2VSLRu0JtJMW0hNDsXiMHWicIcKQW8og+d//+sAgN/57l8Lf29rzB/eDJ+dZsh1+Dxvgd7lkD1Q58Y4+kWgzu3fvYsOUbDOjUPWbfHuPSSU2VBQ3AfeNQX7EI0SKoTlQmdTBXTmpDJQMmQWrW5Kb62QjW+Z1JgdBRSiOxoydyEmir1rMv7Lco4sjcOgyRYYU8JQ5s2XJappyLJ0ETJOZ4casyxct5zvoz4O15jdfxtbFwMFRGcJ+834omQE8ORByCDWk10Mrga6wsLO8d6d0HdGdnE0C8/55q/fY4+kg0vhHtzUYY+oFGeefgRpj3yO0hwXzoTr9XtdrF0IyM/axjqrqTUV3i0tFdd8cwE047tFQZQiIFIA4KNf2ykxI/9X6J3h7ySLjnl4NLJzhICjUccNNFLyGHRgqqd1nsU6vKvhaHwc/SQgVCf/6CfIb4a5spyX6H/zswCAR/6Nb0GPokfUaZqe87HIOrw+mxzitVeDQtnbb/0CZ7cCPWJy9ADnN68AAL74/MsoCJF78/Wg0Hv50asY0TzUyWnKDAOivu3X1+qbliAL+zS1xGOEFJy1VC0f0fb7tY6ooGfUDNZicRBoSuPRGDoNaJtwltG+IIxB2bno/+QATfTl0eYjOPtYyNDeu/EqZicB3ejUCoqEJUQqobrh5x6he91uD8ONMO7GmGFZhDXjymAbT74QxqbCDF1H2UJY/MdEtc5JGqysDZyP2VOw/6GFhyVBh3K5B6miEAsJ9jiNRBR0byUk9Y3CHLoK2cS+L6AzymAKB09qwdaGa3R1gZoo7cfzhzgsw3idSwNUJGKTOAy6YU2DTxqketU+sSZVa39lSrhssQkEGnK8bM2pxhM1CpAE363YEmYKaOlRHge04f7tX+P2jbcBAK5copOepu2FJbqhmCNmtUUjRKCFgqAsfm1Es+cxCqMY+XEqYwq/1Q6LiLZtrOGp5wNV/flvfRsAsLl5DkcnYW843ruPrglzEcMBts6FObd/6z7EnTCH67JCFhV0Z0u6zxKiIKEWkhMDAL+0UIo8CIVl9UopAdklYbbljHsvIoDS1Rh0w9o77XawKCOVGixQpYVib83IJNAeLKKRuGaN0k7wc9MyhaPQTdgS5YzEd47C5zljIauw13ZFDT8PFPLi6AEMMWcqI1DR3lyXBapl6AdTEzugaAQsSmtRUElGWc5gSClcqxS9PApUhT3/zKXLOKYYbX3zHPeHsZYFxEajNQxJ5XttfQN5dhp1UhJwKiI/gGGF98ZjsHIeluiItan/L/berNey5DoT+yJij2ef8c5DzlmVlTUkayBFSk1JlCmLrQGy21IbDRhww2jA8PDqP2L4tQHbgNG2JdgtCYIos1ttSxRFil1VrHnKqpxu3nk48zl7jvDDWhHnVkms9oObLzzxwEree+4+e8eOWLGGb30fDEPfBaMejIaD4/6d08+SglWVq+iqy+QwrupH0D36kUFtBdZqA1NbuL6/IBRKS8gLss/FI9KwPv3kPZghIcdEM8DWdTpXt2/fBLhiCxFAwVYwtWuLsRWerEqxf/AEAPBk7xFSbqWBnkJzRTGKQ6c5O+O/H49GaPJXKL/hrIG+BBOEWZDlGQhXvVuQEokF9lUKV2HVxqBw7P0Cdg/XZlHJDTx7XnuuYl0ZOMZsoXyHvBLyUoXSW2j3ej6dTxJAzP5m0zcoubp+3mphPLdMxAq9Jr3/lVYbCVefmw32w1faiJjNtvY0ck3vqmeALa76+b1VpD26Rn9UIwgZlmp1RqucYhlQNdm1DukSNe+dPJcOvqsLW90u0LK+v7qJOCIbNcYApX3PReb2fhL4QIsrnpWFjJeQbFfLSiPPLeGUcWewyWuIwMY88hLT+t8//j9BQP+uY/t3IXDCCCeEqOUi1luwL0qm0rVwLVo809EZPvuIBMfvv/82ijkZK61S3LxzDQBwMWQa3WYHL7xCxn9tbQcB9/2FSdthiX0smI88z3M34l2CptmSvQ+4kvapqLDHLPev/vY/xD/5L/4pAGBjmxgt0+kEx+z4Ty8OobiRphICsstsWrfauH6FMMad2Vdwfr5H38MGIhiOYY7JkIp6gYOHwEIEVS8WlRBYBIkZQQrGk3O0WyTKnkkDXXNAtrODPKcFPZ3mqDNajA3lYzTh/iX+OhVF8Jg6t73ShWGx3NlsjKpmCu1Gw+kujAcTtPoEPXvy4d8CAM4PHsFMuG+gGGJecbk611B8AOio6aB/NFcM94zpvR0fPMb9Awr6Ls5PMWH4RBC2IBkisra9gm7CwTYH6POzp+gxTHBDlNClNXwF3mGx+dp48Bheun37Cr76wnP0XOw4lbJ2sVhWwQV9SaQcdEEJQCrrCMxxdJ8gUOLFW/x+Lh8r4nOtUDbAPj87x0qHAhA/kjC87q20RaUFAk4maFOjsjAC7TmDXSnAcDCu/ADpm3S4nP/RDwEA4f0RSmZ6bf/SHez8M0pa1Lc3EFl7/oVa/3hMa4WRJaiKGdodcg4aSeImZGOtB8F4jOFwBi+kAz/NKAB/9PgR7jJLb9yIHIPd5TlQSjnKYt9f9Jk65s+qcramqir3c5J+4EBVKYSG5snzPHfI2D4VXGK88uoUVUYBcWOtBc2xlNaLvc+zQvfHB1ysPSeqPI8itG8zXGrzFs73PgAATB59jBur9D0vv3DVsbf1epSkiqIYjYZl2B1gNGDHKX2KNovKSlXBY7h2iBwrLOYbsE0x47kTpS4FYJ3RuloI3CsJFNYJ4R9KszjZjWwiLWm+hv0RRJ/ndwKA11IkvUW/nyA7srKW4FqbnGZd1PjXb5PTvG8SrDHUukCOmrFrk1Hm7Ppy/OyGZ5mPxaIn2OgFOyukhlXs/nwyxlBrBOBYrYUxTuJD1iVQkA0tihn2H1HQt//koYOm+aHvpBakTbx6ATZ4fSSdNQQNsvGt9V3okJwdIRQq63zrCj5swEVDA5gwnbkX+6j5PO6XNUp2yp756i/jV37v9wEAt15+DQCQlX0UT8gGDcYDGD7/m2s+NtYoGdNuttDukX0zZQXNztrRfbKlxWnu2KQ9LOjuQwhItskS2kH1jPHBbbM4fvopz7NCq8m2MPAQ8AHTaTYhM+6LQ47It8G4WcC4LJS80g7iTz6LxawDGTt8uiicBEzDNxDcn2eTtGmaoeA2C1ll0Cl9x/ziADk7wCJsQbEDn2cTBNaIcIwzn0xcoDdPUwdHq8vS2aBGLJiNGJA8d2VVY8QBivQWZ1iWzVEx/DdOzrC6Tn7S5rxAi6OUmNs3gjBE5Nukl3ZcCtoTkOzUF/mir7yuK8cNYf6eRKPBFyCNFoJYVtAMqRPKc8y6lnFWar0IYMuK7CuAGhU8+9y6BizHwmSC8vgxAGC0RwnSfDoCGAaoehtobROLerC1DbfyFVxgZaoac5ZTqbnFaZbNoPmgbrUi1MxCXdSVK9J0O23IgINf7kOcTkcIeE6bcXPRg3nJnZdY+DGUgBaLX4B7661zpM2Cq6IGFmrxi56XWsPJAtkAEBKOHV8avWiXEAvZDCjtfHQDCd+3q4y5L4RGyGs09kusrtF5a+7cwaDNn5EZttdoLd3eWcVmm/ZcixM1rWaIiP3GqjDIOXFT6wKJorijETZQM9y86wl0WLKs4Laa2TwFqkUlxclO6Aqa7z8vaxhOsEvGUUdBjYyDwYPjATiuxNHRAFNmcK/qBfNn6AeQHNOsct+uBwXJfYGmrB3jqQk1EmuHA4WKE3GFMchs0uKnjCUEdDmWYzmWYzmWYzmWYzmWYzmW4+dkfGkF8POsTAuom82mXGb+NJcqgMIIV21yZX1voSEyGV1gwIxhe5++i72HlEGr8jkEZ6VUEMBIysa//zFl6Sa5hxe/+hsAAL/Vg8fwAxXGTl/ME4v7NsY4HT9lCWoAeJZlUwADRd/3fjHBV3+bICX/9L/8Z7h6jao8SUKZhrSZoX9K1ary6DHMnNnFIOFzpi/P+jhgLbX1O/ewtXUDADDn+/fbMbIzivaV+Txs1g4plSvnXsrHYP8z0o8p6hkuNGX6wmaEzY6FIlQI+e/SIsMOQwN8P4JhEp0BQxGquoK20LowwYwhIv2zM2xsUTUin2QwzHCYNLou+3KxTxnhs/1HaHpcNlcVUqsLJTyIFsE7qnnkSICUUpCG3le3TfdmdsYYM7ROejEEN/BWRYmVLmVynn/uJgLOdh6wcKqpJthao6qaHvcxmDHxjtdy5ANKSRQpV5IKg5M9gglIzvjNRIHpkP7ubJy6En87kthmQoG7d29DKcrk7H30Pk4fEJwPXAG8PIwxlxlmHFzxg/c/gmQhz2//xq8uYBj8Lw8JJqe28qkQtijbU5oSuSOp0WhyRXR2fw+n//v3AQDhQ5rzqdZQL1O1/O5/9Y8Q3yXIZikMpCVF+8L95ky4k89thTbHnTu3AQDrKy0cH1J1dzSe44whRndf+Ao2twnC8uDhQ5q7s0O3l1vNG44RtNb6km0wyDibXFUVYmbLslVBeQneaYxxFUAAzmbUEK6yL4Rwn/H8S3vIVvb1DBvrDE3qhjhmDS4o+XdgZ/SdvEbrAoGtoHkeSs6qJcku4oTm19/axLefpzl79WYCXdAeWVQ+Kyhmd9JejA5XH8vZFO2A9lkdt6B5L0RGYjKifbm5w8QZzQYGF2QnZnm60BdSgOBme6E9B/fzuNITaO2YOt9+7xgPPqF72n9ygowzkReDERIm15LSw4wJoPLSkvBErkrzzbtbSHqURf3GC1/B7/7qtwEAE6Hx7tukG/anf/I9lP+OLONy/P8/LDOfxCW7Iy5zHcKxIApjWESbqgbqcwzcgK4qZClV9Af9Q0zPqd1gOrrAyRHZ52I+dVpetEdZ35Iz0qESMIzwaEQRWiwWHiUtZAzfLKvStWQo6TvnQ4L2Uw6NEpaUyMOIyt8YlCV2Nu8BAO5959fx7Ne/Qddg+LUZxWhbcfLQx/ExVf+LKoPm9obB4AxZThn/XmcFgUfrenREVUE9yRAJtolpCjDDZ4wYgWUgvdQiUWuDGRO7zJ986uY5YcbTbjtB1zKkegKCWbdNqSD5e2D0Jf00vm5VEbkEYEtX7vtKRmLUEAgjq99aomZYfsZVv9lsjnRK71MXKYwVAx8cI2XYYdBcu8TGPEXE8EzNNrYqaxSsCZimM0cs4gUxGomtrHTQ6tBzNRr83yR0iISqyF3fiYBGxb7dfD6DGtI7KosC1hZLJpwqqtQJ0qfT1MEHO90WNjeJhTpKWrYIjbLMkDNJnugy3F9eQi7KBTpH13CEY8UsxZhbZXzfR2eN7G/MMFJtDAo+H/NsioBRD2EjABiyqbVGg693fnGC/hOq/A3OaN9USiDZJn+nd+81bDCCzW81AbsOoBbIIIhL1UpLyBPi6jU6f7rdLs7PrSbtBEJZZk/PocgsoU+ZFw5CGTdaiNjuQ3rOxtdl5eyH5ynnR9dOh3EBnzVY2JFa17gM/LBnkdbUOgNQNYr+btF+BL0g0SuNhuHFEnrK+Yp5XcEe2QETzYRSAMx8W+YTp6O7vXsDmxu0h4N6ho0e7b/d9Q5W4kWVHCDURBAy+ZDWqCJeB6MxkDLycOzDj2gtN41AgyuGtmofBoGrxM9nc6TMAlpDLJAYpnRELCHrJTeaBlVJa+ndDx/j6T7b2HSOnNk8NaSDJDcCCVg9UMv2aZRjVk1yjZiZfre3V7G1TfsiK0ocMOrodFxgJFJ82fjSAFCpz+P8wbfpem1gHHuYMYCl2BPm0mf4TRbTc5weEo75g/fegKhpg0fSIGaYnQoCTBgyOLpI8aM3/w0A4MMH5LwnzS7+/M/+jCYolljbInjm17/5LawwvSukXAjZ1rXr4VqIQS/E4SfK4GOmdw9vXcWv/e7vAAC2r1+DzwbWY8PmqQCbO7QJn779I7z3+o8AAFeuXUeHF2BZjNHfJ8d40L/A9g7d32xMxi7wBPwmM+lNcgdFMJf6x/Qlyn8hFjLqjz4ice+oqdDskLFTfoL52FLP1hBzOuBCXSPhhZ7PC3QYgmOYGWx/PMLROTn1frOBXkz3v7ERYs7X6A+nSBhP/dJLL8BnyGhd0UZZ7TVRMavaPM9d/2EYSJQ5wzFEtehjyw2QJjwfdFB1ozbm5/Ruu0kbXe5PWO31cOv2Tfp5K4biADuMaf43NzYZpggcnp7BsAOaT2dQPh8EKoLqEpTt4P7HOJ3zvXKvyP6wj3lG87F33HfMd57O3cH99a+/ildfIaa5YngGVX6eUldfZli6pBJ/WR7iuWefxXf/5f8FALh+/RqeuUPPpXmdf/T2A/z1d38AAGi1I7zKUgw3712DYSx3XHuoeA+c/ou/hnlKBt469dWtNVz9b36XPvvabUs2Bd9cOgW/MHKGBaU2kVFPsLXOsNqNrqNTPhnP8OATWtMHwxJxTM7OfELrZHRxgotjmtOL06Hrx7x27SrimPaQMQshXmOAjOHJdv6kFPCtrMcl5mFiULVsmIuf13V9CQLqBF6YwRVYawZ4fpMO3YEf45wdSa18+NZeYQG7riq+n3SOkA+tUEhEvu0P8VCUfLDcvI2ru+Q06PIEnz4+5O+n66ZpinWWytjcWHfUz8bzoG2/5SRHyEEwpI+Ndbr2xlWa/3YzQfEh4/tNGxOW6qizfJHcqrWD7cWcxFpvBJgx++/HDw4wT+m5ihSu7299awsZJ4SGgxSlZV6ztr7wkHE/0MVwgjbDfPzjj/HoL+g9/9XRGPvnZNN0lUNXn3dil+Pf//AcNbtxMJ7aLOCgMAJCWDkHaYVQ4EsB2P5fdiyG43OcHfO5dbqPgiF86XyKgs8DmBopr4vJLEXKayhnWKgAEHPPW7vbxcbWFQDAsy98BWtXCfbmex5qK7buScSLDna6FkonbDIta8zZO0l2r+HlbxHL94u/9A8g+SyaM4xdVnPUmux7Nulj71OSbxFlhutMrz86H2IypmcJghCKHTt7bjXXV9FjWaNhWgBsHxtCuiCs0IsAO0eO0tqvgq5rdGVbH6GzAUor0h0oR2cvjHFwOLoYBz88Fbo0qNnBrC71flcarv+wgILnMxxUaxS5teFsM7Ickuc09BQqPj9HZ8fI2GH1G13MqkWCX/G6sdY0gEDBAVmn2UEU03nbXV1Dd4USpI1GAs/yDrAj70XKweyz+QS5TfyZGhmvN0+EzvcpZ3PkbD8KZh8+ODrEwSMKpI7395FrlrbptnHzWWI9f/a5O+j2KJhV0kdgI4bnXuKJXKT46Ay5VAhguz+dzLC/R/Y7CiIEkvvNeG6fHuxhn3v5ZF3h9i1KrG5d24GvuR0hkJidcJ/sg/u42Kf7rphRNrp5Cxt3vwoAuPHa1xBdpX0BtWhVEtCuHxdGYTYmH2bOiW3fXwT87VbT5XuO9lM8fUqtQcdHh9CW3Z4hwZ7n4eoZBRrnFwPXzpR0uogYoq28wImW18ClXpEFs/eCSXLRI5iXFSq7poVyTOdUFKLPhNboGAnP0mKWetHmIuBgpL72kPF8yLpyMFFfsxHQBsL+Xmq02M+oQ4Ogtmf2DM2Ek8qBj5TnIeN1VZcpAu43bXgeIrY2oacW0HlmtgeAyAtdi0TErS9x4KNfMst3LeC5wpIPn5NXFSr0mE1/vU3zuRJoSN5v8+kY4ISr8iRCPnu9QAHckmayFDX3J9e54fsBWtzPshUHaHdoTWxttyEa9Jn7532cHFCCYG9YoPgp/p8dSwjocizHcizHcizHcizHcizHcizHz8n4d1QAmWkKtaskGKMviWoKVz42l1iqDAThlbDIHjzZ+wD9E2J/3OwIdJsErbw4G2DKUKs0nyGIqVrVn5VO6+PVe3cAAEkzwYu3iHFzcH6Cj94mfZX1zS1sMNuXLheaR0EQuAZiW6KWMKg4G3EqJQ45wv9Hv/gLuHePskee50Exc1Bp2cKUQcUQjIOnJ3j7DcoyTidz3PGoqhMHnivf14NzjPnztvytNhJ4kqpcw/vH8BmaUZvKMRFWonawVV8LFHx/21sEbahROCH1pt/E8Iyi/WYUoGH1V+I2BgOqwo2GU2yuUTVkfYW+e1YV+PTRY/p3DfR5zq/dWIMfsPZiOYdmmMY7b/5bxFwNtE2xUeChw03/YR5jMKBM06A/Qj6n93xyOoHgLOL62joSSdmviGFlxWyKZsiN94FvSeTQaIQ4OiICncePZ+h1KFu1wpnHg+MDJAndZxA2cHRAcIv5ZOrK5q3OGoqM5vTWy887WMf+kKoYbRXA8L1Vae2qGEICZwf0LO+/8yG2VmmdrjckDBMH2EEC7YtGaAeVuASN3t3dxbUdeu7/53t/ifVVWqev/5ggvX/5J3+D7JAyyEU1xckxPfc/bv8Odq5Rxg7HQzz5o7+ia398Bm9OE5Vy5fnWP/k2Nr5KJDdCCAfXqavaafF9EW5sCitOSRmv8fk+3joieG8QAK02v1tPIMuo4viT1/8GrQYzq7bo97oqcf9jqgp/+vBjJJyVeub2bbz0PO2n9bVNeEyuIr3QsVOWvP7zIkfOMCYpBQKuBlZV7XSYSl0hY6Y3pZSrGHo+w87r3GkAiaKPUjHUWvlocmZXK8CzLH5CuExqzTBf4+euAmGqGcSE76+uMeoTzLXaSpDn9IwfHUzx5IKyvxFX9I6PT/Ecz1FxPMJsRJX2K9uriwztfI7EqXdrvPo1es8ZZ57LPEODod3Ndg8PH1H1tx7XiPnPQk8i4srfN67TGv3Nr9/BBw9p702ryol/C1TodmnvvPDcS3j6hJAYjz89wpQhJ9EGQb/nootH7xHUeTswuMUw3bu9FlROmenXDx/j8JzJE3zlID/L8bMbljBBY5HFlZeqgZKUv+j/iAV7H7zKkm4j5Wx3lU9hmPglCQz8yGLrFEqGBhqjHblDmldQDDPwHSmKQc42ZTAcYprSnopbHaxfYbsf9pBXlrlRQ3FdUsCS0ggUfL2ZAcYl3f8vvPIqfu23iNhq9cpVjEZ0Ls2Z+KOYXqDfpz1yfnqCsxOy34lSyBkpIqSEZPKjYjpDn6uZU0aPdLs9rLHGqa40+p9RJSeNJKqC2XmVxszq8gKY832vRZ67f+VgtTVmLFAtQt+xPHq6huKqhxKAsAzQCxQvTG3FpxeadQoSypJceR5CRj5ICOQZ6/EyGzB0jYQ1iLUIXaWyqGvMRswgOSscA3CQNDEZMilYaeHBAlFAdk4IIAxi930Vtw9MyxRSMKkdVwD9MHTIhMl0hBFftypLVyVqNNsIGJqYxA3IiNZBxFXtNG1i0KLvjlsNqMIyCtYYMxndxVkbhtevF4SOydyJltfSkR2RHvSiVckyqGotHTFNmc1x9oBaTD46pv/++I3XsccEPytJB8XXv0bfkRs0mchOVhqnDAHe//QzjM7oLNy8QWilm7/wK9h6lQTf/d4Kcl7fXl3AsMOjdW3dFmhTOhbQ/gX5KkoCTUaiBVEEweytQldIuUo4N9qx/lpyJ61yHLIf1e/38dEHP6FrxB1cuUFtLM+98CqSFvlSNSS0hXjy/dRVfblbwp2ZVW3cXgaqBVrQRQFEsgMQGUxkdXSDypHDJH4AwT9XQgO11XCWTrfWQk6ronTwyDAIkDOccjqco+R/h/5Ca3yWprjo09k7Y+3cskjRZFKc7V4b6122Dcp3c+dVFQyY5db3sXuFzsWNDvub0sP9T8h2HGOE2YirizJHwvrAjchgZ53W961tRlU1QuveYjBIHSFPM26g2abJkDyHAAAgAElEQVR9u9puOUjs4HgMzeQwXsUaykqiJRlWHhoIy1CqZ3jCPuvx02Ow+cMEf7f154tjWQFcjuVYjuVYjuVYjuVYjuVYjuX4ORlfXgFk4gmY0vUTSOWD4c2oqwUlMGrttFGMXmQjTg4pkzbvH2KjRxF3r72O81PCXs+zMUqLJZaxa+wVWuK3vkOEL70eRd8PH3+CGzeoAphNz3B1k2nJh+fIRpR5iZK20/pSSi30OOx/6wopZytOysr1Dd26cW3Rs6SFw2Qrjr6lrp08QOBHWFllshFf4+KYKlCtRoJ2m7L/w3zoMpXdLbpn2VnBDveUxZsH2P8+VTD93MC3VO/CuOZ9aQDJr2j36hY/k3ANw1lWw2Psu8lrGMWN3N6imRShQcovrMNaJ1vNAM9xxr/ymwA3va5uteAl9Fyd3hV0Gtz8Ph7i5s0bAICVdcqSHp2eOn2kZquD4aDHc+NhY4P6MQfnE2QZffftW8/ivT96nT7DGaewkUOs05ynhYcp0/KORiOU3KfSH17gfESVh1dX6J495ePdTzgz11tHv09ZxnI+ww4TuGyvr+BpRnPabiRo8nueMXFQaoDeKt3n+TB3zeX37txEPqcsadxsIGWSjHFRoMGZIZtWkRAusS70gghES7gqVy0r3H2OemD+6k/+Av/r8F8AAD76gNZMOirgs0abSFKsbtK7KOoM+oTm4+kf/xjl25TJS8oAF4ruY+PbLwMAtv6Dl2A4ay+MQZFazaYcitejkp/P9cyZ1nvG/T66yDG6OOX51ZCcvXt+fQXPrxLxS1ldRW9th98XXWc4zS2cHU/29vHwIVWPTj/dw9MPqKL467/xHdx9kUgcKlM6wh3bP6IiDzn3sZRlDvi26mBcNTPwPAjPZm41ctvLwtdYa0nc2qEq+WotsM6lMtlex6uKm/o9z1VCL/c1257E+aQNxdX1a3mGfMYVOQ3MU66IlhNMmZ5bN67iSpPWje2P8gc5RjPur0ynyFk/a2W1i3aL7mOn2XZ9F5PZHDs3Wctpg0h4SoyxeYsqeeOPTnG9pDXdWUtwd5vuu7dWY2eb9t8z2/Rud1b7GP0i7ZFC+FAsOtqIUhiuPs7HI5S3mFb6N7ddRTTkbP/U28DZm0yUdD6AmtCzTg5PMD6lZ/yv72xDCrJtP7iYoLCCbcvxMxtWuqSuNAyfzYGSAPfkaaHdGtf1gpBJ6BplTu+xKMhuinqCbsK9XGETQ+65yic1IstULwI0I8pqr60CpSUm4L2clzVythnG89y/s/kchmnylTHOdlJ/u+2BYp0rAJIrZTkA49Oa3L72DDZukQ0ty0tSF8wdUJZzaGsPfIWEexFFnmM+JfstyxoeV6bmszEyLjGMuIrRa3fRu01VkVwDJ0OqHoyKHJqrbWUJFO55gYLz51bHThnPSQUYpV3GH0IsqrHSuIqnAuBZTTT+aACBkt9bHAVoMvImiGKU/NlaeAi5Jy9oxFAisF8DgGWV2XerQ0Bxtc2PmggTQo2EUROCCSpKIfDGe3S+WJ29bqcLWL3ZwQAD7vk9PT1312s0Wmi26XpdRgP5gYHH9jsKEpQxvZdpNXF9cbqqFlIkyne9ZEVpe/0b6K6SLVeBj5DXXSuJ0WTfodlOEFtCD2ks/YQjKhJKOIIUoHJ9acZ4i++WARJGPA1PD/HkPvXB7nHf3+HhITqs19aOPGiWqZrsn0AH5JNMJ33sPWBivMN9VCzB0G3R/Xfv3IPP6It5WaFiIhPtA9JWzYxwerZFqaHsvuW1meepQx+FeQjDpF+eqbHG8x9xpRsAwBJDWlcIuBGvrksM2E+azPYwndD9d7s93Lz9PF0vDDDjfvhFK7GEdR0MFpq04gvEbXbtGQA1fyZgqEEopeNV6CQ+thjR1U4i0g4GIIVGEtkKYICCEUFjPpvrCqgqW13PMOB9nSUedE77vRUY+Hx+y7pCwNVFzQ22YZmhxQRGjV7XcSykZY1A2375OSRrXcdegOs36D2+dIuQWw1ZY7dDv58OJUYXLMlRpGD1COxuJLi+Rt+z3qb5b/kCMfto4+mG5bOBkhK9Hq3vZnsVNUuonfcURMUVeK64e3XtEJW60piwbunJOEXINmWl08Au76eoMJj/PUSTl8eX6wByQ6JQgGR41enJEQ6eUvA2m4xgansQFIsNXtaAZqfWkBN7dT1Bq8klVyExGLNjPRtjllrWq47bnJ1mjE6DJuCTjwluefOZ64iZKMSPG7jaZIH10QX6R1S2v/786t9b9rSmQAmJlOdkZApIfintXscZdAkJzY6bhVuasoBk4o5nb1/H4SeWbbJAyVCPi+EUK8xCFfgezvu0yaKKnPCVtS5a98jJW332Li6OCbaSvnMfbQ7eaiVhtGU7NND2kOQpKoocoxEHPPDQaVPglfi+u49aSqxzEGaCAWqG8Dn2q8EpNpqs99dN0NwgJ7a7vYX+iJ6xHUjcuEaLPiwmCBkObCS9w72JxjCnQ6EZx1hboftYWV11jd93n7sGp6cURNjfI73Hep0C4p2tLl5+lQLi9z48wpyN3MHJOW49R5DGxso2Mg4OvvnrBAPyfR/PnVJZX0ofhokFBicnOHxMAcisUhB8UI0mU8wHDFNk3acsDLB2g57bDySqyrJhagQMSYb0UHLjbu0J+LGFLIHfj1g4NJKCc4AMb8EbuZYSJ0/oXosTjWFEsI76jFnaZhnECv3dtW/cwIvfpEBpxYR48mc/pr/7yR4ChiFNoxqzDdoXvV8gaLRsJSj54PYgMDgh52U6naLzlTV8cQghoNggi5S1g7IYO1cIypyOL7DHjKfzo4e4zeug2WhhdkaH5Iz1uvoXE1gzsh746N2idXfRvwAKmvOnn7yFOKDPb169gVrQ/GYpB8m1hscMa404QsQkDwY10hE3QheF0/AJgmAhBJ/Svl/traBlGcN8g5yNv8iGkOzAqRoQHkNOpHLaVTEHi6txA4ppEHzZRKhonrWQUAzvODl8iowZ55752jecjbQsp/cmE8xmBOHJZn3UBX22KMb45ClDUtILdLq0X9bWN/GDv6Y1odcJNnT1Xhff2aH1cy/K0XuNdYy2gN4Gw71iwGvxKRJadrQCG9Uqf4cP3hZoBAZz1iA0ZohkjTVAr0h4ITtDI3IAuxcHuH6dDuiirDFgAppH2RQJJ6laosR/+xI5Ncev5/hwumQB/VkPCxOUygAMgRqMznH85DEAYD6buHMEUkNZNmylYOn7fMEELnWKJLRWbSGqnaVTFAzlhJQORg9PAux8VxYSOU8x4/XR6HaRW0ibrpjpkQhrgsBq7YXAJU5kAKgQujaAoi4x0xaaXWMyZjicETBMImGfTyqgyfbs6u4WRke0By72D1EUzLBsBDQHl0VeY8pw1Zy1toLVNkKG6u82WsjYiT148hC1JVOaTjHjQLNC7HwfzfdJ5HeWOE9D2jkXC40zBQNpbdel1hpL2gJdQvH12s0mdjbZnwhDTPhMmRYGigOGOAwc9FMzKVhe51bCF6HvI7EOd28DEWsne37kgqZ5Wbhgw4+ZSGulhybbqHa3gz4HgLO8hGJCkvW1DezuUkJwjQO2pB0isIyide1guqdnZzg9p3eY55WDn+ZFhRk7+TYZOZ1NkDPxhVQxgjDha3fR4iRaGIVOpFtAAHYtu0BQQsKS6UkXlNQo3YeUESjYvn326BEOnxChiuDk6O76Lm7e4PaZZgtdblcxWYops6yORwMUDMGVQYDGKvk23asE6y+VwWhGz1VVFTQHiForF9hK6UHY89sXrp3ABlgVLsG8DRDyXu1tb7oWCKl8nDPhy+nZhD+t0W6yxmwzRMQBfzKfoqw4ufz4viPU29m9jojP0JwPDy3goLtGG1dIUUK6FrFS17CO0GViPI91bZUxrhWrIXx40rLgZsiY6ATGOHbiuOlywGjaYLcZOxZcU3po8DqVG11Iw6REnnHBr4RBlZN/l6dX3fwH3BLjC0EajaBWpJQZ4+fjqdsLK1EDJ6e0Jlo+zdd2A1hv0vx+5WoTAZMB+SZBk21buxEg4WJLdImIq2JfcqMVQ2gmacxq+AyBjhPpRNzjQoE7QuBrZhrPC8ynNAfjaYEZ2w7fU2iy5vWmF6JImK12mqJvM/U/ZSwhoMuxHMuxHMuxHMuxHMuxHMuxHD8n40srgDYKT6cz/OTtvwUAPHzwiWvsrcsCurIVwBk8B0Xx0Ygptrxl4U2hhzlD/PoXfZSs8xL4HqqCInGpQlcJuHP3Fh49IRjUGleMNreu4fiIMlGr3TWXSfMjIOTqjFIKnoW7COEIVfQl2F7GWaJUaATcPFxWJaSylLM15n36HlNZeNgQ/f3H9B2yRLvL0MVpijqzzcYSM5YbUL6P7XXOvnCVIBASMqDIP1rbxIu/9x0AwAd5gfHDfTfvlkbZ6IWMxdEhkVBIKVBxhtZP2ji7oKrCaVFilZvYVzc28eFnVMHJZnNc36BM3pyhjePzQxjOhGTlEGVJFapsPsQBz+9skqH/hDIocTVBPuFMTczwzrxEo8vQut1VB7eYz1Nc8D2dHD12NP1RmGCbiXoSq7PS6rnqzLPP38EOp43ztz/AjTuv0FxEMQpeY4d9lggZn7usctxouEyUSTbR2KHrHR0cIeMMVGuWImIo6owz2oeDPvpMHyx8zyJmkFUpxgwnDqI2OkyGUulzaLnIbgEEfVhkvDSJO4JgmCFnaEfnU3z4+mN6lrKLJutP9XxaVyIWuPIiVd5e+AevYadJ1eLB9z+BfoPWf2sKDHmdVndX0HyWPtO+zSQxKoC05CZG4CkTgeRViTuv2Kqk/hzsMeRm6JSzuVktsLtN2dznnn0WkUeN6+1uCxHDJvzQw9EhrdPvffdP6ToAnrtJ97+23sLqOs3XZw8f47MHdB9vvf59vP3eewCAX/727+C1r3+T/ja0Fdoxxqw55ac+/ID2exxH0Fy1nk+mTrMuSRIEvI/WfFrT/b2nuPiUfk9ZdStJYxxcTgoBwZl4IaSTmLDwG+Up+Mzj7mkNYYlr/MDRnBvhQfB6Oz4buOy7vVYYhu7egmQdrRXaQyhT7J+/CQD44+99D5tbNNf/6T/+ffzbHxKc2btO8zHDS9ipab//yr0zvPAaZRzL3gyGaaVVqGEYJm21/8Kgi4tTetaDsxT5nGFg89IWbHDleWD9DlUBSv8qUp/2YpjQPQSjCTBlCODeBZ4+pHV6OvKwz4bz+LNDvPaLdP+3bq/jw3esFMZy/KyGYcRCmaeYDynzv/fpu/j0o/cBAHU6Qy3tOWggWNIk8AOnZ7a9Setgrdt0UMQsS1Fz1awqMldpVMKDZpiUMh4k61FaCLZsNBEzFfn61WvIGAJ6eD5FzfAVpRQCY+UXlHM+Ks78V8gcHFQJwA8t0VMDGduHOBxDVHwWsT6pno9Rs2xNGAHdLsH2Rqc+xpwBj/QCHlhLoHCEJPTZ3vYOfG7rSLoreDbgrHuzg8OHBPGrdA5jbWg9Q1XwnldWwgkOakhELzz/SkBwNcdXylHHo0xRcuUh4+pelpZIGU2gIDEIWepCeJgwKcQoryD5/tqdjiNUKRiWW1XaVcc63RUnGVJXJWZckavN1EEujdFuTfiMRCqrChW/wyRpQHDFrqMFlPvuLmKuKJYMzzsfTBbVTOEtVOSCFqI2E7UUBYxH15jnlTuTx2x30lmKktd3ZQxmOUOIZwVqnv8gr5zeoB/6iPj+rU4q9EK7WngCkiHwqGoYJpWZ9Yc4YD3b00ePoHmNNRkF0vM9bAb0fFudNpAzqifLkXNVr55N0GB/Rq2uYeUZQuWs3/0K/d73ULDGpq6AKfuHraQB5bGUhC9c2w+URMCSAwGfzYBGgyVFtlZXsb5O5/9at4eI38UsHePjT+gZz47p7JjMJ2gxYi5qthHzjit0jeERoc/e+vHfYP/hYwDAK699A8++QAgkCymd5drZAAHpYM210QsoNhakd8BCuswXFrpYImW9xb3jPTy1cg6mcGgZLQ0SRvT1ui347O9YHUDhhQuptLpanOm+T9I2ICiwYmK0yPPgS3snVo9QQLBMUgWNDDT/ldTOtzvoT1HzOzJxB589JsI0WdE79Dd83OrQOrjSCpD4/H3w0W3QOg6MhMfV+BlXsvv9EU6HbOfKAqWi+a3SFAGjEJ+55WNrg327UEHyOhXcAnU2L5AycvJsVOI0o716lgmcclvKoAZKhoQbrwZv2586vjQANNwL8PabP8RH7xNrYacTQxqGiGQTaMuMWOdOZLERBPC5XDs8oV4nTxfwlGXLquHzhtVFgZAZxTx/4UTPpucOmrW5SYHUe+++j+tXCCa12l1BzkGkLio0Oxba4LkSv5TSLUerDSgNnAh6BYPNDm2QWAXIWRNNo8SHP/4hAOD86BHfQxdVSot1Nhq4QFXBQ8EBWRg3YLgk74exm9yCg5VQeVhhIVa0m9j95mt0DaXw9v/2JwCAyf3HTrMJYsGp1OHAYDYbY2OVgupM59BsKKOoidU1CgDDKMLGFgumKoXJCcG7BlzOjhIfHkPaNEqUIzIG/ckIAZepVV2jZHxzhQKeIAMkOSBGNkXM8xh5AJ9vmIzH8BhqUCsNPhcwTjN0WuQ0zia0iE9PUzx+Sg5L3F3Biy9/HQCQdE+hPXov80piznDDn3xA76KqKmcM/HDiekijoIHQJ+N5lCrsXCFYSiEUWsyidf1ZMtD+vI8xaxH5QeJgh/NihpIP6CLL4XOQUlQpEp/Xh2WWvcQCaqAdk2gkIswH9Izf/YN/hQ/f4P4KE2DQZ7bUwWMAwGrUgxiQI9M+HCH/kPbL5M0n6EwZo15V8J+/QffxbAu9exSEq9U2v0PpmBhNVWN8QgZj/QprYwJfYPIyVn7HHdz7JxfQDGmS2MbVLdpzjfY6JEO/VrZWEHEf5j1OkDz+8AN8/IAOUfNpDhVYSEqA/pDm4GI0x5xZI996+31s7FDA+MorFOQ3kwbOLigJcT7oYzymAJzYydg2eJ4TTdZaOxHj/+Qf/grN1+AcD+8TVBwqgOTATOgKvm2/EQoSFnajFsGg/b1fwWNmuf37n0JwciKMfJfY0aKiBg4AtS9R15TUsiymRVEgZ2euyI1LVASBwCozyj5z7xt48oTWxGdPjnHjBXLsrt2j97aqD/AM90snNwNUDfoOXQMYs8aWKqEqtqcV7ftP9gL8wfdozg/OfKBmB1ovmPm++dtb+L3nfwsAELVuQdUEiRYZBXHvvznAj79H7+LZIfCjp/R9f+NrHMzY+YLA/gHZwoMihpKXFtdy/EyGDYgmwxMMT8lJOd1/iGJK+1LoCso6xVJA8Zr1tQHLwsETtK99WUO49o0UNfcke7J2+xli4aR6cnGeegz38yIPPtveZ25cx4i1t84Gn1FLCIBQejAeQ/WUhOET0gZEHhTAUD0FwGOWZlNnmM/ouQaTM0yYP2B28RgAkM9TGCuSPplgxnp/k3nphLy1FziRdiHhmMytHmeUhBDMFmxUAMW6wivPFBjw3q+mU1R2v6cxCo9+niT8dxooOWhCDdfL5SkfPgdhga/gy4Umca65l4mDoLKqnF7oeDpzQZivhGOszGtARnSNYVnCs73RnASsIdBgRsV2u+2YRqsiR5VaH0eAjz8URiAMrZ4f3ed4NMSUP6s8D8ZyCqgIhn2ENDvH0SmdXZYhttQ1hKF37wURQmbL9ALfJdilVDDM8JqVlQtEa14PtQgg2AlSIJZGAMhLDT3laysJwVDmIFCIgy/0IZuFHp0pa5TsTJt5ickJQVEfvPMunrz/MQBAjKfosRNjeR/M+Qh1k9adDBR8TuSnsxnyEb37uqwhOFjqbGxh6zYxX/s9Wj8lYtfzmRVTd96WxqDgta5VDZ8DEyE0pOvZpHc4ngyRc/9eluaY83sxNbCzQ77g6voGXuT90meN5x+/foD+gJ4vCp6gYlhnWWSYcItTOp9i7z4lOPqnhyhzWo/3XiNfzPdjFKVlBi0WgXm5SIZoLPR6hZALBnQOZjqhj5q1ocvpDCXff1kYxwTsBcoVC0IlYPsXSm7/yYoBCk6MpNPxgtVVKTRCC5mtnVC6Ugs2cTv/MHA6hkEYOJfIkxKK934qGxilNAfRNIWlIw0sm30rweYKPWs3KdDh2CVEhpAXrZIzVHw2W8KUp08nePcBC7QPTxFxO1ygfEg+m5PGKm7duEF/19oESm7d4ts/f3SI+08p8bDXr7Cf0XwdpCX2U5qvcaVQ8zzWvg8VfvnZvISALsdyLMdyLMdyLMdyLMdyLMdy/JyML60A7j25DwB4560focVVAN8YgLPdKDNIjtRjP0STS+G+VKi4XC4pOEcSLYgbkriJ2YSy/MporLDGRllrZIXV9xi5StI5a5mgrNBt0QU31npIOTvw5INP0R9Q9nz7egjhGoIFhC39w+p/Sac15HkKW236bimBBjflHj68j/ffIgKON374IwDAr/2H38TWOj3foH+KmiN734/gB5Y0RCHmbJAXROizBpjtxW+vbyNk0pbS82G69Hebv/gyvsFQmh/9z3+I7NNHfK8CtkH+Glc+x5Ohg62OpgNscPN1s9FGyVAJKQ2ucxOyqCvMj6lCkzH8Y213A5qzvAFq18SMYuwyHX7igdGK0FELpxc011urVJlrtgOMM8pKjft9bN+g56pMDW1hJDpw8z6bpZhn9M49rhRHSQsTZk88OT7BaPgXNE+dLVdNG02nKCxzls3ZSAWPKzxlZZz6lYZ0mkdGCsxnXDlBgcySJnCmJ0w2ENf0TAfnFxjNKdvSHwkYvnYoJpjyM7aCEmvh5+mFFrAHSkgFTG5y/OAC3/2Dfw0AeO/1jxFVtGZn5hSaGTw716mStlpHaA8Y2vN/v+sy9WEmMGJY08H8HMfv0Vq6df2ruLNJ8BKbxRZYZOTLeYH5BT1L924PP21Y6LalMTVGYu8pQUfOjw+huPqfKIKCAcBXvv4LCFu0viNmzN28/gz+5iFpFD75bB85Q6567Sau7lAWdKO9ihETKZzsPcB3/+gPAQDFmOb2pVdeQciVhKSZoORs8mQydqxjdVE7BEEQBA7eM2DRG08CvQ7theFw6KDbfhjCsxlVXyLwWa/K913W0ma9ZeAhzBmitnsFpsnwJukh4IoLwaUoU9xbXXHQrspmwMvSVfzTeYbRmDJ2w/EA+0dUgV9ZWcE+w9v/9F/+H3jlLu2pFclMxocX+M2vEjPb9ZUEhaa9XyNHwBUBL92A1vQOBlPKtP74nT72xwSjedqfY8ZVmNgzMJru6c54B3VN185nD2BOCdlR79Fe+d4PMvzzN2ldfa0V4iln598qasxz1m8SCmZE3zlKc4Tm89Do5fj3P0YXVAU7O3iIMSM88skAobLVFAPJ56DyIgRcBveUdHDgkAkHAk/Dsz0SVQXNpDKehCMKqXXlWL6LvERt2QJnDDs0NYKY1l6ep5d0xCoIrhCHoY+SoUpKGkfKEjBEK4bCyLPspgs4oq5rDPksPT09w/2P3gAAHD2i6gZ0jRYjOJp+jNpWGKp6oSOGBXRRSgOfyTgM79+00igslNMHopCq6ju9tmsZ2R8PMPyUqiWzKkXJE9llm1jWGkW2qDZ4jJAJvMDpA3rKOL07SCCb0DXimGHjoe/IZfJ55nRtlZaO9bIbBlDcwlEa49pbjLQVGYmIET6BWlRYYWpIbeGqBhYaJw0xjgJwrRxFXaFijVhRSdRc9fNiIE7oPNMCqLlCIth/iVUDADNSCoGK4el1uSACkVKi4ndvtICtQ3j8fnxtIPkcLMvKnQHZPEXBuM8ojJz9NkY7VuWFLnXtoIhlVTuyl+x0jM/eJdTRB2++gf4T8rXagY8mo47segzKCvk5nS9Dk0Fz9W44vMCEz6651AC33mys9tDu0NwkDI1NTeXQNhCeg30KqRb+jJGL+8ZCs5pXEgbjKcYDJsiZTCDYoew2E+zuUlvNSy++iBvMFH/1Gp0XT/b28TFX9w7H5xC85luNCB1Gvq22E2QMMRw8fYi3f0g+WMK+8O6tF2AsxLjWKHkd1waOHVQb6eZaQcLwelsJ6fs6rSaCDlelem1oJnFCVTg7EjdjhyCIwmjBCm6h7mUNwz5LlmWWoBZaC6ezrXXt9oAxpJ8ILM7muiwcIkfrGjmjFzNt4DEsYqYFBgzfbMwyJLwvuw2P5zxGK6C/S6CxZtnXhQfFjyX9GjD8nhmKHcczCMuo7AdQzEwulUHJvlGalQDvLfR6wNTuM0LoHfVLPDgnX/GTM+CI995FqXFa0nNPa0AGbPerGjHrBv608aUB4PvvkXBkNhtCT1mkc+Q7quPED6E4+IljDyE7V0oAqyvkINy4QU6gNiVaLVp05+fnqNi5TeKGE3XWQmI6t1AqHzWfIlOGXipd43ifelXWOi/iMw6U3vzJ+7h9j6Bgnu+7Mq8QwvUX2CVqYDBiAyt8hZAXweHFgYO4KFQuSBlm9Hx/+cOP8dIz5HC1A+EWfw6NNsMzfc93QpT5bI7pgOADvmb5i61nIVvU36TSHAEb3WJFYfubXwUAvCKBj/74zwEAg08eoTgnOJZiiEaoW/AZ8tGoBXxJhsZXPcwKgg9GkUHGTFaeEYgaNO89Kwi/tYN8RjCICAZjhhdATBxVduh7bvHmnofK9hsJem+NVhMVG+PxaI4uszlN57mD2M6zEtY3DKMYPjsca2sUmPT7fcdAtba2guGAAhBTTFyfaaEFFB/MgZUNCCKifgM53FbiwFQ5SuuY1Bkin95LIADNB32hGQdfaGdk0lmO3HDZv9VAyRZ7vSER8PXGhUZqA8BLcaANApWQOHhETtkf/vM/xcG7dP+x10NW0VwX0Tlu3b0OAHjpyosAgNbBGM2nZIBFZhzMx2iDB+fk2L23/zGGjNeptwK89h3qzwMz2RmvAjg4mo0myLhfs9tbcaxdXxw5Qz2sp7bWXcPjBxSADOZ9hExBdW1zHSNmZ/vwvfv4yqskPTOh87sAACAASURBVCGZEa3XW8PzLxOU87w/RZuhmc/s9vDSbWLV8z2B/piciSfHF8hGFAi9/hfUR1jNBnjhG/RMnW4HYw7cs3kGbaksNeA3LKV17eb9v/8f/xcAwGqQIGIYkB9oXFnjfSYUOtxbECcRghZ9ptFKEHAfpGEmQFML1DyPa9dvoOJ9VkOgt9Lhz9TuANbGuGDVXBJ0tikn3zNoJPbA76LJbF/zyqDKaU7X1loIhmQH/rNNMvi7GxV2Q5oDM+tCecQcLPISMmOGt7yGsVCPgpJYuyst/PItmv9P8kfIO+zMKQ3FAXYnGCK/+HN+rvcQnLO9OiAbcVxM4fXouj+ajsAtSkhrA+vJzLSBx2zHPTnGhTrHcvxsx/7TxwCA88PHECziLlAteonkYt+HyiBi25s0AjSZSbHLLNuBp2AKmxCqnEPlewq11bkptYNMaxhYw26dqzTLMJ2Qc3W8f+x61HRZkDwFAE95iAK2l0pBs4trIfcpfGj+bFYRtToAZPnc+RzNpImMIWn7DHX3ZYWUz8eqmSOAFQU3Tlg5VHDQy8D3HMFBHTP0LlAQnFgVcQQR0H7oxBFusNM+Kuao2O6MLk5dUjFmGGlQV5DSitorhD7NgScVwPOktHZMqJBA0rJJc9uqYuDbeZnlyLnvuShqF8RHDc9JLcyLCoV9RxYyKzzXP+l70kHd6qpEwfa5qg1q/o0W3oK5kwM2qSsncSSFwIz9iVAESNq2xcZ3HqSyNlQK1xtWVjW0zQRowGJOPd9HZWxCVjpGU4/tWUMq1OyjpSJFNrMw2ZnrxfaU7/wnbRYtAXVpGWIlFD+fJw0yLlrsPX6Aj98nNvKL/T1oZiANgxAJ93w3Q9oXoTJo2DU/L1Gwzzo46+PRCSXwhnmGlN/FlarCzt0XAAB3d+nsazUD5Bm3mlQ+Su5flTJykmeep1yfplISBSdgbExbGwlWAsC80Ci5uDKeFDg8oTP7YjDD/BVK7O0yLPTey/dQ8bn05NFj5CwPtbm+iudfJJb1lW4LZwcUYOztPcaAW7beev2v6VlnGXZvUNtMY2UdlZUt8YRjB4XRlwLYBfvtj/76L+n5IFxbkzQ1avYlY99DwgHP6voaej1uYWo2L/XS0t+FYQif23iSTuLknGAWifA8LxzTvRDCzal2Z7NGzgFgUcwx5p68dD7FnH3BIs/d/M/mKWLuAQ25d9lUHvI53fNYGMT8YlZWmhC891Eq1KXt7+Q13Sqxscow8GbDJexhNGrW2inKOSqWjvNiH5iQD3nCdu7oYoonI/qOh2PgjBlSp7rCVFnYp4dGyAtLaiex8tPGEgK6HMuxHMuxHMuxHMuxHMuxHMvxczK+tAI45iqMpwQEQ4GqonIEKIHvu+bhOAZYigftTg+7u0RCIlhHL/C7GF5QdHt2euYgfkZoB6GMGyE6zOCltIeCI/eMBZnHgzMM5lRl+eD9FBzM49buKm5ts14OqgUsS3quJGzTWZXWyK3wqC+guGowHZ9iyuX+jZVt7G5RBifgv3v40Ye42qGfdTZ68JTVP8owGZEWWa/Xc0Q4ZVmixzo6XGFHXgsEnF3ytFwwHIUKYFas67/0NahVmoP+W/fx1p/8KwDAjDMD86xElNDvfT9Aze8lnWXw+Lk9T2A2pi+VkA4u0rVVkagNzRmPWgiohP5O1wYVMzf5YYIgoTlFmcHjrM2cSQY6XuQyPTDaafGFgcSM4RayqB38x9fKNatfXFBFbDKZoM0sbKgFmvzvj56ewC8oG1TKEMpbiIEDlN2xVdDLgqSe5zkGxizLHQGNhwUjmG2wDqRACMr+NZstJAw9SosMvLzRDEKw3AwCIVBmVjiZfgYtHINIDWDMLG3z+Qw2CZOnZ5gqWlf3vnEXr+yQJk38iCpA/vEU8Yy+O5MG6gYLebc83N8jIqKD2Skizk5+9P038OZXCAL67Ta9z+B6jJwZ+s6f7qFmtt2g10ClLYufvMwDA2VJCS5BL1ssKjub9SEZjhEkXaSn9M4vphlGTKoQNOlqF8M+hlz9VaHC9hZdo932MZtRZnF7rYeNrl0rIdKM/j3iNfrO6z/AhLP9v/Y7/xGkoqzbEALZnOG/pnKQn8tC8EPO+L50/QW0+GUNZyeOIW50coIp65Y2kwQxV/KSVgshZx8bDAMPOwnAULJCCCeSDg8wluFVSWjD2fKqhmRbYmEvRlROu7OsCpQsgm2qCh22NTqbot2g7/nWr/4y/qf/4fsAgL/Yp2f92pUYT/fJpsTtBvxVep+NKECz4ko1ZpCxXfdU6b4NhfWIquh371YAN8dnYQCwzdi8toLGPhG/+NM5Judkdz75hKp4hQxx7zmCFf3gjbeRWTsNA8gFQceUhbIDbZz22XL87Eafz6rRxTlWuKLXaDSguLJSSUAwo22rnWB1nc7jtV4XYWwZKZlUqSyQMuHAPM0dasDzPIScWTaRgG8rWn4EJS2bHu2R4cUYJ2dkH6eDAWq2ydl4AMlVDyUMIrmw2xY4nPM1UpRIGX6nJZyfUVUGCTMitnvruLJ7AwDwzpuEUKqKEhULSpd1CcVnlZQKcWyhiT48Xr9+oJy4u2yRPehs7KK3y3phwofPBDkqCrHO4umvJg1sXSMfYHx+jP0ntEdDi0BRPt04ACO9S0Lf0lXFjDCuYqGkdPYXFeuPCiDgamAtpNNGE6qAxz8PfQmfq4FFJR2UkGu4pFlqobu1RlUvtBwdfEUKVyFTCq6C485JLM42z/NQc9V1LnI0LBlHPkXK1RLr9wjlQVp0TgVIfockKM5wUG1cZScIAiLuA2AxcuRX2EqxtxAlLyuUpfXd5g4RVAuCKNN38uFcAcZz+GWMh2RDT57s4WiPyeRmKZo8jx3VQKvBFUDWOl5bjRD7rK1oShgm1WqJdSSsc314+BSnF1SFO3/7baBNttjwWfrMC8+gZOhikabI+NxKkgSS2UY94TsNYQig5kppwH5xt7uB3HLvlRrGvhchYHg+js/Ocf8hIYYKC5ucT5GxNnSRlZgzUcj5cIzRZMr30YDHe6u7soqa2xAujugc6Q+/jymTKr34i99C3Frh+yghhUXnaEfaYlA7cpjPPqMWMlFoNJnQxlfaneNJEiOe08+zqsSU24EaSRMho2VshdCYyhEN+b7n/EpxCXIqhHHkMBDSwb9tsU14PiJliaAMwpCJr+oCHq+3ldVV5zsEoQdd0Zrt92me76saff7AvgdstshXuX2jh5rRi+m8j5Srvi0Wgu/n69CMuDCejyKz6zWHEfS+LmYVPntANmX9aIAp630/5Pfan2YYVMz0WgMn7HtkMI6hOYCB53EM5S/24k8bXxoAFsycJFWIJsOvpNaO8VFAO0O02ouxs0mL/+bt552g6ySlhxhc9HF6xoxKRiKwwpfNGFFooSgGMdPuw7+CUc5UzcxkEyRDdAWVqNPhOQou6693uzj6+Ac0ofMLrF97BgAQthai8C4QhHDSCpU2GLExgy4x7dOiD1fa+OrXyck+2qdS/6cfS9xh2n1R5rjgTV+WpWMbqqoCPjt57XbTGdPVa0yTv3vFMUmZIHQsYVEjcth74ylE1+kgurW+hRazeabBZwCAViOBimhTFNkcxuMDvxzB1Cx6K1sIOJA22iBlRrDxnIVYsxTjCTnfzTCCFzCLUhhDsPOqGm2ULIQdSM8Fq6M+M1YmXQgOLrK0gNVOj8PY9TMU0xwR944qpXB8TM8wZUjH6soKysLisFPnCNTCQ8TOxjjTrg/CsouZqnAwJQNxiYFKuP4rISRyNo7Slw6uYPsJjJGwizT0A6xtkWF7/PQxZtwPWPraCYem2oBbrhY25tL/QmvcuEXY+9//z/9j/OB7JJvy2Ufv4DV+/881NxF/QPuhccpU5lpgqBgCtZUANzgIC0tsfovglp3nrqB/QlDg00+O8c7/+W/o/i4oiLzzSy/j2ssE0ygfH6PL2JIw8uFUKr5gB6zjEfS4P87znKzANBtjxMyknzx+ihFTD6cnZ/jwM+on3WaYSZrOHbvYdNDHxRE5pmetAImg9/XM9W0kfLjO0hzDIQfKbCTHxRFOmPnz9gvP48ZzxKSWZeWiR7FeML/mee4kIdpt7qHpeLhzm+Y/1RtATs+SdNooOTA3Ve2gHnmeIxjQu51Y5rCm72ChjUYDDTbYaEQwDA/zPf9zjp1NBgjbC1NJCDbSqKXD8fimhEzpGZumwtdfIAjwlfVd7NVk8959j/bnSx9NIdmRr5VEGJHD0lACXXZq2gEg2C6WzCCXaYU5u4GzyqBkDuhUU+IOAJ5fi/Df/Ro9741wgCdH9Ix/+xE7XFduo5dYSBY9AgD4ynMJFRiNmPeTJwQKy3y4HD+zwVsVZycXyJgteKWTwONg3IdBzE7X7vY2rj97FwDQ6rRRcIKonNEZMMpPMePETpoVTlbEDwQidg6jMESTnds46brkn+Hz7LzZdw584HmYcWImz+Y43CMHZqW35faUHzdQMADJSgVUUCgs7E347uweDgdI+f62V7ews0ls0jsb1F5SjPtoJQx5vDQ5XqAQM3yz4XnwlBUD1yjYeVppk93fvH4T0f/L3nv9yJacd4K/iGPTZ5bJ8nXrmr6mzW3PJtX0kihRGml3Vm4wEAaaBfSw/8QC+zcssMAC+7QLQZodaQx2RqPRiCI5NGJbtjfX9TXlqzIrfR4XEfvwfRFZTZAtzQLiPjDjgV2sm3XynDhhvvi+n2F17Ula0AkaLD/PB6jG8ioCm2DsdaBZ4MDkFJNIL3AcO6OVI1Dqc8k3o41bL4WZcSxT5kZOsgSCA1rPzDiM2gtc0s6IACkfyLJCI7HJbft30PB4YQqKHCHv09LX8LjPA9+DZI6lgcCEE2022aoM3DoXBOGnFGWdArqSkFbVkiHEBqkLOrUGkNsDv+8OlKGUzkqnKHIAn9aqN1q562VJ5g5ERgaOUpKmOXxWYQ18H1qxOqW1LBAB7/EUswY8L+rlKraYCjPUEhXeUxZLDdTZaiG2apS1KnTMzyqBZoliqurORcRbROUI791FeJ/2xKPxGO99TPzC6rcpkdvrncLwO/Q9oMW8+Hh9xSmxC+kAjdAGjvpU4fm7vrqEiOMoISUePeBDnS6cSnySKzzcp4TrgA8OKhmjc3LC/ZU5xf6jwxFe4bi3Wa27GK0c+xDsh2V4zJwe7EJwMnjj2tPYbFA8Kj0Fo91BwNlzaA3HNdxYp0TicmMBm6v03JVa5RxEOHcwbw8Gyiav8tTNYcnjWOVThDYB5QlHtRJCztRlPQlhLPxRwdgCC/ctCiBhGO9knKBguxtPK0ScwKhvrKFgmw2lCpyx2uuwQ313+17HrZ9lkaDGyaGV1gEmo4yv3bfhB6ocCwdhgILnbK4lwH3kwXMw484gRz5l54NyAJNRjNPrc7xURNAct02RoQeGCmPWVJ6hzPtxKa5A2GrGz2hzCOi8zdu8zdu8zdu8zdu8zdu8zdsvSPvMCmCNVfW+9V/+Czw+sW5tbGCxRRnrZsVHo0ZZkXqthNV1ysztXLqMUzZ+7Nr/Hh45gZFys4IsY5im1o4AHZZ8hGw0/slJhCc3qHISLdPJ/+3797DMgg4LF4Y4Oqbq3OlpF51jqi71hh2cdej36xdvQHFWzHqcAb5TNM2nQxwPuIq4suWqkpmaYO0KVRP++f/0LwEAb/xoE2e7lOnr7N530LrAl87fKwh8R0aG0RjwyX3nBmVC6stt5GxILn3hDMxLtRpYYA2xMi5rORAT5G3KvqTHDIPtHrlTfaNeRsSQxsjzoDT772UzYRTpec7Lzh73jx/dQcpiEs3VNQhOQ0a+dCpVUkocsp+M8kJ4TYK+KDZ5nyQj57nih7ErmxujHFTFCz2XDYIqZga3ns0QaieeobXB6dBmtiIsMxxUR4Bg6E7kW3jKTIlRG8wI4OfEQaIodt9TKGXtJR0pXSmNnNWejNZ4xL5s6yuL2D+iZxwPB1Dcj7VqAzVWcD0vq2IzKAoaYOGUC9fWUK6RKNHlN2vwbpPoyeLdLkLWXmHdVZQurWDj80QczzfryCp0xVLs4/JNInU//PgOqqy6+bf/+59j9IiqbB9+h6qMD9/8EO2rVDUua42dZ6h6TdBfa+L6aT+YmDPZNj+mTYTmIhvMtldwynDt/ZMO1tcoY7ogBWKGfF++tAMAOOt23DhWuUbGXpq+SnC6S5WrH7y9hwZnV8MochlRq/g7LgoEJ9Tn3/3bv0WTM7S+V0IpnnmHKs6cJ0niIKA2Yz0uRuj2KdtpIg9lrlyUl1uYMpwkTVMoXnfyNHcVrZRVybzeGOjRL/veDE7sRyUELGoRRqGr+Evfh/KsurDtR+PG42Q8QY9VQEUxhB+ykm5TYo39sWp1D70Oj2+GaR6bPg4yul4/FQh4nAZawWf4KUQASLumZdwXvssepxBOuEkpAcVw50dnY/zziywa05bwc4I72SrM/f0OopBFeIochSu1GzfwC2MQRvR3K4t1vHX3APP2822bm7Sn3L5dwQNWMjwIFGrMw1islVBq0/69vLSCpWWaU0b6UFwNHue0N6dTQBcWz+BBMjQ0igLUGF5aKdVQadDeG4Rl5yFnjbfr9SY2N+iX5biEjOdnPpoiZzTB/p0PUbXqg8tryKxBPI9p3/fgWV8t46HgakSvO8DBIa2hzaUlbG7R3vy1b/wKfcdg5JSK9++8j0GHKiFBECCMrQBNAM3zISkKTLm07bFaY3WhDROyemUxheZ1X0mBgjfnwvjwWOk06Z6hx2td75C9NGWEepOer9aow2d0lDDmHExNQ1khliJzaqm5FZcZZdDsaRj4AgEsVF8iZRjHeFogZWTBNFPIueqRsGBZFCnUazPFUwu3FNLMvE/9GSTECINuj9ap0JqTRzFKtrJiqP8AEs6LeU0uV8rIC+ozxfdZ5LN9Pk1y2P1FaQ27gAQwLkYwReGQRHYz1bqAZ6uZnnFCcdVqGQWPqzRNoC283swqnkZaGGngKDFCazTYEP2pG9ew1aR3fvzJHfSPaayECRBXWcGTY0zRqqPaoj2xvFRDZZGqesIIrLMq9PLFbdTuUMz6YH8PR0yb6nJs+sGbKYZDirUWFhbxbMv6IpZgWClSa6cRBC00EoYS2uJNpVRCyDGrVhnOurTP9XpdFxv5QezEkc5YBA7J1IkZtpdjFIq9lccT9EYUiJyePnIijCuLy4gZHm5r1ul4gH6XkCmjsz7ENlNKjICyMEzjQWtLFQAUK05XWTkzCD1Ijt2iOID0rTJsgJwrxHmWYDyifkryqYM+sx4cAinh+xZplztxOE96zi0g8kOnJOuH4awCyPeWZ5nz801GIxQshKeLDCXug4XFBVRZNFGhQDZhfz2mNalCY8TKxykMghOaN+X7dhee/RcA/HO/tXXu8/GjB4Axj5ikAr5hv+GVEgKOKaaKxmVUryDkuD1Rw09V/mzTAKasRByXtXMM+FntM//1n/3hHwEAFpfa+A///t8CAP7ra29gPKDg8CsvPYnLv/oyAAqoO2zwnRvAsO5P55TeoFQCDcbLFiZFUOIFNtEIODqPGxWcUn9idWkFteP79BmeCRfWWujvEUa2sdFApb5D97dcx5BVBrO0gDA0YI8+ecuVm+2ANgACqwoEgzobdha6hG6XXvJOexsjTffa3CZo3ReXl3HMhtd/82/+FPkxBeFLiy0H2RDQMFzGTrOpK/muX36MOrtcgWApdRNJd+ijxZNV2koVlHgTH2cSnUN63u99n2wphB7j8kXa/FvVy7aSDBgJj40oJ/0EvT7BzaZFjhVn3UAHmJPdfYA3jYnyMRmz8an0ELEFwmQqcOeYjSuT1B30V8oNvmc43HSlVIFkuIVSAtrC4SBmRulGYMQLoYUzPHz4EDeuUf8OJxMkxYynlzEMs8gLxz0KLYZaSnjMmfSAmYqs1k7xKfADFxRIpWEsLIV3XG18xxfQyqDBm8xT169DqQ8AAA8+fg9DPqxUqg3ss+mtthhQYyDsIQIFDGPiK0ahPqFxvzpKIc9YLW6k0eE/iF+iw92Tv/OrqN6gwxvtB3woTVL0jmmh3z88QIv5m5//738Fr/xn4oytrlMgVAlLeOV7PwAAtIMQz/x3FBgpf8YX+Ek70Lt3KWip8nsFhEtqLLXb2GDY5INPbqHXo/m0c2EbmoOX994hJbU8TdBo0FzZuXQVQZsgIsl4gIyfZbffxyEvlH4QoFGnz0vBYy0vsFmjjfbK1auOI1quLaLCHB2rGgzQ+7bKewlLKP/da6/hPe7b5lITOSukbm1uIuIESJbnaHCyJs21gx9bhcGoyJzCpxASQUHjqpxMnDS4OqeqpT3hoKEW7q2UcjyPfj9Bh8d8mg3RKtPffunphoOseZFEZUpQvLYNUI1xcudC5I4DkwuBlOdCITNn8GznoWc0MqsmGHpOej7KtVOUi6WPwTH13TujEkYM+c6see/Vx3D6kA50iTIoLH9CA1Zv3kDi9h5z0MZ9VJsVzNvPt335a18HALSWFvG9b5Oq6zuv/gAdNrkeNatoWYsjz3dm20YVmNrxObJwqAKC4ZvVagUqnx3IKjzPqpUSYithL2bQZqvWWAkClBcJTrnYqNllDAtxCUNWWsxNBgwpdtBxyXosOzuIVAniw4N4btqnsXnS2ceDW6T+vX3hMtqbtF5+bsGqPAukPId+CIU3OWAVwjie4cTkkDzGM2VQW6H1dO3CDj1rpQwjbDAdOE47hLKCoZDCh5Gsejqd4OiYKCOPmCbiSYlLFwkauFCvwIJYjTGOv1TkmduXPCjEnJyytjoHhz0H+4yERMJhXlQAacrJqeEEU8aYZUrAt8lQXoPqAqjzQSiOQ/fvRgDKmqNrNTPThsTAagawmmdcLjnaRBCEjpOljMSUTcQhU/AZDGULnyxFDjoaBblTpy4K41S3Y4/g+AAwmAwR+lbd1Jpj+zAV+js/DBBwkrtSipDxdz969AhTXltV5KEUW4gq+D4zSF6zY6Hdc0eeQJW1KvrCOEuzCYC4RvtR5QIlu9uXtlHl5GelHkByjAAj0eCDHsoZgip96dbODh5wosJac0wmU0ct8qREwYcjCAHF79YUgGfVYwMfydSazBfcpzECjplK5QpqnBzv984cJ9LzBKZWmZftwIq0gOQDSCg8BIGl2EiX/EyUcuqxEMYlxRXDI9M0R8iHpnTcR8H95XkebOhJ9QMeV0q5w+DuLvVFkT/CBzw2S6USSjzWS+USFriYVC5Hbo9XeeqU/C1TWBqBgMeVMcoVdLJcu76GgbOHEPCdbZSN+YpiitGQnmXQ7yNn+KnQBjWmVV29cgmlbU4iBCFCOwcsBDcw0Jz9SpMcnM93/7VNOHj7zCKJaZyfOnR5mMVm3VGGHh84tRc5iHbKY6bw/Jn9y7lrnGP2QgGYpvRcwchDzfvJyO/TbQ4Bnbd5m7d5m7d5m7d5m7d5m7d5+wVpn1kBXGgRhOT3fv+f4eWXCdL2t9/9Dv7sT/5PAEBrqY6QT/PD6Qgxi7Z0+kMMWThi/4CycTpJcXWVIKJeoDDhjJ32MgftC6IQJc40HU6nuMFiEFMucx2cdLHM1wjDFIG0AiMLaDUp+yiMcjDLcV7CK9YLho/IGZRTLjMAevzZP/uLv8RLL1BW+w9Xfw/tRfb9YCGF+spjWFqjqk2SSfzp//oAABAFegb/0oXz6plkCmvXnwUAXL/5Ij1fGDpRDuH5TmBBF8plPDwpnPJR9zhBd+8R3yu9quvXrmN1mfq8yHMovohRGidcMeqfddFktcOFhZYzec25wtAvJLZ26FlMuYnXP3iDbkRqBEwyHY4KnIwYcicrOOSMV7FM93ahvQ4wSdUouEyUOZfTMJDQxlZMDKas8mQrdmEYYsBQhMvXbqD3CT9rLjFltdFkNIAfWg8/eiYvriFlOEAYzKB6UkoHv9NKAUx6Hg8GropSZWVTjZKrAGZZAY+zcYGEq+gm2RQBZ733Tk6gJFX1bDZRK31O5MOg6NKzHLxxC4N3qFosjkbIp/SZbKOG1S+RsMsOe/mV1pcw5Uys1AoBX0/6PhqLdK87Ozs4vH0fAFBZX8TLf/hbAICHLMiyuLWFP3ieKqn3//PfQbRYJEgSVAOAg1QAXK3iVOnJKfVzXijErBppILHIcJe8SLD/iMb6rdt3IX4iMzcZDvAuVwNff/3HCLhikOYJQp7LxmjUF6jytrLcRsEE/+5ph6/lI53S/RVTIA5ofKcqR4nRAdJ4yDkzmue5E/6x6nR+VMYiVzwOD3dxwuT3ZDrFhCFJvhB45hrBbeNyCYU1GLaGsGoKIWYZO5viDI2BsOICcpYzM0rBcMZ0xFiV0WiMwYAy00cHA5yweMWoSCA5B3hp7Yt4ok0+TK//+BN8YZXm9q9tsiDLscb9j3huFQY5vzsN/5y/m5qlO61wE4zzK9KZgs9zb9EPsch+hBdUBq9H/X97qLBxhd4Xa0JBhYFT8TXnVG6N0U5ACfBw3Kdn7A+72FzfxLz9fNvlK7Q3bm4+hu01Eidbb9Tx7vvv0geKBDFX1dNEo8sQa+lLTPj9WihTt5+gzBWUleVVhLxZZrlBwDBMnBM/kp4HYf1UuWIehB7KPo3fRhg7ZdjKSojEGpvrAmOe+3E5chUEu0NkJkOu7VouEFoI1LCP/hmtFYUGvIARBCV6vlo5RpW/o729i/AjVh9MB5gUM2EUuy9FzUVs3qD+u3CdxJiicgWKUTEm1EAxqzZY018B4xAyuZqgNySUDSMiUS3FqLG6L7IcwwlX9CdTZzodeAIBxzuVOHRQ9ZxFYJLCwFiFZt/HiGFqp8MRhgPaP487ExS8rperFZRLNg5ir2BFVU4AmGY5RMSQQg+u0gshoG3VBrRO0jVYzMUUCLhyWK7W0OL9QGnp1rfT0w54+cXSCqOSFhYR8P0HpQgR7wdaG4zHXJUMfBww9PLhvU8wXOa9gSlEYVxy4yfPE4Chudr3oCxtoKOyeQAAIABJREFUYNhHl6G+5VIA02BxIY4F1ETBjqxJt4ejWx8CAM5uf4wh7w2jUQcBq2uvbm+hfYWEkpa3LwEAmpsb51BHCqjyXBAePEV7TViuQ3u0h1abAS5GFkFFc+zk5AQnLCjkaYVQ0vUkAveMQitXJ9JKO4X86ZTnTZbD53cozAxJdxKWkXLlNsMU2paqrZG89GA4Js9FjpShoaPBBAlXCYVSiFmpstwouzjoZJ9RAtMRwKb3nZM9jAbUd2G1DsGxoi+E+85MagirCGrfYVFglNLevXe4j8nE1sIE2ks0ri5eXMfyMv0sPe3eMyx1yxiErODpSeOQXoXWyFPrRZoiS2diQIrnfsJosvFkiCm/l/F0goz7V0hglaHp7cUm2ixY43kaOytUAa5xTO4JieNej5/lBMesfDz8Cb/lwqECqc3IODiH9yP5o7L9WeQQDP9WRQ7DcNaM0XUZPBhGTf0kjNR5fQJOxTlJMwTTz4aAziuA8zZv8zZv8zZv8zZv8zZv8zZvvyDtM4+HPp9iM22wxcTrP/qj/xFf+eIvAQDeefU7SPssjNLP8d69HwMgrtndWx8BAG4xb+745BQhSy7ffPKKI6UP+kfIRuxhEXjY3KBsj+xoDDw6rSc5nW93FstYrtBpPxQaBWdKvLgOn6VqtZnCr9EJuKqqjttiKxdCSFfBUQB0TBm74TTDA6629XtdNMfEEahwJa1QPoTkLPqNF7D+2DMAgNPbb6DBWb/CAJpxxxdvPoXnvv47dA2Wl061cKRcKTwLj4eBQmb5ap6E4Oxe96TjZI2/+OxT9E4iBSMpy9FPMxzsMyG4M0CFJWe3t1ewyLw/X3oIOXObGStLbdBgoY3uBBiz702pXMGZxdUrgcvblFne2NrE/Qf0nnvMg+sMJMAVmUKECFgcprWygtxmXY2AsuRgbZwoi60ELi0t4axPlZNHxz185Ru/QddYaOM//sW/AwB8+N5reOIpqlZ2DinbtbBxGZOU+1wpJwgiz3kCCmOQjihT86MfftdlTK5epQrcyvoV5JbcXxQ47BBO//133sHuPvEuM5073sVYCQx6JBRjeaUCVHkFgMmtRzh9k8Z8cvcY3smU70+gdJOqPWu/8SIWbxJ/JWIighIGobVmgnB2DRoGkitoaxe2cHKXvns6niJksvpjT1Km8u1338BWQNy7WPiQjPv24eiTdLPn4OArzG0YDS0ZvIOMCR2+H6DOPL28WEGHq4QH3V0EfO1rj5FAU3uphft3SYDp8OAUigWFJvkEZxPi/TUadbz0Eq0Z3/j6NzAZ0Xv8t//mz6jP3/8IIWeV//pvvuWqBJuXd5yoxXSSIOXs3nQ6dTYQizGN3aBegWIRobWlVTRa1B937t6BZpLAQr2KYY+exeia40LZ6jtM7ji8xGngdUJq5IyrT9MUGXtfpdMU6Zj6zPKpTvtjdNhzajIcY8J9WllYdLYk7+3muPYCZRlff/PbuMI0zN+8RPm4L2+X8NIa/fzRWYZ3Tugat0ZAn3XyVQoUXP1vMW9n0fPxkPtubDQ0VwsTrZxIwlal5Obk9oaPGxep//7VXZq/n3y8iwts8SFvHSNnoQth4DgJ0vOgmFsivAD51GZ05+3n1ay9UxBHePIZQpisthfx7Ie0B9//6AOolObZ7t4hUusrGZadOMkJ83L3d/ecx2rjhTpWN1jwy0iktgoxyVBYewghIJn7WuFNLPYDxFwxCqQHw+RRPwwQ876ZokDA2Xq/UnYceMdxhXT2BolUMMwB9HPt1pWsME70xIO1PwlQKVMVaXn7MlqrtMYmvUM3byfDBBGjlTauXMeNZz9Hz8txSKbMjD8kfWjm+hWFcB5nRaGgWXhJhIHby3dWiffXrFThs1R957SPHvONsjx31gnt5SbKFbqPaiVChVELA55naZ6jxNeoVKsILHqnoiACFoAoJMBzvtaswOPKiOUeT7MCPV4LpS9QYx+61kITfkDz3fO0EwbThXJxhuMs5XpmtZNOscYCVXG54mws0oMJRjz3Q+ZZB57n7A2k8GaIEd931Q2jgZSrMsfHh0gSGoce93kc15Dwujkc9l2c12g0ETBf7ax7jCyhZ/QQIY953+cxKEOgYHRRZ/8e7n5EKJWzDz9w/LF6u40rT1BctXb1BmprbNHA4od+vT6r/gqDWd3FAypsFVFtgJd19M6G7nnjkDmRMkLCldu4NEbE/RuKmQ+mlgLSWA55gQpzIS3XLJmOMWX+JKSHmPt6eXkRBVfCptMpNO9LVvEu8ANI/qzJMld56/WGbg8oxQFaLFy0sbHmYhuL4BtMU4x4LH347luI2Qpj48olLJ2v2LLFii4UJsxdXFyieL/IDcDf1zmLcXBEsda4P0bnjFB3pUrgLOWq5dhZryQ8BrM8c5ZQvied8JwQEinzYYejMUZji26YIGUdgxHrMYwnQ6QJeyTmuRNgCj0JwetOtzvAiL0TAx1iqcXV6QV67kolwg1BcfHZaIRjFgzsds+gUtb/0GNMrCYJj2OtJTTbmPnwHFe1GoRYZCuri60GttvsyVhvOGu4hNfPXqqgmA8bYBbOSZDfN7Vz81obh376We2zD4DWQdH3nHlpnqW4cZ09hYIMf/OX/xoAsHd4jI8Zpra5uYhHn1BQWCkTTOPj+3v44DYFfF/74ufw1a/QAnxxex2ocsk6G8ILqRMfX/MxFFRerUVsLm0SyIgDMS3heSW+vRIKViPTnocyL4iTgYSy5XB+JiEMfDUzNM54snzhl15CckbBoVLKBfZW9KJIc0zY/8Mv1fClX/+nAIA3Qh9TFsXxtEGLDeRf/uZvY+06HRITe2/THNoaxcLA5xecZQUyC28zmvzPAFTrTaxu0MHbUyzMUABBTMH56eE+Pv6IyPE72xdw4/HL1F+10MEYpREImDiqeALF1RZKZd549u7jy194HgCwvbmN6Zg2GYkCzTotRJVKBac3KOD/r9/9OwDAo91PIDULtUjAMPSi2mxBWfK7PgeXg3GwvQkfANM8d9C53ke3ELPgx3PPPQ9ZUH9stBfQLPP7ZDhM//AhKhzg557n/BSl9JxSVOgHOGbBoLR7igWGUw5YvGdheRs5T5pcKxR8jR+8+iYKDm58L3JrfigMWgxXcN50wxRHbxK05OSV9xDwAVX0p0hYLXPj176Ai7/9FbrGVgsFC8VY2LA0589lAtqSh42C4f4qLTawfIEgNh+++YFT3/LqVkKuwN/+yV8BALZLC3jBs+IgcPcvpPjUAbDG4i+1Gs2tcqWCfo/ufzKZOs+p9vIKCusHComTI+rTTx4QLDQu+Xj8JsGo2u0THO7T4p4+PHPec5VQOoPqo/09hDwsWmyWW6mWkDA88mRwir/4c1onXrz5DH7/D/6A+qBScYt+GIZOXOiDN8kM+tJzT6LCwUG1VMHtD+m9oFBY4EV8c30Fi+zDVC6XoVnl9oznvdATCKZXK50R/AhAkRkkDOeaTqfIchsUpzjr8xxg0SsT1eDX6OdLW9tQvI6cnPUxOqVx/6O37wHht+m7+z185w6P6y5d96ttjS+sUH/98tUYw2v0rk5zD2Ne0EejKTo97l/G0pWEwFtj9hMz0imoVn0fdT4oV5dqeOJZ2mQuxgNI3rRYxwAxPIwYsjTOtRPzCMIQ7TYFy57nYzCkviuFIQbDn6TAz9s/drPBoyqUOwAsb+8gZtgQsgLvvfMWAGA67LpgPkmMg8l1z2i+f/LJQ6R8SIhioNVgb7y1FYQlFm8Ipshzq7wnUOIAN2BIUhRIBDwnfelBW2ygELBAI2k0Cslw7GrNWu1B80lQS0lO6CBYuI1nVZrNjNSVRMyHVcPBkAl8CPZMq7XX0d6gffBIK3gp7S/GL9BmmOKVZ76A5jrtZ4rXDJ1nzipV65lZuzYahTXWzmbqhL7vo1qjPlhosu9WUMJgNFPAtMnIZhyh3qA5vLjUcArMfiCcuqalhniBj5iTg+32AkJOLtdrLUimI4yT1HmwGSPRZXjsfTaM7nROHIw7iDx34K9XS4isSqKMnBCIcaAxAMbuP8IlpbunJ6g3ORAONlAt0T1XyxGGLOrTOaHkeTYeOJPuwPNR4X0mCCMnxlGrVhHzHlWthPA5qTjlwFtoCmmp03P0uxQHjgcjlDn5KbRxHr2+57m+zqcM9+v0cbJP/XH7x6/iZJdgwYHIsbxB4+D6F17Glat0AIwaTRgLceb+l6GAUwCCmu2fwgAleheN5ToqDD99+PA+BIuFXNymJESWGoyHNLfCeIjQqqWaAibnZ/RmsD0NhXUep2mV1/rpAD2OMQfDCSKGXq6ubwBMUTo+PMSQvXSt120pDhFy/BvHMVaWKGZq1mvOx7NUirC6QrFRq9VEwokIe9CL48Ql6W998BGmvPc9WyRosghTqRQgCHg8FueEi9jfG3oKn8f86sqyoyT1B0MohkaX49A9VxxGbq2xNJ1pmjlhN4rNOa5Xs+cdDIYYMsx1OBq75JU165UQThHfD0MYXvPSXOGEKQ237u8itcrvrSb29ile1KxQurxcQxRx0ksCN9ap7+SFFgqOZadJDwknho1dzxQQhJzgkGWUOAlUgQCfv1H3Koi5b5SW2GUxvD32TT6eKJwxNPZ8ytVgpvJuIBz8tChyZE4l8qe3OQR03uZt3uZt3uZt3uZt3uZt3ubtF6R9ZgXQQsk8zGTOhSedF0vvdA9FesafSfHMk1cAAOP+wGWSMj7hr68t4Z13SXr+X/8/38a7XC38+ssv4vMMb9xaW0Ug6fNZkaPMoiWBZ0UOtCvxk7gFZxaFjxKTs4O4jmTEFYsocFUUq18gBIk6AOS/cXqfhUeuPo4thsX1ej2XrUqstcUoQZmrToURWNgg+N3L3/w9dE6oOjcaTRHXOSPQWkfGGTtjsS46xyShylZcjmYeckXmpNxVXrjMVrkUk8IKgCiaZQb2DynTc9ad4PlnSWhmc2MZgc8VIV1Acf97nkdiDiDJXABoLm3gkAVjqpGPx7bonuMgw1KZMmxB4M3IxHqMepl+blWZVDwGwogqdnGthV6XYBzTwQgVJtRmqnBVGwAO8mMzNiedU5RLFf5ugQcfUMZ6fLSLS1uUQdvZWMK7771H/c7ZqYPDj5wvjl+po8RiB7VqExWubI6VQe+QSOKPb67ja18mu5Ifvk+Vq8Goi5QrfWmRILf537jhEn2FMc6iZKkS4LnHaHxHLOry7l//EKPvk+hCbaKRc3bG22rj0m+TaNLal59FaGXyjUZgv+cci9e4ypyA4H+XZtZfXhxiYZMqV9n3X0edfWraFwiKUBxneDih5+r5YuZ/A+neIcS58h/gqrFWkKder8IwfCVJpkhYut2TIdZWaV5EgUStznLlnF199/0PMUl3AADPPvk4nrh5mf/uJZQZkrzQWoBSdE8nJz0MGLJRKjPUqV5BwUtRXIrQG1Am7eDBA5wcUEVx7fo1JFz9klIi5oz/5198ge458p0npppM8dwTV/l6JZT4s/VGA7CZ4ix3EOC9g2PuryEanM1VKsOQIcTHR2OMxtY+ZCaGEgUBqjXKjLfalLWtLbRRqdN4jAOJt99+GwAwfLDr3sGjh0fod8jKQ4sQd5iof2uX+vxP9ido89q7UQlxsU4ZwO2ywUpEa2EtAtotene2Qr5YD/E4ixpI4UPArhkh+hOaF2/cP8TtW/Q9e8UIh2f0Lt4/pt/d0bss681iO05G2rgqUp7lTiY891NMkzkE9OfdbGVZSOn2CKmoMgUAWVG4+VKpNtw43d89wfEJ2yR1aO8+7k1dFfc73/sxhkP6uxeffwFbWzSuFxaaqGhb/fKcX5zmrLwnBQJb+fE9t6gJL3ZCQvloAimsrU7ZLYGu+mEMfMZhhoEPHfDvVY60T9XKYecQyYjuu7pAa6IQBbKcYXaVCpY3aV1M0yF8rkbEpTpWGNraWNlEYmH3DJn2fc/Bvyk8sPVAA80+oSrPYDTTDQC3t41G9N39fABGqSOKYtQYNlkpx6hUrUhJAA1rPySRsax+4cR0AtRqtF+Ua2WEsV0jq86HdpopTEYWWpY7uwYrcGFMjikL1ORpBsmxjKeNs45RSkHbqoHwHERP87vyAOe72u12kDGiotcbolRluwZfIuBrDPs0ppLBwPWjJwOUrD1EGM2gac0mfK4i7mxvocRCKxZ6lyc5Smx1UK3WMGGYaZEXMGrm8+uzFVQU+gjYC84imB4+vIsP36N44v6tD2EYDrq6toK15yhmWrn5NPwlGt9KG8iAkWHWO833XPUIxsx+Fp6rVKNZR5X9NrPAd+in8hL9binVmBQ0PsyoQMSV4Go9xDg5RyVhSSQJ4QRHaizus4ga6n3q8+OTU9cfMiyjXLY2TgZTji0nPH8TnQIcS7ZqZdz83HMAgMeffAz1Kt1fno1wwgJR+0cnGLENQZU9P3MhkXC8POqPcecO0VwqrRaevElCSs2FmosdBDRCrqr7/N2ToXLxdGuhhlaT9sdC5dA87kM/QI1jz2mSYjDmMwbDqCdpAi7EI8sT52WaZDmShPp3Ok1dXFMql1GvM0ydq8aNVh0VrgBOJ2MnFLe7e+Qs7F77+B5uH7HFW3sRt+/vAwBOIrru4lGMKld/pSfRLtM60V4sw+cblEWCkJ+rziJItShEyCJZURihVmf0otKQjLCaJCmGvJZ0hgluH9B7efchV9lzg5Tj+gSzgrQG3LhTnzKe0DjnWvVT22cfAHlhMUa4+DGQwnH2jvcfuXLuF176HIKQHurhw4dIGH+7x7y6ZjnC5gpB6D68+xBvvkMLyu3b9/C9778KAPjNX/8qXn6JYJOLC01EvPAKa3QsCgdbkIjg86EvjioosUdLqgSOuHS6sHrBqZRJzAJhyRvZth9hnz3wXvvWt7F5dYeeFwohL97blymgrTRb6DEMc5Ia5LzSp4WHsE0cgMXlCJnz54HzGfMUe4glY4fp90seVGENXyWsW2umZ0bq+WSIsyMagHqZBt3te/fA5zg8fu0a6lW+djF23EFhQlhaU2E0rJdzzpOm3xs42MUzNx9HIK030BSK1d2EKuHhLn+31AgYqtJsUD+vrT6FIffB8VEXGcOJpoORM8MVQjq1JmOM84exsEMhgAlDNmJloFh5bffRCPt7dMBoNVtYW6GD9+07BHcdDwc4PqJ3oYSEtIf/oORgBwsLi1AZHUovP30V9Qb17/o6/fuDQQ/txR16h6GPjA8jaZrButoLz3OGtJuNCBU+LEUtxsw/cQ27BxSMqLMhlq9RUmDtS8+hfJWCEJTCGQzzJw5hn9mkB8F/KIWHSoMWTS9JcefHxGe49NLvAQBOD0boDendLm8sICw33GWst5X4lPbUjENkPaI8T6LMC/Dy8iI6DLvpn/VRZU7CwmIDY+b1jccEE/P8CHfv0bs46/bxwvMEB/0n3/w6Xv4CHc7SJEOvT2Nld7+HW+zpFTHEOKg2kU5pbLz/zgeYsHLmRANDXsFWhXSLe5Zl7ueXnif48l+/8iP0NW0WX3n6GYTebBm03n2nnTPsHdPacHh4iIPDA/75yH4QNYZ1VSolhKw+ezbSbs6tLK9ie4fm+8baujvQe3z47Jz18NHHHwMADnYf4oSVTpNCQzBvJy8EDCsY9wdjMAIEmufhQFTQ4YXuw9MpcGqDToGYD3Vl4aPs00ZfC2gONcoCfBZEyfPg8z0VwkOfqQD9/hT+e+zBpQVGDLXp8VaQIUXIXJzIl9COm6LQ79O7NwYQDC0xefET+mfz9vNoxhkh+44f55sAMRuVe34ZU/79SrON5WWarzJsoGcPD2DuVRQiYs7e3d0TPHjwNwCA77/yDnZ2iILw+LUbuPk4JcCuPnEDjUXiyKfsEWbyHIrnZBRI5z8aCA9KcGJmOnam6n4QuSOW9bQSRkHyfu1pgdDmM4XB2SHtBx/88DswU/rMY09RAFpeWETA677SKaoV5gOuXkCJ17SFlTZqrBQeBCVoy1PnA7PSBi64EBLCHqq1hhTWFyyAx8HwVE0x4SRIibGqutDuAF6plFBh+Kbne453lmfKreuFFEgZBmj35jD0EHOyN/A95p5RatDyvTqnfdx/QP1R5DmWGdp3gd9Vo15xFJzp8AxZYQ+LMyN4YwwM70eeJ1AwtM9yxj3fQxBaQnqBHsNMx9MpmuxJS89Ia/iIZYTTpOfwrIXSGA8YGiiFCzaHgyYWFxhm3N7E2gXikj14SLHi0XEHyzG9w2q7hRIH7d3OsTu0lqLIUQiEL6H5cNZsUYy5uDpAu0sHfunNfNxuPP4UnniODkKLyyvI+UApMw2tKGa1CTAD7ZSgPc8A7LMMT8620pIPr05zbqgKDDt08E75fYpShILhj6ZSwSIXGRB7lqoHUSjklghrhDsUWe4jpHQJ73qtDiH5QOmFiPjwPE1SjG0igvmm6WiA0YiNtU2O8WQHALDQWMLlq0/wtTM8eEBxHoK7qFQpzti8SLHMJCtw6wPyRf74o49x2mXaxOkpUo7dPGNs/gCB9KADm1zmYgEMFBeNymkJlSrzVys1BxeejCZ4sEf38fDhI3T4cDa1aqVGfwou7fFAlV7ojOBX6w3Uq0SPKlcr8Dn5YPnNWhRIxzRnR70OTljN82w4RMbfM86U8y3tTVMcs/F6P2Hu6SiD57GvYJahymWcpWaMKheqQs8g53NAg/1+S5EHRslCmhA19pz0jEJh4+hehgHTvyaQGPDB8JAPvhMhoXgtMpilqH5C3uG/qc0hoPM2b/M2b/M2b/M2b/M2b/M2b78g7e+pAM6yRbZ5cqYut7zUxnhIJ/U09ZwCVq2x4E7uZ2d0ylbpBEtcPWrWS+gwzKs/9fC9Nwni996dT/DyGyQO8we/9eu4+TgRtRusSFgpAZqrPYWJEfDPvhehzyXcH7z2FtpWlWtlC4VT/GQSObTzF1ryQtQlZQgPhhN8xJWV6WCIPmdyLlyirOfO5UsOqjc2AVa3drhHDGqcgStFNWhBmTAJg1GXSrcxE3WFUZDssZTrDIIzPL4AEoZvFGkKTgCif7qPztEuAODuIT1HvRrh+mXKbJUjg6JgJSyhoLmCprSBYKib1saRQi184mTvEV76/EsACHKSc/pGegZGWFhkgO/+HVVpc0/hN77xNQDAteuUuTM6x4hhSMedEVKrBJgrKK4qGDODKgEkAkCN/ptnBULO3gS+QI8VGvv9PhY4Q7jYaiDi1EmPs0+jYc955UBoaIbQpWIAxeIEocjQavG1I43X3vwB9VOdMnBlP8LNyzROptMEGcM3kyRx/ki51uieUnWos/cIfp8hxwzjWP/a82g+SWM0H01QZ4ERGQXwbGZRCJdp/WntfFXw/Dyb0XpJ3KbcoDkQhBFufUgVpi+y4tX+QQenDE1abFdRaS2cu4zt83OCPGbmo6O0zQ7PstfNVgNVJu+n7SUMrddQHqCd01i3WeW4MgVOGd7ZPcO3vv0aAOD+/V3cvUsZvSeeeAqK/b1+9Mp72DukMb1zgbK1/8M//R0gofv5n9/8X5yi1cqlbfgNWlNGyXBGKgec3+OEie+9kw4y9tSaKuOy5VmW4eiQYKR37t3HofXtGQ6dGq3w6DvK1Spq54RprJrdyuYOFrmCsrzcRomhOaenZ7hz7z4A4AFn5Pd2HznoXatRc1DVyWAMyetmGJccbFIJhaH1ajSULQwKzykS6iiAseq9RiK1FTmt0GehjYe2PDkBbLlZQgGMnNAws7cfRIh5TIRhgIAr33Vj4d4etOGqiJRQ9veFcXCSPFfnYCbz9v9Hs+gcAQmf11UjBEqMFIgrDaS8pmUaCLlS02w20WJl68mUYGDVWhVTRsJ06wJ7uzRfbj04xI/v0c8/+NF7ePyJxwAA3/y1r+E3fu1XAQBLjIYIBCCYslH1I7ff5qlxXq8HRz2ogn6/qIRDIXi20gdAMSRTaIM4tP57EgVX698/7uLsAa0fJ3fI03P1wiUIVgGvLrXg8zrRbi+jxBUBr1SGYMim9Dyn7qjPrbkzj9OZqp70Q/d3QRTCM7Rm5FniREvk4kzUJeZKTakUu+pGnufgLRG+780EwALfAbbsPAsDHyF/ny4ytwd4QjqVwf2DA7z7HlX4BsMxbtygd/f8M1TVWV9bAbg6ufcodSrHk6yAz+t2EATu/oQx50R2LORRwZOWViNhMiuCNYbkqp4nGw5+bFjhczQaumqiMTN/Y1Moh4Tp9zsYdE/4GXPUm6wSO6F9Jhn1YBRXAKMGNNMOdBJhyBUck02BLOI+i1Aw7LCxSH9Xrj6JLa5iZckEMcMqq7VFRAzbEwAEV6SNJ2F4/bXxrTaYCcwZ4Sq30AAswkRKVNiXT0E7hcu9AxqjkQwQ8H7R3lxDpWbpIDNL10IKmNxJECHl/suct7J2StbCD1CpWFqTdF6aSwsLMJfp8+Mh3U+nc4T9Xfq7g6NDfPs7JDx20u3gC5+nauv6+hpGXMUfjCdQ/Lw1VuyvSYmTU3ov1f0jJ0IWxL6rcPueQFLYOr5x79yKD3k+MGZxpG7/DEOuqgXCd1XO084JDvYP+DNd5Akj9zyLKogguepXKkdoMc1icbGJSoXWoFKp4iqvnV4fnSOuWvPzTZORUxU96/ec0ul4mrl4KC5F8BnpVWiCWgLkHw4AhTJQHNuxhjgAYLc3cUie2EkwArLP6xlyJ0TpI4EF7saSxhkA9M3s+wy0ZdEBLI4jPAnJe3OgDKRVjpIz8UBfKUZ70T2bv2en/swD4E+DrGljXMk1rjTxozeoPPz2W7fwpa9+GQApDBYMVRIhPepSYwk5KCD85V/+MvYZivXW+3eQsKR/f5zir75FKpO3P7qLr3+RDoNf/hJByZ584hIW15b4uxeQcbn0zp37+I//6T8DAP79X34Ln/8yHVb+eO0qhOWw2MORUW6Q+0WKmpVT9n2sMQwmv3eCo32CpZ4EhCO/s7wIbXG7Wzt48RtsILq54aAjhZk4a4c8VQ6wOvrdAAAgAElEQVSG6Zfpl14cOU6iznJI3gwR+IiqNHwKITGeUJA66Z5gwAfs9cfICmFtdcGZdWqVw+I7lQZ8XgQl/Nmo0gS/AMgYEgB2di6gycbcRaHh+XZBKRwU5eN793DrER24EpFhmzmbL7D8v1AFQVcBXLi0gTPfHpry2QFQhm5iaa2dsayVIw48DwutBj+LxiL/vL6+gjJjp7e2Nlywf2GbD75x6Kwf8mzsuElpWiDPucTfzWAyugbwJE47tOg0Awp6bn/4Mc726fnqjTp6nKg4O+s61TcjfdQ4+bDaLKHP0ESmpiAzBhWGNQcrDi2CAsa9Z8+Y/89ldnsEFEJC8uHm+lc/jw5j3n1rq2EU+jF9+85LV1BlCxBN8lB8rU+3CfMpLfTB9rFtkqOzqBQ7OeI8z9Fo0cK7wTybyWSKgwOGUh4dYMDck/t7x/jf/o//CwBw9bHHcP0qjd/haIw7t0iR7XCXvuPRxx/B8AGwFsWIGPIT+cBwQO9odXUZqVWB7Y2c9LPhDSTwA+TcH6++9Q5S5oJMJlPHZQn82WGqUqlhY4N4pmWG0fiBgc9B59LSgpOxrpbKmPIYu3P3Lj5iPszB0Sl6fXoXigOrVqOGm09SYPq5F19Aj9XFfvjqa+hz0ktgdvCOYo9MhgHkPGZiP0HEu0VWAIrnkCclDMNMdKwRcLImNnY7kRDMofYFXNDg+RINXrOVJyAES7frAhFD9QSvHdNcY2K/GxJZYTlDhZvLUhi3vJifnduYt3/EJqxpudEzSXRfABx0RVGA7hmN2cn7t9C3SZw0x5AVp+3eWC+Vsc37qhGr6F6hPefR7hE+vkcJsF5vjNdfo3Hf704xHdEAeOlF4u9vry+h2WD+uBcg4z3g/qNjfPDxJ3S9B/tYZD7x8s4Vy3pwdCoYRxdEJIwLozwjEVlV7mKC6V26j3t9Crbvv/EjZDx+mxeu4urzZItx6bkXAU5K51pBpJYOkkEyP0l41jxbukOhlJ4LPD1tnCJ4FEcwVi7+rIMOm2Jf2yQemTRwEv3S8xzvqVDKJfe00W7fhBbuge3eWK+XEcV8H8rAAmR1OkGX97A7dx5id5/ip15/CMnUm81N2h832k20eH9PkyFSe/oMfDdfBYQ7gChoB2GfwQ+1m+9hGKBaoX+v1iRiTiZUKyVEnIRXisZPGHhIObGWZFMUnOhKdOJsCtJRgT4nJ3qVGnontM7mnMQfDc6we581GMYDp5B62ulgyDB0QCJtEBR1YaWNEifurHqr9EIsLNh1XcwSaposdACi8nncCUZoB4+1Bz2j9cysXUoXa5EohrQdiQVWB3/mhecQRKxSyv2ZFBmW1ihpevHiFRfAF+mMYymEgOegfcodpmzyXGkNoW1CWUKywrg0BpphgtVSGXKZ+sNbof68fOUChjeIC//+u+/j8NF9+vm99zDoUYy5vLKJkA8YQRS45OaIY2tdKBye0BownEydiuzm+gbqNRu7KaSs6yBgoNh6pdenWOCkc4wz5hsLwCmMC8gZXLjIHOVird12e3KZ+XbVchnV8iy5YudTkqToM/y40z3DMVNXjo5O0OX91vIFq5UK6rxGba5uARv0facnx+ixInhWaBevEa+UCyn8uwKeg4Qbk7h/oZFL/TiCcRZ6tphgMHMiiGFTs2SfIzjWSpVGwnGYEJyoAuBFzi8ONpEfIHbCJkmWQgsbIwhobRVB//407RwCOm/zNm/zNm/zNm/zNm/zNm/z9gvS/kEVwPOVQC095Bb615vg3VtUSn7lndu4f0yn77W1NZc9+oSN4J++cQM3GNK53G7i5tM3ARDhd3efRRh8351e7z06wt6f/wcAwPdfI3Pbp29ex/XrZKpdayzg6IgycK+/9ibe+jGpMZ70Rtg9oUzZzec+P/NsszUQLVwixxcG0gqxQKDGJ3ipPBQjVvRhyGYyPsCrGd3ns+1VTBV7A0kzUwzNc8SsfJgkKWImv1tFxbjiwWP/HikkOJEAEQQoM1QsNwajEzb97p+5bMoFVuosCjV7FuE5WEWew2V/pdDOw9HzfOeRU+Us2Wqz4bzMAgiXTfE9YMheJq+/+WOMuGKYewo/fIWgfRcYYrHSKEFldG+LC2VUAso+Hj/cwwJDF3Rg3FiBMU491pK6y+UyDGcF4yjE4hqRwbe3t1wFyvc9NBjWdHmHKjbTZOpgdnk2wVmXsqFn3SFOWOEuTQqcnLISWuLja1/9LQDA371OFd3e0Sl6XLlaW11Fh4WDuqenTjlzYWUVpSZlokq+QZsrlLb7AzGr3yvM4CKeESj4GlrMoB7/zc39nYDm61146VmsXiJhIqMo7/TUF26i3CT46S//7tfB1lx8iZ9eovG5amDHhjHGzfPzP6tzMlK+7ztYeMjE9nq9jhZXBS9c3HKQ09OTUzxkr8Djg0McHX4XALC6uoINzpgvL7CaHAxe/S69l3IIXH+K5uHXvnEBSyskbDBMMpyxwXqaKOQ8xvY5s9gdDJELmvcPOj2nyLrQamCFFdlWlpbw+BWCdMdxjBIbQyesnhZFBj4rwI2mKe7fJxjP4f4RHrFAQb/fd8qpgQxxcY2eYYeFYa5fvYb19TXum5rL4l27vINbXPm888kn6HAmO8kKl8W12cQphIOJKa1n0GBVOIUCo4QTPvA8rkx7EpGFd/o+JK8Hvu8h4CpjaJQzGC4K40ydrVG1kgJ5weAWId16bLPVACXCnVIuZlSBefv5NcUelRDSKTeGoefWo8l0ir09nn/HHdQ/YD82L0Rqx5uxBuFl1FmsYGmhiVaNlSfjMsZTfv/FMU7Y0Pq9uw9x+qf/DgDw3VdpX3jq2gVc2eH9vdVAn0Xg3v7gLj68cx8AMOyN3L6/89hjM+87fqYCAoJRMR6AyKqQF0DM49cUPgKmLJhjrmp6ZxgzEuD46BQN9qy79MST8EozVVzbN3meIvKt2qWtrGi3FnoSTnwF0sy8/zw5g8v1B0jtvh7zd8A4T0YN6QS4pJAwfD0pZ5VGYPY9FRarKZUip54sz8UWZ90u7j6itW5/7xAJ76uF8NHt0j53xKbUrXrofNlaSwuwbAlPBq76Ql6HM+XPiNfCgqGenvSc8Fu9XnceuNVa3Xn7hUHkKjhpZj3XDK1TfK0JIzHOemcYs5faZDR1e1ut2UAUWooMPYdOEuw+oPXx0YOHrr/yvHD3BwDjEb0LJQ2WGa5vq4wCEsqJ+viuYq5UAcmrlzynkq2MdkKHDl4tMFOGFcZVAyHFbG82gNegfnrpi7+E9Qs7/Hn6bDLuAZKQce3lNeSMd/aRI+L1WUM4uJ9E4BRhLYJD61lVUkjpKjsGxop8QhiJcvTpPb1aq2OH99q15UXsPqD4af/wABnHdvsHe05YZ2VtFT6PQzCiZTqZImN6UrlUw9pF2ueeeepxp2SZ5zlBcgGkeebQMru7tH/u7R5hwu/K86WLHSqlCCWGTK+trmLB+QqGCAKGHztBHqvOS0gvG/vfvnsPB4f0c5EoZ3AfRDGWOS5ZZGGg7Z0tLDP8vdWsuyrtyckJ7t4lh4LdB4+QMELCj0qY1cgs5N53VJlYlZzIJKCg+J1LKRwazFbphJDweax5ngdwnKE8D/KcV2CkLEpBo1LlqjzHwoPp1EFc0yRFapV7bQdhVmX8h7a/hwPI/xXi3HgX8HnBu797hJMeDY6nX3wZe7u04fzgRz9GyES2CxdoU3h0MkbxMXXyr258Hk3mNH3+xWfxgx8S3HJ//wTgRScTEhl32Hu3Kfh6/9ZDlEuEY/Y9gemUDyh5AZ/x6r6MMOLBdnx0iMxOHHtmMmKGjjTiHDtGOpyvEoWDRDkEqTZY4ANd5/5DHD+ge1pstbBUbfDfGccr8n0JzS+z0WS7hCBEELM1hBRIWCU0h3YwEy/0MRnSQnl6fOSM6HPLjZDSDcCiyJ0kvZTyUwG8e1/GuCC+ytLKnpSuQ7RWbmIJ4+PuHTp8Hux3IFHmzwP9HvWp/ffWk48h5+czvoDmjbvRWoDPk3ei1Oz4IQQ2ODB2C0ClgkaT+Z2VEuoMt1xeXnaKoXleuEktbHAbeIgYWqx1iIihA+sbW648Px5O8MldWoD6/QF2WdHUJiSSUR8qp4Xqdu/YLZpR6Lv3loxOcXLEsOBRBJEw5ME908zQ/fwxSxggOI+N+wfC5H62SuiMDxjEZZQ2aP5pDgJ//ff/CcCQ60ozcJBNmq2Wx0v/3zYHzeWXn2fZT/3+LMtQOBU57T5jx6AQwo0vT/rQvGGW4jKuXSPY58Xti/j4Q5KPPjo6RZ/XjHqNOBqNVhkXr9LGIr37+NI3aaxfvvExUrYvyHHdHTTyIkHGC+Gb71LiZzwdu/F95eI2Gixj3WpU0WTeRTWOEdrDWzCDJ4/Z4qFzeoRDloDujSZIeUwLA5QYJmPKJaxvUKJic20Nl9mKY6VNB3DL+bN9ZMd6eXMdq8wjfOLJG7hzn9bK1954C77P17YwMa0d7PPTvFC4XZAOcQwR4veTgWmAADwvR+CMdUNk/FmhFUocFHthBQ1WRrRGvYenJ1hoEZSoPxohYRPgMIgQcz9K4aPLti+e9BAEn7mNzNs/QrOwIWF8CCvdKATMhBUCjXHWD0dnfRTCJmyaqNi1juPZhYUqFpZpvlRKFRScAGi0CscX3DvtoaJsskvgkBMvhwy5+ujDT7C2TtDMZqOKLu8Xu7v76A3pelEQwSvRPb1/fx8WCSmZe2Vg4Ft1Yi9EheGFWgAVPiYK38fUGh1bGX3fwLfq2kZh3KeDkJn2UGvRvJxCOUsqCA2T8yGLh64nJUKeh5/KZxhASVqPikLh+JQ5xOMpCmkVgO13zxQkC63d//GkdAc93/fPWfKImT0Vr9mhFzplZl1kLpFYpAWOenTgPRuljlNIzDSWhrcJ27RwiWuIEGFok8GeU4wtCgW7Oxuhcf1x4g861yAoZ8zdbNRR43hNeKGDqwIGBSfQrXVCIKWjJvih7xJxy4M+CgsBTTOkNoGb58itITcn4rJ0igkftPMsm1kM+CF8m70XEgkfOkajEepMD3CbsxBO8wAeYFwMISEs3xHa7X9SSLfHCrd3G4qV+HquKcyyFudaVCvhwhXa0ywPTmMNyxusERGWUalwPwYC1qPb0zMovfApMQcAheV2m9m+6wnMuGEQs/vzAMlz3PChZDIaY8I8VQiB9grNhWqjjIztW+49Osb9hxQbTXcPMOGCQ8w82kJpB3e9/vgTePppojc8/dRjTq02y8YOOpwmE+RZwn1AX+37EnVW/C6XY9Q5Hm42qqhxPNxut9Di3wvPIzV2AFNrrj6dYsom772zAU47PMeVwiLHjcFihCVOVLRX19BeoX2s1aT/Li3U3eHe8z1kmq0dVpaxvkmJ5sP9fZx1SItimkzw+ju3uKetoimci4CGOBeZhY5qRXstj1m7Rgjh4KBKKmSGD7jGwDAeNIwkSuw+sNZu4jIn+gNel04HPZwwdzYZZ0j4rNEbDpAxxU3nOQQfKH0h4fHf/qw2T93O27zN27zN27zN27zN27zN27z9grTPhoDa/4rZOdc3QMgnzKtXr+Nf/OG/AAD85m/+Fj76iARh/uo//RWW23Tq/uavfxMAwSD/73/1JwCA1964hV/9FSpHX338Jg5PKZvcH6bOA8yDcPAkB6vQGqOpLXIaSFtyDQKXzSoUXEbg4tYW/tpCqbh6FAgilQOUybSlWi0kQsvxNcpliTyubsRCYC2gLMa92/fw8StvAAAubW4hZOjLNFMOmhiGPiRn0PLUGpnmM6KxP4PrQMBBPbIkdZCNogByW4X4KTArY2YQrtALnaKS9D2nKkaZI1sZ5Osq9alqj8dZijzX6HYGfK+AVbTRCjBMXk6nVjSkmFURCwOPK7fVhcoss2j0p0RgvvnNXwNAZHqAFDct6TjPE/fZJJkg4z4TUn7KZJSulbtn0Sig+T7SNHWwJi0Mrlxho2A/xD7DBBYXmVRc3XI+jFmWzfqgyJEmY+7H0Fn/lCsxyuwrM6uvzdqn3s4/QlrFzsBQCBjGlHoRjcdGu+Lu5icLRj9NpUMI4TKstvKZpqnrf7oOZ/uVcpU3KWe+jlY19byiaKFm1do4Lrt3OBiMnB9et9OBKki84e49Uv9t1krYXiey+tbWGtjzGd3TFSQTmvtn/S7GU0vOn0FUV9kjcntrExUmjtdrVdQqDN2NAviWIO1JxFz9GgwGGLAwhr1Wo7WAxgLBRYUfoM83cry3h3bNZirbWF+nCuDS0hJCJsXbTPh5+KyU0lVH83w2ZpeXlhAximI4HONHr73tPm/fiedUfPUMhnuusv/Tm3CQ30IBBa+V02SKwGYkjUHLo+/+4z/+l3j+c6QGbJPlewf7LkM7Hg+dv5fRAhU2HX719Xfw9h2aT9XWIkrnIMfz9vNpgivwHoQTLEEOTBjeubFzCb/zu78LADgbj1BmkbGlVms2nxVn6kWOnJX5psMRxlw1KNXqyAVXi6MID3epOj7Jc4y4am7n0FF/gv50j/4uCDCyqspZDjD8TgiJswl9z3FvhMiqCFoUhdbQXFrxA3FuPhiEARup+xoqszBXupbWEsJCqhTQZSXGs90D1LmabWQAyV58yghozxqwW6hZ5MR0pGf/BzAaTqxtMOqj06XvzLREieeD9RIU2jglUaPNrHovBDxbefC8mYCSMbPKjxNjkshZmTIvDOwGpAUc7D1JMyjrLay1Qyl5nv27AoLX4TQrIJ3QjHRUAiNmIhHGADefIf/lkGMtlU+dwichb6S7nl3jtS7cumfhisV5VVVfzGC3qnDFORn4CIyFuQpIXp+r7MG60m6iNrW+jgq5tnubdv6NeaGd4ErgCYR2DgjbL7NKrjmnRG6khLLPZcSM5nQeQXVeO4Mrfcbgpwlqf7oJILBBpBUPER6W2iR85Itz/oHnBNrOIVEhxEwRVp1De1gYspEzhVoNA2NmQjJWRMUqYBcZkLGgnVYKmq9Rq9URlyyyJsb+EVW1j4/3cMqq5xWLShG+M5P3SxVMWYCu2+uiUrBwYVEg5+8p8swJyi23qRrXqFcdxalajRHGVvndg8cPXi7FkHYgS+Pena2Me76Ex89Xa1RRZS/pq48FKPOzNGt1tFoU+8s4hsBs3Zn1v4V2C3jcj6EfYWGB7ykM3dllOh0jwqc3twKFE+cx0jj0ixQzVB3JKnHs7NB1wtmMqlw59FkgArd+ByZAUPp/2XuPHsuSLE3sM3HvfdKfi3AP99AqtSjd1VXVJdgcYpoYED0bko1ZzIIYAS74B7jglgQIggS45obcDGeIwZAECHDAIUFOs2eqslRWVmVWiogMHa7dnz95hZlxYceO2fOIyOoRTBCsdxbhL564wq6JY+d85/v8767feRc//OPv+3ake5rN5hgTQWTTVEzYOD49w3HIjp4eYUwokGo+Z9bWl9kXbgCDP5jCkAQ8QxQA/MG3voWvf82LarbbBW5c83Co733nD5nJL7D5AMCrr/j0+C9+8Qv8+Mc/BgBMZ2f4W3/77wAAfvnND/D3/t4/AADs7u1ze9bMjCN4kk6vySGyaTlh8eabPk39+muvoaE0NVM8W4M6wOLg4MIIF5LZQSUUpA0sjrRYQkIG5c6mwTHBTDqDHrZ3PLRxd/8AIjjnSsDRdc/pgZydDtGi9mh32pDJPeTU48dlDScJE593AE0ToYm48AgVc7xIpjT+QNxQWiFinVdg8UOc7JqmgaFaIgmJy1e8c9u/f4CTadxEhpqx9Y11ai6LoP5pascLQFlbZtGElLxRMMage9FP2OH6hycnKOctut4KYRMzL8ukrkgsQBABvyj0yaEpOgN0OkHCoYJzcbFDQzUFUFjb9H3v9Teu01F9zSDgN7Ohn+7t7mI89r+TOkNJnnG3yLGzuYb/b9lLNn2/61fOcZuGdtZaL0CIQ1unEE8gwndTiZjw3bppeIEwdcNBgQsbm7h+zbf7wf4BhiTAaq1/bifHx3j2zD+LTz7fxL0nfuP+1psV3nnH9w8nKmbULPIuzwOvEeRGKYmC6gmEEBhTvcnp8SEc9ce1tQE2qd5kMFjFGsllTGl8Hp2OcEhsYNPJmOto3nj1Ni7RRrPX6/G9G5vCXGItZbC07fz7YX5xWKFgwve/9x38N//tf++Pl7TzC+HASSDuOWiofxcv9E5cFNsGgCFBRw6HZ9ij+x0S7Hw2m+HJUw8JGqz0GTLTNBZnQbpnNEJGTHRKSQjmNFval2VBQkFKGQs/MqAgCNGlG7cxoLkaTQNLNVKo5gzLC+NvNBliMvRz8xCaHaZOrwtLpRVFJ8dg0zt0uwenONyj4Ak5L6PhnGF246qGpI1Lu6VR0/xQ9FtM2T6dnaEiRr5WkCuxGWoKYjbCoSbnVosCmQzlBhpBUpzUWzCzY8yItVCvtTmwd3T0DFtnfj0T3S5M2JxBoa5pLDb+/spGAPRe5iSvwcY4zGf+eJNJiYYgclkrR5eo6KvgSAoD64JzbjlY04johDY2vm9SMXYZgpsNz+fGgCH1fuwHGnwFS1S9QkoUnVAIFs5hEPCFtbEQgcI+mYOUlnxuKxTWiR8gXF1dAUqEzZ2JmwrnYGrDp9OZ4u8AvvaO5SWk5BIVhyx+x1rkOsyFGs75B1lQ/dnVG1cQ1jZjDPMVVI3FlKSPxpMZGmJkbbe76BNLqQxa7cjhCCdrkWwoJCIvhLPcHp4dOgbugiVl+NFbfnmlxnOfSYBrsZ/7eji4BW806xowdL9hw4CkPjFd7K1zMVEhJROTRhZLzf5oWVWoq7BJN7CFf73S7WBnxwc955MhphM/rs+IkVUrDVCQ/tHjBzDOv//g4UVsbXl/qN9bQU6+sbGW/cydbb+RElIgk6E+znDSQiTXWpsKuuFoCH8n7LqzPGMopFxVzHhftAq0Ca7aKjpcy16amqHxYR9TQ3JZhJU2SpTYmBDpFjnX6mW5QqsbkjhhHc+5dtb7PjSGVHzw1ko0CQs/ADTCxgvhuwcq1Dx/i1wjL/z8vbZ+AR0qLZM0xtqyh3aXkmGuwpyC+N0iQ7fv+8zKSguntE6PhkPoSSgKebEtIaBLW9rSlra0pS1taUtb2tKW9nti4sWRZG//2X/5jzx1hDXgSIlQCPtGkUS2gBiVXmCFCxFyIVjIEaKGVAHWJyMJiRBJgCOJD/AXAOuCLsdi8pIT71IsRMf/o7/7b7782l5izrmXRNiT86WEKwtwyy8+rn0BuYNJdIIALMAmDUUS/uP/6R8CADKtmYVQ65wLrpXWDEWRWnE2UErJ9ywSiFl6DEYiSMlRRIgIOxQODDUIUTzfMQJUQXDGEdYxYY3M9QKxxeu7RMTyoYcKj472OZW/tXMZmwSta/e7HEWEsaz5FxjFqtkMjnE0NUOZ4CwL2auizRkoKSJzVkMwoHI2RUlF5JPpFCenB3SIijM/WkoEqlYLxRnPce4j60/uvY9Xd6hw2Tq8/8nnAIDjSQlDBe+rKyt4Ruy44/EMkiJXIbC7utZFi6JHs0mJG6++7c9Xz3FMBb9lbVkUVmmHmjTwxpMgSmy4kN46y5ll11i4Juhnxb4phMBh4b/fITKd48NTWIqoinNhTI5ZSwVJMIzuir+PW3d20CZI+KOH+5hNqX3LGhtUhD0vx5gRrPbWncvI8gBfI/iyzjEiSNnxcQ0CD+C11zYxOvHP9hfv3cd04q+jKHK+3//wP/hTf75ZxdH5spqyGHFZlswcVzcNZgTrNFXFDJ3hLyA4eumQjvFIlmNEzHbD2AXYbLCQaYCL4zydJ5ywmJIm2W+nRzj9lKBpNmRlXULGkMCUkgzgy6ca+iCZazw0NMJn+bVzcDZoMqnnDoGEQ9baSJjgLGADm7F0+OyuJ/j52tff/kvSHS3tX9X+k//6HzoAECigAnu1lElVgWWomF+jw6NpOBAd1m5nDRwiykLq5DFyd0ozD2BIY4AXSiEW1g7HU4aOfTYBMGe6wd/9s3/D/8cEeJtkhm44x33cJpkrn+QW6aUhfoBz78cLcQJJ9kQ89zs/PsOpLQ8w/37IFMTshnVRZeu/+B//ZwBAU2TMGil1LFHJixYjFvIsY4ZIpYu4ZlPTWRuzFFIq5OFzlcW0L2JmsKoaJowCo5wcX2e6jktEqLdwgv2BVmOx9sizMD/7/AEA4OjwGaNiLqxv4SLB4rIsR1nHTFJYE6WL6IUIWY9lGxCS50VrI7mev3/KhgSIrkkyjsZgSjC2+WzGc6RA7G7OOk6C75JusnENRCD5Gk2w+2yfjjH16zp8d5hMouh6mN9CSYlWGVYIktcrCtiQVa0tM2caIbg9ZrMSc0LAxCXAMWIOzsb2gGVIr2ni3Kq0wvFZYNT039Va8xrlnOVn7xDJu7SWyIjVaTDwpSEbm2toEwJuOi0xnfo1eD5vmOgmV4oZcTOtuG14fNN9AcDwbIyc1u6LmxtMlvfoyS7m9B0pBDpdn8n90z/9E3+dTYUq6EHOZ/xd0zTsK56dTXA2praDRUYdJGT0rAVDhQutIENms7EMo85zyTBSY4B50DgONVUiZlUFLI/VxoHZNWezBlPyYWpj8MldDycPmVYIxXPUQh904R0qAwlf4vU/PDEAyT7HmqTEI5mjhJSRYJAZLGUCCzaQgUVLRCitE4r76d7+Pn72S6+g8Df++r/zwrV5mQFc2tKWtrSlLW1pS1va0pa2tN8T+2L+bq6rixE4IeRC1u9FWa+UCCHshF0SMYMTME3MKAYaYx9F9F+RLq13ofcgODJ3PkvBNLlKLkTdlVL4l7G0BupF9i+aUUz/nn99/v8piYSIFeP+uoyBCQXjaGKhvIuR/fPHZarjUEMHx8EIaZMMg1KMuRYi1t45YzkSE0wiaSMI6FCY7AxKqjHqFiscyTPWYjomUo17HwMApge7OOv6aNXJ0yc4fOJrKW++/joyij6eHuxxVC3Um7TbLRRE5tHOu5Bzkm2YjbnoWeQ5NNVPwgFVHYqhg1HMTP0AACAASURBVGyGgiYdmCxroKjWZTKbgjthBlgXKI0dJA2X0xNPaLK/f4IOFexfubTFtM7aSpRzKsLPGxjKYPZaBWoKQY2pjaZjgfUdH13VmcLw1GsTXt25yhTIg0EHd+/d923ab6OkDGDQQcqyFo9VaxqoMG4R6ajTqLwDMDojghbKlEmnOYqUCF2En/p3reVMYtBYGo+nUFTLc/XWDnpU4/Ppp59jfOYjjtdvXsMwULNbCUcZw5zaqycELNWxPNo9hrOBlKhGTc+t08lRl/5ZGFNyZDwQSBiR1NMIBUsZDSkE17HBOabWzpSEEYuZ+0xJruHx7RXHXJhFNETCqyMReOsbG6J4Do5o7eFiNNxZy+PTQkCHmLWN0cDwqCxErO0QMVJPhwyX90KLAAoBF/rBQnYjuS9r41yeHNAlHSW0jU3yN6nukJQSjx97SZyvff3tl1zV0v51m4Cvy1FOM5kBnIB0YQzIZLwrfr7Can52LtRCWQcXXAFnI9nFubWPl30L1lLjdQnJmuz4iiBd4isk/zojuY6H10cIPqWzNhJfQMRspXOsqxokFyQAx5IL8X0Bwdl6BVr34H2YQAwSouxWOI7sGxPrs/x9UW1dQhoCa3m+nIR7ahrOUkgrmJRFuRhpNzIicaAEZ1sF+Ski4TOQWkBQVkdIxdcBIViTTioBWy9m4ZQQ/GxhYwYTkLA2EAA1PF9WmUBO68DZkc+gHTx6ihaRdYwOhyiJDKrb6UFo3/eyVoEicCyEbGcmYKgQT2eKkTpKRckLYxvWR1OZZvK60DHrRPu0NrV/IACEMdChTwiZ+ICRdn9eBqkPh36b6pQ7OU6pnRsozj4aE+c3nVyfpbYz1sAakrfINNrat8e0Lvl+J7VBTXJewgGa7t1y1XXU8HUu9mNrXZSbcIKRXHAukfiIzzvN+HJGXSoY+nJTGsxnYb0Ka5yG6XXDYRntYY0JiVvAeukOwI+hilLE7YDSyTJI0fBxDX3e67TQkKyKksIPPJAUFEvj1Xw+GTTwpFyQ1gjZVmvdQj8IuAFyr1A3zusuw/eZNpHt2KaMpCxWcDbcNI7TfezWuOgvSEiYpGbS0uOqYVgrs6oaaEKRMVukSw7owP65f5TklzuHAHYM9yeF4LEgsDgXyfDaAYEqSUFx/whzlBMi+jtWQNkwjwDSBT/E+0oA0Gr3GSX2MvvCDaCTERrFCwgkd8zze6N0s8SbCn4DPOEIkSG6KjqBjjwPPUlfptub8w5Q3CzGbymlntt8fdGGLoVevuj76evg2ImEZGXheoR47twvg4Cm5zv//fA6DBqtFBNOqESYWybMgSnsU0oZmUeTlHL4rpBRmFPwP/58oZPqLIcM2j9h0AgZHVoX77UuK4aOtDsdbifrHGoSFLVTv7FR1kDRRGqNwfDIwzTu/6bCxkXPnOUyjQGRdXRXfdF9UbQZpmmbEiawVHVWAdJkgm55oVr4wT4vqTD2bEht7pgpLc9aaLf9RDmZjljUVqkOMtpEOgtmQh2NPJQVQqLd9YXQ40mNHbrmo998xCQ1pmrQonZf31jHAYn2lrQKdXLJrGqDwYAhG2uDPj75rd8o7/QGvOEZjyaowgqROn7sIFl2/KWSPBk7REiVc5GJtaYvKCjWDzofR4gammDWE0FeVr+3is8//wwAcPn6JpwgFt/cstisbYA5MVLmOVCSrpalgv5Xbl3Hw/uf+nap59hY8c+7rmsULVrEtUPeIvKVxuLChm93hqrIyIaolIJlUWcJFyC20xpHpx5GfHFjgIPTADkx9F6BXidCdHlIKhVhVv7uAQCNFjgjMoay8efudxVDem3tFgJgvHALYF4HgeHY3sFpE0jnGrlQOx62YU6m34kOhglOfXwbDiLqUjkXHX8k2lfxIpKTJZvThCTBYZGIav47mMaW9q/fdCAbs7F8QEIi+ATOJUzQSCCNCRw+oirTzWKE8jmR9DHnIiRJJcEkhpPGviKEi45Pcm6JyOhsrUngqrHvRY22qNcL2OT6RFynA4ulEAjiaM6C1yWRbISsczwGhHBYJALBOdinSBgyLTtu1sa12Vgb78UEEpuGsY0SmgPbzta8K5VGIjwkqdLNtovXr4P3qPhzax2UDnNv3OTaJrKAJg8/ztNKsBftnIMJnq6UvNGREhzsbWgDlUEhD4ya0xlGp6T7CcHw/0J30KF1M7Ah5rlmX0VlsbzEB2+Do+5YE7BxyesmkLlJOCK4cHMDSxss21S8qfZrET1bYxj2NiLSCwWHrg76rzlWukQ2V1uGhjYwcGEj4TRK2jAw8Z9QHGxtmhpbV8gngcSY2G/lbIZy7Oc/V1Uc1LDJeqzDpk9Y7lfWWB5ESkUnHxKR1ZdxgpHt3RoLrZKECfcPS9qwwJR0N/fECCvECtxptxnmL2QGS5uteWUhAnN6lqynxFKd5xkmdDmNBSStW9YYtDJi9my3WZcXTlDCCInvFyGuQsbACIxEzUGeuFZaC9Qm+L0gU5FQsNAYkMZpXlQ4IX3Muq65VMY5yYEiDkYpFxl4YeOmWwgY2udluYYkjUQDE2IPvEnzAagwMUmGVC+wugoJk/R1f4oY2HGIz8rvIgNzbQpDjgSP/E1jeQ/lEOGgonGwNFEYodCEQHim4/zyEltCQJe2tKUtbWlLW9rSlra0pS3t98S+GAIatHAW3hQJGEi8NKMWCxdf9GFM34uEO9cjTUOWyr3wt/zWC/TN0isEsABFfe66sJhhAyKkMSVleREZjHPuhdT4aSZQKfXcuc9fz18mKxnuOKT6C50lxf0vb//fRWITs3cuFtVLxUW0Ssasr0SEoUmCBggRs07WxWCV1zEi+u6kPaRzTM8dsBbWADL3UTpddLhIfHh4CEWEHpffeBO9NR/tWd308gAqy5lS1xoD11unw9o0FcyF8MIB3cyTnWhJka+RxWhMWlDOIafrb9vGZxIBtLI+JpQxmg6HcRwoIi/JFRTBGZ/tPcGbr98BABzsPYYjzcjx2Ri3rl6jtsuxv++hkOur/no2NzrYOzr255hN8f1ve122XLdwnX43HE+xseGpmj+9e5epOeIjFhyJN8Yw/EAtwJNtfIYQIFkeaIoKVvPqPPIzGmcYYlS+pijZ/XuPcUzEKtdubWNGz805wFGU6+G9R5hQ1nc2nmPn9hV/TiJ+2V6/hckvPwIA3Lx5E+tEdby+3kNN9O4Xt9ex7vl2MByOMTrzxDopXFrKJAMeMoBKs3aUlgaXL/lnK6RGfeSv9SLpFY2ric9YAFhfLSKVs4uwGiGBwFD99NkQIH0ypf01V+UUK33KIiqJLARxmwSGCccwEyFcIrdD3zURzO0ztykKIX5yHn7uEB+hc4hkNXRMwI8RkWQ6wslZMyud7V2SGUretiZGQV/AgbO0L8ECeqGBhSSiKpf0JTjAMoTM8SIuXEJ6FJZoETNhUihe34VMsoVIkABCcPZoYb0O5lxcgyEihAlugdiAIaUurtd8thSOYhMYXapVmhA2BfSCEJHwCELyfcuF90Xi1MQO7hj3mfgeSfZiASVkbMxAkpyGsRY6kJBpGaPrxjIlvbDgUgbYJFPHaC4Ryz6E4LZRMmY0RALpzVXO0f/QtkpEeKH1PYSPxy2sABMggQAsrc2C/razDH1CiQihQEpR0FKiQ3I7nU4LORHWFCQV0Gp3eN2RSjIaRek4PztnUZYh69fAhVITykSpouRMsKkrmCAtJARnrhrb8PxmjIWjNmhYjsJx+YBUbXQDsU6n4GtKF7wKAmdDX7IwqR1/PCN0Q106rA0GdN9tPKNDFO0WKAGMw8ORz7gCTMJnnWWfo7GxJEMLxX3aQvD8C6Xiuk1t0DSGM0m+pCc8Z5FkDnUkUOLssGF4pITkcyipMaO2MY0BGpo/GodOm7Q5CfrYKjroFN5PWulVaLX8Otft9NCmNt0YrGBCWVBjXSQ2CtlaISEC2ZhUEJQtLo3jey0yhRYdr3GG2Yoy5a/HaYEJEclUjYOmDOVau+C+OxpPGanjnOV7YKIWkZIjKYig+ykVwymdU6hI8mEEwf3K2Ni4lrN6jn0xD+QJiIu4ioZ50DqHlAQrzJt+bqFnKwTP5R7mTdfNcylPF7DOMJxYGAuXhTEnmGBQQTG67GX2hRtAnsIW4Jgy+W+ElaUm0sk77tj4PykrmU+tphuhuFIFweuFYzNM6lzycnE9+peyVPya09cv8HCapnlhjWCWZYv1ewmTWPj7u+Cgz3/fv2dokhQy1jRJKdiRlDJhCEyOZa3lyVTIeG1BL89qwIX3lUwgdQCVYsFaCxUEL4PoLRK2VYm4oEoBFSZbrbl2yjrHx+j0vbBlJjNYYoRq5hU0wSZlp48hbRj6jx9jlcS5a2IB1b0BWlQ7mIYnjI1QFsDBEUbdliOAhDdF5aEs2k15Mi4hkBN0Y623iVBGcLL3BDVtaJq6hKDjlT3aNNUzGPj32u0cz576Wqhb16/gt/ceAPA6O5u0gb335DFKmng3Nv299rsC08ovnidHM5TUHuP5BNs7HnKyXlncf+iPXZYlbzYCi1imwBM9HNiRcYmz72tM4jjr9fwEunLRL2r3Pn0Ay1AhLNjCEKbPStLGmpcz9Fb98zw9OkO35+9ldDrDbEr1e+t9lDOCGNUT5E/85u3rN18HAByfzLG9QQywgw66nQAFMQwXuXx5E6cH/vkfHx0gDxNeuumTcX5hZ0gqkI4ztFRokXD7tK75vl69cdVf/2SMDz/0jJbrax10SAPI1HE+EBK4T6yuk0mNb33rNgDg6Z4X0K3GI7RIoLg2BkHgySXPSEiJdk07cCniBjBhjltYFNh/i47ry9iHU0RK+hi53CQJuDlneYFIZIwWagDDPOFcssC5uPDZRNdsaV+eOUdQa5mzhpySmjdiHpYfn10MFsSnxeyQUgEJLDR808OWwkYixSELrtFJ6/q5GxrBkMaUuVYyd6Jn9AssgoIhd7EvOyRsu7BIIZu8bibOTWOi4xfnOcXXL1OG0QVoc+zTXN7nEgfORfipdS5hsoxwvvBlrcAQQAkgFFoJJ6DSCkjarDobvZiw92xMA+vCJlIwhEvAMFRMKRmfc+JrZSHoJcABGmOV1yCDZ5hkXU/bcB2bkho1MWJXsyldv4EluGhjKkzKoO0b4b2lMej4qR851eRnRcEMjVIo9jmUVtC0/sNKnj+0yiBygiDW4QFUsPS7mTPQBO+ULmoCzsuaGdKdsxA61KAFv1JwALKu5uhQKcFqt7UA2wslKGfzOeZULxWOq1sFpmMqHRmPMJv7NugP+iiopnB9fQe9vg9Af35/FyehVpKgsUK65DpjYEQrB8MQ4Ti2pADX9DZJ/XhkngbDC5WyXIPpnEYTROHJJ8yLHK22D1BmuWb28KqsMA/lGbWBI41NZXOAyncUldIIKZG3/L2sriq0KAGgtWYOhdW1VZyOfL8ZT0reAAameVvXsHRNBo7nJaXApUWtdovX6aqq4WhOa1FgQUqJ0xNfPnM2maGiMh3dUvws6lyzZripDTKekMJaFWsOpYp+tNKCGeNVnmM+D8GkSdRLRkh8xGCXQ7JWpusn4iy7UAqXBIDDgJcAmHbAxjnIM/L71wvQ9SS55UT0SZiZVDi+70Io5OLcPumcLSGgS1va0pa2tKUtbWlLW9rSlvZ7Yl+YAcyYiSfCAZ2nsQIACGGBBfgB4yaeJ4hJdKYWIE3J1vk5yCRHAENkP35fyLgbTnffNoWi/AtagHBaa1+Y4Usj7inJSrA065dmDl+WAXwZzNS+IAMYogeNc1CcOTTMXCVEZMLyleR0/UoyK2TQoFMyYR/LHGcH0jQ2hIIMaW8lISmaE2DBwsWsgRWL/KNB501qldybg6LfZl0fZcprx5FRKzOU9LybukZG0LqTo2OsPvDZr63Cv9foKcpwacJAUR/MO+sMxzXVFIaimtaMYUmHbnroWTZPHn+OEbFmobOFauQhDGfjCYwIECkFTYXO0jTUxmC2qhpgjcIr29u4dv06AODuxx/iAhGWXLt8ER/e9ed8dnCKOUWU5nOfzWoVA44o5Z0Wjk49w+h6kePjz/3v/tpf+eusZffLXKFF0bnwngd7Ud+0LrJeigg6UEJFRko4nBz5yObpAWU4KxFZyV6S0xHCQ1cAMEHUhYsXsXXRR0BPR08wIajy6cEchqA0u88OoEJxfA3YqW9rTUQ+jw93Mej7Z/t09wkOtb/Om9d3WE9RCo3JzLeZtQ0uX/FtLRlKmc4pyTVLAUOYCJ1JtCgSbAAmqaknvm9c7HfxM3rv6d4pbl/3kGMrDHLSPyxrg13KRL5x6wZWiKXvw0MP7c1EjREVpXdWejEKmoxl5xyKPGQ/8FwGUEnJmRUjBBd+SxnxFw5xnli4X452AsxKhphhUEjnI3AWgKEsLmoQSmfRBHInq/g8Cg5ViOJCvqy7LO3/TWsCCVmNVo+gTkLCEBmDbWpem7XKOUQsRc6MfI5hTYbXdClkQrwg/Gfw2RxFmR2TlCbEzHGioSkFI3RiD/R/Q2ZeipjVS9Ej4T+eZKyhq4t9Ei5mTEJmRTgXiVpcZMNWwjK22RrLWaWIhYjHsrBxbLmQSfTvR71EFwFKydzKGmFO83iiwgn/Msl4+WxszFIxSzOzmKYsiY4hbVplnN2CiHCyIlMJkY3/WEuBXPk1oqwqVLXPzmgItHKftWkaoGSmUIGGoPamSTLI9KxqY0DkyZhUDZpDX7KwfzLE6oafyzsrhGhJ2taztofnGTNbzjXIaU4uJ2c4PnwCABjScWfTKUpar62tI2FGY+GYDKVBE8jQpEz6dMxkhwxxlmtc3vIIk7WVAR498+vq48e7qGk9Hk/nGFL2zpGuZq/bZtbF2XyOGa1b4/EEFZGaXbmxiZ2rPg3605/+BvsHfv1uMmLIVBpBb04gIrZ8G4csUPQbfd+M/p3/oeA+4ZyBCMdzkVTJQSA6g6HN48Rc1gaTkV/zJ5MZ34s2DZPomUxyptGSvz6vDUoiX6nqGqbyv3tWTjHo+zKWditnxtjxtIYOmpe0ZhrhYMKwqBxsoN8U7E4izzTadAwtJeY0jwXUmNIaNbVXOS1xRvqN8/kck7MIP5WBdVYrFOSHzgjSaUzDuoktnUMQX6CSEqBr7gqFEZHTaUQ/XhKrq1MyavQqx4RCFoiMx1IwQzDrbto4ngQSZv6kjE4kCgZyoTQuWa8RTDDLqYGFDn65iGU6Z/NpRIa9xL4YAhrmB2t48mn1Mm6Uum6iU+kkU+mLhc0gXW5CR5PWrr1cPN0CyabRnyQe9mXsnBDnfhN++gK41PlzpzV84bOQAk6ZOhfE1ZN7STeDL2P1TGGfsQ4vYnvp7HS7yeayCuKqaS+IFM/WCYbsygQbLqTkjVeKzQ+vc6f4dzXADKNCKmYUzKG5c0enM1kAhUBFWPn9B4+wvuMdZ9kSvOgKG0XYmb5bWGRErSxbGUMNrNbI6DpkOcHBI79ABKbO7XffBkj83ZkSgvDg1pmIIy9HaMYeljc/2sPoka8xe/rzvwAATE9myG6+419PJCa0lhip4Sp/Z22hYYoAq6lgCI/eVH4izbIc+3ueufTdO3fwta9/CwBw8eJlPPjYS1r88hc/xcGJX9hqZ6MjTmPl8LTCkE5e1ooXp53XbmE28XDVxlT4xje/CQD4v977ZwyZmgamNBehVbCxZkwp5TH+CM5+WEQc5jPfbzK6Dik6MMZvBqUUizhQ9jsEgksX+u7obII5baBaHYcWqbi7SiOA0ZUU0No/l3a7g5uXvOyFULSRVg1M4zdV6/02DqYe6nFyeswzQCfr4NJFX6tnKoea6b5F+se/lmIxmENjpCgyFHmYpAv0CUa8/9S3+fqrN7E+8DWCT/dPcTwkqY5yxtAuJRVv1Na6BY72PCOssySEqx3XCOZac0DCIhn7cMhK6t8QTHWdLtwsA+EEjz3lZGQJwzm4COAXVH5scZGBAxTPE5F9TrhIfZ9u4hhu5Gzw/wlWS/OHMzAicJVH1relfYlGzm1dV5BE+73SL9CmGp1qblBSx2qSsZ/pjOtZGxbvFjE4KCyPW7fAFh3r9yQEezaRzlwk8kSIEQdEBj0lJQfrrHWJkHf4a3iTma59AgILQGPevUX4VahrFC6yUwsJgOjRnTVcGwNpEVlDA8zOMRQLqWPtHEIBnIfVhtfeL/KnD+PW8jjTQvI9+PmULzVuDKWIMFxmaY23ZxvDgdrG1ggeqxaOJzwvTr+4+TFCR8dTSsxKv07MS4H+So/OrRCGsPO7UgBA0SLW66aBCizaRqKd+zW2v7qCkgS2x5MxrAybNmL4Ng0HFoRqoBhXaz1LKgDRVLAUAN1/8Bk+/fBXAIDTfb+WNsZA0Ua1224h6/lrVkojI0fd6FjP5UT0n0qqF6ydgKZdx42dLbz2lpeoyYTCZ/d9QPnp413MaBNZO7AAeKcXAsAOUsW5sqyCIH2FU4KGQmpsXfDr2WCljz6VQLQJEtuYGraOm2Cu67KIzN1owD6fAAt8q/C8beSckEJyf1QihQgjbtjJZ52OSjS1L1fIVIYxrdM2gUl7uG7gvgBDKJtQE+wkNPlUWa6BALGsKq4t7fYH6BBXAtyIhe+VDvOBRPAbjJQxaqQit0SWC4aDzrVGZabUfmGTU3MQpWosTkb+XjQanBwNqR0dFN3LoNPietywYa6rBjltKNuFhi4yvo7AFix1gSLzz9Y4MJ8FJ73gYENQzKkk8AAYGaSdJD/otL6fzyGSYLCIQVgfqA2+fxJsSvYJkXk4KQsTDi6r6DpkeESAbaDU76B5+cJPl7a0pS1taUtb2tKWtrSlLW1p/7+xL9wehkxCKxcApUA3N3p47XVPfNDUBgcHPm2+++wYZ6Q/4qxLoBAcpl447osycYti6Ek2jV88z+oZfx+ik+eY8V5CsHL+3OlnSimOqgc2J2PMAqwzzbK8DMqZCi6H/7+IVAYiRgzOHYBRnRlFgIiviT+P4JqXZVJfdK8x8mhdhKegajCijJZWEm0iZRGZZhH5kGEwQsTfAbAER7z/8afo9v3vdLcT28BZtHsDugeKEGWACDABrdEmvT8HwVADl3cBggacEjFM58l9tDoE7+j2oQY+MwStmS1VVBUw9X2zOnmK6vghAKBoERnJ1irGJZHAFAJF18M4VKsNQ8XgupqhaXyfLsQcnxGpSW2Ijao2OCJimqZpcOP2KwCAx3un+Md//ksAwHQ2BFq+nfq6QJcyQoX2Uc29kzlmpX9vOp2inJC+0Ftv4NrWFgBg9+kjvPnO1/37iPDjgrJtZWlgmaRELPQ7zjKf63eDVYLh0vXs758iEkFILo53xjBrmnUOSgTdQG+jk1Nmy9QZIKSPOA4Ga9jZ9uQ908kMJcEwzk5HGI995i0nmK+pY6H/9nYb1Yn/7sbGFs6eekjN1bVt5BTN2n18imd7R3St8U+aldcUfaxri5DYb2Ut1lkspMDmtu9vlor7D07PkFObVtZCGMqOqj5qyqSOZiXrWd5/8ASXr1721zogwd1mzBqLnU4nZgCd8wLIdH1lGSHmo1kg8wkZNpuIOoOvX+sEYo5I6BChfGDx7PORPc7WJ+8JBwhnFz5IxbjhHA6e+Ox7b20Tra4f19ZYCEdzvYzZkqV9eUYcSDidTtFv+w6yur2OLhFslRVwSlmZw5M5w0HzXMKwVjZF153gvqfOZdsWSMECuYcV/Fpw+toulGxImiGksDFzLCJSQckIdYvmOIsBZxdKE1iwPSX84PKH6E84EQWXIWKZhbEqpomEPxfgkS7+RcyMO2ciOtU6/q5zaXkGGJrIY8hIQMeMgJDJWEyy8QHBoV1CuEMvNDRnQS0sQAyj9XwOqmiA0AoyEK0h6u6F1hQm6owpWGbfruYT5FQ+0O52EDQfnW2QE1S9u+bH+Gw054yMyh36a3696g9W4QjiLqaS772pfV+rypJ170QumTzDCAkTShbmI0yO/fw9OXoKXflszoDQNsZpRnJlUkAH+GOW8RqQ5xkmc/+daTlHSeu0ZVF2gYbO3W510CPf4+nTZ7h7z2cAD09G0DTf6yJjZmzOqFYNpIlIqqAP3JgGFem+zuYlpuRzOCHQJ+H1wKY5ntkFyHKADPpMHj3nxNcVAIqAYmpCX3KcTdauhuV1TsERggcu8aPpd2XZcNsZFX3nXOUgdCSUlqgI/js3DbIqircDgMgztFv+yyvdLj/D4fAUwRst2j2sku+2t3+IxgZSKtD9JaikhLhQKQUlI2O8zgJKCEwXNZ6QrzWpmBTHCMvrsZGWkQ6mqbk/trSFoExuaINeobGx4p/PyqALp4P+XkJxlRVoE8utFgLDmR9/OTOog8eNdWB9ZgGJQL9urEXODKRRSYEh2inM3gJBZV4IFdvMgqERzN4uHBP/zWcT1PQssjxDRhlnJWQAh6CuZoyke5l9sRA8TXzz2QjrqwQHcDNI+Ea5tLOJrQ0Pmcq0wOnQTwKjsymm0zLeCAAk9XHnLdJLp25LTE3z5ubc718IIz1PYchvP//+y+Gnixu88DeViViA2SWMob/rPOep2/21JbiPc9ccjtfWoUNFeKcSsSZIIkkfJ/BSpDSzTGcW2f8aZ5BRO4+fPsOjn/7CH1tLXP2qh0h2r+1AEpxCJ6syL/cSMOQUwzo+dl3XSd0ZeIAHuE5WtGECy1Z1horgoO2VASZjvyj019bRuuA3EpKgD+PhES/+nZaABm3YWqtME105B0lMeSjHkB1/jNYVPwGUUwdR0qBRGVr9IEfRhev4993pCbIp3WO2gp9+9gkAYPWm7/NoDMMtnu4+xeGh3zz//X/w9/HZ/fv++vsZdOGvqd9ZwXDuD/j4yYE/rmohJ1ZPKyOT19HpKb7y6qsAgHtPD/H48QMAXg6kCRNNwvwZYL5WSm5zY0yEEUjJkAEA6HV8W69teIexNjWGJ/7ahE0cLos47hIG2oqe96t3buPK9Vv+uUwnWCcH4v33f8ZiU0mtLgAAIABJREFUuT/60Y/wf/zv/9Q3WV1h/5lvp3fepI1SS+PuQ99eMznFmDAMuZtjdkCLU1bj0g2/0a/mDbpdv6BnNC4qEceWD8rQooyG6/7aWc5tnbcybG/6xrl37zEA4NMPPsaUNqpFrnH1iq8bmYxnLMTrYHH/c//93z7eR97zx+upwMDWQocWmW63sxBAco6YxqRguBEg8PAx1RwEJ1ACOg8OrUNGUiNZZqHp2efKRJh0cAAdmBFNJkyHQHQ6hYgwUu9WhDmBnFXlFxcAmI7O8MH7PwcAXLt5B6+8+RX/flJfKK2IrHpL+9JMB2hgOcHohOB3lwcYdDzjcLHexkblx+Jad4oxQfQro1ATrM2Q+nFjDNOdCxchmYCJzpqOlP6NiRu1JqknZPimRSz5cDaBNgvu40LKBMJJ63gSmIRwcInUUnSCkjKEgOaWkqeoxlquK/JM1XQZzjKruK+JD20Q66bC+mkSqJUTjjfBzkRG7XA/QFwHpbTxXhJYvnCADPAvC0iCirlG8JrGvoysYImSXivB8hbHxyfYPfLzplQOq6s+ODgYrEETPJMFxI2DcP55F3nB9Ui2aliywtkOdJjWdcaVMx3aBWUyh0GQGgFaHT+ntTptZmBclwLdrt8YBtbR+dlZhNmbAi2q9y+ERiPCulQzK7eyNTYGxApObNKVsZjTPFw7x/r2ItNM/587hznNyZPTKU5pzmJ0sHFcIz+dznB85DecH/32E9x/RLB9JVnGIlcKFU1kYaNXzuYIGPi6NDg+JDhlliMvwuZC4WB/FwDw5OlTWKpZ1yu+HcU8stlaa/i1klHUW6QVF86iTc+gJDbKpjFwBJs0cHGuFhpa+DU0a2m0adNp6Hl3uy1sDohZHQ4jCmzXsKgp6DmdTnFW+r7SuAYdKrMJcF3Puusv7Ww4wsGB91uODk/Qo2Bqt9dNgp5djCYEj2VX0UUma4nFzSCtj0pJdIixNM9aXBr0sNzlc4dNqVaa5xQtNQqCvc/nFhUxnY6rGt2Zv1/KMWB9dQUXt2h+bBWYka84q2uYUOKkNEJ42zqBvWPyLbm8T3lGXnj/NkCcs1aL54mmrhnlmtOzLLIsso4KhyILpTdAlLSrY8DtOaUEb6bxz+1k7xBTYqhfv7iBjvHnqY2JgS4j0SVI98tsCQFd2tKWtrSlLW1pS1va0pa2tN8T+8IMYNDg+vjTD/Fp7YkZXnnlDo6I8W5nZxsXKDvTaWtkmY/O7FxcI0gZcHLiIzPzKmalvA5dhFLEBFkCP4LDF+9P07Cz4P8L0USc0+/IqqVsnuehoOdZQM9DVlO2zxfp+f0uIfaURczZVPMowvWss4zrMAwBjRk9L1ibwF1EuI7I5iSk5YhiiMoKKXzFLwCNmrVRmtkIzdAXdctWDktwxPqohYwifQ2ltp0MbHA+1W+IKWrtwgYkRWQa0yyws51Qhqyk53ph8wIm9Lt+odFe9Vmd9avXUJPoJyDQJi0YbSnLKBpkBEWUx/dZY8YWfQhN2Ts4BL2lXieHFb5Qu26IBaqvUde++0/nDc6OfSFxq2OgQ6F/WSIniML7jw7wAWV+vns1iMorOIqCnQ6P8eknvwYAjIb72BgUdDyNvPDXt9JtY2/PRxFnlX8W33/nHY4M/eKjX/O9PH72BN9+7TUAwNaFVSYycY1FSfpMgYHUuUTjKrGFvu7O9UmCEmwQNEZcaqGa+CzjbDJFHeCKFswG1e608fWveijqm697DT/bWPz6w48BAFcvbuOv/sm/BQB49dZNvPcTT7ijnMIbr/p7qaYVdtb8Ob/xlW8AAIazCpPSt//uyWPs7/k5Y688Q6vy554+GuKdN/w5m3mDV9/0Rf0dInKZnJ1xBDyDZvImZx2KwPyVF8go4qiVxvXtAX3H98En+weYU7/rdtcwo2jirKoiG5gQaHcDUcIUA8ruFkH7qGhh46KPzrfbHZ4nvLZohMZwgbwAnuz7ew+wz0xJOMrSQMlIzCQkBEHxpWgY+hWzI4IzQ16vKEEeUJRRK8lIC6Ukw7VymutXehI1ZYtmszGuXr9M7ZyhJKKbuchQG9KXEoBxBZb25VqL+nQ9r/Foz8N03eQIhw/vAgCuXXsFF7evAAC217uYVv77o6nDWBK8nrpYWTWwBOk1RkCRAGzZNJC0TiidCLonybuQVZNScBTd41JofW9khELKSHyhtEzSNfQ3yYSIhDHUJuRZQkS4qknGZNCby4RgdlCZsiArwWPYw9tpXuRMYCwj8dqFAU0Dft9BxGC9i+zZQRPRWcnRdysEi0sLF5E6Ism6aySEOwwnlVzGYqxBTSrjs+kQ1ZTILoRgSFquNVqhHYikDMbwuEYmmWm7LmuYQCZXVOCCEmMDqSyKIsDzBeYEq5zVNSRBDTOZoUfZwJVuD31aP7pE2oKyxJgyTTkEWuseLdPfXI8M0spgXPoskbYGHYLPF6yfajEm3djxvMSU2WUlMiLucMbi5MyvpXfvP8KQyF8Gm/5YTV0xNHB4eooDIph5+vARKoJsbm5soFsEiH6NhhigM5qnV3sF+oSUsTjC2ci3f5ZpXL/hUS+bmxewu+uzYuPTYyY7s7T+1FUNG8jvbCT3amREvKUi7cIBl3f8+lHO/UM5Oj3BZEyZRSXjGFEZCoL+DQbr2KSSkW7HP9drW+u4tLVK7TjFwZH3yY8mc1SkaXx4MEbR8fe1sjbA67d8ede1axepnR0mZ55M7/ToGA/vk07xeAxnvV82HJ3hyrYnvVtdGzBEMjBgS4BLc2zTRNgzIsRcSa+jCQBFoTGgbHCHsq7lrIKiAZcVGprWqyxX7AMYeJF4AKhqhykxibYHvo1WV3sYrAT9aESMqpSoqIyiSkikAIvTcTpneH+aUW02rrFyVkW9PhdZWxVBWLUs0aYxUuQaeVbTceOcCDgmyVIQTOBCjxhFlkd/QoQ8pCeEnlC22BiB2dT7T/sHx2jcvwIENCMq4YsXL+OnP/HO7f7+U1y44DvH5uYFXCRn59KVK9jc8p2g3c+ws+UfYJ8Ep4+HFaY0Mcwp/Qx4nHyAW0jvtfj3YcDQxfDlhRrB81Vv4bM6vhbPb8TOSzyk9Yd/2Q3c+Q1g+j5v3hI4aLrhTM/HkFIbKa0BJMyqDmHxaVF637oIiV1g5hOR7VMqxYVDUkro8DpQ5ivJC5I9OoWhwdQvcrz7za9Q0zk0AQL85BkcsSO2trfoGFlkGhVxIV1ZXQGIacm5xXsfE3MTEVCi2NjCRsdDFA7HYyhi+3KVQTsIvRuDXAfsOm2qFNBQQOL0yacoiEa6m60g27xNDyNHQxskOx+iaHtR9Xbh/0qbIyO658mjXRT9gNV2mOx7uEhPA0b5vvre+79CRrDCLGGYVJTiH0/OcDb0v1tb7aBQYSPRxtGJn2CHwxFG5Fx3+z5w8odvvYbxqd90f/ybBlPydA6PTnC45+EP3/rjP8b/+ZMP/fUlrJYBagsHFn9NgyjWRnp0Y01knJOSmcsKYhorh8eoaPMjEdn/ut021jY8vn80HuNN2oT90Xe/CwD45//3j/H2G28CAHZ2LuI3738AANhY38Df/Bt/EwBw+9YdTCZTbrM3b3toqyE66IeHRxhc8P3gZ7/8CX72nj/Glc0drFPd2e3tLfz6fc/kurm2hbfeoA0g9RmpFMtpCCjUVVggMnRoc1aoLEqUqNhOWzRXbQz6cNa3x+rKSoiRoNXtMfxEAlhbo/HUDLmmsEN1EoONNaxeGCBYnAMU4/clgDwUYQhgVo/pfYKWWIGcxKBt5VCLuMELY9IIyTDXlA0xzEapYLb/KMyrEd4mpYRCgHb7fr69pvDmbT/GN9evwdEGsK4bPCT4z929GWbGt1lLOhydzrC0L9eC09MqMuyN/Pzxm8P7+O1H3rHYGvwCVwmafe2Vt7Fz6SYA4MLKOi/6wTGtMwdjiOHYGZa5cdMySqWgYda8LFMc/AtwUa0lH1cKxX1SaMMbwEzpBWeH607JsbPGLEhQxJIFF1lnnUTo5Uz5LwUEwriWCLA9BxfXR6sS1lMTGfnCOy5KLlibiHdbwxAta8FzjEUc2zGIHI+Xy8hwqKSItblptQqc12PwF+6vQzYwMgaNKtqEaQ1srff5WjMd6oUrmJrY1224JwlDc4OpDZyJm1lnQts5hvnBWFRB54FkjzKdwVKwtKgbpvnvtNpcY6aliE5tgPLZGiVBAE9Hx2gmPllQZLfQW/E+RG0rCIKyGdNAkryTog0gAHQzeoZFjobWpdIJlgeYzqc43Pebg/39E4ypzvHCNvEWKAlHm46qnGNGpRfj2Zz7Y6fVQpsgoMYAJZWSBObVa9sXsE4Jjv69h3j4wPsZ5ewMOUH0N9ZWcHzsx1/TODgKpMyntLGsa4bHCgEYlv1AhEgKxxB9CItXX/E+TOjeDx8+wT1iLm0qINDVCpUjD89LSd6Y37nh/fPXr19Cl+C4d588QocC6SvrW1hd8/eV5T1ICuqvrLWxvenLLFq0bxidHuDkwK+xu90+njw7pst3KGiDVxQd9Ij3YWtzi+vwQuDVCsESDpWNGHMJIAssoFrFcispEQg6wzg1jeFNX7+TY6Xr+4rONFpBzkFlaEI/Ng3vJQLzZ6/fQ6tN12QiX4jMLSRBjtEIhm8KKAzJbxEsBB/h48lLWGvRkD+mVB5cYAhHrKm2YemnIs+QM123TWTaBASNSVs3zDa/0g+JtS20qPh7sLGGTkOMvirDhBjhh8MxTk69vzmZlQvcKy+yJQR0aUtb2tKWtrSlLW1pS1va0n5P7ItZQOnvzqUbePWtdwEAP/5n/wSzmd9tHh0d4DFptN279xC3bvmI480bN3H16lUAwPam36UWrTkmE9pvuj729nw0eTqf8SbVEyaEsGANgZipCyYT+BJH9xKCGQe1AMkM2bQXEb445yJULMnIKaWe+/7LGD7PZwNfpPn3omMsZABdZNR8ToQ+pJgJEqa0AijqlorGKqW4oFYoxVWrWkgQAhFKhsg/EIR6q6qEoSjHcDTG5ZDhg8XRsY+wucaxUOZ8QhG/1XWIUCwLxxDQypQIcTxjDIexHCIMIG9R1qbbRXHBR5w2a2BKxxBSwlExdVPN0cl9ZCujyGhd1V64B4A2FpbYPpWzLP7puiuYUtuUkzHmGd07Fd8WeR/V2Gfj+oMV9Aa+ONg2BnskCNtuRjD0fqdX4HruX690g9h8g5Oxj/hN5w7TQ5+VnM/nGFKk8vDsEFOCs3TbK3j7LQ+hLOm96XSCNj3bV7avYHji4RbTco6zmY8A7Vy+jLsP/lcAwKwu4agdAneVzjNoiuJVTC4CwAkfjQex+CXZ6e9922fw1lZ9VHZ7+ykePPxHvs2dYVjqtWvXGBKxurqKP/+nf+6v9aaPUv6dv/23cEQCvu+99x6m1Df/6A//CO+++xYA4ONPPsaQsqCPHz9G3vhjnwx9RnhsDb7yjs88/+AH38FHH/gMYNE43N7xMLauVrhz5zoAYPTRJ1ihgvEBEQjsFTmaKoVR+3GfZxkzdeVZzuLzmZIYE/vZzz94CAA4ndYMEammNW694uewxjqUlAkRApiPPInAWqGxSrpP61s+S7q+uQ5FWUEhJWcOq9pA14FhzTFUSCCiBgJTY2MtC8xPpnNUlsOJPJ6EzJ6b26x1McLsYhs4578P+H4QiCXgLIyl66h9n7+01kWniLARh0DYlCNznvH25N49HDs/T9RCoKSI49K+POv1fP/fvryFsxMfzT85mKEilM1n9+7h0088HHTno9/iW3/wHQDA61/9DnpdP48F2GRlAOEItu8cZ67msxwHp34NGJ6cwAo/s690B6gDoQplCz2xV0L2Qhpc1komTVBKMNGKlrFMgWnrXNTHMmlpRfgcPhMQ1sRAgiTgYIkQRDiV6J2aRaHswGZsm4USFH8JDWc+q5R8xnmmbH8eyceQwnHphKa11ArFeqFaZTzGtU60eBPGD2EBJwIENPwVUCpmHwUxBBZawYgWvV8xu6BwDoo1Q4MvIKAFjXcnGF6a5220KGOkZQuWyPzm8wq2DGyMRGo1WMeA0jArK+vodv3vOipDQ+yyaCpoylBK0n7LnEGLMmjz2mI88WuicRUksXHkVQYV2LBbBQpC0WiCWwpoWEJLtLI2TCuQudVRt8+CM1edTgeG2D+7lCFpao2S2rRJWFFbrYyJwOAEupQR6uVtdKifDc/8ulvAoUfO00YvwxlBK1vdFlYpAyWsQTn2Y67IM55bg++aq4wZc2sbmeSliMRFEFFV2QIYULlNmMv7gy7aHT/eJ80Eip5hUbTQocybMAYrHd++33j3qwCAV65dxHzix++0mmF9w6OfBhs76A02qAkEZlUYtw4Fka+AWNi1ztChzK0+GKHb8dc275fMDtrptFDQ6831VWjqvy0ilKkwQx3WT4CJU1q5QvD3s7xgJlolJZO5TKZ+zZnNp5zNUkqgoDU2z3LUYYxkGWc5YRq0KY25tuqveXWlxyRHVWORUT/ILJAV5Js6jZXeiI83OvO+caYo46gEl280NmrxNk0khqwwh80Wt1YSDSZN0COsGAYubM3zWZG1IChzX5cjjIZ+7IyGxNDbbqN1wfsZWaa5PZrGYDL1ffDo5ARDIt+r6hpCLHUAl7a0pS1taUtb2tKWtrSlLW1p+F0yEI4w0VLixk1P4vDhr97n2qrd3X202z76e3p6jMMDX2j74P593L7tMwS3bvus4NalbeQUAdrZuYjbd3wU8sHnuziiQs+6rjl6J6TiaFtKchCCJlJIrkGDEFGLyiXVpAv38nxNX5qFK8sSpxTJLoocXaqxCDpeqXyD8z9OjkHRG2uihkzCQBNoq00SWbTGYE71YJAaOdEbu/NZwxAFVSETKKCyWO/A9QdKxnZSguv9ZG3Qavx5Nuhp91QbQ6L+PbANLBUESxv15JwxLKmQ6ZwlHAxllCZnI3Sofg9KMCGJVJqv2cO9Yx1nqBtpUfZGZi0UG1S4nBXIKXIxG5/BEK1zSzsYqnGZUoROihqZpojNYAvtSz4zpDdv8LNXWQ7dJnrptWuYKZJ/mPjjCiuZCKfIFePSZ2dDrFPN21rnAvZINOvNr72DB1TsvU4Uz6PRFObM39Pa2gUmA5DQOD4j2vWZRS/zUbV/+zvfxdalawCA/+5/+F8AAD/+1UdQzkeZJk2G1+946Q1TnvIz/PzePdy7ew8AUJuGC5ID/TGkZjmEeiYWCIUisUF8xzmH/+0f/xMAwAWKKI3nhvunkgqDFd92vU4HVyibf3RyjDde9TWAP/rhDwEAly5fxiGRQq2srGA28+1189ZtvP/BbwAAd+9+hvd/5XURrWkwOfURtn/33/szAEC710WPKp1//uHPUZOm1PXtHUjhI4d7B8f40Q+/BwC4/2QflrLWre4VaosWDGWN4RwE3Uu7lTHxS5YXLKmg4bBLGqYffPwotAyaKmQOczx87N/v93vcgA+f7KImcqTX37iILj2DHuk/Fa02Z+UB4Yt3AGhtUVHftMawhqMQErYJY540FjVQUuSzLiuULkSs4/gE1W4Ci3NTyIT4ZogohUBKYK1hNIEQFhkRCgUpldwpgKKQQqqQ+AeEw/zM9385OcDKKmWdGgcp4jmX9uVYQ5mGtfV1XCXkjdQOu49INuV0jIoQBJNPPsQZ1Rk/3TvBm1/5GgDg5iu+LnBzY511xuqmgaW6lX4vh6Yo+nQ8RKZ93+t1CozDPErzSrsdM9J1Y5Ax2YvgDJq1lrOOPhMdMtQ0RznHNYXWWpbYccb4QnkAk8mM771L/AJFu0CIZXsNX9aB4Jogay1fq0cG0Xmor8/nc0YDVVUTyqygpeYaIqkko16EkNCsZ0ZjXCrW/yqk5s+lUEyAAhdJ3b2/EMY+XSccuFRRRMIYL7NBsh3WhTJHiMayrEQWyHaMCNMObG0CsAC9dpezLAIOoglZCAtLJw1Zy063ixbNaf1eF5IyaGY+j7qHjUET5qkmCMsBlohrSjS4sOr9vMGFNSjKppkyQ9YK8gVtaMraKMrIOAe+16LlICUhhqoGGa0vrdrgzi3yB/IMh4Qw6bT8OYaTGibUg9VeXxEAVjo9zgCWsxn6O75e++3Xb+Jw17/+xS9+BcCjVUaUwSzLGjtU271xcRs9Qho9vvcp7n/+abh19nEC2inTCg21XWNtROR4EVZ/v3A8RgCHBw89GVtJc/nZcLqosUntv7FSYG3dPyPhgHdu3wAAfO0dX2O/ub6Gw2dE8GcN2sQ7UPQGOCMihrsPH+Lufe+3D9odvP2WP8bGjkdmddqX8Pi+R8h8+OvPUBF/R5FJtFsBTZWhRf5rv9tBHmSXCOkF4zA2fs20pkEWuCOymCXPigIZ+dpNBRzT89wjVNXZtAQjDKRFHiTDWgXmhOSZzCqUJI3QLzJsX/A+zPVL/l7WVvssy5RZwFI/MAAyQpR1ZIFuz59bK8nES6KIa2Ih/bmdabzsAt1jkDeDi8SLOuhWZpJla4SLmr9aS0bqwM6QUwa7rQpoQdrC9LzLukRDvjxg4VyQVWm4v9W1wZT2FXVlkOnF/cR5+8INYJ4Fp0FiQGxP3/zGdzA+o7S+qXDvc6+NdnxyjOGZnwxOh8fYJWayTz/zg+POK6/jxk3v/Ao02LroH8o7b97EeOSLVp8+fYYj0mtpLFh3KJjwDAb+tTM80QuRbLiEYPYhP6GHzVmEhb4IpimEQE4CkJ/cvcfHDlCbLM+wvuEHkMo0bwy1E1AmpNAlI1ittZgTpHFYkoM/nbCGTl3XqAkSNuitYJOgiSm5jbUWho5dEcShgx4K3hhH9jMpYvGqFCLo0WJ6NsbB0DtuYZJ3KwOc0HVUVrK4qtUKOTmplTFoEZyv1WqjonbqUhuNp1NUQ9owdzu8sSryHFkdNAaTDSyAKjinLDLagiLH39QVLDmjqGdQxg9kV5VoSChWUee3eQ5oIs5Y6UFs+eCE6q7w5nh+8ATNgZ+4ysZBEoNoTQXx9byB1v5eJATCqpu3W+hu3PDfsRVOHvj+e/3GK0DmJ7RrV/wxfvyTX/HkvrKyytCN21eu4UMSm3VNC9/71h8AAH7w6g18vuv7dz3x13Ni5qgaP55QbOCv0gbr2pULGO550pOfvfcedonFLM9zaLrH2oaFJWWSWnT8oxByFGP1QuS+b57SRDttojZNp9PGgASlp5MJenRf//6f/Rm+++1v+0sNi2g5R39AIsGrA7xDzuV4OsWf//MfAwBG4zNUNMllucJf+Wt/AgDYpEVGW+DokSe8+a/+0/8cH33k55QBNF771rf8vVYlPvzQE1Gt97q4s+MJpwrqo51uFzMiH3DOeYgEPMQoFJrnecYF/s4Z1kYTtCg3tcEJjVWlSoxngQ1zBksO6Gw8xmrbH6/XjeyXisaN1jrC0W18X8CyPI+TigWZAcA2gZXRj3FlLfo9InyYz4Agum4adtQNGt74ZVkonhfQDPWUPI9orRkmBlioLBAeWSgboFsUnFEC+0SCJJRGFvDjUmJCQZnBWg8lyQvNpxWsSmDHS/tSzBJEq6kbdMixW9ucYUoOspE5BK0vk8N9PH7qoeXDs7/Ao4cewvz61zx50zvvfgU3iDCm3+1A5OSQywyKnLnHjx8iI+ekaGnMx+QbkHPSKXoI7oQxLsZghYE1tIHKNDvA89mUiVZYMhCOReidtR7uCe/gBDKak+EZB06HY4LC5dGRLLKMNoRAZ6ULQ+t4Yy1EIMQyBiU5shWN99PTIW84jQOPz067A0mwzkxqJuCATDauAcHtuUv9ayGgqD1ykWj0QTJ03zR1dCBDqYdwiCPX8bpkjWOSDAnNotiyNpDEVA0RndSwIXaigaMAk8oVanpdV0OcEZSznE6wGoTXgz6ZysBkF0KhIZbN6ekZQP0qg4EMrIs0hzWwMLRBVw5Y3fC+XX/1AshHxdyOMB37dp+VMwSiwiwEK3XOzSykQo+IwFrdAvMusVDOSia6y7tdHFK5yuj4ATWFRTkjX6uqeeN7efMi9ja9P1SXcwxW/Np19fplLqNwH/h1d/90BEF+3s7FTdwhcpaNrUs4oXXz17/+DHc/8j6CqWouvQnBcyjBbW5M1HTzepaBmCuW+jgHfH7vc9829LvGVDDwfboyBm3afN65dg1/9G2/Pq6uDPDW228AAG5QwLap5zij65yNS3RW/L1Mzxp89sj7Rj//6GPsH/g2/errr2DQ9xv2ixf9c4MzOHz2MwDABx98iIN9P490+y0u3ZLSwZFvlxUt9Im0JCtoA+gElDqke3VM6qdVxkoDOst4szStJtjb82vQlPSghdRMiDSalDig+2rnBWoK2s7mFW+mLq50cGXbE91trHkIa1FkCYRbwYhI7FK7ANnVrBusMgUVSJjCxixvoU3PsMgkGlqbK1v7+QGAQWTuzingXBQCMuj2QkQ2cungiIzQB95C4Ewjp3msqQPbu+I+7aRERbDtsioxpfmxagAZtCEzwxvel9kSArq0pS1taUtb2tKWtrSlLW1pvyf2hdvDkE5vGoec6HLnoQAYwA9+8EO89ZaPOvz244/w6ac+cn90dIiTEw85CZGS3afHePC5jzq8+todvP76KwCA27eu4cKm36H/+Cd/gWfPfCbge9/7Pp4+89GGkPGQQkYii8YAgWLVWM6slWXFUJQnT57gR3/sIySRkCXC4xKOFSilsLru4XD60TM8pevoUebtye4TXLri4Wa7B3voEUTu0sYmrm5t8zEqyvqNx+NIzUpRxtl8zlm/pmmwTVGWdq/HkYko/OBfc+afdviyalAQRbKwMbKY6YxT69ICFaXCTx8/QZciZTPa7j94fICKIqMbO9uRjKYsOeJirOFoyvhshGLgM8DTQ8omlhVGJOvQ6nZQUVbYKgVDEdqmiSQ8xhhIiiJ2qaA5KwrMqI1GR/sAaQOJ+YT4jgFX1WhRlCUX/nzWAdWY9GGURfvEPyuxcwuS4KX14SOcPfAePYHaAAAgAElEQVRkIi7voaQiXtH2zzATEg3BOzOVQRF8prW1yYXV88kI+aHvgy0I3Ljk27fb9Znuq5ev4vRj36ePD09xSmQo7965heOhvxdT5fjeu16ywExGODnwkbCS+sTbr/4/7L1prGbJeR72VNXZvv3uS9++t7unu6dn3zjD2USRI5EUSa0kRTqSJTmCF8GIEyeOHdhAbCMJEBiQAwPxj8RyZC1QEtiRnUjWQkoixX1Izr72vtyl736//fvOd7aq/HjfqnMHMgcBgjA/eOtH98VdzndOnap6t+d9nlO49wwRLL16YwvLU5TVPre0hK0JffadgxslPNP33BqyFUBI5TLxf2EcQ4O+R5bENlRzZjRLYrcO0kmCNKS5+Xv/xd/BU1z1O7VyCl3OtFqdqf2Dffwvv/WvaB79ChrcMP7id17C7j49642b1/DIo1Rt+Oxnfhr3XLgAAMj5OXY3tvAv/9tfBQC8/u1X4HkEfajXZrG6Rr/b6XQwv0AZvckghhlbCBp9r1pvoM1Q1DxPIQO7Lzwn2+F5ykEoh3GKG+tUCckY/qGU7+Da0vcRcrXbFHASDmHgY36a4ZtSuPUW8pqGkk6KQUiJ4phYja3Sagjg2PeDgEl27L4QGaYbpc6e4YxpKBUC3rfaaId0sJWBwPfAx3QJKwERZxRM3BVGgcucJ3kG5ipCnjLUbzzB0YD2U4ESnhprjTxl4ig1hTubd+hrIVyW9GR8/0ZRlJBem2VeWFxCo27lcwQChpjtb27h+hWqnvd6Y2xvEpy8M6As+5V33sY5JnW6+OADuPQAoRAaU1MYjukc2z/YR71O635+cdqREWXcSpCmY0hLGCQVmNsE8WSAIfsM47RwMO3RYIjnniLfwWhbMdeQlh5dl60TeaGRcPkoKQxG/HU/ZZtTaOR83cl4DMFVhdNnVjHPMlWAxKBLZ0a/33F2OuMNkCQx70tAeT7qvK8DP3DoosJoB683BZx+qC2uS20gLKLIwBF7COE54gsIASu6l+YJdFYSnwGAltLJOVA9zV5bwOPfKbSBtPs7T0v9P2ElOxQmha2wJRjwO1TKh8/VjSTLMOaKfpEaNC3dPj9LkpXkFDrPnUTQJJlAMEQ/ROGklIyxFQiNhH9upEEtJFtaDWqIGYWz2+thi+H1g04bzRnyLZpTXA0KC8QJ3Uic5phlyF19ZhqGKxpJVgA8T/Vq3dH099tbPGPKtUpkae7ahe45fxZS0ay2D48wN00oFF14mEysrAjDGVt1LCyfAgCsnTmFlTNn6Vnq0+53B72xgysWxjh47CS1hHvaSu7C4BgEGsKhSgqtHekalEFvQH5SzJWfQmsI3lABgHtWaa9+5MNP48PPf4S+7wvMTM/wtel3N7e28RLDWXcP+lhetVVaic098i2GnQnOrBIyb+38eYQNehcp29cbV9/FF/6ECOhef+ttVG11fSp0kiai0MjZ59O+cL5UGNX4TQj4DNkMPB+RlTCRnoOBC11Ylw+D3hBdjh8KrjBHvueQPAIJCiaJSYriGHxWIuBrLy5MY3HRSl1YrceStEUKCXDFF0JAwVbRVdk+JX3UQ/qDuRb5ArNzTdQju+aBLKZrx+kEmbakQzkUkylxURBSGkgmYQy8kozOysrQc1XsXSAzBjmvD5nS5/W6Ywz7I/4MTTIPAPr9LvaPyPcZxhNHZhX5IWZnp/B+4/3rgw42lLrF2mxVIJjnUWtgdpagWM88U2oC7u7uYI8X2NYWQUEn4xGGrAlzcLCNTpu+vnrtMmI+iH7t134NTQ40Pvuzn3XaLV/4whcAAIPBwJXW0yRBHLOuYBwjZrjWcDhBbvGweYq/+bf+Y34UZo08XhI1xvUc5oVGWnCv3OwM7mzSATW1QIto+fyam+Sty/u48+pLAMhAPM+aaMPRCBkbltbUlNN/qTPUsFGrI2IWotFohMUFCgDDqILUiUSXvXwaBgXjS0I+m9J2D0f7A/5s4Z5HKYXAK1mUEtsPNRwBtudugeZWeXW0twhS2L91C8vLViMvLNnIjHHMZRo5Qt7tRwwBmPQHiJl5qGeMg1J4gY82w41MVgYlRVGgyrpTvoV9Go19DrRFNkYTzL6kBxiM2VEoFAQ7NQEfzIGvkLGxS3LS+aGLKBfvDHtt9Fg8VVcm8Jjtq8YMh0VYQX9E9yPyHCZnyKmuO0fAr9bgs0ZRkWWoN2n+Ll95g35ufKxyD2OoQljt7rqe4C9/8kdpnoYZIobmbN45xMtXCC6SMmwnNznOtsgIfWN0BdttCrDvF5egeS1VooozxoXWKHILA7CYcziH5vgwxjiDo8wxtkkA+9wT1OOD3hgFZYOBJMWv/N1fAQD89M9+1vWVXX39TczUKfEhG/ROJrrAi999leYgAz74yJMAgCDy4fGB+KMvfAQ//bM/AwCYnZlGweug/S6xFP7RP/in2LlNYvKpL6EsjKpWwZe/TWLyvjS4sk7Gvao8CIYq3VcQ3KVSqTj4khACFcsspzzH2uX7HgwfvL2DLrZ3yXCXMHA4HL+SEpIhKYVA2VecF04TS3gStSlaExWGKYljwbghIUz6DGMc0y/w3h5hn/Hay7w/GxGgbTujCBExu1tVhY5tr8gzjDmoi9lhGBY5KnUL31Oup8nzPJeUGY76SNiJzQwcM1nM+20yGmPA/bxJMnFQrd1xhm5C89jvDl3yLfKBC9N/sb/6ZPx/O2wywffKQMNTAtWQnMAwrKFmYZHKQ79NZ3Iz6qBSpTWSsLe/efsabl8nndG3rr6DR29RH/Li8ins7JL9/sa3voPTp2ivzU3PIubE6I2rdJ71Bn3o1OrvJUjGDNWbDJBwj06WZw7CPD07jb/8cz8FAC7gEaIkBKVv2KSogLHsmmEIyefHhJmUdaGR877Yax9ib59syp3tTdxzgWF7rVnsWcHuYR8NTvI1OZFbCRoQFurpKdf+4Pk+pFcmZ10ACE1s1CgZPHMUELaBL59A8/wb30NuCdBN7vQX0zx1rSaWnVoY5YJIbQp3fijpweP2BQXh2ih0liGxbSWFZd/MkeY2sJ1QMAfACOP6FbWQyO0how0yPptSnsckTcv2DV0yr0aVGoqMuSGyEXxB1w7Y+GWBgbBRZKXlAthJmhJEEsBw2MM2tzTobIL6NPkO1RonsjzPsYd2j9pQrG9XnVmEp/hc9yaO4dAYDZ+TdbYQIIyE4vc5GU/QOSJ7tzg3hYtnaB13WzV3je27+7i+QXsk5qCj1ay4gPio3UVUpXuenRdOtzGoRRA8p6R5yWvZ8T1I2P7OoigdfmGU6+3KC+2YWoXWyDjYs4FNnhVo1enzHrx4Lz758Y8DAH7khz6M2Vlav5N+Hyn3pmtD6y7uDaEZXyu1jwNO3sqwhmqToJ4fvPgQzl2k5OzKyjTSDiVFb92+AwB465WX8O7bBIlNkhhBjZ41TVIk7P/1ez1wvhVFUUfIfcMBB4AeDALWeEy9AJFlmRbKPXeW5U5HL0kSB+v0eZ49JVDwvChtoDwLnzWlALtSaHBbxsxUw8G4rV6y8gLXNkHtSXQbuSBIJQAEYeASv4EnoWPuQWQ+lEZVIqzO8vMFyJnXpJ4rDFMrxq4dnNXuJwgBzeeZF/plIFqIsjVKCYwZmn7UG7oiQs6Fll6/7/ZyboCEE2tH7T10OrYtL0eFobfNZtNKmH7PcQIBPRkn42ScjJNxMk7GyTgZJ+NknIwfkPH+OoCiLFfbbNC9Fy++h+jE6qtMkgm++pWvAwAefuRh/P2//w8AAL/5m78BAPjm17+CNKFINh4N0OtSNW0wGuKQK2ubm1uYZgjil770JVy6SGxG775N8JU333rLQdqKonDVNKUk1s5Qpu/U2ftguKJ4z7kVV+Ern+mYxp8p8/BFkaPL+m/9QQ+tacoQaoaTxHnmsnj3XLqIyhRlXq5fu4Y3r1D2dHp6CgVX8hD6OL1I1dFHzhKMzZelRuFoNEaoymyKnVStygwgwTDtAzM8bzhCzPppvkRZjRBl43GWJFCMBas35+BxGqBZZda0RhWDPcoGbR/sY+kUVQCDauigPVoLZAyRPDo4Qpsrtiln/ypSwmIb0kkGMJPXOI4xOKBMa6tWceQ1RVFAMkGFJZRpH7WRchVjph5CMeNSI93B7j5N9vVX3sGzHyRIkjlDsOGi2kKlSVmYpLeHokkwwMwUMFw1Hg2H6E5YH6Y5g8oMwRx0g551IivIcvpdPe6izpAZpRMMejQ3QVRx2T2dpKhwJfWVVynr3QhCzMxS9c6oELfvbvM9eTg6oIxjrVLDjT491zdeeRs3dmhuPCb5uLl3hLeuU7W5PUjw2o0bAIDzK6fRZlKlV998E3DkCAaWJshBOvPcLg9muyt/7nSHjsMOjYFkKIfmIyAZJzAMKfwbf/Ov4xd/4ZcAACIr0GsTVOz2tRuI1og5MOf7qdca+OynPkvzFXjIOQMu8gw/8RNE9vL0k09BMKxikhe48w2qGH79H/1zAMBwYweNR+m6/Vc3XZ3syvpVjLqU2frAw4/hBrORXVg7jS5Xqe4rSD+wWqkgjEqtu0pg94VwpCfS85w+U601A79CFTczYPgKACuRJ0QB37Gcheh0WE9pHEODdSmD0LH32qqxNHBkAoXRJSNubhwJE2QJvZVC4A9+798BAFYXKHM32/ChmLW2UMCp07TW7zt7AU3OPk4Kib0urVkLt5dSIWBNp+Fw6LKdYRgiZbjIYByjz3Dyo/4Q+x16t8MunX3NqAbDEJ5KaBxr6o29Htb7FtozjXNzdLZdOLOARXOAk/H9HsegQ7a6pI3Loqs0xYiJYkyWOJKferWKc/cQa/LIsjWOerh5l6ofu3c3EDBU6fK7b+H2Bp1N167fRucM7YGl5SVH0PbKy0T0tHVnCxHruDUbDRScW27VKmg0WbOzWYXHBDNRrYWCq1TWRksjS0iUMY7oREOXcDMlj7Fq0/cKrQFbbZM+elbXtj/AmLPkU605R5I2PzuFU6cIfVOziAYJBymEIEZSgPZnWcUvtb6MzlAwZi1JySZCayT2IDYl7DCQAihstS1153ZYqaLJqJKQzwOtSxIVnSXuuaUv4FnEhzZIuJ2i3+8htoQ2vPc1BAzDH4UnHdGGkMeI4KSEsreqc0ewEWdMbjeZwPB5KpRyumueJ6Eyeodxv4OixwzQrI1badXRZBIQ2azCCJr/yagDaZEdEK4qkyQJEvYpfLa1USVAFNi2H4OM21myPClbF2R5TxACiquqYyZXS9IMBfs9vWGMy1cI9uwJg8X5Wb62j06Hqiw31rdxhYnbrB5xBh/D8V2+Z2CTkU0Xzl9EwCimUZq6tSdVuT6sDSsK48heNIR1r1AITesWQJ5nzn4LkQNsd3JGuOlc49Enyc79/M/9R/jQ80QqtzzbwvCQqt29rS0MBrTugyrB/pQxuHSGWl5OnwbCFp3Z1WYLzSXyW+Zm1xCw77a9fgevvUj7OR2yHvF4BA2u8voBMrYjR/khdEz7Oo4njpipOR6hwSzpliTGKOkqYsr3Saca3NLF+92HIDY7AFEQIGD7rTXZpVxnJfu9VO/V8dS2Si7RZLRYNarC2PjFau4qz2l3FsYCZcnuy2NIOosEMMbgTWYvHw5ondTrVSwsECx4fn7WMcZXGxUy1gA5Ab6FibLfk2TILQrRDx2raxJPMGCUzVH7CDt7jJI8bGPCrSm1qmXJDVHnmCeoVRzqISk8SN+y5lYwP03re2ZuFq3m/wsIqC3P6qIUO4fQUA4jK5yT0T/o4eWXabJ6/QF+/ud/DgDD1wDc3drFPIt+J5MMY2bsq9RqDts+3Wqixovxj/7gj3DzEkHE+r0B3xBw7yVifHzs8SecVEOSTPDIY7Qptvcm8Jn++uKFU9AMi7BlYq3Lg5ka7iz8RKHF8FM/CuCxsOWAD4NBt+MYu5rVOqpn6LnaWzuoMQ5+rTXvICUQAstT9LwBH8ZFmjuHPPJ8FFwy1tAO9qmk50ReJbRjxhQMQ+m3j1AwE5CKlLuGBY3Se8kA2yOSDeFVqdxvuKw+6o6gWKTznouXHGNirieQzmEtjUU8GeHggIx/yJDU5vwMurxw08xgyIdPVImc0xsGwkFcjBbwC96IDLvo9fvO0E5MFY0ZCuIDbw7+hDZC/8YXcXGRA0NJAda33o1RPPNDAIDFhx5CwT0Cg/XLiJYokMjCBswUQT28mTNIOnSYLvksGNvexcsvUeA+f888Tl9gx9oD+nzIpcZAW4ZIbZynNRjR/9VaE/sMN473d7G5TgbkRx5/FCEHLtfW34C3TM+1noXI2KL77MD14wLfuEXrfDQZIWLG1oP2Dt5iZsx3t9vOWKAwDvZb8NqkZcwwpWMst+6HIDiJg/QYgYwx9BMO0Bv1Jn7l7/0dAMAv/5W/AsHBc3tzG1/7Jom/tzs9vP0q9VV+5FM/BgC4vb6Bjz1L7+JrN9/EzFk6HB9aPYcWs3zmSYYwprV0+Qt/jm/99/8CACDW6fku/Gefw7958ffoe1IgZxjP1uaW62tNkhR7DD8+u3oKBxz0DFg6pNDG9UZGUeTmSMrSATICMHwerNxzAS/8FO2LP/jd/4M+oz90Dqgf+Gi06OdypBAf0dqbOruGRYYQAZ0SwmWZeY/rbaDsYzLHmN6klA4qZAB866t/AgDY36Lgv1qrQLABDOpVVKzo9/IqVhbIoBdZjj4ny1rsxFYrdcgaHfiZlqi36PvxuI+dDYLPHnW6GLDDOs5TBx+zUiU/8aMvQEj63uLsNAKPgtLDogdZZxhYWMMCn5X3XTqP0eYIJ+P7O2yrnEDZj6uVcokWJYSTQJikGSacMInHPXTbtC4ydtByrR0zZZzE2D0kqFuaa+ztcl8tgDbb7LevXHe9Nv0h2Wa/5uPcvZSge+zhpzA3R9D46UaAat320/lOZH0wHLlkqWQH2cjimAyUcDApAZSMj4GHyIo9u57ahgsY/EoFPguYHx4cQXMwmxYac9O0n1fWVjE3Z4WwbVCVO6dMACVDqYELqmEEDEOfk8nECTXbXuY8zUnYGYA0Br6FjioJcF+czmMHLa96TTQ5kLA0+pnWKDLh3ov1CwI/RIUFwAENw8nXQd5DxkFuwQ6hkcoxGysErpUDRhzrKcycZIE0GokFg3Eg5SWx62kKwhAB94b5VYWEE7W397fRYehwPaRnOn3uHM6cZxmCZssJ0o/HY0ht2zAC16sad49cX/nuNgVb1WoTBRuuqamW669Ok7ELHNNJ6nwHAe3YYzsdOosmkwxWw8ZH4SByB+2uk+wpUo2722Rvr9+6i702+TDSsiqPU+SR9cuAI4baNzsdCCtTcNDGhPsjIQQU2x3BMgC5nrh+XZiSJd4YXdppUSZxBKSDJir2K++9eBE/+amfBgA8+8wz8BjW2d7dhx6T/ese7OCAk8uS15KI6o5Nda41g9ZpClYai2tozVDSXCoPnUPaw2+++BV8+2t/CgC4/9JZ2Lsw7CuIQpeBS5ZhZHv5fA/eiBPJeYY4o88fduh+knjoGLqlkC75qYQHw2eA9oWTjpn36rhf0zVUSEnfre09CA5EC53ACb15HhRPWL05hdklOne8WhWplWLgNeh7vtvXWVG4FolCCtuWC+lF7l1ASVy9Tjb5+jX6Px2P3PsMKnXMLVJBYXFpFtXQytJU4EW0Zm0hxhQS0iVRQqTsN6ZxiqHl6zjqoG/ZceMEmvft8sJZ/owAiZXY8EJwPQWVNIGVbgqjyMUg87Nzrm/7e40TCOjJOBkn42ScjJNxMk7GyTgZJ+Nk/ICM/0cQUADvqSq4xIUoRYjr9TrOnqXM+MH+Dv7ZP/unAIA766TLMrewgAbDHbIsxVGbMvjRZOQqilNTDZf1W1+/jTb/jv3sxx9/HD/2CYKV2f8B4PDwEP0+Rc63L7+OmRmKgLvdniuTGlcBLPXQpBBlxlHBQZ+C0MMKQ0QMZxGyXDt4mx+G6HHmc/6HK8g5aocQuP+++919WeZGm60ThXaZC62Ny6b4HhwzVZ7HSLn6srOz6wgzbPO2UgopwxWn6nVUOKughIHk6pwMIzAaBKemI5xZpIxXxeeqU5JDcPo3DCKcm2Gom5cj4efd7+XY79P1FpdXUWdduP7eDs1HmkAJW3ov9eaUVE7DLPACRJy1zPIcSY+yVV4e8rNE0FyIj+ME2RRBFPTSRVxcpKzma3/8Ev78S18GADywRGvpq9cKDCc0H7/86BMoBpSl7u5uopZzRbc6g9op0ra6+soVPPji1wAAT3yU4KRX7+zi3W9Rhupnnvm7mD1PVbqtWCLlbNWk38fEsstmBSKr9+jR9+bmpzBIbPl+gG6PMjnffec6fuLjTAKz04H0qCrTWqng4ICqWIIJajBJkQxontdOL+IBzqJ3BzG2Duk+UkTIbQO0JyEY0pDbzBwAT1m9ubJhnzQlObue6/cwhSZcEViep4rd3/5P/3N84uO0p0QvxpAZyHb7HSiGvl5aXsNGSGvpiDOgt9Y30G3Te73noUt46H5qKK/VK8i5umhyjW//b78PAHjrn/wm6gy7OftXiRhmcGkFnd+ldaV8r4QKeQYzTNaQZ6l7RmMK7DDRw2XWBoTO4TNRS7NZL9nFpIJiGKNUEtIwvEOF+NRnPg8AWD1Pc/61L3wBG9epGgsvR0cz++xOgdU12tcv/PLnoXqUqTb772CKqwpW10fDOAZOXehjZ2WJlhCyZP0TAvjIj3wMAPC7/ytlO3N4SIY0p0GeoMOQ78Nejs09+r5SEsaK6LJYri8jJD7v9cYsZEiZdYMU3SN6X/3eABkz9klfQtRZw5Ez/JkXIODK0bCbwhT02XVRQVBl+Eweo8msbmbcdjpFJ+P/h2GMY05REk5nr4BAwendzA/AFgo727vY4ZYLxXAihCEWmAkwh0GSWNKQGPUp1mBrVqE4k767tVkSK3Hl7dz5h/H0M88DAJ576jmETLqVjkdIrWZhUQBMThbmBVKunFlBbKkFPLaDCtJhyIz0nPi1H0RoMUSrUuMKkFSImPDh1PIyzp09CwAYxWOHCoDWqIS0Zqdnm67q6AibihIhQfqDbI9Nhjy2JB45EkYXtQ8PXAXQQlmhM8f2GYW+I5KpBQJSM6Fa7sNnm9iankadCU4sYiESHmTVwkwL5xspFTjkki60Y7iMh2N3r2NLtpPlxxAjKUxu4XeubgJT5O9pNSmZKhlRlCWOaVSmCQQT2dUbC4hmqMoiGw3c7ZOPZm37wSCGZgh8bWoadcnkLHoIj9lKI63Q5Hd7lBYY7NM8vsv2TnseZljn9dz5+xAxkmE8zjFk0rI4TjBhGGxugIzX0MBWQzONsMKkOUHodAB1bpAw4dVwFGOfmaNH49iJcwvWfNNGQ4ChrVGAGrfQZFmGEaOwhqP8GBmbcRU+odj+COFYqI0x8JjwSwmFXNjKmiBheBBRT5VZK08tEtrjk5/6OD784Wfp75RB55BspYiHjmCuVm9g1CKftN+l/wftQ4RcDWotrGB6iqrelUoTFa7YDg962HiNiNbuvPpt1EBzM8vMrIOdbfSHNM9pPIEfWfsJB708zsIroDEa0fePGJ6aDocoWCcyjELWmCTUkq22aWPICQeweHoBKxeIiOqRJ2mOrl69jrfffg0AsLuziZjnv1qNML9IPsyZs2uYn7GQ6gwpvwvbAqW0dPu9QAkHhTAOjZBmOTIm2dNawmf/esT2uN87stKcMMMeOkNWOdiroeKRnxTVIvgVsqsBt2gZEcDnMxGej5wPZ6MFMgsln4yR5rbVRwKMJkhSy/zZLGG1RjhorpQGPqOcfJmDj2RUA9/FB99rvG8AaGnG6QMt5laXfXWl7cHU1DQ+/3lyqL7z3Rdx6xZhrlfXKCj82c9+Hj47LHe37+KLXyRq2dXVZccyuL297dgMR6MRpqaoZP2BD3yAJ0U7COWNGzdx6hQdEgsLi5ifZciAH2Fvhxy0OCuQZZaZp6QXc46YKCVXhdbQ7GQbpZ2BCxi2GGgJbWc/MwgrtLGWzzeRcU+N0dptdgo0eYNYHL9ESa0stDMWve4ukpQOv6PDIwwGtIHXN9YRMjxTKFpctVoNPl8wCAJ4VmDWFIALVj3nHDYrFZyZYYPJweTA9zCI6Z5zIzHDB14jBMa8JAZj7QRuu70BukcUuKwu0OGfJxMYZrIM4aHKB40QAo0mOQ2jQc/BY6RSEJmVyGDYkB+6+RiMY9RZTP5oWMXZM9Sn8pH/5Ffwzd+gRfziOwQ/vLukMH1EG/Lw2luYmuGDeTxAvkFsklr6GPLh/qVvvoHbNynYMyP6/6sjgeknKeA5/4Fn0Wfc/ai/iy733mV5jgGL3ZssQa1Bm/rhx+nekMVocMB5YX4OZprWTz0MEVTpeh/76At45cYBX7uHFm9IOy/zi/M4s0bXazUiDI8IfrV1eBUh1/iX6iEOMnreLNUIApbkGFsacuMYz4wsYHGfnuc59t5ElyyhUgqEDJv963/tbwAAphotvPi7fwgAqDVqmDrD0iZVHw88QEHddH0WK6fIKH3ru98GAAyzFA8/RAHUx559HlsHBJUU9UWEXVrTX//nv423fvP/AgA0hgXOfZ5YzO7/W58DAPw3/8M/wQofnps6Rcz7xfd9t9+7va5jO9zf27aIMMeI2qhGqDpomHD9piSRwE6lkrh5m96FP1NF7Sytt2dfeAEA8MHnfxjbTE/+L3/7f8Yb75LB+dhHP4KPfYqC1bnzp7Hz9ks8p22EjM+3DqWFXtNtSHfuCFNKokhIB8sSAO5/nIz7+dcIQn/rxmUYK/VRZNABM4xCIWFHMhUewoDWY8Kw1jCswpum95aoCgrrNIcVzJ6nfTtTFIjZmI37PdRbdI0z3BcjJgahYaOlgZzZdmsBMFth6Fq1gtU1gvc+cGEVbw/u4mR8f4eVJTGFdjYMpuwMFArw2WmfW1rAhYfJoYqToRNznmqSfb143/2YZQbvYb+HW3coCXLQbqPJNCnklnkAACAASURBVPlBGGLEfTB7d3eQsRd0apXOg5XFBSiGIfW7h2iC+5BkAUtFlxdwQYcSAhO2zbY3TwmgsEyXErAYJ63hnENPCSf6HoljoslWZFkqTLP9gVxwfyeEcbIN0uRIuHUiZ+crL0o5lUmRQLJzNRr2MYrJHiTjGB0Onre3N5Bzq0Do897xPPgM7Y6UQsB2VaKA52j+fSdr5R/rZbJcClJpB9/0lHK+igRcwlhrzUEqJYYt+3FgNeVxTIpGlaziQnouiFRSurOz0MbRz9szNjcawkoxQWLCTmiSxVhaoHXz9IefRehzv+grJBY+7A2wvX4bALA6v4gaP2vc0wjZM83iAQQnBwMAbWZzbbOU1CA3OFuhOT0XNRDx14NJB12G/MeTtHTUjYbh3tIG22hPKjRqDMnzpaN8jMcFun3u2TQaFYaDLi5o5DZZd4yi3/pz1SBAjQMzk2aQlim0FmI8ps8ZDIbImU3cBui+HyBgikyRZxCW1V1JFByYx8UEOSfChQIuXiB/4IFLjwEAnnjkUWRsxw9391BYts+4j9yje4pqFczM0DxZRvmJklAV2oezs4uocduPnoyx9RbxGLz7nRdx9Q1iNQ/NBE89+iAA4NQpCqqOuv1yDUrjAiJqli8LKZnt55WAV3AylBODQmtEDI8MgrCE7h7nxOgNkXKLkvTrWDtNrV733keJqZUzl9CcoWd58403MeTgf+nUKTxwH8loLZ9egtbMTL9zByqj88r3bDLnWOwiSx/BmByKf0f5nkt0KQnce4kKA9evk1/Z6x+iUrH9xoBhibI8lcisDzbRKGALT7aXP0XGsQSUB8EJVBgPwvbcV6uouj5f6ebPhmGmyCEt4/BkAMW07QutCmpL3PsYhJjndTA91XS9gd9rnEBAT8bJOBkn42ScjJNxMk7GyTgZJ+MHZLxvBTBNbHasJC85zi+htXEClkpJrK4SAcfc3AK6PYIGTLFOVmtq2sEYo1odq5cpov7gk09gkQXRb1y/jnWGjL700kvY2mLdL24CFkJg/5BY69Y3NnH+PLFrnrvnLM6doc++98FLKJjE4J3LVxAzFtKKKppjopue9GCrA0IYCM+W6gTy1FY8OdMmpKtWQQKam8FTox281GgDyXo4Qryns5fmCwa25qi1wWhE2YrLV66hz2xakyRGn6ElnW7bNSxjjv4PwxCRZGFLaRC7CoN05CpxrCEV3d+NjRh7+1z14FTrOAEGwjYKS2xyI3TFk8jts0BC+/TuqpUKctY5mzCkNgiqqPLqMdKDYniNARwxBox2DbNSKSeSOmL9mKIoXNY1TXP0+pSxqdbrDs7ywNNP4dQ99G7XmW21v34dey9/iebo1huQMTU06/EAxRHBDhDVsXmFMtnTFQ/tRykD/q8n9H5OP/cwPvMZ0qEKGlVs7xKsYnd3D11mRBQQ6HcJRlfxlctyzcwR7G+q0YLKKfM4FdSRcEP27du38dUv/REA4ENP/xBORQxPyXpY4cpJxLDKs2sXcMQ6L2+++l0gpn3z0KWzmD91FgCQR3Uc9Ok+vvnyFTD5IwzPrQHK9ZqXFSjPD5Bw1ZfI7MrK4EOXaD5mmTGqXq1j5ZEVAMDB3i42LlOGcGVqGtsZVXiO5maxci9lxCwj10/++Kfw2LNP0X3GCc6uUEVg7+YWfv+/I5bPzu9/AzO8Jk794ifw2D+kquO//8N/DwAYTkb4wAsEH4v/7OvYHtsKse8ydr7nIeLPbDbqrnpgM76+Jx0BhhACirOWnvIQcOZ2Z/8I336Vzp0PffJ+jAc0p1GPMovN1jTO3ktQz0988qfxiU/Q+njs8SedntKwP8DSpccBAJOKgO5f5c+0UHLlsui60JBWP7Aw8Fgc1vOVI2DQAGpTdP598LmPAABu37wGjzN3XuSh8Gmv5qRIT5+daqSsTZRzNV/rPmrGQsWrJdtgKBE1KGs/X2+hsUDPGy4KLDEsfDG081XqLRVhgAaLCy8tzmB5idb90vIsWvP0fR8Brr5eQstOxvdnWKZIY8oKGrRxVV9dCMe2V6lP4d4HCUUT1pq4u0lQ4wprga2snnbCxJ3BAAlDgMMwwrlzZGMXFxewvc1sxe02JiwubgnEhqMBXmZG0M1b63jkccrKnzt3DoGFmvoSGWtJmrwoGbMtHaWWjoQkLTTAEDnpKcfSJ6VEEFjWRV7rXuDQNoFXCmHnReay/J4vXfUlT1MEdtLYn5E+IDibjyzGsEtV0p27W0gY5pWmOY5Yp7W9u4ucq7CVC3TmSaEhLPS8MMi5ypgVCYRlATUaYPhaPNFQyj6LcrdjfQSji5LABQrWkSh07rSO4yRxvo1lGVR+hNCzQtSlNi6kKKkZoR0KJVMJJDNxFsdgofIYGd2AUQPdKMTCLNuu+x/F6ll69sefoPd965VX0eEK4N7mdeQxzV1FlhW2LE8xYf9QGomIfYsZPgengxoWVgg5pmpT6PM8Hw5iDBJb5vQdK7vWhUNCzTAEsBZV0WpU3TNZm77XbsOw/7eyNIfFBfIdAj9wxDOWcE+YAsrCB4UEF3RRrYaYP0dIi4fu97HP8Prtu9vYZqbQQyaBi1ONgis5yhSosAZeGCkMmBhtkscO+qc8hUcfIdt86cIlvrcCHWZWl7pAwRXpdNBDYm2iaSBixsd7l2nudNBAOE3PN7+whgEjm/av3sC3vkxon3deeQmtOs3ZE888jtMMnwazhM/MzGNhkXyWrc0UOfscnij19bTRkFbx3JSKAcpWvQPPET1KTzn6XikVYibQafeGzh9Och+NWZrTqMlERK1Z3PfY0zT/MytIGR65srqG2fl5vieBuE/xQfdgB0JyxZ+LlsVk7PZb4ClH8GegIHj+feWX5FLa4Id/mPySI447jg63UDDTrDA+Mv67QpFWtJ0Py5JsPyMIo/eQwNmDR4gSmj7VmnKIoixJ0WfmcZ9tdKQyNAP67EY1wtI8nd/3nFvFEgu+S2H/AdIswNhiu7/HeN8AcMBQveP9fyRLUNLM23664wLrUvmYm6ENYvjQ7Q9HCPhBv/vSy3jjLerdOXP6DM4xvf+jj3wAFy8Q3KzT6eFbL36TJmaKHm5hYRH7DF/pdDrYvkuO6fVrV7HCjmdrZgZvv0PQrcl4hMefp/4ah4k3hSu5Kpk742m0gcgsM1/Zx4PMyhjkOE69PRxaccaO69OTUrqgtFaruaA5Z2M9iSeOeWg0GmF7m+Byuwe7aLcJ+ieEQcZ46a3dDUhevfctnnPzrBiKBaGcUZPGAF4JNegx/HE/ldjslEKw/ICQsCX7vjMc0AaSn9H3PeRgB9kPnTimxwxTSnnwrTERAumxNWKNWVivv0cUu+rmjpyH8ThGjXu8Cm3Q4UO6Vm+gx8yvzeYUFk7Tu51l+OmkfRGvbRJufffG6ygGtD4Cz8eEReFzvwLF7GI/9dEXcO7jf4nun4OBmfkFR9+8c3CAPZZtuLu7h5AhNcYYF2RpVb79DWbnPJrqIQRBX691xuiwc7C5eRenp+hQ2r27ixFDXFQyxPws7YGEWeHi3h527tLfzUxV8egHzgIA7ru4isY0BWStxfNAlQKdp58/wMuvvwMAuMXsoUEYOgmQDRZwtfP4OtMYa1MgtHh0Y/Dpn/kMAGBpgYzun/zxn+KRJ4hq+vy5s7iP4Rav/u4fIonpXs99+GmMV+m5nvsQwRbPnDqNhA9EFfrYeoWC9D/4r38V+XdIQNZrVPDQX6XPe+pv/xKudJhum+UxPvrhj2B/QAYzeu0NqEnJKhnymaGExIU1goMoFGjOz/HPLdRTlv0yUjlYnOcrKHYwX37lXYTMHFxvVpA76i9ar0k6cdSILzz3YSfYvN/pg9tyMTcziwqLv17bnsLVG2QYzjEEo14rYc3ufxBmX7gO4FJk3kAgY7rn8xfJ4C/Nz2Fvm9a/hwAFQ82KcQbFiZFWo4UqG+mUz7MsHUN2KJERBnVE1rCEFUwnLPMgcswIWks138cc94IusGFEVEHlFCUF5s6dxcppciaWZluosCPfm6TYOqA9vH1nF+t393Eyvr/DtvMaDcf0Z4Rx0D8YScyzoP51zett/tR51GdozwsLm1QKyYTWW5zkzkFutmawskJ77tSpJYS83q5fX8eEnZ0K97CJwuBoj2zz+o0N7B/Rmba2tooFdtDCIHT3JKGxdIZaOGygpDzl+oqELpkdPaGRpZT1ytKkbNuwwa6RzmlTKJCz3NRkNMKYk9iBJ6AszE5qeHzG12pklXwlkBV0jh3u72Nvk1pYtrY2YWc1CCsYc/Ky2zkCOOE651gNAVjYpMkg3X733FFQaO18i0mSIC9ifncl7NP1ExeF81uMph5mAMiz3LELSyXhB2STK8zk6wWhez4hpGP6LaCPsVBmrpdJSAN4fE9M5y/zwjnwShrknDjt9/ro9cnxnJubw+LSWQBAq0YJobqn8C638Ux2d9BO6KxXxjgYo1LGdqtgZnER99z/BABg+RIlKYJWw0lDxOMUdzbIT9pu92GMFREXyG3QBOP6vAbsl2VZ4SCixhRod2h9d7sdJyI+PVV3fYT9fscZ+CqLideiEFXLLusJVDgRN780i4ucKKzPLMKKiXV7I8eae2OdfNPt7W10uScP+QiRZ3s6JcJb9Dvtbg8D9jMqSmB1mXznJp+3JklQ8JrwogAZJ/kO9rsYT+hcn52bwwMP8zxeJJ4Dv7XoAjkzKLDNki6X33oVe/z14tIiHniQoKYX77+EhPs0U/ZBg/oUzl8g33M4HmPIwacpCncGCSEQMsTT8yR8TiZZFlwJ361dIaQLFpWnUAwsB8TI9XdqoXDArTDRFPlAjakFzHFPZFSfc/3NYbUBISwscoSDQzp37m7tYm6KGTMZHhxn+TFek/KsFJL2GgAUKFwACwicWyV//kPPfAgA8PZrr2P79jW+goTilozIl6hUObisVOH5VZ4PPmP9UoReIEfGZ1QGQLPPUQsEIo/2TiQ91BcoMF9o0Rm7tLSAs0u0z+aWZl1g3mo2oTjGGI4S9Pt07e3OCIMJXfuH8BD+Q+MEAnoyTsbJOBkn42ScjJNxMk7GyTgZPyDjfSuAlp1GqRJGYKAdREFKCelERo1rNgZKEVfpsgDSZfdWT6+ixhC4b33rO7iXMxb1es01NP/Mpz+HWoOqEKvcaL64uIRrV6nCcHiwjQ5XzdpHe7h6haoiXhA65OWHnnveibVaJIWQZel6NOpgd+8OXaPdRp8rEsYYJ8zqWVHQSYzMwhmFQJN1sDLkOGQmrDzPce3mS/w50kEsqlw1m56ackQQ/X4fd1n3JklGMMy+ND01jTqLqtZrpTiq1bSBkI51Sinl4DDGGFdtq9cqqFjGqsJAoGziBQha59mGZ1O4MrUQwsF0J2kOzU3zeZq5rE2VIakCx9jPpHDsbtAa3M+OQjv9ehgDVGvcoM1VuMGgC8VQFS2kqwyub2y6inK1WoWYY8gaZzihU8RcMerf3YMQ3FC+uOQ0Yfq725gcUCZk6cJZnHmIYHse60IVRjho7O7ePjY2CW5sNBxMIJmMoHn9iMh3FZ3xmB6wN2kjZKHbwWHsss2NmVO4/xH6vNwY3L1LkJh2f4iaoopnMqT7r3rAjEX5zixjfpqbhoWPkMXuZdiEV6Nsz6OPruFezj5OuDKnPAXDGdBut4fdPcqY9jo9PP0kZQW/9OdfxN4RVauk8vBDzxO0YTKka6ytrmF9g6BhnUEXMwydOvfoA/i3v/avAAAHxRiP/ihV/hqs8TfJY5d5vvbHX8fv/8NfpXd1aw+VRSZx+q9+GU/8EukYHR0dItmkfXv/w5SVGvT6+M7XqdofxxN3jkRR5ODa6WSC+vwi/04XIe8By+7mKeWybcfZMDzPd9dYXpjGPJM36TxDjffl/Bzdp5ES+6wvuL2x5/ZemqQYMUFRsxbidb7Xf/c7vw2/oIxo9Yepeb5ZXynhnca4zKf0pGM7NCiRE0KUBFUV1gRbmZtBssu6g9GsIyKQUjhWsbBWg2BygYSz4ikUCnD1NJ6gZijzuVCfQsVW9rMYLa6EzNZbmGP4/YWLlOmcmZ9Dc5kqNtXpBmrMFqe0wcYWZbe//J13cXWX5snLDXyuSp6M7984zmpt975Qx9A6oiQWKZLYMe8p5aHKLM12k6RZ7ipo1XodSwwhq1YrqLKd8zwfMwwnu3DhPKYYOn4Pa2JGoY8Bw0Jz3XXQ+avDIW75VwBQNb9eoc+uNmp47DmCjlsbkENCMzKl0+04wpVB9xC9Ln2djGOHdjBcodcQDopa8X0YZu9N4omzKUUyhmaCDgM4yGuTK4CNRgWKCS6SZIKjfaqkd9s9+Gxj680Gxiy6niQJpG+1V0tNT5+fRUnPEVX5fhWBslpkCinfd17kmCSMxLGgAalQYVSDQAknzwrt0JtZph25l5DSoXMEE3tJ6bnqaV4UrtLhGe18H8CD5PMjhUbEtjlLmDRsMoEWloFcIWC0THc0xMZdOptq9RBVj8hCfDsHOnTay8iF84GSrHDtCH6gXEVrcf4cHnyOUFr1M/fyfObosQ3bO1pHd8jvMy99qjQZO5bpahgh4LP8kNlIjemjxsRufuBjzCiK3GhXPY0nMUYjrnyPJ8Q8CyCwet6edAiTiqdgeTvCIECVCWYarZaDjM7MLeD8JTpHH2Tm2GQUO7RPOhlhwm0u+WSM9U1aY8GffQkvv3PZzXsYWCJBhuUmGqmyTNZw62e/f4RdRsQlUuF+ngOf0UdQAXr8GS9/67u4yiRjw/1drK1RK8fFxx7FMqM8lAImDPUdWGnDMMLcAlUko+ptTDI7j4kjNwSUQ4dUogCFI7QpCZ0cIkcqeBx2eJ5ytlnAoMq2rdVsOMIgySynWmcwhsXOw8Cx+/d7bQxH5LcP2/t47Ttkm2++8zoeuEC2LWSIdlEkEHwNz6tAcLuWKSQyRsIYTzjW4iw3sBRbcwy3vO++S1AZVypNBiHonv3AR1Dlc1X5jnzK7reiSGEx0EVeOB8nDCLH+jpb87A0T9c4tbKGU4v0Hs+u0B6bXphCq8ZC8IHvfPU4HqF9ROv41uYObt0l27zf7iGze/FzH8d/aJxUAE/GyTgZJ+NknIyTcTJOxsk4GSfjB2S8vw6gtHooJdmLVKrUHBOi1NRT0vWYCSEdlbH9eVFopJwpXl1Zwy/+wl8BQJUwjzNvhYEj2vCCEB//5I+/5xokMUAZ/Mm47zR+8jx3umaVSgU17peanp5DnNp+M/q/12sj4T6xfn8Hb7xJvWR7e3sYMwZ5emYGM9x32GItn8DzkHLGKctyLGrqR0tR4IAJXJJJ4nDnQRAi4CxXjSskw34DLZtRVQJnVymzkhdDFNx/MDe3CMk0tNONBiohPcsNTrJLKRy2GkpAS0f8DSmsFIB0ldRQluQ1dh6DIHD9UtDvaVVyNNFhnrsm/TzPSxkQ27wqJfVsgCoYPmdWirw4JrlhUF7aoMbNxjajtH7nNvoseaGiEJJp/DvDEV5/4026Pa1x70XKVrWm6N0XnS0ETfpdFfgQedkH6XEmW2oDzbnP+GgLgrUTxywO0x8OsbFBhENXL19B+4gqco2pKQxi+t1hew8F9wi0WtOw0PCEs7bSDzE9S5nwCBNMsc6lnsTwhRWLyTDD91pIjXnuY0TIRAsoMMcyFpAxjg6ZWrx5CorlHlRYcRklowHP6izVSmIEW+Vt1JvUzwBg0uvhM5/4KADg/EoD/+b3SItv+7CD3R2qwu3epQzh/fc/AI8zYvOnFpHyHqlkGr80918CAL7x0ndQ43dgMQHKGPzxr/8O/fx//C0sC3rWCz/1I3jsr30WALD63GO4cfcOAGDn5joUN59oxq33j9owPZrzqvAcMVCWFdBMduAHIVL+u1QDMfcN2aqfp5QjfgHKjKOUEoLPriceewAb9JoxmeTY2+GKZ5syiIPBEP0x3YdRAVZOU/9Bv3vkSATadzfwv/+L/4l+/2ATTz5I2bneIc3neGHOVd8LU+B4P6AlqTGmcBUcQCAMaS6nuXrwwv0XcIkJaqpBC4LPj7wSYMyIhHgcY8wV4JT3fbS4gKBlNTZ9tHg9nllbwxrL8dTqVZxepXteWJjFmNeN7e1tNRoOSbC7f+SILrq9MS5fp2zzu7cOEAtLyNOAyk56AL/f472kbLZPrNRjNaKA1fHRUE6KAcYAxXvtgYCBz+WNhYV5TM9QdV8pAclrMiWdBADAufMXcGqBzsXpGcpYZyhwlvfZ4vIEipEWwmgoblL0pYEfUsVNegJ73K/j0Au9PnYYLXH7zjqO9qkK1O0cOiRG6AWoMVFCxfa5KeGQMPUwQsSogFxrFEzW5kkJwVUKKeG0Rq2FKkBnDABMt2bQYNKk0fLA6YVFUQU+9/WNkxwFoxAOuYfXSOF0EeEVpQYoFBT3j8nAg7LSVCZBGFoCFyuJIaG5QS4IfIegEoV2xC9+ANdP50npehEtEVWBwtkqXRSO56CgyaZrKwnNZ6eCQIP7pBKm4h92U7fGIASYFwgyK3DYoUN0b/8AC0w8ItgX63eGSMaWGj9zsjhpVsAVBvMMI670LoZ1BEwCMxrROz7Y38HGdUJ6bW4fIGbfTkmFSU4+2njQdzIWUeCXKCeu2Mlco1rn6m41QoU12CYjHyFXBvWximjg+6jwu6uzPxopCZ+vG0lZVnEFICx5jx/BKGuHlfOPZqZoPvX0jOv9Go8G2Nu8Q5MwqWF2lio8mTTwuS/x+sY6DL9bp8OoNCTrCnoKkFyNVdUmak2ugjZbCBhVNzwiX3fj5qv4ypf+BABw+Y23oQQ91xPPPocPPEbIoOnZWYfW6w87iC3xI79wCbie4DSPXV+8p6SrHukiZ24QQHl+6fvbnjeDYyg06WRQPBUgYrszOzONGiMT6s1ZtFrMGJGRbb59bQsd7m3TSiGKmOBHhRgyoVD7YAu3rlMldePOOtgU4sw82cHANygs/4cXwufwR5scgeUyCRSEa6rWrkRWZZt+7twaRh3igIj7PQSK3rMf+cj4POgNEoxZS7pSpzlv1QLMsqxdvdnAPFf3pucWscAIpJXTazi9RFXLWnPK9aoKsBwODApGCh6NxhgwT8bu7j52maRmd7+HowH5MN2RwSR5f43e9w0AHZRJa0htGxiB8lwQJTGmLkUdjw9rkIoMJeuXyNBkeFtrZs6RkOTHtPqMzv/CNYw2UAxNa04tOwOmjzVWKyXcAoxT4OXXXwUAXGGI6ObWujukZ6brGPS4QdpUMT1Lhm9ufs6VV4+4GTlSFURT03wfGvv2kPYkFlZYYDOq4DTDVefn5h08zer2QWsH8wrD8JgBniC1Yo8Z4PGiiqK6a3S/8qffAAB4kA7CoGHcOxKiDN40jHMwhTbuALKwuCKZQFrWQmOOaTyW0DR5LCosdOnI2nCzMMeExQtDwpUoA0SADw/reMC4oLPGMNIoqqDNxCnV1hTCioWXSrQZjnvrzgYWlskR19ww7/W7qM/T94x3E0NuJM72Bo6YJh0UiNhRSIXAPgvYDxnicuX6TVx+h4zMcDBCiwW9s1yj197haxyiyULZkMoFtlPzzLikFC49SLAV5AX2tunvcuXjnRt07cVqBJ+NY6vWxAyz4g5i2rCFDKBYXL0oEmck/bDmYFuF1lCWwTMIYfhUKmHWCoa/9kSI3S06GDZu3sD9a/R595+ex2d/jILB3/jX/yd+/Td/CwDw7AeJWevJJ5/C6bVlflWajDcAneVYZzjXsz//OYScwJgwydAf//rvQNyiZ/n0Zz6N8AIFF5c+/3H3vq9cvYL9XTo0Q89Hl+GUbzEkZXdjy7Flpjpz8CUFAcVOWT9NcMjkMONJiiFDHpSDR0qnW6q1dmteeZ7TGfUCD4GhgzmTNbz1OhFRvfiVrwMARu2uI4jy6g386I//JADg9PkLmOH18eZX/gQTNgCkXUvnxD3nqVE+ikoIehD40JycMFI4xjAhZalHBOAZZoat8HytfuTH8QaTTaxfvuKcw34eI5yh+5ifn0PB0JG1+wgSfOa+S5hmds5ma8oxkUVRiErku/nYZ4a6797ews0N2n93NiiIW5hfQIX34eFhFxkb9kx7mHCSR0XT8BjmM8oKyPwEAvr9HpaNj5gay0DOnb+iTL4Zox0RmRLH7QHbC6OdUa+GFUhOLOVFqVuW5zkEB3XVahWRI5DhpCIUZvhswHQLFauZZkpSBZNrF1BKAVy/eQcAcLhHa3CvfYCUk295YSD4XAzqBlbNyg8Eosjax8h9tuE9kisBUWfx62oVAQcEtWqIJmteVqMqhIWQ8/03GiEihnBVohZ4uyBNJsi4vUF6HkL7XDBImTDld/6MzhHPk47RD0IhZuhfXhRI2AfwUuVaSXSeuvso9fcKKEOfVxjAt1x0hXGaheS2WXF6CU+W5DZ0b/oYGZB29gImg7SBufKcbqqIpHOG60zKNo5jZMzQ6Hk+wO8tTlMYZmvsdLvoM+tixWkrFw76mhck3g4AkzxxzNE6kDCccMihsbNDgX7m09l869Y1XL9K/tok1og4oQ8U0FwgKLJSc7nihQg4UD6zSjYsUh7mmcisXq9gyALx/XYbImdSrSxxCcZACVSYCTnyy8ClnHOFOvuNjUoI39pjU8DnJL30fBR5Sc7DE4KCA4rJaIj1W5TgyPp9LDLZy/m1FfQeIRbVIs8w5j3gLuFLVLh1xA8DTDPhyqVLF5Em5G/OLS7B57W5t0uJzctvvYHbNwl+XZ1p4umnqe3jieeeR433Tn93Fx32teJxjJjX+v4R2fz9Xhdd9tGSOHFJAel5LsERxwkO22RTkrxAxC1MtmhktIEl2PeUKNe8EogY0riwvIgpTjy1D/u4dvmyuzYAvHP9Bq7doZapSq2BC/cSWeTahYvuPPM96Xy+IAox5CCsz9qRi/MNx0aqhEDASQGjlTuXvLDqWqzyLMMeE1vFMV1rcWkBg7OUTD3ci2Bl1lvNBsBnTZxqeipwwQAAIABJREFUhJxQsUQtp1cWUOXzxfc91DiWCCvVktjIl9jr0LsY7x9iMKB30R7SfhsMJ0h4bUzS1BEY9UaZ0500nocEdG4mUQOZHuL9xgkE9GScjJNxMk7GyTgZJ+NknIyTcTJ+QMb76wDmttFcO7iWMAbCZhylcCUhqkCVX9thK0BK+q42JAAnnVDk2mXMpZQuw1AUueO6lsegXTabkqfmPY3w9v5Ih89m1STC05QFevhpqnQ88OTjUJamVnmMi+DStC3lK+kqEmPhfsHR6Idh6PR5AmMQcqrG8zx3rwZAwtnz3MEmBVLLXzsuqWclFCRTs8NIV1Y2qDnSEguDqAQ4BnWTrlInpXKf7XnqGN2tOPY+yvdm/06Y98p82Hcrj+nGCc4t2+dyv2uhJapsrNYGKESpJWSrVIUuqBEWcFnUpaVFDDn71G0fYWbGdl9LJyWRpimGXDHSA8rkqKMDgMk8MLuKnR7NzeERcHaRv19poNHgSlJrHneuU5b2iOmgX3nnFtptut784jImXPHKJj0krENXb0whYDiIVOX8PvgoVf2CKEKDM2m+AuBTVqdIDKYX6Ou8O0CyR9WVfDLG1bffojnwaI56kz7WOcvUqIe45xxVkmqNuiPZyQZ91CyJQ62G3Fbd3f8CMWejr16/iTvXKKOqdI7BEWUDEWdYmaI19vDFC/DOE2mJJUn4R//4H2N5heCpT3/wKdx3Lz3j6eVTCHitX1w7g42rpA/4ype/DABYnp/HUFJGUs/M4qGfoIb+bpZg6yZBbCeDIaosv3B3YxO3btI12nZesszRN0NKt3aNMUgZ8miKHB3O5I3HE0d9bskfYMpz4vh6lkJA2t8RGZZn6X1987V3kEzo3Z67j0ioRqMRdm/RvY0GHbz4pwSfebzfB1KqeKrOOp597Axd4/UbuLFO9/0Uw5emp4G+JS0oBArOHNZqERQ39/u+Ku8bBoNrlO3+0+8SWiFdvoBnPkuyJQe//2/RZ6Keh89fxANPPknzm0zQY1h7a5aqfjL0sdsmOMw7t27j6nWSCen0eo4kQQQRDnq0nw6HMbxgju+JqjeNbg/VCu3Z0A9LqLjnIbdU+0ZDOGr5AtXi+KlwMr4fIx5bAqiy8o1jCA6qgluIk7b8a4DxXGnBVubyvEChbeuCctUUJRTywsogFQ6m5kmDhNs9ciZMyHWOwlbKhMAxtn5nM5SvHAzaC4AGI240k5hMrayixrT1Ua2CgCWQJsnY+Qu+p1w1wVbfpSng9HwLwLdncuDB42pO6Aduz0npuQqN72xjKW81hkLqNM4EpGJIN4STSZIAogrN+2HbZtlTYI5+Xg1CaD43M7hLwxiJHBa1EIKlQZ3d1UbAsqgVonD+SaYFFMPrjWcg2K8qtAC4ZcQyu4nCOFKfQFUcmYvOiECGvq/Q5DLnJEmhwLBahmNWamOnAz1OYngMBddZhoyRON2ujzaTZs1YKH7oo7VEla39UYweayj2B2OEXO2JgiqqDHsTvkG/TYiKfkH24NbNW2gzYRmU7xAQKDQKrt6FnkKVYb+BL53+7T1nCEJXDUOHwIoCD60mQQanqwEGPSYN6fYdrFZBOs0/K9U1yTRCacmFFKqM3qpEFSRMBpTGQ9QbdHYaqSDYt3SIa50hZp3f7fU72FoneRGpgZArsPWZKawt03zsLS1gzLbcEg612z1UD6nCtrC0gGZtiu/Th7LSTiLA/h6Tf+ySPzHJCzz8+AcBAA8+8jguPkSka1EYYO8Okd5tbdzBLpPhTSYxJryHt/fonQwnE4yZWAfHWheKInfPOEkNcvYjMq1RK6wkBP+ZEZCmJENzhIwoSQertcj5w2+98y6uXCXbZeHe+3u76B3Qezs82kXCMgqjcR/LXElVUjjyxqBeQcJrts3zPzdbh5WRnMQxCljtauMgzk3PAAxP1gZ4+bvfds8LAPXIw+oZaklaPLWKOtvVqXrdkffkQkKy32iJhUbjEa4zIU+73UWSW3ZEuPNAFxlSlKSOvQETKFnfL6xA8s+dtioAFUWOSEvmApmNoXwP4Grx9xrvGwDaCSyKsgdMCulgiUDZRwBTasgdD9icQZKFwwnTr1s2xsz11o3HY4y4B6ffa0OykfEY80yQU8atS98xRUZRhFaLDi4hNaQoxVMv3PsM/y19RyrjYF4ojG2ToB4de2+AO7AzVXrbpV9ZwiZRFJiwcTKFcWKQdI+WAdUGacJh8M0x6KUoJKTdTcJA8gGr45Fjtbx+hQSnQ+W5RU6OZMnuZnvy1HGImTje6yHc94x9F8ecBqN1WeJH+bxal319Ds5jjPtelmfu0Mzz3H2t8wIZO/BFUeChT3/avgQAQHNqCius7XZnfQP7fOh4fkCabACm61XEtuw9oAPO741Rr9IheP6JR9D+LsEF1NEh+vzcpy8sY/40BTyj7hFGXTqgOhPGu/c6pB8FYNDvIkvoM3Q2RpP7TKv1FjRDgKUsqazynK6ligL9AQsiS0AzdKc1MwXBvTFJI8IOl/XT3gE8NrqWeXWcpdAsCHucFStPxhgy811zehaKe1nNJIIVUbKiw0qG6LBuzhe/+EcY9+g+nv/AGoYM9+t3xpieoXe3MDuNt1n4/i997vMAgCtXLuO110k/8+tf/yp+4ed+nu5JKjRq3Hc7GmGe99kDFylAvP7OOzj3BAnXPvCBJ3BnhwzLzsEeNrco+ExGMRLGq3/na99Ewe+2zlDgtChQY2hs1u047SsppXMetTY44t4TAYkuz7vVCSzywq07Y4yDhhZF7tg3hSihOZfffhOvvE5wnEceJzjJzNws2ny9/5u9N4u1LDvPw7611p7OfMeaq7p6YnezyWaLzXkSSVuxYMlObMkBYiURLATIQxIkQQA/BBASBAgQxAEU5M1IYMABEiSRBUQeIlmRJdEkmy2SItlkN7uGrrlu1a07n3kPa8jD+te/9m022y8xX3jWS926dWqfvddew7/+/xsW0xMc7fkD7D/73XdxcdPf66/9m1/EpcseAqKFwh/9ie+z3/t9D9H+N77yKRwv/WdPDg9gDr1n0PowZ/5kd2MDVR3eocA//ppXLvsuKZjdzL6DA/fLAIDRU09jSEFIsr6Fe2P/3Hd3dvADel/3KagYT8ZoCJTSCIlZFYxwcwzW/eG+GK6jt+Z/Hmyeh0r8+ywSP1eyog+R+E3DCXkaSi6CYqFhk3GfRFsZwf+s25z8uJRM0aUgScmEE3XGAkIEtUkLUIK0kSnvGYH/VNclNKn7CRjeO6q6wnJJfmxVyZwra6OitKCDV92UPFayRKHXCzD/DD06VPQGfSRJNH3f2PbjcI24UBApX1clkqPojh4hoXWx6OSsQRDWbwmLtA3bM4EyooFwn84heCJLOOYAhu1f64bh2kbXSAlWaXUNxXKHhg8dDpaTI4cnPsDcffQAtzO/pmwMuugTJHbQ7/G6L9MkHogtWIkwQPyNQ7wP20CFYBQSIpyqpUMSoLlO8H4QPJcTZ+GCMTcULOkLmHqBJSVuhAXyJPIfP/3xL/j/S33b7w1QUZJheXKCaXlMfWqR0CEsExYP6PBuBv59j/Ic6xd9gqysLe7u+T17v5qhV/j72+700CPjaqFSPN73e9cDUhbeuf8QizIa0yviOGaJ4tgnVwnHLU1lYJy/1+EwJGQVwEnrOKZ7vQ4c3f/xwSFKWiMTJzGng2bg4FVVhEBnqeCEA5IMmmK+ZjlDTUqiWU9CEtcz7GG6rLFHXPM7t+6gnPvvHvZ7aIjHPjuewNX+84OiwBHt2cekS3B8fACQP9+HXnwJH/2wh4t2ekPIyvfBk/0jaOMPzY4OtYPNTZy/5A/ET115nosW+zsP8eb3vgsAePfHb2FMe2mjNaaU/A73kHYKHud5lmOxpIQswJxgWAcZ9CKMQRX6NChgK8VxtnQOImQ1RORMwlkcUHL+xzdu495DH5+8/KKPMy5ffRr5yN/n4fgE06mPqa69/QMc7/pE0vbZbZ5b3aKHahk+T561+10Y0O+O5ljSadAJhwHxD69cvIpyRrBfDXztX3p6yIJ4dZcvb2P7rIcZ93s9LEkrYWf3kH2NDydTTBd+fCxIofTo8ABHBEldagmVZXyNLiUQOnkPkg6UyHIYolz0yAdw2BsBpPSbdgrmpCJRXIDRWrPaKCz4DPXT2goCumqrtmqrtmqrtmqrtmqrtmqr9nPSPhgCStki5yJx3Fp7qooVIGRNXTE0QyWKCdoBQWRsdUpsZEGVvuVyioYqgMtlyaXW2eQEXYKDBDigbXnawAmMCRImIPD001cBAOcvbLECphApjAqQGKqWQISCAKQUUDJ6djHMseWpx7Y5NioLGmv5WSwMLKkTOjiGlyqlACZGR0JwqLBZ5wLCFULLFtzSa/34C2rO2mjyPjrcO0AT1P/qJWdlBQRSqpTKVjVTCsnvrlWsZREBZy2yQFx3ju9PSXUaTkQVQxngXm3lUGchuaLrGLoojeNMWLsqo5KU/+8GwYAa7XDnrq+4TGczvvyN6ze5n86s+wrKZtaF1X78pOubuEAqoUePJhhd8HCA7eevQtf+M+PJHAuCOB1RtW08mcG6QPYtIcjzcGNthOFwnfsjKMYuFwuuXCZpyBQukJIoUa/o4WDPVzD3dnfQ7ftMjtI5xpRVKxqN/prPNB0u/didL5bICQckIeEIclUvTmAICjI52MXmls+S95ZLZH2fPQ2QpmY2x8GuJ0gLUeOll/xndXmEt254SOONnX08/6zPzB7MK/zeH/4LAMCP3/LQ2N3Hj1h1Ks9z/Mpf9wIow14fJRGxh4MBDD3L9R96AZcv/+ovoyAI4veu/QDLUBGtLdYoG3r/+Am+/00PpVjuHWG0uU59TT6HRYaHJCagnWX1sFNjCQ5dEihwFugN/FgI88JofUqdOIw1rTXBwn1/XXvbV8vu33mIOXkezanS98KFDm6SOM/VCyOMRv7+37nxCI/2/WfvPjrBy6940ZXPvvYK3nzbVzzPnvdZwbNnL8A+8dnTO4+uY/exr+IuNNDP/ECe2Q5Dwediie/s+z57h9TzSrnEP/qnfwjAixkEhePyZAoRVJJVgjkJxQTlZOMABP+kIkW+5gV5hmfPoD8itdrOEIKyyUnaQ4fI6jlVWBQsQFBtIZKYQTSOqz2JBGcqrbBY5RF/9k3TuqRFDdDaZa1BRRCucrFARftqVdVIaC3v9YZYJ0GjAMes6wp1TagTXbO6dlVVqEgIZFEvGd0ipWAvV1DVxLSqbSUcjg59Bl8lEmtUCds6ex79AQlmCIFt2VIY9ReBSqLgWyi8JUKxwrjWBi6IPoT9TAj2BXNSRO88Z+DqIJAiQds0rHQIhchA9YBIIETwPRb8HcJlEEGhU6asCGqtZOvb7TO+ynJ4dIRHj3y15/69JcO08jyGWU5K9iC0rmHxrnBvwhlY2quMMZGWIqJqsrGO1Y+VELEa1QREiERKFUAHi4biOCkARXtNo2sYqvp21kb41Mc+T++AIKx5DkeqhYlUDAmcTo4wPvGVuoPHGhOiLzxzxr/jZy5eZDGg4sxlqG1STBzPsbblx9361afQ2/R9Nm3mePTQX+8WCVEdHk28gjJ8mFFQlbGbD1nsQgiBOviqLSsI7X8/2AgVpQqGnqWqTYy1DDAjRNGyalCbEI8ZqJJiXBqPZdUAtLZOugvMqXqXjKdMSzncfYS8+wAAsHHuHEYbfh9IaF2dz8e4f90LsTy8fZtFbMZ1hQlBUYUDFhRHL2YT7JX+nu7e8dfdebSPvOvX7LMXLqFPInTbGxdQV7QHGMt+cgOaY93uAAVVnp3RONj1/XvjrbfwF9/7CwDA8ZPHjKRzzqAiWkFYLzpZwnDubp5hFuDjQpIAFVGEGD5rmTrGKAGneXw7KfgdQmiEULCqGuzv+733yeEeFgFaTnN84/w2FIk7aWFRByElY/Bkn0TZpMP2to99RqMBJlTpPaHqnXt0zBXfR7uHODrxscx8uUSHxtiLz5/AUuVtUjWYUSUviNwsyhpP9vx7gxQ4OgoVvkOUtFY2VntvVXixIsCvPx2CIef9Ar0Qw3U7SClWFGkHovDVPpfm6GT+8x2qTqqsy9Bu5yRMiJJbe7MSCjINMHXLaIKf1j7YBoKC/vlszFjvydExTxBtGhzT73VdQ9EirZTDq7/gOUYXiFdkbH3qAJIqf2OjQQ/C+YdWiWoplDmgdSDzNyQ4IKmbGj980wevOzu7uH/Pv6jNzRxwwYxVwREMRgZJVQE+vIj4FXTssvxz0NFiuXZrW7w6tA5A1ktuwx8AA5LTAi2obICqSIgAR3MOLvAWpIAIss4CjDtPVYIuYXv/+r//7wIADvb2+aAxOz7EgqB11WyOZkYm0GXlJbgASGOjRC89awoHwxufQJYyEYENZhOpIvxORCXTAIdpP6t0AiZ8uki9VjE8tzFkAGSeoaYdM00Dl7KAoGeVaQGV+kl469YtHleT2Qx/+if+sHLurD/cvfLis1jr0OK+OMHhke+D8bCPASl57Y6nELS6HE4rPKTF5T7BTKzMMSTYikwznD3nVUW3trbZNmA5n6Khfqy1Bujg/RQdeCbzKW+iRipWIN2bjDGnhTQ3HUharEaDnA8sJ5MA7QHzb4yp2EB52jnB4yf+55u3b+LDL3koxFPPPY9zFz1stkOqUmVpceNH3jZjMT/BdOn7cTo9wGziF7/DeQPx0G8o8+WSY/adXX+AWS4WPD/VYo7X3/gWAOBzn/oMLhMvsdvr4MG7fiH86Gc9F02XJe7d8YcqYS30zH/f7R9ewy1SPJuOx0gI4vLh5z4EG2CwNDaskDihsbuoayY3ykRBu6Bs56GiAFAua4xn/lnAiRPBENA2BD3LcuZK1nWDb7/xpn9HexOGim+OfGDyhc9/AtPaX+9br7+FL1zy4+0Tr/Tw9W97Vdd/+v+8jvmckgKVQEm2IlPi/f2fv/enOAjjcTaHsyTRrnKMLc05Z1Eu/MbR9DWKdR8M1dveRDgB4LT/juNKo6zDZge4KiQfaiiS1E9HfjPpZB1WSZZ5DkEy6ElvHaLwz6iKHgqaF4XKkND8E7QGWCk5cWVUVCx0TgD0LEtdA5V/RpEIZAFbt2o/s1ZSAsk6gcmJX1cn0xPMZmRsPZ+dskkK29VouI76olcODLY6Skl06DQl8xxJSlYNELwWWhdVnB3AAXowTXbWwhBEeD6dMIerqkpMyZbGQWMx7/N3vkBwyoIVcWULWuqTqwBZHYQEhVS8NxtOKIOVg611TLmwsJyA9Pwl/6OygCIoYVAbF3CsRWAg0GK5QJJipUgEQwKdswg03i9++S8BAJ59/kXcvesh34dHE5REf0A9R1mGvU9CZgS9bCqmvIg6cA4rKPqOxjgUAealNJ8Sy9ogDSreqWN47LKk9c9KPownUqFp6PDf7SHtkfqmNljM/RhKkoITuCltAipNUZCK5mg0wJAOdQe7u3hAfb338D7eOfAHwL0d3xkPHz3B1UtXAQBZp4Bd94mnrQ9lGKz7cWWH69inAPlg7xAPSNH58QFRJbTlgLzfydnOZjgYwtGabRuLhrLzjdHcZ0HlVDdNTGSUNaolUVTKGlM6EMyXc6axSKFC2p3jGmMNBP37YjbH4x3PrT86PoFtIv0FdEgfrW3gIsFfR5s+7j2eHOLNNzzc8t7ODnJKCjgbFTUTIZmvC2UxJwrQhCC4i1pjTGPpRzdu4/wPPGf86aeXbD21fe4c1si6bGPd/5k4hynBKvf2nuCQrAKuX7/BvD+nDQbEpy+ylAsiIZ7L+gM0xKFTMmELKit9ghAgalGA4xoL0LvlftQNJ3ITKaOlnOwgofh2f/8Qt+/55ImuK2Qhu0Pzs16WsHVQbE1xjuy3AIVHpCVwdHTMRRBtLKqQACD6z9Gkgg5nmmWDkpKw00rgkGyeZtUNFER5mSyWGFASpKwDNcpiHgowusGE1ttKGwhaNztFgjSoEtPcU0qgGPq51+l0WDk27/fQoWR2XmRwdOhLsh5Sul6vR+txphCyYsI6XouEBFJam7V1HMM756DaZ5b3aavU7aqt2qqt2qqt2qqt2qqt2qqt2s9J+8AKYDnzWYLbN6/hIQk6OGtbYiIWLFNlHUO3lLRotK+0zOeBiBsFRhbzeRR6EAIzyv4PBgMWaRBCMPSJhUmERGCTNrqGSqnqt9Vhcvn9BzdOQR4/EjJ8IWMm5Skpy7YYiuPMYsvjrqV+yjot7VO1NRA2ik84qvA4pRgy6AJsRESFQ+Ecq5wKK7haKFxU+JLxIzj/jK/CXHruWYQHaJoShkra5WyOkvxC6mWJmrx6dFnBEqSrIoXA5ZNDKPZNVG0052nlzyAOIwRX9VwQf8gSyCDzlKRQBNcdbG+gQ4IeulVNkGkCe9NXhFz4fypjKNFAJWzu2el28fCBr/Ac7O/hEaUp7lCGaO/gEOe3PXS01yvQUAZ8OrO4u+czOY8nc5SEXXjw+ADThf/M5pavtpw5s80GnGtr6yjI+8whjmOhJCxCZsvBEBRyIPyzlvUYuyTUYvoFQ4iKToZu6qtzGFso8ocZdoB75LtGglZIih5KUrTqFhLv3vHZ8sXkDqsuyk6OyZJ8eXbexsmBh3VaBCWoER7c9pnnx3snuPXAZ/qubHZxht7L05cv4Oplf0+Pdu6h0/dZrC9+yZP/v/vt76CiLOPx0RF++7d/21/j4mV89LVXAQCf/cyn8cVPvub7lJTbHu88xM4t/66Ojo6wJFjFyeM97D3xfdMfDNAd+oxXBcv+R92Rz3y9/e51HE9DJk2jomxhoQqGqQkgZt4qjTyoW3FFWrQ8MUXM5jY1k6V/9KMbOKRs5ic+/TJu3fLj6cozl+hddPC5T3uD3CwpcPGCzzJevrCNGcEzv/atH+Gf/ZGHsxbFAIbG2PfeuuvfFRKs94JynMB0EcQkZnDkyeogIFoQJ0Uk8LWz/j5Onhxw9ldbwDqCbOYdJH2ah9Bs2pwMfMY3H60jL3w/p0UHCanMpXmBLCVRjqxAlxTKUikhCRImqHLohOJ531gTUQDWRe/QNGPfNWUNDLshrdrPqs2mURE3oAbmizGrZVrrTu1hYV0/OW4AS5WFqa8KD4cDNjqGszBBvdo2PL+MaVhooDbGQz4B3o+zLGMI+aJcsHCXMQ1q+n+VqRk+KGU0kg7jKkkUQ8+cELBk4u6rj3R7ziFImjYu0BWcV88EQR55T29YEU+0RMuKRMLKIIASYUlhTjpnuVqYCBX3A4BpA1JGZeynrj4DALhw4Sm8+srH/LPW3uMOAHRVwVLFXEpBktFAUzWoFh5RUS/9ulTrCpYgm3U5Z/ipSLmoh7opffwAQGUKCc3dAAUXFrw3K9Vhs/tO0UGXqkT9YohF48fQ+PgEMkDFgq+q8HEcQCbpSVA7HGFIatL9Ise9O+/6axDipbr3BE+O/fhK++tRRRgKBwSNr+7tYkmfny/mWJL5bTBXX19fx3kS2hitraGXxzUqqHZWqFlExWoDhUAvCRWSuB+kSsBR1VU5heWSfp8ANmCBtYGl+wi/6qaSx1VVV9h55CtNRaeDnMq/RZogTagye3SAxwSlPiThl/2DCZ4Q8kZoB61ImAsGWYil0ow9/JI0wbuEIKkIjVJrr8YJAN97823sPvbw6q3NM3j5w17B+ktf/ipeeN5TEyqqBC+PDnHruhcPfLzzADXFQJPxCRoSc2nqimPWjUGBnCqvLIJoHcZkwL5cNgxBVELyvIUQPI+ckSFk5jhcCgVB0G4lEzaylyLFCZm4X795Fw8f+phiMFrHGaqKXX3Oo50unz/H1d9SeyN3AKgbjde/+R1/jRvX2e9RAmhovgcfYN3oqFZvRShUwjnJz3V8MkNRhvhao0MosSz0wWKKOcHvnTNwRDHrDQZwVBHVQhA1Asg70YuXzzZFAkdKy2lRIB8QNDTvQJDvscoKhtjyecaZqOgvBRyvV4K92aUTvDY4Bz7T/LT2gQfARw99qdk2xzi3TTylJBrPQkR7CI9Pov8oDJ7s+WB/7+BO+CXDKbXWfKiDiNKye4fRBsK1+GihOWtP8fSChCwg0JCiz7KOUE3nLAKQMSzuYRGlD7R4idFM3tp4AEyCFQUcP1/b7NwZDZh4SAwDTCgFxojQRHdS8mYiIfg7JMAwjlwpFPRzAscqoJqgo05GflNajFCs+Z/XLqVcTrdO8MFWCoGEoDkgRTc7nsNWBA9SjnkQUkQuolLRigFSQgWceDjASgnHmyjgAjwolR5GBg/TCHBhawxO3vVcq8ArEcogowUnzzIUHf8dvdEQ/aE/rDx+tIFzl7zq4s4DD1f87hvfwoyghs8++xw2SEWut7GGGfGibj3exT06MPb6Azz7rOcJXiD407MfegFDOpQYY1GR6ljdlKwQVzeaNwXdGD5IKyrND0cDzK2/jyyzKAhe0+3lSIV/Fyd6iqYiSO96gp4JfAZSi3UplnP/2SrLcEjBXN4HLl3xHK61zgVcOEdS06bEnPgHJ2P/2Z1H1/DOHQ9PWdohQAHB0ckSXToKXTqzjmfO+D4927kM9Rf+BBpsOtI0hab5lhc5ClqIHu0+xpM/9BtfdTLGgFbNx2Q2e/3d6wAdbPoqbmQbly+gofFz4dIlLImPtHd4gN1bflM6T7y52hrskhJcWZW8QRhjoi2Mc5AUjMo0hSTIYhh3EkAqfd+esmNxFgvqr9e/9RfY3PaHur/xN38Jc+IQD0iNtNvtoEPctssXvgRNAaipLS5d9uOm+P41ZNQ3FgXosViFMBGKuT8SAs74757VJasTCjjmIQsHdKUfQxdzf7FuX6MOBzKtmK+rpEKHxlhaKCwoISEILpMVGknh71klDokltT5Topdo/llWLYgsraFBpc05GxM/AG9CmYr/55TNhjVIixKr9rNt42M/95eLBWakQmhMHZWnVVSCdhZ8aHLOYDamQJxU6fZ2oxx7U2uUgV9oNCfArHOwZIoo23tUAAAgAElEQVRtrGVeUEjg5WnCEEoHx8bzaRKpBDJRnDxWSgAU0AXqhYT0wRj8ONQUUBlXQetwCIscwBAAaR2pGk5I2KDmaAVD7oEYCAqRQpDKpwnQO2dZTdPDSIOyrYVwkZYSYP4WErIlFx/6qEeQvJ5LOGGcSQsQt1uCz9RQykHTAiJob3HOMdRzWS7YtDxNFXM4msZAEu8zSVOG3wVOnHKWYwEnEg/Lgw/E047fu1SWQkq/r05nJ9j9keepiaAwLVxrzDiG4HZ7A1zt+gPvqDvC+rpPxO4/9Bz0x4928PiYFDynrRjNlZhTUnq+OOHnhpPob/hExJULVwEAL33oeVwgA21tGpS011dVwwKH2thAV4QTEoohwnzZMLyQQCKlw6XsJOiQXP+o34GhQ2Q9q6GDEjgdvIQx0CXFbsagJvh9ltfoE/1ia2PI+4dzFgtKwh8TN/Lg8AQp7cF5kTH8N8166HTp0Jd1kFOSWGUJDB0Auz06dFuNvO7QO4z2bI8e7aLf9Qel6zduBiowQPSBh3ce4A5ZLrmmxjbBcfvdLs6d8RDVyWQMp5d0/w6OYmB6VOxPJtjZ8XvzeDKHowN2EMcHPH9ScDrHQRPvbUKq3brRqKmfhVLIM+Im18e4ccfHdO/cfBdTgklvnj2Li0/5/fbSZR8jDAZDpjIVNkFKcWNjLPpDMj5vLBrSKHBCQoTYgf5sas1Hl1SmnBhxFtAs9tFgsSTtExiM1gMElPbgE2A5Cwczix78OJBKQVGSpAFQ2pAY8++13++xcrZTjvfYottDh+akTBPYoJORIcI3w1oFE6HOIuH13Trb2pNjAtcYy9Dbn9ZWENBVW7VVW7VVW7VVW7VVW7VVW7Wfk/aBFcCT47v+BxEJh0a3PPCEZSiQLzdGoRVdWv6//t8dn75l219K2uh9ZizaSqHyvQRG6Ria4Q1mo0Rn8NMRUkRTVWuhqeoVynen4JttKKiImUjdqjyE/KVtZb3D8wJeNY0N64Xg68v3+bCD4BSVdZGsqZRkJaYU8aVIa6EJBqvtFt1nVCvNZMyYNsbB2ODl5Fg9VArJ1UBBmQbR7cMSQToTAh02lRTcKZK11NCyyEXsW3i1I8BnyvgZrYsiQbVlTxLhBFcRlQj3bKLKo1SQ9D5lorBBGSonEywJ+jcgsQ5rLe7c9JDHJ7s7OCLlw0SpmOWVwNPP+arfuQsXcfVpL7AR1DQ3t7bRkEqorit+QCUT1NTns+kcYHJ2imPKuh+S4Eev08HGhn8v88WEM06iaZAS9HL74gDqQlBWK7HxghebuVoH9coFww9qDUwnvip5oTfAgPy9kADpJkEDZA+Z9f0woPGDtQOUBJNZzAfYGnnBmMnJHuZLn2EbrK3h3iMPHVkfjbj6PCEo2XQ+58y/A/Dqqx72ORwMcfWSz8Z94bXXcPyIKo3krbMxWkeXYIeqsigISpatDXDx6lUAwPFsyuJNW+t9TIlwfWvHQ2Nu3bnNcJ28KFh4p40AcAAkwSa0diz48g9+9/f9B7SNXp9Sxgy4c1iWPiv41q07GBDcsv6//zkr6bkWZCLOc8dZzsVC4x2Ci1aJRkVjrG7msEHgiTLnNQTm45glDcJYDSwsww+i0q9LgC88RybAgwB524QIqoCtOen/FrAeFiasMkEYAaJV+WxV6YSEDIRxlKfWsoAmCM3aNoIifk601jZrDeCC+nCCy2fWsWo/29aQt1tjKl9NA5AmOe+lQoD3T4c0+u42mverYLCtlw1MmHPOAlylS5AGuL4EXBAxEtE/L0BqjHP+2vDCGAF6VFWIa6tQ0as2UUhIFMwpWueM9WIn8GgaGSp81sDReJMQXI0Ka5hz5pSfr279HiaMe3B1o2mianUiI+0j0DCkFYAMFUVwRU46wagXIQV08EUkMRUnuzAEH0vSJYtPOTj2D1TK8L5vnUGnE+DYLaVzWvcG3QySnluJVmXeOQhEdWRL+0codkoR45YGin9vXXwWOMU+bf0igaAFh6sUxiHq8bkoBoTW/Q17uHT5KgAgIwE3LTMWDFyUERVmrYUkFEJ/tI6MxkGnk+LCJS+c8qEX/B596dJTUOSpe3iwDyuCMInlGE0owbBPf18ElyRgjWipjksHjkMTKdBZ9/e6sTHiOWIrAxOEeKgSVZUVShKMscawP+BofYQtEshZX1tDTpDeqtIQVCkK6vNSSfSHpAiepSwCk3VSFBSPJS0P5yZNMCKl8JTGxlmcBVsQyhxZFuDaClvbntKSd7qYkudcTfDryWIMVfhr9Nb62KDYZ6PfhaMxs1xM2C+vqZYoKdaak7jJyWSCGQkVmRb8GkJxNdCHf0GFMtJ+bt3zVeH5yYRjOOskim6X+tdiZ8/TVeZliZTmSG+Qo0PKuUdHPuZ6vPsICxJfW5QaKamsdrtdRkCkuULVxNiyofcY5hvg+PaNAvtLamgwla0V9wqZ4/wZXzXtBc/i6jwqUl5VIn5WGwcXvEWFQijGBvRflkX0g26JTAopWIzOKskqw9I5qCAuyduw4HGuEKl4Eh7tAADCaa7iJ9YiDbyTn9KEe8/Bpt0s1VzLqozYUyEjBxCtA5U4HTDwxA88hJZSl5ASqlUyDphVYSwHgv5fwqHN/122OHROiFPqoODNzvHdFXmB75PhpX6fUmjgBITniNzB+HxtSOr7/V4IEY11W4t32/YgtLY6oRDRaD1sJOEz7c+F6/3RH3slzGxwGR0yXv721/4Qw4uk1HVmgIImjTDWQ0bgA7wsKJ6FSV8DimZCmilUpEJZpEA3i9jqAyqnLxqH3PlFp5fR4O8YVKTR65JdyNxPTqUHuPE9z3OzG4+x+Yy/XlGdxz/8H/+cnpcChaaBzAhTniX8Pqy1LJ+bZSkUgfKDHYFAzht3otSpd6HrsOErXlCctVGxVIZDhD41XtuHDR5j1nDAXUmBL37hFwEAz1/0AW8ny3iSosV39MFygCyd3kjDZE8DXLeltCclONECCMig6Chl/IyQDGMOm5dScU46CX5WIIGkRU5JhcD2lDLFf/4P/2f/EerzxhpIgiwnTvF86uQd/Cd/+7cAAK9+5GUcHvsFe0R8zUVd4d6OP2Te293DtXse8v3Daz9ATQe9jfV1fPIjnhvz1Vc+g5de9ka2H/8FbyDfSSJs5HC6wP/1x38EAPidf/A/4YAM7rM8Z8iXgIWj/vuPPvspf88q4WSC1hoJKc0qlSCjw7E1urWmCB5vgaeSphmSJARqim02hGjP92iPkmcZGjo0B1hzWdZo6FnqumaJ9rqpOZjO0hQ53VOSF9i++GG0m1IqWqaoqIyslOIxK6Xkz7THseINxDKns24qfpb3rkmhD4JiXlsl2bYg8sZpNJTUs1pDE6ejnsxw55pXVv2d/+X/eE/GbtX+dbUFEesm4wNWISyKhAMBuCh932gTOSKwfNgL+0GqJFMQnK45eZJlKc8H29RMh1AiwjdBSRmkefwdHAzBRVWwGKJmmmjL8LVv/Ev/6ZCgFFEhsJ328PSR1kXacuI4veejZYHUvogf13F4Sv5IK8Y4lfB4n+vBIWFeTlSN/Md/8E8AAJfOnUNJwfLbb7+NPsG119aH3I9JlkJRUtG2+lFwYjjGSVJFywilEla4NFbD0gGp0SWMDbynoIQped0XsBgTFO/gYBcN4QRloliRXIkCv/+/e15zoAJKqZiz5JyDMXF/lEy9cfzunI0BdHiHzlpWlUwSwYlfBzAVBlKwaqtpHS4C98468H3CSRDSHtZqPrReuXIZH/2IV50PVkCJVJwEVzJSiyQkq54nScaHuqacMnQyCKRnWYqs599bluZISL06VTlrIahMgs+hTnICmulCQvDBwBjHNhDOGI4XkiTnvcGKBP/Zf/nf+m4g3rZMJBwdLJPEoOj4A+InPvNV/NIv/arvg6vPYNCn+I6SunqxxJN9z8N/cPsu7t31lK69e3eQ0fvKO0Nsn/NUk0989tP42C98GgAw3KKCg9Os+LxYlrj/xB/q/uzr/wLf+Po3AAAHB1ME5X3jDCTBwv+DX/f6AtI2rbl1uojDHFEZCxW2pSciW+/QtRKegSaikjQevMxppf5AnahJJ2O5KFEFe4mqYUqGUooTMXme8P7nhMSlF318wbocUvG7UkrxfitELI6I1sEwJIE8pzbGlZrWQWMaHhOAQ5IECL/iZze00tRNw+uEVCmPMd1oniNWV2hqf6CfHe5jNvNz/+/+V7/zvnvzCgK6aqu2aqu2aqu2aqu2aqu2aqv2c9I+EAIaxCYe7T2CJKGBNM2hyMC1Xa16b9UqKAQZG7KNjKqAqCoE9QSjG8gAAyybaLAuHEACHHWA+BVpFEgZ9CJMM1HQ5J9hIJHRveRZjvdWOFWrYpQkSVS9FKJliOlOVZXC79rZhfbzRuGUFiTB2lOfCddiiKiUp3zLQibetbLu7c93KRMIrTk7maYOBaWrEpEgD9lY5Th7kCYJZ/ICoR+JgW0IImIkgkuUhoOjjF3RSbCZ+QzUoLYMx0k7vgKYFgXGh6FaOAQhQFBNBxgMPRSh2rRYaj9WCul+UtVVShiq2KFdjXUC0rX7Nwwcus9GM9E/H/RRtCCsWgXVRdeqQrvohShj1SQKEUV4KlpqcS3dH1hjOUPFlUpjGMonlGNclJCCMzLOGfa8rJclaqpirg995VYUBbQLMMKWDyaikI9THn7knyXCl0KeySjJZHtn2xXAFineRqU8pST0YhL7CQCkgikDcMGi1/FQyb/za/8O/sO//RsAgCLPkIbr4n2aA26T8ud/8d/917j37jsAgM9d+TDOTv21Z69/Az+mKmGAgL38yY+hT8T8zc0RfuWrfxUA8PbNm/jffu9/BQA0y/JUhtuS2lqoviulOPsrpeTsX6/f5c+UpWaRF8AxLKXTIdhqlqOug4piFIXycz8qFYZxo41muFlCUJuNYReC5pAUEpYMiuta83wvq4rX1uPJPrYvgp8h/Bl+bgvaSBlFmtrrS1jinLOoAyzN6Kg4LATPEWdb3miIGVhHqh7W2pbqoYviGk5yFbEyJaYHvsr/4MbrqMa3sWo/2zYlAahr77yLJc2FXr+HtPBr8rDbUtQGELb6XEYIZRHmixKggi4yKSDDWqlrCILJ15MJjqmiX5VLroQF9WSZFby+SOFQhgpD1kGffMmyroKdkzjMoM8m54wSQsyiw0WEg0W7eBcrchH12UIkoQ2Yjk0K0fo8IwJPwTRF/EtEVLQu5Fo3IqXi4mdDCo15L2cz9sWyjJUCRHENJdJY5XQy0maC+I0AVIsyEiq6Bji1DrBAhHCQunWvgK/+0nVTJZGQMbRQllUykySW+oVsUDene02K+F60tq1KqvKCf/R70e4b+P0u9LMEIKlkJ52ADTGOTFrwTcd33hbo47221pAqrEGA4+qjRUnlyllpkFKl2dG7cElEkDmRwFK1xzqNVIYKjkBDlepqNoMr/ZpM24U36CZhD2MMlgvax3UDkOBYp1MgKwjml6dQKqCtQndKrp564ff4MytYi4islApwklSfw35gNTSbqqcYjTzl4pOf+AS++pWvAgC6wwyC9vJm4Z8jcQIz2uOuXrqKbRK9e3M+xg7tzfb4BLsnPl67lqXoJP65nv+IR+mcO3cO3YHf28rKIu/7uODunYf4hv06AGB//wiDPvnMZmn06wsv1Ek4joeiKEsbpq6UYnEkg0g5U4yOEiw04wSzZiAR900rVJzDAHtJokf9vG4j/NoYGFYRcgD7exqO77TRjNji8R8+D4Kpt1aZSFkT8fPhs1Iw+houqv4nsOy1K1SkR0mP66RnDPFoi2plJM9Poys0pCQ/Pz7A5NirqY73n8C00IXv11YVwFVbtVVbtVVbtVVbtVVbtVVbtZ+T9oEVwOuUJfj6G3+KpCCJ/v4IvZ7P6BXdPhLijBV5FznxbqrlAt3ARyMeE6opzI4/mdbX78GQBHtzYQtyzWcR9TvX2Zsj6XXgFv4v04CjvbiOs0SKPv/y81GAQSWY9XwG4kQopHSCf+HpF/lZ3k/YQEp5Knv4fpywdiXw/fg31toWl0id4uW0K3nv/V2bcyZbGPX2v7XvL2CQU9GqvjiHDuGtizRl/LmzlmVtbaagiIcURC+Es5ylsy76JiWIOPxcCGyTLUPVGaOCzwSXznPAZlUH1p6hz3ZRT/wVb9+cYP0cSea6jKu4c3vI/dTmWqrgq2QlDGH6rbSc4jHGBDtF5jKW2gIkSGB0m7brYJtQAYyeKQI4xb8CPE8sVD+EFJyxs9pCcJbIQtH4zp2DZN+YkKGNGR5nHRyVuIWNlVulFKYkmHK8/xgJ8TWSxmOzm6wPRfPJaoeDA5/ZPz4acz91Oh22TDh79gyLPkQqjIhS5NZwWlsIGatDrc/7pBVl1kJ12FiWkLfa4Etf+GsAgF//W7+FhpaJ2bRCj9aBjDJ6RatK7WDx9DkvaPL3/tO/i7//9/4HAMDDb/wZHHHeHh7sc9X0+69/CwDwm//Nf49PfM5zBOcHU2SNv96vfPmv4I3vfxMAcOf2jZZoC9hDiatmUqEtOJV3giWEZI5tkkjuO2sdNjc9wbskoSilFPp9/76XyyX3/3K5ZHSARxBE3lA74wh4CxZF/dXolliKNEhz6vMsx2CNBJlw5hQiIXxHO9v/fsgD15J+rqmSqrXh6kJW5FztbqTgLLSQgit8Phsdvock923NnFUhJASCcJPCZO7H5p1br8NS1W8oF5Cj960Hr9q/xnZAtinf/vPX8e5tX1Hv9Ls4S+vEc888i9HIc4WyJENG/KVukjKCYEoVaaNL1BOylTg8hiERiTRJkBGiYjFeYHfHi0idTCfokYT9cEDS8v0+pPJzJ+slEIXf0/O8i4SqhEZo9ox85qWXeI0P2Xy4NpevzQE8xfJr2TDFSllcgyJiRAjZsqaIlUMhxKnv9M2i9QG+Nqnb8feGbVoLwyXIwPsTFgxzMtYiJeRSlnfZn9EZA4voJSjf0wfOIgpEWLDevkD0TYRwpyqX4QaDpY8XtPH/3jiJJsQC7fVKgTmFzqrIX+ducbBNrFYxvwk2okachXMhPon7IHOI25UQSK78WGthSCYjzdJ2GZa6P6Jp0iTubdoY6BCrJIoF8BbLZbS6CII9VsTvM23BvYQra6ZuMDvy8cz0YA8gO4QRVcpqGEyD32Zd4ojE38bjKdtDdQddbJ31c+DCuW1skDhMThXJxhnoUF2V7f04vntvOUKVXuNQdPp8r/6XJRxxzDu9HJ/8xOcBAF/60pexRfZssynY6iwI8kipsEmx9bkLm7h4xnP9knKB2S6JuU2OoMmT8f7Nazg59AI+717ztiCf+tLn8bHX/N6c9Uc4Q754r776C3jju9538MGTIxjimzt7uloP+FfpfnI60ftrc1xD9dnEcdUSOom2gxFhBWc5/lMtmzXRQuxF7zwJlYZxnvJ3aKOZs2xMtGjyYkun8QYSglFVsl2JjAV1oCUgF70SwXoSVteQJM4ihYZI4v6fpCFoFYxAqskKRjoDyUJcAoqvN4chz/bp3j3MSDfBLOaM7vtp7QMPgHcf3gIAfO0bfwxBHbd99gK2tq8AAEab2yh6frD2ukN0C7/hzCfHKBpfVu5OCUbw43cwv+4HVfloFyXBVpYfeQ7lyA+q0f07kLlXKioLCXPfHxhnZC7dfOxZPB1UsZIKgga6NQY3yf/tO/t7SGhh+I1f+zvoEBn6p4mz/LRD33uho++FdLI/WUv4BcD7HgDfCwUN/94+GLYPlO1rh983pOxTJBKKPJGcVsgpaEyS6AOU2CwS/RMHSQqpBf2ZyBQlKXhqAAmN1iyJflA+SCf/HfEINQ3ShgbjtG6QKX/o7ooOdu744GCqH2FIQXQnH/LiXpnFTxzC/Z9h4TOwdH9+4yRVxbrm/2da/RngQ8ZYVOTfI4Xgw5sxNgpWiQhxGow8hGE8mbAIgnaI5FsdleNgXfS3sVG9LYiNmFSy4afwOzf1XYSazmYl3vjGt30fzKb4zMdfAAD2cznYe4A3r/t/3z9eYjwp6TsaKOqPLOtgSPf9yqsfxVOX/fyztS/7r/cTOFpQ6tpg84z3d8p6Q9iwIQoL5ygZ4ywUqWiFxQ5WoCl9P66vbeOjL3vy87JscHgU3qHlhZdQkx4CEaC5VvB701WJvTs+MN0+cxYZiRYMqgUOJn7TXez7MXP9O9/D+jmfTPjWn32LvY0OZoe4tOkVSG++ew2C4MIqTVjgJ8yPPM+xJHhtmuetOR7nWZZ148Zc15iSWt1y5vvxaO8JnvqQTxr117fZF0pKyTDSTqc4lcgIQQbD1axDXUUYaTh4ZWkWlYNNAw7amgadwU+uUVFcK0L2hJRRMKGuYKkPJP3Z7XZjUGENnA1KgTEJJYWEE1FB0BCMin08jfEEcwBJmqEkqPDxk3dxtPNj/yyzByD/eBiTQtcRvr1qP5vWNH7M3rt/CzeveV/NrfNbSAh2NRr0YGh9kIlEh4QvZlJiSfDRo10v6HCyu4vjPT8X7fgQkgSBYC3yAI92BjUl17SzbMQcxNw6eY71Nb9Gnbl4AR2Cqc2NwuGxH0N3H+9iSeP3L//ab0CR1+v7qbZ46GX7YBAhZOCpEQ5PjoNAh5ZBdUu/WqoozIVTCuMBziVYudTfhYg/h31HCIRF3hhE3CP9LkkFQSsBKR2bseedHuom+AC2EsOQfBgJEaODZaiYaambShUPD0o6jqglwOJYAWZqrOFDvkYSkz8iJjqds4w7dK5qx670O3cqgdpW82y/rvCZAAk2wsFwItrBskhQgQQh4VDxOpZYi4T2lJDoVSrHou1dGKDn2qIMSVjr4EiAa3wywcGepx6cpVhSII4JD9WjW7YOs2lIyB5iTOPeLGfYHBL0kgTvZuMFbpK/4aPDE8xpWpRVGRVXlWQ/4QvntnD50jkAwNa2T4SmaQ4VkvTdAVIqkiRpzrGWhWCIqhBNNLCnA7pFCuXzhbhw5Sl8/OOfAQCc2zqH+cx/ZjGbI6UEDFkKwgoJS/DgNAPOnfex9drWFqSJ/oYdgkj2EokJ7clv7Nz1z73/GPuHvm8vPfMcBrQelHWF7bOeu7C5vom9A/8ZBwdi50SqQRyusCaq2NtWYkfKhGlLLnGtA1lwGYgQUCmj0JBFLABIJXGKFhHGekjoW81exxCiBV+OVAcg+mw7YXmOR4XxmHxRUffPHxT5eraVUAlJFk37PmCbORT5ggprEc580IITCxZRvTR4Zjpn4QI+GRK68WeeanKE5YE/0LvZIbogSHrh0PwrtuYVBHTVVm3VVm3VVm3VVm3VVm3VVu3npH1gBTCIFkynE5brH65toqqCH8cSgk6kiaoA4X+urMbe938IAOjQCbp7eIDDa2/568JgQHC/5s4tHBDxNN0skJKtgZyN0Ux9FXFy7P88OTlGj1IJ5cEhkr7PINbVEgePvBT9j2/+GKN1n+mYLcbo9DZPPVMbytm2gZBScpqiXe1rw69idc9ChwrbeyqIbejfewVy2lVBAEyKFi0IaLuaiJZQSfAeK6yGCIINzjGpO23VyItEMVSiaSz65LuSEfTOCIGgsSKcRUL3V8iUhTYgDSqSjK6shlVBdtpfo2oq5EmQGx5gPPYZoGc+skAdoCVwqEtKm8k8QlvpPq2NVbP274WLEv3SCbiQFWlCZlHB0rNYxIqcUwlyGkvn1nJc2BzQ9zhouvbwrIdBbG9toWd9BqWej3HvyD/LxACSZNClkqgDrMZozOYz+plsP5yImWdnODNqTBQYuX/vIe4/8NmZfpZjOiPBhtz3yyATGFJS5850BkJSwGjN1bvlUmC+8J8fT76JH3S/DwBYy/13f/KVp7G97auxurJY5gT96vRZ1MeLw4R+l1AkKmSWVAmu5+h2/f/77Jd+FaAx9uMf38CVi77PagckVPXdJHns2jmUOs6nDZojX//mN/Fk11fw189so9vz135xtIbZxEtMH499he2tP/0D/L9/9AcAgNfffhu//rf+JgDgz7/7PSyMv+ePPPNRvH3b2w0IZ1DWNX8n4G1LUs6OtQVVJFsc+L/TkpcJVKV/FzlZWjzzzLNoXKh015x5LmQRBaeU4u8UQkSfNGppGsVSsixDXfn+nc0WyEmgwznNsFMp1ak1JnzHKQiojFXyAHWCTJGkPhubEzqgWexjPn3iP1tNYUjyuqlrvl6SJNxPy+UETeXHtKSKTmdwGWrkYYQnu9dx/OQHAAC9HKMg9EWaJwwbByxEuoKA/qxbTdWUo5Mj1OT/KhXgaF8tFxNMae6nKaBpXWwSieMDDxG6f8NXDg8fPUZG2eRhIiFpjxpPlhiT8MuwyNnj0xlgSoibILYkrGEPseH6AN2+n1OuanDwxFcV3nzzLRQjP/eFsJyZZzESF6sDQEChhH+PFbso6BWq2tFji0p2APz+yBYqUiACcWT0SORqnGUhJC+YFYQXIpzSy7uHGMHw2p8QqkAlGe/BzkURjzQvkBEc1BrD9iwOBpIqfwHRItCwAIrW2lch4Cv0waNPWMP3Z4yFo/3PBl9eG9EvxtZcRRFKcN81RqOmz0MIb0sARBsRYSOKwrWqKM6wf6AXa6MeDUuAjPu1bkFmZa0haWxuDbvY2PDV4uGox+J1HRpfadbDwbFflx49esQ+dFqDBf5qo4PFMHTd4PDIx4jnCFrvbAueGgu3WNQljolmce/uA0ypurWWOWz3fUUreNDNFnMc0n08OZwh1EmbRvNeqoTCsvZzZDxb4IgEVc5s+3He73fRG/i979y58xgM/P4pZMLWIMJFKzAhgTzAleie60rj3HlfWfzM538RTz39NH3fCYrGxypKZtDOz79U0L6QSh4HjQWSJMCTNUq2q1BYI/rUaDCAEP55JyQk8+DWu/gziruGm99DQpQunTjsEIIgSSUy2j+quoHVQTwozKFY6XMtwSclY3zdFmH0H41xl//TMiIqTRTv49o6hjGlqptAppkAACAASURBVI1gi9BoG0BcNomUrpb9CBxgyQpNW8vCL4lTPKdYnAWSPTYlLAOzPbUkVO8kf7ejGNrUCzQkMoRmAUMVQOiSrXGMtTw/IeJKGJALVtjox2nBUP3ZyR6WFH+LukQW4gVnTgkCvl/7wANgWCSFjF53db3AdOY3EOQSIgRUNoEjdVCnEmiCYz3ce0Sd1WDzGT/Q8mKIMS00i7SHCW1OB3aJrYmHez7VS7DT99fbmxGfZ32LF8TGGAgZFBgFugRpcw7QNhwUmlMeWqG1/eZC2dm6eABJlIVV4UBGm4xRcaOVggd046KZrLJA5mghTzM2MOXNyVo4xtUb5gvAWlbWkvRXwB8CmPdEl6iNRiqDmWiGlNQTO3mKDr0LaQFHOICTWRkFNoP/kHUI2GtjLHsGSZuzmTxSC1iaTLILrf3CYAN0wGlA+Emzv3eInkft4ey5Ee7t+QW2ruYwNkzqLPoAtYhpQclIKRVVwBoH11o8LG3MIny31hwanOZJOhTGT7K/+tEL+NXPvkRfo3Dk/A26i17d6tikGBzfAwD0Zrv4+//knwMA/uzaYySWNqJBVFuz1mFJvojW9unbM16grG3ipqsF3r15FwBw7dotCOrrXr+LAWHoRyPfL9PxATY3/fdtn/RxNA5Bu0Sjw7hJkIZJvVwiS/yi8tzVp/z/G2xic+RhoUcnRxgfeWijKPoYEixLSMSgBgL1gqCmM1qgqiVe+9QvAwD+vX/rN3Hnlu+bGzevMcdMZQU2Rn6DPa7oXUBgMqZFuq6wWPPjbrqQ6JFfUa/f5SDJNA22z/iNVmQeCjoe72J7+xkAwG/++r+NZ5/z/37j63+CZuk7dfvpV3Crc5P6oIbhZyHIaWOimbEQPG/KsuQ1wBgT/e6kwGDdr0flnDg8AAZdgsZKyYpteR6TF1pr/DTecPiONtQ5BHgbmyN0KRGzt7sLRRzF9Y1N1O+D02/7VbbnSJZTEA6L8oQg8lO/Hs9P7qMgs2U0GsGpzDUN84OmZRUPx6lETsmO1Po/q+k9jI9/BABYTvdQkOF4IS0fPg0AFfiusIGOu2o/w1aGhNB4xgbh2josSQVxPD5h+FKvKKC6xOVMOyhJjfnRE88bWUzn2Oz5damREjUpf86shCEVZ9sAYuq/s2waTEuaazQvCmVR9Mnf1Qj2lp2XNfZP/PUOJhUub/p1YDAcYq5bsE5/tUgQah3kjItQK8/4obkY9mYRuToOmjlqUrQPfdFwWUnBh8c2L435cVKwL7SntAV4qWMuvhDxvsWpfS0mbsK+miSJd572vQNFsUXVNFBNMHqnZ3FgGoNyjoMBYU2Ehgrr91/q/8C/k0EV2lqGujXGQgdPRgk2UjdWM9/fWoPgR+h4DZURauvAcZd0EffpQXkhlgI3G1TR4fnkANAkGmtkcP6h56/g1Y96qP3581tsst3v+n1VmwRPiJv31ls38M47Hnp+b2cfJfHwdNlAUB+oJMeYePZhPFpn+PmEVAzhKxclHj/xSbK9w32UBP/PixRLSgg6GrtZLpES1t0YAR0geS2utcpEVGS1QENw+JLoFEUi0ITYeTbHjA7HTV0hO+MPiUolHGcKEb0CU4Li9wcbeO1zXwEAfOHLX0WqfH/dun4L2xseatoZrqFP8EyTxkNC0NRwBpjToa5aLNCjA5vIRxgMfIygZAGpKvpO/y6SwQgF7VVSpahMgJyWqIjqkHcG6A0oEXt8jLIJ3rcsmoDQPHwzxJCRp6mt5cOIFCJ6ODKZTiJNgmpx9AuVLvqItznEUojT60r4U8axG/ZBAQetZbwefaeCat17+A7BcQGU5DknJJiD6ayGJlVOXVLRoJqjoXU1sQYy+HguFyhp3PmDYOBxpkhIuTsnDrUV4GvUywr13MejZnaCNOh8QERajzPIWmr679dWENBVW7VVW7VVW7VVW7VVW7VVW7Wfk/aBFcBwOncAwxIWszlSgh51+w00qRM1oobp0Ik/TSBHJFBwmwQfKgXX8VWYE5XjMUHdDvcPMKaspbYGT1MZeDYo8CZlDvtU0buYpiipKlhaAxfK5taiS+X0NZsiMf4aKaJv1nvVP9/7O9fKgkmZoGlB5wBfHWQIKCSOKON0NJnjmSsecipELB8La2FIbEFzFg9MZFVScHXAe9JR1RKCS/8SMbshCeppXRQmyfOo9CeRQGaUCW5ilaKfS1YpSwY+W1RXJRo6+tdWQwQ/lNJnJwAglQ4NVTNlp49OSu9W+azDqDdAve/Hwc6Dx3juNZ9FauoFeonPHu3PxwxRtaSw1W6+UpPyO8lawh5Lgme2PSVl8pPvTQjB1Zk0SfAiiXicfeZlqHPP+98XfSQzf/8bl6/6+zycwsJX0NKtK/jkX/bjausjh9A07t65fwuP9/34rcqSvycQ+utGsrqbsRoBUXNwMMG9ux72tJg1nDyajMc4OvbZx+11PxesqbGYUHVVN0hD5itPIULxVAMbA58N6uY5Lq77frpywcNC1tY3cTRrqI86KCcehnLznev42Gsfpz5IOLsLCDRj/5ngL9jd2MJHP/YJ/90G7C9pmhqPSDHsylPPwNGYOB7TOE4sDg59tms5n2PZ+P93vHeCiwQdPbM5RENZTV1WWBo/+Q+o8gYl8cUv/yIA4Be+8pcwPvJ99PILL0E9JlXU6RymDMJFAhllR+uKxmuahOmLLMsYTmKtacEf0xbkRCINAhfkF7Scl0jSmHmsSsvXYDGJVsVZa/0TUHFjDH93u1qodYPZzM+dLFEMT1oul8j66tQ1tDFReEIp5HnwXU24ir84eYKTu98DAOTCj8dulmFAcB6tK4DWxxILNFTZV2mGjOD8RW8Nw/Uz3DcAMJ/NoOb+O/pJF8uFryYvZo9ZLCp1Eoay78ZZnDLwWrWfSSvnseLB6DtYzGYE6YVBQeNm1O2iIFhymuYoCZ2ze0ziassGltaGMRpYgu2XtWN12GXtsKC9fryco6L9m5CN2OxmOE/wMKMSVLSHndQVDqjycFJWGBJ9xBq0fPdoHzzFfnAtGKOBafmghqZp3zJlA0NVm0ZHKH6e5ShCiCNE9AlVcQ4rEapmriVG43hMt9BonrLBFcpI26ipX+qqZgXPPC9QkOeoEgnKZkFXdqwxBkcee0AUSHGGYwE4izTEE8ZG+JzSLeiq4rkbhN/qaoklVWeM0wxvS/OEESvl0qOJAMA0FhkhHwIawljDsUyrJ+BE9MN1MCyIEdRqBBSrJOoGLJJW9AZ46SW/N//iL34Wzz37LACg3++gQ9D4IiiLIcVVquI+++IreO3+AwDAzZv3cP++p/o8ePgQD4hi0LiGocg1iQsaY1DroM6e8z3vHR5i/9BXvsezBQQhI2ZaYG/soZzbcx/LZJ0CW2t+n94bz3E092NXOAcRYLBCIqFOWOvmuLDlq5xnz/hrDPojJCT84nSFgz0/P51KkVBlcOtMwlVmYzRTcgzFPZeeehaf+/yXAAAXLj+Pwx2P+Nh7coTp1L+vC1ccClLeNVRNXFZeFA7wa8OSYtZ6UaJD39Eb9LCW+/9XI0MC33/BWnm0sYEPvfQhf5+XriC0g/EE6Q0v6risG0zGhI5yiqdITXu+cI7VOVOZtpRTWurlxkJTxTDNE6785SSg4yt2VFGXMhbkrI1T0hgIFYWcuHIflJRgGeoMIRih55zj8dMWhwEkK8VHNU/HiAsItNSELTSdY5aLCZq5j68cQeuFaRACROsMLEFAm7qCo3VMSsdK7J1ujoIobimt49Y5NCGGq5sID5fgPpWIVVXpklit/CntAw+AvMDpBg0Fuk1Ts0pc3hsyD0WpDJoCu1Q4LLS/9EM6jO1Nl5ju+wE4cxaSJkVWFNg860vhWaeDPsnuHj7ew5Tgkl1SIHV1gyX89y3KBoogJ9VsjoTu9TwyABQsJz9pBA+0cf+xbAvnUBJU4mRhMK8o2Ak9pCdYklF20hlhvAiKhCMsGv+iREcgpfK7WCyAoPRDUtoCAjKUsYWMUE9jGDZxGlrSsp4IykNCML9svlhGWWoNBMcNKMVG6s7VOJ75PnU00VEb6GAOKyUrhpXVEooGo0skQ2mLQrKtR+bI/H15AW/c8BhwOVjA0CI+nmlI5xc/2AkPQNMAr77q4ZcLKnnPJjPUpHDUNDoatzrHgbhugPdypNrNQaOh+7xy6Tn8jd/6jwEAl86dw4w28eH6Bl5YpwWZeAYXriyxnPv3WZYNvvKClzr+UlPBEL/l9s59/O4/+l0AwHe//Qa/gyXBO9JM8GZtreHDyP17Ozgh7kyS5KzSJtOEX2SAY5gmQm116Vi1a7DWQx7ku0uNPPX92MkNEhksWTyksLc+wJMnfsFpphNUtInffriLtQ1/IHjpxWdjQkcCSZ+ghLTuXXnp0zh71m/KBwcHmJJS2mi4hg4p/Q56fV7wFPWtriJ8yTQah0/85qRgsEXf3clz6KU/SCRwmNL9BU7n+tlnOUmxWBxDkJpZkvWxcdF/94Pdm2xwa4RkI9UQRDbGRK6oMJBsGCwZqmKQIu8Q9AURH8McQQEsaMPsFgmyTkrXdizlnSvFSZksy07ZxACnVXyVUmgCqRMtK5I8x4I4gNWiQtKN6p8AIJREQYfSVKbQDPdymO3fAQCMH7yJnBI+HYLv9Xp9OHqWTGWoaf2pFsdwtA4XRYZ86BNBg/VNhuYG+FPS2cDm2iXqDo35wo8rPMywnN71PysLYSPMTph/BdFg1f5/b9bQ+KmnKJd+XtSLJcux23qBYdePi621NYZdNbbBkwNvQXM4oXcuJPSc+LACyOhImXW7UaXSGA4mZxpYEiy8R1kqlyR8iDCuZhW/rEiR0jxyKoOjQ0V9SnWTkiuqlbC1kQPTmAiL005D0HfO6SA7nsw4KZdA8N7R6XQwEhREpQVABttOCLCWJa1nxoFV+iQkHxj8XI9qhuGU6qyLXL3G78eTyRg1mWCnWYIOrbFCpag5zIgHW88jo2vYcMqMwu2pSpG2EqSW1gGno2m9h9TRntyEQ1CDmk7oQlhOJEqFVlzg0NDnbWPw3LPPAQAW4QCoNR9ynHPQtK6YJlIddFXxPlbT+m1cAx0SdQ7Iifu9vn0OT13137F97jKSHlETOgUsxUS1oT1JSZQm2B6M8MJLXsfh8jMvYbbwv79z/yG+850/BwBce/uHqCj2DNxqDwENKooaU/p/e3tHmNChyVrNcv0ulayiPp369zkQEoqSzv1egZqSEFWtom2Us2yl1O9nOLftn+sCxbTD/oD1GPYOpzjc91ytpXVMB1nf6EIFeo4zUBQbdzpeD+PKU89hm5T3m9qr8AIA0oQPTcuqQR0s1MLBy0qGSadIQF2EutJYoz1js5ujT/cxLg0yehc5wn6RY0D8nm7RYf5qo2uUAW4+mWG2CAl7zfDqUKg4bfiexJ9FTBY4ZyOfuLIcX4TDtT9AEmTZ1GwBokSkYynhIg1ERggoq4RKIJ7uBF9PSkAFqpWUkbLlHBTt2S5ejBMx2sQESLOYY0Hq5q6qkIQ9m3jzUsbCk2k0Sk4aNXxLearic6cpOkWgekW3gyrQyoRk7uxCCJQnRLtqlgyDTiCQt6hv79dWENBVW7VVW7VVW7VVW7VVW7VVW7Wfk/aBFcCYjYvG4S6xqMmDqKzmKIiUqJIMRUMGmpMxvvOjGwCA2wSFc06gQ6bag7UBhps+u6HShFWltHMMaYQDFpTV013KcFUViPuOyXyJYuBLo+PxBI6gFKlp0MyoQmPcT1QAZUvcoXGWT/taG9wmo/qTMoOwPitycHAXALDz8G1sn/FZmN7WU1Adn0XvyQ4eH/msw/p6CrnpT+ubWwOUM59praoA12mgKMuRpBmaltDDaY+8qIwkW+VywMNyw6k9SwsEtUajXRTUdAr7x74PBhvryEixcUr9sjFcw5z8vdI8YdUgbWs0AYogJfpdygQPBJODh8bD+t66aWCEv8bFF3M00r+YqkxhS8qamC6LylSlw1e+6onMdcgiNQ1XX+q65t9XZclVzul0jgVlF6dTDzNYzBcwBG+zUqM79GPplY99Ejfe9XCRP//u2+j2fEZxfW0dn/+yN0+988Ar0b5z4xYaqnY22qGuo1Jk6PM8Vfj/2HuzZsuS7Ezoc9++pzOfc8eYIzJyqMzKqsqaVJJKapPUjdQYRgMCGRhP/AP6nSfMeOFX8IK1GW3QGAKBELRaKqkldY1ZmZVDZAwZ453vmc8efbvzsJb7vvWglMxoioc6/pARee+Jc/bZ24e11vet79u5TuIk/d4nnqri1COrOILgik1VGlxc0P0oiva9rG284uZGN3hxQpWajJ9PtxtDRlRhi8IE6YDmT9JXHtFaVBkSpsemcYHD62O+UPZkXB3jzgFVHjdK4UfPCW1rhMQpK/69Xh4iiGhOW6Oh+O+ChXz2+xOg4YZ3Y9FN2ahcSNy4QaIsg94AG64Qd1iB1GqgwybT6HdwekpVsN74AJvTJwCA6cVpSxuyBiWLULj7qcIIC0Ycf/D9n+KMEYpXeY1sQxXTxasn/j3CuK3AWteMHwTeo1DIltJhmwZ9VmHLSu2pKNlqCcX8NVeRlMJSiRWA1iEuXxLdSKgY42t0D0pReqqyvIJM2yuooDdkbhpE7L/m/h8ARBgg5J+XuvKIoVMJxRXkv7IGgaA5sTz7DJcvPqHnNYwR8bxXzFFTqhW6qMsSpUNM8xIJV3ltHGFnTBV1qQJfObfCiXMMIZlGk6+X6LKYVHL7y3j1nJ5bsXzqKTrCCPydUmPb8W99GBbhyasSGa+nrMgRMaIrhfEm1tPZRevDZTSeMqXOrbk0Trx4RSWBIVecR90eugmtkWJTImMGTBRGnsbo2hjCKPTrqSpL5Bn9PBmMMGBD+iCAV9BLAoPci4C2aJar7AspUDrEMa+wYeqotdqjGjNmWayWK8+s6HYTpJL2fdXAoy+6MX6uQ0WeReEErgqt/R4bCOupnGisp5tJAI0jRAp4upb77CLLPEvImgbKUSht49kttdZXEAnVKvwxTVNYgcgpq0chFLOtqNWBXltqAwvn5dkiqcb710oE7rOl8IhFWWmUjMzWTeOZKSKQ+Na3f5Wvu92n3fcy1vjztipq//f1eo35BZ01C/ZUXSzm2DBVUkNiOKH2mGvXriPjc/HDTx9jd0rnyO7+jqe4G037/jrPcHbmBIoKxExZ7w77GAwIWUv6Qwx3ib5uVIyKkT8XNxgYP0fzIsfFlGKHs+kCeeXosS3dT0B6JpFjqFRGI3fKjdYgdfNDAobnfaO1b3OxsCj4+Zf8GcTs4mdfVx5BK7XGckn3rMz20GFETkggZOQnYUR9OBoRxQvAarmEsY750aP5CaJD5xwvJw2LJoqm9b40jWeEVFojYOaeCpWnDhtToeb3KBi5ms7mePyEzvHo/BzTGT2jJy9e4pg9Ay8uFsj5mVd17cV3Wvqk8FTyQAYt9RtXBZRa1VDbNC3DSLY5iDs/tdY+bk87XU99hbQ+tifDeUdV9h/Wtmih/T1E69tsrxiwG2PgFFUCx1K4ggAKIbxH33J2gXpNsV0SKIQpxxZwpowW1rZryIl1aa3beEIFV9ix0h+rLs5ohIVkml+cSiS8T4SB9ErAm03WsoCUgAm+GOP7eyWAQRD6TcbalvO+Xi0Qx5T02dp4Swhb1FiziXI4YHpS3GmVk0IBKZ1ylfEUorCTIGCKXhIpSIYvC05c8k0OdYMOuKJaeLnZQCgvqxoogYKpHLBtL83PWT7wgw9kgJoTqGenFzhiNUMb9rFmeeAf/fAHAIBrt3ZQRxRwrUUPEUvtCg1vUG1XFYKENrMlFD79nBbL6zeoV+v6QCLM6WcBNEonMRu0UvBXzd+vJq+8xyC80oMUBIEPMMMwBhjGfvDiGI+PaUP8R3e+jvKSuNqxp3rWCFjdSlp4mFhJeNsABBZRzHTVEAgEBY2nz5h+s17j9bdpM04HxveHyEpAcwOIDBQaVjsSEpBsExIxrzZN5BVFqNb8WqnAf3eta29H4p+lrrFx6l3pEK9/6R0AwPHJMf7oD0nN8/D+fey/ST1+eVbgBz8g6wSeMnj18hwF0yeUjNq+S9EuGBkIhLLnPydgucOKKT9V1fEKayfHMxy9ojmTZbmnhSipIHlDjARQ8+HizLujpI8Nz58oltg/oMKCCTRml0S/29vtImXF2zhWWLG8f7yhAKhsCljtekwiSA7EDg6G/oDL8hIpP3MhBCwfRPmG7mNga9SsXFVFXZ8oQUiUzC05Oz/372dKWgv7e0MI6RKYCL0uJY693X3krMx7+fIpQv57GseoHZXG9b+IAB98RIn5/Gef+e+XFTOcv6C5u1EBLM91I4S/PlcO6aYphHB0XOM3cSmkTzitsaiYLpfEIRrjKKD054unj5DPuGd5WcHyPjI82EfvOgUbwigUhes7DL2xvJPxruvaz9M4jlHxZ8O2hYWyLK8ok1ZIXMPFld4Cv/dKg/XpMwDAyZMfYI9VWFUU+e/oevqaqkRj3CFZoeR+sKo0CIf0mtHuoVfjzYoaqsfPcZ+sHyIpkV3Swd4JJApN1x/3d3HnrV8DADz7aIWmpPtkAmwTwP8fhlNGbioDcLGjrkpPqSpFhA33Cc6mM28GXuYV5tz/2zjKoDDg/B+BDfx67sQRBlwIChqLjNsAukngaZjeTkECFQc4yyJHzRdysBti0KcC16TXR+io/aaB5fDDK3ULAe0LnRJLDubP5xvknABWjcaGFYxnHIxqU/siTyJCNHxtFSyYuY3SNGiY4repK2jTUtkAIFUROoq+axgYGO2C5QzGFZMA+OKsVD5odCqKnbSHql7yA5J+PVsbeIPnMi9ak2sATvHfBbxxpHxbjRDiSo9dq56oRNjKwcOg4WvVle+GQtpL/HPJuf++1rUvEksp/d4FAIMB2xN4iqjxaohWitaUWtdwh2VTG5Q3OJAtufeuyhGETF9Puggi7uurS7xiU/XHT19hvE995W+8+SUMuT3DFWFniyVOTyk5Wi9nvieu003R4bmUpB2sWME6jHpoJLfvcB9fbQwqpv4vNyUuLikB3KwKr4YdQPhzOlTSN3SteH7VTau3MO510efbWxnjbbTKqoS9kiRM+cx2sUxdWb8ms83GJxRRqGBcIr1YtsG+Usg5XkhcO0gn8UVnYyvftpEmPa/fYBqJgltadOyuJ2jVboX1FNG4M4TiOdvUK+iS7k1eVL5IUnKiZGqDizmtM1lqzFZsD3F2jinTWTdF6RVSLYTvTfP6DkL4WB5SeqsDAeP7eJWV3valqGufWJXc01lXlbeZyfPc99Af7u3i+o1bAIBut+t/boRF47QtvFKn9HGDhfD9xNYaCOXiPnklAdS+IOIo4+qKQqkyDTYLOgfz6Sli6dp0UnRYZdW1djXCwrie5bpG5a2pLILIJXVdX8RWaQoV034Evo+2gU8ErGkbEOP+CD2n8p2VyKdn/poD88UJ4JYCuh3bsR3bsR3bsR3bsR3bsR3b8UsyvtgH8Ip/VskZubDWK1KqtO+QaVxMz71qURLHGDDlqAZlura0qB39sRMhYaSggvGozP7uBAELbATQeG9MVT19RHQAe3qBKVNHz3WGG4yWJEkHBdNFQhVAGJflW2+2flWswfvHBAqfvaBs+cGrKTaWrlmvczz82d8AAIY7XBXsjpEpup44GsBwCSgyNXLO8rs6xCann/+L7/8Qf/mYqlzf/Qrdr//sO2/iO68RGig3MxhWO8zy7Mr1SU/XkVf8a3oxqwMFEValo47WnoqlpUJe07U+2gRI3/4dAMCf/ugVnv34LwAA/+BbhJQdpDXGA0L0dFkids85CMEij9TI79CwKkZZ0r356SekPDi60UV3Qs+irpdQmpVhwxhSERTeGKAouFoSKPiymTfTFV5NEFYi4OpNVZa+GmSk9FUdV1GKohgxC9rs7N4g9BPA5fQSx0dsur67j/WSqlV11cBWM/6+3JSrGwwc8mIbRFym292doK5bGoex9L0Ort9B0FBFfbVkqvD0BOdnVAGaTle+whzHkUe767rCgNXgxv0QE3Z93zkk9FTGIVRFFbivvvcWbtwimvGjx49weUbzY7LTQ+QqwUZi7SphR/z7wRBLVi5d5tqLfHTjbmv8qw0kV2mVUp6i0h0RsvWdb/9D7LFP02KzwWq+8c+wM3CIl0TKvpOewlhqWK5AN9pgwwibbmrvPymDGMtLuk9NEiPgCnLkGrZtge6Yrvn2a1/Ck1N6hv/yX3+CNaPQxgBO70uXGrHba1jowkAgsCG/VnvEXIoW/e+kKZYLqmaeHr3Ahk17L8/P+d9Zv29ZXaOb0mev5xc4efSYru/NN1txirpFp924iubneX7Fk6dFuMuy9Ahf3ehWNMY1ogMePVifv8Lpwz8HAIzS2KNtRZEhZI++Vd5+tkMMqrLwFNC4N8CA/aIGnQHWK1oLqneI8T4pvEWOZr+ZelpOtzdGXdNrO4mCronOtXP4LmYvaU8xVuDvKDJux/8Hw3A5XwWBVw6WsLDOl7QGDO9jVZEj5/2tzLU3QJbMCICQUM4DVwo4obk6K5HzwzWlRsoMn2G3B4D3B15bZV16YaNuJ4bleCEvS9RuLw8sakclrHIIVgV3dEUhLEqG7NZFjdmKRV4yjYyFRYqqwpxVk9cb+n2oAkTss1fqkHisoPNfRrxnBMKrm748nWLNSEaX2wRu37qBwY5TBRZOvw22LlHwd9FNA8lex1IICL7vg96A36vrhWmCULVUcG1aoZjVCpeONrmYelXCMdNkb984QK/LvmvWegZBY4xnzoRK+c+udO4/x9FZIQUSZlLJJICs+Pw0ISo2DleF8kiN0cbT4RwrwlpA+XYc6dFAJSQcWGIjgYjPNhUQopSEgVdV7sYDLyT3+PFDnF3QeXVyMcWS0efu5BBLvuya509WVrBMdesNdtBNW29Wp5WzXObQPDf3Dm4gTCj2uXj+Q3qPvMAm2/MdCwAAIABJREFUpzdeZSXynGmfukHE3qdSCU8lbNCiPLmLJSPgjdeoBWTv8NAL7j17+RJnUzozyjpq1dxNg5I/84xbIZbrDKET0zHWCygO+gMkfIauVis0zv8t7WDFipppyOsjUmjAjJ15BdEwahmlHo0yWkNXTpXWeT3WXtm+rHPw1ETSm3gF7PnmDCWjUfPSYO2Q79hRUiV6Izo79m7fxoAFdF5Mz3DE53R1xe82DGOvPu3RNgsvHKSsbOnVlu4ZAKw3mRfXW68zZBxHlF4ksPIiVGVV+X1ntcphGce6du0aekylNca0Hp+2ZaR5P3AhvbiaBbcyAICE9xjWtUHqEXqnJC4h+JqKzQzrc7oHyGYIeS8RjYJmBqSLda1tGRfGmFaNXCqk/Czift8LxXW7PaQ99mLkfbqMyzY2LTI0vFFLFaE3oX+3UzWYOcHOIieV7i8Y26N7O7ZjO7ZjO7ZjO7ZjO7ZjO7bjl2R8cQ+gE0upa5Tc4xUIASWcx5ZBzRUDiwbzGSEjMAYF9xO5KkADjZA556NOFzFn0dXRCULu57KfP8cui2OYbgx5lwRHsg5VTR6+OIG4JNRpmShoZ0YUKs9tV2EIFI4X3XqAXZVpF9yY+fzVHO9/+AgAcN70oHrsLba5QBhxqYn5+JsmxTilKldprPcPstbCukqCFfjpY3q/P/nwM0x7hK78H4/pvix1Dfz6WwCA3769i8M+Va2WsynWjEJkTQVduwqJ8TxflTLiKEOonEVWINAwkjBfr7BiHnasdjF/QkjHz/7mX8KuqErx7CVVFN555y4O9liYZBl7iXuIBpKrglJGGA4ZEa0qPHhI/P0qpCpdZ6+L0nJ1stEemQjjCMahM0GEGVek15XBa7cI/XTNvo1p2qZtHXj52qqYYsN9YFIN/esrX/VWkCxVXhYZZNjaEDR8TY8ffIA5y/Ie3r4NwwiTZV59ltW4f/suAGBvp4ddtlS4fvMQlquWl9MZnjylZ6ffeB1BQ3Pz//qjH9E9n2Xk8weq8/T7zhIlgOF+GEj4ntRSNyi5FPZqSuiTsRZf/zpZUPze7/4eUq68/W9/VOGcK2yTSQdJwPLious9ns7O6Bm/WMywWPKzCCXefY/mXRwHWC7omouqhlCuylt7qwvL/RrZeok5I6wff/YAJ5eE/PT7Ywz3CfnpS4tYcWWQK3erZYmEUcEyy3H8/DkAIF+tffd1URXItRMQyTEeEvqclfSCYp3h2p0vAwB2r91COKHf//kP/wzlnNaFsLZdy1ekjXMWCOr1uqRzDiAKlOfuN8ZAJdwb++ABnnzyU7qPQYiIq22J622KLGqHFg46uORKal9oLF9QI/zq2m0o/r6dNPSVSHNFPKEV3DCInZ1DGPleRBkEbbN6VV7p/WuR7sujBwCA80//GomgZ1UghXM5G0QD1Cw44GwzRLGBcKiPATJG3K+Ne8i4Sb/WxxjcINTv2s2v+P6UbocuermpEfNa0ZX2FcKmBI4e0R5wbe8OsvQhvV/xDCZohW624xc0/P5SeUEQq2sYuB7QBiVL36+DFUqPrrQS5IFDlKIAXUaAe1Ki4862PPOiFYEUiN0pm4awgvaBDa+/TbHBdEGf0Rt1oUBzYp2t0TBKHoYSzp2lARDxOnb9iU1joZlFMVuVOJux1+4qR8HIyiLbYMMy/a43Oe0E6LI9kUaExjiBF4sqo9dcrDY4ZqGSJ0/PML2g/W1/TPGEtV3cO6B9M5lIQDkRqcL3eQstPSpmr8jSO2TO2lZMIoyidp8KbGupleWoWABiNl+h4j5uw2J6O8MECbNDOlGIJd/fssgh+Tv2Bj2PyGmjUdf0zN2fIhReeEyFAQTv8amMUTVs+1IGMMwCKfIcQ2aNCJ4/1sqfQ3Bs0J6xbSO49T1LLh4KpIGDyqq6RSDm6zXmS/quWtfIeT7OL2fY8DNyFiZRnGA8oVjrxsE+Ju7shkDmrAcWC6yYzTEe9jFa0XP8kx/+CQCyyHLWbtYIjzpJpbyVkRXC/70xFrXbuPk6whyY7NJnf/2b30DGSNkqy3BxQWdzHAgMBoz8wOKS0aM1o83L1cbrC0ShwphFAg/3RhgPCeHTMN6CRDc1NO/rTvCjG0nvjfvq+TE0203t7F3HZOeQ71nsWT0LjqeDUHmGSlWUqJ2gTaTQMPo4L0usMkbU6wbaibJwX5oyAt0h9Ydeu3Ub9pTO48ZKf78s4MVGokDB8rnq9g4YtMibEp4dBUiUHOedX15izf7FZVX5GEyqttfY/TslAxiOCbMsx5x7gXcnE4AZQbCtkJr7NLJgYe0RYf37XWXcWGE9a8fCQPB1eIG5yiKfU0yYXR7DsKZHRzSIGJm1WYPCsTL4ewsZeERSQHikTyrl2VS9bh8pMyPTTq/1j3T5hWzjHq01DJzWiUTI9lY71256m6rp0RGqKsMXjS9MAB21oC4r7wNjJUj5BMBqeel8iaGsRLPiYH+zgmaaYsCcQqM1+sz13F2uMGG1ofurCik3T8q8woID0xfdHUSsGopd2hCDVxe+cbZoAu+BYoRFwU2ygQEqtyE2DZIriR9Az5l7h/G9j1/i6SltSuvhDoYVLZDlyafYYe+z05zNdK+/Bmi6njCwCFiUo2oEEj60LnWGP/6AAsxpVkHFNLnXLB7yvRcLXP8pBXbfnLyHMZuFDiAw+5B+LiYxAqf8ZQF3yGys+34VNE+0/niEuMsmqvUahhOU28MYn7wi8Yx//DtvYnlKG029JDEJhZ5vSu/0hshzZyRrvECOkAqdlCmB6yVevKDAvneTPruQMw9pS5iWphnC4/1pHGEdsSfQusZkh56jW5hlWeLaIYv6FMYf3FB3UTMHxzTWJ4me4tJY2LJNCisObiMhEfFCzZZzXDpzzDBAyYp4d25T8JtlGS4vqIn55uEEt25RI3GSRpgyvUOp0Jt6j0YjhJY27+l0zdfcQHBEI4T11IG6NpDOPRikNgsAvTTEDtOMlnz/L+Yr7FyjhOeb33oPjz6hwDo2BrcP6OebqkTNSpD3DobAnK7bJTkqiTBhtUZjAcGb/o1rN5Fl9Nxk1EF35JLgCiF7azpvvc8efIBnHCh8/PFPsOEDbO/aHRwc0iFz9/Aa+ky1OTvnzbpcQbmgrqm9SarQhTdvrpTyiVUkA6wKun+X/BlineN6TPel2x1glhMFptNNEMXOxErCERYa04CXCNZ8L0RTIeCNtN8feMpJknZxckqJy9NHn6LP79fpJIg5qBmwYEKiNE7mTmwnxrNHdO++8fohBAepYdxBxAmlsa0nlguWdGN8oiqDAC4brKvSH2rmymvquvRBiKNerqanePXRXwMA0nqKkBv2wyiE5gNzdll5o+nOkNcQJFCzcFZVIWVhB5n0kCS0BxR1jX6HAt2gNoglF9xKpsWLlla+WF54RbEMOSYjKgQs5muMxt+mZ3h+DmvpmrbjFzdYAwZV1aDkfbgoNQLNCYi0nlLVGO29MLOiwsp5/nEhYzzo4HBAe32KBiHTl4KqguDkx6BtCYmiGDsDmk+K+YDHxwuczWhPGO/2kXBAslzn2JROabZV5AthW3U7J1RhBZY5ffbJ+Qwvj2htr4rK0zCzokLpPMdcAKqsb8NIZMzKRCS2oNdMvczWOL+kve5yusE6o2va5JQIRt3neO027Y83Dg69AFqadCC8QmpJ4gv82cL7nfGzKApP9VQyQMCF8lBIL8bRSVOMX6Nk5f5rdzA9I8GljMUkmmyNigP1nckAxtBGV+aZFx9rTITACUMIDc1K2467KwKLsnGFVQWhnGdxQGInALRVrcp6A9y9TSrHzvdRiMB/r6IqvcdgXdZXjLWt98nzBXFdo6ldYQqwfD6ORwMErlBrG1SlS+QuEfMe4+KeKA7RZTG98WiA0ajn7/OQE9heJ8SS55hpGgz79JopF1bzSlPbCQAZxvBMONv46w+tQM3fsjZAySI6pVeCNmg4RI67fTS5E0hpvCJuHIWIlEt+JKrUxVJcIC20v3cSBqFTq42Vp7YaKWFcW1WUerETVyCpywzPHlFM8sEPf4qcCzuvvfk23vkKqbdODva9mJlTmEQtvK9jvllAcQy5Wc+w4jm2EQIbd04LwIZOaZYps00FEbfXueY9ZbkpvICLBSC4IN/I0LchrFlgzmqDmMWAgl7oExutWzr3bDbzvoJxGKDDiVDM91PXtY83a61Rlk5l86pqp/Vqo9Yar2btDlgp1c8VW11SKgPh9yXTGGjHBpUWARfXAr4fy9kZpi8/p/u1OMeww/Ojl3iFf12Wft5bphsjDOFYppCtuXsQhL6YA8D7fgZSwk9aZxHaaB9vQLTFKF1rn4yHUYj+mM79siygF18s0LalgG7HdmzHdmzHdmzHdmzHdmzHdvySjL+XDQSsgQMfA2G9iEqdLXFZUwbfS0eIAnauh8B4QlloErFwShzhBgss3Dg5RnJJ0GmTFCi5SrGaL/CUZXyfBRJvmHsAgJClfxUC5NzJWicx5pdE6RCxwoYbq4v5GoqrQYG4mv26JlWFDx+RR+FRXWHGMtdTnUNfELS7Of0cq4SqdIve6wCAz6caIVc0JoMI19hbr04CBFyOffzwFA8+p+sIezFExjS/xHl+lTiak+hMbhpMuFJ59vIEn/6IkMO3f+tbaFLHUWifwZzpY0IV2HAFSIgAMSMyKSoIZxUwluh8m5qX/9G/8xvoBvTz//t//ef0XT7/HKam6upgXEOEhDSVdY3aOPnaEgu233j5/BzdLqOOE/r9LL9EylWdGApWsgBAuEFcM5Kxqr1YTpx0cOPe2/TejKYsV0tPn9lsSqzWriJpEcVOqMCiQ4/iSsUmwHrGyO3lBcIRzbtJPseb98n64Wy6QcZoyOz5JVYRVVjv3yUEUCmFM6YOfPjpA3RiV/k0Xnp4f28Po9GQr0mj2bCQDNM1rDXeHqVpGtiC5arjyMudG619lTxSAoarrp2E3vdwZw8NG2I1RYWjpyQ2MhoPMNBUlc02Mzx/SfOqVyuUjhZUOUqBxoSrpHEIKDjp5AJCOmGUCMMRSf1Lq6GZBnb3Gq2xe7ffwr/5CQkfXcynAFM2Xz7b4PMDEqYZph04o5prjMrvH+zg/ffJKqUoG+z2ad2ragPJYgZB0Faz8ryE4QJlNDkAAOx96asIu3Q/VqsVnjwnpHpdbCCUa9oWviAmhfTeocv5Jf8MSBix3mDl/dDOji/w4GfvAwBSJbDHKHSZbyC4EvmMqbT379yABqGPRVHh62/dp/u4miFlqo0IlRcLIHDECaaweIy1yBmhiOMYQeCsS7T399Ki8dSQUIbej6vm57k+f46wpIpvICWiiPaPTqcDxRXFPMuheZ7maxYtGEwgmKJbNkv0eO6mvQkC/uzSlFjzfruzJ1DyHqU2tb8vlqvXdZG1PqShgg1YWCJSKDUtyrj/ZRQL2ru24xc3HFvC1vBeYHVVe39AE0hkjPSRbxXNt9FghMmY1oCz/+lHEYaOCrdZQ7oF2khPM86qBivHwIhiHNyk/SYd0Bqez6dY87xfZRUGrmpdVrC836dR5MWsrLWQoUNw6GdVXeL4nPbYh0+e4ficBaUsUDUttdX5jDnxobqoYc/Zg7WOoXnPawyw2rj1rLFk9LzSEeqgw/ePLujpqxWOjmktFG/tIWH/QwGLhs9VK1pRtkAGXmDDnUu61igZvYGxHm2TQkDyOd7rRrh3l/aVt958G+sV7V+Pf0biag8e/Awnr4ix0E1TxB0+3+PI0+jLOofiGKzSFQxbWoiwtXBy9DCyMmBkQgZe7EXIVtjCiAZj3isif83Kf1dtAM3PvtENjEMaRetb5sSHqiJDzihuYy20Q2bTCMMhfUae135fyYrM78POpkhrS96OAJ4/fY4pewImceS95aw1qBiRM02NxtkXMJpoYH3sIY32bTVSBG2bSyC9wEakQkjlBM4SflY9KLYvKhrTssygvG+fUqL9d0GALj+vvHTei8bL/A96KVKOd8qiwIzFXobjEXZHFIOptIOky/s92/tkywU++/gjAMAH7/8ADoBdbdZI2can2+sgZOuB2lMNLTKej3WxRlkxpXo5xaJg64e6QsECMyWsFxfKmT7bTboAM/HmywxnJ/QsqnINJVqBJRU6pk7gKenOD9nUDQRbgIkrPoCr1Qrn/GxX67WnLvbSCF1G/hL2GK4koH17UtviAWO9t58V0nuENhAQggWP+NpkoLxdhbEaoWxtaFwrVWMr77MIK6D43picWUsvHmJ+Qj6qERrEXYp3AgHPlhBXBOtqLwJjmDZOe1/McyIMQ8+EqMoKVeXo+bH39nPCL64Fiu6jhICnXnn0twEgOF5Ihj0vwPW3jS9MAH8ePHRwr0DDVE7dVN4brbvTx/6EaHRBo9Hhxa5Y4SYMQ4xY1aZaTJG9JOpDI4B8wRzk6cL7WKFqVY1Ej00klULhEsCxwekpJTGdOEbGBturskS8P+Lrb5X3wisL78lDeoC5volwSEG2XGY4eUIBcH32CsGYPjszNEHPFh8AbJ593ungmBf6zmiAfRaTfPnjnyDO6DpMd4gC/Av2qFFZhps9ukcyinDMSev7P/sEG7dRxl1wHgFjWiN71+5Yy8b7Es3nF1hntHFYZVuz2aqABF2rNCHe+9obdP/q3wcA/Pf/3T/DySv67CgQSEY0YayQ3lS405EoNkx9yQzeuk+JQpES9/1yfQ7ncZ0ohTlz+he5QBrwcy4CT72NIolTVoLs8uY53NlByhvs3q5s1cyqCnnlAo/aLwrvRWQtBP+70lq8ekXP0yxX2NshmtrXf/U38dMPPgAAPPvsZ+jyQg2ZanH/3h3P6c9Xc3zyEVFwx+OB38Cm55eoeSNfLTM07Jezy30BL16c+sMEaGkfNm79G4Uw2BvTXJnsDBEErneHqbQLjQ7305wdnWO1oiDEQuDpQ6Ig3nv7Fva4INEUZxgzxG/4GT9+dgLF/SE3b3UQc//qxdlLrOZ0H8+jU9y4dZOfhULAm8SXv/ZdehbDCeaXrLYrFPI19RxYfYmnzx/wM9pHxs+lz2u52EwR8DbSlRaXz6kHNltcQHHApVcLzNZMzWmALpsDD/dp7Q13DpEwHeZiNsXTZ9Rvl+drz5uHlJ42MUgSGD7w9w7oPQIZwDiKiKiQs4rYJz97H5YD06TfwYy9kkLTIOEizmCHgtioP0Rfc0Hicgap6T2yosKtO1RQEZHyXpGBEYj5cPSGtjBI+X2LovC9yUnc9QdfYHO/L2WV9kq4hqnrxeLCB18q7nnjWViLioMQGO3p2hX7EtZmAcFKqGUDJM7Ty8Ani2iA8gUZ3K+CCGHCymsbej5UfOHgJh5Ahkwfi0coK/YlQg3tlEvXb0PJz7Adv9ghuO0gDENEHFwFsL7/qjEGhimeg+4uxtxHtX9wA0NWraw4qKk2a4glnVtVVnlPXWE0NAfZdVZgw3OviVLsHbg+JN7Luz1UmVMdbVBwkWptMsxzem1eZF5BUtcFAj7rHVV8sVzgFc/N07NzlNoFbgkKZ86tS99b13h/LI28ogRwvqxhQp6nUccHT7VuUPN6aISAdEbYrAStun0UfL4v8gYN09DPXzzxxcbBcOADbinhCzeul1jAwPD+KGwDx91qdAHBCVkYCChHQx/2ces27T2jHhWvZssNHn1GnqjGWtx9jYvgUQLDlLZGG2y4XaVsSl+EdP2VRjae0maFhWHqOWrtX1vpxveLku8u+F4zpS0OfHIEKDjzueaKlyAs2p5Tvs9SwFMel7MLXHCRfjNfYDzgJDPqotIuEHf/ge9HlsJixR6m6+XCFyoC4du8ycuU7781xtU3MGDf6eli3fZONRECtEqzCSeco16CQdfpUqRIE7qA4YDOtusH13xivFmucMHe0EWZeyVRi8qrmwtBvsoAvHdhGkY+2R33eog5Dp3NFijPqfC4sy7R69O86o9SjLpMx+Yzcb1Z4tlzinHOLy/8e+gXz/D5Z9Tqs7+3D6VYPVY56qNpW7eMRcnFnPU6916ahdEwXDCRUkIx3XNvnwqe9974Enr83ObZGscXHCPAIkrZV7jRCDiuMhJ+D0q4CGsUEHGLR6UtLrloe3L8CosprS0prKe8Gm2hmXatHUfRGJ/0VWVNvZUAYASSpM8vkaj5NboxPvFzibGUAeDo0lai8cqfknQsQC1HridZNwaoac8rl46ivUDKD7mfpFc8uw2sm2NStG2yToXdaJ9YqlAhYlAoTmLvI7nerPxZbww8XdVdW2PQUr9h/e9FoBA4zQOtoV0dKI6928LfNrYU0O3Yju3Yju3Yju3Yju3Yju3Yjl+S8YUIoBukdMkIoBHe3ytOUnz57W8AAG7evAfLjZLTl8891W3Ff2aLJfI9oqD11yXMlCuOky58pQwGB1wizFWDnCksasAZvArQMHUtbzRqrlY1RoDBF9i8wckRVY/ONzVuTFjdh7Pi5XKBz7kZ/IGuYFNGgbREtaD37mqJLmflbxyw71azwMsTomZcXAjMIhZE2LmBR4KqKeLRT3HjFlWg1uNDnMAJPVCl5zBO8O0DQgCXsw2OZoSyfHZ6jHt3CcmYlQVSxapSolUvTcECNEJizVUJYTVKrtbWqqESGQBrKxQM8R8fn0F/7S4A4J0vfwkA8Hv//j/Bn/7R/0yf/fEUX36XqsOT0RhZTsjsIJ7g8YfU7CqbHJMRVYRWoOrYbjoEa2cgUBnmXLGZFwXWXIAKG+tpMsZo/On3/gwAvLDKzs4uEv777njs6ZbDwQBdFuHpdRJYS38PmcZmhMCTJ1Qp/nh6iWeviDb5WtTFSNL93x3sYperWLXJvCpab0AVEX1ZYnnKDfibDbILekZnaQzBNKokDdFw1fvoyQvETIP9rd/+BwCAf/WvvodTpkRobVrFJyG8KEiiIrx2h6q8N2/tYH+fPXV2aS3UWuGdd96lf6dCGH72P/rJX6HnVK9WCyzOmdbZbGBTuj5Xpcu1wZyriWHQR8L3Ka9zLwx0cvII0xl95u7eHiKuDL37la/Q9zt6iuOnRDNBo1GVzufL4NVnhKTeu/O2V6w640pgLIEhI3qRqfGcKVebiwYBz4lVHKLeo2vt9cYYXSdKqWGlwChKEXFVdjM9x/OXhChZ2yBkPysYi5ifYW0qNHUrygIQzdTRrIQQeMSIrq209waSQYDpjK7v9Xs3kK2p+njvPj2fi9MppksWwGgqXF7S758cnWMafB8A8E/+w//AbyZxHHtap0P3qqrySDYEAPYNCzsDT5OSVwQWVCeF5Yr5/Iwo6Ov5MWKuBIZJAsEVTAjlq/yBirzIQcGUq1LnmIyd4FMNKxnlkAGcJF5fGYiC0L46W3lVWsPrRgiD0lEKTQPJ4iGBKNFlhCQXCsr5EdYHyKNr2I5f8GAVUGIgsMqmjJ1yGFQQYH+HmAKHu2NIRiw2iznWLNayYQVKWRe4O6S50u8luLxkymBeeB9d2TRweFBpao8oW6fcGAZI2KtWNCXq0rU9GBSs2lmXBUTfrUWFgkVNmMGIKttgzWJdqAtM+kRVFYiwvqD9yBQZgh6zivhsaaxEzQqIm8LChs6nGF790TTuP0BTV9Acqzh63qiTImRO6dnZHEtNrRpHT5/BMAMliQOkjM7o5kpF3y3KKy0bjdEeDZKw3gsukjFqRl6XyymGvDfdZEXqX/3Od70I2auTl+ixGub169cRWbqni2yFTHPMEQiEjoXgGEAiR8OIo2ms9/bTukatHYqoYYzzrzM4PaG9p8uMrSSOETNSaUSIDlPxwiDw9Httjacfu3abOFSIeJ/Os9y/73q9we4usZUOkxQZsxam84WPJ107SBxJVEx5qvMcBX8vJQMopuMGaK4gI9p7D773Lt3Hp09PsFg4L0q01NdAoMOo38HeCLus4DnqdXHtkM6xN9+gtp9bt+8h5fajoiywYbXMqmjRcF2XXjgn7MeETALoMltFQXjEVIhWYKYxEkv21NtsKuwfOI/V6/jmtyimvnOb0F8Zhl4ASEp49coym+HVcxKNe379Fno9Wu9DppMaCI+SNk0DbRgNFAIQdA8C1cOgz9faTRGmFGsd3CLGy+Gd+5D8XBaX55hPCdE1ukLIDAPbaI9cRbJBw3Ho3h6dq0EQAZ72ucYZ+xtfXEyhmEEVheEV+jFQOySdhZSiUHrEq6or5MxMaGqLOOK9wQJOHc5otDmBZ4u28aiQLZwsZeDpoBABbOUUsxvokgX/lvS9g0ajy7FTJ1b+WUgZelZXIKRXA/aKooYQPABQgYDyvolhK7ZZV8j4LE86KTrM1EgC2iMaazxDsq6MFx0MpITkfUwIASnderH+73/b+PslgMZ6OVkppaf1jUZjHF4nWlmja1ycUEA9XyyweUlUwcIZbwdAh6lrozs3ENygybE6OfHwZS6Mh4GH0mDON7+xtAkiCtCwuXfRABdMI93o2sugwgqsKse/bdU/3YiiGDFnLnFRYDKgzzvPKnTvUZBaPLrEKV//2/cpuHnroIsdvp5nZY4l8+Av5hl6N2jDwP23MZs/BQBUL44Qx3RYgHvGvvnN72CHew9eHU/x/BVR/ISKYEf0sKerDW6woeTVxLvPiUEIYMqGqqESCFOWvFYGASecSSJQM3a+XuZgZhkShul/47e/DaZY40/+p3+GYkbf9eauQM19Ww8/u8CnT6l36jd+4x3UghUb+fl0u11oS8F0ZUsgZPpmsUGe0WtjqxBEtKFoa7G5pKQt7VEStEaIUz5cP//8mTcujroJDq7Tfd/pDTHkQ7fLvZ29yRjamY83GoPugN/PIhjTYgnSCNdv0yZW1iXef5+sG777G3T/n86P8eABJY6hDMFnFvYPB7h1i+xHBASePKfXlDrHG3fIwuP6De4R6P27+OQTSjQ++OnHvm8hDkN0+Abv7fRw7YDoHUoKvPk6JeFff++bdP2Qnnd/cHAN996gPsl7nx/hK+/Sa6cXr/DZp9QXV9UFxj2QKWwxAAAgAElEQVSX/NCcmC42OD+m+39+vkYU0QEQRwkGPfr5ycUcM6Z49npd/Of/8X8BAPit75CK2P/wP/4LZAva5GBr3zvYlCXWl7TOHnz+GQKmTI1YNbAbx1CsHDZfL2GYolDoBooPk9HrX0WncYqlKdIOU4R5byib2isBns6OkFUUoAJoe4+V9IbXla49l3/I1yFEu9GXZemTImGMnysqTHDIUu+r9RKSqY6PH1OwN9nto1PR91vMZpBMM95oi5PnNHeL9dobt1pYVNbRqGgdqkB6yegoTPwzClWA0m30CD01pD8Y4oItJhqWlBbVCiFfWxAnnvJbVZWnEymlfHGow2tsICwupkRVUekIEfeQxJ0EmvtM63KDeMJ9FaFBd0hr8fYdWm/nJ0d4/oTmfJFnAPcwBqVF6hTi6vbATDs95Ou72I5f8OAAoqhqT3uSUvge0WEv8QmglBJnZzQvLs6X2HBhtWK1z0FkcefbVIQajoc4ekg07suLKQSreDeaVA4Bsjsq1yt+Py7KVJU3dNdVg4LVN0uD1uB5ucGElYjjMMCqchLrTL2bjDDkgDw+MdgbcB9Qp4s1se9QbZaQnOz5BCUUMEVrnaQ5UYJVEC5JRIWSi9FlnnulSuWK2fE+BGeim8Uapaa1aCA9jcpI6emNkMLL9LvijxKBV0AMrEDEeyGMhYKTn1dYTum956dn2GP7he4e/fnWu29jxnSz7/+bv2wDYQNvqr7OMmgumsfdBLGjsbrWpbpCXTsVx9obXldN6eOxqq4hA0ddVPjjP/4/6V5zIDnsDxBxj7+KI+yypsNwOEDccWroof/urrUCwiDj5KjUFSpO0qI4wGjiqI1DrDKnHCxQcqFQha0audu/i80KkuOafjdBxImVMKK1WrIGNSf0/94//E0AwIuXZ3jw8CkA4MOPn2K5YiXagHrxAOD2jT3scbywN5rg9dcpXnj9DTrnx3v7nro4Wyx90qeU9Irml5dH3o6s0cb3/g+YYlwohZx/nxclAtYlSDodhJyszJdLHJ1QTHf93h38ynd/DQCwP6Gz6tmzJ36NKBW2itNFhfNjooY+f/YQE064FBdpwyT187WsMhhO/tNOH+ODa3zrdhFxDCnjGJbnbJhwL3mlITmGnp9f4JyL5vlmCUimTaKB5H47IVv10n6f419eJwBQ5qW3hYHRCFm9PI4UQtXSdF2LgeQsLgzge+WslbBcQs3Lyu8vgQrQ4dYsg8yvAWePQVoMTrlX+PeTMvBrIVCq7XuHQOMALC5sS1h0ONcIo8D3f4ow8HsNpABcUcCVeo3wZ74MrlBEm8Ynz3GofJtLFMXocCwuJP1Z121co6vaKw4HUiIIXJuc8RZpNg5hXBva3zK2FNDt2I7t2I7t2I7t2I7t2I7t2I5fkvH3QgBxhYooRFvhFlJgyqjCiyefYzFrK/eC6QXuTxUHKNn7pVQxQqfKlFeoGfq1gfJ0ACEa5Bll3QPj6FAx7JQbe2vbevIYC8vZ9yqgzBcA4ritPLjR7/dxf0BZ9tMHP8DvfvfrAIDlMEWVUFXnw8sdCFYH/ewJVTxmlxLXD6iiMZ4u8a3XCfm8LErUQ3q/k53fxcevCKnJP/lL6AtC+A7fISTnrd0DTJl2tinmqFm57GBn13vd1ZHxIhONbVr42vWrGoP1gu7zaJSglzjouoHzjxdRiBGbzM8u5zi9oPfe32PEQCj8+m9+h75LP8CDn/wZfV55iQ5TPD9/8gl27nB1Y6BQshG5VU7Vy6DkSo4JOt6YsykbGE2VsrWuoBgpUCrAH/4vfwgAmIyJ7rC3dw2TCaF3e5MJuj1Geo3EkgUF1psL2DWhL850uzsaeMSlKkts2Pg02R3DgJ75xbqCZMraG2+8g5GrPHNj8mt3b2HF81VYAcEN9v1BhIQrqqYJcO+t9+i9gxA32bcx7hBiVOQbvPUm0TSUMPjsU0JyAgCDHl3Hu+++hruMLG82uTc2dap2q+UasaRrKpMMNw+JCvxf/tN/iiEjP59+/BGC4HsAgLOTEzRMAc5ZqGB/bweLGZs+rzcoSnrOtw4PYARVmzdZg83CKYJp/Cf/+A8AAMyWQp4vvadTU8tW7U4FvvL84qO/QpfFY95kJbuiLLBZzfl7W1/VjIcT9Nlkeb2aIna+QrX2BtS7I/p9bYRvLn9+eoSa0XWJFgG01qLi96iL1jxdXPkz4up1WZa+sbrQBSL2y5OBhGKlvFDGXjluyqpLO3GIxrDS7/4BLi7pfnUihW//yrcAAIPhEFXlhCW0n4fNFQGjOGrVx7QTMNK1v2YVdVqj+iaFnNM8zOZEMRdNhd6AxW1UhA2rDwsh/Pw1xnjKyWhC9/Hs6TNcu0b7UtzveY+lxdxiZ+LQIIOalc1CZfAhK6SenhDqEwXAekrz2zQGA6ZtpZ0O8vWM36ODmmmpug6wrpmmux2/uMEol9XCqwIKKxEzSisBrBc0by7O5zhn1GmxalAzFazxqnUCmgmeJomwYSOs4+kGkTdLDqAYOUGsUDKyk7NoVZEVKLjiPitKNE4ZMQpQlkwzBrzSR60NSRYDSJkGGTSt0EO5XkEY2uf6aRc7Y96bKqBi5EeBK/+BhORzydYALKM9TegZK6ENvDJf1QDaMZMi+nPUCWBrer/pMkPMaGAYdxEnToxBQvM5F8WRV3d26Iw2jUe8RKAQs4iHMAaGqWyBlN7Pdr7aIHfPgmvxne4EX/sGoVhBqPDqiND4TZVhxvvAdLpAxOdL2kugHArBYiQqCBAwnbWqLGq+5rKuUVQOGTQIXIwWaPzNv/4L+jm/djIcYcT783gywcEhIUbj4dAjgIN+H3GHPVQZ+ZIACv5+q9XSK2ZDoPUMDKWn/Pd7Ej0+ayJGzXRZw/DcrMsSCSNao9EYExZ5MWhQaEZD6trHgl9+i2KLuzevY8RChMv5Es9eELulEcBkQM/izq0d7LIZ+3g4xi7TMCVf5zrLEPGZIyXw1lsUx331a9+AO5mWs1M8f0otC08fforpKbGptHaiONKxZIHaIOaWht3xDjopnQ2z+RqnrwgBXJ5fQLOa/qsXhO69/+P3cXl+zvdRevEhLSosWZn8+YvH2L1GrRUDZnX0hEDB6GNR5AgYNY46HYz5nAgEIFwWIIX3/8tZMKY8P4VhhOroxedYL+jzSCjK0WolQlZRl6KNpQTHAta24lSmLv11xGHghcxUIL0ypgJgrKOA0qVZAy+2EwUKCSOVIgX2+IwajsZIeE+pmgS6cegnn8FWeQN2IYRXJq91BcWqykIKH8OYUKPhNbdaMYtI1943+Cq3UKPx+4CE8PfGERAD2SKYENKLU1krPIIfxTEUn+9WGO+jqFmMripyZEwfb+rSt5TESnkFKGu1Zw2apkapv9ij9++XAP7csJ5WmWUZalblXE4X3pARQiAOXN8K/dHoxgdOQkQQvAELa73Rai4iaIY9k50h8UcALysMKSEYxl4vVyiZKjnY2YNlxap6XWOX4e1O2vEKWG4TjKIINdPHzp89wU8U/V4N+qg4obl99ya+sU+bwZTVqI5mSzxc0u/PNwvos5I/+xBzNmt9pS2KXaIR2K+lEHPivx8wpxwY4oxl11fnLzHkICowIeZntNjvv30b2gWHViDgCeskuIUUXg436SVQvPFGULCcAdZNhZAN2OuiwovnxJ9JOkQREB2DmDe5t977ujfd/PhHf47HT58CAIyocPMOLayqbqCUszvg3gKtUJVOIh4QjVPq6kAL+uxMzxCzqmiiRjg5J2rL6TEFmJ+YDyAil6wnGA/pwDm8cx3X79F9vHZwA30OTjK2CBHVIcBQ/9nZKeasKvXld96GYBrJ4+fnuJyyomkQ4zbTQV2/xu39Xch7FGSP968jYKUrpQRqtrG4OF36vpEkkkhier9vvkd0qefjV/jRj35CT7Yf4/YtmjNlViJk64293QluvHaX7pMVGPG8qvgZZnmNC1aZe//HH+E7/+A36Ppe20PDwcHtm7dgv/MrdE3nF3j/ferJO2Za1/7+BHfvOd11i91doo586Z138f0fUYB/dPIxBMtbj8Yd/Nf/7X8FAN7K4+Hjxz5pQiBhHE2jMV7BLr94geOnpDpWVr9D79XpYMrU0Wwz84f8/s7YK2QtG4swoueVqAa9lNX2OHEP4g7OWCH26PiV3zWFtH7/kFL40yDpdP1c96brUvpkcLFYoHZroZZQkaOJWpSsFtgf72I6o+91cJOuZ7MusWGV0M5gBw8eE733177zDbz3TSoUWbR0zyBoVW5dAhgI6a9fV7XvuyBlMLrWENLTPuqqgGXZrumUgoBhFKHxtBUFEUT+89x3bJoGO1wgyDiBHI1H6Azpu1bzGbJT2n961+75YDSQFiiZWhcBN/a46MLBrykqNHzI1EUNzaqiup+gWDEPLwpRbCihQDBA1NB1b8cvbnCbMuIw8mbVpq5RseKmznLkbKtTVA3yysnjKyQJnTuNU9mExpwDviQJUHIQ0qQRLpnqudhUCFjpORx2scP0S1diXdcaa/6MzLTm7nvdCTp92gfUKkfFgfFqs0E6Iap9wHtQucxwzknry8sFZErzqmwqrFldO41CDLnnTYO+nwwUQi6ERipCyWfixkqvnClUhNSbLxdeSXFvl86cQW8M1wegjYV1KpXQ6GgnRV8j8X3G0vfcCU66s3Xmk+A4DqG4EARjfY+OiiKArZa0sFg7VXP+PCUlhnu0f3/l699Gd0hr/KOP38cyO+f7USMRTO0SAUre+qUrzMO21yasV8Osa+33qcZYx1KDEiEup7S2HZ2xzNaYXtCenKRHODumxGY46SPt0H45GIwwGXPfPqtCi6A1Ij85P0PO/VRJJ/Yq3k1ZeuPwTjeG4gykP3D2XRHGXDzM87U3vB52O57WrusaeensjipvIeDaRNIkxmv36LvcvLmPyxntadYajLkP9frhBLv8OWna8wqjBVOjTV349owo7CBxrSjjEaKEPmf/2i52ODlWUYyPONmYXdD9lAro9VnDQkQ4OKTneevOPZyd0bn56MlzzPj8Ozs5xp9/Rn19nz+lwvfLZ09hWYFcRsmVrEhB87w5Pz3FMYMPu/s0f4qmRs1xNqCRRC75UQgSR/4LYDleE9J4xfWc13pZV5gv6cw8Pj9FzcUX0x7HdOZzAcYIQHKCV/FakJZsIwBgPj2H5jkRSumY7JAQcJ3xSgr41ITndFMbD4YEQvm1vDPo4x6rm/fSuFWzFYCEU/N0rRfGq7ZbKNRcANZN4/I1hKHyff1VrQHeFwumq4fGQjs6bhB6mwdY6edgYxpPrXTJmLDS9/0bK3wSb6VEwNZNgVJeVTfPNqg1J3uu11U3KBhEE41GwPcm06WnlzZNDcvvYaxB7eK4v2VsKaDbsR3bsR3bsR3bsR3bsR3bsR2/JOPvMIKnP3XToLwKaTsUbpP5DD0y1gF2raEL4OFjaw1RtwDYpIfaqeRMJhjfJoqczad4YFggwmpkXEmKTqlSIrOcPF8AZFWJMxaA2DQNcm7SXGvgJlw23EC5KgCn35+fr/DjV1RNnLz1LtZcEXjw0w+gOSsfj0bYO6HK1uSAqpQHh9exy1/w+uQaLhjxemQ7OFnS953pFbRDB5IeRq+RKXma0odPyxWqiirn1XyFzg5V8WazM9RcUR+Pv+a9z4SQrY9IyeaaVnmEIe0mnhra6AaSVckCABE77r527z5ybrg+PWVlx70YO6xoGQrg1uvvAAAu5xn+978gD6Jv/sqvYuc6Pa/j0xlqpnWqyFU0Euiam1Ob3FP1kqhBaVkYoIygKqZ9JAqDHqFAIZce1+sVSucZtFogY8rv2cUxPv6QUK44STFk76Veh5737u4BMq5UPXz8kadsNmWJIdPhDq7dwDt3qRI2v1xizfc36TGlTdRIuPk56adQfD+apkHEVePdAwXJ5tfz85dYHhFNrvfrvw0AeOPeTaSO6iQ1ZudEFz6bnfqq1PWDQ6RcUY+i1FeBMka/JpMxyiV9lxs3r+PgkK6ZBAscEiYxYIQyiROkTFuJudq5vDyH4YbgXq+HW3dYxezuPbw8ZZqr/DGWC7q+9XKEv2LTd8VUoaaoIJwiJFpVN3EF5QIENlNClc5OmOoynKDidZrluffbXGGNkpH7nd09xKzStppdos9qZR8+pkrnw6ePYbhKvV7NW4NiaC/2ImzgvZWEVFBp66kDELL76cefAACeP/vcV8eUbPx1jPdGSId7fP1TCK6ILtmQ19YZ+gOaP4+fHqPHleKdycjvL9Za8hMCIXyuadtTQbX269cY7SuqEPD0tqgovTLfarOGZFSyZq/Estf31KnQJoj5M6SQ3gM07fS9UI9T+Ozs7sBwhbDYrHD3Du0/QX+ImnlIDQLkjv62WWLIa5J73SFlBMnKpdPpJQIW1MqPTrEI6EXDYAABojsn4Ut05Bd7DW3Hv/3hqttKWngykrGonVlwI2D4PJMy8J6tgWgRQC9TWVVYcqW7W0VIGRV7Z2+MfEH7/tMXR7hgBkYFgw1/TFwzZdNa5M4/q84gmIp1PYoxGtG+Pd/kXnkvqzQChx82tK5fnk9xsWCREhnheE575Hy9wZyFz3qxxk6fKG6O3SNVBqOceMIIXRaf6imFgkU3KhH4eS2UhGSRrp0B7aVBk6PKnZpnBbBwlBUVcr5mFcqWxgXrGU8ubijK0rOcup2uF7VoTAPB+4NUCh1GPJNOz6usOoEaKQMYZi/ESQ/D3QP+jAgZxx/7+9dxcEDtKjYIUFUUzzjmkGksFIs3hQZgQguCQCJw5uiBRcioZBBIpB1WjnY+ZE2NBbN3Ti/PcXxG9PReN/05BsSQWTs7/IxVqFDwmV6UhW956PV7WLLCci8dYm+fvtdgNIaneTjhj9B61dFOMoTkNpIA1lPq8yLzaKaSyrf7JE7x2VpcZ3zj1p2bePqczqu61BgO6TX9Xoo+n6FxkiAIHIOK9tCqND7+CuMCEXs1itUaklt2ekmKgM+2u/fegm0cXZLu13J24a+52xvg/n0SmLl7/z4ePSR6b7/7194Tejqb4xGL1xy9ovMaVQnB9NgoCr3AiDUGJceFy80GL1/Sv9u/RsymxlhUjMonUQDJTC8ZBAicAFDg9FFJMNTws5CMtFZ5jRUzfOq6QNxhpp3WsNLFCK3AiRANDDOGnLm6gMWGFU+nFxewzGZLItme08J6sURtWpVMNzeapvHMPWMNhkw9Hg676DFrsKoLgPeBQBivKis4xm+s9bRQKYxXhm0a3bZkGN0KEFU1IlZDr536Zt2gcKq1Qv7c1PWe0CLw1+0ZSsLAOtVgSSqw9HO0iB00tHb3MUDEDALBaL/WpaftV3XTfrYuW0qpqSH4FwYGzf8bI3gHX+rGoObFJoz1akJFUXkeatRYLx9+1RHTvQcsUK4I0iz3drF5g5K+b3z71/DVr38VAPD9f/7f4ONP6MD5+OkFEqZWDEe0SFdliQVPgrDWWHBfw0R38BpTBjtVjc2GKUx1zYasgBPv+qtPT/CsiPlnGQ5Y8XFwcoSHDymAzGqN8xltqsFLOnji7iPsjek6bu6M8Card6WDGJdMHXm0sXjGXOGZFqjXTEfUfAjZE/zalKieCSIYOnvw6vgTvMtqo1IqL1V7dXR4YpfzAjlTFKI0QtM4M2jtVYhUGECwNHgaS5LhBTC/pAAzikZIOGXrpRJV5RLEKa5fJ5uK3//9P8C6JKrm9773ZygF0Qskw+OoFaSk67BNBWsL/mwAheP9JwgCJ5st0OVF6yweal2jqimwjlRLb4vCCBEH51W1wckpPU/nZ/vRRx9CS6bWicDLS3/yyU9wekTUizt3X8d/9Af/KQBguNtH3fCi+IwSnxeLC3yWsWXEaoU379P8ybLM92Nu1ksoppoe7IyQCFbXdNQZKNxmNduyfMdbb0Br3L1LCWAaRQh4UaeJgqhdrxhvLKLCbE4b/Xg8xsU5zY9eswPlrBHWS1iG8vuDHnqHnOSyUmq9LvA5mwfrpsTt194EAAyG+7h2g/oClIqQbVqVz5CTUmcWrkMFzUFgs85bTrkxfr0rpXzQdXRMB5zVGpZ7EiEDPH9G1x/dvu3ptnVdYbWie5NnG/TZlDphNc3L05co+D1s159vsLWFI06Ipj2MK1X5pMilptoaTLmPsFgukbJS18HOAEFMC22QRnj8kuZHICwS7r10tKdu0oGzOf7o08f4vf+nvTdpsuS60sS+O/j0/A0xR2ZGJjKRyMwCODRZJAtkkSwai22mhVTSomVa9rDV/9Av0B+QNjJt1BupF93WKlWXkaoSW5yqABAoDJmJRI4xv9GnO2hxzz0esG6iN21YCH4WQFjkC3/u1+9whu98388CQ+p6s8Ljj34PADi6cw+HlBSy9oo8Dv2/ucLU6ZyDUP0WG33Htm3hydmo1hvIZ38XrkH37OCZ2bGUKbpNhJEaXgRSCgiWz6HxEgrLiwAlmk4nfACb5SWWBDNO8h4W3zaeWf+yWUg8LJZzzBdh3StbYX0R5ow3Dps03PMoFSjLEMRPCgcTm48H+9Is9u1Y62AoC9h5AUHJtzxVmFEPlNASl+SANdbBoWcUBICm8VgSFLGoFSeYvnbvPsbk4D979hy/+fuwBj54+BgXlPRBFFpvLZa0fxSZYqdmVJbYJXr9yekZOgoqYDpcVgRhugjn+HuPjjEnxyjfvQHnw2dPlwucLsP+oM4XWM2p94/WWaIkPPftLDG9HsZj62CCkuDmNSQ/t2wcQ9UjlPnDD55jZxKusTfLUWTESD1RKAjumo8y/ruw8/SOFhB635orbOkmtsTAcaLIK8XSE5A9Q+qS+qiF79jZ11pB0Pc1dYWtUVhzf/zNP8HWdjiXXp4e49UJybq0PaTNUiBnjIM1PdthQlA9e6X3SwiFHXrPZ90pXauD6aLMgkNrI+usgSJoX2dabDbUl3hGYuJKctLfebD8UpJkyF6E5OGkKDHKQ0/13TvXsSSo8slxgDxWtYFwfQ9pQuegtx1aKiK0TcVB387OLvddRykmCY8JMavfe+N1fPxJkLS6ODnDhIJ/BYGExjdLE/7bSOUqrYOld2tdhw2171hvkRFDtDQ1FAX65XSMb34ntAq8fj/0yL989gxzkk7IsgIH1Kc32T5EMQ3nVV5OoIi9uaoqTgqk9B0ik9zvrXUvLGGdhKR367oOF7T3RzjvaFyioXdVt2BpEJ1kGFFLQCY1By5Oee7v7FZhXl5cnmN+Ge7fdA0XfLyUkIRDd8LxPXnveV3GFhAtFWpa94vFCnEJTfI0tCQg9MeZ7gp0kT4Tg1PnPQf8QgiMiC04z3P2X9frFTwlRZXsE8aWgl1nHSeznegZW4u0iO2AgLdYUKJruVxDk08X+3YTIWEi1FMK6MhMKjy8j0F1j48Voo9/YtuHh4BSfYDYB6KOA/A0T1niKibV69pzH/OmbiAoaa6EgyJ5FC88Q5mVlJDui2UgBgjoYIMNNthggw022GCDDTbYV8S+uAIYf3ACikk+FIsfitZD2IgvSOCiOKlz8D4KgBOEC57hULWxOCPB4mpnhsefhczQz9+vMJ/HjJHGmj7/8GXISG+aCobSB0flBN/9Qci23D46wNGNkMGezyv84ue/7r+biC8ePQ3XePfZCm1JTbKdwXtPQxbG+xSmpki8EQCJKFvSF1o1G7w6Cz9/+NEzHBDz551bh/jBm4EJ8kf338SHdahuvXtm8fQ0ZFyebMLv3PxdzJ7/Jvx8/0eoqFLZVHPcvh104Zq2ZWiX0oorCzPSo3mxWXLzapopiJhxtJ4b3q0QMMSQ9ejZO7i+//VwPTmjMVqxZlDbJfjVL38Znuu3v8ePv/8jAMD1gyO0IlS33npzhd99FCo+ry5DpnKz8VA+Cvl6rClDIrMUoLlSKMnMZAbAdBoymHu7oZI2v7zEMk4yIRgmKKVCTtXM7awEyvDznPSrzk2DnLKkR0dvMKxyuTqDoPn2+OFDvEMw0h/97B8jo997YnU18xMIEbJx1eU53n8nXPvZs2ess9O0NfauhTEo8xTXDm/TrcYKrePszNHRDXzrHwVymGpxgZTIhbIUzNInvWd0tGCV4BbTbYJNLl/h6aPw4cPqCIKqvnXbYvcgQH50lkJTZnBEFXIxdpyJr9oKh9dC9dE5zWtO6Rwvn4dnfPXiGPujsP4uqfnctwaOsoWqyBCz275xcHG9Sw9JVbsFZd12ugqC4Evn5ycsEH94sI9tgswEWBTBk7zDkphw50Re0nYWgrLsWSGJYQxQeUrqrlSJjEKq6LPrMVsa8OrEcjsusSG90HFWsGDtWdWgbcOaOzrYw5K1wyiLahU++Cgw99567Q6OjgKURsLghJr6dZpwhjhNUm50j9XroihwSWskTVMkSYRZKZjYFC9Tzvq5rsNHq1B1f2srjEvTbrCgDGyWNQw7S1LBsBtb1QyPLYixuFpXsJEptVNwJsJgBZNnibzEqAzZ/iyRSOl9plRduHx2hlc0dpvLFbaK8NlWNNh5PdxnW9eY5QRRlRIu+eJG88H+85snnVdnJAxVzTor4Khqo5WEi1Ax75jdr7USINIHR2fLum5wTjDoNJWwVZiH5qNHEDrsNc+fP8cj0sJ8djrHis6uSOahlWLEwni2jWtH1Dpx4xCzPaoAbr3A+Uk4hztn8Pw87EfHr8J6+eSzU6xc2PNsodBVYe2sbA1LYshN0+KS2goy2li17yBluNbaLlARLE7LDNduh31uNirR0P6wZTQckTMtT8Jcn58cw64IArjJMaMq0U62h5Kg4OOyREpwRC8Fa5HF4kHT1syYbIVDxeLdCe+hXdtwlTAzbRC4BrDaRBi1Qk7nXWMkTo7DeK0WK7x2/w4A4M0/+hr7Zp3xOL8IqAsT4biNRUXVwMpU6CIkT6GHoirPe5YHMCKikvU652tERlDT9YQTCRIkhM6B1OzfNZHgT4G1AZVQ8HSN1hnUa9KD9GdY3pbxdeIAACAASURBVAsVsjLJsHRhLr16FnyMs7MLWKo4luMcOWnSCTiYuofw7RNZjhISaayoEOJCeIkR7b1v3H0D3/tOmGOPP/oA42lkUrbMLCml5Gpr7H2SUkRuE0AAhtpgfONga0LwNBmzlCZZgSKPayCc13dHY6zIb0kSiawIPliRZlguqB3I1EGcHUBlW27r0UzUkoKIsWGdZ61J0zmuKkF6tARb7jpq9ekafofz+QLrZUC1lcUIen+fxi7jdytSBUfn+vFnj8N4ffA+aiKEsVf2eukNU3QLL+AQoeceBHjjsy+RYFTVZrPBjKCoWZpwNdBahy5iWXx4zjjuAFB3DpbIIqeTEXb3w/iOypJL2V3Xsr+T5RkEVWY1QSidbHk8hAe3ciRJ1kM14XBJ6L9NtUF7QcoFLT1Tobka2DYaNhJ7ejAzl1IJ+72RmAnwcL5fFymhfZQGHL1wawRrYeqkQEoorchKm/qCkU+dsxAuPOt4UqIsIhOxY9SjhYndIX/QhgrgYIMNNthggw022GCDDTbYV8S+sAIYmwmlVVAuNv4mnDmabm1DESWwmIwAyjJ630tFxAqWVAJtpBjuLJp5+Pl/+Z//V1ycUG+SLrA7Dhm7iXAw83CNxZyy9reu4ei1kJW/XC1w82bIMr7xxmt4/jyQEnz84WfYkH5G27S4oGzhrx+F75hbjRj3JuUW1kvKUOkS+/e/Fe7ZGCZN0JSVKESH0oVqiTEtTk7CvZ2vWxjS5Xt7tcb9O6EaeGsCLEaEbafeiKPfvoCmysWznRyvPg09h2/dPMQ2aePVtYGIDaK+j89T+rmaL5FTFrJMM3jCTQsIWLp2Y4CWsgOL7iFSyuLvUIP05fkFltTbo4XH734TKmXV6StMJ2H8X708RzoJFZw7d7+Gj0/CZ95599+GMUCNacwwtw4bG5vYBVckpkULrcOzO1tzr0pTkT7LpoOgKegBKBkJNRJ4IjPIswQTqhwuq9DIDdHLF8wmJZ6SXs5ms0ZBFQ3ngcdEMvKDP/0hEmpq/mwUev2W5jp2iimPb12HLNf2/i7GRGm9v7+HA8o0FXnO2lZr6pex1nE1vBxPcIuIXx5fO8SGqlzGdv0akJI1m2IjPaTH7l7oidvZ2kZC1d/6co6UejeLUQ5pY99WA0VZec4tGY/JLFRVJ8pjNAr3f3xygXfefQcAcHJ2gjG9zyKfYr0KlTpB42XaJTzdm7ui2yMkkFJDPpxFTVnLkyehp+LO/gHGNK/Ojp+hpqbpF8fP4V2onu7NZtxfsagqzJ+GKtunzx/Rc3SwEf9vU2gag84Z3msgFQxVvmVnIaMOWuwxBvoKmxDY3iO9pd0cz4huWyYZdrbC+7xcVkiyMB5L6u3QMoGhdf9P/sl/g5w1AyW2DsO7lVr1vT1C8XuM+51SivWzmqbp+6iNYQp8CEBQ6nB+cY7HTahE30OQFCknW/x3tWuhKcPcNIo12rztUFJP3nhENN4wSKkvoKlWkFH2Y7aNw/3wLs7sCD6nLPRsioz6wCIa4e6d67j7enjWi/kpsjysESEUBFW7s2SE+iKMrz/9ORRTQA32pZmPPSQaKb3nvMxY26rIM2Sj8L7WG4uW5nVnHYTsSREAoDUGFc0r4yVeXYQ1/N7Dp7i4DBnw+cU5k4xAyliY74mNUo3Dw7BP37/3Oma7YW15pbCm829Z1ajaSBFf4/QyVB1PFuH/DTQE9b3Yrqew73QBUYb9TScZCFwES5T0wlToWGd0hfPjgCjqqoYrogev3cOICD8aZ5lyfkT3uZtex3pBMgvrS8wJHVAfFHzWFNmIq6aAgInVdhv9mhqOqqtegSUQlHR8r5vNBoY0Bot2hDISmEVCGWf5uatqgQ8/CmfYum5Y41NnIyyImt8jgdJhzLwP73u1brHchGpbC8P9xmqkkcaeZAkoFdEV4Mrlhnrsqq5h6nvnDHQk4EAGCKrsKN3T4JOrIoWK7WpIE80/S+9ZZmi5XGN+Ft7RcnGGiipTINRSniiMZuGZxuUIJZGGJUqiol4yLQQmRGAlvOWKLoiiX0ByJXJnZ4avfS2c+wkqGJIk8LDwkt6X6HuqIzpDwLJOnTUNqjX5Wm3KPljXaVjanzM/hvBhjqUkVZLmI+yQj1wWGQRJf52/PMWHnwQSmPOLeayfIU1zOCYaokdC32tujUFD8y0cOZEYTbHmsqC5lCaCz6p6PceSeAfWaQ5NMlXOTPteOTgs6V08/jBIID15/ClS8gvySQkZiV2Uh+ioAq7B1XAvwHIfkR+kay1La3RX+1Ov9ORJCOYMkKnid1BRtdzYFpb8x4PDa7hLZDpOOGw2VKH0Or5+eN+jcmKvq3Yp983Bu8+dzbEaCGHRkC/StAYL6j3mXrpcwMU9rF73Pnpnwd34ChC0FiOZnveC+/dSmUBwrKQhiFRCpSmyLMqYFNxbGtFiMtcYjcNc2pqWSKmyubszQUrXaNuWK9Vd26AT5HP8AfvCAFBSIKKhQO8ak0Tgm98MkMLyxm1Y1uYqIAjS5dyVAJD+33kBv46iyB7Hr0j369krjHeilswEWZz8G8vCkJFVajbewoTYEM8vF/jNr0JQcvzyJbP0PfrkM6woqDPG4/dPwvd8TALykBop6Qe1KoOizSXXKTRNUt82EERs4MihrasL1izJ4bBN/76pO/zmncAOefryFb7xIMAYdnb24GljmGyFe8gmGRbXgp7burnAq+dBQPS7934CibAx6KRmWIJzngOGuCDW1QYzcjDTfMwwk0VrsIiiq6mDjJiBNsHlCemLJeHvTGVRb0hjpG2wJv2x3e0MT4np8rxaYUWXc9bD1cSemYTxb3yNlt7tqqtYXHicFtghkpcyaaEJKnZ2ucAs8P7g+PkzGoOKNw6BcDgCgFIaLUECV12F7JQ2aXKaJ9mYdR1XyzkLekspcef1EIDvHxziIRGS1FWFna0AF/mHT8Lv2sbhRz8Km8j2bIZyEg75yra4sRWedfn8GI7exZOHj3FOmlj3bwXSIuvBkCAhBabEKHf/wX2cvQpjaq3lIFFrjTKylNGm25rmc0QiknYwJQUsMTpu6jU7ELoo0M0oKUGBnpAC+SQchoE9K1zv9PQUz5+HoHm5nON733kLAHB04ybwbgi+DG1mYpRD0ZhiVcPFZm+AxZRFKmEcwbzpyBoXI9wg5lVpOnzy6GMarw8xj03p0y3s7IeXv7O1A0nJgt8tAqyyWV5ws7pIt6HSHj4eWdW8sdHnhVWSDxfW31MaKTlTp69eYX8/OHYXlx02y/CODo62+EDPx7sM9WlISKvLK/z0Z0GHsSwzfi+d80hITN7aGkticJWixmwaA6S40Xt+t0mSfE4nMB5wQkt066iD6gAdvv+chNv3doCS3i1cgxGxGnadQ0qEWBcvljitwrUvyWmbTiZQxDKsYLG9G+aa1wlWNL5pmWCfEg7a1jglZzkKOQtZwBJ0NPM1HK1JZxUyWkPWAdkkrLP68h0Iep+DfXmmCP5oYCKJJnb39nDjWni3k+mUIUTPXh7jfBWcgs1qDUs0dirCk9MEIxKoLkZjGEo8reoAawOA2e4+EzkkUsGSIxWP663JGIeHIcmQpApzStpuqpoTVqcXF6x/1VngmOb7ku7HCsk6rtIDguB0IzmDIGcfrkNC350KgqybDjUlMNrzE7h52FdePH+GOTFL3768xP71SN5kSDEe2CN46vWdCVZJ2NuePnqFZb2g8drjtS2lhKM90jnLe0/bhrFtTcNC1FmWoqVzyXc9WV5raiaNabsN1m24v44Stq0v0BExzPNnz/DyZThHJrMtaNqDjs/OsSLG4PlqDiEIgpjEsyUHBMEVISCjFiIUt+YoBGItAPDOYjQJz5gR/FS3Co40Jb0TcHQu1Z2B0JEcxrMAO8fF0l/pHQqJfwDIdIIdImVp2i2k1IPy5NNHOD0hyB3N0ddu3cb1mwFuHuGVQCC1iDpoZVEwk+VmUyNNowMfk5iOAyGVSFyL7NrrCxzTmRg+SpBGZfr9OQYDUkLR76y1aCNCURgmZUm0hyQH3ZkWraKAQcZASSMlDVxdjKFpoD78+Pf4+HE4KztnsLMTfI4bRzchf/WC7onccychk8gUaZkgB0owo7oXHoqguZG4RmnJfqz3HScNWjmHpXU4n25jlJPO4qTAeBye6/AwjNfpxSUuiWyxqjYc+AorAjkNAi9Z/8otbwqe1oe3BjkFxGVZ8nqqmw6SgkElBe9HQspeC5vG3FiHMRWHbt66g/1rYS3P53NEqTsvFJPXOYBZzSNmNujo9gRtLrKOdk0ffErJcFwPhyUVK3IZiS87NJQILRJLXjvgXIfkCkNwbMXgIhoEPAXrKhWwpKGdjhzyqE+aTfsxgIclYryYMJLwoDoGdmYTPgOKPEOSRW1CiZYmqrQd3H8C4zlAQAcbbLDBBhtssMEGG2ywwb4i9oUVQCo+oZwKqFGIFd88SPDf/uNvAgD+/VODT18SOULS64UZY/uMAAXFaZ5x46bUiuEY149uYEp6fuNyjIuzcL2t3Rl2ieziH/7uAwDAerVGRdluISRnx1abS1hqQj47uYTWlD2HxLuPiF7XUtOl6JBRZ6+3Cs4T3CwZceZHpWko7wDoWspAlWPYJpaGF5AgWnVRQXYhw7N/6z7+gZrKN598hOl+gKtOQ3IHDw5fQ06Z1vNH78DT8yWjEVPcKwhu3ldKh2wlAE9wjaqpsHcYspZC50AXMp+LusHTKmQdDkuH69thPE7XBS5Jn242jhW065hk4edHnz5CswmN5jsP3sDZIlTnNgA+eR6e8eTVOXKCDOxNQ2auyjzON6c0ygkSStVMZYqMq+IZNitqmG1H+P6fh+rh7/8+SEqowoI4ObBZtFzhW68W3NDurURLjd9JzNIIhY4ygScnJ5xRyrIMr90OY350dBOfUVP5L/76r3Dn7h0AwMefhKzbt771Hdy+E57l//mbv8WUdIw+efIYb90JDeq58SwTcvPWTdzOKVNDZAdaCKY6lsIio4z70Y0jeKq4WGtgYmUNgqtK8Z5HoxFSyvrYziEiFJx1DMvRzoVKEYD1+RlDW0ak3aTTFCqlbKfQqKjS/t677+HJk1Dx3N7exvffDtTb+9v7sH/5i3DfdD/puIShRm1VeDhKq1kp4aOsgTfY3Q0wpO98L5AF7UxnnKU93D9AmYe18OLZQ8zPA6TqJMlxSKQQP/3TH+PPfvB9AMB//V/8DADw8PEj/Mt/9S8BAL/84LfQET6hFDRji2TfJC4lJGV/45gDwNe/HSDcnzz8FHOCrjVNiyU91+RgC2WcnAJ48jiMzeU8ZDgPdq+ha2JlvEOSxQysgbKRhrtAOiHYljEwNE6x4d05x+9Wqc9DRLtY5YRCRVpTpmrQIcy9+SVl9LBCvSSSFSXhqSKX5zkKqtS9/sY1zihmk10eg/NXgajD6xHruXnVoKb95fq4hGxCxl23FlPaqJsq7KXzuYOP2pBOcEUmmY5QKIKxNUukUVhVCygi6BjsyzMq+sBegVRpraGTWKl2WFVRM8oDKu5dBtJFRAKRCyUpkxK0XjLqQSYFdrbDni2FxYLIjbIkgaPqVoRejvKMqwonZwusqnAOwgOa1uir4zNMCcLkHbCmfSpqulqho4wXUin5uaTOWbJQOAtJKARQBUsKi2ISkDyyGANUKTMXFzBU4dlUF1jM6ffNGtWCNDeJ4n76R7eZDMN0FZoq7AOJ8sgi2RYc6y9a13H2PEJBPTwmJKcw3ZqgpufabFYsueGkhaC9q0OHeRXu4+QirL9pZ6CoyvLs1VMmzDo4OMQFkVM8f3GKqpnTGHQwVM301EKRFRMk8Z5cDS97Ij4R0SZOwkRaeifxxt2A0JhR+8NqtcHTZ8F3Oj5ZMDpn1TQwrFsmISKxBQ2GtgqxHtQ2NRKaY3o6xgERmW1tz9Bswvx59/338fwZySEQPPLWa69hSn7S2dkcT548Du+qrbFFmsBvvnmfq2aTsu0rbrH6JcFkGEI4FLTv7+5soalIrslZvlfhemkQHSt2kHw2G+d5/25NBxElCZRgpBqUZlITyTJoNhaloLzDy+fB93nnt7/Fycvw3NvTbdx/K0g3vfngHhIVWgEiGkioIGsQvkJDUwXWQ0AmkehEsdRIrNbqPMf+LLxP01V49TKcDW1boyJf9uT4EpNpOM++8Y038L1/FMgIv//dcEZ/+Ohj/Lu//r8BAO9/+DGqdVh7SZZARSI2IVmr0UsJxeuFfFqlsLsT1ufF9jYWJPtysW6QRA5JAQgib1JaMSQ2SpEY4zEilFM5GbMmYN216Fg/UjFBn/fgNpYeJiv4PoV0LFUjIPnvpIiYtBC6NNS21jl2zCApbJJWoCUUQq4VRgS51xosg6fouqmSgYALQFrmcDLuxwm3UAjRw8o3tYUlUk1DpD7CGyhGiNmgnQigazfcSgVpoSPEX1kkrKf4H7cvDgBjD8yoZ7DZaOB//8t/DwB455M5pgcBcrK1VSIlyMY0zTAiDaIyCm0WObMrrtsG26Tddnh4yNpyWiocn4RNZ7IzxcFB0Dl79EGAqzVtww5XlmXsWC8uF0jo2mmaQTJLo8fLeS8kDQDSGdbmUFL0ED4dPg+Esm0Sq+xJFGE0SKk87uoC1SV9gwUkiaNjeh2v3w4T/Te/+Ct8/NHvAACz8zBG69NLHO2EMvbp5Rrjw/B8u9f2mUVJ+F5DxMFzn0aEm1jrMCUhXCEUa7hY41CTNpMaeyhNmn+ZxDQLG+9nJOY6yVpcG4cg8vHTz5CW1HtXZsyStF6eoqSgYjFKsCZ21qYJi/DazQcwtLNVOOGgxHkF20X8ds2MbB4TXL8Z5sfbe0Go/MG3G5w+DQfLq09XePowDKp1joWyLSwvvvju4TivgKqquA9lMp2gITjOyelLdATf/Zuf/xV+/WsSvKZN4a0338SLl6EXLS0kHhF08eHHH+MJBYn//J//M9z72ps01v07KAqCxTnPDr70Fo6ee2t7F2uCJC8XF+gIk71cnqMg8fkYuKS5YiZROMmB77rawFM0qKGY1TLRCbZIyLYsCS6QaJgrbGx/S6yu/9df/59QKlzjxz9+G9/4eoBuZ2kOQ5uRLKlnUgjkpHnYJQKWNl5pHBJaT11t4SnR8s7/+5fhWt/+Ab7xoxDIVW2D8Thcb7O64PeydAIX8wC7qdZnOL0Ih+Db3w0BaTmZoSRH0zkT9WAB59kZzbMiCL2CBFh91Nnpt+sJ9Tj++U9/gr/5639H7ypDRk7nw4efMmRN6ww3iDn4BgX564tXSC7CuplMCk5iNc5B0XvurO0hnmnKTsFVi0kBcyVAdM71+oDeoyYWuZOLU/g87BmnLe2Vp59BJJFVtERNzvS4LFFS39TkNQAE6dn6OsGyqw3GxNJ29OANdHVwYuvVCuPd8Kyb+Zp1zRbL56w32JjI/tfwvi8SiSwPYzrbvonl4wAXtbrB/Dwk5cbbNbzvg/DBvhxTlLjMkxQbCvROzhas9ye85XNVKI2E4NF75QQpeespwRW1VshJR81DoCQne+Ykjm6GhI+wTdABAzAqMjgT/vbyMsyxyljYFWXzVIKui1DPFqqNjlgGkPam9wo1JfZM7C+BZCZRLzT3wlvfwwqtcwDNVUHQKOsFa1/l6QwypxaJ8SvWKE3KEiBH3VbAhoKADenv5bLDhPbVtuowI7ji4bVDXvvGGe7Vg1DQtEY7YthNdIpt0jgdjydYnxHEdb3gsU7SBJrZWSU2m7CGn78KZ9GqWiEh5/Dk7AR1FKTXCheXIQA8PjuHo36ufJQF3TGwjja0zpFTfzNaiU5EiLZlp9h4x2PqncCt14I/tn+dGMuNwp3zsGd8+tkpnn0akr3n55eoKQnfVQ3iSZzEyB09M2XbGSgfGZMV65XKRLHO8kcfP8bFRRinWRn2sadPX8DTnPjk4cf45GHgd7DO400KlB587QH2rgfIsYTjhISU1KLgPevUCQgWCJ9ub6GKvXCrZey2QWsNBJ1tCfVh5VmBhBInmVIM462bCjXp2tq2gqVEfp5I7vlSHAA4+NimU13g0YdBS3Nx8gI39sLzHh3dxBukG7gzmSAviP27iVBK2QenUiGLsFhnoBLq/5Rcs4Cj+7xx6xpuEU/G060tPHw/9PW9fPkcad4nXVbLcO3JOMHtWyGB/vVvfg0AcP/+XTx5HObmk8+e4OwsrBePDqmLepYKPgYm2kP62JZBgQ2A6VZYF3v7l1jSPtEslqhpEo7yDBPqlczHo579kwLOrnO8n51czOFozrdNx98txBWIpxfcNhVhygKOuQgEBJSKcYeGpASN6RpU1FPYNA1WHSVuaG7UpkFN82d+3iIj/b1pouEPw7w53B0hzwn6SetX6pzjo/HhDqjNGotNDU8J6jQdw1BhQyoDZWk/tWFPhzMcsSmlmRG8NQaxMTrRYC4WBY/kPyEEP0BABxtssMEGG2ywwQYbbLDBviL2hRVAJCGDkk46jCjJu7IJ3nkvQAq7eQ1HONF7917H9nbIIgkhmdShIUarpjPMgiOFx6iMJcsEjuLQquvgqPS2aQ2WBNPQaWQzq7GgDH1ZlmiI3sf7hDWv6mqDnCoj0AJdhA26WIrN4CkrZWFhiSxFCsFlWS9TKIISUFUX3jus6FlSPUFGkIJ2WaCeh3D+8fEcE2LzfP3+A/z9r38e/phKuS9PHRpiUx1PZ7hzI2Swblw75HI5hPgcgU6scMTK0KiYsJ6e85abr1OZII1QEN+iJm2dNCthRbi/s3XIulVNy0QAJ/M57r8ZskSttbBdZDOcw6chO6lHQCEDRKS6DFma+XGH14kMpal+i1ebAC+oMsEVI+M8nAljaqzCmrKxOREL7N5KsXMtZM+O7s1w/V7IWp49t3j0ATVCOwlDGa1T0mITXiAyH1jTZzjarsXLF6GCfPLqHOfn9HnRAdRkvzcLY/fBu7/D8iJAX7Msx29+E/QZV6sV/uIv/gIA8PZ3vo2EtAmrqkbjKRt7RZsuvgABdaUZ32L/WhivJEtQU8ZrXW3winSwYnVpLEeoaa0kOkNKjHPap+go03pyegxNY7p1MIVKCIrShffj2gSG4LN16xk29ODBXXzrW6Hqd3R0BKVildYip4kt0gjXAMMthZiwvh3qBlnMqsFgTkx5KWUhf/bjn+EH3wmwkadPH8MSdEoknlmsfGvh8zCvHp08wrN/G6qB/9u//lcAgFxmvA61lMwiZr0PcGwAJpUQMZNnLGI3dMwyOucZLrKzs4Mf/+TPAAD/8NH7aGlv+Nl3f4obt+6Ev4PFhKDn7737HgDg148eIqYef/+bX0FQ4/r+4XVs74b3kidJD+UUgqGfFWklaa0Z5nu16qdUD5GHB05OA+RrNM6Y3WyuA1NdOn+MkQrj2G1qpFQhrjcrtATpqT5tkdP+t6bq8OnzT1EQVNv7BqszYnptN7A7AW2wWFxiE0mCLjeoaV+8Noq6fhYysvwBaF4+BgC8PP6Qq/lp0iFLwnrPxCiQIgz2pVpO+9LB3h6kCu9f5wmsi7qrDjlltcvxGFvE9puPSmgRCSz6irSl6p7wHiOqhLlUY2c/7Jfr5ZzPmqZuWXPsYhm+23uLoqBKSD6GlZF1VDCroUgySNo3pE5RdbE63q+RWPVzQoQJCCJKip/xltmKLWkheuuZWEWmCtmMqjBpCks9BkJ6dLRXNM6wTp4lMrTnzyx2NuFZy1KxpuuD+/egaQ/qOgcRBXR9D6eM+1UxGjEDNpS8woDu4T0zqgFUvfVeoaVs/fmC4KJVxRqPZxenTHbWWcf6gdY6tPSdZl1BXdFpBQAlUuhYabVVz5TsPEy856vIMO8Ri0qS2mOEVpiSlt3WVoHbN4gYbbnA2XE4X16eXOLsNLx/Q+gYCcW6Zt6DCe1qB7w6D2fz5fwSDx+G/ePlixN4gsBZKmG9/8GHePpZQCudHr9kOHFejgLRBwLp4JQgoAoOxhBCphfMC/AsBN9OExFRqSW2ibAuy3JuHarbDhsi5GmpmjKbpdBF1IrLARXZKz28D9+3Wizh6RmDvhudobG6als0m/Bul5fnqCMp0Z3ruHc/IKG2ZttQNK/atua1HXWRu6blalWSJgw/da4AFIvxMWnMLmn83X39Ndx+LbS5pF5gj37/9NHHQCQFSwQMkYm9//tPcE7+9a9/G9jDU63x7LNQAezqFoLmrnAags4fJTOGaGsh4KjqFFt2vJfIi/B8167dYITexcUFTNvRWE9wjTSXt7Z30FGl/ZjavE5OTrEmHcknnz1DYyIaYYwRIZeUzLga6L1nxt7IPishudVKCsGIRCEUV+dsZ7EkDd7NqoLVAQnQRsZT16DdhHu+aDs4qhYqW+HVPNzHg9v7uEEtRRm9HysFCiKMk34LFY3Ry9UCl5piDefgY0uJX2CWrOndEexWWoaHa2Egad04WPi4l3owm62UFiKKUP8B+8IAcDwLg3LtaMZ0vqvLHDvj8GXFzj4mBJUwnUFtYl+AhRRRcJRKqE0D34WBlVozTrlyBjb2HCiFjLDfq8szrEX8fRj8xWKF+qMABw14WWLoyXNEOVYhBW7fCwx127vbMIL61CKbvEw4ELXew0aWHqXhaPNQzkIT/G6HKPCFB7K40asE+TiU7+tU4pwgA11b4+w4bGy6q/DgzQAfPCeoFnQJX5KswHSCt+7eBgBM8gygAzgoqf6HuN2OArOdvT2M6buNX2PTRfYwMAymKAI9LhBYC2Va0feHw26yfYTqLLwLqwQcwVOqtoWjQKJDhUoHyJcc1xAEY5ztBKhCddFicxKucXdyA4vVOY0BsGqIvQg1HNH1V20F2PBuO8Kf+6bjeTXdTbFDwq62lbh9PzghJ59VkLTpPH82ob/3WF0Sw5r16IiuerlY4in1/XnhsK7oGWH5oF0TU+25N9wjqI2ASamvLknxnH7/b/6Pf407rwfIyf7RPibb4Z4iZNPaPkD33jNLxlBS5gAAGbJJREFUpZcK42mAPKR5CUf317ZtLxxOgZ6XY4YYqTRFRjAIVXRQVfi5MxlATItpmfT9fuzsK15v6/NLhnHcf+N1huY2dY1+Vil46iMoKMhpmhaEZApwaTrEW+fQ0br2wmNKDLT/4p/+9wCAH/7J25iQ8zAuJAxBcIr9aR8oGweo6IQ42KbvWQKoNyVCZ9ICkU45TQUHUMYZaGJ6k16gjod7nEDoYZqJVCjLcPBt7e3wofDy1XMOVrwz2FDQ9uZbgR21qSo8+SRIs7x4+hT7xES3s7vLTS5CSoY4GesYztWb7/t2vUe1CZv4aFRy0O8cYAmvlWcZdISJjsP3nbR/BtGEOZivT1DaMGdH4hxLEuyeTBfII/RFh3VRrecMwd1bbrAkOZzlqxe4PgtQvpWb4IPH4Xqn65u4kYfP3J38Db0TASUirNWjmEQ66wzjGUEHRYeEggsvErgh/vvSraTEw8HBDgytnXJrjJgP897xXpJnOUOqhPBMIx8ZIZ3t2BG2ENCUhJ0mJbEKA8aaHubvLNo2JnjpWhJQxBJqvYKnw0hCoKN+1yTNsU09ckU5QUtnSjzvhBXM/uetgIxnswOvOQHFwYtk8e+edRzOQBCMNJMahpKN0hkoWvsqL5HNwv5cR5p867EmKaC7d+7gW98MPAfXDo7QEgRLyl482jkHG6UHyPstyjE07afGOHbaU63hXMt/Fx9AasmQtY72gLrewFCipetqCGqS6mzDchNJKlARQ2dbtdB0Ho1IsD5PUyiCPBrXwlAffmM7+LhYJTh5qQSwJN/M03eItoX0sUdU4Og6BbZHE+CPgt+yXjV4+jwklc+IKbKpOzQESV6sa1REo+/gcUxyX8Y4nBF02EHye4w9ia+Oj3ES9/22gieobe49NiRT8MHf/ZoD3+vX9rlP89p+bGkwvUg6DPsZeZpiL8rqlGsWOW+qGg050UKGwEulBbJx2FvTpOCgbjTaYmmq+eUpJ0N0mjFbdISCGtvA0vmfaODoVpD8ee3WG/AUIK1Wa1yeheC4bVtY8jPjGqtbCxVJ3a1nZk8oxW1QWguMxzTv6axanZziiQ/v4vj5K1hKglsn0G2IYRwbhoQv5havjkPA9fDjxwCAMk+wIYhi3WzQEJx1VI6go29vDTOWZ7lCTv248YwQXsJTNic7TLFHXCBN3XAyNU1SjKjKJGUCY2MPKyV2lMLlfMHPHfvfkzxDllMPupJwvNF5Zvx2XFwB7zWtsyw1ppVnBvG6M1hvepmvLguJU0Ps9/ANpKK9su1Q+TAfzfIC1UX4u1p0OCYlghgAikxgByTLNJphRQWMF0iha2qz0CkcSceMvQUJHqCMUP3EACRbIgWgdN96Yek9exM5YAFYyy1if8gGCOhggw022GCDDTbYYIMNNthXxL6wAphSg+k0ew3OUOZLNtjbIbKRJENehOh0s1kg9kd7B5TbIaqNGZH5WcUMhxKCRUtTnTLRSV1VqEgUtqsqpLuhuliSrtxiUbFmlxQCGbEyjkYjjLfDve7t7+DB/VC1KbKctfuibpmDY30S4TwS2WfwYzI/8xYFZdCiZlmmBCYjgvgJgYTFGXeg0oa/Y0bizPXKI1ckuLwfMvutATQ1Z984nOHerev0HR2XpiEFMxz6K83vL14G6GBnHLJRuEYjz1GpkLm7sB4VvYAkM3CU7WlqoKVMpRbUbOoKPP40CMzqUYINJQl0Y6Aom7ixawhNGojuGJULmbJrRSjT7+YTZC5CJlJsj0PV6XT9FMKH31dNiyrCO4TA6jxkQJyPJCqmJ1GRFsUo3P9sq8TRG+Hd714vkCK823vLAGGYn9f47EnIBgmb4cWnBHVDis6H+1ytF7BRLFnlnBmK88flKajXGgeJxlkWBlp7gY8+DiQwi8fPsLsbvnP73m388CeB+fK/+6/+hN9PzDw7bxCXk5Q9K2JuLXRsJE/TXrw9ijB3ghnikqTPhuaJwGwvzI/DaykskdsYcwFNVbaoXaN10uvoweLmjQA/FUKgIRhprRVDaY1xME2sRBLjmdGoKAOepgKCiBtSqWGpIteZDh3pHD16EmCTf/vLf4MJNcE/evEZVjpkosb7W+guN/TdFjIKpSeC4Z45V88kxpTFm29WyCjj5W3Ha0ELyXqKxjiIWD2M1QPRN33rPEESq4GjEVcJd3Z3e3ipNYhlkVgZ/d7bb+PuvQDLGY1GfcUDApqr8haOqo8B/knQ7JKYQW2vD9a1HRxV6KUvIGhPaeoKFaECZrNt+Auqsm0HKLY8fABNsI+6XmBFsNrm4ikmVBncch4jWpOXj6P26DbmL0MmOf3gCVwRMq3vbxL86j3KqBd7qNM/Dp+5+Tr0KGipbuUBAq2zBtBhz4YrkBBE1MAC1PAutWKkQGv6yutgX55Jgg1tz2ZYEAwpkRoqQg1FgiyJ1VvP54vzHtJGaCJBCk3HKArrHRRtjOVoFIk20dS9TlqSaEja0yrSJN1UG6yp8uNrA0HtFLMyx2yLzuaD63j9Vqge7W/tACCESVzDUkL7iEBxrKGVSA3nI/smuDLVw6s936c3oeIQ/j2QPQHhrI/Qs/HeIfIpoSt2iVCtrTGS4f5v3LmJg+s7NM4Wmu5DJUkgoQFg2n5MG6p+jIoR71FtU/UsiFIxgkcKwZUkLSVEFHsOv4Jpg/4YEJjV43jUzRrLRUQeWEYQtF3He2AUAi/yEokj2Kqp0XWEzNq0aCrajxLFsEMPgTnt/TLqO8IhpfFKZAKnIhmXxojYJre3S+zvh4runMiHmtaiporR/KLCyXHYj87P1zA079rWIs1iu0T6HzBjN23LzMdCekwIYZWohJml//YXP8eHHwUd5Qf37+LWa8H/uPlf/hQA4JSCi7BUpZmZUgnB52CeTTCL8Fjr2A8VkgiWihHSSG6jNJQKcynRikkC9w720dD4wntoeucNMcAmBmhitSpJUE6pLUIVYAKdYoWOxkZv1njwIKCs9k+26dZasJo5BGQkHcoSeEFkclmGMfniY1q/nz16CPkZkTeultglQsa33/426y+36yU2BK28WG1giN15OiX9z3HB0NK6axiimI0KRNyw9Q6e/Im8UCiIlTpqKAoIXr9KJny9NMu5hcd5z5V97wzDzSOxW1GUuH6d/BfnGG6eZTkTu3gICGYN730s72JrlIeN+oJdh5r0NlOdsBafsR2TKiWZRtOE+a30Dn2HR5fGZ5EoZkTU0tTwJvx8qgTqSNIlqCJtJY434TzeXc8A0vndTK4hSSOaxrMSgfANWtJnFCXt76Kv6DnXQUU2WA/WV/XOII1Bj7RcKf1D9oUn99Y4OPt3brwBZwhOd6ixuR1x7hqGNg7vDJSKTFASGQleR1zieJTDEfxACcFY/9YCK4LlnW067FJpPdvfx+FhGLBpGZzwWzfXDJlIEoWCJvxkNsFsLwQMeZFhNibR8rxARhOvIwhGA9f3CwgBecV5icKR0mtYuu8oUqutgPbRyfZMMay1RkPU28p1WNDm0uV7WBE8ZjIKk3WkFFKarLev72OLSt62rQHP9D59X5kU6AgGcHwc4BOTyQwpwVrWtsJoTNDAZccTfbVqIMgRF6LA5ZwOANq0Ti7OsCZoxtZre0ypu6g65ILE2LMVNAgeoS0WJkADsioEjtPtuwxV+OCDJ7hx+054xrHC+UX4zHrVYiVDELk1A+qKJAcoYFiv5iiKeBBo1DES9Q0ksRmq1HLfxTbR728fjrF9gxyQhYIikdRyOkFD8Jmmqlj41DkLQwcboTXQemCfnOw/GZd4TDCljS5xfT/Mn+/lGa6TAHF79w3kRdiMohNwVfQ71VdgyELyBpbluqdIDis1fAaRvlkzTBpKIGI9vHTcSyFliaIM99Q0Oep1SAZU5HBBt+hofU4nJTKaj03dcCAk5Qgdzc3Npsb/9D/8jwCAgmi/8zznxEiqE0QhF+ddD+X0nmE10ZnKsox7Fm69cQM//NPQD2iN4WBLKsk9Ik3bssMX4R/TyZRhTN77AGRHcAjiweFxtd+vDzre++VfAQiH61WG4JYgbVrrK7IMGp4SN0mSfC54jN99jXo3m6ZhGvowfpSUcYYg5/0Bc/VZpJSc1PDeI58R7l8p7oWqm4odxdnOHkYmQJ9jUGvRQYow18psCj8L97Gc3mSmVpkmqAiutaAe49M8R7MKUKxP2x0klDiptzNkk7CHel1inIZ73TanmDWh11ZtU490uoeo8ADXASr2JBh4Fw9aDUG9QonUQPPFTGOD/ee3kvpJXv+jeygJmr6qLLou4nE94o9CeqaUz9IUmpzamOAwtk+KruqKIW3eeTQE6/Tecj+uVJJp9Q+IzXi1XGGz6SniZ9PgOF3fn+EGScC8dvsejq4HCNytW3chXOgFtsQmKIVltnFhPXdCWHcladu7eMzgKCygYr8d+p4f7yxElCdAhij6rYoJUnIsVZSPqObYpj7Yu3df5/2oaRoOHgDBtO5O9/3fNbUaiO1tRHdis+k4APHOATYmoCVL7BjZwnWUmKGAp+kMLP0stYSMT+saVFXsAfQBVk/vJTp/0Veo1itUxAjetB22iTU8zRJcLEJLjHGG+7ac8wxB7flYexp/SAN4St6KDh0lB4s0Y/bBjJxVqTyyjAoH0xH298P5cnm5wWIR7unsZIlL8t2ckb0gNz2T6Rw0nY+jcYk98gN3xmPklPjwxrBvadoWLT0vYo+UBEScr0Jw/zuE5z5HKWS/r3t5hVGaTAhmm9ZJxn3vEBKOkmFpMkFWUHKia2HbuHZE/1lEX0wCETbpBfeo5XmJw8Nw7tTVCv/in/1TAH0gl+qM5X9UknBwDAtImptSZ3w9x8zUDUsdGAemCc1yBUWJX9u0WBFnQF2t+FzPqXCTJhmyIp5zmp+rtQKWOBGEThGRht61OFuEQOj8cQjQBUTf1ST6c1MqxYGod4YTKuHzNO46wh8V/+w9OHnhvOghnjTeQGjZims/ppkd7JXkyxVYPDz7dHXdsJ+R5hlEE9ZOxBA7ODhKELRS8tikY8eyOmlSckAp2IcQqLNQELqQBaQhHyEzaOkGbdcgkdSfOtrG7k5Yc0UUsm8biMgMrwFBbTXGWkgRJU8Qzm0ElQOuuv0BGyCggw022GCDDTbYYIMNNthgXxH7wgrg0bVAHPDD7/0EkqpfndXYND38KjZpAmCYAARwVZ8LoOpBzIhZz9mIqm5QU9Olc57hCkJIjKhyFjWFrO1JQ7TWKKiCVoxG0JEZzFqGrl4/vAHh3qO7oEjYeyTx3pTiyoQQAp6yR60VqCnL5UlUvoPnTI61Bl7ETBRYq0w5BRmhNjIBqBl94yN5gsAkDz/fv3WASCjmPCeaQnWRRs62hqslhsZgUs6Y8KOuLVQefj9OO+TEjLhYWSgiCZhkJRoSkuxi1bLxLELviwyGMmJV63FJukgzuYZeUIYBBZwLvz+vAwnP/MkLJg1ZtRbXFmGufPPuH+O5DuP/m4tf4WAU7v+1Gxm8j1pT1ORsFEg3HJNpgjk1+b48foWt7ZDhnk5yrFfh/hvR0mtLIIi0Zf9ohPEkQkMbnL0I9/zi6RoZ6UF1dQWVxGwJwXK0xhaNVzYt8S0SzZ4eXsd1Yss60AIJjU1TTLCJ8J8I+3SeGfWkF8zK5F3PgCmF7HWfXMvVPkvl2NY6qJjd0xkEZYCEb+C6nrxHkNi2zDNIEg6nfmDYtkUXtWRyhZoywQ4KKUErm6ZDSwxkzmpsT0I26tpeeNZiNGIoRd00GFGVK0szrNYkipwkDHldEbPpuBzzmi339lHRfbRt05M7CMHZ8kTrXlyV/m53dw8t/dy2LRLK3DZNwyybs9kMm1XUb1phVIaJ81EaUQefh4PGzK7WutffszbsPQgVu/gssaKntebKofceKmaHAagIyxaaIV9JkjDcM6X7cM597rqGMsKuTybj8vKciXWK2Tbq5wTLIphvknjoyNIqMq7GTnZ2mbRHewlPVYpMhfuH9xB5eJ9WeMyoi1ynEyyXpAl49gxv7QfY5x8f/A43tun+R2G9WSeQUNZytK6gaTyW44TFclXr0FHGsZa9CPBgX57dJHa/P//ZT5mBtm47PieEADYEy/LW8/lYFDkkQdli14G1Fg0heS7mCywWYa9fVw2TMbSVQduESpdONAqCMI1oA08SCU8V/7IsMSa42ThLoKlyIpIMmiBRxWQEH9crPZP1ktmOoRxTKYYsOh+QDJNHnI9wPdxLgKtSHo73IJc4CCJhgOyZCuN3pHmBo1uhCrOzt4uYuO9sDRDMTioFz0zKfaWxraP/YvnePByjFwIcLRLoOCbuMK5FTWMWCUhMZ5FShUEpjV7H2cDSgdt1lv0u68B75+lFgFtuVmtckl5oWSZ4/Y1wNh/sHjKa5vzyFI5ZvwwSOpNjq4TwAYYLhJaehlph0AjksT1Gpyw0HauxxjkmQMlyjbII+8pkMsaSyGjKosQWMTDbzrEuc0LzsswzpOQ75XmOLdKQm01yjGne5YlkWGSWJcgIGcFoDgjo2D6gEmZIhZRcLQ4dNhF+rPg+Yo05zLq+khd9Bw/LxD9SexZplzJBdOriOJqWyUgD3DXen7E94kwLZKMI5QTuZOF9bRGhV5pmKCMBmlLwRGjnnIaT0RfBFQ28WA03XPlsTQtvYgvFBpFHxpdAOQvoolRNWaeziGRHyvdQ7KyApBfdODBkVicpTBd9Y4sxsbaeP/mIxrmHPQshe31ArVirUbreF4fzV2DB8bUpCNbP9J8jShTM/NlXGiWAuHgi0iGQXsUx19CEXBLeXCFjcoi3oaSCIj/ZiqhJKlnz0jsPT0zKSDSSPMzNNM+YoIWJd5DB0vy2RiL2yymp4Wk/UL7GlJ5xtjXDmAgxc6ritd0SglqchPcMB9XK8/7iTQ8TlSL+5w/bF57c21vBmUjuf49PC+t6vL0Qot+9AWafCb04n/9i4T2c6SERcWNzzsOJPqCMUCrv/ecgX/HKUXxVKcUY43DNvgwcHZLZdAobsbE0MxIRWMXCd/TlYXjPm4FT8op8YtxQPDyI4ctrGLqu8ZbhXFJnUMR2pBF6LwAgjxBRYXG0FybU9e28Hzch+SCS3nDVVgnBC6FzkWVpDCGiSLfA4pIOIdMgIXbIldmg21CpeMtzX98sC5Dex58eoyHoQGYspkV4rrUDTpaEw0cCGYOUtkPHvRkkgGrOGcstRxkevwhOZek1tmevAwB2ypfAVoCc7E0nuCDZgo76orb2cnhaCOtqiaaNgpgjdjZXmw18pDanv/MQ7JhMyzV2t0PwJrMWqqRA+SCDrwn21uQYU99kMeoDm1EZ4Y8ZRlGoN9GI73zjbMDfA/DWXkl2xA3FcQDgrWVWsiRJeD4a23Efp1QUGAGwJDAq0NMpS6EQsRTOAC3Br1xjkSRhk9Y6YQcnrsM0nyIhR32zWSEjiHCSpixkK5Vl/LuHxO4oQJ+ePQswrFE5YrhXXdfYmoXgQUiJzYYY4rxHRsHSkuC1RVHwobu1tYWnT5/yGMT123UdO6n7+/sM1Yywi+cvXzC8I00Svg8pBAeDp8fHWNN9VJsNphTcxO+o65rvw1rLgaO6wtKptUbHFODZ5+Qcwt8ZZvkTV2iipRAMoeuc+dy+1LOQ0nco/bld1ScR3y/RkHN+eXnGz5WPJmjJUc/4gCtQNeSYaoeOoDbaaXgdGeoyxEaUysS+LwlN0J28GOPJMsyJ6uwYURknH2nc2wuQ+zdvjiAIeq5pnkNWSOLhO98gnYfvLu/uodNRMsRAEAzJFwIdJcYG+/JstBVgcfeKEm1NSdMkQUf7ohQyCAQjSAhERjwlNCKCLG5npmtgoqxA16GNIu6m42Sj9z2zIWSQrAHA7HnCe+47z9McCTnw8B4tMQc6L1mqRacCeUt06xjTUzkYZrnrRcuFc+xOaOGuBFN98KFp3zQQ8FERHeyPAyKBJA9FKQ9FkExanihkinIcgpXGeKwpSSXgkVLi0WoNFfd1Z9FEySTf7wdxfxdwELElwAt2op316Frqe5K9fEIM3L337LRJKfuskfewtM67znAPrjEWtYkJ1fDvy1WFhiCRUhdYr+K5miCJ/olI0bEfoZATvNtqug94RICYhECPZBOQMdnuRZ8AlbE/tPfNwzPHMzHFmKS/xL7GdBreue/CfQGBswEI/ddp9C2E5+RamY9QksyGTjJ+n5CAIB8nssgmiYKLiQDRs632DQWBHbL/vQzz7Mq/CyE5SWJs2wfdV9zbVoEZSLXSvKa490olzF7tIGHJ32ndFeZaCD6bjQWWJJexvowsvR4Z+XajPOFEgHVgvzHRkuVUDEtjOdCrQlM1qLvYArKCI9bIqqph2N9sUZAvGKW6hEq4D380KpjN00Gy8HySaJbwcM6ii3PaxrV8BUorBMMpIRUz/UL0wbgXlv1exVINkoXP4T0nRXGl91/IK8G76BNc8f8Cngsq8H2iFs6zTw3XIU2jvIyCooCr7YK/YYWDJuillA6IbLtCwEbOAOHRxuQxJfw7q/qEsk4hqP1I+BaKkqkj7VHE1hptIcmRMBSltCggqFUpT90VxlPB/qaQApLngYNVfZzxH7MBAjrYYIMNNthggw022GCDDfYVMXGVyGCwwQYbbLDBBhtssMEGG2yw///aUAEcbLDBBhtssMEGG2ywwQb7itgQAA422GCDDTbYYIMNNthgg31FbAgABxtssMEGG2ywwQYbbLDBviI2BICDDTbYYIMNNthggw022GBfERsCwMEGG2ywwQYbbLDBBhtssK+IDQHgYIMNNthggw022GCDDTbYV8T+P2nA07XOWSU1AAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "# Reconstructions\n", "train_batch = next(iter(train_dataset))\n", "valid_batch = next(iter(valid_dataset))\n", "\n", "# Put data through the model with is_training=False, so that in the case of \n", "# using EMA the codebook is not updated.\n", "train_reconstructions = model(train_batch['image'],\n", " is_training=False)['x_recon'].numpy()\n", "valid_reconstructions = model(valid_batch['image'],\n", " is_training=False)['x_recon'].numpy()\n", "\n", "\n", "def convert_batch_to_image_grid(image_batch):\n", " reshaped = (image_batch.reshape(4, 8, 32, 32, 3)\n", " .transpose(0, 2, 1, 3, 4)\n", " .reshape(4 * 32, 8 * 32, 3))\n", " return reshaped + 0.5\n", "\n", "\n", "\n", "f = plt.figure(figsize=(16,8))\n", "ax = f.add_subplot(2,2,1)\n", "ax.imshow(convert_batch_to_image_grid(train_batch['image'].numpy()),\n", " interpolation='nearest')\n", "ax.set_title('training data originals')\n", "plt.axis('off')\n", "\n", "ax = f.add_subplot(2,2,2)\n", "ax.imshow(convert_batch_to_image_grid(train_reconstructions),\n", " interpolation='nearest')\n", "ax.set_title('training data reconstructions')\n", "plt.axis('off')\n", "\n", "ax = f.add_subplot(2,2,3)\n", "ax.imshow(convert_batch_to_image_grid(valid_batch['image'].numpy()),\n", " interpolation='nearest')\n", "ax.set_title('validation data originals')\n", "plt.axis('off')\n", "\n", "ax = f.add_subplot(2,2,4)\n", "ax.imshow(convert_batch_to_image_grid(valid_reconstructions),\n", " interpolation='nearest')\n", "ax.set_title('validation data reconstructions')\n", "plt.axis('off')" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "TF2_VQ_VAE_training_example.ipynb", "provenance": [] }, "kernelspec": { "display_name": "Python 3", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.6" }, "widgets": { "application/vnd.jupyter.widget-state+json": { "state": { "00a640f960744579a63f30d7e61837c4": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_9ca42d6547c64311b1156142caf73c91", "placeholder": "​", "style": "IPY_MODEL_5387493f8b574b56954240e78d876d8f", "value": " 0/0 [00:00<?, ? MiB/s]" } }, "030005c3a52d4e6f82615b727d9234bd": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_237a38f3fd0846c1bac1fcb6295fc4e9", "IPY_MODEL_b5e4f8c518504149aaa2be0262804143" ], "layout": "IPY_MODEL_64160543e7954c4b81eed5d1e943b68c" } }, "0f03e9a4f7c2447a8bd7d08fa3127655": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "Dl Completed...: ", "description_tooltip": null, "layout": "IPY_MODEL_62235fc6bbfe4f6db6a7f1a6d6ed18f2", "max": 1.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_45e0726050d249ffaea57c16b7655946", "value": 0.0 } }, "237a38f3fd0846c1bac1fcb6295fc4e9": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "Extraction completed...: ", "description_tooltip": null, "layout": "IPY_MODEL_a1b8844fe3ad4d08bc434d92c62f1cbe", "max": 1.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_a4521605bfc9403b8dc32c21cdd9d55b", "value": 0.0 } }, "267b37f12bc145d38224d0a4c59c987c": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "initial" } }, "2758f9b231ae40899210407e66a1bd40": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "2ad4f36074a94ae8b5e0bcd22e072f70": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_bf11f99ee0a74f1a988c9e5f56c0dff3", "IPY_MODEL_30da0090afc04341b5c80c2cf94384b4" ], "layout": "IPY_MODEL_3f0ae4a43f3f41b4bc4406108ec7291a" } }, "30da0090afc04341b5c80c2cf94384b4": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_d76a99aed0534570bbf9b2ab73ed9d39", "placeholder": "​", "style": "IPY_MODEL_c21fdf120e3f4ec09ee396e848ec8630", "value": " 10000/0 [00:06<00:00, 1668.80 examples/s]" } }, "3a08302c69514afba1934f6475737397": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_6407c4a2a79a4efca9785dd96c069b4c", "placeholder": "​", "style": "IPY_MODEL_fcc716d80f6345ca8d9623a7d8a4e85a", "value": " 0/0 [00:00<?, ? url/s]" } }, "3f0ae4a43f3f41b4bc4406108ec7291a": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "4276da9ba50e48ab9a1b218085764e34": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "initial" } }, "44a7f5fe14424d2cbb5a4f3c521e33cf": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "initial" } }, "45e0726050d249ffaea57c16b7655946": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "initial" } }, "4e796662246d4229acdee5850ce64405": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "50a5e80086fd4f34ae81f5e8da9f9333": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "initial" } }, "5207e75e672f4a3a8fa7d158539c02c8": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_0f03e9a4f7c2447a8bd7d08fa3127655", "IPY_MODEL_3a08302c69514afba1934f6475737397" ], "layout": "IPY_MODEL_bb0a3b1e948e4a21a3922652705159fe" } }, "5387493f8b574b56954240e78d876d8f": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "54cf8fd1ae5a448693baf588194fbfa0": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_8d58fa285b19459fbea0efa476a4a577", "IPY_MODEL_bfa04142613c4c658ebf20c65da4360e" ], "layout": "IPY_MODEL_4e796662246d4229acdee5850ce64405" } }, "62235fc6bbfe4f6db6a7f1a6d6ed18f2": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "6337f911938942cb888a2056ef4115ce": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "6407c4a2a79a4efca9785dd96c069b4c": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "64160543e7954c4b81eed5d1e943b68c": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "64672cb5e965471881bd2da3078e63c1": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_bdf77430a1e54af0b7edbbbaacb8e234", "IPY_MODEL_81bd4784bfde47f89d5770160649e69d" ], "layout": "IPY_MODEL_a344044558404f7a83f1e389cb606958" } }, "6ab2bd198da24a5a87c7ea1d404a47dd": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "6f481dd0f52d4df0a13df36adf3b5f0e": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "7900c1a3bbec4ab4a5d8bbf3976bd2c0": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_a1686659229a43118f7b71340c6f02d4", "IPY_MODEL_00a640f960744579a63f30d7e61837c4" ], "layout": "IPY_MODEL_ccc2fa3cc33942d8b464f947b525aeb8" } }, "801f6cd9eb6642429ed8e3020d335db4": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "81bd4784bfde47f89d5770160649e69d": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_e01b0a8160234e849a9da58afdb9c8ca", "placeholder": "​", "style": "IPY_MODEL_9a3950f2dc21480c9efe6d0c5a8988c5", "value": " 50000/0 [00:30<00:00, 1676.83 examples/s]" } }, "84c8f0362d554463959e93ce8e82e332": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_ebd4b63e7a4f4654858e38c97533bc55", "IPY_MODEL_ea9e592b7d7145f191453be6f4bcae46" ], "layout": "IPY_MODEL_2758f9b231ae40899210407e66a1bd40" } }, "854ddf758f764ac0a53dde8dc6e448e5": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "8d58fa285b19459fbea0efa476a4a577": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "danger", "description": "100%", "description_tooltip": null, "layout": "IPY_MODEL_6337f911938942cb888a2056ef4115ce", "max": 50000.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_44a7f5fe14424d2cbb5a4f3c521e33cf", "value": 49999.0 } }, "970e7fb38822400395e6674a42f07213": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "initial" } }, "9a3950f2dc21480c9efe6d0c5a8988c5": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "9ca42d6547c64311b1156142caf73c91": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "a1686659229a43118f7b71340c6f02d4": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "Dl Size...: ", "description_tooltip": null, "layout": "IPY_MODEL_6ab2bd198da24a5a87c7ea1d404a47dd", "max": 1.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_970e7fb38822400395e6674a42f07213", "value": 0.0 } }, "a1b8844fe3ad4d08bc434d92c62f1cbe": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "a344044558404f7a83f1e389cb606958": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "a4521605bfc9403b8dc32c21cdd9d55b": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "initial" } }, "a8cd42db14e647d897fe869e8e47a54f": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "b192ee599f1a429b9ca73c713e2d9aa7": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "b5e4f8c518504149aaa2be0262804143": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_b9442fafcebe478da9cb0486a3cd4bdc", "placeholder": "​", "style": "IPY_MODEL_d48f0b8d2fc84dc8977d944df482c66a", "value": " 0/0 [00:00<?, ? file/s]" } }, "b9442fafcebe478da9cb0486a3cd4bdc": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "bb0a3b1e948e4a21a3922652705159fe": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "bd8f2e96c0b24222aac0acf4e5dab0e3": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "bdf77430a1e54af0b7edbbbaacb8e234": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "info", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_854ddf758f764ac0a53dde8dc6e448e5", "max": 1.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_267b37f12bc145d38224d0a4c59c987c", "value": 1.0 } }, "bf11f99ee0a74f1a988c9e5f56c0dff3": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "info", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_6f481dd0f52d4df0a13df36adf3b5f0e", "max": 1.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_4276da9ba50e48ab9a1b218085764e34", "value": 1.0 } }, "bfa04142613c4c658ebf20c65da4360e": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_a8cd42db14e647d897fe869e8e47a54f", "placeholder": "​", "style": "IPY_MODEL_b192ee599f1a429b9ca73c713e2d9aa7", "value": " 49999/50000 [00:00<00:00, 94821.94 examples/s]" } }, "c21fdf120e3f4ec09ee396e848ec8630": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "cbf776079db64c8cbbb022f616196cd5": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ccc2fa3cc33942d8b464f947b525aeb8": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "d48f0b8d2fc84dc8977d944df482c66a": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "d76a99aed0534570bbf9b2ab73ed9d39": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "e01b0a8160234e849a9da58afdb9c8ca": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ea9e592b7d7145f191453be6f4bcae46": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_801f6cd9eb6642429ed8e3020d335db4", "placeholder": "​", "style": "IPY_MODEL_bd8f2e96c0b24222aac0acf4e5dab0e3", "value": " 9999/10000 [00:00<00:00, 97906.75 examples/s]" } }, "ebd4b63e7a4f4654858e38c97533bc55": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "danger", "description": "100%", "description_tooltip": null, "layout": "IPY_MODEL_cbf776079db64c8cbbb022f616196cd5", "max": 10000.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_50a5e80086fd4f34ae81f5e8da9f9333", "value": 9999.0 } }, "fcc716d80f6345ca8d9623a7d8a4e85a": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } } }, "version_major": 2, "version_minor": 0 } } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: readthedocs.yml ================================================ # Read the Docs configuration file # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details version: 2 sphinx: builder: html configuration: docs/conf.py fail_on_warning: false python: version: 3.7 install: - requirements: requirements.txt - requirements: requirements-tf.txt - requirements: docs/requirements.txt ================================================ FILE: requirements-test.txt ================================================ mock>=3.0.5 tensorflow-datasets>1,<4 docutils ================================================ FILE: requirements-tf.txt ================================================ tensorflow==2.12.0rc0 tensorflow-probability==0.12.2 ================================================ FILE: requirements.txt ================================================ absl-py>=0.7.1 numpy>=1.16.3 dm-tree>=0.1.1 wrapt>=1.11.1 tabulate>=0.7.5 ================================================ FILE: setup.py ================================================ """Setup for pip package.""" from setuptools import find_namespace_packages from setuptools import setup def _get_sonnet_version(): with open('sonnet/__init__.py') as fp: for line in fp: if line.startswith('__version__'): g = {} exec(line, g) # pylint: disable=exec-used return g['__version__'] raise ValueError('`__version__` not defined in `sonnet/__init__.py`') def _parse_requirements(requirements_txt_path): with open(requirements_txt_path) as fp: return fp.read().splitlines() _VERSION = _get_sonnet_version() EXTRA_PACKAGES = { 'tensorflow': ['tensorflow>=2'], 'tensorflow with gpu': ['tensorflow-gpu>=2'], } setup( name='dm-sonnet', version=_VERSION, url='https://github.com/deepmind/sonnet', license='Apache 2.0', author='DeepMind', description=( 'Sonnet is a library for building neural networks in TensorFlow.'), long_description=open('README.md').read(), long_description_content_type='text/markdown', author_email='sonnet-dev-os@google.com', # Contained modules and scripts. packages=find_namespace_packages(exclude=['*_test.py']), install_requires=_parse_requirements('requirements.txt'), extras_require=EXTRA_PACKAGES, tests_require=_parse_requirements('requirements-test.txt'), requires_python='>=3.6', include_package_data=True, zip_safe=False, # PyPI package information. classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', 'Intended Audience :: Education', 'Intended Audience :: Science/Research', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Topic :: Scientific/Engineering :: Mathematics', 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: Software Development :: Libraries', ], ) ================================================ FILE: sonnet/BUILD ================================================ load("//sonnet/src:build_defs.bzl", "snt_py_library") package(default_visibility = ["//visibility:private"]) licenses(["notice"]) snt_py_library( name = "sonnet", srcs = ["__init__.py"], visibility = ["//visibility:public"], deps = [ ":distribute", ":functional", ":initializers", ":mixed_precision", ":optimizers", ":pad", ":regularizers", "//sonnet/nets", "//sonnet/src:axis_norm", "//sonnet/src:base", "//sonnet/src:batch_apply", "//sonnet/src:batch_norm", "//sonnet/src:bias", "//sonnet/src:build", "//sonnet/src:conv", "//sonnet/src:conv_transpose", "//sonnet/src:custom_getter", "//sonnet/src:deferred", "//sonnet/src:depthwise_conv", "//sonnet/src:dropout", "//sonnet/src:embed", "//sonnet/src:group_norm", "//sonnet/src:leaky_clip_by_value", "//sonnet/src:linear", "//sonnet/src:metrics", "//sonnet/src:moving_averages", "//sonnet/src:once", "//sonnet/src:recurrent", "//sonnet/src:reshape", "//sonnet/src:scale_gradient", "//sonnet/src:sequential", "//sonnet/src:utils", ], ) snt_py_library( name = "distribute", srcs = ["distribute.py"], deps = [ "//sonnet/src/distribute:distributed_batch_norm", "//sonnet/src/distribute:replicator", ], ) snt_py_library( name = "functional", srcs = ["functional.py"], deps = [ ":optimizers", "//sonnet/src/functional:haiku", "//sonnet/src/functional:jax", "//sonnet/src/functional:optimizers", ], ) snt_py_library( name = "initializers", srcs = ["initializers.py"], deps = [ "//sonnet/src:initializers", ], ) snt_py_library( name = "mixed_precision", srcs = ["mixed_precision.py"], deps = [ "//sonnet/src:mixed_precision", ], ) snt_py_library( name = "optimizers", srcs = ["optimizers.py"], deps = [ "//sonnet/src/optimizers:adam", "//sonnet/src/optimizers:momentum", "//sonnet/src/optimizers:rmsprop", "//sonnet/src/optimizers:sgd", ], ) snt_py_library( name = "pad", srcs = ["pad.py"], deps = [ "//sonnet/src:pad", ], ) snt_py_library( name = "regularizers", srcs = ["regularizers.py"], deps = [ "//sonnet/src:regularizers", ], ) ================================================ FILE: sonnet/__init__.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Sonnet built for TensorFlow 2.""" from sonnet import distribute from sonnet import functional from sonnet import initializers from sonnet import mixed_precision from sonnet import nets from sonnet import optimizers from sonnet import pad from sonnet import regularizers from sonnet.src.axis_norm import InstanceNorm from sonnet.src.axis_norm import LayerNorm from sonnet.src.base import allow_empty_variables from sonnet.src.base import Module from sonnet.src.base import no_name_scope from sonnet.src.base import Optimizer from sonnet.src.batch_apply import BatchApply from sonnet.src.batch_apply import merge_leading_dims from sonnet.src.batch_apply import split_leading_dim from sonnet.src.batch_norm import BaseBatchNorm from sonnet.src.batch_norm import BatchNorm from sonnet.src.bias import Bias from sonnet.src.build import build from sonnet.src.conv import Conv1D from sonnet.src.conv import Conv2D from sonnet.src.conv import Conv3D from sonnet.src.conv_transpose import Conv1DTranspose from sonnet.src.conv_transpose import Conv2DTranspose from sonnet.src.conv_transpose import Conv3DTranspose from sonnet.src.custom_getter import custom_variable_getter from sonnet.src.deferred import Deferred from sonnet.src.depthwise_conv import DepthwiseConv2D from sonnet.src.dropout import Dropout from sonnet.src.embed import Embed from sonnet.src.group_norm import GroupNorm from sonnet.src.leaky_clip_by_value import leaky_clip_by_value from sonnet.src.linear import Linear from sonnet.src.metrics import Mean from sonnet.src.metrics import Metric from sonnet.src.metrics import Sum from sonnet.src.moving_averages import ExponentialMovingAverage from sonnet.src.once import once from sonnet.src.recurrent import Conv1DLSTM from sonnet.src.recurrent import Conv2DLSTM from sonnet.src.recurrent import Conv3DLSTM from sonnet.src.recurrent import deep_rnn_with_residual_connections from sonnet.src.recurrent import deep_rnn_with_skip_connections from sonnet.src.recurrent import DeepRNN from sonnet.src.recurrent import dynamic_unroll from sonnet.src.recurrent import GRU from sonnet.src.recurrent import LSTM from sonnet.src.recurrent import lstm_with_recurrent_dropout from sonnet.src.recurrent import LSTMState from sonnet.src.recurrent import RNNCore from sonnet.src.recurrent import static_unroll from sonnet.src.recurrent import TrainableState from sonnet.src.recurrent import UnrolledLSTM from sonnet.src.recurrent import UnrolledRNN from sonnet.src.recurrent import VanillaRNN from sonnet.src.reshape import Flatten from sonnet.src.reshape import flatten from sonnet.src.reshape import Reshape from sonnet.src.reshape import reshape from sonnet.src.scale_gradient import scale_gradient from sonnet.src.sequential import Sequential from sonnet.src.utils import format_variables from sonnet.src.utils import log_variables __all__ = ( "BaseBatchNorm", "BatchApply", "BatchNorm", "Bias", "Conv1D", "Conv1DLSTM", "Conv1DTranspose", "Conv2D", "Conv2DLSTM", "Conv2DTranspose", "Conv3D", "Conv3DLSTM", "Conv3DTranspose", "DeepRNN", "Deferred", "DepthwiseConv2D", "Dropout", "Embed", "ExponentialMovingAverage", "flatten", "Flatten", "GroupNorm", "InstanceNorm", "GRU", "LSTM", "LSTMState", "LayerNorm", "Linear", "Mean", "Metric", "Module", "Optimizer", "reshape", "Reshape", "RNNCore", "Sequential", "Sum", "TrainableState", "UnrolledLSTM", "UnrolledRNN", "VanillaRNN", "allow_empty_variables", "build", "custom_variable_getter", "deep_rnn_with_residual_connections", "deep_rnn_with_skip_connections", "distribute", "dynamic_unroll", "format_variables", "functional", "initializers", "log_variables", "lstm_with_recurrent_dropout", "merge_leading_dims", "no_name_scope", "nets", "once", "leaky_clip_by_value", "optimizers", "pad", "regularizers", "scale_gradient", "split_leading_dim", "static_unroll", ) __version__ = "2.0.3.dev" # ________________________________________ # / Please don't use symbols in `src` they \ # \ are not part of the Sonnet public API. / # ---------------------------------------- # \ ^__^ # \ (oo)\_______ # (__)\ )\/\ # ||----w | # || || # try: del src # pylint: disable=undefined-variable except NameError: pass ================================================ FILE: sonnet/distribute.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Utilities for using Sonnet with TensorFlow Distribution Strategy.""" from sonnet.src.distribute.distributed_batch_norm import CrossReplicaBatchNorm from sonnet.src.distribute.replicator import create_variables_eagerly from sonnet.src.distribute.replicator import Replicator from sonnet.src.distribute.replicator import TpuReplicator __all__ = ( "create_variables_eagerly", "Replicator", "TpuReplicator", "CrossReplicaBatchNorm", ) ================================================ FILE: sonnet/functional.py ================================================ # Copyright 2020 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Simple functional APIs for TF2.""" from sonnet import optimizers as oo_optimizers from sonnet.src.functional import haiku from sonnet.src.functional import jax from sonnet.src.functional import optimizers # Utilities for converting Sonnet code into pure functions. variables = haiku.variables transform = haiku.transform transform_with_state = haiku.transform_with_state without_state = haiku.without_state # Utilities for working with tensors on device. device_get = jax.device_get device_put = jax.device_put # Utilities for transforming pure functions. grad = jax.grad jit = jax.jit value_and_grad = jax.value_and_grad # Optimizers. optimizer = optimizers.optimizer sgd = optimizer(oo_optimizers.SGD) adam = optimizer(oo_optimizers.Adam) rmsprop = optimizer(oo_optimizers.RMSProp) momentum = optimizer(oo_optimizers.Momentum) # Avoid accidentally exporting the private API. del oo_optimizers, haiku, optimizers, jax __all__ = ( "variables", "transform", "transform_with_state", "without_state", "device_get", "device_put", "grad", "jit", "value_and_grad", "optimizer", "sgd", "adam", "rmsprop", "momentum", ) ================================================ FILE: sonnet/initializers.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Initializers.""" from sonnet.src.initializers import Constant from sonnet.src.initializers import Identity from sonnet.src.initializers import Initializer from sonnet.src.initializers import Ones from sonnet.src.initializers import Orthogonal from sonnet.src.initializers import RandomNormal from sonnet.src.initializers import RandomUniform from sonnet.src.initializers import TruncatedNormal from sonnet.src.initializers import VarianceScaling from sonnet.src.initializers import Zeros __all__ = ( "Constant", "Identity", "Initializer", "Ones", "Orthogonal", "RandomNormal", "RandomUniform", "TruncatedNormal", "VarianceScaling", "Zeros", ) ================================================ FILE: sonnet/mixed_precision.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Sonnet mixed precision built for TensorFlow 2.""" from sonnet.src.mixed_precision import disable from sonnet.src.mixed_precision import enable from sonnet.src.mixed_precision import modes from sonnet.src.mixed_precision import scope __all__ = ( "disable", "enable", "modes", "scope", ) ================================================ FILE: sonnet/nets/BUILD ================================================ load("//sonnet/src:build_defs.bzl", "snt_py_library") package(default_visibility = ["//sonnet:__pkg__"]) licenses(["notice"]) snt_py_library( name = "nets", srcs = [ "__init__.py", "resnet.py", ], deps = [ "//sonnet/src/nets:cifar10_convnet", "//sonnet/src/nets:mlp", "//sonnet/src/nets:resnet", "//sonnet/src/nets:vqvae", ], ) ================================================ FILE: sonnet/nets/__init__.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Common network architectures implemented as Sonnet modules.""" from sonnet.nets import resnet from sonnet.src.nets.cifar10_convnet import Cifar10ConvNet from sonnet.src.nets.mlp import MLP from sonnet.src.nets.resnet import ResNet from sonnet.src.nets.resnet import ResNet50 from sonnet.src.nets.vqvae import VectorQuantizer from sonnet.src.nets.vqvae import VectorQuantizerEMA __all__ = ( "MLP", "Cifar10ConvNet", "resnet", "ResNet", "ResNet50", "VectorQuantizer", "VectorQuantizerEMA", ) ================================================ FILE: sonnet/nets/resnet.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """ResNet components.""" from sonnet.src.nets.resnet import BlockGroup from sonnet.src.nets.resnet import BottleNeckBlockV1 from sonnet.src.nets.resnet import BottleNeckBlockV2 __all__ = ( "BlockGroup", "BottleNeckBlockV1", "BottleNeckBlockV2", ) ================================================ FILE: sonnet/optimizers.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Sonnet optimizers built for TensorFlow 2. All optimizers implement the `snt.Optimizer` interface. """ from sonnet.src.optimizers.adam import Adam from sonnet.src.optimizers.momentum import Momentum from sonnet.src.optimizers.rmsprop import RMSProp from sonnet.src.optimizers.sgd import SGD __all__ = ( "Adam", "Momentum", "RMSProp", "SGD", ) ================================================ FILE: sonnet/pad.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Paddings.""" from sonnet.src.pad import causal from sonnet.src.pad import create from sonnet.src.pad import full from sonnet.src.pad import reverse_causal from sonnet.src.pad import same from sonnet.src.pad import valid __all__ = ( "causal", "create", "full", "reverse_causal", "same", "valid", ) ================================================ FILE: sonnet/regularizers.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Regularizers.""" from sonnet.src.regularizers import L1 from sonnet.src.regularizers import L2 from sonnet.src.regularizers import OffDiagonalOrthogonal from sonnet.src.regularizers import Regularizer __all__ = [ "L1", "L2", "OffDiagonalOrthogonal", "Regularizer", ] ================================================ FILE: sonnet/src/BUILD ================================================ load("//sonnet/src:build_defs.bzl", "snt_py_library", "snt_py_test") package(default_visibility = ["//sonnet:__subpackages__", "//docs/ext:__subpackages__", "//examples:__subpackages__"]) licenses(["notice"]) snt_py_library( name = "base", srcs = ["base.py"], deps = [ ":once", ":types", ":utils", # pip: tensorflow ], ) snt_py_test( name = "base_test", srcs = ["base_test.py"], deps = [ ":base", ":test_utils", # pip: absl/testing:parameterized # pip: numpy # pip: tensorflow # pip: wrapt ], ) snt_py_library( name = "build", srcs = ["build.py"], deps = [ # pip: tensorflow # pip: tree ], ) snt_py_test( name = "build_test", srcs = ["build_test.py"], deps = [ ":build", ":test_utils", # pip: tensorflow ], ) snt_py_library( name = "reshape", srcs = ["reshape.py"], deps = [ ":base", ":once", ":types", # pip: numpy # pip: tensorflow ], ) snt_py_test( name = "reshape_test", srcs = ["reshape_test.py"], deps = [ ":reshape", ":test_utils", # pip: absl/testing:parameterized # pip: numpy # pip: tensorflow ], ) snt_py_library( name = "leaky_clip_by_value", srcs = ["leaky_clip_by_value.py"], deps = [ # pip: tensorflow ], ) snt_py_test( name = "leaky_clip_by_value_test", srcs = ["leaky_clip_by_value_test.py"], deps = [ ":leaky_clip_by_value", ":test_utils", # pip: absl/testing:parameterized # pip: tensorflow ], ) snt_py_library( name = "scale_gradient", srcs = ["scale_gradient.py"], deps = [ ":types", # pip: tensorflow ], ) snt_py_test( name = "scale_gradient_test", srcs = ["scale_gradient_test.py"], deps = [ ":scale_gradient", ":test_utils", # pip: absl/testing:parameterized # pip: tensorflow ], ) snt_py_library( name = "conv", srcs = ["conv.py"], deps = [ ":base", ":initializers", ":once", ":pad", ":utils", # pip: numpy # pip: tensorflow ], ) snt_py_test( name = "conv_test", srcs = ["conv_test.py"], deps = [ ":conv", ":initializers", ":test_utils", # pip: absl/testing:parameterized # pip: numpy # pip: tensorflow ], ) snt_py_library( name = "depthwise_conv", srcs = ["depthwise_conv.py"], deps = [ ":base", ":initializers", ":once", ":utils", # pip: numpy # pip: tensorflow ], ) snt_py_test( name = "depthwise_conv_test", srcs = ["depthwise_conv_test.py"], deps = [ ":depthwise_conv", ":initializers", ":test_utils", # pip: absl/testing:parameterized # pip: numpy # pip: tensorflow ], ) snt_py_library( name = "conv_transpose", srcs = ["conv_transpose.py"], deps = [ ":base", ":initializers", ":once", ":types", ":utils", # pip: numpy # pip: tensorflow ], ) snt_py_test( name = "conv_transpose_test", srcs = ["conv_transpose_test.py"], deps = [ ":conv_transpose", ":initializers", ":test_utils", # pip: absl/testing:parameterized # pip: numpy # pip: tensorflow ], ) snt_py_library( name = "custom_getter", srcs = ["custom_getter.py"], deps = [ ":base", # pip: tensorflow # pip: tree ], ) snt_py_test( name = "custom_getter_test", srcs = ["custom_getter_test.py"], deps = [ ":base", ":custom_getter", ":test_utils", # pip: tensorflow ], ) snt_py_library( name = "deferred", srcs = ["deferred.py"], deps = [ ":base", ], ) snt_py_test( name = "deferred_test", srcs = ["deferred_test.py"], deps = [ ":base", ":deferred", ":test_utils", # pip: tensorflow ], ) snt_py_library( name = "initializers", srcs = ["initializers.py"], deps = [ ":types", # pip: numpy # pip: tensorflow ], ) snt_py_test( name = "initializers_test", srcs = ["initializers_test.py"], deps = [ ":initializers", ":test_utils", # pip: absl/testing:parameterized # pip: numpy # pip: tensorflow ], ) snt_py_library( name = "linear", srcs = ["linear.py"], deps = [ ":base", ":initializers", ":once", ":utils", # pip: tensorflow ], ) snt_py_test( name = "linear_test", srcs = ["linear_test.py"], deps = [ ":linear", ":test_utils", # pip: absl/testing:parameterized # pip: numpy # pip: tensorflow ], ) snt_py_library( name = "batch_norm", srcs = ["batch_norm.py"], deps = [ ":base", ":initializers", ":metrics", ":moving_averages", ":once", ":types", ":utils", # pip: tensorflow ], ) snt_py_test( name = "batch_norm_test", srcs = ["batch_norm_test.py"], deps = [ ":batch_norm", ":initializers", ":test_utils", # pip: absl/testing:parameterized # pip: numpy # pip: tensorflow ], ) snt_py_library( name = "pad", srcs = ["pad.py"], deps = [ ":utils", ], ) snt_py_test( name = "pad_test", srcs = ["pad_test.py"], deps = [ ":pad", ":test_utils", # pip: absl/testing:parameterized # pip: tensorflow ], ) snt_py_library( name = "metrics", srcs = ["metrics.py"], deps = [ ":base", ":once", # pip: tensorflow ], ) snt_py_test( name = "metrics_test", srcs = ["metrics_test.py"], deps = [ ":metrics", ":test_utils", # pip: tensorflow ], ) snt_py_library( name = "once", srcs = ["once.py"], deps = [":utils"], ) snt_py_test( name = "once_test", srcs = ["once_test.py"], deps = [ ":once", # pip: absl/testing:absltest # pip: absl/testing:parameterized ], ) snt_py_library( name = "recurrent", srcs = ["recurrent.py"], deps = [ ":base", ":conv", ":initializers", ":linear", ":once", ":types", ":utils", # pip: tensorflow # pip: tree ], ) snt_py_test( name = "recurrent_test", srcs = ["recurrent_test.py"], deps = [ ":initializers", ":recurrent", ":test_utils", # pip: absl/testing:parameterized # pip: numpy # pip: tensorflow # pip: tree ], ) snt_py_library( name = "regularizers", srcs = ["regularizers.py"], deps = [ ":types", # pip: tensorflow ], ) snt_py_test( name = "regularizers_test", srcs = ["regularizers_test.py"], deps = [ ":regularizers", ":test_utils", # pip: numpy # pip: tensorflow ], ) snt_py_library( name = "sequential", srcs = ["sequential.py"], deps = [":base"], ) snt_py_test( name = "sequential_test", srcs = ["sequential_test.py"], deps = [ ":sequential", ":test_utils", # pip: absl/testing:parameterized # pip: tensorflow ], ) snt_py_library( name = "axis_norm", srcs = ["axis_norm.py"], deps = [ ":base", ":initializers", ":once", ":types", ":utils", # pip: tensorflow ], ) snt_py_test( name = "axis_norm_test", srcs = ["axis_norm_test.py"], deps = [ ":axis_norm", ":initializers", ":test_utils", # pip: absl/testing:parameterized # pip: numpy # pip: tensorflow ], ) snt_py_library( name = "group_norm", srcs = ["group_norm.py"], tags = [ "notap", # TODO(b/208346960): investigate flake on tpu. ], deps = [ ":base", ":initializers", ":once", ":types", ":utils", # pip: tensorflow ], ) snt_py_test( name = "group_norm_test", srcs = ["group_norm_test.py"], deps = [ ":group_norm", ":initializers", ":test_utils", # pip: absl/testing:parameterized # pip: numpy # pip: tensorflow ], ) snt_py_library( name = "moving_averages", srcs = ["moving_averages.py"], deps = [ ":metrics", ":once", ":types", # pip: tensorflow ], ) snt_py_test( name = "moving_averages_test", srcs = ["moving_averages_test.py"], deps = [ ":moving_averages", ":test_utils", # pip: absl/testing:parameterized # pip: tensorflow ], ) snt_py_library( name = "utils", srcs = ["utils.py"], deps = [ ":initializers", # pip: absl/logging # pip: tabulate # pip: tensorflow ], ) snt_py_test( name = "utils_test", srcs = ["utils_test.py"], deps = [ ":initializers", ":test_utils", ":utils", # pip: absl/testing:parameterized # pip: numpy # pip: tensorflow ], ) snt_py_library( name = "test_utils", testonly = 1, srcs = ["test_utils.py"], deps = [ # pip: absl/testing:parameterized # pip: tensorflow ], ) snt_py_library( name = "dropout", srcs = ["dropout.py"], deps = [ ":base", ":types", ":utils", # pip: tensorflow ], ) snt_py_test( name = "dropout_test", srcs = ["dropout_test.py"], deps = [ ":dropout", ":test_utils", # pip: absl/testing:parameterized # pip: numpy # pip: tensorflow ], ) snt_py_library( name = "bias", srcs = ["bias.py"], deps = [ ":base", ":initializers", ":once", ":types", ":utils", # pip: tensorflow ], ) snt_py_test( name = "bias_test", srcs = ["bias_test.py"], deps = [ ":bias", ":test_utils", # pip: tensorflow ], ) snt_py_library( name = "embed", srcs = ["embed.py"], deps = [ ":base", ":initializers", ":types", # pip: tensorflow ], ) snt_py_test( name = "embed_test", srcs = ["embed_test.py"], deps = [ ":embed", ":initializers", ":test_utils", # pip: absl/testing:parameterized # pip: tensorflow ], ) snt_py_library( name = "batch_apply", srcs = ["batch_apply.py"], deps = [ ":base", # pip: numpy # pip: tensorflow # pip: tree ], ) snt_py_test( name = "batch_apply_test", srcs = ["batch_apply_test.py"], deps = [ ":base", ":batch_apply", ":test_utils", # pip: absl/testing:parameterized # pip: numpy # pip: tensorflow ], ) snt_py_library( name = "mixed_precision", srcs = ["mixed_precision.py"], deps = [ ":custom_getter", ":utils", # pip: tensorflow # pip: tree ], ) snt_py_test( name = "mixed_precision_test", srcs = ["mixed_precision_test.py"], deps = [ ":base", ":mixed_precision", ":test_utils", # pip: absl/testing:parameterized # pip: tensorflow # pip: tree ], ) snt_py_library( name = "parallel_linear", srcs = ["parallel_linear.py"], deps = [ ":base", ":initializers", ":once", ":utils", # pip: tensorflow ], ) snt_py_test( name = "parallel_linear_test", srcs = ["parallel_linear_test.py"], deps = [ ":linear", ":parallel_linear", ":test_utils", # pip: tensorflow ], ) snt_py_library( name = "types", srcs = ["types.py"], deps = [ # pip: numpy # pip: tensorflow ], ) ================================================ FILE: sonnet/src/__init__.py ================================================ # Copyright 2021 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ ================================================ FILE: sonnet/src/axis_norm.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Generic axis normalization module.""" import collections.abc from typing import Optional from sonnet.src import base from sonnet.src import initializers from sonnet.src import once from sonnet.src import types from sonnet.src import utils import tensorflow as tf class LayerNorm(base.Module): r"""Normalizes inputs along the given axes. This is a generic implementation of normalization along specific axes of the input. :class:`InstanceNorm` is a subclass of this module, it normalizes over the spatial dimensions. It transforms the input ``x`` into: .. math:: \d{outputs} = \d{scale} \dfrac{x - \mu}{\sigma + \epsilon} + \d{offset} Where :math:`\mu` and :math:`\sigma` are respectively the mean and standard deviation of ``x``. There are many different variations for how users want to manage scale and offset if they require them at all. These are: - No ``scale``/``offset`` in which case ``create_*`` should be set to ``False`` and ``scale``/``offset`` aren't passed when the module is called. - Trainable ``scale``/``offset`` in which case create_* should be set to ``True`` and again ``scale``/``offset`` aren't passed when the module is called. In this case this module creates and owns the scale/offset variables. - Externally generated ``scale``/``offset``, such as for conditional normalization, in which case ``create_*`` should be set to ``False`` and then the values fed in at call time. Attributes: scale: If ``create_scale=True``, a trainable :tf:`Variable` holding the current scale. offset: If ``create_offset=True``, a trainable :tf:`Variable` holding the current offset. """ def __init__(self, axis: types.Axis, create_scale: bool, create_offset: bool, eps: types.FloatLike = 1e-5, scale_init: Optional[initializers.Initializer] = None, offset_init: Optional[initializers.Initializer] = None, data_format: str = "channels_last", name: Optional[str] = None): r"""Constructs an ``LayerNorm`` module. Args: axis: An ``int``, ``slice`` or sequence of ``int``\s representing the axes which should be normalized across. Typical usages are: ``1`` or ``-1`` for normalization over just the channels and ``slice(1, None)``, ``slice(2, None)`` for normalization over the spatial and channel dimensions whilst avoiding the batch and/or time dimensions. create_scale: ``bool`` representing whether to create a trainable scale per channel applied after the normalization. create_offset: ``bool`` representing whether to create a trainable offset per channel applied after normalization and scaling. eps: Small epsilon to avoid division by zero variance. Defaults to ``1e-5``. scale_init: Optional initializer for the scale variable. Can only be set if ``create_scale=True``. By default scale is initialized to ``1``. offset_init: Optional initializer for the offset variable. Can only be set if ``create_offset=True``. By default offset is initialized to ``0``. data_format: The data format of the input. Can be either ``channels_first``, ``channels_last``, ``N...C`` or ``NC...``. By default it is ``channels_last``. name: Name of the module. """ super().__init__(name=name) if isinstance(axis, slice): self._axis = axis elif isinstance(axis, int): self._axis = (axis,) elif (isinstance(axis, collections.abc.Iterable) and all(isinstance(ax, int) for ax in axis)): self._axis = axis else: raise ValueError("`axis` should be an int, slice or iterable of ints.") self._eps = eps self._data_format = data_format self._channel_index = utils.get_channel_index(data_format) self._rank = None self._create_scale = create_scale self._create_offset = create_offset if self._create_scale: self._scale_init = ( scale_init if scale_init is not None else initializers.Ones()) elif scale_init is not None: raise ValueError("Cannot set `scale_init` if `create_scale=False`.") if self._create_offset: self._offset_init = ( offset_init if offset_init is not None else initializers.Zeros()) elif offset_init is not None: raise ValueError("Cannot set `offset_init` if `create_offset=False`.") def __call__(self, inputs: tf.Tensor, scale: Optional[tf.Tensor] = None, offset: Optional[tf.Tensor] = None) -> tf.Tensor: """Returns normalized inputs. Args: inputs: An n-D tensor of the ``data_format`` specified in the constructor on which the transformation is performed. scale: A tensor up to n-D. The shape of this tensor must be broadcastable to the shape of ``inputs``. This is the scale applied to the normalized inputs. This cannot be passed in if the module was constructed with ``create_scale=True``. offset: A tensor up to n-D. The shape of this tensor must be broadcastable to the shape of ``inputs``. This is the offset applied to the normalized ``inputs``. This cannot be passed in if the module was constructed with ``create_offset=True``. Returns: An n-d tensor of the same shape as inputs that has been normalized. """ self._initialize(inputs) if self._create_scale: if scale is not None: raise ValueError( "Cannot pass `scale` at call time if `create_scale=True`.") scale = self.scale if self._create_offset: if offset is not None: raise ValueError( "Cannot pass `offset` at call time if `create_offset=True`.") offset = self.offset if len(inputs.shape) != self._rank: raise ValueError( "The rank of the inputs cannot change between calls, the" " original call was rank={} but this call was rank={}.".format( self._rank, len(inputs.shape))) mean, var = tf.nn.moments(inputs, self._axis, keepdims=True) normalized = tf.nn.batch_normalization( inputs, mean=mean, variance=var, scale=scale, offset=offset, variance_epsilon=self._eps) return normalized @once.once def _initialize(self, inputs: tf.Tensor): """Setup of rank specific values.""" self._rank = len(inputs.shape) # Turns slice into list of axis if isinstance(self._axis, slice): axes = tuple(range(self._rank)) self._axis = axes[self._axis] # Create scale and offset variables dtype = inputs.dtype if self._channel_index == -1: params_shape = [inputs.shape[-1]] else: # self._channel_index == 1 params_shape = [inputs.shape[1]] + [1] * (self._rank - 2) if self._create_scale: self.scale = tf.Variable( self._scale_init(params_shape, dtype), name="scale") else: self.scale = None if self._create_offset: self.offset = tf.Variable( self._offset_init(params_shape, dtype), name="offset") else: self.offset = None class InstanceNorm(LayerNorm): """Normalizes inputs along the spatial dimensions. See :class:`LayerNorm` for more details. Attributes: scale: If ``create_scale=True``, a trainable :tf:`Variable` holding the current scale. offset: If ``create_offset=True``, a trainable :tf:`Variable` holding the current offset. """ def __init__(self, create_scale: bool, create_offset: bool, eps: types.FloatLike = 1e-5, scale_init: Optional[initializers.Initializer] = None, offset_init: Optional[initializers.Initializer] = None, data_format: str = "channels_last", name: Optional[str] = None): """Constructs an ``InstanceNorm`` module. This method creates a module which normalizes over the spatial dimensions. Args: create_scale: ``bool`` representing whether to create a trainable scale per channel applied after the normalization. create_offset: ``bool`` representing whether to create a trainable offset per channel applied after normalization and scaling. eps: Small epsilon to avoid division by zero variance. Defaults to ``1e-5``. scale_init: Optional initializer for the scale variable. Can only be set if ``create_scale=True``. By default scale is initialized to ``1``. offset_init: Optional initializer for the offset variable. Can only be set if ``create_offset=True``. By default offset is initialized to ``0``. data_format: The data format of the input. Can be either ``channels_first``, ``channels_last``, ``N...C`` or ``NC...``. By default it is ``channels_last``. name: Name of the module. """ if utils.get_channel_index(data_format) == 1: axis = slice(2, None) else: # channel_index = -1 axis = slice(1, -1) super().__init__( axis=axis, create_scale=create_scale, create_offset=create_offset, eps=eps, scale_init=scale_init, offset_init=offset_init, data_format=data_format, name=name) ================================================ FILE: sonnet/src/axis_norm_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.axis_norm.""" from absl.testing import parameterized import numpy as np from sonnet.src import axis_norm from sonnet.src import initializers from sonnet.src import test_utils import tensorflow as tf class LayerNormTest(test_utils.TestCase, parameterized.TestCase): def testSimpleCase(self): layer = axis_norm.LayerNorm([1, 2], create_scale=False, create_offset=False) inputs = tf.ones([2, 3, 3, 5]) outputs = layer(inputs).numpy() for x in np.nditer(outputs): self.assertEqual(x, 0.0) def testSimpleCaseVar(self): layer = axis_norm.LayerNorm([1, 2], create_scale=True, create_offset=True, scale_init=initializers.Constant(0.5), offset_init=initializers.Constant(2.0)) inputs = tf.ones([2, 3, 3, 5]) outputs = layer(inputs).numpy() for x in np.nditer(outputs): self.assertEqual(x, 2.0) def testSimpleCaseNCHWVar(self): layer = axis_norm.LayerNorm([1, 2], create_scale=True, create_offset=True, scale_init=initializers.Constant(0.5), offset_init=initializers.Constant(2.0), data_format="NCHW") inputs = tf.ones([2, 5, 3, 3]) outputs = layer(inputs).numpy() for x in np.nditer(outputs): self.assertEqual(x, 2.0) def testDataFormatAgnosticVar(self): c_last_layer = axis_norm.LayerNorm([1, 2], create_scale=True, create_offset=True) c_first_layer = axis_norm.LayerNorm([2, 3], create_scale=True, create_offset=True, data_format="NCHW") inputs = tf.random.uniform([3, 4, 4, 5], 0, 10) c_last_output = c_last_layer(inputs) inputs = tf.transpose(inputs, [0, 3, 1, 2]) c_first_output = c_first_layer(inputs) c_first_output = tf.transpose(c_first_output, [0, 2, 3, 1]) self.assertAllClose(c_last_output.numpy(), c_first_output.numpy()) def testSimpleCaseTensor(self): layer = axis_norm.LayerNorm([1, 2], create_scale=False, create_offset=False) inputs = tf.ones([2, 3, 3, 5]) scale = tf.constant(0.5, shape=(5,)) offset = tf.constant(2.0, shape=(5,)) outputs = layer(inputs, scale, offset).numpy() for x in np.nditer(outputs): self.assertEqual(x, 2.0) def testSimpleCaseNCHWTensor(self): layer = axis_norm.LayerNorm([1, 2], data_format="NCHW", create_scale=False, create_offset=False) inputs = tf.ones([2, 5, 3, 3]) scale = tf.constant(0.5, shape=(5, 1, 1)) offset = tf.constant(2.0, shape=(5, 1, 1)) outputs = layer(inputs, scale, offset).numpy() for x in np.nditer(outputs): self.assertEqual(x, 2.0) def testDataFormatAgnosticTensor(self): c_last_layer = axis_norm.LayerNorm([1, 2], create_scale=False, create_offset=False) c_first_layer = axis_norm.LayerNorm([2, 3], data_format="NCHW", create_scale=False, create_offset=False) inputs = tf.random.uniform([3, 4, 4, 5], 0, 10) scale = tf.random.normal((5,), mean=1.0) offset = tf.random.normal((5,)) c_last_output = c_last_layer(inputs, scale, offset) inputs = tf.transpose(inputs, [0, 3, 1, 2]) scale = tf.reshape(scale, (5, 1, 1)) offset = tf.reshape(offset, (5, 1, 1)) c_first_output = c_first_layer(inputs, scale, offset) c_first_output = tf.transpose(c_first_output, [0, 2, 3, 1]) self.assertAllClose(c_last_output.numpy(), c_first_output.numpy()) @parameterized.parameters("NHW", "HWC", "channel_last") def testInvalidDataFormat(self, data_format): with self.assertRaisesRegex( ValueError, "Unable to extract channel information from '{}'.".format(data_format)): axis_norm.LayerNorm( 3, data_format=data_format, create_scale=False, create_offset=False) @parameterized.parameters("NCHW", "NCW", "channels_first") def testValidDataFormatChannelsFirst(self, data_format): test = axis_norm.LayerNorm( 3, data_format=data_format, create_scale=False, create_offset=False) self.assertEqual(test._channel_index, 1) @parameterized.parameters("NHWC", "NWC", "channels_last") def testValidDataFormatChannelsLast(self, data_format): test = axis_norm.LayerNorm( 3, data_format=data_format, create_scale=False, create_offset=False) self.assertEqual(test._channel_index, -1) @parameterized.named_parameters(("String", "foo"), ("ListString", ["foo"])) def testInvalidAxis(self, axis): with self.assertRaisesRegex( ValueError, "`axis` should be an int, slice or iterable of ints."): axis_norm.LayerNorm(axis, create_scale=False, create_offset=False) def testNoScaleAndInitProvided(self): with self.assertRaisesRegex( ValueError, "Cannot set `scale_init` if `create_scale=False`."): axis_norm.LayerNorm( 3, create_scale=False, create_offset=True, scale_init=initializers.Ones()) def testNoOffsetBetaInitProvided(self): with self.assertRaisesRegex( ValueError, "Cannot set `offset_init` if `create_offset=False`."): axis_norm.LayerNorm( 3, create_scale=True, create_offset=False, offset_init=initializers.Zeros()) def testCreateScaleAndScaleProvided(self): layer = axis_norm.LayerNorm([2], create_scale=True, create_offset=False) with self.assertRaisesRegex( ValueError, "Cannot pass `scale` at call time if `create_scale=True`."): layer(tf.ones([2, 3, 4]), scale=tf.ones([4])) def testCreateOffsetAndOffsetProvided(self): layer = axis_norm.LayerNorm([2], create_offset=True, create_scale=False) with self.assertRaisesRegex( ValueError, "Cannot pass `offset` at call time if `create_offset=True`."): layer(tf.ones([2, 3, 4]), offset=tf.ones([4])) def testSliceAxis(self): slice_layer = axis_norm.LayerNorm( slice(1, -1), create_scale=False, create_offset=False) axis_layer = axis_norm.LayerNorm((1, 2), create_scale=False, create_offset=False) inputs = tf.random.uniform([3, 4, 4, 5], 0, 10) scale = tf.random.normal((5,), mean=1.0) offset = tf.random.normal((5,)) slice_outputs = slice_layer(inputs, scale, offset) axis_outputs = axis_layer(inputs, scale, offset) self.assertAllEqual(slice_outputs.numpy(), axis_outputs.numpy()) def testRankChanges(self): layer = axis_norm.LayerNorm((1, 2), create_scale=False, create_offset=False) inputs = tf.ones([2, 3, 3, 5]) scale = tf.constant(0.5, shape=(5,)) offset = tf.constant(2.0, shape=(5,)) layer(inputs, scale, offset) with self.assertRaisesRegex( ValueError, "The rank of the inputs cannot change between calls, the original"): layer(tf.ones([2, 3, 3, 4, 5]), scale, offset) def testWorksWithFunction(self): layer = axis_norm.LayerNorm((1, 2), create_scale=False, create_offset=False) function_layer = tf.function(layer) inputs = tf.ones([2, 3, 3, 5]) scale = tf.constant(0.5, shape=(5,)) offset = tf.constant(2.0, shape=(5,)) outputs = layer(inputs, scale, offset) function_outputs = function_layer(inputs, scale, offset) self.assertAllEqual(outputs.numpy(), function_outputs.numpy()) def testShapeAgnostic(self): layer = axis_norm.LayerNorm((1, 2), create_scale=False, create_offset=False) inputs_spec = tf.TensorSpec([None, None, None, None], dtype=tf.float32) params_spec = tf.TensorSpec([None], dtype=tf.float32) function_layer = tf.function(layer).get_concrete_function( inputs_spec, params_spec, params_spec) scale = tf.constant(0.5, shape=(5,)) offset = tf.constant(2.0, shape=(5,)) outputs = function_layer(tf.ones([2, 3, 3, 5]), scale, offset) self.assertEqual(outputs.shape, [2, 3, 3, 5]) for x in np.nditer(outputs): self.assertEqual(x, 2.0) scale = tf.constant(0.5, shape=(3,)) offset = tf.constant(2.0, shape=(3,)) outputs = function_layer(tf.ones([3, 4, 6, 3]), scale, offset) self.assertEqual(outputs.shape, [3, 4, 6, 3]) for x in np.nditer(outputs): self.assertEqual(x, 2.0) def test5DDataFormatAgnostic(self): c_last_layer = axis_norm.LayerNorm([1, 2, 3], create_scale=False, create_offset=False) c_first_layer = axis_norm.LayerNorm([2, 3, 4], create_scale=False, create_offset=False, data_format="NCDHW") inputs = tf.random.uniform([3, 4, 4, 4, 5], 0, 10) scale = tf.random.normal((5,), mean=1.0) offset = tf.random.normal((5,)) c_last_output = c_last_layer(inputs, scale, offset) inputs = tf.transpose(inputs, [0, 4, 1, 2, 3]) scale = tf.reshape(scale, [-1, 1, 1, 1]) offset = tf.reshape(offset, [-1, 1, 1, 1]) c_first_output = c_first_layer(inputs, scale, offset) c_first_output = tf.transpose(c_first_output, [0, 2, 3, 4, 1]) self.assertAllClose( c_last_output.numpy(), c_first_output.numpy(), atol=1e-5, rtol=1e-5) def test3DDataFormatAgnostic(self): c_last_layer = axis_norm.LayerNorm([1], create_scale=False, create_offset=False) c_first_layer = axis_norm.LayerNorm([2], create_scale=False, create_offset=False, data_format="NCW") inputs = tf.random.uniform([3, 4, 5], 0, 10) scale = tf.random.normal((5,), mean=1.0) offset = tf.random.normal((5,)) c_last_output = c_last_layer(inputs, scale, offset) inputs = tf.transpose(inputs, [0, 2, 1]) scale = tf.reshape(scale, [-1, 1]) offset = tf.reshape(offset, [-1, 1]) c_first_output = c_first_layer(inputs, scale, offset) c_first_output = tf.transpose(c_first_output, [0, 2, 1]) self.assertAllClose( c_last_output.numpy(), c_first_output.numpy(), atol=1e-5, rtol=1e-5) def testInstanceNormCorrectAxis(self): layer = axis_norm.InstanceNorm(create_scale=True, create_offset=True) inputs = tf.ones([3, 4, 5, 6]) layer(inputs) self.assertEqual(layer._axis, (1, 2)) def testInstanceNormCorrectNCW(self): layer = axis_norm.InstanceNorm( create_scale=True, create_offset=True, data_format="channels_first") inputs = tf.ones([3, 4, 5, 6]) layer(inputs) self.assertEqual(layer._axis, (2, 3)) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/base.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Base Sonnet module.""" import abc import functools import inspect import os import pprint import sys from typing import Any, Callable, Dict, Optional, Sequence, Tuple, Type, TypeVar from sonnet.src import once from sonnet.src import types from sonnet.src import utils import tensorflow as tf T = TypeVar("T") TFFunctionType = type(tf.function(lambda: None, autograph=False)) # pylint: disable=invalid-name APPLY_NAME_SCOPE = "__snt_with_name_scope" ALLOW_EMPTY_RESULT = "__snt_allow_empty_result" def no_name_scope(method: T) -> T: """Decorator to wrap a method, preventing automatic name scope wrapping. By default, any method on a module is considered as a forwards function, and so any variables / modules created by the method will be scoped as belonging to the module. In some cases this is undesirable, for example when implementing ``.clone()`` / ``.transpose()``, as in those cases we want the new module to have the scope of wherever the ``.transpose()`` call is made. To allow this, decorate any methods with ``no_name_scope``. Args: method: the method to wrap. Returns: The method, with a flag indicating no name scope wrapping should occur. """ # NOTE: This logic is tied to ModuleMetaclass.__new__, if anything is # changed here corresponding changes will be needed there. setattr(method, APPLY_NAME_SCOPE, False) return method class ModuleMetaclass(abc.ABCMeta): """Metaclass for `Module`.""" def __new__( cls: Type[Type[T]], name: str, bases: Tuple[Type[Any], ...], clsdict: Dict[str, Any], ) -> Type[T]: methods = [] for key, value in clsdict.items(): if key == "name_scope": continue elif key.startswith("__") and key != "__call__": # Don't patch methods like `__getattr__` or `__del__`. continue elif isinstance(value, property): # TODO(tomhennigan) Preserve the type of property subclasses. clsdict[key] = property( value.fget if not value.fget else with_name_scope(value.fget), value.fset if not value.fset else with_name_scope(value.fset), value.fdel if not value.fdel else with_name_scope(value.fdel), doc=value.__doc__) elif inspect.isfunction(value) or isinstance(value, TFFunctionType): # We defer patching methods until after the type is created such that we # can trigger the descriptor binding them to the class. methods.append(key) clsdict.setdefault("__repr__", lambda module: module._auto_repr) # pylint: disable=protected-access new_cls = super(ModuleMetaclass, cls).__new__(cls, name, bases, clsdict) # pylint: disable=too-many-function-args for method_name in methods: # Note: the below is quite subtle, we need to ensure that we're wrapping # the method bound to the class. In some cases (e.g. `wrapt`) this is # important since the method can trigger different behavior when it is # bound (e.g. in wrapt `FunctionWrapper.__get__(None, cls)` produces a # `BoundFunctionWrapper` which in turn populates the `instance` argument # to decorator functions using args[0]). # Equivalent to: `cls.__dict__[method_name].__get__(None, cls)` method = getattr(new_cls, method_name) method = with_name_scope(method) setattr(new_cls, method_name, method) return new_cls def __call__(cls: Type[T], *args, **kwargs) -> T: # Call new such that we have an un-initialized module instance that we can # still reference even if there is an exception during __init__. This is # needed such that we can make sure the name_scope constructed in __init__ # is closed even if there is an exception. # NOTE: We disable pytype since (somewhat surprisingly) this method is bound # with the new class and not the metaclass. module = cls.__new__(cls, *args, **kwargs) # pytype: disable=wrong-arg-types # Now attempt to initialize the object. try: module.__init__(*args, **kwargs) finally: exc_info = sys.exc_info() # The base Module constructor enters the modules name scope before # returning such that other functionality in the ctor happens within the # modules name scope. ctor_name_scope = getattr(module, "_ctor_name_scope", None) if ctor_name_scope is not None: ctor_name_scope.__exit__(*exc_info) del module._ctor_name_scope # TODO(tomhennigan) Remove `_scope_name` after next TF release. ran_super_ctor = ( hasattr(module, "_name_scope") or hasattr(module, "_scope_name")) if exc_info[0] is None and not ran_super_ctor: raise ValueError( "Constructing a snt.Module without calling the super constructor " "is not supported. Add the following as the first line in your " "__init__ method:\n\nsuper(%s, self).__init__()" % cls.__name__) module._auto_repr = auto_repr(cls, *args, **kwargs) # pylint: disable=protected-access return module def safe_compare(a, b) -> bool: try: return bool(a == b) except: # pylint: disable=bare-except # Some equality checks might be buggy (e.g. `tf.Tensor == None`), in those # cases be defensive and assume `a != b`. Note that an exception is also # thrown when a and b are ndarrays of >1 element. # TODO(tomhennigan) We could be smarter about comparing ndarrays. return False def auto_repr(cls: Type[Any], *args, **kwargs) -> str: """Derives a `__repr__` from constructor arguments of a given class. >>> class Foo: ... def __init__(self, x=None, y=42): ... pass ... >>> auto_repr(Foo, "x") "Foo(x='x')" >>> auto_repr(Foo, "x", y=21) "Foo(x='x', y=21)" >>> auto_repr(Foo, None, 42) Foo() Args: cls: a class to derive `__repr__` for. *args: positional arguments. **kwargs: keyword arguments. Returns: A string representing a call equivalent to `cls(*args, **kwargs)`. """ argspec = inspect.getfullargspec(cls.__init__) arg_names = argspec.args # Keep used positionals minus self. arg_names = arg_names[1:(len(args) + 1)] # Keep used kwargs in the order they appear in argspec. arg_names.extend(n for n in argspec.args if n in kwargs) arg_values = inspect.getcallargs(cls.__init__, None, *args, **kwargs) # pylint: disable=deprecated-method # Extract default parameter values. defaults = argspec.defaults or () defaults = dict(zip(argspec.args[-len(defaults):], defaults)) is_default = lambda n, v: (n in defaults and safe_compare(v, defaults[n])) names_and_values = [(name + "=", arg_values[name]) for name in arg_names if not is_default(name, arg_values[name])] # Add varargs. names_and_values.extend(("", arg) for arg in args[len(argspec.args) - 1:]) # Add varkwargs. names_and_values.extend( (name + "=", kwargs[name]) for name in kwargs if name not in argspec.args) single_line = cls.__name__ + "({})".format(", ".join( name + repr(value) for name, value in names_and_values)) if len(single_line) <= 80: return single_line else: return "{}(\n{},\n)".format( cls.__name__, indent(4, ",\n".join(fancy_repr(n, v) for n, v in names_and_values))) def fancy_repr(name: str, value: Any) -> str: repr_value = pprint.pformat(value) if name: repr_value = indent(len(name), repr_value).strip() return name + repr_value def indent(amount: int, s: str) -> str: """Indents `s` with `amount` spaces.""" prefix = amount * " " return "\n".join(prefix + line for line in s.splitlines()) @utils.decorator def wrap_with_name_scope( method: Callable[..., T], instance: Any, args: Sequence[Any], kwargs: Dict[str, Any], ) -> T: """Decorator that calls the given function in the module name scope. Args: method: The bound method to call. instance: `Module` instance. args: Positional arguments to `method`. kwargs: Keyword arguments to `method`. Returns: `with instance.name_scope: return method(*args, **kwargs)` """ if instance is None: instance = args[0] args = args[1:] method = functools.partial(method, instance) try: module_name_scope = instance.name_scope except AttributeError as exc_value_from: exc_value = AttributeError( "The super constructor must be called before any other methods in " "your constructor. If this is not possible then annotate all the " "methods called with `@snt.no_name_scope`.") raise exc_value from exc_value_from with module_name_scope: # snt.Module enters the module name scope for all methods. To disable this # for a particular method annotate it with `@snt.no_name_scope`. return method(*args, **kwargs) @utils.decorator def wrap_with_name_scope_no_exception( method: Callable[..., T], instance: Any, args: Sequence[Any], kwargs: Dict[str, Any], ) -> T: """Patches the given method so it enters the modules name scope.""" if instance is None: instance = args[0] args = args[1:] method = functools.partial(method, instance) with instance.name_scope: # snt.Module enters the module name scope for all methods. To disable this # for a particular method annotate it with `@snt.no_name_scope`. return method(*args, **kwargs) def with_name_scope(method: T) -> T: """Patches the given method so it enters the modules name scope.""" if os.environ.get("SNT_MODULE_NAME_SCOPES", "").lower() in ("0", "false"): # For debugging purposes name scoping can be disabled using the environment # variable `SNT_MODULE_NAME_SCOPES` (note: this does not apply to __init__). # This can help to make stack traces shallower and should have no # behavioural effect (unless your code relies on string variable names). return method elif not getattr(method, APPLY_NAME_SCOPE, True): # The function has been annotated to say that no autoscoping should be # applied, so do not patch it. return method elif isinstance(method, TFFunctionType): # Autograph cannot convert functions that have try/catch. method._decorate(wrap_with_name_scope_no_exception) # pylint: disable=protected-access return method elif hasattr(method, "__snt_once_wrapped__"): # Special case methods decorated with @snt.once so the name scope is pushed # inside the function body rather than outside. This removes the overhead of # entering/exiting the name_scope just to do nothing. return once.once(wrap_with_name_scope(method.__snt_once_wrapped__)) # pylint: disable=no-value-for-parameter else: return wrap_with_name_scope(method) # pylint: disable=no-value-for-parameter NO_VARIABLES_ERROR = """ {module!r} does not currently contain any {property}. Most Sonnet modules create variables the first time they are called with an input and requesting variables before this typically indicates a coding error. You should refactor your code such that you request module variables after you pass an example input to the module. For example: module = {module!r} output = module(input) params = module.{property} If the module is stateless consider using `snt.allow_empty_variables(module)` to suppress this error: module = {module!r} snt.allow_empty_variables(module) params = module.{property} You can annotate your own subclasses directly if you prefer: @snt.allow_empty_variables class MyStatelessModule(snt.Module): pass """.strip() def allow_empty_variables(module_or_cls: T) -> T: """Allows ``{trainable_,}variables`` to return empty results. >>> mod = snt.Module() >>> mod.variables Traceback (most recent call last): ... ValueError: ... pass an example input to the module... >>> mod = snt.allow_empty_variables(mod) >>> mod.variables () Args: module_or_cls: A :class:`Module` instance or subclass to decorate. Returns: The input module or class. """ setattr(module_or_cls, ALLOW_EMPTY_RESULT, True) return module_or_cls def assert_tf2(): if not assert_tf2.checked: with tf.init_scope(): assert tf.executing_eagerly(), "Sonnet v2 requires TensorFlow 2" assert_tf2.checked = True assert_tf2.checked = False class Module(tf.Module, metaclass=ModuleMetaclass): """Base class for Sonnet modules. A Sonnet module is a lightweight container for variables and other modules. Modules typically define one or more "forward" methods (e.g. ``__call__``) which apply operations combining user input and module parameters. For example:: >>> class MultiplyModule(snt.Module): ... def __call__(self, x): ... if not hasattr(self, 'w'): ... self.w = tf.Variable(2., name='w') ... return x * self.w >>> mod = MultiplyModule() >>> mod(1.) Sonnet modules are a layer on top of :tf:`Module`, implementing automatic name scoping as described in the original RFC :cite:`agarwal2019stateful`. """ def __init__(self, name: Optional[str] = None): """Initializes the current module with the given name. Subclasses should call this constructor before creating other modules or variables such that those modules are named correctly. Args: name: An optional string name for the class. Must be a valid Python identifier. If ``name`` is not provided then the class name for the current instance is converted to ``lower_snake_case`` and used instead. """ assert_tf2() super().__init__(name=name) if getattr(self.__init__, APPLY_NAME_SCOPE, True): # Enter the name scope so subsequent code in the contructor (e.g. creating # submodules) happens inside the modules name scope. This is exited when # the subclass __init__ returns (this is implemented in ModuleMetaclass). self._ctor_name_scope = self.name_scope self._ctor_name_scope.__enter__() @property def variables(self): r"""Sequence of :tf:`Variable`\ s owned by this module and it's submodules. See :tf:`Module.variables` for implementation details. NOTE: Most Sonnet modules create variables lazily (e.g. the first time they are called). As such just after construction there are typically no variables. To mitigate a common error (calling ``.variables`` or ``.trainable_variables`` before any variables are created) these properties will raise an exception if their result is empty. See :func:`allow_empty_variables` if you want to suppress this error. Returns: A sequence of variables for the current module (sorted by attribute name) followed by variables from all submodules recursively (breadth first). """ variables = super().variables if not variables and not getattr(self, ALLOW_EMPTY_RESULT, False): # Raise a useful error if the collection is empty. Typically this # indicates that the user has requested the property before the module has # been connected. In many situations this can cause hard to diagnose # problems (eg. if you are trying to copy the initial state from one # module to another by zipping both module variables and assigning one to # the other). raise ValueError( NO_VARIABLES_ERROR.format(module=self, property="variables")) return variables @property def trainable_variables(self): r"""Sequence of :tf:`Variable`\ s owned by this module and it's submodules. See :tf:`Module.trainable_variables` for implementation details. NOTE: Most Sonnet modules create variables lazily (e.g. the first time they are called). As such just after construction there are typically no variables. To mitigate a common error (calling ``.variables`` or ``.trainable_variables`` before any variables are created) these properties will raise an exception if their result is empty. See :func:`allow_empty_variables` if you want to suppress this error. Returns: A sequence of variables for the current module (sorted by attribute name) followed by variables from all submodules recursively (breadth first). """ trainable_variables = super().trainable_variables if not trainable_variables and not getattr(self, ALLOW_EMPTY_RESULT, False): # Raise a useful error if the collection is empty. Typically this # indicates that the user has requested the property before the module has # been connected. In many situations this can cause hard to diagnose # problems (eg. if you are trying to copy the initial state from one # module to another by zipping both module variables and assigning one to # the other). raise ValueError( NO_VARIABLES_ERROR.format(module=self, property="trainable_variables")) return trainable_variables class Optimizer(Module): """Base class for Sonnet optimizers.""" @abc.abstractmethod def apply(self, updates: Sequence[types.ParameterUpdate], parameters: Sequence[tf.Variable]): """Applies `updates` to `parameters`.""" pass ================================================ FILE: sonnet/src/base_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.base.""" import abc from absl.testing import parameterized import numpy as np from sonnet.src import base from sonnet.src import test_utils import tensorflow as tf import wrapt class BaseTest(test_utils.TestCase): def test_basic(self): m = LambdaModule() self.assertIsNone(m(None)) def testWrappedMethod(self): mod = WraptModule() scope_name, y = mod(3) self.assertEqual(scope_name, "wrapt_module/") self.assertEqual(y, (3**2)**2) def testControlFlow(self): mod = ControlFlowModule() f = tf.function(mod).get_concrete_function(tf.TensorSpec([])) self.assertEqual(f(tf.constant(1.)).numpy(), 1.) self.assertEqual(f(tf.constant(11.)).numpy(), 11.**2) class TestModuleNaming(tf.test.TestCase): def test_single_name(self): mod = base.Module(name="simple") self.assertEqual(mod.name, "simple") self.assertEqual(mod.name_scope.name, "simple/") def test_construct_in_scope(self): with tf.name_scope("foo"): mod = base.Module(name="bar") self.assertEqual(mod.name, "bar") self.assertEqual(mod.name_scope.name, "foo/bar/") def test_enters_name_scope_in_call(self): mod = ReturnsNameScopeModule() for _ in range(3): self.assertEqual(mod(), mod.name_scope.name) def test_enters_name_scope_in_other_method(self): mod = ReturnsNameScopeModule() for _ in range(3): self.assertEqual(mod.alternative_forward(), mod.name_scope.name) def test_subclassed_module(self): mod = SubclassedReturnsNameScopeModule() for _ in range(3): self.assertEqual(mod.alternative_forward(), mod.name_scope.name) self.assertEqual(mod.alternative_alternative_forward(), mod.name_scope.name) def test_submodule_created_late(self): m = TreeModule() self.assertEqual(m.name, "tree_module") self.assertEqual(m.name_scope.name, "tree_module/") leaf1 = m.new_leaf() self.assertEqual(leaf1.name, "tree_module") self.assertEqual(leaf1.name_scope.name, "tree_module/tree_module/") def test_does_not_evaluate_property_methods(self): mod = PropertyThrowsWhenCalledModule() with self.assertRaises(AssertionError): mod.raise_assertion_error # pylint: disable=pointless-statement def test_overridden_name_scope(self): mod = ModuleOverridingNameScope() self.assertEqual(mod(), mod.name_scope.name) self.assertEqual(mod.alternative_forward(), mod.name_scope.name) def test_patched_callable(self): with tf.name_scope("foo"): mod = base.Module(name="bar") mod.foo = get_name_scope # `foo` is not a method so we do not re-enter the name scope. self.assertEqual(mod.foo(), "") def test_property(self): mod = PropertyModule() mod.some_property = None, None # None, None for the linter. getter_scope_name, setter_scope_name = mod.some_property self.assertEqual(getter_scope_name, "property_module/") self.assertEqual(setter_scope_name, "property_module/") def test_property_no_name_scope(self): mod = PropertyModule() mod.no_name_scope_property = None, None # None, None for the linter. getter_scope_name, setter_scope_name = mod.no_name_scope_property self.assertEqual(getter_scope_name, "") self.assertEqual(setter_scope_name, "") def test_ctor_no_name_scope(self): mod = CtorNoNameScope() self.assertEqual(mod.ctor_name_scope, "") self.assertEqual(mod.w.name, "w:0") def test_ctor_no_name_scope_no_super(self): msg = ("Constructing a snt.Module without calling the super constructor is " "not supported") with self.assertRaisesRegex(ValueError, msg): CtorNoNameScopeNoSuper() def test_invalid_name(self): msg = ".* is not a valid module name" with self.assertRaisesRegex(ValueError, msg): base.Module(name="$Foo") def test_modules_not_numbered_in_eager(self): mod = RecursiveModule(2) self.assertEqual(mod.name_scope.name, "badger/") self.assertEqual(mod.child.name_scope.name, "badger/badger/") mod = RecursiveModule(2) self.assertEqual(mod.name_scope.name, "badger/") self.assertEqual(mod.child.name_scope.name, "badger/badger/") def test_module_numbering_in_graph(self): with tf.Graph().as_default(): mod = RecursiveModule(2) self.assertEqual(mod.name_scope.name, "badger/") self.assertEqual(mod.child.name_scope.name, "badger/badger/") mod = RecursiveModule(2) self.assertEqual(mod.name_scope.name, "badger_1/") self.assertEqual(mod.child.name_scope.name, "badger_1/badger/") def test_ctor_error_closes_name_scope(self): with self.assertRaises(ErrorModuleError): # If super constructor is called then a name scope is opened then an error # is thrown. The metaclass should handle this and close the namescope # before re-throwing the exception. ErrorModule(call_super=True) self.assertEqual("", get_name_scope()) def test_ctor_error_handles_ctor_not_opening_name_scope(self): with self.assertRaises(ErrorModuleError): # If super ctor is not called then the name scope isn't opened. We need to # ensure that this doesn't trigger an exception (e.g. the metaclass trying # to __exit__ a non-existent name scope). ErrorModule(call_super=False) self.assertEqual("", get_name_scope()) def test_forward_method_closes_name_scope(self): mod = ErrorModule(call_super=True, raise_in_constructor=False) with self.assertRaises(ErrorModuleError): mod() self.assertEqual("", get_name_scope()) def test_get_attr_doesnt_enter_name_scope(self): scope_names = [] class GetAttrModule(base.Module): def __getattr__(self, name): scope_names.append((name, get_name_scope())) return super().__getattr__(name) mod = GetAttrModule() with self.assertRaises(AttributeError): mod.does_not_exist # pylint: disable=pointless-statement self.assertIn(("does_not_exist", ""), scope_names) def test_get_attribute_doesnt_enter_name_scope(self): scope_names = [] class GetAttributeModule(base.Module): def __getattribute__(self, name): scope_names.append((name, get_name_scope())) return super().__getattribute__(name) mod = GetAttributeModule() with self.assertRaises(AttributeError): mod.does_not_exist # pylint: disable=pointless-statement self.assertIn(("does_not_exist", ""), scope_names) class VariableNamingTest(tf.test.TestCase): def test_variable_names(self): mod = RecursiveModule(3) self.assertEqual(mod.w.name, "badger/mushroom:0") self.assertEqual(mod.child.w.name, "badger/badger/mushroom:0") self.assertEqual(mod.child.child.w.name, "badger/badger/badger/mushroom:0") class AutoReprTest(tf.test.TestCase): def test_order_matches_argspec(self): module = RecursiveModule(trainable=False, depth=2) self.assertEqual(repr(module), "RecursiveModule(depth=2, trainable=False)") def test_defaults_ignored(self): module = RecursiveModule(1) self.assertEqual(repr(module), "RecursiveModule(depth=1)") def test_does_not_fail_with_hostile_input(self): r = RaisesOnEquality() self.assertFalse(r.equality_checked) module = NoopModule(r) self.assertEqual(repr(module), "NoopModule(a=hostile)") self.assertTrue(r.equality_checked) def test_args_are_repred(self): module = TreeModule(name="TreeModule") self.assertEqual(repr(module), "TreeModule(name='TreeModule')") module = TreeModule("TreeModule") self.assertEqual(repr(module), "TreeModule(name='TreeModule')") def test_long_repr_multi_line(self): module = TakesSubmodules([TreeModule() for _ in range(6)], name="hai") self.assertEqual( repr(module), "\n".join([ "TakesSubmodules(", " submodules=[TreeModule(),", " TreeModule(),", " TreeModule(),", " TreeModule(),", " TreeModule(),", " TreeModule()],", " name='hai',", ")", ])) def test_repr_wildcard(self): module = WildcardInit(1, 2, 3, foo="bar") # NOTE: This is not a valid piece of Python, but it is unambiguous and # probably the most helpful thing we can do. An alternative would be to # special case `__init__(a, *args)` and not render names preceding *args # but this is unlikely to be common in the ctor. self.assertEqual(repr(module), "WildcardInit(a=1, b=2, 3, foo='bar')") def test_repr_non_bool_equality(self): class FooModule(base.Module): def __init__(self, a=((-1., -1.))): super().__init__() # auto_repr tests default values for equality. In numpy (and TF2) equality # is tested elementwise so the return value of `==` is an ndarray which we # then attempt to reduce to a boolean. foo = FooModule(a=np.array([[2., 2.]])) self.assertEqual(repr(foo), "FooModule(a=array([[2., 2.]]))") foo = FooModule(a=np.array([[-1., -1.]])) self.assertEqual(repr(foo), "FooModule(a=array([[-1., -1.]]))") class ForwardMethodsTest(tf.test.TestCase): def testFunctionType(self): mod = ModuleWithFunctionAnnotatedCall() self.assertIsInstance(mod.forward, base.TFFunctionType) self.assertIsInstance(mod.forward_ag, base.TFFunctionType) def testEntersNameScope_call(self): mod = ModuleWithFunctionAnnotatedCall() self.assertEqual(mod.forward().numpy(), b"module_with_function_annotated_call/") # TODO(b/122265385) Re-enable this assertion. # self.assertEqual(mod.forward_ag().numpy(), # b"module_with_function_annotated_call/") def testEntersNameScope_concreteFunction(self): mod = ModuleWithFunctionAnnotatedCall() self.assertEqual(mod.forward.get_concrete_function()().numpy(), b"module_with_function_annotated_call/") # TODO(b/122265385) Re-enable this assertion. # self.assertEqual(mod.forward_ag.get_concrete_function()().numpy(), # b"module_with_function_annotated_call/") class AbcTest(tf.test.TestCase): def testAbstract(self): msg = "Can't instantiate .* abstract method" with self.assertRaisesRegex(TypeError, msg): AbstractModule() # pylint: disable=abstract-class-instantiated def testConcrete(self): mod = ConcreteModule() x, scope_name = mod(2.) self.assertEqual(x, 4.) self.assertEqual(scope_name, "concrete_module/") self.assertEqual(get_name_scope(), "") def testCallMethodsOnParent(self): mod = ConcreteModule() self.assertEqual(mod.foo(), True) class CustomGradientTest(test_utils.TestCase): def test_custom_gradient(self): if tf.version.GIT_VERSION != "unknown": # TODO(tomhennigan) Enable this once TF 2.0.1 comes out. self.skipTest("Requires TF > 2.0.0") mod = ZeroGradModule() with tf.GradientTape() as tape: y = mod(2.) g = tape.gradient(y, mod.w) self.assertAllEqual(g, tf.zeros([2, 2])) class ZeroGradModule(base.Module): @tf.custom_gradient def __call__(self, x): if not hasattr(self, "w"): self.w = tf.Variable(tf.ones([2, 2]), name="w") with tf.GradientTape() as tape: y = tf.reduce_sum(self.w ** x) dw = tape.gradient(y, self.w) def grad(dy, variables=None): assert variables return dy * 0, [dw * 0] return y, grad class LambdaModule(base.Module): def __call__(self, x): return x def get_name_scope(): with tf.name_scope("x") as scope_name: return scope_name[:-2] @wrapt.decorator def wrapt_decorator(method, instance, args, kwargs): if instance is None: raise ValueError("Expected instance to be non-null.") scope_name, y = method(*args, **kwargs) return scope_name, y**2 class WraptModule(base.Module): @wrapt_decorator def __call__(self, x): return get_name_scope(), x**2 class ControlFlowModule(base.Module): def __call__(self, x): if x < 10: return x else: return x**2 class ErrorModuleError(Exception): pass class ErrorModule(base.Module): def __init__(self, call_super, raise_in_constructor=True): if call_super: super().__init__() if raise_in_constructor: raise ErrorModuleError("Deliberate error!") def __call__(self): raise ErrorModuleError("Deliberate error!") class RecursiveModule(base.Module): def __init__(self, depth, trainable=True): super().__init__(name="badger") self.child = None if depth > 1: self.child = RecursiveModule(depth - 1, trainable=trainable) self.w = tf.Variable(1.0, trainable=trainable, name="mushroom") class AbstractModule(base.Module, metaclass=abc.ABCMeta): @abc.abstractmethod def __call__(self, x): pass def foo(self): return True class ConcreteModule(AbstractModule): def __call__(self, x): return x**2, get_name_scope() class TreeModule(base.Module): def __init__(self, name=None): super().__init__(name=name) self._leaves = [] def new_leaf(self, name=None): leaf = TreeModule(name=name) self._leaves.append(leaf) return leaf class ReturnsNameScopeModule(base.Module): def alternative_forward(self): return get_name_scope() def __call__(self): return get_name_scope() class SubclassedReturnsNameScopeModule(ReturnsNameScopeModule): def alternative_alternative_forward(self): return get_name_scope() class PropertyThrowsWhenCalledModule(base.Module): @property def raise_assertion_error(self): raise AssertionError class ModuleOverridingNameScope(ReturnsNameScopeModule): @property def name_scope(self): return tf.name_scope("yolo/") class CommonErrorsTest(test_utils.TestCase, parameterized.TestCase): def test_not_calling_super_constructor(self): msg = ("Constructing a snt.Module without calling the super constructor is " "not supported") with self.assertRaisesRegex(ValueError, msg): DoesNotCallSuperConstructorModule() def test_calls_method_before_super(self): msg = "super constructor must be called before any other methods" with self.assertRaisesRegex(AttributeError, msg): CallsMethodBeforeSuperConstructorModule(allowed_method=False) def test_annotated_method_is_allowed(self): self.assertIsNotNone( CallsMethodBeforeSuperConstructorModule(allowed_method=True)) @parameterized.parameters("trainable_variables", "variables") def test_requests_variables_before_they_exist(self, property_name): class MyModule(base.Module): pass mod = MyModule() err = "MyModule.* does not currently contain any {}".format(property_name) with self.assertRaisesRegex(ValueError, err): getattr(mod, property_name) @parameterized.parameters("trainable_variables", "variables") def test_allow_empty_variables_instance(self, property_name): mod = base.Module() mod = base.allow_empty_variables(mod) self.assertEmpty(getattr(mod, property_name)) @parameterized.parameters("trainable_variables", "variables") def test_allow_empty_variables_class(self, property_name): mod = NeverCreatesVariables() self.assertEmpty(getattr(mod, property_name)) class NoopModule(base.Module): def __init__(self, a=None): super().__init__() self.a = a class RaisesOnEquality: equality_checked = False def __repr__(self): return "hostile" def __eq__(self, other): self.equality_checked = True raise ValueError("== not supported") def __ne__(self, other): self.equality_checked = True raise ValueError("!= not supported") @base.allow_empty_variables class NeverCreatesVariables(base.Module): pass class ModuleWithFunctionAnnotatedCall(base.Module): @tf.function(autograph=False) def forward(self): return get_name_scope() @tf.function(autograph=True) def forward_ag(self): return get_name_scope() class CtorNoNameScope(base.Module): @base.no_name_scope def __init__(self): super().__init__() self.ctor_name_scope = get_name_scope() self.w = tf.Variable(1., name="w") class CtorNoNameScopeNoSuper(base.Module): @base.no_name_scope def __init__(self): pass class PropertyModule(base.Module): def __init__(self): super().__init__() self._setter_scope_name = None @property def some_property(self): getter_scope_name = get_name_scope() return getter_scope_name, self._setter_scope_name @some_property.setter def some_property(self, my_property): self._setter_scope_name = get_name_scope() @property @base.no_name_scope def no_name_scope_property(self): getter_scope_name = get_name_scope() return getter_scope_name, self._setter_scope_name @no_name_scope_property.setter @base.no_name_scope def no_name_scope_property(self, my_property): self._setter_scope_name = get_name_scope() class DoesNotCallSuperConstructorModule(base.Module): def __init__(self): # NOTE: Intentionally does not call super constructor. pass class CallsMethodBeforeSuperConstructorModule(base.Module): def __init__(self, allowed_method): if allowed_method: self.no_name_scope() else: self.with_name_scope() super().__init__() @base.no_name_scope def no_name_scope(self): pass def with_name_scope(self): pass class CustomMetaclass(type): TAG = "__custom_metaclass__" def __new__(cls, name, bases, clsdict): new_type = super(CustomMetaclass, cls).__new__(cls, name, bases, clsdict) setattr(new_type, CustomMetaclass.TAG, True) return new_type class CombiningMetaclass(base.ModuleMetaclass, CustomMetaclass): TAG = "__combining_metaclass__" def __new__(cls, name, bases, clsdict): new_type = super(CombiningMetaclass, cls).__new__(cls, name, bases, clsdict) # pylint: disable=too-many-function-args setattr(new_type, CombiningMetaclass.TAG, True) return new_type class ModuleWithCustomMetaclass(base.Module, metaclass=CombiningMetaclass): def __init__(self): super(ModuleWithCustomMetaclass, self).__init__() self.init_name_scope = get_name_scope() class CustomMetaclassTest(tf.test.TestCase): def testSupportsCustomMetaclass(self): m = ModuleWithCustomMetaclass() self.assertEqual(m.init_name_scope, "module_with_custom_metaclass/") self.assertTrue(getattr(ModuleWithCustomMetaclass, CombiningMetaclass.TAG)) self.assertTrue(getattr(ModuleWithCustomMetaclass, CustomMetaclass.TAG)) class TakesSubmodules(base.Module): def __init__(self, submodules, name=None): super().__init__(name=name) class WildcardInit(base.Module): def __init__(self, a, b, *args, **kwargs): super().__init__() del args, kwargs if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/batch_apply.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Merges a number of leading dimensions of an input tensor to manipulate it.""" from typing import Any, Callable, Optional, Sequence, Union import numpy as np from sonnet.src import base import tensorflow as tf import tree class BatchApply(base.Module): """Merges a number of leading dimensions of an input tensor to manipulate it. Merges a number of leading dimensions of a tensor into a single dimension, connects the provided module, then splits the leading dimension of the result to match the input. Input tensors whose rank is smaller than the number of dimensions to collapse (e.g. all scalar values, which are tensors of rank 0), are passed unaltered to the provided module. This is useful for applying some module to each timestep of a Time x Batch x N tensor. If a module is hard coded to only support 2D (Batch x N) then the full 3D Tensor cannot be provided. BatchApply will 'merge' the first two dimensions of the sequence tensor by reshaping to a (Time * Batch) x N Tensor, and then the internal module can be applied. The result of that operation is reshaped such that its first dimensions are split to match the leading dimensions of the input. """ def __init__(self, module: Callable[..., tf.Tensor], num_dims: int = 2, name: Optional[str] = None): super().__init__(name=name) self.module = module self.num_dims = num_dims def __call__(self, *args, **kwargs): example = first_leaf(args, kwargs) if example is None: raise ValueError("BatchApply requires at least one tensor input.") num_dims = self.num_dims merge = lambda x: merge_leading_dims(x, num_dims=num_dims) split = lambda x: split_leading_dim(x, num_dims=num_dims, example=example) # Merge leading dimensions of inputs. # Example: [T, B, N] -> [T*B, N] args = tree.map_structure(merge, args) kwargs = tree.map_structure(merge, kwargs) # Compute merged output. # Example: [T*B, O] outputs = self.module(*args, **kwargs) # Split leading dimensions of output to match input. # Example: [T*B, O] -> [T, B, O] return tree.map_structure(split, outputs) def first_leaf(args, kwargs) -> Optional[Any]: flat_args = tree.flatten(args) if flat_args: return flat_args[0] flat_kwargs = tree.flatten(kwargs) if flat_kwargs: return flat_kwargs[0] return None def split_leading_dim( x: Optional[tf.Tensor], example: tf.Tensor, num_dims: int, ) -> Optional[tf.Tensor]: """Split the first dimension of a tensor to match an example. See :func:`merge_leading_dims`. >>> x = tf.ones([6, 1]) >>> example = tf.ones([3, 2, 1]) >>> snt.split_leading_dim(x, example, 2) If ``x`` is not a :tf:`Tensor` or :tf:`Variable` then is is returned unchanged: >>> snt.split_leading_dim('not a tensor', example, 2) 'not a tensor' Args: x: A tensor with leading dim merged. example: An Tensor with leading dim not merged. num_dims: The number of leading dimensions of example to use. Returns: A tensor with leading dim split, or the input unchanged. """ if x is None or not isinstance(x, (tf.Tensor, tf.Variable)): return x static_shape = example.shape[:num_dims] + x.shape[1:] if static_shape.is_fully_defined(): # pytype: disable=attribute-error return tf.reshape(x, static_shape) # Shape can't be inferred statically. leading_dims = tf.shape(example)[:num_dims] other_dims = tf.shape(x)[1:] dynamic_shape = tf.concat([leading_dims, other_dims], axis=0) return tf.reshape(x, dynamic_shape) def maybe_prod(s: Sequence[Union[int, None]]) -> Optional[int]: try: return np.prod(s) except TypeError: # Can happen if the input contains `None`. return None def merge_leading_dims( x: Optional[tf.Tensor], num_dims: int, ) -> Optional[tf.Tensor]: """Merges leading dimensions of a tensor. See :func:`split_leading_dim`. >>> x = tf.ones([3, 2, 1]) >>> snt.merge_leading_dims(x, num_dims=2) If the rank of ``x`` is less than ``num_dims`` it is returned unchanged: >>> snt.merge_leading_dims(x, 4) If ``x`` is not a :tf:`Tensor` or :tf:`Variable` then is is returned unchanged: >>> snt.merge_leading_dims('not a tensor', 1) 'not a tensor' Args: x: A :tf:`Tensor` to merge. num_dims: The number of leading dimensions to merge. Returns: A :tf:`Tensor` with merged leading dimensions or the input unchanged. """ if x is None or not isinstance(x, (tf.Tensor, tf.Variable)): return x # Check if the rank of the input tensor is well-defined. if x.shape.dims is None: raise ValueError( "Can't merge leading dimensions of tensor of unknown rank.") # We can only merge the num_dims leading dimensions if the rank of the given # tensor is sufficiently large. if num_dims > x.shape.rank: return x static_shape = [maybe_prod(x.shape[:num_dims])] + x.shape[num_dims:] if static_shape.is_fully_defined(): # pytype: disable=attribute-error return tf.reshape(x, static_shape) # Shape can't be inferred statically. tensor_shape = tf.shape(x) leading_dim = tf.reduce_prod(tensor_shape[:num_dims], keepdims=True) other_dims = tensor_shape[num_dims:] dynamic_shape = tf.concat([leading_dim, other_dims], axis=0) result = tf.reshape(x, dynamic_shape) # We lose some static shape information from the above reduce/slice/concat # dance, so we explicitly pass it in from what we computed earlier. result.set_shape(static_shape) return result ================================================ FILE: sonnet/src/batch_apply_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.batch_apply.""" from absl.testing import parameterized import numpy as np from sonnet.src import base from sonnet.src import batch_apply from sonnet.src import test_utils import tensorflow as tf EXAMPLE_INPUTS = ( ((1, 2, 3), 1), ((1, 2, 3), 2), ((1, 2, 3, 4), 3), ((1, 2, 3, 4, 5, 6), 4), ) class BatchApplyTest(test_utils.TestCase): def test_simple(self): m = batch_apply.BatchApply(AddOne()) x = tf.zeros([2, 3, 4]) y = m(x) self.assertAllEqual(y, tf.ones([2, 3, 4])) def test_no_output(self): m = batch_apply.BatchApply(NoOutputModule()) y = m(tf.ones([1, 1, 1])) self.assertIsNone(y) def test_kwargs(self): m = batch_apply.BatchApply(KwargsModule()) y = m(tf.ones([1, 1, 1]), is_training=True) self.assertIsNone(y) class MergeLeadingDimsTest(test_utils.TestCase, parameterized.TestCase): @parameterized.parameters(object(), (np.ones([]),), 1, None) def test_x_not_tensor(self, x): self.assertIs(x, batch_apply.merge_leading_dims(x, 1)) @parameterized.parameters(*EXAMPLE_INPUTS) def test_static_shape(self, x_shape, num_dims): x = tf.ones(x_shape) y = batch_apply.merge_leading_dims(x, num_dims) y_shape = (np.prod(x_shape[:num_dims]),) + x_shape[num_dims:] self.assertEqual(y.shape, y_shape) @parameterized.parameters(*EXAMPLE_INPUTS) def test_dynamic_shape(self, x_shape, num_dims): merge = tf.function(batch_apply.merge_leading_dims) x = tf.TensorSpec([None for _ in x_shape]) cf = merge.get_concrete_function(x, num_dims) y_shape = (np.prod(x_shape[:num_dims]),) + x_shape[num_dims:] y_shape_dynamic = cf.output_shapes y_shape_dynamic.assert_is_compatible_with(y_shape) x = tf.ones(x_shape) y = cf(x) self.assertEqual(y.shape, y_shape) @parameterized.parameters(*EXAMPLE_INPUTS) def test_dynamic_shape_has_static_info_in_graph(self, x_shape, num_dims): y_shape = (np.prod(x_shape[:num_dims]),) + x_shape[num_dims:] @tf.function def merge(x, num_dims): y = batch_apply.merge_leading_dims(x, num_dims) self.assertIsNotNone(y.shape.dims) y.shape.assert_is_compatible_with(y_shape) # Make sure we have static shape info except for the trailing None. self.assertNotIn(None, y.shape[:-1]) self.assertIsNone(y.shape[-1]) return y # Fill `None` in the last dimension (which won't be merged). x = tf.TensorSpec(x_shape[:-1] + (None,)) cf = merge.get_concrete_function(x, num_dims) y_shape_dynamic = cf.output_shapes y_shape_dynamic.assert_is_compatible_with(y_shape) class SplitLeadingDimTest(test_utils.TestCase, parameterized.TestCase): @parameterized.parameters(object(), (np.ones([]),), 1, None) def test_x_not_tensor(self, x): self.assertIs(x, batch_apply.split_leading_dim(x, None, 1)) @parameterized.parameters(*EXAMPLE_INPUTS) def test_static_shape(self, i_shape, num_dims): x_shape = (np.prod(i_shape[:num_dims]),) + (2, 2) y_shape = i_shape[:num_dims] + x_shape[1:] x = tf.ones(x_shape) i = tf.ones(i_shape) y = batch_apply.split_leading_dim(x, i, num_dims) self.assertEqual(y.shape, y_shape) @parameterized.parameters(*EXAMPLE_INPUTS) def test_dynamic_shape(self, i_shape, num_dims): x_shape = (np.prod(i_shape[:num_dims]),) + (2, 2) y_shape = i_shape[:num_dims] + x_shape[1:] # Build a concrete function with fully dynamic input dimensions. x = tf.TensorSpec([None for _ in x_shape]) i = tf.TensorSpec([None for _ in i_shape]) split = tf.function(batch_apply.split_leading_dim) cf = split.get_concrete_function(x, i, num_dims) y_shape_dynamic = cf.output_shapes y_shape_dynamic.assert_is_compatible_with(y_shape) # Make use of the concrete function with fully specified inputs. x = tf.ones(x_shape) i = tf.ones(i_shape) y = cf(x, i) self.assertEqual(y.shape, y_shape) @parameterized.parameters(*EXAMPLE_INPUTS) def test_dynamic_shape_has_static_info_in_graph(self, i_shape, num_dims): x_shape = (np.prod(i_shape[:num_dims]),) + (2, 2) y_shape = i_shape[:num_dims] + x_shape[1:] @tf.function def split(x, i, num_dims): y = batch_apply.split_leading_dim(x, i, num_dims) self.assertIsNotNone(y.shape.dims) y.shape.assert_is_compatible_with(y_shape) self.assertNotIn(None, y.shape[1:]) return y # Build a concrete function with fully dynamic input dimensions. x = tf.TensorSpec((None,) + x_shape[1:]) i = tf.TensorSpec((None,) + i_shape[1:]) cf = split.get_concrete_function(x, i, num_dims) y_shape_dynamic = cf.output_shapes y_shape_dynamic.assert_is_compatible_with(y_shape) class NoOutputModule(base.Module): def __call__(self, x): return None class KwargsModule(base.Module): def __call__(self, x, is_training=None): if is_training: return None class AddOne(base.Module): def __call__(self, x): assert len(x.shape) == 2, "Requires rank 2 input." return x + 1. if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/batch_norm.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Batch normalization module.""" from typing import Optional, Tuple from sonnet.src import base from sonnet.src import initializers from sonnet.src import metrics from sonnet.src import moving_averages from sonnet.src import once from sonnet.src import types from sonnet.src import utils import tensorflow as tf class BaseBatchNorm(base.Module): r"""Batch normalization module. This implements normalization across the batch and spatial dimensions. It maintains moving averages of the mean and variance which can be used to normalize at test time. The constructor is generic and requires the user to pass in objects to compute these. At training time we use the batch statistics for that batch and these are then used to update the moving averages. At test time we can either use the moving averages of the batch statistics (``test_local_stats=False``) or we can use the local statistics (``test_local_stats=True``). It transforms the input ``x`` into: .. math:: \d{outputs} = \d{scale} \dfrac{x - \mu}{\sigma + \epsilon} + \d{offset} Where :math:`\mu` and :math:`\sigma` are respectively the mean and standard deviation of ``x``. Note that this module automatically uses the fused batch norm op if the data format is ``NHWC``. There are many different variations for how users want to manage scale and offset if they require them at all. These are: - No scale/offset in which case ``create_*`` should be set to ``False`` and ``scale``/``offset`` aren't passed when the module is called. - Trainable scale/offset in which case ``create_*`` should be set to ``True`` and again ``scale``/``offset`` aren't passed when the module is called. In this case this module creates and owns the ``scale``/``offset`` variables. - Externally generated ``scale``/``offset``, such as for conditional normalization, in which case ``create_*`` should be set to ``False`` and then the values fed in at call time. Attributes: scale: If ``create_scale``, a trainable :tf:`Variable` holding the current scale after the module is connected for the first time. offset: If ``create_offset``, a trainable :tf:`Variable` holding the current offset after the module is connected for the first time. """ def __init__(self, create_scale: bool, create_offset: bool, moving_mean: metrics.Metric, moving_variance: metrics.Metric, eps: types.FloatLike = 1e-5, scale_init: Optional[initializers.Initializer] = None, offset_init: Optional[initializers.Initializer] = None, data_format: str = "channels_last", name: Optional[str] = None): """Constructs a ``BaseBatchNorm`` module. Args: create_scale: whether to create a trainable scale per channel applied after the normalization. create_offset: whether to create a trainable offset per channel applied after normalization and scaling. moving_mean: A metric which tracks the moving average of the mean which can be used to normalize at test time. moving_variance: A metric which tracks the moving average of the variance which can be used to normalize at test time. eps: Small epsilon to avoid division by zero variance. Defaults to ``1e-5``. scale_init: Optional initializer for the scale variable. Can only be set if ``create_scale=True``. By default scale is initialized to ``1``. offset_init: Optional initializer for the offset variable. Can only be set if ``create_offset=True``. By default offset is initialized to ``0``. data_format: The data format of the input. Can be either ``channels_first``, ``channels_last``, ``N...C`` or ``NC...``. By default it is ``channels_last``. name: Name of the module. """ super().__init__(name=name) self._eps = eps self.moving_mean = moving_mean self.moving_variance = moving_variance self._data_format = data_format self._channel_index = utils.get_channel_index(data_format) self._create_scale = create_scale self._create_offset = create_offset if not self._create_scale and scale_init is not None: raise ValueError("Cannot set `scale_init` if `create_scale=False`") self._scale_init = scale_init or initializers.Ones() if not self._create_offset and offset_init is not None: raise ValueError("Cannot set `offset_init` if `create_offset=False`") self._offset_init = offset_init or initializers.Zeros() @utils.smart_autograph def __call__(self, inputs: tf.Tensor, is_training: types.BoolLike, test_local_stats: types.BoolLike = False, scale: Optional[tf.Tensor] = None, offset: Optional[tf.Tensor] = None): """Returns normalized inputs. Args: inputs: An n-D tensor of the data_format specified above on which the transformation is performed. is_training: Whether the module should be connected in training mode, meaning the moving averages are updated. test_local_stats: Whether local batch statistics should be used when ``is_training=False``. If not, moving averages are used. By default ``False``. scale: A tensor up to n-D. The shape of this tensor must be broadcastable to the shape of ``inputs``. This is the scale applied to the normalized inputs. This cannot be passed in if the module was constructed with ``create_scale=True``. offset: A tensor up to n-D. The shape of this tensor must be broadcastable to the shape of ``inputs``. This is the offset applied to the normalized inputs. This cannot be passed in if the module was constructed with ``create_offset=True``. Returns: An n-d tensor of the same shape as inputs that has been normalized. """ use_batch_stats = is_training or test_local_stats if self._create_scale: if scale is not None: raise ValueError( "Cannot pass `scale` at call time if `create_scale=True`.") if self._create_offset: if offset is not None: raise ValueError( "Cannot pass `offset` at call time if `create_offset=True`.") self._initialize(inputs) if scale is None: scale = self.scale if offset is None: offset = self.offset mean, variance = self._moments(inputs, use_batch_stats) if self._fused: out, mean, variance, _, _ = tf.raw_ops.FusedBatchNormV2( x=inputs, mean=mean, variance=variance, scale=scale, offset=offset, is_training=use_batch_stats, epsilon=self._eps, data_format=self._fused_data_format) else: out = tf.nn.batch_normalization( inputs, mean=mean, variance=variance, scale=scale, offset=offset, variance_epsilon=self._eps) if is_training: self._update_statistics(mean, variance) return out @once.once def _initialize(self, inputs: tf.Tensor): input_shape = inputs.shape rank = len(input_shape) self._fused = (rank == 4 and self._channel_index == -1) self._fused_data_format = "NHWC" if self._channel_index == -1 else "NCHW" if self._channel_index < 0: channel_index = self._channel_index + rank else: channel_index = self._channel_index self._axis = tuple(i for i in range(rank) if i != channel_index) # Ensure all the variables are created on the first call mean, variance = tf.nn.moments(inputs, self._axis, keepdims=True) self.shape = mean.shape self.moving_mean.initialize(mean) self.moving_variance.initialize(variance) dtype = inputs.dtype if self._channel_index == -1: params_shape = [inputs.shape[-1]] else: # self._channel_index == 1 params_shape = [inputs.shape[1]] + [1] * (rank - 2) # Creates scale and offset parameters - required for fused_batch_norm # trainable set to with_scale and with_offset which gives no-op if false self.scale = tf.Variable( self._scale_init(params_shape, dtype), name="scale", trainable=self._create_scale) self.offset = tf.Variable( self._offset_init(params_shape, dtype), name="offset", trainable=self._create_offset) if self._fused: with tf.init_scope(): self._fused_constant = tf.constant([]) def _moments(self, inputs: tf.Tensor, use_batch_stats: types.BoolLike) -> Tuple[tf.Tensor, tf.Tensor]: if use_batch_stats: if self._fused: # The raw ops version of fused batch norm calculates the mean and # variance internally but requires tensors to be passed in. mean = self._fused_constant variance = self._fused_constant else: mean, variance = tf.nn.moments(inputs, self._axis, keepdims=True) else: # use moving stats mean = self.moving_mean.value variance = self.moving_variance.value if self._fused: mean = tf.squeeze(mean, self._axis) variance = tf.squeeze(variance, self._axis) return mean, variance def _update_statistics(self, mean, variance): if self._fused: mean = tf.reshape(mean, self.shape) variance = tf.reshape(variance, self.shape) self.moving_mean.update(mean) self.moving_variance.update(variance) class BatchNorm(BaseBatchNorm): """Batch normalization with exponential moving average for test statistics. See :class:`BaseBatchNorm` for details. Attributes: scale: If ``create_scale=True``, a trainable :tf:`Variable` holding the current scale after the module is connected for the first time. offset: If ``create_offset``, a trainable :tf:`Variable` holding the current offset after the module is connected for the first time. """ def __init__(self, create_scale: bool, create_offset: bool, decay_rate: float = 0.999, eps: types.FloatLike = 1e-5, scale_init: Optional[initializers.Initializer] = None, offset_init: Optional[initializers.Initializer] = None, data_format: str = "channels_last", name: Optional[str] = None): """Constructs a ``BatchNorm`` module. Args: create_scale: whether to create a trainable scale per channel applied after the normalization. create_offset: whether to create a trainable offset per channel applied after normalization and scaling. decay_rate: Decay rate of the exponential moving averages of the mean and variance. eps: Small epsilon to avoid division by zero variance. Defaults to ``1e-5``. scale_init: Optional initializer for the scale variable. Can only be set if ``create_scale=True``. By default scale is initialized to ``1``. offset_init: Optional initializer for the offset variable. Can only be set if ``create_offset=True``. By default offset is initialized to ``0``. data_format: The data format of the input. Can be either ``channels_first``, ``channels_last``, ``N...C`` or ``NC...``. By default it is ``channels_last``. name: Name of the module. """ with tf.name_scope(name or "batch_norm"): moving_mean = moving_averages.ExponentialMovingAverage( decay_rate, name="moving_mean") moving_variance = moving_averages.ExponentialMovingAverage( decay_rate, name="moving_variance") super().__init__( create_scale=create_scale, create_offset=create_offset, moving_mean=moving_mean, moving_variance=moving_variance, eps=eps, scale_init=scale_init, offset_init=offset_init, data_format=data_format, name=name) ================================================ FILE: sonnet/src/batch_norm_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.batch_norm.""" import itertools from absl.testing import parameterized import numpy as np from sonnet.src import batch_norm from sonnet.src import initializers from sonnet.src import test_utils import tensorflow as tf class BaseBatchNormTest(test_utils.TestCase, parameterized.TestCase): def testSimpleTraining(self): layer = batch_norm.BaseBatchNorm( moving_mean=TestMetric(), moving_variance=TestMetric(), create_scale=False, create_offset=False) inputs = tf.ones([2, 3, 3, 5]) scale = tf.constant(0.5, shape=(5,)) offset = tf.constant(2.0, shape=(5,)) outputs = layer(inputs, True, scale=scale, offset=offset).numpy() self.assertAllEqual(outputs, tf.fill(inputs.shape, 2.0)) self.assertEqual((0, 1, 2), layer._axis) def testSimpleTrainingNCHW(self): layer = batch_norm.BaseBatchNorm( moving_mean=TestMetric(), moving_variance=TestMetric(), create_scale=False, create_offset=False, data_format="NCHW") inputs = tf.ones([2, 5, 3, 3]) scale = tf.constant(0.5, shape=(5, 1, 1)) offset = tf.constant(2.0, shape=(5, 1, 1)) outputs = layer(inputs, True, scale=scale, offset=offset).numpy() self.assertAllEqual(outputs, tf.fill(inputs.shape, 2.0)) self.assertEqual((0, 2, 3), layer._axis) def testSimpleTraining3D(self): layer = batch_norm.BaseBatchNorm( moving_mean=TestMetric(), moving_variance=TestMetric(), create_scale=False, create_offset=False) inputs = tf.ones([2, 3, 3, 3, 5]) scale = tf.constant(0.5, shape=(5,)) offset = tf.constant(2.0, shape=(5,)) outputs = layer(inputs, True, scale=scale, offset=offset).numpy() self.assertAllEqual(outputs, tf.fill(inputs.shape, 2.0)) self.assertEqual((0, 1, 2, 3), layer._axis) def testSimpleTraining3DNCDHW(self): layer = batch_norm.BaseBatchNorm( moving_mean=TestMetric(), moving_variance=TestMetric(), create_scale=False, create_offset=False, data_format="NCDHW") inputs = tf.ones([2, 5, 3, 3, 3]) scale = tf.constant(0.5, shape=(5, 1, 1, 1)) offset = tf.constant(2.0, shape=(5, 1, 1, 1)) outputs = layer(inputs, True, scale=scale, offset=offset).numpy() self.assertAllEqual(outputs, tf.fill(inputs.shape, 2.0)) self.assertEqual((0, 2, 3, 4), layer._axis) def testNoScaleAndOffset(self): layer = batch_norm.BaseBatchNorm( moving_mean=TestMetric(), moving_variance=TestMetric(), create_scale=False, create_offset=False, data_format="NHWC") inputs = tf.ones([2, 5, 3, 3, 3]) outputs = layer(inputs, True) self.assertAllEqual(outputs, tf.zeros_like(inputs)) def testSingleBatchInference(self): layer = batch_norm.BaseBatchNorm( moving_mean=TestMetric(), moving_variance=TestMetric(), create_scale=True, create_offset=True) inputs = tf.ones([1, 1, 1, 1]) outputs = layer(inputs, False) self.assertAllEqual(outputs, tf.zeros_like(inputs)) @parameterized.parameters(True, False) def testWithTfFunction(self, autograph): if "TPU" in self.device_types: self.skipTest("Test not working on TPU") # TODO(tamaranorman) enable on TPU layer = batch_norm.BaseBatchNorm( moving_mean=TestMetric(), moving_variance=TestMetric(), create_scale=False, create_offset=False, data_format="NHWC") layer = tf.function(layer, autograph=autograph) inputs = tf.ones([2, 5, 3, 3, 3]) scale = tf.constant(0.5, shape=(5, 1, 1, 1)) offset = tf.constant(2.0, shape=(5, 1, 1, 1)) expected1 = tf.zeros_like(inputs) expected2 = tf.fill(inputs.shape, 2.0) for is_training, use_batch_stats in itertools.product((True, False), (True, False)): outputs = layer(inputs, is_training, use_batch_stats) self.assertAllEqual(outputs, expected1) outputs = layer( inputs, is_training, use_batch_stats, scale=scale, offset=offset) self.assertAllEqual(outputs, expected2) @parameterized.parameters(True, False) def testWithTfFunctionTfArgs(self, autograph): if "TPU" in self.device_types: self.skipTest("Test not working on TPU") # TODO(tamaranorman) enable on TPU layer = batch_norm.BaseBatchNorm( moving_mean=TestMetric(), moving_variance=TestMetric(), create_scale=False, create_offset=False, data_format="NHWC") layer = tf.function(layer, autograph=autograph) inputs = tf.ones([2, 5, 3, 3, 3]) expected = tf.zeros_like(inputs) for is_training, use_batch_stats in itertools.product((True, False), (True, False)): # NOTE: The use of `tf.constant` means we require graph control flow outputs = layer(inputs, tf.constant(is_training), tf.constant(use_batch_stats)) self.assertAllEqual(outputs, expected) def testUsingTestStats(self): layer = batch_norm.BaseBatchNorm( moving_mean=TestMetric(), moving_variance=TestMetric(), create_scale=False, create_offset=False) inputs = tf.ones([2, 3, 3, 5]) scale = tf.constant(0.5, shape=(5,)) offset = tf.constant(2.0, shape=(5,)) outputs = layer(inputs, True, scale=scale, offset=offset).numpy() self.assertAllEqual(outputs, tf.fill(inputs.shape, 2.0)) outputs = layer(inputs, False, scale=scale, offset=offset).numpy() for x in np.nditer(outputs): self.assertAllClose(x, 2.0, rtol=1e-5, atol=1e-3) def testIsTrainingFalseFirstCall(self): layer = batch_norm.BaseBatchNorm( moving_mean=TestMetric(), moving_variance=TestMetric(), create_scale=False, create_offset=False) inputs = tf.ones([2, 3, 3, 5]) outputs = layer(inputs, False) self.assertAllEqual(outputs, tf.fill(inputs.shape, 0.0)) @parameterized.parameters("NHW", "HWC", "channel_last") def testInvalidDataFormat(self, data_format): with self.assertRaisesRegex( ValueError, "Unable to extract channel information from '{}'".format(data_format)): batch_norm.BaseBatchNorm( moving_mean=TestMetric(), moving_variance=TestMetric(), create_scale=False, create_offset=False, data_format=data_format) @parameterized.parameters("NCHW", "NCW", "channels_first") def testValidDataFormatChannelsFirst(self, data_format): test = batch_norm.BaseBatchNorm( moving_mean=TestMetric(), moving_variance=TestMetric(), create_scale=False, create_offset=False, data_format=data_format) self.assertEqual(test._channel_index, 1) @parameterized.parameters("NHWC", "NWC", "channels_last") def testValidDataFormatChannelsLast(self, data_format): test = batch_norm.BaseBatchNorm( moving_mean=TestMetric(), moving_variance=TestMetric(), create_scale=False, create_offset=False, data_format=data_format) self.assertEqual(test._channel_index, -1) def testNoScaleAndInitProvided(self): with self.assertRaisesRegex( ValueError, "Cannot set `scale_init` if `create_scale=False`"): batch_norm.BaseBatchNorm( moving_mean=TestMetric(), moving_variance=TestMetric(), create_scale=False, create_offset=True, scale_init=initializers.Ones()) def testNoOffsetBetaInitProvided(self): with self.assertRaisesRegex( ValueError, "Cannot set `offset_init` if `create_offset=False`"): batch_norm.BaseBatchNorm( moving_mean=TestMetric(), moving_variance=TestMetric(), create_scale=True, create_offset=False, offset_init=initializers.Zeros()) class BatchNormTest(test_utils.TestCase, parameterized.TestCase): def testSimple(self): layer = batch_norm.BatchNorm(False, False) inputs = tf.ones([2, 3, 3, 5]) scale = tf.constant(0.5, shape=(5,)) offset = tf.constant(2.0, shape=(5,)) outputs = layer(inputs, True, scale=scale, offset=offset).numpy() self.assertAllEqual(outputs, tf.fill(inputs.shape, 2.0)) class TestMetric: def __init__(self): self._foo = None self._built = False def update(self, x): if self._built: self._foo.assign(x) else: self._foo = tf.Variable(x) self._built = True @property def value(self): return self._foo def initialize(self, x): self._foo = tf.Variable(x) self._built = True if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/bias.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Bias module.""" from typing import Optional, Sequence from sonnet.src import base from sonnet.src import initializers from sonnet.src import once from sonnet.src import types from sonnet.src import utils import tensorflow as tf class Bias(base.Module): """Bias module. Example Usage: >>> N, H, W, C = 1, 2, 3, 4 >>> x = tf.random.normal([N, H, W, C]) >>> scalar_bias = snt.Bias(bias_dims=[]) >>> scalar_bias_output = scalar_bias(x) >>> assert scalar_bias.b.shape == [] Create a bias over all non-minibatch dimensions: >>> all_bias = snt.Bias() >>> all_bias_output = all_bias(x) >>> assert all_bias.b.shape == [H, W, C] Create a bias over the last non-minibatch dimension: >>> last_bias = snt.Bias(bias_dims=[-1]) >>> last_bias_output = last_bias(x) >>> assert last_bias.b.shape == [C] Create a bias over the first non-minibatch dimension: >>> first_bias = snt.Bias(bias_dims=[1]) >>> first_bias_output = first_bias(x) >>> assert first_bias.b.shape == [H, 1, 1] Subtract and later add the same learned bias: >>> bias = snt.Bias() >>> h1 = bias(x, multiplier=-1) >>> h2 = bias(x) >>> h3 = bias(x, multiplier=-1) >>> reconstructed_x = bias(h3) >>> assert tf.reduce_all(tf.equal(x, reconstructed_x)) """ def __init__(self, output_size: Optional[int] = None, bias_dims: Optional[Sequence[int]] = None, b_init: Optional[initializers.Initializer] = None, name: Optional[str] = None): """Constructs a `Bias` module that supports broadcasting. Args: output_size: Output size (output shape without batch dimension). If `output_size` is left as `None`, the size will be directly inferred by the input. bias_dims: Sequence of which dimensions to retain from the input shape when constructing the bias. The remaining dimensions will be broadcast over (given size of 1), and leading dimensions will be removed completely. See class doc for examples. b_init: Optional initializer for the bias. Default to zeros. name: Name of the module. """ super().__init__(name=name) self.output_size = output_size self.bias_dims = bias_dims self.b_init = initializers.Zeros() if b_init is None else b_init @once.once def _initialize(self, inputs): utils.assert_minimum_rank(inputs, 2) input_shape = inputs.shape bias_shape = calculate_bias_shape(input_shape, self.bias_dims) input_size = input_shape[1:] if self.output_size is not None: if self.output_size != input_size: raise ValueError("Input shape must be {} not {}".format( (-1,) + self.output_size, input_shape)) self.input_size = input_size self.b = tf.Variable(self.b_init(bias_shape, inputs.dtype), name="b") def __call__(self, inputs: tf.Tensor, multiplier: types.FloatLike = None): """Adds bias to `inputs` and optionally multiplies by `multiplier`. Args: inputs: A Tensor of size `[batch_size, input_size1, ...]`. multiplier: A scalar or Tensor which the bias term is multiplied by before adding it to `inputs`. Anything which works in the expression `bias * multiplier` is acceptable here. This may be useful if you want to add a bias in one place and subtract the same bias in another place via `multiplier=-1`. Returns: A Tensor of size `[batch_size, input_size1, ...]`. """ self._initialize(inputs) if multiplier is not None: return inputs + (self.b * multiplier) else: return inputs + self.b def calculate_bias_shape(input_shape: types.ShapeLike, bias_dims: Sequence[int]): """Calculate `bias_shape` based on the `input_shape` and `bias_dims`. Args: input_shape: Shape of the input being passed into the module. The leading dimension is the mini-batch size. bias_dims: The dimensions that bias should be applied over. The remaining dimensions will be broadcast over. Returns: bias_shape: Tuple corresponding to the shape of bias Variable to create. Raises: ValueError: If the user attempts to add bias over the mini-batch dimension, e.g. `bias_dims=[0]`. """ input_rank = len(input_shape) if bias_dims is None: # If None, default is to use all dimensions. return input_shape[1:] elif not bias_dims: # If empty list, use a scalar bias. return () else: # Otherwise, calculate bias_shape from bias_dims. bias_shape = [1] * input_rank # Populate bias dimensions. for dim in bias_dims: if dim < 0: dim %= input_rank if dim == 0: raise ValueError("Cannot apply bias across the minibatch dimension.") elif dim >= input_rank: raise ValueError( "Dimension %d (bias_dims=%r) out of range for input of rank %r." % (dim, tuple(bias_dims), input_rank)) bias_shape[dim] = input_shape[dim] # Strip leading unit dimensions. start = input_rank for dim in range(1, input_rank): if bias_shape[dim] != 1: start = dim break return tuple(bias_shape[start:]) # Do not apply across minibatch dimension. ================================================ FILE: sonnet/src/bias_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.bias.""" from sonnet.src import bias from sonnet.src import test_utils import tensorflow as tf class BiasTest(test_utils.TestCase): def test_output_shape(self): mod = bias.Bias(output_size=(2 * 2,)) with self.assertRaisesRegex(ValueError, "Input shape must be [(]-1, 4[)]"): mod(tf.ones([2, 2, 2])) def test_output_size_valid(self): mod = bias.Bias(output_size=(2 * 2,)) mod(tf.ones([2, 2 * 2])) def test_bias_dims_scalar(self): mod = bias.Bias(bias_dims=()) mod(tf.ones([1, 2, 3, 4])) self.assertEmpty(mod.b.shape) def test_bias_dims_custom(self): b, d1, d2, d3 = range(1, 5) mod = bias.Bias(bias_dims=[1, 3]) out = mod(tf.ones([b, d1, d2, d3])) self.assertEqual(mod.b.shape, [d1, 1, d3]) self.assertEqual(out.shape, [b, d1, d2, d3]) def test_bias_dims_negative_out_of_order(self): mod = bias.Bias(bias_dims=[-1, -2]) mod(tf.ones([1, 2, 3])) self.assertEqual(mod.b.shape, [2, 3]) def test_bias_dims_invalid(self): mod = bias.Bias(bias_dims=[1, 5]) with self.assertRaisesRegex(ValueError, "5 .* out of range for input of rank 3"): mod(tf.ones([1, 2, 3])) def test_b_init_defaults_to_zeros(self): mod = bias.Bias() mod(tf.ones([1, 1])) self.assertAllEqual(mod.b.read_value(), tf.zeros_like(mod.b)) def test_b_init_custom(self): ones_initializer = lambda s, d: tf.ones(s, dtype=d) mod = bias.Bias(b_init=ones_initializer) mod(tf.ones([1, 1])) self.assertAllEqual(mod.b.read_value(), tf.ones_like(mod.b)) def test_name(self): mod = bias.Bias(name="foo") self.assertEqual(mod.name, "foo") mod(tf.ones([1, 1])) self.assertEqual(mod.b.name, "foo/b:0") def test_multiplier(self): ones_initializer = lambda s, d: tf.ones(s, dtype=d) mod = bias.Bias(b_init=ones_initializer) out = mod(tf.ones([1, 1]), multiplier=-1) self.assertAllEqual(tf.reduce_sum(out), 0) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/build.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Utility function to build Sonnet modules.""" from typing import Any, Callable import tensorflow as tf import tree def _int_or_none(o): return isinstance(o, (int, type(None))) def _promote_shapes(o): """Promotes lists of ints/Nones to :tf:`TensorSpec` instances.""" if isinstance(o, (list, tuple)) and all(_int_or_none(e) for e in o): return tf.TensorSpec(o) return o def _maybe_tensor_spec(shape, dtype): return tf.TensorSpec(shape, dtype) if dtype is not None else None # TODO(tomhennigan) Use TensorNest in types here. def build( f: Callable[..., Any], *args, **kwargs ): r"""Builds a module by creating all parameters but not computing any output. >>> mod = snt.nets.MLP([1000, 10]) >>> snt.build(mod, [None, 28 * 28]) TensorSpec(shape=(None, 10), dtype=tf.float32, name=None) >>> mod.variables (, , , ) Args: f: A function or callable :class:`Module` that will create variables. *args: Positional arguments to supply to ``f``. Note that positional arguments that are sequences of None/ints are converted to :tf:`TensorSpec` instances. **kwargs: Keyword arguments to pass to the module. Returns: The output of ``f`` with any :tf:`Tensor`\ s replaced by :tf:`TensorSpec`. """ f = tf.function(f) args = map(_promote_shapes, args) # NOTE: We use a concrete function to ensure that weights are created and # initialized, but other stateful ops (e.g. updating weights) are not. cf = f.get_concrete_function(*args, **kwargs) return tree.map_structure(_maybe_tensor_spec, cf.output_shapes, cf.output_dtypes) ================================================ FILE: sonnet/src/build_defs.bzl ================================================ """Sonnet specific build rules.""" def snt_py_library(name, **kwargs): """Proxy for py_library. Internally we override this to enable type checking via PyType (more information at https://github.com/google/pytype). Args: name: library name. **kwargs: keyword args passed straight to py_library. """ native.py_library(name = name, **kwargs) def snt_py_test( name, deps = [], tags = [], main = None, gpu = True, tpu = True, **kwargs): """Runs a py_test. Args: name: test target name to generate suffixed with `test`. deps: additional dependencies for the test targets. tags: tags to be assigned to the different test targets. main: main script to be run for the test. gpu: Whether the test can be run on GPU. Note ignored by test. tpu: Whether the test can be run on TPU. Note ignored by test. **kwargs: extra keyword arguments to the test. """ if main == None: main = name + ".py" native.py_test( name = name, deps = deps, tags = tags, main = main, python_version = "PY3", **kwargs ) ================================================ FILE: sonnet/src/build_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.build.""" from sonnet.src import build from sonnet.src import test_utils import tensorflow as tf class BuildTest(test_utils.TestCase): def test_call_with_shape_lke_object(self): output_spec = build.build(tensor_identity, [1, None, 3]) self.assertEqual(output_spec, tf.TensorSpec([1, None, 3])) def test_output_spec(self): dtype = tf.float32 if self.primary_device == "TPU" else tf.float16 inputs = {"foo": [tf.ones([], dtype), None]} output_spec = build.build(lambda x: x, inputs) self.assertEqual(output_spec, {"foo": [tf.TensorSpec([], dtype), None]}) def test_does_not_trigger_sideeffects(self): mod = IncrementsCounter() output_spec = build.build(mod) self.assertIsNone(output_spec) self.assertEqual(mod.counter.numpy(), 0) def tensor_identity(x): assert isinstance(x, tf.Tensor) return x class IncrementsCounter(tf.Module): def __call__(self): if not hasattr(self, "counter"): self.counter = tf.Variable(0) self.counter.assign_add(1) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/conformance/BUILD ================================================ load("//sonnet/src:build_defs.bzl", "snt_py_library", "snt_py_test") package( default_testonly = True, default_visibility = ["//sonnet:__subpackages__", "//docs/ext:__subpackages__", "//examples:__subpackages__"], ) licenses(["notice"]) snt_py_library( name = "goldens", srcs = ["goldens.py"], deps = [ # pip: absl/testing:parameterized # pip: numpy "//sonnet", # pip: tensorflow ], ) snt_py_test( name = "api_test", srcs = ["api_test.py"], deps = [ "//sonnet", "//sonnet/src:test_utils", # pip: tensorflow ], ) snt_py_test( name = "checkpoint_test", timeout = "long", srcs = ["checkpoint_test.py"], data = [":checkpoints"], shard_count = 10, tags = [ "notap", # TODO(b/203294224): investigate flake on GPU. ], deps = [ ":goldens", # pip: absl/logging # pip: absl/testing:absltest # pip: absl/testing:parameterized "//sonnet/src:test_utils", "//sonnet/src/distribute:replicator", "//sonnet/src/distribute:replicator_test_utils", # pip: tensorflow # pip: tree ], ) snt_py_test( name = "distribute_test", srcs = ["distribute_test.py"], shard_count = 10, deps = [ ":descriptors", ":goldens", # pip: absl/testing:parameterized "//sonnet", "//sonnet/src:test_utils", "//sonnet/src/distribute:replicator", "//sonnet/src/distribute:replicator_test_utils", # pip: tensorflow ], ) snt_py_test( name = "doctest_test", srcs = ["doctest_test.py"], deps = [ # pip: absl/testing:parameterized "//sonnet", "//sonnet/src:test_utils", # pip: tensorflow # pip: tree ], ) snt_py_library( name = "descriptors", srcs = ["descriptors.py"], deps = [ "//sonnet", # pip: tensorflow ], ) snt_py_test( name = "descriptors_test", srcs = ["descriptors_test.py"], deps = [ ":descriptors", "//sonnet", "//sonnet/src:test_utils", # pip: tensorflow ], ) snt_py_test( name = "function_test", timeout = "long", srcs = ["function_test.py"], shard_count = 10, deps = [ ":descriptors", # pip: absl/testing:parameterized "//sonnet", "//sonnet/src:test_utils", # pip: tensorflow ], ) snt_py_test( name = "goldens_test", timeout = "long", srcs = ["goldens_test.py"], shard_count = 10, deps = [ ":goldens", "//sonnet", "//sonnet/src:test_utils", # pip: tensorflow ], ) snt_py_test( name = "pickle_test", timeout = "long", srcs = ["pickle_test.py"], shard_count = 10, deps = [ ":goldens", # pip: absl/testing:parameterized "//sonnet/src:test_utils", # pip: tensorflow # pip: tree ], ) snt_py_test( name = "saved_model_test", timeout = "long", srcs = ["saved_model_test.py"], shard_count = 10, deps = [ ":goldens", # pip: absl/testing:absltest # pip: absl/testing:parameterized "//sonnet", "//sonnet/src:test_utils", # pip: tensorflow # pip: tree ], ) snt_py_test( name = "xla_test", timeout = "long", srcs = ["xla_test.py"], shard_count = 10, deps = [ ":goldens", # pip: absl/testing:parameterized "//sonnet/src:test_utils", # pip: tensorflow # pip: tree # tf: compiler/jit:xla_cpu_jit # tf: compiler/jit:xla_gpu_jit ], ) snt_py_test( name = "keras_test", timeout = "long", srcs = ["keras_test.py"], shard_count = 10, deps = [ ":descriptors", # pip: absl/testing:parameterized "//sonnet", "//sonnet/src:test_utils", # pip: tensorflow # pip: tree ], ) snt_py_test( name = "build_test", timeout = "long", srcs = ["build_test.py"], shard_count = 10, deps = [ ":descriptors", # pip: absl/testing:parameterized "//sonnet", "//sonnet/src:test_utils", # pip: tensorflow # pip: tree ], ) snt_py_test( name = "copy_test", timeout = "long", srcs = ["copy_test.py"], shard_count = 10, deps = [ ":goldens", # pip: absl/testing:parameterized "//sonnet/src:test_utils", # pip: tensorflow # pip: tree ], ) snt_py_test( name = "tensorflow1_test", timeout = "long", srcs = ["tensorflow1_test.py"], deps = [ "//sonnet", "//sonnet/src:test_utils", # pip: tensorflow ], ) snt_py_test( name = "optimizer_test", timeout = "long", srcs = ["optimizer_test.py"], deps = [ ":descriptors", # pip: absl/testing:parameterized "//sonnet/src:test_utils", # pip: tensorflow ], ) ================================================ FILE: sonnet/src/conformance/__init__.py ================================================ # Copyright 2021 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ ================================================ FILE: sonnet/src/conformance/api_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for Sonnet's public API.""" import importlib import sonnet as snt from sonnet.src import test_utils import tensorflow as tf class PublicSymbolsTest(test_utils.TestCase): def test_src_not_exported(self): self.assertFalse(hasattr(snt, "src")) def test_supports_reload(self): mysnt = snt for _ in range(2): mysnt = importlib.reload(mysnt) self.assertFalse(hasattr(mysnt, "src")) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/conformance/build_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests modules support `snt.build`.""" from absl.testing import parameterized import sonnet as snt from sonnet.src import test_utils from sonnet.src.conformance import descriptors import tensorflow as tf import tree BATCH_MODULES = descriptors.BATCH_MODULES RECURRENT_MODULES = descriptors.RECURRENT_MODULES def if_present(f): return lambda o: f(o) if o is not None else None class BuildTest(test_utils.TestCase, parameterized.TestCase): @parameterized.named_parameters(*(BATCH_MODULES + RECURRENT_MODULES)) def test_build(self, module_fn, input_shape, dtype): module = module_fn() build_output_spec = snt.build(module, tf.TensorSpec(input_shape, dtype)) actual_output = module(tf.ones(input_shape, dtype)) actual_output_spec = tree.map_structure( if_present(lambda t: tf.TensorSpec(t.shape, t.dtype)), actual_output) tree.map_structure(self.assertCompatible, build_output_spec, actual_output_spec) def assertCompatible(self, a: tf.TensorSpec, b: tf.TensorSpec): self.assertTrue(a.shape.is_compatible_with(b.shape)) self.assertEqual(a.dtype, b.dtype) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/conformance/checkpoint_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests checkpointing with Sonnet.""" import os from absl import logging from absl.testing import absltest from absl.testing import parameterized from sonnet.src import test_utils from sonnet.src.conformance import goldens from sonnet.src.distribute import replicator as snt_replicator from sonnet.src.distribute import replicator_test_utils as replicator_utils import tensorflow as tf import tree class TestCheckpoint: """Wraps a tf.train.Checkpoint to make it more convenient for testing.""" def __init__(self, golden=None, **kwargs): if golden is None: root = absltest.get_default_test_tmpdir() else: root = os.path.join( "sonnet/src/conformance/checkpoints/", golden.name) self._root = root self._prefix = os.path.join(self._root, "checkpoint") self._checkpoint = tf.train.Checkpoint(**kwargs) def save(self): self._checkpoint.save(file_prefix=self._prefix) def restore_latest(self, assert_consumed): status = self._checkpoint.restore(tf.train.latest_checkpoint(self._root)) if assert_consumed: # Ensures that all values in the checkpoint have been consumed by some # checkpointable Python object. status.assert_consumed() return status def with_soft_placement(f): """Wraps `f` such that it runs with soft device placement.""" def wrapper(*a, **k): with tf.device(None): return f(*a, **k) return wrapper class GoldenCheckpointsTest(test_utils.TestCase, parameterized.TestCase): """Adds test methods running standard checkpointing tests.""" @goldens.all_goldens def test_save_load(self, golden): """Test a basic save/load cycle.""" module = golden.create_module() checkpoint = TestCheckpoint(module=module) all_variables = golden.create_all_variables(module) # Save zeros into the checkpoint. self.assertNotEmpty(all_variables) self.assertEqual(all_variables, module.variables) for variable in all_variables: # TODO(tomhennigan) Perhaps limit the range/switch to random to avoid # overflow/underflow in the forward pass? variable.assign(goldens.range_like(variable)) checkpoint.save() old_y = golden.forward(module) # Overwrite zeros with ones. for variable in all_variables: variable.assign(tf.ones_like(variable)) # Check restored values match the saved values. checkpoint.restore_latest(assert_consumed=True) for variable in all_variables: self.assertAllClose( variable.read_value(), goldens.range_like(variable), msg=variable.name) # Test the output from the module remains stable. if golden.deterministic: tree.map_structure(self.assertAllClose, golden.forward(module), old_y) @goldens.all_goldens def test_save_then_load_new_instance(self, golden): """Checks that a checkpoint created for one instance can restore another.""" module_1 = golden.create_module() checkpoint_1 = TestCheckpoint(module=module_1) variables_1 = golden.create_all_variables(module_1) module_2 = golden.create_module() checkpoint_2 = TestCheckpoint(module=module_2) variables_2 = golden.create_all_variables(module_2) for v1, v2 in zip(variables_1, variables_2): v1.assign(goldens.range_like(v1)) v2.assign(tf.ones_like(v2)) checkpoint_1.save() checkpoint_2.restore_latest(assert_consumed=True) # Assert the parameters in both modules are the same. for variable in variables_2: self.assertAllClose( variable.read_value(), goldens.range_like(variable), msg=variable.name) # Assert the output from both modules are the same. if golden.deterministic: tree.map_structure(self.assertAllClose, golden.forward(module_1), golden.forward(module_2)) @goldens.all_goldens def test_restore_on_create(self, golden): """Tests that Variable values are restored on creation.""" # Create a module, set its variables to sequential values and save. module_1 = golden.create_module() checkpoint_1 = TestCheckpoint(module=module_1) variables_1 = golden.create_all_variables(module_1) for variable in variables_1: variable.assign(goldens.range_like(variable)) checkpoint_1.save() golden.forward(module_1) # Create a different module, restore from a checkpoint, create parameters # and assert their values are sequential. module_2 = golden.create_module() checkpoint_2 = TestCheckpoint(module=module_2) status = checkpoint_2.restore_latest(assert_consumed=False) variables_2 = golden.create_all_variables(module_2) status.assert_consumed() for var1, var2 in zip(variables_1, variables_2): self.assertAllEqual(var1.read_value(), var2.read_value(), msg=var1.name) # Assert the output from both modules is the same. if golden.deterministic: tree.map_structure(self.assertAllClose, golden.forward(module_1), golden.forward(module_2)) @goldens.all_goldens def test_restore_golden(self, golden): """Test restoring from a golden checkpoint still works.""" module = golden.create_module() checkpoint = TestCheckpoint(golden=golden, module=module) variables = golden.create_all_variables(module) for variable in variables: variable.assign(tf.zeros_like(variable)) checkpoint.restore_latest(assert_consumed=True) for variable in variables: self.assertAllEqual( variable.read_value(), goldens.range_like(variable), msg=variable.name) class ReplicatorCheckpointTest(test_utils.TestCase, parameterized.TestCase): def replicator_or_skip(self, replicator_fn, use_function): replicator = replicator_fn() if not use_function and isinstance(replicator, snt_replicator.TpuReplicator): self.skipTest("TpuReplicator does not support eager mode.") return replicator @test_utils.combined_named_parameters(goldens.named_goldens(), replicator_utils.named_replicators(), test_utils.named_bools("use_function")) def test_save_restore(self, golden, replicator_fn, use_function): replicator = self.replicator_or_skip(replicator_fn, use_function) with replicator.scope(): module = golden.create_module() variables = golden.create_all_variables(module) def forward(): per_replica = replicator.run( lambda: golden.forward(module)) return tree.map_structure( lambda args: tf.stack(replicator.unwrap(args), axis=0), per_replica) if use_function: forward = tf.function(forward) if self.primary_device == "TPU": # TODO(b/132329316) Remove when `xla.compile` allows tf.device(TPU). forward = with_soft_placement(forward) # Assign sequential values to the weights. for index, variable in enumerate(variables): variable.assign(goldens.range_like(variable, start=index)) # Create a checkpoint and save the weights. checkpoint = TestCheckpoint(module=module) checkpoint.save() # Compute a forward pass of the previously saved module. before_save_ys = forward() # Assign different values into the weights and do another forward pass. The # result should be different. for variable in variables: variable.assign(-tf.ones_like(variable)) if golden.deterministic: y = forward() self.assertNotAllClose(y, before_save_ys) # Restore from the checkpoint and assert the module is in the same state. checkpoint.restore_latest(assert_consumed=True) for index, variable in enumerate(variables): # Parameters should be restored to their previous values. self.assertAllEqual( variable.read_value(), goldens.range_like(variable, start=index), msg=variable.name) if golden.deterministic: tree.map_structure(self.assertAllEqual, forward(), before_save_ys) @test_utils.combined_named_parameters(goldens.named_goldens(), replicator_utils.named_replicators()) def test_restore_from_golden(self, golden, replicator_fn): replicator = self.replicator_or_skip(replicator_fn, use_function=False) with replicator.scope(): module = golden.create_module() variables = golden.create_all_variables(module) checkpoint = TestCheckpoint(golden=golden, module=module) checkpoint.restore_latest(assert_consumed=True) for variable in variables: self.assertAllEqual( variable.read_value(), goldens.range_like(variable), msg=variable.name) @test_utils.combined_named_parameters(goldens.named_goldens(), replicator_utils.named_replicators(), test_utils.named_bools("use_function")) def test_restore_from_non_distributed(self, golden, replicator_fn, use_function): replicator = self.replicator_or_skip(replicator_fn, use_function) # Save a checkpoint from a non-distributed model. module = golden.create_module() normal_variables = golden.create_all_variables(module) for index, variable in enumerate(normal_variables): variable.assign(goldens.range_like(variable, start=(index + 1))) checkpoint = TestCheckpoint(module=module) checkpoint.save() # Create the same model (new params) in the replicator scope. with replicator.scope(): module2 = golden.create_module() replicator_variables = golden.create_all_variables(module2) # Ensure the distributed params are != the values in the checkpoint. for normal, distributed in zip(normal_variables, replicator_variables): distributed.assign(tf.zeros_like(distributed)) self.assertNotAllClose(normal.read_value(), distributed.read_value()) # Restore the checkpoint and ensure the parameters are the same. checkpoint = TestCheckpoint(module=module2) checkpoint.restore_latest(assert_consumed=True) for normal, distributed in zip(normal_variables, replicator_variables): self.assertAllEqual( normal.read_value(), distributed.read_value(), msg=normal.name) if golden.deterministic: def run_forward(module): forward = lambda: golden.forward(module) if use_function: forward = tf.function(forward) if self.primary_device == "TPU": # TODO(b/132329316) Remove when `xla.compile` allows tf.device(TPU). forward = with_soft_placement(forward) return forward() y_before = run_forward(module) y_after = run_forward(module2) tree.map_structure(self.assertAllEqual, y_before, y_after) @test_utils.combined_named_parameters(goldens.named_goldens(), replicator_utils.named_replicators()) def test_restore_on_create(self, golden, replicator_fn): replicator = self.replicator_or_skip(replicator_fn, use_function=False) # Save a checkpoint from a non-distributed model. module = golden.create_module() normal_variables = golden.create_all_variables(module) for index, variable in enumerate(normal_variables): variable.assign(goldens.range_like(variable, start=(index + 1))) checkpoint = TestCheckpoint(module=module) checkpoint.save() golden.forward(module) # Create the same model (new params) in the replicator scope. with replicator.scope(): module = golden.create_module() checkpoint = TestCheckpoint(module=module) status = checkpoint.restore_latest(assert_consumed=False) golden.forward(module) status.assert_consumed() replicator_variables = module.variables for normal, distributed in zip(normal_variables, replicator_variables): self.assertAllEqual( normal.read_value(), distributed.read_value(), msg=normal.name) @test_utils.combined_named_parameters(goldens.named_goldens(), replicator_utils.named_replicators(), test_utils.named_bools("use_function")) def test_restore_on_create_in_replica_context(self, golden, replicator_fn, use_function): replicator = self.replicator_or_skip(replicator_fn, use_function) # Save a checkpoint from a non-distributed model. module = golden.create_module() normal_variables = golden.create_all_variables(module) for index, variable in enumerate(normal_variables): variable.assign(goldens.range_like(variable, start=(index + 1))) checkpoint = TestCheckpoint(module=module) checkpoint.save() golden.forward(module) with replicator.scope(): module = golden.create_module() def forward(): return replicator.run(lambda: golden.forward(module)) if use_function: forward = tf.function(forward) if self.primary_device == "TPU": # TODO(b/132329316) Remove when `xla.compile` allows tf.device(TPU). forward = with_soft_placement(forward) checkpoint = TestCheckpoint(module=module) status = checkpoint.restore_latest(assert_consumed=False) result = forward() status.assert_consumed() if golden.deterministic: result_iter = iter(replicator.experimental_local_results(result)) first_replica = next(result_iter) for next_replica in result_iter: tree.map_structure(self.assertAllEqual, first_replica, next_replica) if not golden.has_side_effects: replicator_variables = module.variables for normal, distributed in zip(normal_variables, replicator_variables): self.assertAllClose( normal.read_value(), distributed.read_value(), msg=normal.name) def setUpModule(): # If a physical GPU is available make sure TF sees at least two. gpus = tf.config.experimental.list_physical_devices(device_type="GPU") if len(gpus) == 1: logging.info("Splitting one physical GPU into two logical GPUs.") tf.config.experimental.set_virtual_device_configuration( gpus[0], [ tf.config.experimental.VirtualDeviceConfiguration( memory_limit=1024), tf.config.experimental.VirtualDeviceConfiguration(memory_limit=1024) ]) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/conformance/checkpoints/BUILD ================================================ load("//third_party/bazel_rules/rules_python/python:py_binary.bzl", "py_binary") package(default_testonly = True) licenses(["notice"]) py_binary( name = "generate", srcs = ["generate.py"], strict_deps = False, deps = [ # pip: absl:app # pip: absl/flags # pip: absl/logging "//sonnet/src/conformance:goldens", # pip: tensorflow ], ) filegroup( name = "checkpoints", srcs = glob(["**/*"]), ) ================================================ FILE: sonnet/src/conformance/checkpoints/README.md ================================================ # Golden checkpoints Golden checkpoints represent checkpoints generated from stable Sonnet code. We have unit tests that ensure we don't introduce checkpoint breaking changes to Sonnet. To generate a new checkpoint first add an entry in `goldens.py` describing the module you want to add. For example: ```python @_register_golden(snt.Linear, "linear_32x64") class Linear32x64(Golden): """Tests Linear without a bias.""" def create_module(self): return snt.Linear(64) def forward(self, module): x = range_like(tf.TensorSpec([1, 32])) return module(x) def create_all_variables(self, module): self.forward(module) return module.w, module.b ``` Then run the `generate` binary to generate new golden checkpoints: ```shell $ bazel run :generate -- --dry_run=false --golden_dir="$PWD" --alsologtostderr ``` At this point your golden checkpoint will be created and registered to run whenever `goldens_test` runs: ```shell $ bazel test :goldens_test ``` ## Regenerating old checkpoints WARNING: In general once a checkpoint is checked in it is only safe to regenerate it if your module has zero users. If you are making an additive change to a module (e.g. adding a new parameter) then consider making a new checkpoint and ensure that you can load from both the old and new checkpoint. If you absolutely need to regenerate the checkpoint and know what you're doing then you can do so with: ```shell $ bazel run :generate -- --dry_run=false --golden_dir="$PWD" --alsologtostderr --filter=my_checkpoint_name --regenerate ``` ================================================ FILE: sonnet/src/conformance/checkpoints/base_batch_norm_1x2x2x3/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/base_batch_norm_scale_offset_1x2x2x3/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/batch_norm_1x2x2x3/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/batch_norm_scale_offset_1x2x2x3/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/batch_norm_training_1x2x2x3/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/bias_3x3x3/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/cifar10_convnet_2x3_2x2_1x3x3x2/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/conv1d_3x3_2x2/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/conv1d_lstm_3x3_2x2/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/conv1d_transpose_3x3_2x2/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/conv2d_3x3_2x2/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/conv2d_lstm_3x3_2x2/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/conv2d_transpose_3x3_2x2/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/conv3d_3x3_2x2/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/conv3d_lstm_3x3_2x2/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/conv3d_transpose_3x3_2x2/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/cross_replica_batch_norm_1x2x2x3/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/depthwise_conv2d_3x3_2x2/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/dropout/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/ema_2/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/embed_100_100/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/generate.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Binary to generate golden checkpoint tests.""" import os import re from absl import app from absl import flags from absl import logging from sonnet.src.conformance import goldens import tensorflow as tf FLAGS = flags.FLAGS flags.DEFINE_string("golden_dir", "sonnet/src/conformance/checkpoints/", "Directory where golden files are to be found.") flags.DEFINE_string("filter", ".*", "Filter to a specific golden by name.") flags.DEFINE_bool("regenerate", False, "Whether to regnerate existing checkpoints.") flags.DEFINE_bool("dry_run", True, "Whether to actually apply changes.") def safe_mkdir(directory): if FLAGS.dry_run: logging.warning("[DRY RUN] Would create %r", directory) else: logging.info("Creating %r", directory) os.mkdir(directory) def safe_unlink(path): if FLAGS.dry_run: logging.warning("[DRY RUN] Would delete %r", path) else: logging.info("Deleting %r", path) os.unlink(path) def main(unused_argv): del unused_argv for _, name, cls in goldens.list_goldens(): if not re.match(FLAGS.filter, name): continue checkpoint_dir = os.path.join(FLAGS.golden_dir, name) exists = os.path.exists(checkpoint_dir) if exists and not FLAGS.regenerate: logging.info("Skipping %s since it exists and --regenerate=false", name) continue logging.info("Processing %s", name) if not exists: safe_mkdir(checkpoint_dir) else: # Clear out old files. for file_name in os.listdir(checkpoint_dir): safe_unlink(os.path.join(checkpoint_dir, file_name)) # Create the module to checkpoint. golden = cls() module = golden.create_module() golden.create_all_variables(module) for var in module.variables: var.assign(goldens.range_like(var)) # Create a checkpoint and save the values to it. checkpoint = tf.train.Checkpoint(module=module) if FLAGS.dry_run: logging.warning("[DRY RUN] Would save %r to %r", module, checkpoint_dir) else: file_prefix = os.path.join(checkpoint_dir, "checkpoint") logging.info("Saving to checkpoint %s.", file_prefix) checkpoint.save(file_prefix=file_prefix) if __name__ == "__main__": app.run(main) ================================================ FILE: sonnet/src/conformance/checkpoints/group_norm_2_1x3x4/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/gru_1/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/instance_norm_1_1x3_2/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/layer_norm_1_1x3_2/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/linear_1x1/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/linear_nobias_1x1/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/lstm_1/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/lstm_8_projected_1/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/mean_2x2/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/mlp_3x4x5_1x3/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/mlp_nobias_3x4x5_1x3/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/resnet50/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/resnet50/checkpoint-1.data-00000-of-00001 ================================================ [File too large to display: 30.8 MB] ================================================ FILE: sonnet/src/conformance/checkpoints/sum_2x2/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/trainable_state/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/unrolled_lstm_1/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/vanilla_rnn_8/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/vqvae/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/vqvae_ema_eval/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/checkpoints/vqvae_ema_train/checkpoint ================================================ model_checkpoint_path: "checkpoint-1" all_model_checkpoint_paths: "checkpoint-1" ================================================ FILE: sonnet/src/conformance/copy_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests copying Sonnet modules.""" import copy from absl.testing import parameterized from sonnet.src import test_utils from sonnet.src.conformance import goldens import tensorflow as tf import tree class CopyTest(test_utils.TestCase, parameterized.TestCase): @goldens.all_goldens def test_copy(self, golden): m1 = golden.create_module() golden.create_all_variables(m1) m2 = copy.deepcopy(m1) self.assertIsNot(m1, m2) # Check that module variables are recreated with equivalent properties. for v1, v2 in zip(m1.variables, m2.variables): self.assertIsNot(v1, v2) self.assertEqual(v1.name, v2.name) self.assertEqual(v1.device, v2.device) self.assertAllEqual(v1.read_value(), v2.read_value()) if golden.deterministic: y1 = golden.forward(m1) y2 = golden.forward(m2) tree.map_structure(self.assertAllEqual, y1, y2) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/conformance/descriptors.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Module descriptors programatically describe how to use modules.""" import collections from typing import Callable, Union import sonnet as snt import tensorflow as tf class Wrapped(snt.Module): @snt.no_name_scope def __init__(self, wrapped: snt.Module): super().__init__() self.wrapped = wrapped class Training(Wrapped): @snt.no_name_scope def __call__(self, x: tf.Tensor): return self.wrapped(x, is_training=True) class Recurrent(Wrapped): """Unrolls a recurrent module.""" def __init__(self, module: Union[snt.RNNCore, snt.UnrolledRNN], unroller=None): super().__init__(module) self.unroller = unroller @snt.no_name_scope def __call__(self, x: tf.Tensor): initial_state = self.wrapped.initial_state(batch_size=tf.shape(x)[0]) if isinstance(self.wrapped, snt.UnrolledRNN): assert self.unroller is None # The module expects TB...-shaped input as opposed to BT... x = tf.transpose(x, [1, 0] + list(range(2, x.shape.rank))) return self.wrapped(x, initial_state) else: x = tf.expand_dims(x, axis=0) return self.unroller(self.wrapped, x, initial_state) def unwrap(module: snt.Module) -> snt.Module: while isinstance(module, Wrapped): module = module.wrapped return module # TODO(tomhennigan) De-duplicate this, BATCH_MODULES and goldens.py. ModuleDescriptor = collections.namedtuple("ModuleDescriptor", ["name", "create", "shape", "dtype"]) ModuleDescriptor.__new__.__defaults__ = (None, None, None, tf.float32) BATCH_SIZE = 8 # pylint: disable=unnecessary-lambda BATCH_MODULES = ( ModuleDescriptor( name="BatchNorm", create=lambda: Training(snt.BatchNorm(True, True)), shape=(BATCH_SIZE, 2, 2, 3)), ModuleDescriptor( name="Bias", create=lambda: snt.Bias(), shape=(BATCH_SIZE, 3, 3, 3)), ModuleDescriptor( name="Conv1D", create=lambda: snt.Conv1D(3, 3), shape=(BATCH_SIZE, 2, 2)), ModuleDescriptor( name="Conv1DTranspose", create=lambda: snt.Conv1DTranspose(3, 3), shape=(BATCH_SIZE, 2, 2)), ModuleDescriptor( name="Conv2D", create=lambda: snt.Conv2D(3, 3), shape=(BATCH_SIZE, 2, 2, 2)), ModuleDescriptor( name="Conv2DTranspose", create=lambda: snt.Conv2DTranspose(3, 3), shape=(BATCH_SIZE, 2, 2, 2)), ModuleDescriptor( name="Conv3D", create=lambda: snt.Conv3D(3, 3), shape=(BATCH_SIZE, 2, 2, 2, 2)), ModuleDescriptor( name="Conv3DTranspose", create=lambda: snt.Conv3DTranspose(3, 3), shape=(BATCH_SIZE, 2, 2, 2, 2)), ModuleDescriptor( name="CrossReplicaBatchNorm", create=lambda: Training(snt.distribute.CrossReplicaBatchNorm( # pylint: disable=g-long-lambda True, True, snt.ExponentialMovingAverage(0.9), snt.ExponentialMovingAverage(0.9))), shape=(BATCH_SIZE, 2, 2, 3)), ModuleDescriptor( name="DepthwiseConv2D", create=lambda: snt.DepthwiseConv2D(3), shape=(BATCH_SIZE, 2, 2, 2)), ModuleDescriptor( name="Dropout", create=lambda: Training(snt.Dropout(0.5)), shape=(BATCH_SIZE, 3, 3)), ModuleDescriptor( name="Embed", create=lambda: snt.Embed(10), shape=(BATCH_SIZE,), dtype=tf.int32), ModuleDescriptor( name="Flatten", create=lambda: snt.Flatten(), shape=(BATCH_SIZE, 3, 3, 3)), ModuleDescriptor( name="GroupNorm", create=lambda: snt.GroupNorm(2, True, True), shape=(BATCH_SIZE, 3, 4)), ModuleDescriptor( name="InstanceNorm", create=lambda: snt.InstanceNorm(True, True), shape=(BATCH_SIZE, 3, 2)), ModuleDescriptor( name="LayerNorm", create=lambda: snt.LayerNorm(1, True, True), shape=(BATCH_SIZE, 3, 2)), ModuleDescriptor( name="Linear", create=lambda: snt.Linear(10), shape=(BATCH_SIZE, 1)), ModuleDescriptor( name="Sequential", create=lambda: snt.Sequential([lambda x: x]), shape=(BATCH_SIZE, 2, 2)), ModuleDescriptor( name="nets.VectorQuantizer", create=lambda: Training(snt.nets.VectorQuantizer(4, 6, 0.25)), shape=(BATCH_SIZE, 3, 4)), ModuleDescriptor( name="nets.VectorQuantizerEMA", create=lambda: Training(snt.nets.VectorQuantizerEMA(5, 7, 0.5, 0.9)), shape=(BATCH_SIZE, 5)), ModuleDescriptor( name="nets.Cifar10ConvNet", create=lambda: Training(snt.nets.Cifar10ConvNet()), shape=(BATCH_SIZE, 3, 3, 2)), ModuleDescriptor( name="nets.ResNet50", create=lambda: Training(snt.nets.ResNet([1, 1, 1, 1], 4)), shape=(BATCH_SIZE, 3, 3, 2)), ModuleDescriptor( name="nets.MLP", create=lambda: snt.nets.MLP([3, 4, 5]), shape=(BATCH_SIZE, 3)), ) RNN_CORES = ( ModuleDescriptor( name="Conv1DLSTM", create=lambda: snt.Conv1DLSTM((2, 2), 3, 3), shape=(BATCH_SIZE, 2, 2)), ModuleDescriptor( name="Conv2DLSTM", create=lambda: snt.Conv2DLSTM((2, 2, 2), 3, 3), shape=(BATCH_SIZE, 2, 2, 2)), ModuleDescriptor( name="Conv3DLSTM", create=lambda: snt.Conv3DLSTM((2, 2, 2, 2), 3, 3), shape=(BATCH_SIZE, 2, 2, 2, 2)), ModuleDescriptor( name="GRU", create=lambda: snt.GRU(1), shape=(BATCH_SIZE, 128)), ModuleDescriptor( name="LSTM", create=lambda: snt.LSTM(1), shape=(BATCH_SIZE, 128)), ModuleDescriptor( name="VanillaRNN", create=lambda: snt.VanillaRNN(8), shape=(BATCH_SIZE, 128)), ) UNROLLED_RNN_CORES = ( ModuleDescriptor( name="UnrolledLSTM", create=lambda: snt.UnrolledLSTM(1), shape=(BATCH_SIZE, 1, 128)), ) def recurrent_factory( create_core: Callable[[], snt.RNNCore], unroller, ) -> Callable[[], Recurrent]: return lambda: Recurrent(create_core(), unroller) def unroll_descriptors(descriptors, unroller=None): """Returns `Recurrent` wrapped descriptors with the given unroller applied.""" out = [] for name, create, shape, dtype in descriptors: if unroller is None: name = "Recurrent({})".format(name) else: name = "Recurrent({}, {})".format(name, unroller.__name__) out.append( ModuleDescriptor(name=name, create=recurrent_factory(create, unroller), shape=shape, dtype=dtype)) return tuple(out) RECURRENT_MODULES = ( unroll_descriptors(RNN_CORES, snt.dynamic_unroll) + unroll_descriptors(RNN_CORES, snt.static_unroll) + unroll_descriptors(UNROLLED_RNN_CORES)) OPTIMIZER_MODULES = ( ModuleDescriptor( name="optimizers.Adam", create=lambda: snt.optimizers.Adam(learning_rate=0.1)), ModuleDescriptor( name="optimizers.Momentum", create=lambda: snt.optimizers.Momentum(learning_rate=0.1, momentum=.9)), ModuleDescriptor( name="optimizers.RMSProp", create=lambda: snt.optimizers.RMSProp(learning_rate=0.1)), ModuleDescriptor( name="optimizers.SGD", create=lambda: snt.optimizers.SGD(learning_rate=0.1)), ) IGNORED_MODULES = { # Stateless or abstract. snt.BatchApply, snt.Deferred, snt.Module, snt.Optimizer, snt.Reshape, # Metrics. snt.ExponentialMovingAverage, snt.Mean, snt.Metric, snt.Sum, # Normalization. snt.BaseBatchNorm, # Tested via `snt.BatchNorm`. # Recurrent. snt.DeepRNN, snt.RNNCore, snt.TrainableState, snt.UnrolledRNN, # Tested via `snt.nets.ResNet`. snt.nets.ResNet50, snt.nets.resnet.BottleNeckBlockV1, snt.nets.resnet.BottleNeckBlockV2, snt.nets.resnet.BlockGroup, } ================================================ FILE: sonnet/src/conformance/descriptors_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.conformance.descriptors.""" import sonnet as snt from sonnet.src import test_utils from sonnet.src.conformance import descriptors import tensorflow as tf BATCH_MODULES = descriptors.BATCH_MODULES RECURRENT_MODULES = descriptors.RECURRENT_MODULES OPTIMIZER_MODULES = descriptors.OPTIMIZER_MODULES IGNORED_MODULES = descriptors.IGNORED_MODULES class DescriptorsTest(test_utils.TestCase): def test_coverage(self): all_modules = frozenset(test_utils.find_all_sonnet_modules(snt, snt.Module)) tested_modules = { type(descriptors.unwrap(d.create())) for d in BATCH_MODULES + RECURRENT_MODULES + OPTIMIZER_MODULES } self.assertEmpty(all_modules - (tested_modules | IGNORED_MODULES)) if __name__ == '__main__': tf.test.main() ================================================ FILE: sonnet/src/conformance/distribute_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests Sonnet and TF Distribution Strategy.""" from typing import Callable, Tuple from absl.testing import parameterized import sonnet as snt from sonnet.src import test_utils from sonnet.src.conformance import descriptors from sonnet.src.conformance import goldens from sonnet.src.distribute import replicator as snt_replicator from sonnet.src.distribute import replicator_test_utils as replicator_utils import tensorflow as tf class TpuReplicatorTest(test_utils.TestCase, parameterized.TestCase): @test_utils.combined_named_parameters(goldens.named_goldens(), replicator_utils.named_replicators()) def test_variable_creation_in_replica_context(self, golden, replicator_fn): tf.random.set_seed(None) replicator = replicator_fn() with replicator.scope(): mod = golden.create_module() @tf.function def forward(): step = lambda: golden.create_all_variables(mod) return replicator.run(step) # TODO(b/132329316) Remove when `xla.compile` allows tf.device(TPU). with tf.device(None): variables_per_replica = forward() self.assertLen(variables_per_replica, golden.num_variables) for per_replica_variable in variables_per_replica: self.assertSameValuePerReplica(replicator, per_replica_variable) def assertSameValuePerReplica(self, replicator, per_replica): per_replica = replicator.experimental_local_results(per_replica) first_replica = per_replica[0] for nth_replica in per_replica[1:]: self.assertAllEqual(first_replica, nth_replica) @test_utils.combined_named_parameters(descriptors.RNN_CORES, test_utils.named_bools("dynamic"), replicator_utils.named_replicators()) def test_unroll( self, core_fn: Callable[[], snt.RNNCore], input_shape: Tuple[int], dtype: tf.DType, dynamic: bool, replicator_fn: tf.distribute.Strategy, ): replicator = replicator_fn() with replicator.scope(): core = core_fn() def step_fn(): def forward(): unroll = snt.dynamic_unroll if dynamic else snt.static_unroll sequence = tf.ones((1,) + input_shape, dtype) state = core.initial_state(input_shape[0]) return unroll(core, sequence, state) return replicator.run(forward) # TpuReplicator doesn't support pure eager mode. if isinstance(replicator, snt_replicator.TpuReplicator): step_fn = tf.function(step_fn) # TODO(b/132329316) Remove when `xla.compile` allows tf.device(TPU). with tf.device(None): out_sequence, final_state = step_fn() self.assertSameValuePerReplica(replicator, out_sequence) self.assertSameValuePerReplica(replicator, final_state) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/conformance/doctest_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Ensures that code samples in Sonnet are accurate.""" import doctest import inspect from absl.testing import parameterized import sonnet as snt from sonnet.src import test_utils import tensorflow as tf import tree class DoctestTest(test_utils.TestCase, parameterized.TestCase): # Avoid running doctests inside a `with tf.device` block. ENTER_PRIMARY_DEVICE = False def setUp(self): super().setUp() if self.primary_device != "TPU": # `TpuReplicator` cannot be constructed without a TPU, however it has # exactly the same API as `Replicator` so we can run doctests using that # instead. snt.distribute.TpuReplicator = snt.distribute.Replicator @parameterized.named_parameters(test_utils.find_sonnet_python_modules(snt)) def test_doctest(self, module): # `snt` et al import all dependencies from `src`, however doctest does not # test imported deps so we must manually set `__test__` such that imported # symbols are tested. # See: docs.python.org/3/library/doctest.html#which-docstrings-are-examined if not hasattr(module, "__test__") or not module.__test__: module.__test__ = {} for name in module.__all__: value = getattr(module, name) if not inspect.ismodule(value): if (inspect.isclass(value) or isinstance(value, str) or inspect.isfunction(value) or inspect.ismethod(value)): module.__test__[name] = value elif hasattr(value, "__doc__"): module.__test__[name] = value.__doc__ num_failed, num_attempted = doctest.testmod( module, optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE, extraglobs={ "snt": snt, "tf": tf, "tree": tree, }) if num_attempted == 0: self.skipTest("No doctests in %s" % module.__name__) self.assertEqual(num_failed, 0, "{} doctests failed".format(num_failed)) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/conformance/function_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Ensures that all Sonnet modules support ``tf.function``.""" from typing import Callable, Tuple from absl.testing import parameterized import sonnet as snt from sonnet.src import test_utils from sonnet.src.conformance import descriptors import tensorflow as tf ModuleFn = Callable[[], snt.Module] BATCH_MODULES = descriptors.BATCH_MODULES RECURRENT_MODULES = descriptors.RECURRENT_MODULES OPTIMIZER_MODULES = descriptors.OPTIMIZER_MODULES IGNORED_MODULES = descriptors.IGNORED_MODULES class FunctionTest(test_utils.TestCase, parameterized.TestCase): @test_utils.combined_named_parameters(BATCH_MODULES + RECURRENT_MODULES, test_utils.named_bools("autograph")) def test_trace( self, module_fn: ModuleFn, input_shape: Tuple[int], dtype: tf.DType, autograph: bool, ): module = module_fn() forward = tf.function(module, autograph=autograph) forward(tf.ones(input_shape, dtype=dtype)) @test_utils.combined_named_parameters(BATCH_MODULES + RECURRENT_MODULES, test_utils.named_bools("autograph")) def test_create_variables_eagerly( self, module_fn: ModuleFn, input_shape: Tuple[int], dtype: tf.DType, autograph: bool, ): module = module_fn() f = snt.distribute.create_variables_eagerly(module) forward = tf.function(f, autograph=autograph) forward(tf.ones(input_shape, dtype=dtype)) @test_utils.combined_named_parameters(BATCH_MODULES + RECURRENT_MODULES, test_utils.named_bools("autograph")) def test_trace_batch_agnostic( self, module_fn: ModuleFn, input_shape: Tuple[int], dtype: tf.DType, autograph: bool, ): module = module_fn() forward = tf.function(module, autograph=autograph) input_spec = tf.TensorSpec((None,) + input_shape[1:], dtype=dtype) cf = forward.get_concrete_function(input_spec) cf(tf.ones(input_shape, dtype=dtype)) @test_utils.combined_named_parameters(BATCH_MODULES, test_utils.named_bools("autograph")) def test_trace_batch_apply_batch_agnostic( self, module_fn: ModuleFn, input_shape: Tuple[int], dtype: tf.DType, autograph: bool, ): module = snt.BatchApply(module_fn()) forward = tf.function(module, autograph=autograph) input_shape = (8,) + input_shape input_spec = tf.TensorSpec((None, None) + input_shape[2:], dtype=dtype) cf = forward.get_concrete_function(input_spec) if isinstance( descriptors.unwrap(module.module), (snt.nets.VectorQuantizer, snt.nets.VectorQuantizerEMA)): # TODO(tomhennigan) Make VQ and VQ-EMA batch agnostic under BatchApply. return cf(tf.ones(input_shape, dtype=dtype)) @test_utils.combined_named_parameters(OPTIMIZER_MODULES, test_utils.named_bools("autograph")) def test_optimizer_dense( self, optimizer_fn: ModuleFn, input_shape: Tuple[int], dtype: tf.DType, autograph: bool, ): del input_shape, dtype # Unused. parameters = [tf.Variable([1., 2.])] updates = [tf.constant([5., 5.])] optimizer = optimizer_fn() optimizer_apply = tf.function(optimizer.apply, autograph=autograph) optimizer_apply(updates, parameters) # TODO(petebu) Add a test with completely dynamic shapes. @test_utils.combined_named_parameters(OPTIMIZER_MODULES, test_utils.named_bools("autograph")) def test_optimizer_sparse( self, optimizer_fn: ModuleFn, input_shape: Tuple[int], dtype: tf.DType, autograph: bool, ): del input_shape, dtype # Unused. if self.primary_device == "TPU": self.skipTest("IndexedSlices not supported on TPU.") parameters = [tf.Variable([[1.], [2.]])] updates = [ tf.IndexedSlices( tf.constant([0.1], shape=[1, 1]), tf.constant([0]), tf.constant([2, 1])) ] optimizer = optimizer_fn() optimizer_apply = tf.function(optimizer.apply, autograph=autograph) optimizer_apply(updates, parameters) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/conformance/goldens.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Golden test cases.""" import abc from typing import Sequence, Tuple from absl.testing import parameterized import numpy as np import sonnet as snt import tensorflow as tf _all_goldens = [] def named_goldens() -> Sequence[Tuple[str, "Golden"]]: return ((name, cls()) for _, name, cls in list_goldens()) def all_goldens(test_method): return parameterized.named_parameters(named_goldens())(test_method) def _register_golden(module_cls, golden_name): def registration_fn(golden_cls): _all_goldens.append((module_cls, golden_name, golden_cls)) golden_cls.name = golden_name return golden_cls return registration_fn def list_goldens(): return list(_all_goldens) def range_like(t, start=0): """Returns a tensor with sequential values of the same dtype/shape as `t`. >>> range_like(tf.ones([2, 2])) >>> range_like(tf.ones([2, 2]), start=5) Args: t: A tensor like object (with shape and dtype). start: Value to start the range from. Returns: A `tf.Tensor` with sequential element values the same shape/dtype as `t`. """ return tf.reshape( tf.cast( tf.range(start, np.prod(t.shape, dtype=int) + start), dtype=t.dtype), t.shape) class Golden(abc.ABC): """Represents a golden checkpoint file.""" @abc.abstractmethod def create_module(self): """Should create a new module instance and return it.""" pass @abc.abstractmethod def create_all_variables(self, module): """Create all variables for the given model and return them.""" pass @abc.abstractmethod def forward(self, module, x=None): """Return the output from calling the module with a fixed input.""" pass class AbstractGolden(Golden): """Abstract base class for golden tests of single input modules.""" deterministic = True has_side_effects = False # Tolerance to be used for assertAllClose calls on TPU, where lower precision # can mean results differ more. tpu_atol = 1e-3 @abc.abstractproperty def input_spec(self): pass @abc.abstractproperty def num_variables(self): pass def forward(self, module, x=None): if x is None: x = range_like(self.input_spec, start=1) return module(x) def create_all_variables(self, module): self.forward(module) variables = module.variables assert len(variables) == self.num_variables, ( "Expected %d params, got %d %r" % (self.num_variables, len(variables), variables)) return variables # pylint: disable=missing-docstring @_register_golden(snt.Linear, "linear_1x1") class Linear1x1Test(AbstractGolden): create_module = lambda _: snt.Linear(1) input_spec = tf.TensorSpec([128, 1]) num_variables = 2 @_register_golden(snt.Linear, "linear_nobias_1x1") class LinearNoBias1x1(AbstractGolden): create_module = lambda _: snt.Linear(1, with_bias=False) input_spec = tf.TensorSpec([1, 1]) num_variables = 1 @_register_golden(snt.Conv1D, "conv1d_3x3_2x2") class Conv1D(AbstractGolden): create_module = lambda _: snt.Conv1D(output_channels=3, kernel_shape=3) input_spec = tf.TensorSpec([1, 2, 2]) num_variables = 2 @_register_golden(snt.Conv2D, "conv2d_3x3_2x2") class Conv2D(AbstractGolden): create_module = lambda _: snt.Conv2D(output_channels=3, kernel_shape=3) input_spec = tf.TensorSpec([1, 2, 2, 2]) num_variables = 2 @_register_golden(snt.Conv3D, "conv3d_3x3_2x2") class Conv3D(AbstractGolden): create_module = lambda _: snt.Conv3D(output_channels=3, kernel_shape=3) input_spec = tf.TensorSpec([1, 2, 2, 2, 2]) num_variables = 2 @_register_golden(snt.Conv1DTranspose, "conv1d_transpose_3x3_2x2") class Conv1DTranspose(AbstractGolden): create_module = ( lambda _: snt.Conv1DTranspose(output_channels=3, kernel_shape=3)) input_spec = tf.TensorSpec([1, 2, 2]) num_variables = 2 @_register_golden(snt.Conv2DTranspose, "conv2d_transpose_3x3_2x2") class Conv2DTranspose(AbstractGolden): create_module = ( lambda _: snt.Conv2DTranspose(output_channels=3, kernel_shape=3)) input_spec = tf.TensorSpec([1, 2, 2, 2]) num_variables = 2 @_register_golden(snt.Conv3DTranspose, "conv3d_transpose_3x3_2x2") class Conv3DTranspose(AbstractGolden): create_module = ( lambda _: snt.Conv3DTranspose(output_channels=3, kernel_shape=3)) input_spec = tf.TensorSpec([1, 2, 2, 2, 2]) num_variables = 2 @_register_golden(snt.DepthwiseConv2D, "depthwise_conv2d_3x3_2x2") class DepthwiseConv2D(AbstractGolden): create_module = lambda _: snt.DepthwiseConv2D(kernel_shape=3) input_spec = tf.TensorSpec([1, 2, 2, 2]) num_variables = 2 @_register_golden(snt.nets.MLP, "mlp_3x4x5_1x3") class MLP(AbstractGolden): create_module = (lambda _: snt.nets.MLP([3, 4, 5])) input_spec = tf.TensorSpec([1, 3]) num_variables = 6 @_register_golden(snt.nets.MLP, "mlp_nobias_3x4x5_1x3") class MLPNoBias(AbstractGolden): create_module = (lambda _: snt.nets.MLP([3, 4, 5], with_bias=False)) input_spec = tf.TensorSpec([1, 3]) num_variables = 3 @_register_golden(snt.nets.Cifar10ConvNet, "cifar10_convnet_2x3_2x2_1x3x3x2") class Cifar10ConvNet(AbstractGolden): create_module = ( lambda _: snt.nets.Cifar10ConvNet(output_channels=(2, 3), strides=(2, 2))) input_spec = tf.TensorSpec([1, 3, 3, 2]) num_variables = 22 def forward(self, module, x=None): if x is None: x = range_like(self.input_spec, start=1) return module(x, is_training=False, test_local_stats=True)["logits"] @_register_golden(snt.LayerNorm, "layer_norm_1_1x3_2") class LayerNorm(AbstractGolden): create_module = ( lambda _: snt.LayerNorm(1, create_scale=True, create_offset=True)) input_spec = tf.TensorSpec([1, 3, 2]) num_variables = 2 @_register_golden(snt.InstanceNorm, "instance_norm_1_1x3_2") class Instance(AbstractGolden): create_module = ( lambda _: snt.InstanceNorm(create_scale=True, create_offset=True)) input_spec = tf.TensorSpec([1, 3, 2]) num_variables = 2 @_register_golden(snt.GroupNorm, "group_norm_2_1x3x4") class GroupNorm(AbstractGolden): create_module = ( lambda _: snt.GroupNorm(2, create_scale=True, create_offset=True)) input_spec = tf.TensorSpec([1, 3, 4]) num_variables = 2 @_register_golden(snt.BaseBatchNorm, "base_batch_norm_1x2x2x3") class BaseBatchNorm(AbstractGolden): create_module = ( lambda _: snt.BaseBatchNorm(True, False, FooMetric(), FooMetric())) # pytype: disable=wrong-arg-types input_spec = tf.TensorSpec([1, 2, 2, 3]) num_variables = 2 def forward(self, module, x=None): if x is None: x = range_like(self.input_spec, start=1) return module(x, is_training=False, test_local_stats=True) @_register_golden(snt.BaseBatchNorm, "base_batch_norm_scale_offset_1x2x2x3") class BaseBatchNormScaleOffset(AbstractGolden): create_module = ( lambda _: snt.BaseBatchNorm(True, False, FooMetric(), FooMetric())) # pytype: disable=wrong-arg-types input_spec = tf.TensorSpec([1, 2, 2, 3]) num_variables = 2 def forward(self, module, x=None): if x is None: x = range_like(self.input_spec, start=1) return module(x, is_training=False, test_local_stats=True) @_register_golden(snt.BatchNorm, "batch_norm_1x2x2x3") class BatchNorm(AbstractGolden): create_module = (lambda _: snt.BatchNorm(True, True)) input_spec = tf.TensorSpec([1, 2, 2, 3]) num_variables = 8 def forward(self, module, x=None): if x is None: x = range_like(self.input_spec, start=1) return module(x, is_training=False, test_local_stats=True) @_register_golden(snt.BatchNorm, "batch_norm_scale_offset_1x2x2x3") class BatchNormScaleOffset(AbstractGolden): create_module = (lambda _: snt.BatchNorm(True, True)) input_spec = tf.TensorSpec([1, 2, 2, 3]) num_variables = 8 def forward(self, module, x=None): if x is None: x = range_like(self.input_spec, start=1) return module(x, is_training=False, test_local_stats=True) @_register_golden(snt.ExponentialMovingAverage, "ema_2") class ExponentialMovingAverage(AbstractGolden): create_module = (lambda _: snt.ExponentialMovingAverage(decay=0.9)) input_spec = tf.TensorSpec([2]) num_variables = 3 has_side_effects = True def forward(self, module, x=None): if x is None: x = range_like(self.input_spec, start=1) return module(x) @_register_golden(snt.BatchNorm, "batch_norm_training_1x2x2x3") class BatchNormTraining(AbstractGolden): create_module = (lambda _: snt.BatchNorm(True, True)) input_spec = tf.TensorSpec([1, 2, 2, 3]) num_variables = 8 has_side_effects = True def forward(self, module, x=None): if x is None: x = range_like(self.input_spec, start=1) return module(x, is_training=True) @_register_golden(snt.distribute.CrossReplicaBatchNorm, "cross_replica_batch_norm_1x2x2x3") class CrossReplicaBatchNorm(AbstractGolden): create_module = ( lambda _: snt.BaseBatchNorm(True, False, FooMetric(), FooMetric())) input_spec = tf.TensorSpec([1, 2, 2, 3]) num_variables = 2 def forward(self, module, x=None): if x is None: x = range_like(self.input_spec, start=1) return module(x, is_training=False, test_local_stats=True) @_register_golden(snt.Dropout, "dropout") class DropoutVariableRate(AbstractGolden): create_module = lambda _: snt.Dropout(rate=tf.Variable(0.5)) input_spec = tf.TensorSpec([3, 3, 3]) num_variables = 1 deterministic = False def forward(self, module, x=None): tf.random.set_seed(3) if x is None: x = range_like(self.input_spec, start=1) return module(x, is_training=True) class AbstractRNNGolden(AbstractGolden): def forward(self, module, x=None): if x is None: # Small inputs to ensure that tf.tanh and tf.sigmoid don't saturate. x = 1.0 / range_like(self.input_spec, start=1) batch_size = self.input_spec.shape[0] prev_state = module.initial_state(batch_size) y, next_state = module(x, prev_state) del next_state return y @_register_golden(snt.Conv1DLSTM, "conv1d_lstm_3x3_2x2") class Conv1DLSTM(AbstractRNNGolden): input_spec = tf.TensorSpec([1, 2, 2]) num_variables = 3 def create_module(self): return snt.Conv1DLSTM( input_shape=self.input_spec.shape[1:], output_channels=3, kernel_shape=3) @_register_golden(snt.Conv2DLSTM, "conv2d_lstm_3x3_2x2") class Conv2DLSTM(AbstractRNNGolden): input_spec = tf.TensorSpec([1, 2, 2, 2]) num_variables = 3 def create_module(self): return snt.Conv2DLSTM( input_shape=self.input_spec.shape[1:], output_channels=3, kernel_shape=3) @_register_golden(snt.Conv3DLSTM, "conv3d_lstm_3x3_2x2") class Conv3DLSTM(AbstractRNNGolden): input_spec = tf.TensorSpec([1, 2, 2, 2, 2]) num_variables = 3 def create_module(self): return snt.Conv3DLSTM( input_shape=self.input_spec.shape[1:], output_channels=3, kernel_shape=3) @_register_golden(snt.GRU, "gru_1") class GRU(AbstractRNNGolden): create_module = lambda _: snt.GRU(hidden_size=1) input_spec = tf.TensorSpec([1, 128]) num_variables = 3 @_register_golden(snt.LSTM, "lstm_1") class LSTM(AbstractRNNGolden): create_module = lambda _: snt.LSTM(hidden_size=1) input_spec = tf.TensorSpec([1, 128]) num_variables = 3 @_register_golden(snt.LSTM, "lstm_8_projected_1") class LSTMWithProjection(AbstractRNNGolden): create_module = lambda _: snt.LSTM(hidden_size=8, projection_size=1) input_spec = tf.TensorSpec([1, 128]) num_variables = 4 @_register_golden(snt.UnrolledLSTM, "unrolled_lstm_1") class UnrolledLSTM(AbstractRNNGolden): create_module = lambda _: snt.UnrolledLSTM(hidden_size=1) input_spec = tf.TensorSpec([1, 1, 128]) num_variables = 3 @_register_golden(snt.VanillaRNN, "vanilla_rnn_8") class VanillaRNN(AbstractRNNGolden): create_module = lambda _: snt.VanillaRNN(hidden_size=8) input_spec = tf.TensorSpec([1, 128]) num_variables = 3 @_register_golden(snt.TrainableState, "trainable_state") class TrainableState(AbstractGolden): create_module = lambda _: snt.TrainableState(tf.zeros([1])) input_spec = tf.TensorSpec(()) num_variables = 1 @_register_golden(snt.Bias, "bias_3x3x3") class BiasTest(AbstractGolden): create_module = lambda _: snt.Bias() input_spec = tf.TensorSpec([1, 3, 3, 3]) num_variables = 1 @_register_golden(snt.Embed, "embed_100_100") class EmbedTest(AbstractGolden): create_module = lambda _: snt.Embed(vocab_size=100, embed_dim=100) input_spec = tf.TensorSpec([10], dtype=tf.int32) num_variables = 1 @_register_golden(snt.Mean, "mean_2x2") class MeanTest(AbstractGolden): create_module = lambda _: snt.Mean() input_spec = tf.TensorSpec([2, 2]) num_variables = 2 has_side_effects = True @_register_golden(snt.Sum, "sum_2x2") class SumTest(AbstractGolden): create_module = lambda _: snt.Sum() input_spec = tf.TensorSpec([2, 2]) num_variables = 1 has_side_effects = True @_register_golden(snt.nets.ResNet, "resnet50") class ResNet(AbstractGolden): create_module = (lambda _: snt.nets.ResNet([1, 1, 1, 1], 9)) input_spec = tf.TensorSpec([1, 8, 8, 3]) num_variables = 155 has_side_effects = True def forward(self, module, x=None): if x is None: x = range_like(self.input_spec, start=1) return module(x, is_training=True) @_register_golden(snt.nets.VectorQuantizer, "vqvae") class VectorQuantizerTest(AbstractGolden): def create_module(self): return snt.nets.VectorQuantizer( embedding_dim=4, num_embeddings=6, commitment_cost=0.25) # Input can be any shape as long as final dimension is equal to embedding_dim. input_spec = tf.TensorSpec([2, 3, 4]) def forward(self, module, x=None): if x is None: x = range_like(self.input_spec) return module(x, is_training=True) # Numerical results can be quite different on TPU, be a bit more loose here. tpu_atol = 4e-2 num_variables = 1 @_register_golden(snt.nets.VectorQuantizerEMA, "vqvae_ema_train") class VectorQuantizerEMATrainTest(AbstractGolden): def create_module(self): return snt.nets.VectorQuantizerEMA( embedding_dim=5, num_embeddings=7, commitment_cost=0.5, decay=0.9) # Input can be any shape as long as final dimension is equal to embedding_dim. input_spec = tf.TensorSpec([2, 5]) def forward(self, module, x=None): if x is None: x = range_like(self.input_spec) return module(x, is_training=True) # Numerical results can be quite different on TPU, be a bit more loose here. tpu_atol = 4e-2 num_variables = 7 # 1 embedding, then 2 EMAs each of which contain 3. has_side_effects = True @_register_golden(snt.nets.VectorQuantizerEMA, "vqvae_ema_eval") class VectorQuantizerEMAEvalTest(AbstractGolden): def create_module(self): return snt.nets.VectorQuantizerEMA( embedding_dim=3, num_embeddings=4, commitment_cost=0.5, decay=0.9) # Input can be any shape as long as final dimension is equal to embedding_dim. input_spec = tf.TensorSpec([2, 3]) def forward(self, module, x=None): if x is None: x = range_like(self.input_spec) return module(x, is_training=False) # Numerical results can be quite different on TPU, be a bit more loose here. tpu_atol = 4e-2 num_variables = 7 # 1 embedding, then 2 EMAs each of which contain 3. has_side_effects = False # only has side effects when is_training==True # pylint: enable=missing-docstring class FooMetric(snt.Metric): """Used for testing a class which uses Metrics.""" def initialize(self, x): pass def reset(self): pass def update(self, x): pass ================================================ FILE: sonnet/src/conformance/goldens_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests goldens cover all modules.""" import inspect import sonnet as snt from sonnet.src import test_utils from sonnet.src.conformance import goldens import tensorflow as tf class CoverageTest(test_utils.TestCase): def test_all_modules_covered(self): allow_no_checkpoint = set([ # TODO(petebu): Remove this once optimizer goldens check works. snt.optimizers.Adam, snt.optimizers.Momentum, snt.optimizers.RMSProp, snt.optimizers.SGD, # Stateless or abstract. snt.BatchApply, snt.DeepRNN, snt.Deferred, snt.Flatten, snt.Metric, snt.Module, snt.Optimizer, snt.Reshape, snt.RNNCore, snt.Sequential, snt.UnrolledRNN, # Tested via snt.nets.ResNet snt.nets.ResNet50, snt.nets.resnet.BottleNeckBlockV1, snt.nets.resnet.BottleNeckBlockV2, snt.nets.resnet.BlockGroup ]) # Find all the snt.Module types reachable from `import sonnet as snt` all_sonnet_types = set() for _, python_module in test_utils.find_sonnet_python_modules(snt): for _, cls in inspect.getmembers(python_module, inspect.isclass): if issubclass(cls, snt.Module): all_sonnet_types.add(cls) # Find all the modules that have checkpoint tests. tested_modules = {module_cls for module_cls, _, _ in goldens.list_goldens()} # Make sure we don't leave entries in allow_no_checkpoint if they are # actually tested. self.assertEmpty(tested_modules & allow_no_checkpoint) # Make sure everything is covered. self.assertEqual(tested_modules | allow_no_checkpoint, all_sonnet_types) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/conformance/keras_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests Sonnet and Keras compatibility.""" from absl.testing import parameterized import sonnet as snt from sonnet.src import test_utils from sonnet.src.conformance import descriptors import tensorflow as tf import tree BATCH_MODULES = descriptors.BATCH_MODULES RECURRENT_MODULES = descriptors.RECURRENT_MODULES # TODO(tomhennigan) Add tests with Keras optimizers. # TODO(tomhennigan) Test Keras compile/fit. class KerasTest(test_utils.TestCase, parameterized.TestCase): @parameterized.named_parameters(*(BATCH_MODULES + RECURRENT_MODULES)) def test_build_without_batch(self, module_fn, input_shape, dtype): # For Keras test that building with unknown batch dim is supported. layer = LayerAdapter(module=module_fn(), dtype=dtype) layer.build((None,) + input_shape[1:]) # For Sonnet just call with the example input. mod = module_fn() mod(tf.ones(input_shape, dtype)) # Some modules (e.g. Sequential) are parameter-less. snt.allow_empty_variables(mod) # Test that module variables look the same. by_name = lambda c: sorted(c, key=lambda v: v.name) abstract = lambda v: (v.name, v.shape, v.dtype) for collection in ("variables", "trainable_variables"): for m, l in zip( by_name(getattr(mod, collection)), by_name(getattr(layer, collection))): self.assertEqual(abstract(m), abstract(l)) @parameterized.named_parameters(*(BATCH_MODULES + RECURRENT_MODULES)) def test_sonnet_module_as_layer(self, module_fn, input_shape, dtype): mod = module_fn() layer = LayerAdapter(module=module_fn(), dtype=dtype) example_input = tf.ones(input_shape, dtype=dtype) # Check outputs are the same. for m_y, l_y in zip( tree.flatten(mod(example_input)), tree.flatten(layer(example_input))): self.assertEqual(m_y.shape, l_y.shape) self.assertEqual(m_y.dtype, l_y.dtype) # Some modules (e.g. Sequential) are parameter-less. snt.allow_empty_variables(mod) # Check that variables are the same. self.assertEqual(len(mod.variables), len(layer.variables)) self.assertEqual( len(mod.trainable_variables), len(layer.trainable_variables)) # Check that Keras layer freezing works layer.trainable = False self.assertEmpty(layer.trainable_variables) def test_build_with_updating_module(self): # Calling the module creates and updates `w`. mod = ModuleWithUpdateInCall() mod(tf.ones([])) self.assertEqual(mod.w.numpy(), 1) # Calling build() should not trigger updating `w`, just creating it. layer = LayerAdapter(ModuleWithUpdateInCall()) layer.build([]) self.assertEqual(layer.module.w.numpy(), 0) def test_layer_with_model(self): layers = [ LayerAdapter(snt.Linear(3)), LayerAdapter(snt.Linear(2)), LayerAdapter(snt.Linear(1)) ] model = tf.keras.models.Sequential(layers) model.build([None, 4]) for idx, input_size in enumerate([4, 3, 2]): self.assertEqual(layers[idx].module.input_size, input_size) output_shape = model.compute_output_shape([None, 4]) self.assertTrue(output_shape.is_compatible_with([None, 1])) self.assertEqual(model(tf.ones([1, 4])).shape, [1, 1]) @parameterized.named_parameters(*(BATCH_MODULES + RECURRENT_MODULES)) def test_symbolic_model(self, module_fn, input_shape, dtype): module = module_fn() inputs = tf.keras.Input(input_shape[1:], dtype=dtype) layer = LayerAdapter(module=module, dtype=dtype) output = layer(inputs) model = tf.keras.Model(inputs=inputs, outputs=output) example_input = tf.ones(input_shape, dtype=dtype) # Check outputs are the same. for m_y, l_y in zip( tree.flatten(module(example_input)), tree.flatten(model(example_input))): self.assertEqual(m_y.shape, l_y.shape) self.assertEqual(m_y.dtype, l_y.dtype) def test_layer_adapter_custom_method(self): module = ModuleWithCustomForward() inputs = tf.keras.Input([], batch_size=1) layer = LayerAdapter(module=module, method="forward") output = layer(inputs) model = tf.keras.Model(inputs=inputs, outputs=output) self.assertEqual(model(tf.ones([])).numpy(), [2.]) self.assertEqual(model.trainable_variables, [module.w]) def test_keras_layer_inside_sonnet_module(self): mod = ModuleWithLayer() mod(tf.ones([1, 1])) self.assertEqual(mod.submodules, (mod.dense,)) self.assertLen(mod.variables, 2) self.assertLen(mod.trainable_variables, 2) # Test that layer freezing does not change tf.Module tracking. mod.dense.trainable = False self.assertLen(mod.variables, 2) self.assertLen(mod.trainable_variables, 2) def test_to_config(self): mod = LayerAdapter(ModuleWithLayer()) with self.assertRaises(NotImplementedError): mod.to_config() def test_from_config(self): with self.assertRaises(NotImplementedError): LayerAdapter.from_config(None) # TODO(tomhennigan) Make this part of the public API? class LayerAdapter(tf.keras.layers.Layer): """Adapts a Sonnet module to conform to the Keras Layer API. >>> layer = LayerAdapter(snt.Linear(1)) >>> assert isinstance(layer, tf.keras.layers.Layer) We support building without ``__call__``, even with unknown dimensions: >>> layer.build(input_shape=[None, 28 * 28]) Of course now features of Keras work as expected, for example layer freezing: >>> [v.name for v in layer.trainable_variables] ["linear/b:0", "linear/w:0"] >>> layer.trainable = False >>> layer.trainable_variables [] """ def __init__(self, module, method="__call__", dtype=tf.float32): super().__init__(dtype=dtype) self.module = module self._module_call_method = getattr(module, method) self._output_shapes = None @classmethod def from_config(cls, config): raise NotImplementedError def to_config(self): raise NotImplementedError def _trace_and_initialize(self, input_shape): if self._output_shapes is None: self._output_shapes = tree.map_structure( lambda spec: spec.shape if spec is not None else spec, snt.build(self, tf.TensorSpec(input_shape, self.dtype))) return self._output_shapes def compute_output_shape(self, input_shape): output_shapes = self._trace_and_initialize(input_shape) return output_shapes def build(self, input_shape): super().build(input_shape) # Trigger variable initialization by tracing the module. self._trace_and_initialize(input_shape) # Make sure Keras variable tracking finds our weights. # Keras has a setattr override which can be used to register weights in a # similar way to `Layer.add_weight`. By setting `_sonnet_weights` we trigger # this mechanism and module weights are found in `Layer.trainable_variables` # and `Layer.variables`. snt.allow_empty_variables(self.module) self._sonnet_weights = self.module.variables def call(self, inputs): return self._module_call_method(inputs) class ModuleWithLayer(snt.Module): def __init__(self): super().__init__() self.dense = tf.keras.layers.Dense(10) def __call__(self, x): return self.dense(x) class ModuleWithUpdateInCall(snt.Module): @snt.once def _init(self, x): self.w = tf.Variable(tf.zeros(x.shape), name="w") def __call__(self, x): self._init(x) self.w.assign_add(tf.ones_like(self.w)) return self.w.read_value() class ModuleWithCustomForward(snt.Module): @snt.once def _init(self, x): self.w = tf.Variable(tf.ones(x.shape), name="w") def forward(self, x): self._init(x) return x + self.w if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/conformance/optimizer_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Conformance tests for models and optimization.""" from absl.testing import parameterized from sonnet.src import test_utils from sonnet.src.conformance import descriptors import tensorflow as tf BATCH_MODULES = descriptors.BATCH_MODULES RECURRENT_MODULES = descriptors.RECURRENT_MODULES class OptimizerConformanceTest(test_utils.TestCase, parameterized.TestCase): @test_utils.combined_named_parameters( BATCH_MODULES + RECURRENT_MODULES, test_utils.named_bools("construct_module_in_function"), ) def test_variable_order_is_constant(self, module_fn, input_shape, dtype, construct_module_in_function): """Test that variable access order is consistent in built in modules.""" logged_variables = [] mod = [None] if not construct_module_in_function: mod[0] = module_fn() x = tf.zeros(input_shape, dtype=dtype) @tf.function(autograph=False) def f(): with tf.GradientTape() as tape: if not mod[0]: mod[0] = module_fn() mod[0](x) # pylint: disable=not-callable # Leak out the variables that were used. logged_variables.append( [(id(v), v.name) for v in tape.watched_variables()]) # NOTE: This will run `f` twice iff `f` creates params. f() if len(logged_variables) == 1: self.skipTest("Module did not create variables in forward pass.") else: assert len(logged_variables) == 2 self.assertCountEqual(logged_variables[0], logged_variables[1]) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/conformance/pickle_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests pickling Sonnet modules.""" import pickle from absl.testing import parameterized from sonnet.src import test_utils from sonnet.src.conformance import goldens import tensorflow as tf import tree class PickleTest(test_utils.TestCase, parameterized.TestCase): # TODO(tomhennigan) Add tests with dill and cloudpickle. @goldens.all_goldens def test_pickle(self, golden): m1 = golden.create_module() golden.create_all_variables(m1) m2 = pickle.loads(pickle.dumps(m1)) self.assertIsNot(m1, m2) # Check that module variables are recreated with equivalent properties. for v1, v2 in zip(m1.variables, m2.variables): self.assertIsNot(v1, v2) self.assertEqual(v1.name, v2.name) self.assertEqual(v1.device, v2.device) self.assertAllEqual(v1.read_value(), v2.read_value()) if golden.deterministic: y1 = golden.forward(m1) y2 = golden.forward(m2) tree.map_structure(self.assertAllEqual, y1, y2) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/conformance/saved_model_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests using tf.saved_model and Sonnet.""" import os from absl.testing import absltest from absl.testing import parameterized import sonnet as snt from sonnet.src import test_utils from sonnet.src.conformance import goldens import tensorflow as tf import tree class SavedModelTest(test_utils.TestCase, parameterized.TestCase): @goldens.all_goldens def test_save_restore_cycle(self, golden): module = golden.create_module() # Create all parameters and set them to sequential (but different) values. variables = golden.create_all_variables(module) for index, variable in enumerate(variables): variable.assign(goldens.range_like(variable, start=index)) @tf.function(input_signature=[golden.input_spec]) def inference(x): return golden.forward(module, x) # Create a saved model, add a method for inference and a dependency on our # module such that it can find dependencies. saved_model = snt.Module() saved_model._module = module saved_model.inference = inference saved_model.all_variables = list(module.variables) # Sample input. x = goldens.range_like(golden.input_spec) # Run the saved model and pull variable values. saved_model.inference(x) v1 = saved_model.all_variables # Save the model to disk and restore it. tmp_dir = os.path.join(absltest.get_default_test_tmpdir(), golden.name) tf.saved_model.save(saved_model, tmp_dir) restored_model = tf.saved_model.load(tmp_dir) # Run the loaded model and pull variable values. v2 = restored_model.all_variables y2 = restored_model.inference(x) if golden.deterministic: # The output from both the saved and restored model should be close. y1 = saved_model.inference(x) # TODO(b/161972382): The restored model doesn't seem to specialize the # graph with implementation selector, so the original model uses CuDNN # calls, whereas the restored model uses the non-specialized graph which # still contains a regular Tanh op. tree.map_structure(self.assertAllClose, y1, y2) for a, b in zip(v1, v2): self.assertEqual(a.name, b.name) self.assertEqual(a.device, b.device) self.assertAllEqual(a.read_value(), b.read_value()) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/conformance/tensorflow1_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests Sonnet 2 with TF1.""" import sonnet as snt from sonnet.src import test_utils import tensorflow.compat.v1 as tf class TensorFlow1Test(test_utils.TestCase): def test_requires_tf2(self): if tf.version.GIT_VERSION != "unknown": self.skipTest("This test only runs if testing against TF at head.") with self.assertRaisesRegex(AssertionError, "requires TensorFlow 2"): snt.Module() if __name__ == "__main__": tf.disable_v2_behavior() tf.test.main() ================================================ FILE: sonnet/src/conformance/xla_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests Sonnet and XLA.""" import functools from absl.testing import parameterized from sonnet.src import test_utils from sonnet.src.conformance import goldens import tensorflow as tf import tree class XLATest(test_utils.TestCase, parameterized.TestCase): @goldens.all_goldens def test_compile(self, golden): mod = golden.create_module() golden.create_all_variables(mod) @tf.function def forward(): f = lambda: golden.forward(mod) out = tf.xla.experimental.compile(f) if len(out) == 1: return out[0] else: return out if out else None if self.primary_device == "TPU": # TODO(b/132329316) Remove when `xla.compile` allows tf.device(TPU). with tf.device(None): xla_out = forward() atol = golden.tpu_atol else: xla_out = forward() atol = 1e-3 if golden.deterministic and not golden.has_side_effects: out = golden.forward(mod) tree.map_structure( functools.partial(self.assertAllClose, atol=atol), out, xla_out) @goldens.all_goldens def test_jit_scope(self, golden): mod = golden.create_module() golden.create_all_variables(mod) @tf.function def forward(): with tf.xla.experimental.jit_scope(): return golden.forward(mod) xla_out = forward() if self.primary_device == "TPU": atol = golden.tpu_atol else: atol = 1e-3 if golden.deterministic and not golden.has_side_effects: out = golden.forward(mod) tree.map_structure( functools.partial(self.assertAllClose, atol=atol), out, xla_out) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/conv.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Convolutional modules.""" from typing import Optional, Sequence, Union import numpy as np from sonnet.src import base from sonnet.src import initializers from sonnet.src import once from sonnet.src import pad from sonnet.src import utils import tensorflow as tf class ConvND(base.Module): """A general N-dimensional convolutional module.""" def __init__(self, num_spatial_dims: int, output_channels: int, kernel_shape: Union[int, Sequence[int]], stride: Union[int, Sequence[int]] = 1, rate: Union[int, Sequence[int]] = 1, padding: Union[str, pad.Paddings] = "SAME", with_bias: bool = True, w_init: Optional[initializers.Initializer] = None, b_init: Optional[initializers.Initializer] = None, data_format: Optional[str] = None, name: Optional[str] = None): """Constructs a `ConvND` module. Args: num_spatial_dims: The number of spatial dimensions of the input. output_channels: The number of output channels. kernel_shape: Sequence of kernel sizes (of length num_spatial_dims), or an integer. `kernel_shape` will be expanded to define a kernel size in all dimensions. stride: Sequence of strides (of length num_spatial_dims), or an integer. `stride` will be expanded to define stride in all dimensions. rate: Sequence of dilation rates (of length num_spatial_dims), or integer that is used to define dilation rate in all dimensions. 1 corresponds to standard ND convolution, `rate > 1` corresponds to dilated convolution. padding: Padding to apply to the input. This can either "SAME", "VALID" or a callable or sequence of callables up to size N. Any callables must take a single integer argument equal to the effective kernel size and return a list of two integers representing the padding before and after. See snt.pad.* for more details and example functions. with_bias: Whether to include bias parameters. Default `True`. w_init: Optional initializer for the weights. By default the weights are initialized truncated random normal values with a standard deviation of `1 / sqrt(input_feature_size)`, which is commonly used when the inputs are zero centered (see https://arxiv.org/abs/1502.03167v3). b_init: Optional initializer for the bias. By default the bias is initialized to zero. data_format: The data format of the input. name: Name of the module. """ super().__init__(name=name) if not 1 <= num_spatial_dims <= 3: raise ValueError( "We only support convoltion operations for num_spatial_dims=1, 2 or " "3, received num_spatial_dims={}.".format(num_spatial_dims)) self._num_spatial_dims = num_spatial_dims self.output_channels = output_channels self.kernel_shape = kernel_shape self.stride = stride self.rate = rate if isinstance(padding, str): self.conv_padding = padding.upper() self.padding_func = None else: self.conv_padding = "VALID" self.padding_func = padding self.data_format = data_format self._channel_index = utils.get_channel_index(data_format) self.with_bias = with_bias self.w_init = w_init if with_bias: self.b_init = b_init if b_init is not None else initializers.Zeros() elif b_init is not None: raise ValueError("When not using a bias the b_init must be None.") def __call__(self, inputs: tf.Tensor) -> tf.Tensor: """Applies the defined convolution to the inputs. Args: inputs: An ``N + 2`` rank :tf:`Tensor` of dtype :tf:`float16`, :tf:`bfloat16` or `tf.float32` to which the convolution is applied. Returns: An ``N + 2`` dimensional :tf:`Tensor` of shape ``[batch_size, output_dim_1, output_dim_2, ..., output_channels]``. """ self._initialize(inputs) if self.padding_func: inputs = tf.pad(inputs, self._padding) outputs = tf.nn.convolution( inputs, self.w, strides=self.stride, padding=self.conv_padding, dilations=self.rate, data_format=self.data_format) if self.with_bias: outputs = tf.nn.bias_add(outputs, self.b, data_format=self.data_format) return outputs @once.once def _initialize(self, inputs: tf.Tensor): """Constructs parameters used by this module.""" utils.assert_rank(inputs, self._num_spatial_dims + 2) self.input_channels = inputs.shape[self._channel_index] if self.input_channels is None: raise ValueError("The number of input channels must be known.") self._dtype = inputs.dtype self.w = self._make_w() if self.with_bias: self.b = tf.Variable( self.b_init((self.output_channels,), self._dtype), name="b") if self.padding_func: self._padding = pad.create( padding=self.padding_func, kernel=self.kernel_shape, rate=self.rate, n=self._num_spatial_dims, channel_index=self._channel_index) def _make_w(self): weight_shape = utils.replicate(self.kernel_shape, self._num_spatial_dims, "kernel_shape") weight_shape = weight_shape + (self.input_channels, self.output_channels) if self.w_init is None: # See https://arxiv.org/abs/1502.03167v3. fan_in_shape = weight_shape[:-1] stddev = 1 / np.sqrt(np.prod(fan_in_shape)) self.w_init = initializers.TruncatedNormal(stddev=stddev) return tf.Variable(self.w_init(weight_shape, self._dtype), name="w") class Conv1D(ConvND): """``Conv1D`` module.""" def __init__(self, output_channels: int, kernel_shape: Union[int, Sequence[int]], stride: Union[int, Sequence[int]] = 1, rate: Union[int, Sequence[int]] = 1, padding: Union[str, pad.Paddings] = "SAME", with_bias: bool = True, w_init: Optional[initializers.Initializer] = None, b_init: Optional[initializers.Initializer] = None, data_format: str = "NWC", name: Optional[str] = None): """Constructs a ``Conv1D`` module. Args: output_channels: The number of output channels. kernel_shape: Sequence of length 1, or an integer. ``kernel_shape`` will be expanded to define a kernel size in all dimensions. stride: Sequence of strides of length 1, or an integer. ``stride`` will be expanded to define stride in all dimensions. rate: Sequence of dilation rates of length 1, or integer that is used to define dilation rate in all dimensions. 1 corresponds to standard convolution, ``rate > 1`` corresponds to dilated convolution. padding: Padding to apply to the input. This can be either ``SAME``, ``VALID`` or a callable or sequence of callables of size 1. Any callables must take a single integer argument equal to the effective kernel size and return a list of two integers representing the padding before and after. See snt.pad.* for more details and example functions. with_bias: Whether to include bias parameters. Default ``True``. w_init: Optional initializer for the weights. By default the weights are initialized truncated random normal values with a standard deviation of ``1``/``sqrt(input_feature_size)``, which is commonly used when the inputs are zero centered (see https://arxiv.org/abs/1502.03167v3). b_init: Optional initializer for the bias. By default the bias is initialized to zero. data_format: The data format of the input. name: Name of the module. """ super().__init__( num_spatial_dims=1, output_channels=output_channels, kernel_shape=kernel_shape, stride=stride, rate=rate, padding=padding, with_bias=with_bias, w_init=w_init, b_init=b_init, data_format=data_format, name=name) class Conv2D(ConvND): """`Conv2D` module.""" def __init__(self, output_channels: int, kernel_shape: Union[int, Sequence[int]], stride: Union[int, Sequence[int]] = 1, rate: Union[int, Sequence[int]] = 1, padding: Union[str, pad.Paddings] = "SAME", with_bias: bool = True, w_init: Optional[initializers.Initializer] = None, b_init: Optional[initializers.Initializer] = None, data_format: str = "NHWC", name: Optional[str] = None): """Constructs a ``Conv2D`` module. Args: output_channels: The number of output channels. kernel_shape: Sequence of kernel sizes (of length 2), or an integer. ``kernel_shape`` will be expanded to define a kernel size in all dimensions. stride: Sequence of strides (of length 2), or an integer. ``stride`` will be expanded to define stride in all dimensions. rate: Sequence of dilation rates (of length 2), or integer that is used to define dilation rate in all dimensions. 1 corresponds to standard convolution, ``rate > 1`` corresponds to dilated convolution. padding: Padding to apply to the input. This can either ``SAME``, ``VALID`` or a callable or sequence of callables of size 2. Any callables must take a single integer argument equal to the effective kernel size and return a list of two integers representing the padding before and after. See snt.pad.* for more details and example functions. with_bias: Whether to include bias parameters. Default ``True``. w_init: Optional initializer for the weights. By default the weights are initialized truncated random normal values with a standard deviation of ``1 / sqrt(input_feature_size)``, which is commonly used when the inputs are zero centered (see https://arxiv.org/abs/1502.03167v3). b_init: Optional initializer for the bias. By default the bias is initialized to zero. data_format: The data format of the input. name: Name of the module. """ super().__init__( num_spatial_dims=2, output_channels=output_channels, kernel_shape=kernel_shape, stride=stride, rate=rate, padding=padding, with_bias=with_bias, w_init=w_init, b_init=b_init, data_format=data_format, name=name) class Conv3D(ConvND): """`Conv3D` module.""" def __init__(self, output_channels: int, kernel_shape: Union[int, Sequence[int]], stride: Union[int, Sequence[int]] = 1, rate: Union[int, Sequence[int]] = 1, padding: Union[str, pad.Paddings] = "SAME", with_bias: bool = True, w_init: Optional[initializers.Initializer] = None, b_init: Optional[initializers.Initializer] = None, data_format: str = "NDHWC", name: Optional[str] = None): """Constructs a ``Conv3D`` module. Args: output_channels: The number of output channels. kernel_shape: Sequence of kernel sizes (of length 3), or an integer. ``kernel_shape`` will be expanded to define a kernel size in all dimensions. stride: Sequence of strides (of length 3), or an integer. `stride` will be expanded to define stride in all dimensions. rate: Sequence of dilation rates (of length 3), or integer that is used to define dilation rate in all dimensions. 1 corresponds to standard convolution, ``rate > 1`` corresponds to dilated convolution. padding: Padding to apply to the input. This can either ``SAME``, ``VALID`` or a callable or sequence of callables up to size N. Any callables must take a single integer argument equal to the effective kernel size and return a list of two integers representing the padding before and after. See snt.pad.* for more details and example functions. with_bias: Whether to include bias parameters. Default ``True``. w_init: Optional initializer for the weights. By default the weights are initialized truncated random normal values with a standard deviation of ``1 / sqrt(input_feature_size)``, which is commonly used when the inputs are zero centered (see https://arxiv.org/abs/1502.03167v3). b_init: Optional initializer for the bias. By default the bias is initialized to zero. data_format: The data format of the input. name: Name of the module. """ super().__init__( num_spatial_dims=3, output_channels=output_channels, kernel_shape=kernel_shape, stride=stride, rate=rate, padding=padding, with_bias=with_bias, w_init=w_init, b_init=b_init, data_format=data_format, name=name) ================================================ FILE: sonnet/src/conv_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.conv.""" from absl.testing import parameterized import numpy as np from sonnet.src import conv from sonnet.src import initializers from sonnet.src import test_utils import tensorflow as tf def create_constant_initializers(w, b, with_bias): if with_bias: return { "w_init": initializers.Constant(w), "b_init": initializers.Constant(b) } else: return {"w_init": initializers.Constant(w)} class ConvTest(test_utils.TestCase, parameterized.TestCase): def testPaddingFunctionReached(self): self.reached = False def padding_func(*unused_args): padding_func.called = True return [0, 0] conv1 = conv.ConvND( num_spatial_dims=2, output_channels=1, kernel_shape=3, stride=1, padding=padding_func, data_format="NHWC", **create_constant_initializers(1.0, 1.0, True)) conv1(tf.ones([1, 5, 5, 1], dtype=tf.float32)) self.assertEqual(conv1.conv_padding, "VALID") self.assertEqual(conv1.padding_func, padding_func) self.assertTrue(getattr(padding_func, "called", False)) @parameterized.parameters(0, 4) def testIncorrectN(self, n): with self.assertRaisesRegex( ValueError, "We only support convoltion operations for num_spatial_dims=1, 2 or 3"): conv.ConvND( num_spatial_dims=n, output_channels=1, kernel_shape=3, data_format="NHWC") def testInitializerKeysInvalidWithoutBias(self): with self.assertRaisesRegex(ValueError, "b_init must be None"): conv.ConvND( num_spatial_dims=2, output_channels=1, kernel_shape=3, data_format="NHWC", with_bias=False, b_init=tf.zeros_initializer()) def testIncorrectRankInput(self): c = conv.ConvND( num_spatial_dims=2, output_channels=1, kernel_shape=3, data_format="NHWC") with self.assertRaisesRegex(ValueError, "Shape .* must have rank 4"): c(tf.ones([2, 4, 4])) @parameterized.parameters(tf.float32, tf.float64) def testDefaultInitializers(self, dtype): if "TPU" in self.device_types and dtype == tf.float64: self.skipTest("Double precision not supported on TPU.") conv1 = conv.ConvND( num_spatial_dims=2, output_channels=1, kernel_shape=16, stride=1, padding="VALID", data_format="NHWC") out = conv1(tf.random.normal([8, 64, 64, 1], dtype=dtype)) self.assertAllEqual(out.shape, [8, 49, 49, 1]) self.assertEqual(out.dtype, dtype) # Note that for unit variance inputs the output is below unit variance # because of the use of the truncated normal initalizer err = 0.2 if self.primary_device == "TPU" else 0.1 self.assertNear(out.numpy().std(), 0.87, err=err) @parameterized.named_parameters( ("SamePaddingUseBias", True, "SAME"), ("SamePaddingNoBias", False, "SAME"), ("samePaddingUseBias", True, "same"), ("samePaddingNoBias", False, "same"), ("ValidPaddingNoBias", False, "VALID"), ("ValidPaddingUseBias", True, "VALID"), ("validPaddingNoBias", False, "valid"), ("validPaddingUseBias", True, "valid"), ) def testFunction(self, with_bias, padding): conv1 = conv.ConvND( num_spatial_dims=2, output_channels=1, kernel_shape=3, stride=1, padding=padding, with_bias=with_bias, data_format="NHWC", **create_constant_initializers(1.0, 1.0, with_bias)) conv2 = conv.ConvND( num_spatial_dims=2, output_channels=1, kernel_shape=3, stride=1, padding=padding, with_bias=with_bias, data_format="NHWC", **create_constant_initializers(1.0, 1.0, with_bias)) defun_conv = tf.function(conv2) iterations = 5 for _ in range(iterations): x = tf.random.uniform([1, 5, 5, 1]) y1 = conv1(x) y2 = defun_conv(x) self.assertAllClose(self.evaluate(y1), self.evaluate(y2), atol=1e-4) def testUnknownBatchSizeNHWC(self): x = tf.TensorSpec([None, 5, 5, 3], dtype=tf.float32) c = conv.ConvND( num_spatial_dims=2, output_channels=2, kernel_shape=3, data_format="NHWC") defun_conv = tf.function(c).get_concrete_function(x) out1 = defun_conv(tf.ones([3, 5, 5, 3])) self.assertEqual(out1.shape, [3, 5, 5, 2]) out2 = defun_conv(tf.ones([5, 5, 5, 3])) self.assertEqual(out2.shape, [5, 5, 5, 2]) def testUnknownBatchSizeNCHW(self): if self.primary_device == "CPU": self.skipTest("NCHW not supported on CPU") x = tf.TensorSpec([None, 3, 5, 5], dtype=tf.float32) c = conv.ConvND( num_spatial_dims=2, output_channels=2, kernel_shape=3, data_format="NCHW") defun_conv = tf.function(c).get_concrete_function(x) out1 = defun_conv(tf.ones([3, 3, 5, 5])) self.assertEqual(out1.shape, [3, 2, 5, 5]) out2 = defun_conv(tf.ones([5, 3, 5, 5])) self.assertEqual(out2.shape, [5, 2, 5, 5]) @parameterized.parameters(True, False) def testUnknownChannels(self, autograph): x = tf.TensorSpec([3, 3, 3, None], dtype=tf.float32) c = conv.ConvND( num_spatial_dims=2, output_channels=1, kernel_shape=3, data_format="NHWC") defun_conv = tf.function(c, autograph=autograph) with self.assertRaisesRegex(ValueError, "The number of input channels must be known"): defun_conv.get_concrete_function(x) def testUnknownSpatialDims(self): x = tf.TensorSpec([3, None, None, 3], dtype=tf.float32) c = conv.ConvND( num_spatial_dims=2, output_channels=1, kernel_shape=3, data_format="NHWC") defun_conv = tf.function(c).get_concrete_function(x) out = defun_conv(tf.ones([3, 5, 5, 3])) expected_out = c(tf.ones([3, 5, 5, 3])) self.assertEqual(out.shape, [3, 5, 5, 1]) self.assertAllEqual(self.evaluate(out), self.evaluate(expected_out)) out = defun_conv(tf.ones([3, 4, 4, 3])) expected_out = c(tf.ones([3, 4, 4, 3])) self.assertEqual(out.shape, [3, 4, 4, 1]) self.assertAllEqual(self.evaluate(out), self.evaluate(expected_out)) class Conv2DTest(test_utils.TestCase, parameterized.TestCase): @parameterized.parameters(True, False) def testComputationPaddingSame(self, with_bias): expected_out = [[4, 6, 6, 6, 4], [6, 9, 9, 9, 6], [6, 9, 9, 9, 6], [6, 9, 9, 9, 6], [4, 6, 6, 6, 4]] conv1 = conv.Conv2D( output_channels=1, kernel_shape=3, stride=1, padding="SAME", with_bias=with_bias, **create_constant_initializers(1.0, 1.0, with_bias)) out = conv1(tf.ones([1, 5, 5, 1], dtype=tf.float32)) self.assertEqual(out.shape, [1, 5, 5, 1]) out = tf.squeeze(out, axis=(0, 3)) expected_out = np.asarray(expected_out, dtype=np.float32) if with_bias: expected_out += 1 self.assertAllClose(self.evaluate(out), expected_out) @parameterized.parameters(True, False) def testComputationPaddingValid(self, with_bias): expected_out = [[9, 9, 9], [9, 9, 9], [9, 9, 9]] conv1 = conv.Conv2D( output_channels=1, kernel_shape=3, stride=1, padding="VALID", with_bias=with_bias, **create_constant_initializers(1.0, 1.0, with_bias)) out = conv1(tf.ones([1, 5, 5, 1], dtype=tf.float32)) self.assertEqual(out.shape, [1, 3, 3, 1]) out = tf.squeeze(out, axis=(0, 3)) expected_out = np.asarray(expected_out, dtype=np.float32) if with_bias: expected_out += 1 self.assertAllClose(self.evaluate(out), expected_out) class Conv1DTest(test_utils.TestCase, parameterized.TestCase): @parameterized.parameters(True, False) def testComputationPaddingSame(self, with_bias): expected_out = [2, 3, 3, 3, 2] conv1 = conv.Conv1D( output_channels=1, kernel_shape=3, stride=1, padding="SAME", with_bias=with_bias, **create_constant_initializers(1.0, 1.0, with_bias)) out = conv1(tf.ones([1, 5, 1], dtype=tf.float32)) self.assertEqual(out.shape, [1, 5, 1]) out = tf.squeeze(out, axis=(0, 2)) expected_out = np.asarray(expected_out, dtype=np.float32) if with_bias: expected_out += 1 self.assertAllClose(self.evaluate(out), expected_out) @parameterized.parameters(True, False) def testComputationPaddingValid(self, with_bias): expected_out = [3, 3, 3] expected_out = np.asarray(expected_out, dtype=np.float32) if with_bias: expected_out += 1 conv1 = conv.Conv1D( output_channels=1, kernel_shape=3, stride=1, padding="VALID", with_bias=with_bias, **create_constant_initializers(1.0, 1.0, with_bias)) out = conv1(tf.ones([1, 5, 1], dtype=tf.float32)) self.assertEqual(out.shape, [1, 3, 1]) out = tf.squeeze(out, axis=(0, 2)) self.assertAllClose(self.evaluate(out), expected_out) class Conv3DTest(test_utils.TestCase, parameterized.TestCase): @parameterized.parameters(True, False) def testComputationPaddingSame(self, with_bias): expected_out = np.asarray([ 9, 13, 13, 13, 9, 13, 19, 19, 19, 13, 13, 19, 19, 19, 13, 13, 19, 19, 19, 13, 9, 13, 13, 13, 9, 13, 19, 19, 19, 13, 19, 28, 28, 28, 19, 19, 28, 28, 28, 19, 19, 28, 28, 28, 19, 13, 19, 19, 19, 13, 13, 19, 19, 19, 13, 19, 28, 28, 28, 19, 19, 28, 28, 28, 19, 19, 28, 28, 28, 19, 13, 19, 19, 19, 13, 13, 19, 19, 19, 13, 19, 28, 28, 28, 19, 19, 28, 28, 28, 19, 19, 28, 28, 28, 19, 13, 19, 19, 19, 13, 9, 13, 13, 13, 9, 13, 19, 19, 19, 13, 13, 19, 19, 19, 13, 13, 19, 19, 19, 13, 9, 13, 13, 13, 9 ]).reshape((5, 5, 5)) if not with_bias: expected_out -= 1 conv1 = conv.Conv3D( output_channels=1, kernel_shape=3, stride=1, padding="SAME", with_bias=with_bias, **create_constant_initializers(1.0, 1.0, with_bias)) out = conv1(tf.ones([1, 5, 5, 5, 1], dtype=tf.float32)) self.assertEqual(out.shape, [1, 5, 5, 5, 1]) out = tf.squeeze(out, axis=(0, 4)) self.assertAllClose(self.evaluate(out), expected_out) @parameterized.parameters(True, False) def testComputationPaddingValid(self, with_bias): expected_out = np.asarray([ 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28 ]).reshape((3, 3, 3)) if not with_bias: expected_out -= 1 conv1 = conv.Conv3D( output_channels=1, kernel_shape=3, stride=1, padding="VALID", with_bias=with_bias, **create_constant_initializers(1.0, 1.0, with_bias)) out = conv1(tf.ones([1, 5, 5, 5, 1], dtype=tf.float32)) self.assertEqual(out.shape, [1, 3, 3, 3, 1]) out = tf.squeeze(out, axis=(0, 4)) self.assertAllClose(self.evaluate(out), expected_out) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/conv_transpose.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Transpose convolutional module.""" from typing import Optional, Sequence, Union import numpy as np from sonnet.src import base from sonnet.src import initializers from sonnet.src import once from sonnet.src import types from sonnet.src import utils import tensorflow as tf def smart_concat(v1, v2): if isinstance(v1, tf.Tensor) or isinstance(v2, tf.Tensor): return tf.concat([v1, v2], 0) else: return v1 + v2 def smart_lambda(func, v1, v2): if isinstance(v1, tf.Tensor) or isinstance(v2, tf.Tensor): return func(v1, v2) else: return [func(x, y) for (x, y) in zip(v1, v2)] class ConvNDTranspose(base.Module): """An N-dimensional transpose convolutional module. Attributes: w: Weight variable. Note is `None` until module is connected. b: Biases variable. Note is `None` until module is connected. input_shape: The input shape of the first set of inputs. Note is `None` until module is connected. """ def __init__(self, num_spatial_dims: int, output_channels: int, kernel_shape: Union[int, Sequence[int]], output_shape: Optional[types.ShapeLike] = None, stride: Union[int, Sequence[int]] = 1, rate: Union[int, Sequence[int]] = 1, padding: str = "SAME", with_bias: bool = True, w_init: Optional[initializers.Initializer] = None, b_init: Optional[initializers.Initializer] = None, data_format: Optional[str] = None, name: Optional[str] = None): """Constructs a `ConvNDTranspose` module. Args: num_spatial_dims: Number of spatial dimensions of the input. output_channels: Number of output channels. kernel_shape: Sequence of integers (of length num_spatial_dims), or an integer representing kernel shape. `kernel_shape` will be expanded to define a kernel size in all dimensions. output_shape: Output shape of the spatial dimensions of a transpose convolution. Can be either an iterable of integers or a `TensorShape` of length `num_spatial_dims`. If a `None` value is given, a default shape is automatically calculated. stride: Sequence of integers (of length num_spatial_dims), or an integer. `stride` will be expanded to define stride in all dimensions. rate: Sequence of integers (of length num_spatial_dims), or integer that is used to define dilation rate in all dimensions. 1 corresponds to standard ND convolution, `rate > 1` corresponds to dilated convolution. padding: Padding algorithm, either "SAME" or "VALID". with_bias: Boolean, whether to include bias parameters. Default `True`. w_init: Optional initializer for the weights. By default the weights are initialized truncated random normal values with a standard deviation of `1 / sqrt(input_feature_size)`, which is commonly used when the inputs are zero centered (see https://arxiv.org/abs/1502.03167v3). b_init: Optional initializer for the bias. By default the bias is initialized to zero. data_format: The data format of the input. name: Name of the module. """ super().__init__(name=name) if not 1 <= num_spatial_dims <= 3: raise ValueError( "We only support transpose convolution operations for " "num_spatial_dims=1, 2 or 3, received num_spatial_dims={}.".format( num_spatial_dims)) self._num_spatial_dims = num_spatial_dims self._output_channels = output_channels self._kernel_shape = kernel_shape self._output_shape = output_shape self._stride = stride self._rate = rate if padding == "SAME" or padding == "VALID": self._padding = padding else: raise TypeError("ConvNDTranspose only takes string padding, please " "provide either `SAME` or `VALID`.") self._data_format = data_format self._channel_index = utils.get_channel_index(data_format) self._with_bias = with_bias self._w_init = w_init if with_bias: self._b_init = b_init if b_init is not None else initializers.Zeros() elif b_init is not None: raise ValueError("When not using a bias the b_init must be None.") def __call__(self, inputs): self._initialize(inputs) if self._output_shape is None: output_shape = self._get_output_shape(inputs) if self._channel_index == 1: output_shape = smart_concat([self._output_channels], output_shape) else: output_shape = smart_concat(output_shape, [self._output_channels]) else: output_shape = self._output_shape output_shape = smart_concat([tf.shape(inputs)[0]], output_shape) outputs = tf.nn.conv_transpose( input=inputs, filters=self.w, output_shape=output_shape, strides=self._stride, padding=self._padding, data_format=self._data_format, dilations=self._rate, name=None) if self._with_bias: outputs = tf.nn.bias_add(outputs, self.b, data_format=self._data_format) return outputs @once.once def _initialize(self, inputs): utils.assert_rank(inputs, self._num_spatial_dims + 2) self.input_channels = inputs.shape[self._channel_index] if self.input_channels is None: raise ValueError("The number of input channels must be known") self._dtype = inputs.dtype if self._output_shape is not None: if len(self._output_shape) != self._num_spatial_dims: raise ValueError( "The output_shape must be of length {} but instead was {}.".format( self._num_spatial_dims, len(self._output_shape))) if self._channel_index == 1: self._output_shape = [self._output_channels] + list(self._output_shape) else: self._output_shape = list(self._output_shape) + [self._output_channels] self.w = self._make_w() if self._with_bias: self.b = tf.Variable( self._b_init((self._output_channels,), self._dtype), name="b") def _make_w(self): """Makes and returns the variable representing the weight.""" kernel_shape = utils.replicate(self._kernel_shape, self._num_spatial_dims, "kernel_shape") weight_shape = kernel_shape + (self._output_channels, self.input_channels) if self._w_init is None: # See https://arxiv.org/abs/1502.03167v3. fan_in_shape = kernel_shape + (self.input_channels,) stddev = 1 / np.sqrt(np.prod(fan_in_shape)) self._w_init = initializers.TruncatedNormal(stddev=stddev) return tf.Variable(self._w_init(weight_shape, self._dtype), name="w") def _get_output_shape(self, inputs): input_shape = inputs.shape if inputs.shape.is_fully_defined() else tf.shape( inputs) if self._channel_index == 1: input_size = input_shape[2:] else: input_size = input_shape[1:-1] stride = utils.replicate(self._stride, self._num_spatial_dims, "stride") output_shape = smart_lambda(lambda x, y: x * y, input_size, stride) if self._padding == "VALID": kernel_shape = utils.replicate(self._kernel_shape, self._num_spatial_dims, "kernel_shape") rate = utils.replicate(self._rate, self._num_spatial_dims, "rate") effective_kernel_shape = [ (shape - 1) * rate + 1 for (shape, rate) in zip(kernel_shape, rate) ] output_shape = smart_lambda(lambda x, y: x + y - 1, output_shape, effective_kernel_shape) return output_shape class Conv1DTranspose(ConvNDTranspose): """A 1D transpose convolutional module.""" def __init__(self, output_channels: int, kernel_shape: Union[int, Sequence[int]], output_shape: Optional[types.ShapeLike] = None, stride: Union[int, Sequence[int]] = 1, rate: Union[int, Sequence[int]] = 1, padding: str = "SAME", with_bias: bool = True, w_init: Optional[initializers.Initializer] = None, b_init: Optional[initializers.Initializer] = None, data_format: str = "NWC", name: Optional[str] = None): """Constructs a `Conv1DTranspose` module. Args: output_channels: Number of output channels. kernel_shape: Sequence of integers (of length 1), or an integer representing kernel shape. `kernel_shape` will be expanded to define a kernel size in all dimensions. output_shape: Output shape of the spatial dimensions of a transpose convolution. Can be either an integer or an iterable of integers or `Dimension`s, or a `TensorShape` (of length 1). If a `None` value is given, a default shape is automatically calculated. stride: Sequence of integers (of length 1), or an integer. `stride` will be expanded to define stride in all dimensions. rate: Sequence of integers (of length 1), or integer that is used to define dilation rate in all dimensions. 1 corresponds to standard 1D convolution, `rate > 1` corresponds to dilated convolution. padding: Padding algorithm, either "SAME" or "VALID". with_bias: Boolean, whether to include bias parameters. Default `True`. w_init: Optional initializer for the weights. By default the weights are initialized truncated random normal values with a standard deviation of `1 / sqrt(input_feature_size)`, which is commonly used when the inputs are zero centered (see https://arxiv.org/abs/1502.03167v3). b_init: Optional initializer for the bias. By default the bias is initialized to zero. data_format: The data format of the input. name: Name of the module. """ super().__init__( num_spatial_dims=1, output_channels=output_channels, kernel_shape=kernel_shape, output_shape=output_shape, stride=stride, rate=rate, padding=padding, with_bias=with_bias, w_init=w_init, b_init=b_init, data_format=data_format, name=name) class Conv2DTranspose(ConvNDTranspose): """A 2D transpose convolutional module.""" def __init__(self, output_channels: int, kernel_shape: Union[int, Sequence[int]], output_shape: Optional[types.ShapeLike] = None, stride: Union[int, Sequence[int]] = 1, rate: Union[int, Sequence[int]] = 1, padding: str = "SAME", with_bias: bool = True, w_init: Optional[initializers.Initializer] = None, b_init: Optional[initializers.Initializer] = None, data_format: str = "NHWC", name: Optional[str] = None): """Constructs a `Conv2DTranspose` module. Args: output_channels: An integer, The number of output channels. kernel_shape: Sequence of integers (of length 2), or an integer representing kernel shape. `kernel_shape` will be expanded to define a kernel size in all dimensions. output_shape: Output shape of the spatial dimensions of a transpose convolution. Can be either an integer or an iterable of integers or `Dimension`s, or a `TensorShape` (of length 2). If a `None` value is given, a default shape is automatically calculated. stride: Sequence of integers (of length 2), or an integer. `stride` will be expanded to define stride in all dimensions. rate: Sequence of integers (of length 2), or integer that is used to define dilation rate in all dimensions. 1 corresponds to standard 2D convolution, `rate > 1` corresponds to dilated convolution. padding: Padding algorithm, either "SAME" or "VALID". with_bias: Boolean, whether to include bias parameters. Default `True`. w_init: Optional initializer for the weights. By default the weights are initialized truncated random normal values with a standard deviation of `1 / sqrt(input_feature_size)`, which is commonly used when the inputs are zero centered (see https://arxiv.org/abs/1502.03167v3). b_init: Optional initializer for the bias. By default the bias is initialized to zero. data_format: The data format of the input. name: Name of the module. """ super().__init__( num_spatial_dims=2, output_channels=output_channels, kernel_shape=kernel_shape, output_shape=output_shape, stride=stride, rate=rate, padding=padding, with_bias=with_bias, w_init=w_init, b_init=b_init, data_format=data_format, name=name) class Conv3DTranspose(ConvNDTranspose): """A 3D transpose convolutional module.""" def __init__(self, output_channels: int, kernel_shape: Union[int, Sequence[int]], output_shape: Optional[types.ShapeLike] = None, stride: Union[int, Sequence[int]] = 1, rate: Union[int, Sequence[int]] = 1, padding: str = "SAME", with_bias: bool = True, w_init: Optional[initializers.Initializer] = None, b_init: Optional[initializers.Initializer] = None, data_format: str = "NDHWC", name: Optional[str] = None): """Constructs a `Conv3DTranspose` module. Args: output_channels: An integer, The number of output channels. kernel_shape: Sequence of integers (of length 3), or an integer representing kernel shape. `kernel_shape` will be expanded to define a kernel size in all dimensions. output_shape: Output shape of the spatial dimensions of a transpose convolution. Can be either an integer or an iterable of integers or `Dimension`s, or a `TensorShape` (of length 3). If a None value is given, a default shape is automatically calculated. stride: Sequence of integers (of length 3), or an integer. `stride` will be expanded to define stride in all dimensions. rate: Sequence of integers (of length 3), or integer that is used to define dilation rate in all dimensions. 1 corresponds to standard 3D convolution, `rate > 1` corresponds to dilated convolution. padding: Padding algorithm, either "SAME" or "VALID". with_bias: Boolean, whether to include bias parameters. Default `True`. w_init: Optional initializer for the weights. By default the weights are initialized truncated random normal values with a standard deviation of `1 / sqrt(input_feature_size)`, which is commonly used when the inputs are zero centered (see https://arxiv.org/abs/1502.03167v3). b_init: Optional initializer for the bias. By default the bias is initialized to zero. data_format: The data format of the input. name: Name of the module. """ super().__init__( num_spatial_dims=3, output_channels=output_channels, kernel_shape=kernel_shape, output_shape=output_shape, stride=stride, rate=rate, padding=padding, with_bias=with_bias, w_init=w_init, b_init=b_init, data_format=data_format, name=name) ================================================ FILE: sonnet/src/conv_transpose_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.conv_transpose.""" import itertools from absl.testing import parameterized import numpy as np from sonnet.src import conv_transpose from sonnet.src import initializers as lib_initializers from sonnet.src import test_utils import tensorflow as tf def create_constant_initializers(w, b, with_bias): if with_bias: return { "w_init": lib_initializers.Constant(w), "b_init": lib_initializers.Constant(b) } else: return {"w_init": lib_initializers.Constant(w)} class ConvTransposeTest(test_utils.TestCase, parameterized.TestCase): @parameterized.parameters(0, 4) def testIncorrectN(self, n): with self.assertRaisesRegex( ValueError, "only support transpose convolution operations for num_spatial_dims"): conv_transpose.ConvNDTranspose( num_spatial_dims=n, output_channels=1, output_shape=None, kernel_shape=3, data_format="NHWC") def testIncorrectPadding(self): with self.assertRaisesRegex( TypeError, "ConvNDTranspose only takes string padding, please provide either"): conv_transpose.ConvNDTranspose( 2, output_channels=1, kernel_shape=3, padding=None) def testBiasInitNoBias(self): with self.assertRaisesRegex( ValueError, "When not using a bias the b_init must be None."): conv_transpose.ConvNDTranspose( 2, output_channels=1, kernel_shape=3, with_bias=False, b_init=lib_initializers.Ones(), data_format="NHWC") def testIncorrectOutputShape(self): c = conv_transpose.ConvNDTranspose( num_spatial_dims=2, output_channels=3, kernel_shape=2, output_shape=[1], data_format="NHWC") with self.assertRaisesRegex( ValueError, "The output_shape must be of length 2 but instead was 1."): c(tf.ones([3, 5, 5, 3])) @parameterized.parameters(*itertools.product( [True, False], # with_bias ["SAME", "VALID"])) # padding def testGraphConv(self, with_bias, padding): conv1 = conv_transpose.ConvNDTranspose( num_spatial_dims=2, output_channels=1, output_shape=None, kernel_shape=3, stride=1, padding=padding, with_bias=with_bias, data_format="NHWC", **create_constant_initializers(1.0, 1.0, with_bias)) conv2 = conv_transpose.ConvNDTranspose( num_spatial_dims=2, output_channels=1, output_shape=None, kernel_shape=3, stride=1, padding=padding, with_bias=with_bias, data_format="NHWC", **create_constant_initializers(1.0, 1.0, with_bias)) defun_conv = tf.function(conv2) iterations = 5 for _ in range(iterations): x = tf.random.uniform([1, 3, 3, 1]) y1 = conv1(x) y2 = defun_conv(x) self.assertAllClose(self.evaluate(y1), self.evaluate(y2), atol=1e-4) def testUnknownBatchSizeNHWC(self): x = tf.TensorSpec([None, 5, 5, 3], dtype=tf.float32) c = conv_transpose.ConvNDTranspose( num_spatial_dims=2, output_channels=2, kernel_shape=3, data_format="NHWC") defun_conv = tf.function(c).get_concrete_function(x) out1 = defun_conv(tf.ones([3, 5, 5, 3])) self.assertEqual(out1.shape, [3, 5, 5, 2]) out2 = defun_conv(tf.ones([5, 5, 5, 3])) self.assertEqual(out2.shape, [5, 5, 5, 2]) def testUnknownBatchSizeNCHW(self): if self.primary_device == "CPU": self.skipTest("NCHW not supported on CPU") x = tf.TensorSpec([None, 3, 5, 5], dtype=tf.float32) c = conv_transpose.ConvNDTranspose( num_spatial_dims=2, output_channels=2, kernel_shape=3, data_format="NCHW") defun_conv = tf.function(c).get_concrete_function(x) out1 = defun_conv(tf.ones([3, 3, 5, 5])) self.assertEqual(out1.shape, [3, 2, 5, 5]) out2 = defun_conv(tf.ones([5, 3, 5, 5])) self.assertEqual(out2.shape, [5, 2, 5, 5]) def testUnknownShapeDims(self): x = tf.TensorSpec([3, None, None, 3], dtype=tf.float32) c = conv_transpose.ConvNDTranspose( num_spatial_dims=2, output_channels=2, kernel_shape=3, data_format="NHWC") defun_conv = tf.function(c).get_concrete_function(x) out1 = defun_conv(tf.ones([3, 5, 5, 3])) self.assertEqual(out1.shape, [3, 5, 5, 2]) out1 = defun_conv(tf.ones([3, 3, 3, 3])) self.assertEqual(out1.shape, [3, 3, 3, 2]) def testGivenOutputShape(self): c = conv_transpose.ConvNDTranspose( num_spatial_dims=2, output_channels=2, kernel_shape=3, output_shape=[5, 5], data_format="NHWC") out1 = c(tf.ones([3, 5, 5, 3])) self.assertEqual(out1.shape, [3, 5, 5, 2]) @parameterized.parameters(True, False) def testUnknownChannels(self, autograph): x = tf.TensorSpec([3, 3, 3, None], dtype=tf.float32) c = conv_transpose.ConvNDTranspose( num_spatial_dims=2, output_channels=1, kernel_shape=3, data_format="NHWC") defun_conv = tf.function(c, autograph=autograph) with self.assertRaisesRegex(ValueError, "The number of input channels must be known"): defun_conv.get_concrete_function(x) @parameterized.parameters( (1, (3,), 128, 5, "NWC"), (2, (4, 4), 64, 3, "NHWC"), (3, (4, 4, 4), 64, 3, "NDHWC")) def testInitializerVariance(self, num_spatial_dims, kernel_shape, in_channels, output_channels, data_format): inputs = tf.random.uniform([16] + ([32] * num_spatial_dims) + [in_channels]) c = conv_transpose.ConvNDTranspose( num_spatial_dims=num_spatial_dims, kernel_shape=kernel_shape, output_channels=output_channels, data_format=data_format) c(inputs) actual_std = c.w.numpy().std() expected_std = 1 / (np.sqrt(np.prod(kernel_shape + (in_channels,)))) # This ratio of the error compared to the expected std might be somewhere # around 0.15 normally. We check it is not > 0.5, as that would indicate # something seriously wrong (ie the previous buggy initialization). rel_diff = np.abs(actual_std - expected_std) / expected_std self.assertLess(rel_diff, 0.5) class Conv2DTransposeTest(test_utils.TestCase, parameterized.TestCase): @parameterized.parameters(True, False) def testComputationPaddingSame(self, with_bias): expected_out = [[4, 6, 4], [6, 9, 6], [4, 6, 4]] expected_out = np.asarray(expected_out, dtype=np.float32) if with_bias: expected_out += 1 conv_transpose1 = conv_transpose.Conv2DTranspose( output_channels=1, output_shape=None, kernel_shape=3, stride=1, padding="SAME", with_bias=with_bias, **create_constant_initializers(1.0, 1.0, with_bias)) out = conv_transpose1(tf.ones([1, 3, 3, 1], dtype=tf.float32)) self.assertEqual(out.shape, [1, 3, 3, 1]) out = tf.squeeze(out, axis=(0, 3)) self.assertAllClose(self.evaluate(out), expected_out) @parameterized.parameters(True, False) def testComputationPaddingValid(self, with_bias): expected_out = [[1, 2, 3, 2, 1], [2, 4, 6, 4, 2], [3, 6, 9, 6, 3], [2, 4, 6, 4, 2], [1, 2, 3, 2, 1]] expected_out = np.asarray(expected_out, dtype=np.float32) if with_bias: expected_out += 1 conv1 = conv_transpose.Conv2DTranspose( output_channels=1, output_shape=None, kernel_shape=3, stride=1, padding="VALID", with_bias=with_bias, **create_constant_initializers(1.0, 1.0, with_bias)) out = conv1(tf.ones([1, 3, 3, 1], dtype=tf.float32)) self.assertEqual(out.shape, [1, 5, 5, 1]) out = tf.squeeze(out, axis=(0, 3)) self.assertAllClose(self.evaluate(out), expected_out) def testShapeDilated(self): if "CPU" == self.primary_device: self.skipTest("Not supported on CPU") conv1 = conv_transpose.Conv2DTranspose( output_channels=1, output_shape=None, kernel_shape=3, rate=2, padding="VALID") out = conv1(tf.ones([1, 3, 3, 1])) self.assertEqual(out.shape, [1, 7, 7, 1]) class Conv1DTransposeTest(test_utils.TestCase, parameterized.TestCase): @parameterized.parameters(True, False) def testComputationPaddingSame(self, with_bias): expected_out = [2, 3, 2] expected_out = np.asarray(expected_out, dtype=np.float32) if with_bias: expected_out += 1 conv1 = conv_transpose.Conv1DTranspose( output_channels=1, output_shape=None, kernel_shape=3, stride=1, padding="SAME", with_bias=with_bias, **create_constant_initializers(1.0, 1.0, with_bias)) out = conv1(tf.ones([1, 3, 1], dtype=tf.float32)) self.assertEqual(out.shape, [1, 3, 1]) out = tf.squeeze(out, axis=(0, 2)) self.assertAllClose(self.evaluate(out), expected_out) @parameterized.parameters(True, False) def testComputationPaddingValid(self, with_bias): expected_out = [1, 2, 3, 2, 1] expected_out = np.asarray(expected_out, dtype=np.float32) if with_bias: expected_out += 1 conv1 = conv_transpose.Conv1DTranspose( output_channels=1, output_shape=None, kernel_shape=3, stride=1, padding="VALID", with_bias=with_bias, **create_constant_initializers(1.0, 1.0, with_bias)) out = conv1(tf.ones([1, 3, 1], dtype=tf.float32)) self.assertEqual(out.shape, [1, 5, 1]) out = tf.squeeze(out, axis=(0, 2)) self.assertAllClose(self.evaluate(out), expected_out) class Conv3DTransposeTest(test_utils.TestCase, parameterized.TestCase): @parameterized.parameters(True, False) def testComputationPaddingSame(self, with_bias): expected_out = np.asarray([ 8, 12, 8, 12, 18, 12, 8, 12, 8, 12, 18, 12, 18, 27, 18, 12, 18, 12, 8, 12, 8, 12, 18, 12, 8, 12, 8 ]).reshape((3, 3, 3)) if with_bias: expected_out += 1 conv_transpose1 = conv_transpose.Conv3DTranspose( output_channels=1, output_shape=None, kernel_shape=3, stride=1, padding="SAME", with_bias=with_bias, **create_constant_initializers(1.0, 1.0, with_bias)) out = conv_transpose1(tf.ones([1, 3, 3, 3, 1], dtype=tf.float32)) self.assertEqual(out.shape, [1, 3, 3, 3, 1]) out = tf.squeeze(out, axis=(0, 4)) self.assertAllClose(self.evaluate(out), expected_out) @parameterized.parameters(True, False) def testComputationPaddingValid(self, with_bias): expected_out = np.asarray([ 1, 2, 3, 2, 1, 2, 4, 6, 4, 2, 3, 6, 9, 6, 3, 2, 4, 6, 4, 2, 1, 2, 3, 2, 1, 2, 4, 6, 4, 2, 4, 8, 12, 8, 4, 6, 12, 18, 12, 6, 4, 8, 12, 8, 4, 2, 4, 6, 4, 2, 3, 6, 9, 6, 3, 6, 12, 18, 12, 6, 9, 18, 27, 18, 9, 6, 12, 18, 12, 6, 3, 6, 9, 6, 3, 2, 4, 6, 4, 2, 4, 8, 12, 8, 4, 6, 12, 18, 12, 6, 4, 8, 12, 8, 4, 2, 4, 6, 4, 2, 1, 2, 3, 2, 1, 2, 4, 6, 4, 2, 3, 6, 9, 6, 3, 2, 4, 6, 4, 2, 1, 2, 3, 2, 1. ]).reshape((5, 5, 5)) if with_bias: expected_out += 1 conv1 = conv_transpose.Conv3DTranspose( output_channels=1, output_shape=None, kernel_shape=3, stride=1, padding="VALID", with_bias=with_bias, **create_constant_initializers(1.0, 1.0, with_bias)) out = conv1(tf.ones([1, 3, 3, 3, 1], dtype=tf.float32)) self.assertEqual(out.shape, [1, 5, 5, 5, 1]) out = tf.squeeze(out, axis=(0, 4)) self.assertAllClose(self.evaluate(out), expected_out) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/custom_getter.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Custom getter for module members.""" import contextlib from typing import Any, Callable, ContextManager, Iterable, Optional, Type from sonnet.src import base import tensorflow as tf import tree _DEFAULT_CLASSES = [base.Module] @contextlib.contextmanager def _patch_getattribute(cls, new_getattribute): orig_getattribute = cls.__getattribute__ # pytype: disable=attribute-error cls.__getattribute__ = new_getattribute try: yield finally: cls.__getattribute__ = orig_getattribute def _custom_getter( getter: Callable[[Any], Any], classes: Optional[Iterable[Type[Any]]] = None, instances: Optional[Iterable[Any]] = None) -> ContextManager[Any]: """Applies the given `getter` when getting members of given `classes`. For example: >>> class X: ... values = [1, 2] >>> x = X() >>> x.values [1, 2] >>> with _custom_getter(lambda x: x + [3], classes=[X]): ... x.values [1, 2, 3] >>> with _custom_getter(lambda x: x + [3], instances={x}): ... x.values [1, 2, 3] >>> x.values [1, 2] Args: getter: A callable to apply to each element of the class members. classes: The classes in which the getter is applied. If `None`, defaults to `set(o.__class__ for o in instances)`. If `classes and `instances` are both `None`, defaults to `[Module]`. instances: The instances in which the getter is applied. If `None`, the getter will apply in all instances of `classes`. Returns: A context manager in which the custom getter is active. """ # Workaround for the fact that we can't annotate the type as `Collection` in # Python < 3.6. if instances is not None: instances = frozenset(instances) if classes is None: if instances is None: classes = _DEFAULT_CLASSES else: classes = frozenset(o.__class__ for o in instances) stack = contextlib.ExitStack() for cls in classes: orig_getattribute = cls.__getattribute__ # pytype: disable=attribute-error def new_getattribute(obj, name, orig_getattribute=orig_getattribute): attr = orig_getattribute(obj, name) if (instances is None) or (obj in instances): return getter(attr) else: return attr stack.enter_context(_patch_getattribute(cls, new_getattribute)) return stack def custom_variable_getter( getter: Callable[[tf.Variable], Any], classes: Optional[Iterable[Type[Any]]] = None, instances: Optional[Iterable[Any]] = None) -> ContextManager[Any]: """Applies the given `getter` when getting variables of given `classes`. If a member is a nested structure containing any variable, `getter` will be applied to each variable in the nest. For example: >>> class Times2(snt.Module): ... def __init__(self): ... super(Times2, self).__init__() ... self.v = tf.Variable(2.) ... ... def __call__(self, x): ... return x * self.v >>> x = 42. >>> times2 = Times2() >>> with tf.GradientTape() as tape: ... y = times2(x) >>> assert tape.gradient(y, times2.v).numpy() == x >>> with custom_variable_getter(tf.stop_gradient): ... with tf.GradientTape() as tape: ... y = times2(x) >>> assert tape.gradient(y, times2.v) is None >>> with tf.GradientTape() as tape: ... y = times2(x) >>> assert tape.gradient(y, times2.v).numpy() == x Args: getter: A callable to apply to each variable of the class. classes: The classes in which the getter is applied. If `None`, defaults to `set(o.__class__ for o in instances)`. If `classes and `instances` are both `None`, defaults to `[Module]`. instances: The instances in which the getter is applied. If `None`, the getter will apply in all instances of `classes`. Returns: A context manager in which the custom getter is active. """ def wrapped_getter(x): x_flat = tree.flatten(x) if any(_is_variable(it) for it in x_flat): return tree.unflatten_as( x, [getter(it) if _is_variable(it) else it for it in x_flat]) else: return x return _custom_getter(wrapped_getter, classes=classes, instances=instances) def _is_variable(x): return isinstance(x, tf.Variable) ================================================ FILE: sonnet/src/custom_getter_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ import doctest from sonnet.src import base from sonnet.src import custom_getter from sonnet.src import test_utils import tensorflow as tf class CustomVariableGetterTest(test_utils.TestCase): def testDoesNotModifyNonVariables(self): class MyModule(base.Module): v = tf.Variable(21.) d = {} my_module = MyModule() self.assertEqual(21., self.evaluate(my_module.v)) with custom_getter.custom_variable_getter(lambda v: v * 2): self.assertEqual(42., self.evaluate(my_module.v)) my_module.d["foo"] = "bar" self.assertEqual(21., self.evaluate(my_module.v)) self.assertEqual("bar", my_module.d["foo"]) class DoctestTest(test_utils.TestCase): def testDoctest(self): num_failed, num_attempted = doctest.testmod( custom_getter, extraglobs={"snt": base}) self.assertGreater(num_attempted, 0, "No doctests found.") self.assertEqual(num_failed, 0, "{} doctests failed".format(num_failed)) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/deferred.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Enables module construction to be deferred.""" from sonnet.src import base class Deferred(base.Module): """Defers the construction of another module until the first call. Deferred can be used to declare modules that depend on computed properties of other modules before those modules are defined. This allows users to separate the declaration and use of modules. For example at the start of your program you can declare two modules which are coupled: >>> encoder = snt.Linear(64) >>> decoder = snt.Deferred(lambda: snt.Linear(encoder.input_size)) Later you can use these naturally (note: that using `decoder` first would cause an error since `encoder.input_size` is only defined after `encoder` has been called): >>> x = tf.ones([8, 32]) >>> y = encoder(x) >>> z = decoder(y) # Constructs the Linear encoder by calling the lambda. The result will satisfy the following conditions: >>> assert x.shape == z.shape >>> assert y.shape == [8, 64] >>> assert decoder.input_size == encoder.output_size >>> assert decoder.output_size == encoder.input_size """ def __init__(self, constructor, call_methods=("__call__",), name=None): """Initializes the `Deferred` module. Args: constructor: A no argument callable which constructs the module to defer to. The first time one of the `call_methods` are called the constructor will be run and then the constructed module will be called with the same method and arguments as the deferred module. call_methods: Methods which should trigger construction of the target module. The default value configures this module to construct the first time `__call__` is run. If you want to add methods other than call you should explicitly pass them (optionally), for example `call_methods=("__call__", "encode", "decode")`. name: Name for the deferred module. """ super().__init__(name=name) self._constructor = constructor self._target = None for call_method in call_methods: if call_method == "__call__": # Has to be handled separately because __call__ cannot be overridden at # the instance level. # See: https://docs.python.org/3/reference/datamodel.html#special-lookup continue setattr(self, call_method, _materialize_then_call(self, call_method)) @property @base.no_name_scope def target(self): """Returns the target module. If the constructor has not already run this will trigger construction. Subsequent calls to `target` will return the same instance. Returns: A `Module` instance as created by `self.constructor()` . """ if self._target is None: self._target = self._constructor() self._constructor = None return self._target @base.no_name_scope def __call__(self, *args, **kwargs): return self.target(*args, **kwargs) # pylint: disable=not-callable def __str__(self): return "Deferred({})".format(str(self.target)) def __repr__(self): return "Deferred({})".format(repr(self.target)) def __getattr__(self, name): if name != "_target" and hasattr(self, "_target"): if self._target is not None: return getattr(self._target, name) raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, name)) def __setattr__(self, name, value): if name != "_target" and hasattr(self, "_target"): if self._target is not None: setattr(self._target, name, value) return super().__setattr__(name, value) def __delattr__(self, name): if name != "_target" and hasattr(self, "_target"): if self._target is not None: return delattr(self._target, name) super().__delattr__(name) def _materialize_then_call(module, method_name): def wrapped(*args, **kwargs): return getattr(module.target, method_name)(*args, **kwargs) return wrapped ================================================ FILE: sonnet/src/deferred_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.deferred.""" from sonnet.src import base from sonnet.src import deferred from sonnet.src import test_utils import tensorflow as tf class DeferredTest(test_utils.TestCase): def test_target(self): target = ExampleModule() mod = deferred.Deferred(lambda: target) self.assertIs(mod.target, target) def test_only_computes_target_once(self): target = ExampleModule() targets = [target] mod = deferred.Deferred(targets.pop) for _ in range(10): # If target was recomputed more than once pop should fail. self.assertIs(mod.target, target) self.assertEmpty(targets) def test_attr_forwarding_fails_before_construction(self): mod = deferred.Deferred(ExampleModule) with self.assertRaises(AttributeError): getattr(mod, "foo") def test_getattr(self): mod = deferred.Deferred(ExampleModule) mod() self.assertIs(mod.w, mod.target.w) def test_setattr(self): mod = deferred.Deferred(ExampleModule) mod() new_w = tf.ones_like(mod.w) mod.w = new_w self.assertIs(mod.w, new_w) self.assertIs(mod.target.w, new_w) def test_setattr_on_target(self): mod = deferred.Deferred(ExampleModule) mod() w = tf.ones_like(mod.w) mod.w = None # Assigning to the target directly should reflect in the parent. mod.target.w = w self.assertIs(mod.w, w) self.assertIs(mod.target.w, w) def test_delattr(self): mod = deferred.Deferred(ExampleModule) mod() self.assertTrue(hasattr(mod.target, "w")) del mod.w self.assertFalse(hasattr(mod.target, "w")) def test_alternative_forward(self): mod = deferred.Deferred(AlternativeForwardModule, call_methods=("forward",)) self.assertEqual(mod.forward(), 42) def test_alternative_forward_call_type_error(self): mod = deferred.Deferred(AlternativeForwardModule, call_methods=("forward",)) msg = "'AlternativeForwardModule' object is not callable" with self.assertRaisesRegex(TypeError, msg): mod() def test_name_scope(self): mod = deferred.Deferred(ExampleModule) mod() self.assertEqual(mod.name_scope.name, "deferred/") self.assertEqual(mod.target.name_scope.name, "example_module/") def test_str(self): m = ExampleModule() d = deferred.Deferred(lambda: m) self.assertEqual("Deferred(%s)" % m, str(d)) def test_repr(self): m = ExampleModule() d = deferred.Deferred(lambda: m) self.assertEqual("Deferred(%r)" % m, repr(d)) class ExampleModule(base.Module): def __init__(self): super().__init__() self.w = tf.Variable(1.) def __str__(self): return "ExampleModuleStr" def __repr__(self): return "ExampleModuleRepr" def __call__(self): return self.w class AlternativeForwardModule(base.Module): def forward(self): return 42 if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/depthwise_conv.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Depth-wise convolutional module.""" from typing import Optional, Sequence, Union import numpy as np from sonnet.src import base from sonnet.src import initializers from sonnet.src import once from sonnet.src import utils import tensorflow as tf class DepthwiseConv2D(base.Module): """Spatial depth-wise 2D convolution module, including bias. This acts as a light wrapper around the TensorFlow ops `tf.nn.depthwise_conv2d`, abstracting away variable creation and sharing. """ def __init__(self, kernel_shape: Union[int, Sequence[int]], channel_multiplier: int = 1, stride: Union[int, Sequence[int]] = 1, rate: Union[int, Sequence[int]] = 1, padding: str = "SAME", with_bias: bool = True, w_init: Optional[initializers.Initializer] = None, b_init: Optional[initializers.Initializer] = None, data_format: str = "NHWC", name: Optional[str] = None): """Constructs a `DepthwiseConv2D` module. Args: kernel_shape: Sequence of kernel sizes (of length num_spatial_dims), or an integer. `kernel_shape` will be expanded to define a kernel size in all dimensions. channel_multiplier: Number of channels to expand convolution to. Must be an integer greater than 0. When `channel_multiplier` is 1, applies a different filter to each input channel producing one output channel per input channel. Numbers larger than 1 cause multiple different filters to be applied to each input channel, with their outputs being concatenated together, producing `channel_multiplier` * `input_channels` output channels. stride: Sequence of strides (of length num_spatial_dims), or an integer. `stride` will be expanded to define stride in all dimensions. rate: Sequence of dilation rates (of length num_spatial_dims), or integer that is used to define dilation rate in all dimensions. 1 corresponds to standard ND convolution, `rate > 1` corresponds to dilated convolution. padding: Padding to apply to the input. This can either "SAME", "VALID". with_bias: Whether to include bias parameters. Default `True`. w_init: Optional initializer for the weights. By default the weights are initialized truncated random normal values with a standard deviation of `1 / sqrt(input_feature_size)`, which is commonly used when the inputs are zero centered (see https://arxiv.org/abs/1502.03167v3). b_init: Optional initializer for the bias. By default the bias is initialized to zero. data_format: The data format of the input. name: Name of the module. """ super().__init__(name=name) self.channel_multiplier = channel_multiplier self.kernel_shape = kernel_shape self.data_format = data_format self._channel_index = utils.get_channel_index(data_format) stride = utils.replicate(stride, 2, "stride") if self._channel_index == 1: self.stride = (1, 1) + stride else: self.stride = (1,) + stride + (1,) self.rate = utils.replicate(rate, 2, "rate") self.padding = padding self.with_bias = with_bias self.w_init = w_init if with_bias: self.b_init = b_init if b_init is not None else initializers.Zeros() elif b_init is not None: raise ValueError("When not using a bias the b_init must be None.") def __call__(self, inputs: tf.Tensor) -> tf.Tensor: self._initialize(inputs) outputs = tf.nn.depthwise_conv2d(inputs, self.w, strides=self.stride, dilations=self.rate, padding=self.padding, data_format=self.data_format) if self.with_bias: outputs = tf.nn.bias_add(outputs, self.b, data_format=self.data_format) return outputs @once.once def _initialize(self, inputs: tf.Tensor): self.input_channels = inputs.shape[self._channel_index] if self.input_channels is None: raise ValueError("The number of input channels must be known.") dtype = inputs.dtype weight_shape = utils.replicate(self.kernel_shape, 2, "kernel_shape") weight_shape = weight_shape + (self.input_channels, self.channel_multiplier) if self.w_init is None: # See https://arxiv.org/abs/1502.03167v3. fan_in_shape = weight_shape[:2] stddev = 1 / np.sqrt(np.prod(fan_in_shape)) self.w_init = initializers.TruncatedNormal(stddev=stddev) self.w = tf.Variable(self.w_init(weight_shape, dtype), name="w") output_channels = self.input_channels * self.channel_multiplier if self.with_bias: self.b = tf.Variable(self.b_init((output_channels,), dtype), name="b") ================================================ FILE: sonnet/src/depthwise_conv_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.depthwise_conv.""" from absl.testing import parameterized import numpy as np from sonnet.src import depthwise_conv from sonnet.src import initializers from sonnet.src import test_utils import tensorflow as tf def create_constant_initializers(w, b, with_bias): if with_bias: return { "w_init": initializers.Constant(w), "b_init": initializers.Constant(b) } else: return {"w_init": initializers.Constant(w)} class DepthwiseConvTest(test_utils.TestCase, parameterized.TestCase): def testInitializerKeysInvalidWithoutBias(self): with self.assertRaisesRegex(ValueError, "b_init must be None"): depthwise_conv.DepthwiseConv2D( channel_multiplier=1, kernel_shape=3, data_format="NHWC", with_bias=False, b_init=tf.zeros_initializer()) @parameterized.parameters(tf.float32, tf.float64) def testDefaultInitializers(self, dtype): if "TPU" in self.device_types and dtype == tf.float64: self.skipTest("Double precision not supported on TPU.") conv1 = depthwise_conv.DepthwiseConv2D( kernel_shape=16, stride=1, padding="VALID", data_format="NHWC") out = conv1(tf.random.normal([8, 64, 64, 1], dtype=dtype)) self.assertAllEqual(out.shape, [8, 49, 49, 1]) self.assertEqual(out.dtype, dtype) # Note that for unit variance inputs the output is below unit variance # because of the use of the truncated normal initalizer err = 0.2 if self.primary_device == "TPU" else 0.1 self.assertNear(out.numpy().std(), 0.87, err=err) @parameterized.named_parameters(("SamePaddingUseBias", True, "SAME"), ("SamePaddingNoBias", False, "SAME"), ("ValidPaddingNoBias", False, "VALID"), ("ValidPaddingUseBias", True, "VALID")) def testFunction(self, with_bias, padding): conv1 = depthwise_conv.DepthwiseConv2D( channel_multiplier=1, kernel_shape=3, stride=1, padding=padding, with_bias=with_bias, data_format="NHWC", **create_constant_initializers(1.0, 1.0, with_bias)) conv2 = depthwise_conv.DepthwiseConv2D( channel_multiplier=1, kernel_shape=3, stride=1, padding=padding, with_bias=with_bias, data_format="NHWC", **create_constant_initializers(1.0, 1.0, with_bias)) defun_conv = tf.function(conv2) iterations = 5 for _ in range(iterations): x = tf.random.uniform([1, 5, 5, 1]) y1 = conv1(x) y2 = defun_conv(x) self.assertAllClose(self.evaluate(y1), self.evaluate(y2), atol=1e-4) def testUnknownBatchSizeNHWC(self): x = tf.TensorSpec([None, 5, 5, 3], dtype=tf.float32) c = depthwise_conv.DepthwiseConv2D( channel_multiplier=1, kernel_shape=3, data_format="NHWC") defun_conv = tf.function(c).get_concrete_function(x) out1 = defun_conv(tf.ones([3, 5, 5, 3])) self.assertEqual(out1.shape, [3, 5, 5, 3]) out2 = defun_conv(tf.ones([5, 5, 5, 3])) self.assertEqual(out2.shape, [5, 5, 5, 3]) def testUnknownBatchSizeNCHW(self): if self.primary_device == "CPU": self.skipTest("NCHW not supported on CPU") x = tf.TensorSpec([None, 3, 5, 5], dtype=tf.float32) c = depthwise_conv.DepthwiseConv2D( channel_multiplier=1, kernel_shape=3, data_format="NCHW") defun_conv = tf.function(c).get_concrete_function(x) out1 = defun_conv(tf.ones([3, 3, 5, 5])) self.assertEqual(out1.shape, [3, 3, 5, 5]) out2 = defun_conv(tf.ones([5, 3, 5, 5])) self.assertEqual(out2.shape, [5, 3, 5, 5]) def testUnknownSpatialDims(self): x = tf.TensorSpec([3, None, None, 3], dtype=tf.float32) c = depthwise_conv.DepthwiseConv2D( channel_multiplier=1, kernel_shape=3, data_format="NHWC") defun_conv = tf.function(c).get_concrete_function(x) out = defun_conv(tf.ones([3, 5, 5, 3])) expected_out = c(tf.ones([3, 5, 5, 3])) self.assertEqual(out.shape, [3, 5, 5, 3]) self.assertAllEqual(self.evaluate(out), self.evaluate(expected_out)) out = defun_conv(tf.ones([3, 4, 4, 3])) expected_out = c(tf.ones([3, 4, 4, 3])) self.assertEqual(out.shape, [3, 4, 4, 3]) self.assertAllEqual(self.evaluate(out), self.evaluate(expected_out)) @parameterized.parameters(True, False) def testUnknownChannels(self, autograph): x = tf.TensorSpec([3, 3, 3, None], dtype=tf.float32) c = depthwise_conv.DepthwiseConv2D( channel_multiplier=1, kernel_shape=3, data_format="NHWC") defun_conv = tf.function(c, autograph=autograph) with self.assertRaisesRegex(ValueError, "The number of input channels must be known"): defun_conv.get_concrete_function(x) @parameterized.named_parameters(("WithBias", True), ("WithoutBias", False)) def testComputationSame(self, with_bias): conv1 = depthwise_conv.DepthwiseConv2D( channel_multiplier=1, kernel_shape=[3, 3], stride=1, padding="SAME", with_bias=with_bias, **create_constant_initializers(1.0, 1.0, with_bias)) out = conv1(tf.ones([1, 5, 5, 1])) expected_out = np.array([[5, 7, 7, 7, 5], [7, 10, 10, 10, 7], [7, 10, 10, 10, 7], [7, 10, 10, 10, 7], [5, 7, 7, 7, 5]]) if not with_bias: expected_out -= 1 self.assertEqual(out.shape, [1, 5, 5, 1]) self.assertAllClose(np.reshape(out.numpy(), [5, 5]), expected_out) @parameterized.named_parameters(("WithBias", True), ("WithoutBias", False)) def testComputationValid(self, with_bias): conv1 = depthwise_conv.DepthwiseConv2D( channel_multiplier=1, kernel_shape=[3, 3], stride=1, padding="VALID", with_bias=with_bias, **create_constant_initializers(1.0, 1.0, with_bias)) out = conv1(tf.ones([1, 5, 5, 1])) expected_out = np.array([[10, 10, 10], [10, 10, 10], [10, 10, 10]]) if not with_bias: expected_out -= 1 self.assertEqual(out.shape, [1, 3, 3, 1]) self.assertAllClose(np.reshape(out.numpy(), [3, 3]), expected_out) @parameterized.named_parameters(("WithBias", True), ("WithoutBias", False)) def testComputationValidMultiChannel(self, with_bias): conv1 = depthwise_conv.DepthwiseConv2D( channel_multiplier=1, kernel_shape=[3, 3], stride=1, padding="VALID", with_bias=with_bias, **create_constant_initializers(1.0, 1.0, with_bias)) out = conv1(tf.ones([1, 5, 5, 3])) expected_out = np.array([[[10] * 3] * 3] * 3) if not with_bias: expected_out -= 1 self.assertAllClose(np.reshape(out.numpy(), [3, 3, 3]), expected_out) @parameterized.named_parameters(("WithBias", True), ("WithoutBias", False)) def testSharing(self, with_bias): """Sharing is working.""" conv1 = depthwise_conv.DepthwiseConv2D( channel_multiplier=3, kernel_shape=3, stride=1, padding="SAME", with_bias=with_bias) x = np.random.randn(1, 5, 5, 1) x1 = tf.constant(x, dtype=np.float32) x2 = tf.constant(x, dtype=np.float32) self.assertAllClose(conv1(x1), conv1(x2)) # Kernel shape was set to 3, which is expandeded to [3, 3, 3]. # Input channels are 1, output channels := in_channels * multiplier. # multiplier is kernel_shape[2] == 3. So weight layout must be: # (3, 3, 1, 3). w = np.random.randn(3, 3, 1, 3) # Now change the weights. conv1.w.assign(w) self.assertAllClose(conv1(x1), conv1(x2)) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/distribute/BUILD ================================================ load("//sonnet/src:build_defs.bzl", "snt_py_library", "snt_py_test") package(default_visibility = ["//sonnet:__subpackages__", "//docs/ext:__subpackages__", "//examples:__subpackages__"]) licenses(["notice"]) snt_py_library( name = "distributed_batch_norm", srcs = ["distributed_batch_norm.py"], deps = [ "//sonnet/src:batch_norm", "//sonnet/src:initializers", "//sonnet/src:metrics", "//sonnet/src:once", "//sonnet/src:types", # pip: tensorflow ], ) snt_py_test( name = "distributed_batch_norm_test", srcs = ["distributed_batch_norm_test.py"], deps = [ ":distributed_batch_norm", ":replicator", # pip: absl/logging # pip: absl/testing:parameterized "//sonnet/src:test_utils", # pip: tensorflow ], ) snt_py_library( name = "replicator", srcs = ["replicator.py"], deps = [ # pip: absl/logging "//sonnet/src:initializers", # pip: tensorflow ], ) snt_py_test( name = "replicator_test", srcs = ["replicator_test.py"], deps = [ ":replicator", ":replicator_test_utils", # pip: absl/logging # pip: absl/testing:parameterized "//sonnet/src:initializers", "//sonnet/src:test_utils", # pip: tensorflow ], ) snt_py_library( name = "replicator_test_utils", testonly = 1, srcs = ["replicator_test_utils.py"], deps = [ ":replicator", # pip: absl/logging # pip: tensorflow ], ) ================================================ FILE: sonnet/src/distribute/__init__.py ================================================ # Copyright 2021 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ ================================================ FILE: sonnet/src/distribute/distributed_batch_norm.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Distributed batch normalization module.""" from typing import Optional, Tuple from sonnet.src import batch_norm from sonnet.src import initializers from sonnet.src import metrics from sonnet.src import once from sonnet.src import types import tensorflow as tf class CrossReplicaBatchNorm(batch_norm.BaseBatchNorm): """Cross-replica Batch Normalization. At every step the full batch is used to calculate the batch statistics even within a distributed setting (note only with ``snt.(Tpu)Replicator``). See :class:`BaseBatchNorm` for details. Attributes: scale: If ``create_scale=True``, a trainable :tf:`Variable` holding the current scale after the module is connected for the first time. offset: If ``create_offset``, a trainable :tf:`Variable` holding the current offset after the module is connected for the first time. """ def __init__(self, create_scale: bool, create_offset: bool, moving_mean: metrics.Metric, moving_variance: metrics.Metric, eps: types.FloatLike = 1e-5, scale_init: Optional[initializers.Initializer] = None, offset_init: Optional[initializers.Initializer] = None, data_format: str = "channels_last", name: Optional[str] = None): """Constructs a ``CrossReplicaBatchNorm`` module. Args: create_scale: whether to create a trainable scale per channel applied after the normalization. create_offset: whether to create a trainable offset per channel applied after normalization and scaling. moving_mean: An object which keeps track of the moving average of the mean which can be used to normalize at test time. This object must have an update method which takes a value and updates the internal state and a value property which returns the current mean. moving_variance: An object which keeps track of the moving average of the variance which can be used to normalize at test time. This object must have an update method which takes a value and updates the internal state and a value property which returns the current variance. eps: Small epsilon to avoid division by zero variance. Defaults to ``1e-5``. scale_init: Optional initializer for the scale variable. Can only be set if ``create_scale=True``. By default scale is initialized to ``1``. offset_init: Optional initializer for the offset variable. Can only be set if ``create_offset=True``. By default offset is initialized to ``0``. data_format: The data format of the input. Can be either ``channels_first``, ``channels_last``, ``N...C`` or ``NC...``. By default it is ``channels_last``. name: Name of the module. """ super().__init__( create_scale=create_scale, create_offset=create_offset, moving_mean=moving_mean, moving_variance=moving_variance, eps=eps, scale_init=scale_init, offset_init=offset_init, data_format=data_format, name=name) @once.once def _initialize(self, inputs: tf.Tensor): super()._initialize(inputs) # Always use the unfused op here as mean/var are calculated before the op is # called so no speed-up is gained from the fused op self._fused = False def _moments(self, inputs: tf.Tensor, use_batch_stats: types.BoolLike) -> Tuple[tf.Tensor, tf.Tensor]: replica_context = tf.distribute.get_replica_context() if replica_context is None: raise TypeError( "Cross replica batch norm cannot be called in cross-replica context.") if use_batch_stats: # Note: This uses var=E(x^2) - E(x)^2 instead of the more numerically # stable var=E((x-E(x))^2) as this means that with XLA the all_reduces can # be combined and a fusion removed giving significant speed-up. # If you see NaNs in your model please try the alternative formula and # file a bug with your use-case. mean = tf.reduce_mean(inputs, self._axis, keepdims=True) mean = replica_context.all_reduce("MEAN", mean) mean_of_squares = tf.reduce_mean( tf.square(inputs), self._axis, keepdims=True) mean_of_squares = replica_context.all_reduce("MEAN", mean_of_squares) mean_squared = tf.square(mean) var = mean_of_squares - mean_squared return mean, var else: # use moving statistics mean = self.moving_mean.value variance = self.moving_variance.value return mean, variance ================================================ FILE: sonnet/src/distribute/distributed_batch_norm_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.distribute.batch_norm.""" from absl import logging from absl.testing import parameterized from sonnet.src import test_utils from sonnet.src.distribute import distributed_batch_norm as batch_norm from sonnet.src.distribute import replicator import tensorflow as tf class CrossReplicaBatchNormTest(test_utils.TestCase, parameterized.TestCase): # Avoid running tests inside a `with tf.device("TPU:0"):` block. ENTER_PRIMARY_DEVICE = False def testDefaultReplicaContext(self): layer = batch_norm.CrossReplicaBatchNorm(False, False, TestMetric(), TestMetric()) inputs = tf.ones([2, 3, 3, 5]) scale = tf.constant(0.5, shape=(5,)) offset = tf.constant(2.0, shape=(5,)) outputs = layer(inputs, True, scale=scale, offset=offset).numpy() self.assertAllEqual(outputs, tf.fill(inputs.shape, 2.0)) def testWithMultipleDevicesMirrored(self): if self.primary_device == "CPU": self.skipTest("No devices to mirror across.") elif self.primary_device == "GPU": devices = tf.config.experimental.list_logical_devices("GPU") else: devices = tf.config.experimental.list_logical_devices("TPU") strategy = replicator.Replicator([device.name for device in devices]) with strategy.scope(): mean_metric = TestMetric() var_metric = TestMetric() layer = batch_norm.CrossReplicaBatchNorm(False, False, mean_metric, var_metric) scale = tf.constant(0.5, shape=(5,)) offset = tf.constant(2.0, shape=(5,)) def foo(): inputs = tf.random.normal([2, 3, 3, 5]) outputs = layer(inputs, True, False, scale, offset) return inputs, outputs inputs, outputs = strategy.run(foo) local_mean_metric = strategy.experimental_local_results(mean_metric.value) local_var_metric = strategy.experimental_local_results(var_metric.value) self.assertAllEqual(local_mean_metric[0].numpy(), local_mean_metric[1].numpy()) self.assertAllEqual(local_var_metric[0].numpy(), local_var_metric[1].numpy()) mean = local_mean_metric[0] var = local_var_metric[0] for inp, out in zip( strategy.experimental_local_results(inputs), strategy.experimental_local_results(outputs)): expected_out = (inp - mean) * tf.math.rsqrt(var + 1e-5) * scale + offset self.assertAllClose(out, expected_out) def testWithTpuStrategy(self): if self.primary_device != "TPU": self.skipTest("TPU strategy only runs on TPU's") strategy = replicator.TpuReplicator() with strategy.scope(): mean_metric = TestMetric() var_metric = TestMetric() layer = batch_norm.CrossReplicaBatchNorm(False, False, mean_metric, var_metric) scale = tf.constant(0.5, shape=(5,)) offset = tf.constant(2.0, shape=(5,)) @tf.function def run(): def compute(): inputs = tf.ones([2, 3, 3, 5]) outputs = layer(inputs, True, False, scale, offset) return inputs, outputs return strategy.run(compute) inputs, outputs = run() local_mean_metric = strategy.experimental_local_results(mean_metric.value) local_var_metric = strategy.experimental_local_results(var_metric.value) self.assertAllEqual(local_mean_metric[0].numpy(), local_mean_metric[1].numpy()) self.assertAllEqual(local_var_metric[0].numpy(), local_var_metric[1].numpy()) mean = local_mean_metric[0] var = local_var_metric[0] for inp, out in zip( strategy.experimental_local_results(inputs), strategy.experimental_local_results(outputs)): expected_out = (inp - mean) * tf.math.rsqrt(var + 1e-5) * scale + offset self.assertAllClose(out, expected_out) class TestMetric: def __init__(self): self._foo = None self._built = False def update(self, x): if self._built: self._foo.assign(x) else: self._foo = tf.Variable(x) self._built = True @property def value(self): return self._foo def initialize(self, x): self._foo = tf.Variable(x) self._built = True def setUpModule(): # If a physical GPU is available make sure TF sees at least two. gpus = tf.config.experimental.list_physical_devices(device_type="GPU") if len(gpus) == 1: logging.info("Splitting one physical GPU into two logical GPUs.") tf.config.experimental.set_virtual_device_configuration( gpus[0], [ tf.config.experimental.VirtualDeviceConfiguration( memory_limit=1024), tf.config.experimental.VirtualDeviceConfiguration(memory_limit=1024) ]) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/distribute/replicator.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Replicator Distribution Strategy.""" import contextlib from typing import Callable, TypeVar from absl import logging from sonnet.src import initializers import tensorflow as tf T = TypeVar("T") def replica_local_creator(next_creator, **kwargs) -> tf.Variable: """Variable creator that by default creates replica local variables.""" if kwargs["synchronization"] == tf.VariableSynchronization.AUTO: kwargs["synchronization"] = tf.VariableSynchronization.ON_READ if kwargs["aggregation"] == tf.VariableAggregation.NONE: kwargs["aggregation"] = tf.VariableAggregation.ONLY_FIRST_REPLICA if kwargs["trainable"] is None: kwargs["trainable"] = True return next_creator(**kwargs) class Replicator(tf.distribute.MirroredStrategy): r"""Replicates input, parameters and compute over multiple accelerators. ``Replicator`` is a TensorFlow "Distribution Strategy" implementing the programming model described in the TF-Replicator paper :cite:`buchlovsky2019tf` and TensorFlow RFC :cite:`buchlovsky2019distribution`. ``Replicator`` enables data-parallel training across multiple accelerators on a single machine, it supports eager execution and :tf:`function`. To get started create a ``Replicator`` instance: >>> replicator = snt.distribute.Replicator() Replicator provides a scope inside which any new :tf:`Variable`\ s will be replicated across all local devices: >>> with replicator.scope(): ... mod = snt.Linear(32) Additionally replicator provides utility functions to apply a module in parallel on multiple devices. First we need to define some computation that runs on each GPU. The "replica context" object provides us a way to communicate between replicas (e.g. to perform an ``all_reduce``): >>> def forward(): ... # Compute a random output on each GPU. ... x = tf.random.normal([8, 28 * 28]) ... y = mod(x) ... # Synchronize the value of `y` between all GPUs. ... ctx = tf.distribute.get_replica_context() ... y = ctx.all_reduce("mean", y) ... return y Finally we use the run API to apply ``forward`` in parallel on all accelerator devices: >>> per_replica_y = replicator.run(forward) """ @contextlib.contextmanager def scope(self): with contextlib.ExitStack() as stack: stack.enter_context(super().scope()) stack.enter_context(tf.variable_creator_scope(replica_local_creator)) yield # TODO(tomhennigan) Remove this once TF 2.3 is released. try: TPUStrategy = tf.distribute.TPUStrategy except AttributeError: TPUStrategy = tf.distribute.experimental.TPUStrategy class TpuReplicator(TPUStrategy): r"""Replicates input, parameters and compute over multiple TPUs. ``TpuReplicator`` is a TensorFlow "Distribution Strategy" implementing the programming model described in the TF-Replicator paper :cite:`buchlovsky2019tf` and TensorFlow RFC :cite:`buchlovsky2019distribution`. ``TpuReplicator`` enables data-parallel training across multiple TPUs on one or more machines, it supports :tf:`function`. To get started create a ``TpuReplicator`` instance: >>> replicator = snt.distribute.TpuReplicator() This provides a scope inside which any new :tf:`Variable`\ s will be replicated across all TPU cores: >>> with replicator.scope(): ... mod = snt.Linear(32) Additionally replicator provides utility functions to apply a module in parallel on multiple devices. First we need to define some computation that runs on each TPU. The "replica context" object provides us a way to communicate between replicas: >>> def forward(): ... # Compute a random output on each GPU. ... x = tf.random.normal([8, 28 * 28]) ... y = mod(x) ... # Synchronize the value of `y` between all GPUs. ... ctx = tf.distribute.get_replica_context() ... y = ctx.all_reduce("mean", y) ... return y Finally we use the run API to apply ``forward`` in parallel on all TPU devices. This must be run as part of a :tf:`function` since ``TpuReplicator`` uses XLA to compile and replicate our function to run in parallel over all TPU cores: >>> @tf.function(autograph=False) ... def all_forward(): ... return replicator.run(forward) >>> per_replica_y = all_forward() """ @contextlib.contextmanager def scope(self): with contextlib.ExitStack() as stack: stack.enter_context(super().scope()) stack.enter_context(tf.variable_creator_scope(replica_local_creator)) yield def create_variables_eagerly(f: Callable[..., T]) -> Callable[..., T]: """Wraps a function and attempts to create variables using eager mode. Example usage: >>> model = snt.Sequential([snt.Linear(1) for _ in range(100)]) >>> @tf.function ... @snt.distribute.create_variables_eagerly ... def f(x): ... return model(x) >>> _ = f(tf.ones([1, 1])) On a CPU only machine ``f`` will run ~4x faster (700ms vs. 3s), the benefits are more pronounced in a distributed setup since eager variable creation can skip a number of checks that are required in graph mode (e.g. checking whether the variable has already been created) which end up ping-ponging RPCs. Args: f: Any function. Returns: A function running `f` in a context where variables are created eagerly. """ def wrapper(*args, **kwargs): with contextlib.ExitStack() as stack: # The two hacks below enable a large speedup when initializing large # models on TPU pods. # TODO(b/141243467) Remove these workarounds. stack.enter_context(_eager_initial_values()) stack.enter_context(tf.variable_creator_scope(_eager_variable_creator)) return f(*args, **kwargs) return wrapper def _eager_variable_creator(getter, initial_value, **kwargs): """Attempts to force variable creation to be eager.""" eager_initial_value = None if isinstance(initial_value, tf.Tensor): eager_initial_value = tf.get_static_value(initial_value) if eager_initial_value is not None: # If we have an eager initial value we can create variables in eager mode. with tf.init_scope(): return getter(initial_value=eager_initial_value, **kwargs) else: # Fall back to creating in whatever context we're in with user input. return getter(initial_value=initial_value, **kwargs) @contextlib.contextmanager def _eager_initial_values(): """Attempts to force all initializers to create eager tensors.""" all_initializers = {cls: cls.__call__ for cls in initializers.Initializer.__subclasses__()} def patched_call(self, shape, dtype): """Monkey-patched verison of `Initializer.__call__`.""" cls = type(self) orig_call = all_initializers[cls] try: with tf.init_scope(): return orig_call(self, shape, dtype) except: # pylint: disable=bare-except if not tf.executing_eagerly(): logging.exception( "Failed to create initial value eagerly for %s shape=%s dtype=%s", type(self).__name__, shape, dtype) return orig_call(self, shape, dtype) try: for cls in all_initializers: cls.__call__ = patched_call yield finally: # Restore for cls, orig_call in all_initializers.items(): cls.__call__ = orig_call ================================================ FILE: sonnet/src/distribute/replicator_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.replicator.""" import os from absl import logging from absl.testing import parameterized from sonnet.src import initializers from sonnet.src import test_utils from sonnet.src.distribute import replicator as snt_replicator from sonnet.src.distribute import replicator_test_utils as replicator_utils import tensorflow as tf def _create_variable_in_cross_replica_context(replicator): with replicator.scope(): v = tf.Variable(1.) return v class TrainableVariable: def __call__(self): if not hasattr(self, "v"): self.v = tf.Variable(1.) return self.v def _create_variable_in_replica_context(replicator): o = TrainableVariable() def create_var(): replicator.run(o) # TpuReplicator doesn't support pure eager mode. if isinstance(replicator, snt_replicator.TpuReplicator): create_var = tf.function(create_var) create_var() return o.v def all_variable_creators(): return (("cross_replica_context", _create_variable_in_cross_replica_context), ("replica_context", _create_variable_in_replica_context)) class ReplicatorTest(test_utils.TestCase, parameterized.TestCase): # Avoid running tests inside a `with tf.device("TPU:0"):` block. ENTER_PRIMARY_DEVICE = False @test_utils.combined_named_parameters(replicator_utils.named_replicators(), all_variable_creators()) def test_variable_synchronization_default(self, replicator_fn, create_var): replicator = replicator_fn() if replicator is None: self.skipTest("No replicator supplied.") v = create_var(replicator) self.assertEqual(tf.VariableSynchronization.ON_READ, v.values[0].synchronization) @test_utils.combined_named_parameters(replicator_utils.named_replicators(), all_variable_creators()) def test_variable_aggregation_default(self, replicator_fn, create_var): replicator = replicator_fn() if replicator is None: self.skipTest("No replicator supplied.") v = create_var(replicator) self.assertEqual(tf.VariableAggregation.ONLY_FIRST_REPLICA, v.aggregation) @test_utils.combined_named_parameters(replicator_utils.named_replicators(), all_variable_creators()) def test_variable_trainable_default(self, replicator_fn, create_var): replicator = replicator_fn() if replicator is None: self.skipTest("No replicator supplied.") v = create_var(replicator) self.assertTrue(v.trainable) @test_utils.combined_named_parameters(replicator_utils.named_replicators(), test_utils.named_bools("trainable")) def test_variable_trainable(self, replicator_fn, trainable): replicator = replicator_fn() if replicator is None: self.skipTest("No replicator supplied.") with replicator.scope(): v = tf.Variable(1., trainable=trainable) self.assertEqual(trainable, v.trainable) @test_utils.combined_named_parameters(replicator_utils.named_replicators(), (("assign", "assign", 1.), ("assign_add", "assign_add", 1.), ("assign_sub", "assign_sub", -1.)), test_utils.named_bools("cross_replica")) def test_assign(self, replicator_fn, method_name, value, cross_replica): replicator = replicator_fn() if replicator is None: self.skipTest("No replicator supplied.") with replicator.scope(): v = tf.Variable(0.) update_fn = lambda: getattr(v, method_name)(value) if cross_replica: # NOTE: Explicitly not running inside replicator.scope (fn should handle). update_fn() else: # TpuReplicator doesn't support pure eager mode. if isinstance(replicator, snt_replicator.TpuReplicator): update_fn = tf.function(update_fn) replicator.run(update_fn) for component in v._values: self.assertAllEqual(component.read_value(), tf.ones_like(component)) @test_utils.combined_named_parameters(replicator_utils.named_replicators(), test_utils.named_bools("cross_replica")) def test_read_value(self, replicator_fn, cross_replica): replicator = replicator_fn() if replicator is None: self.skipTest("No replicator supplied.") with replicator.scope(): v = tf.Variable(0.) if cross_replica: values = [v.read_value()] else: # TpuReplicator doesn't support pure eager mode. if isinstance(replicator, snt_replicator.TpuReplicator): read_value_fn = tf.function(v.read_value) else: read_value_fn = v.read_value values = replicator.run(read_value_fn) values = replicator.experimental_local_results(values) for component in v._values: for value in values: self.assertAllEqual(component.read_value(), value) @parameterized.parameters(True, False) def test_falls_back_to_graph(self, autograph): if os.environ.get("GITHUB_ACTIONS", "") == "true" and autograph: self.skipTest("Autograph generated code has syntax error.") init = FailsInEagerMode() value = tf.function( snt_replicator.create_variables_eagerly( lambda: init([], tf.float32)), autograph=autograph)() self.assertEqual(value.numpy(), 1.) @parameterized.parameters(True, False) def test_requires_eager(self, autograph): init = MyOnesInitializer() value = tf.function( snt_replicator.create_variables_eagerly( lambda: init([], tf.float32)), autograph=autograph)() self.assertEqual(value.numpy(), 1.) @parameterized.parameters(True, False) def test_eager_variable_creator(self, autograph): variables = [None, None] eager_ones = tf.ones([]) @snt_replicator.create_variables_eagerly def f(): if variables[0] is None: graph_ones = tf.ones([]) # NOTE: `graph_ones` will be resolved by `tf.get_static_value`. v1 = tf.Variable(graph_ones) v2 = tf.Variable(eager_ones) # Even though we're in a tf.function here, eager_variable_creator # should have popped us into an init_scope so we have eager variables. with tf.init_scope(): self.assertEqual(v1.numpy(), 1.) self.assertEqual(v2.numpy(), 1.) variables[0] = v1 variables[1] = v2 tf.function(f, autograph=autograph)() class MyOnesInitializer(initializers.Initializer): def __call__(self, shape, dtype): assert tf.executing_eagerly() return tf.ones(shape, dtype) class FailsInEagerMode(initializers.Initializer): def __call__(self, shape, dtype): if tf.executing_eagerly(): raise ValueError("Eager mode") return tf.ones(shape, dtype) def setUpModule(): # If a physical GPU is available make sure TF sees at least two. gpus = tf.config.experimental.list_physical_devices(device_type="GPU") if len(gpus) == 1: logging.info("Splitting one physical GPU into two logical GPUs.") tf.config.experimental.set_virtual_device_configuration( gpus[0], [ tf.config.experimental.VirtualDeviceConfiguration( memory_limit=1024), tf.config.experimental.VirtualDeviceConfiguration(memory_limit=1024) ]) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/distribute/replicator_test_utils.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Utilities for tests working with replicator.""" from typing import Callable, Sequence, Tuple import unittest from absl import logging from sonnet.src.distribute import replicator as snt_replicator import tensorflow as tf def _replicator_primary_device() -> snt_replicator.Replicator: # NOTE: The explicit device list is required since currently Replicator # only considers CPU and GPU devices. This means on TPU by default we only # mirror on the local CPU. for device_type in ("TPU", "GPU", "CPU"): devices = tf.config.experimental.list_logical_devices( device_type=device_type) if devices: devices = [d.name for d in devices] logging.info("Replicating over %s", devices) return snt_replicator.Replicator(devices=devices) assert False, "No TPU/GPU or CPU found" def _tpu_replicator_or_skip_test() -> snt_replicator.TpuReplicator: tpus = tf.config.experimental.list_logical_devices(device_type="TPU") if not tpus: raise unittest.SkipTest("No TPU available.") logging.info("Using TpuReplicator over %s", [t.name for t in tpus]) return snt_replicator.TpuReplicator() Strategy = tf.distribute.Strategy def named_replicators() -> Sequence[Tuple[str, Callable[[], Strategy]]]: return (("TpuReplicator", _tpu_replicator_or_skip_test), ("Replicator", _replicator_primary_device)) ================================================ FILE: sonnet/src/dropout.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Sonnet dropout modules.""" from typing import Optional from sonnet.src import base from sonnet.src import types from sonnet.src import utils import tensorflow as tf class Dropout(base.Module): """Randomly drop units in the input at a given rate. See: http://www.cs.toronto.edu/~hinton/absps/dropout.pdf Dropout was originally described by Hinton et al. TensorFlow deviates slightly from this paper by scaling activations at training time rather than test time. """ def __init__(self, rate: types.FloatLike, noise_shape: Optional[types.ShapeLike] = None, seed: Optional[int] = None, name: Optional[str] = None): """Constructs a Dropout module. Args: rate: Probability that each element of x is discarded. Must be a scalar in the range `[0, 1)`. noise_shape: (Optional) Shape vector controlling the shape of the random noise used to apply dropout. If not set this will be the shape of the input. If set it should be broadcastable to the input shape. seed: (Optional) Random seed to be passed to TensorFlow ops when generating dropout tensor. name: (Optional) Name for this module. """ super().__init__(name=name) self._rate = rate self._noise_shape = noise_shape self._seed = seed @utils.smart_autograph def __call__(self, x: tf.Tensor, is_training: types.BoolLike) -> tf.Tensor: if not is_training: return x # NOTE: Even if `self._seed` is a constant value (e.g. `2`) this will # produce a different random dropout each call (the per-op seed is used # in conjunction with the global seed and some persistent state to produce # random values). # c.f. https://www.tensorflow.org/api_docs/python/tf/random/set_random_seed return tf.nn.dropout( x, rate=self._rate, noise_shape=self._noise_shape, seed=self._seed) ================================================ FILE: sonnet/src/dropout_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.dropout.""" from absl.testing import parameterized import numpy as np from sonnet.src import dropout from sonnet.src import test_utils import tensorflow as tf class DropoutTest(test_utils.TestCase, parameterized.TestCase): @parameterized.parameters(np.arange(.0, .9, .1)) def test_sum_close(self, rate): mod = dropout.Dropout(rate=rate) x = tf.ones([1000]) rtol = 0.3 if "TPU" in self.device_types else 0.1 self.assertAllClose( tf.reduce_sum(mod(x, is_training=True)), tf.reduce_sum(mod(x, is_training=False)), rtol=rtol) @parameterized.parameters(np.arange(0, .9, .1)) def test_dropout_rate(self, rate): mod = dropout.Dropout(rate=rate) x = tf.ones([1000]) x = mod(x, is_training=True) # We should have dropped something, test we're within 10% of rate. # (or 30% on a TPU) rtol = 0.3 if "TPU" in self.device_types else 0.1 kept = tf.math.count_nonzero(x).numpy() keep_prob = 1 - rate self.assertAllClose(kept, 1000 * keep_prob, rtol=rtol) def test_dropout_is_actually_random(self): mod = dropout.Dropout(rate=0.5) x = tf.ones([1000]) tf.random.set_seed(1) y1 = mod(x, is_training=True) y2 = mod(x, is_training=True) self.assertNotAllClose(y1, y2) @parameterized.parameters(True, False) def test_with_tf_function_with_booleans(self, autograph): """tf.function compilation correctly handles if statement.""" layer = dropout.Dropout(rate=0.5) layer = tf.function(layer, autograph=autograph) inputs = tf.ones([2, 5, 3, 3, 3]) expected = tf.zeros_like(inputs) for is_training in (True, False): outputs = layer(inputs, is_training) self.assertEqual(outputs.shape, expected.shape) @parameterized.parameters(True, False) def test_with_tf_function_with_variables(self, autograph): """tf.function correctly handles if statement when argument is Variable.""" layer = dropout.Dropout(rate=0.5) layer = tf.function(layer, autograph=autograph) inputs = tf.ones([2, 5, 3, 3, 3]) expected = tf.zeros_like(inputs) is_training_variable = tf.Variable(False, trainable=False) for is_training in (True, False): is_training_variable.assign(is_training) outputs = layer(inputs, is_training_variable) self.assertEqual(outputs.shape, expected.shape) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/embed.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Embedding module.""" import math from typing import Optional from sonnet.src import base from sonnet.src import initializers from sonnet.src import types import tensorflow as tf class Embed(base.Module): """Module for embedding tokens in a low-dimensional space.""" def __init__(self, vocab_size: Optional[int] = None, embed_dim: Optional[int] = None, existing_vocab: Optional[types.TensorLike] = None, densify_gradients: bool = False, initializer: Optional[initializers.Initializer] = None, trainable: bool = True, dtype: tf.DType = tf.float32, name: Optional[str] = None): """Constructs an Embed module. Args: vocab_size: Number of unique tokens to embed. If not provided, an existing vocabulary matrix from which vocab_size can be inferred must be provided as existing_vocab. embed_dim: Number of dimensions to assign to each embedding. If not specified, we use ``6 * sqrt(sqrt(vocab_size))``. If an existing vocabulary matrix initializes the module, this should not be provided as it will be inferred. existing_vocab: A ``[vocab_size, embed_dim]`` vocabulary matrix. Will be converted to a tf.float32 tensor. If provided, neither or vocab_size or embed_dim should be provided as they are inferred. densify_gradients: If True, we convert the embedding gradient from an ``tf.IndexedSlices`` to a regular tensor before sending it back to the parameter server. This avoids excess computation on the parameter server. Use this option for moderately sized embeddings, e.g., a vocabulary size on the order of up to thousands. For embeddings larger than these, e.g. a vocabulary size on the order of tens or hundreds of thousands, set this to False. initializer: Initializer for the embeddings. By default, embeddings are initialized via a truncated normal distribution. trainable: if True, the embeddings will be updated during training. If False, they are fixed to their initial values. dtype: The dtype to use for the embedding. Defaults to float32. name: Name for this module. Raises: ValueError: if neither one of ``vocab_size`` or ``existing_vocab`` is provided, or if ``existing_vocab`` is provided along with ``vocab_size``, ``embedding_dim``, ``initializer`` (as these should be inferred). """ super().__init__(name=name) if vocab_size is None and existing_vocab is None: raise ValueError("Must provide one of vocab_size or existing_vocab.") if existing_vocab is not None and (vocab_size or embed_dim or initializer): raise ValueError("If `existing_vocab` is provided, none of `vocab_size`, " "`embedding_dim`, `initializer` are needed.") if existing_vocab is None: if embed_dim is None: embed_dim = embedding_dim(vocab_size) if initializer is None: initializer = initializers.TruncatedNormal() vocab = initializer([vocab_size, embed_dim], dtype) else: existing_vocab = tf.convert_to_tensor(existing_vocab, dtype=dtype) vocab_size, embed_dim = existing_vocab.shape vocab = existing_vocab self.vocab_size = vocab_size self.embed_dim = embed_dim self.densify_gradients = densify_gradients self.embeddings = tf.Variable(vocab, trainable=trainable, name="embeddings") def __call__(self, inputs): if self.densify_gradients: embeddings = dense_gradient(self.embeddings) else: embeddings = self.embeddings return tf.nn.embedding_lookup(embeddings, inputs) def embedding_dim(vocab_size: int): """Calculate a reasonable embedding size for a vocabulary. Rule of thumb is ``6 * sqrt(sqrt(vocab_size))``. Args: vocab_size: Size of the input vocabulary. Returns: The embedding size to use. Raises: ValueError: if ``vocab_size`` is invalid. """ if not vocab_size or (vocab_size <= 0): raise ValueError("Invalid vocab_size %g." % vocab_size) return int(round(6.0 * math.sqrt(math.sqrt(vocab_size)))) @tf.custom_gradient def dense_gradient(x: tf.Tensor): """Identity operation whose gradient is converted to a ``tf.Tensor``. >>> embedding = tf.Variable(tf.random.normal([3, 3])) >>> with tf.GradientTape() as tape: ... y = tf.nn.embedding_lookup(dense_gradient(embedding), [1]) >>> tape.gradient(y, embedding).numpy() array([[ 0., 0., 0.], [ 1., 1., 1.], [ 0., 0., 0.]], dtype=float32) Args: x: A ``tf.Tensor``. Returns: The input ``tf.Tensor`` and a dense identity gradient function. """ def grad(dy): if isinstance(dy, tf.IndexedSlices): return tf.convert_to_tensor(dy) else: return dy return x, grad ================================================ FILE: sonnet/src/embed_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.embed.""" from absl.testing import parameterized from sonnet.src import embed from sonnet.src import initializers from sonnet.src import test_utils import tensorflow as tf class EmbedTest(test_utils.TestCase, parameterized.TestCase): @parameterized.parameters([1, 10, 100]) def test_vocab_size(self, vocab_size): e = embed.Embed(vocab_size=vocab_size) self.assertEqual(e.vocab_size, vocab_size) self.assertEqual(e.embeddings.shape[0], vocab_size) @parameterized.parameters([1, 10, 100]) def test_embed_dim(self, embed_dim): e = embed.Embed(vocab_size=100, embed_dim=embed_dim) self.assertEqual(e.embed_dim, embed_dim) self.assertEqual(e.embeddings.shape[1], embed_dim) @parameterized.parameters([(1, 1), (10, 10), (100, 100)]) def test_existing_vocab(self, vocab_size, embed_dim): existing_vocab = tf.ones([vocab_size, embed_dim]) e = embed.Embed(existing_vocab=existing_vocab) self.assertEqual(e.vocab_size, vocab_size) self.assertEqual(e.embed_dim, embed_dim) self.assertAllEqual(e.embeddings.read_value(), existing_vocab) @parameterized.parameters([True, False]) def test_densify_gradients(self, densify_gradients): e = embed.Embed(1, densify_gradients=densify_gradients) with tf.GradientTape() as tape: y = e([0]) dy = tape.gradient(y, e.embeddings) if densify_gradients: self.assertIsInstance(dy, tf.Tensor) else: self.assertIsInstance(dy, tf.IndexedSlices) def test_initializer(self): e = embed.Embed(1, 1, initializer=initializers.Constant(28.)) self.assertAllEqual(e.embeddings.read_value(), [[28.]]) def test_pinned_to_cpu(self): with tf.device("CPU"): e = embed.Embed(1) spec = tf.DeviceSpec.from_string(e.embeddings.device) self.assertEqual(spec.device_type, "CPU") @parameterized.parameters([True, False]) def test_trainable(self, trainable): e = embed.Embed(1, trainable=trainable) self.assertEqual(e.embeddings.trainable, trainable) @parameterized.parameters([tf.float32, tf.float16]) def test_dtype(self, dtype): if dtype == tf.float16 and self.primary_device == "TPU": self.skipTest("float16 embeddings not supported on TPU.") e = embed.Embed(1, dtype=dtype) self.assertEqual(e.embeddings.dtype, dtype) def test_name(self): e = embed.Embed(1, name="my_embedding") self.assertEqual(e.name, "my_embedding") self.assertEqual(e.embeddings.name, "my_embedding/embeddings:0") if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/functional/BUILD ================================================ load("//sonnet/src:build_defs.bzl", "snt_py_library", "snt_py_test") package(default_visibility = ["//sonnet:__subpackages__", "//docs/ext:__subpackages__", "//examples:__subpackages__"]) licenses(["notice"]) snt_py_library( name = "haiku", srcs = ["haiku.py"], deps = [ ":utils", # pip: tensorflow ], ) snt_py_library( name = "jax", srcs = ["jax.py"], deps = [ ":utils", # pip: tensorflow # pip: tree ], ) snt_py_library( name = "optimizers", srcs = ["optimizers.py"], deps = [ ":haiku", "//sonnet/src:base", # pip: tensorflow # pip: tree ], ) snt_py_library( name = "utils", srcs = ["utils.py"], deps = [ "//sonnet/src:utils", # pip: tensorflow # pip: tree ], ) snt_py_test( name = "haiku_test", srcs = ["haiku_test.py"], deps = [ ":haiku", # pip: absl/testing:parameterized "//sonnet", "//sonnet/src:test_utils", # pip: tensorflow # pip: tree ], ) snt_py_test( name = "jax_test", srcs = ["jax_test.py"], deps = [ ":jax", # pip: absl/testing:parameterized "//sonnet/src:test_utils", # pip: tensorflow ], ) snt_py_test( name = "optimizers_test", srcs = ["optimizers_test.py"], deps = [ ":haiku", ":optimizers", # pip: absl/testing:parameterized "//sonnet", "//sonnet/src:test_utils", # pip: tensorflow # pip: tree ], ) ================================================ FILE: sonnet/src/functional/__init__.py ================================================ # Copyright 2021 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ ================================================ FILE: sonnet/src/functional/haiku.py ================================================ # Copyright 2020 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Implements part of the Haiku ("Sonnet for JAX") API in TensorFlow 2.""" import collections import contextlib import functools import itertools import threading from sonnet.src.functional import utils import tensorflow as tf Transformed = collections.namedtuple("Transformed", ("init", "apply")) TransformedWithState = collections.namedtuple("TransformedWithState", ("init", "apply")) # pylint: disable=not-context-manager class TensorVariableCallbacks(threading.local): """Holds callbacks that are notified when TensorVariable are used.""" instance = None # Thread local singleton instance. def __init__(self): super().__init__() self._recording = False self._callbacks = [] def notify(self, variable): if self._recording: assert isinstance(variable, TensorVariable) for callback in self._callbacks: callback(variable) @contextlib.contextmanager def __call__(self, callback): self._callbacks.append(callback) recording = self._recording try: self._recording = True yield finally: assert self._callbacks.pop() is callback self._recording = recording TensorVariableCallbacks.instance = TensorVariableCallbacks() def notify(f): """Wraps `f` such that callbacks are notified about it being called.""" @functools.wraps(f) def wrapper(self, *args, **kwargs): TensorVariableCallbacks.instance.notify(self) return f(self, *args, **kwargs) # pytype: disable=wrong-arg-count return wrapper def defer_property(name): return property(fget=notify(lambda self: getattr(self.tensor_value, name))) def safe_read_tensor_value(variable): """Reads variable value or raises an exception.""" value = variable.tensor_value if value is None: # pylint: disable=implicit-str-concat raise ValueError("".join(( "Attempted to read a TensorVariable in a context where it has no ", "value. This commonly happens for one of two reasons:", "", " 1) You created a model in one transformed function and directly", " accessed the model variables (e.g. via `model.variables` or" " `model.w`) inside another transformed function.", " 2) You are trying to read a model variable outside of a", " transformed function.", "", "For (1) you can safely do this if you do not read the value of the", "variable (e.g. you just use metadata like `v.shape` or `v.dtype`).", "If you want to read the value of the variable then you must pass in", "the value (e.g. pass the result of `f.init(..)`).", "", "For (2) to read variable values inspect the result of a transformed", "function (e.g. look at the `params` dictionary returned from ", "`f.init(..)`)."))) # pylint: enable=implicit-str-concat return value def defer_read(): return property( fget=notify(lambda self: (lambda: safe_read_tensor_value(self)))) def defer_raise_notimplemented(): def _raise_notimplemented(): raise NotImplementedError return property(fget=notify(_raise_notimplemented)) def defer_indexed(f): return property(fget=notify(lambda self, i: f(self, i.indices, i.values))) def defer_assign(map_fn=None): """Returns a function implementing notify+assign.""" @notify def wrapped(self, v): if v is not None: v = tf.convert_to_tensor(v, dtype=self.dtype) if map_fn is not None: v = map_fn(self.tensor_value, v) if self.initial_tensor_value is None: self.initial_tensor_value = v self.tensor_value = v return v return wrapped class TensorVariable(tf.Variable): """Implements the tf.Variable API but backed by a tf.Tensor.""" def __init__(self, value, trainable, name=None): # NOTE: Intentionally not calling super ctor. self.initial_tensor_value = value self.tensor_value = value self._trainable = trainable self._name = name self._shape = value.shape self._dtype = value.dtype self._device = value.device # Properties. # NOTE: These do not notify since they do not result in TensorFlow operations. shape = property(fget=lambda self: self._shape) dtype = property(fget=lambda self: self._dtype) trainable = property(fget=lambda self: self._trainable) name = property(fget=lambda self: self._name) device = property(fget=lambda self: self._device) # Dense assign. assign = defer_assign() assign_add = defer_assign(tf.add) assign_sub = defer_assign(tf.subtract) # Sparse assign. batch_scatter_update = defer_raise_notimplemented() scatter_add = defer_raise_notimplemented() scatter_div = defer_raise_notimplemented() scatter_max = defer_raise_notimplemented() scatter_min = defer_raise_notimplemented() scatter_mul = defer_raise_notimplemented() scatter_sub = defer_raise_notimplemented() scatter_update = defer_raise_notimplemented() scatter_nd_add = defer_indexed(tf.tensor_scatter_nd_add) scatter_nd_sub = defer_indexed(tf.tensor_scatter_nd_sub) scatter_nd_update = defer_indexed(tf.tensor_scatter_nd_update) # Load not supported. load = defer_raise_notimplemented() # Shape ops. set_shape = defer_property("set_shape") get_shape = defer_property("get_shape") # Read dense. initialized_value = property( fget=notify(lambda self: self.initial_tensor_value)) read_value = defer_read() numpy = defer_property("numpy") value = defer_read() eval = defer_property("eval") # Read sparse. gather_nd = defer_indexed(tf.gather_nd) sparse_read = defer_indexed(tf.gather) # Serialize. to_proto = defer_raise_notimplemented() # Misc. count_up_to = defer_raise_notimplemented() def __repr__(self): return "TensorVariable(shape={}, dtype={}, name={!r})".format( list(self.shape), self.dtype.name, self.name) __str__ = __repr__ # Math ops. __add__ = defer_property("__add__") __sub__ = defer_property("__sub__") __mul__ = defer_property("__mul__") __div__ = defer_property("__div__") @functools.partial(tf.register_tensor_conversion_function, TensorVariable) @notify def tv_to_tensor(value, dtype=None, name=None, as_ref=None): """Converts a TensorVariable to a tf.Tensor.""" del as_ref tensor_value = value.tensor_value if tensor_value is None: # TODO(tomhennigan) We should probably not notify in this case. tensor_value = tf.zeros(value.shape, dtype=value.dtype) if dtype is not None: tensor_value = tf.cast(tensor_value, dtype=dtype, name=name) return tensor_value def create_tensor_variables(): """Defines a scope in which `TensorVariable`s are created. >>> with snt.functional.variables(): ... v = tf.Variable(tf.ones([]), name="v") >>> v.tensor_value Returns: A context manager that forces tf.Variable to create TensorVariables. """ def getter(next_getter, **kwargs): del next_getter initial_value = tf.convert_to_tensor(kwargs["initial_value"]) trainable = utils.first_non_none(kwargs["trainable"], True) name = utils.first_non_none(kwargs["name"], "Variable") name = utils.get_name_scope() + name + ":0" return TensorVariable(initial_value, trainable=trainable, name=name) return tf.variable_creator_scope(getter) variables = create_tensor_variables @contextlib.contextmanager def track_tensor_variables(): tensor_variables = [] with TensorVariableCallbacks.instance(tensor_variables.append): # pylint: disable=not-callable yield tensor_variables @contextlib.contextmanager def track_new_variables(): new_variables = [] def getter(next_getter, *args, **kwargs): var = next_getter(*args, **kwargs) new_variables.append(var) return var with tf.variable_creator_scope(getter): yield new_variables @contextlib.contextmanager def track_initial_state(): var_state = {} def callback(v): r = v.ref() if r not in var_state: var_state[r] = (v.initial_tensor_value, v.tensor_value) with TensorVariableCallbacks.instance(callback): # pylint: disable=not-callable yield var_state def initial_value_by_ref(tf_variables): # TODO(tomhennigan) Consider rolling own ref class comparing by name/shape. return {v.ref(): v.initial_tensor_value for v in tf_variables} def final_value_by_ref(tf_variables): # TODO(tomhennigan) Consider rolling own ref class comparing by name/shape. return {v.ref(): v.tensor_value for v in tf_variables} def transform(f) -> Transformed: """Transforms a function using Sonnet modules into a pair of pure functions. The first thing to do is to create some `snt.Module` instances: >>> with snt.functional.variables(): ... a = snt.Linear(10, name="a") ... b = snt.Linear(10, name="b") Next, define some function that creates and applies modules: >>> def f(x): ... return a(x) + b(x) Now we can convert that function into a pair of functions that allow us to lift all the parameters out of the function (`f.ini`) and apply the function with a given set of parameters (`f.apply`): >>> f = snt.functional.transform(f) To get the initial state of the module call `f.init` with an example input: >>> x = tf.ones([1, 1]) >>> params = f.init(x) >>> params {<...>: , <...>: , <...>: , <...>: } You can then apply the function with the given parameters by calling `f.apply`: >>> f.apply(params, x) It is expected that your program will at some point produce updated parameters and you will want to re-apply `f.apply`. You can do this by calling `f.apply` with different parameters: >>> new_params = tree.map_structure(lambda p: p + 1, params) >>> f.apply(new_params, x) If your network contains non-trainable state (e.g. moving averages) then you will need to use :func:`transform_with_state`. Args: f: A function closing over `Module` instances. Returns: A transformed function with `init` and `apply`. See docstring for details. """ return without_state(transform_with_state(f)) def transform_with_state(f) -> TransformedWithState: r"""Like :func:`transform` but supporting non-trainable state. See :func:`transform` for more details. It is possible for the network to maintain internal state (e.g. for a module like `BatchNorm` that may want to maintain a moving average): >>> with snt.functional.variables(): ... ema = snt.ExponentialMovingAverage(decay=0.5) >>> f = snt.functional.transform_with_state(ema) When initializing this network we are returned the parameters (any "trainable" :tf:`Variable`\ s) and all other state (any non-trainable :tf:`Variable`\ s): >>> params, state = f.init(3.0) >>> params {} >>> state {<...>: , <...>: , <...>: } To apply the network we simply call it and get back updated values for our non-trainable state: >>> y, state = f.apply(params, state, 3.0) >>> float(y.numpy()) 3.0 >>> y, state = f.apply(params, state, 6.0) >>> float(y.numpy()) 5.0 Args: f: A function closing over `Module` instances. Returns: A transformed function with `init` and `apply`. See docstring for details. """ def init_fn(*args, **kwargs): """Applies `f(*a, **k)` and extracts initial variable values.""" with create_tensor_variables(), \ track_new_variables() as new_variables, \ track_initial_state() as prev_var_state, \ track_tensor_variables() as tensor_variables: # NOTE: Intentionally discarding result. f(*args, **kwargs) params = initial_value_by_ref(v for v in tensor_variables if v.trainable) state = initial_value_by_ref(v for v in tensor_variables if not v.trainable) # Reset variable values. new_variables = {v.ref() for v in new_variables} for v in tensor_variables: r = v.ref() if r in new_variables: # Variables created inside the function have their values nullified. initial_tensor_value, tensor_value = None, None else: # Variables that already existed have their value reset. initial_tensor_value, tensor_value = prev_var_state[r] v.initial_tensor_value = initial_tensor_value v.tensor_value = tensor_value return params, state def apply_fn(params, state, *args, **kwargs): """Applies `f(*a, **k)` with variable values passed in.""" initial_values = {} for r, t in itertools.chain(params.items(), state.items()): v = r.deref() initial_values[r] = (v.tensor_value, v.initial_tensor_value) v.assign(t) try: with track_new_variables() as new_variables: out = f(*args, **kwargs) if new_variables: raise ValueError("Apply function cannot create new variables.") state = final_value_by_ref(p.deref() for p in state.keys()) return out, state finally: # Reset values to their initial state. for r, (tensor_value, initial_tensor_value) in initial_values.items(): v = r.deref() v.tensor_value = tensor_value v.initial_tensor_value = initial_tensor_value return TransformedWithState(init=init_fn, apply=apply_fn) def without_state(with_state: TransformedWithState) -> Transformed: """Returns init/apply functions that ignore state.""" def init_fn(*args, **kwargs): params, state = with_state.init(*args, **kwargs) if state: raise ValueError("Stateful networks must use `transform_with_state(f)`") return params def apply_fn(params, *args, **kwargs): y, state = with_state.apply(params, {}, *args, **kwargs) if state: raise ValueError("Stateful networks must use `transform_with_state(f)`") return y return Transformed(init_fn, apply_fn) ================================================ FILE: sonnet/src/functional/haiku_test.py ================================================ # Copyright 2020 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for Haiku compatibility layer.""" from absl.testing import parameterized import sonnet as snt from sonnet.src import test_utils from sonnet.src.functional import haiku as hk import tensorflow as tf import tree class TensorVariableTest(test_utils.TestCase, parameterized.TestCase): def test_initial_value(self): with hk.variables(): v = tf.Variable(tf.ones([])) self.assertIsInstance(v, hk.TensorVariable) self.assertAllEqual(v, 1) self.assertAllEqual(v.read_value(), 1) self.assertAllEqual(v.tensor_value, 1) @parameterized.parameters(None, True, False) def test_trainable(self, trainable): with hk.variables(): v = tf.Variable(1., trainable=trainable) if trainable is None: self.assertTrue(v.trainable) else: self.assertEqual(v.trainable, trainable) def test_name(self): with hk.variables(): v = tf.Variable(tf.ones([]), name="v") self.assertEqual(v.name, "v:0") def test_name_with_scope(self): with hk.variables(), tf.name_scope("foo"), tf.name_scope("bar"): v = tf.Variable(tf.ones([]), name="v") self.assertEqual(v.name, "foo/bar/v:0") @parameterized.parameters(([],), ([1, 2, 3],)) def test_shape(self, shape): with hk.variables(): v = tf.Variable(tf.ones(shape)) self.assertEqual(shape, v.shape.as_list()) @parameterized.parameters(tf.float32, tf.int32) def test_dtype(self, dtype): with hk.variables(): v = tf.Variable(tf.ones([], dtype=dtype)) self.assertEqual(dtype, v.dtype) def test_attributes_do_not_notify(self): with hk.variables(): v = tf.Variable(1.) s = tf.Variable(1., trainable=False) def f(): for c in (v, s): self.assertIsNotNone(c.shape) self.assertIsNotNone(c.dtype) self.assertIsNotNone(c.trainable) self.assertIsNotNone(c.name) self.assertIsNotNone(c.device) f = hk.transform_with_state(f) params, state = f.init() self.assertEmpty(params) self.assertEmpty(state) out, state = f.apply(params, state) self.assertIsNone(out) self.assertEmpty(state) def test_read_captured_variables_included(self): with hk.variables(): v = tf.Variable(1.) s = tf.Variable(1., trainable=False) f = hk.transform_with_state(lambda: (v.read_value() + s.read_value())) params, state = f.init() self.assertEqual(params, {v.ref(): v.tensor_value}) self.assertEqual(state, {s.ref(): s.tensor_value}) def test_captured_variable_from_other_function_raises(self): def f(model): if not model: model.append(tf.Variable(1.)) model.append(tf.Variable(1., trainable=False)) return sum(model) f = hk.transform_with_state(f) model = [] params, state = f.init(model) self.assertLen(params, 1) self.assertLen(state, 1) with self.assertRaisesRegex(ValueError, "TensorVariable .* has no value"): f.init(model) def test_assign(self): with hk.variables(): v = tf.Variable(tf.ones([])) v.assign(tf.zeros([])) self.assertAllEqual(v.numpy(), 0) self.assertAllEqual(v.read_value().numpy(), 0) self.assertAllEqual(v.tensor_value.numpy(), 0) def test_assign_add(self): with hk.variables(): v = tf.Variable(tf.ones([])) v.assign_add(1.) self.assertAllEqual(v.numpy(), 2) self.assertAllEqual(v.read_value().numpy(), 2) self.assertAllEqual(v.tensor_value.numpy(), 2) def test_assign_sub(self): with hk.variables(): v = tf.Variable(tf.ones([])) v.assign_sub(1.) self.assertAllEqual(v.numpy(), 0) self.assertAllEqual(v.read_value().numpy(), 0) self.assertAllEqual(v.tensor_value.numpy(), 0) class NetworkTest(test_utils.TestCase, parameterized.TestCase): def test_transform(self): mod = snt.Linear(1, w_init=tf.ones) snt.allow_empty_variables(mod) self.assertEmpty(mod.variables) f = hk.transform(mod) x = tf.ones([1, 1]) params = f.init(x) self.assertLen(params.items(), 2) self.assertAllEqual(params[mod.w.ref()], [[1.]]) self.assertAllEqual(params[mod.b.ref()], [0.]) y = f.apply(params, x) self.assertEqual(y, [[1.]]) params = tree.map_structure(lambda p: p + 1, params) y = f.apply(params, x) self.assertEqual(y, [[3.]]) def test_initial_values_preserved(self): with hk.variables(): v = tf.Variable(0) v.assign(1) def assert_values(): self.assertEqual(v.initial_tensor_value.numpy(), 0) self.assertEqual(v.tensor_value.numpy(), 1) assert_values() f = hk.transform(lambda: v.assign(2)) assert_values() params = f.init() assert_values() f.apply(params) assert_values() def test_variables_in_transform_set_to_none(self): mod = snt.Bias() f = hk.transform(mod) params = f.init(tf.ones([1, 1])) # Will create `mod.b`. self.assertIsNone(mod.b.tensor_value) self.assertIsNone(mod.b.initial_tensor_value) y = f.apply(params, tf.ones([1, 1])) self.assertAllEqual(y.numpy(), [[1.]]) self.assertIsNone(mod.b.tensor_value) self.assertIsNone(mod.b.initial_tensor_value) def test_disallows_variables_in_apply(self): _, apply_fn = hk.transform(lambda: tf.Variable(1)) with self.assertRaisesRegex(ValueError, "Apply function cannot create new variables"): apply_fn({}) def test_state_returns_initial_value(self): with hk.variables(): # NOTE: Initial value defined outside transform. v = tf.Variable(0, trainable=False) f = hk.transform_with_state(lambda: v.assign(1)) params, state = f.init() initial_v = state[v.ref()] self.assertEqual(initial_v.numpy(), 0) y, state = f.apply(params, state) final_v = state[v.ref()] self.assertEqual(y.numpy(), 1) self.assertEqual(final_v.numpy(), 1) def test_state_counter(self): with hk.variables(): v = tf.Variable(0, trainable=False) f = hk.transform_with_state(lambda: v.assign_add(1)) params, initial_state = f.init() for _ in range(2): state = initial_state for i in range(10): y, state = f.apply(params, state) self.assertEqual(y.numpy(), i + 1) def test_state_ema(self): with hk.variables(): ema = snt.ExponentialMovingAverage(decay=0.5) ema = hk.transform_with_state(ema) params, state = ema.init(3.0) y, state = ema.apply(params, state, 3.0) self.assertAllClose(y.numpy(), 3.0) y, state = ema.apply(params, state, 6.0) self.assertAllClose(y.numpy(), 5.0) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/functional/jax.py ================================================ # Copyright 2020 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """A subset of the JAX API in TF2.""" import functools from sonnet.src.functional import utils import tensorflow as tf import tree def device_put(t, device=None): return tree.map_structure(utils.run_on_device(lambda x: x, device), t) def device_get(t): return tree.map_structure(lambda x: x.numpy(), t) # TODO(tomhennigan) This should be cached. def jit(f, device=None): if device is None: device = utils.get_first_accelerator() # TODO(tomhennigan) Enable XLA compilation (experimental_compile=True). return tf.function(utils.run_on_device(f, device)) def grad(f, argnums=0, has_aux=False): """Returns the gradient function for `f`.""" value_and_grad_f = value_and_grad(f, argnums=argnums, has_aux=has_aux) @functools.wraps(f) def wrapper(*args, **kwargs): if has_aux: (_, aux), g = value_and_grad_f(*args, **kwargs) return g, aux else: _, g = value_and_grad_f(*args, **kwargs) return g return wrapper def value_and_grad(f, argnums=0, has_aux=False): """Returns the gradient function for `f`.""" @functools.wraps(f) def wrapper(*args, **kwargs): """Computes `f` and returns derivatives of the output wrt input(s).""" params = tree.map_structure(args.__getitem__, argnums) with tf.GradientTape(watch_accessed_variables=False) as tape: tree.map_structure(tape.watch, params) out = f(*args, **kwargs) if has_aux: out, aux = out grads = tape.gradient(out, params) if has_aux: return (out, aux), grads else: return out, grads return wrapper ================================================ FILE: sonnet/src/functional/jax_test.py ================================================ # Copyright 2020 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for Sonnet JAX interop layer.""" from absl.testing import parameterized from sonnet.src import test_utils from sonnet.src.functional import jax import tensorflow as tf class JaxTest(test_utils.TestCase, parameterized.TestCase): def test_jit_copies_to_device(self): accelerators = get_accelerators() if not accelerators: self.skipTest("No accelerator.") with tf.device("CPU"): x = tf.ones([]) self.assertTrue(x.device.endswith("CPU:0")) for device in accelerators: y = jax.jit(lambda x: x, device=device)(x) self.assertTrue(y.device, device) def test_device_put(self): accelerators = get_accelerators() if not accelerators: self.skipTest("No accelerator.") with tf.device("CPU"): x = tf.ones([]) for device in accelerators: y = jax.device_put(x, device=device) self.assertTrue(y.device.endswith(device)) class GradTest(test_utils.TestCase, parameterized.TestCase): def test_grad(self): f = lambda x: x ** 2 g = jax.grad(f) x = tf.constant(4.) self.assertAllClose(g(x).numpy(), (2 * x).numpy()) def test_argnums(self): f = lambda x, y: (x ** 2 + y ** 2) g = jax.grad(f, argnums=(0, 1)) x = tf.constant(4.) y = tf.constant(5.) gx, gy = g(x, y) self.assertAllClose(gx.numpy(), (2 * x).numpy()) self.assertAllClose(gy.numpy(), (2 * y).numpy(), rtol=1e-3) def test_has_aux(self): f = lambda x: (x ** 2, "aux") g = jax.grad(f, has_aux=True) x = tf.constant(2.) gx, aux = g(x) self.assertAllClose(gx.numpy(), (2 * x).numpy()) self.assertEqual(aux, "aux") def get_accelerators(): gpus = tf.config.experimental.list_logical_devices("GPU") tpus = tf.config.experimental.list_logical_devices("TPU") return [tf.DeviceSpec.from_string(d.name).to_string() for d in gpus + tpus] if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/functional/optimizers.py ================================================ # Copyright 2020 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Functional optimizers.""" import collections import functools from typing import Callable, Type from sonnet.src import base from sonnet.src.functional import haiku import tensorflow as tf import tree TransformedOptimizer = collections.namedtuple("TransformedOptimizer", ("init", "apply")) def optimizer(cls: Type[base.Optimizer]) -> Callable[..., TransformedOptimizer]: """Converts a snt.Optimizer subclass into a functional optimizer. To wrap a Sonnet optimizer class simply pass it to :func:`optimizer`: >>> adam = snt.functional.optimizer(snt.optimizers.Adam) This will give you back a function that drives the constructor of the optimizer and returns a pair of functions that give you the optimizer state and a way to apply it: >>> optimizer = adam(learning_rate=0.01) NOTE: We provide convenience wrappers for the builtin optimizers so you can just use `opt = snt.functional.adam(learning_rate=0.01)` if you prefer: >>> optimizer = snt.functional.adam(learning_rate=0.01) To make this example useful lets create a simple network to test: >>> with snt.functional.variables(): ... net = snt.nets.MLP([100, 10]) >>> def loss_fn(images, labels): ... logits = net(images) ... x_ent = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, ... labels=labels) ... loss = tf.reduce_mean(x_ent) ... return loss >>> loss_fn = snt.functional.transform(loss_fn) >>> x = tf.ones([1, 1]) >>> y = tf.constant([1]) >>> params = loss_fn.init(x, y) To get the initial state of our optimizer (e.g. m/v terms in Adam) we need to run the `optimizer.init` function: >>> opt_state = optimizer.init(params) Now we can run a single training step by taking gradients of our network and applying one step of our optimizer: >>> grad_apply_net = snt.functional.grad(loss_fn.apply) >>> def train_step(x, y, params, opt_state): ... grads = grad_apply_net(params, x, y) ... params, opt_state = optimizer.apply(opt_state, grads, params) ... return params, opt_state Teach the network to always predict one: >>> target = tf.constant([1]) >>> dataset = [(tf.random.normal([1, 1]), target) for _ in range(10)] >>> for x, y in dataset: ... params, opt_state = train_step(x, y, params, opt_state) Args: cls: A :class:`~sonnet.Optimizer` subclass to functionalize. Returns: A transformed optimizer with `init` and `apply`. See docstring for details. """ @functools.wraps(cls.__init__) def wrapper(*args, **kwargs): with haiku.variables(): opt = cls(*args, **kwargs) # pytype: disable=not-instantiable return _wrap_optimizer(opt) return wrapper def _split_on_trainable(opt_state): trainable = {} non_trainable = {} for param_ref, value in opt_state.items(): if param_ref.deref().trainable: trainable[param_ref] = value else: non_trainable[param_ref] = value return trainable, non_trainable def _merge(a, b): """Merges two dictionaries and returns a new one.""" c = dict(a) c.update(b) return c def _wrap_optimizer(opt: base.Optimizer) -> TransformedOptimizer: """Returns a functional optimizer.""" def init_opt_fn(params): """Creates initial optimizer state.""" def f(params): params = [p.deref() for p in sorted(params.keys())] updates = [tf.zeros_like(p) for p in params] for p, zero in zip(params, updates): p.assign(zero) opt.apply(updates, params) f = haiku.transform_with_state(f) trainable, non_trainable = f.init(params) opt_state = _merge( {r: v for r, v in trainable.items() if r not in params}, {r: v for r, v in non_trainable.items() if r not in params}) return opt_state def apply_opt_fn(opt_state, updates, params): """Applies the optimizer and returns updated parameters and opt state.""" def f(opt_state, params, updates): flat_params = [p.deref() for p in sorted(params)] updates = tree.flatten(updates) opt.apply(updates, flat_params) params = {r: r.deref().tensor_value for r in params} opt_state = {r: r.deref().tensor_value for r in opt_state} return params, opt_state f = haiku.transform_with_state(f) trainable_opt_state, non_trainable = _split_on_trainable(opt_state) trainable = _merge(params, trainable_opt_state) (params, opt_state), _ = f.apply(trainable, non_trainable, opt_state, params, updates) return params, opt_state return TransformedOptimizer(init=init_opt_fn, apply=apply_opt_fn) ================================================ FILE: sonnet/src/functional/optimizers_test.py ================================================ # Copyright 2020 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for functional optimizers.""" from absl.testing import parameterized import sonnet as snt from sonnet.src import test_utils from sonnet.src.functional import haiku from sonnet.src.functional import optimizers import tensorflow as tf import tree sgd = optimizers.optimizer(snt.optimizers.SGD) adam = optimizers.optimizer(snt.optimizers.Adam) class OptimizersTest(test_utils.TestCase, parameterized.TestCase): def test_sgd(self): with haiku.variables(): params = [tf.Variable(1.)] params = {p.ref(): tf.ones_like(p) for p in params} opt = sgd(learning_rate=0.01) opt_state = opt.init(params) grads = tree.map_structure(tf.ones_like, params) params, opt_state = opt.apply(opt_state, grads, params) p, = tree.flatten(params) self.assertAllClose(p.numpy(), 1. - (0.01 * 1)) def test_adam(self): lin = haiku.transform(snt.Linear(1)) x = tf.ones([1, 1]) params = lin.init(x) optimizer = adam(learning_rate=0.01) opt_state = optimizer.init(params) # Step + (m, v) per parameter. self.assertLen(tree.flatten(opt_state), 5) @parameterized.parameters(True, False) def test_adam_with_variable_lr(self, trainable_lr): lin = haiku.transform(snt.Linear(1)) x = tf.ones([1, 1]) initial_params = lin.init(x) with haiku.variables(): lr = tf.Variable(0.01, trainable=trainable_lr, name="lr") optimizer = adam(learning_rate=lr) initial_opt_state = optimizer.init(initial_params) # Learning rate, step + (m, v) per parameter. self.assertLen(tree.flatten(initial_opt_state), 6) grads = tree.map_structure(tf.ones_like, initial_params) params, opt_state = optimizer.apply( initial_opt_state, grads, initial_params) tree.assert_same_structure(initial_opt_state, opt_state) tree.assert_same_structure(initial_params, params) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/functional/utils.py ================================================ # Copyright 2020 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Utility functions for the JAX API in TF2.""" import functools from sonnet.src import utils import tensorflow as tf import tree def get_first_accelerator(): tpus = tf.config.experimental.list_logical_devices("TPU") if tpus: return tpus[0].name else: gpus = tf.config.experimental.list_logical_devices("GPU") return gpus[0].name if gpus else "/device:CPU:0" def run_on_device(f, device): """Runs `f` under a tf.device context on the given device.""" f = utils.smart_autograph(f) @tf.autograph.experimental.do_not_convert @functools.wraps(f) def wrapper(*args, **kwargs): with tf.device(device): args = tree.map_structure(tf.identity, args) kwargs = tree.map_structure(tf.identity, kwargs) return f(*args, **kwargs) return wrapper def get_name_scope(): with tf.name_scope("x") as ns: return ns[:-2] def first_non_none(*args): return next(a for a in args if a is not None) def compose(f0, *fs): """Composes a sequence of functions. >>> f1 = lambda a, b: f"f1({a}, {b})" >>> f2 = lambda a: f"f2({a})" >>> f3 = lambda a: f"f3({a})" >>> f = compose(f1, f2, f3) >>> f("a", "b") 'f3(f2(f1(a, b)))' Args: f0: The first function to apply. *fs: Other functions to apply in sequence. Returns: A function that is the composition of the input functions. """ def wrapper(*args, **kwargs): return functools.reduce(lambda x, f: f(x), fs, f0(*args, **kwargs)) return wrapper ================================================ FILE: sonnet/src/group_norm.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Group normalization implementation for Sonnet.""" import collections.abc from typing import Optional from sonnet.src import base from sonnet.src import initializers from sonnet.src import once from sonnet.src import types from sonnet.src import utils import tensorflow as tf class GroupNorm(base.Module): r"""Group normalization module. This applies group normalization to the inputs. This involves splitting the channels into groups before calculating the mean and variance. The default behaviour is to compute the mean and variance over the spatial dimensions and the grouped channels. The mean and variance will never be computed over the created groups axis. It transforms the input ``x`` into: .. math:: \d{outputs} = \d{scale} \dfrac{x - \mu}{\sigma + \epsilon} + \d{offset} Where :math:`\mu` and :math:`\sigma` are respectively the mean and standard deviation of ``x``. There are many different variations for how users want to manage scale and offset if they require them at all. These are: - No ``scale``/``offset`` in which case ``create_*`` should be set to ``False`` and ``scale``/``offset`` aren't passed when the module is called. - Trainable ``scale``/``offset`` in which case create_* should be set to ``True`` and again ``scale``/``offset`` aren't passed when the module is called. In this case this module creates and owns the scale/offset variables. - Externally generated ``scale``/``offset``, such as for conditional normalization, in which case ``create_*`` should be set to ``False`` and then the values fed in at call time. Attributes: scale: If ``create_scale=True``, a trainable :tf:`Variable` holding the current scale. offset: If ``create_offset=True``, a trainable :tf:`Variable` holding the current offset. """ def __init__(self, groups: int, axis: types.Axis = slice(1, None), create_scale: bool = True, create_offset: bool = True, eps: types.FloatLike = 1e-5, scale_init: Optional[initializers.Initializer] = None, offset_init: Optional[initializers.Initializer] = None, data_format: str = "channels_last", name: Optional[str] = None): """Constructs a ``GroupNorm`` module. Args: groups: number of groups to divide the channels by. The number of channels must be divisible by this. axis: ``int``, ``slice`` or sequence of ints representing the axes which should be normalized across. By default this is all but the first dimension. For time series data use `slice(2, None)` to average over the none Batch and Time data. create_scale: whether to create a trainable scale per channel applied after the normalization. create_offset: whether to create a trainable offset per channel applied after normalization and scaling. eps: Small epsilon to add to the variance to avoid division by zero. Defaults to ``1e-5``. scale_init: Optional initializer for the scale variable. Can only be set if ``create_scale=True``. By default scale is initialized to ``1``. offset_init: Optional initializer for the offset variable. Can only be set if ``create_offset=True``. By default offset is initialized to ``0``. data_format: The data format of the input. Can be either ``channels_first``, ``channels_last``, ``N...C`` or ``NC...``. By default it is ``channels_last``. name: Name of the module. """ super().__init__(name=name) if isinstance(axis, slice): self._axis = axis elif isinstance(axis, int): self._axis = [axis] elif (isinstance(axis, collections.abc.Iterable) and all(isinstance(ax, int) for ax in axis)): self._axis = axis else: raise ValueError("`axis` should be an int, slice or iterable of ints.") self._groups = groups self._eps = eps self._data_format = data_format self._channel_index = utils.get_channel_index(data_format) self._create_scale = create_scale self._create_offset = create_offset if self._create_scale: self._scale_init = ( scale_init if scale_init is not None else initializers.Ones()) elif scale_init is not None: raise ValueError("Cannot set `scale_init` if `create_scale=False`.") if self._create_offset: self._offset_init = ( offset_init if offset_init is not None else initializers.Zeros()) elif offset_init is not None: raise ValueError("Cannot set `offset_init` if `create_offset=False`.") def __call__(self, inputs: tf.Tensor, scale: Optional[tf.Tensor] = None, offset: Optional[tf.Tensor] = None): """Returns normalized inputs. Args: inputs: An n-D tensor of the ``data_format`` specified in the constructor on which the transformation is performed. scale: A tensor up to n-D. The shape of this tensor must be broadcastable to the shape of ``inputs``. This is the scale applied to the normalized inputs. This cannot be passed in if the module was constructed with ``create_scale=True``. offset: A tensor up to n-D. The shape of this tensor must be broadcastable to the shape of ``inputs``. This is the offset applied to the normalized ``inputs``. This cannot be passed in if the module was constructed with ``create_offset=True``. Returns: An n-d tensor of the same shape as inputs that has been normalized. """ self._initialize(inputs) if self._create_scale: if scale is not None: raise ValueError( "Cannot pass `scale` at call time if `create_scale=True`.") scale = self.scale if self._create_offset: if offset is not None: raise ValueError( "Cannot pass `offset` at call time if `create_offset=True`.") offset = self.offset if len(inputs.shape) != self._rank: raise ValueError( "The rank of the inputs cannot change between calls, the" " original call was rank={} but this call was rank={}.".format( self._rank, len(inputs.shape))) inputs = tf.reshape(inputs, self._inputs_reshape) mean, var = tf.nn.moments(inputs, self._axis, keepdims=True) normalized = tf.nn.batch_normalization( inputs, mean=mean, variance=var, scale=None, offset=None, variance_epsilon=self._eps) outputs = tf.reshape(normalized, self._outputs_reshape) outputs = outputs * scale if scale is not None else outputs outputs = outputs + offset if offset is not None else outputs return outputs @once.once def _initialize(self, inputs: tf.Tensor): """Setup of rank specific values.""" self._rank = len(inputs.shape) # Turns slice into list of axis if isinstance(self._axis, slice): axes = tuple(range(self._rank)) self._axis = axes[self._axis] # Create scale and offset variables dtype = inputs.dtype if self._channel_index == -1: params_shape = [inputs.shape[-1]] else: # self._channel_index == 1 params_shape = [inputs.shape[1]] + [1] * (self._rank - 2) if self._create_scale: self.scale = tf.Variable( self._scale_init(params_shape, dtype), name="scale") else: self.scale = None if self._create_offset: self.offset = tf.Variable( self._offset_init(params_shape, dtype), name="offset") else: self.offset = None num_channels = inputs.shape[self._channel_index] if num_channels % self._groups != 0: raise ValueError( "The number of channels must be divisible by the number of groups, " "was channels = {}, groups = {}".format(num_channels, self._groups)) if self._channel_index == -1: self._inputs_reshape = [-1] + list( inputs.shape[1:-1]) + [self._groups, num_channels // self._groups] self._axis = [a if a != self._rank - 1 else a + 1 for a in self._axis] else: self._inputs_reshape = [-1] + [ self._groups, num_channels // self._groups ] + list(inputs.shape[2:]) self._axis = [a if a == 0 else a + 1 for a in self._axis] self._outputs_reshape = [-1] + list(inputs.shape[1:]) ================================================ FILE: sonnet/src/group_norm_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.group_norm.""" from absl.testing import parameterized import numpy as np from sonnet.src import group_norm from sonnet.src import initializers from sonnet.src import test_utils import tensorflow as tf class GroupNormTest(test_utils.TestCase, parameterized.TestCase): def testSimpleCase(self): layer = group_norm.GroupNorm( groups=5, create_scale=False, create_offset=False) inputs = tf.ones([2, 3, 3, 10]) outputs = layer(inputs).numpy() for x in np.nditer(outputs): self.assertEqual(x, 0.0) def testSimpleCaseVar(self): layer = group_norm.GroupNorm( groups=5, create_scale=True, create_offset=True, scale_init=initializers.Constant(0.5), offset_init=initializers.Constant(2.0)) inputs = tf.ones([2, 3, 3, 10]) outputs = layer(inputs).numpy() for x in np.nditer(outputs): self.assertEqual(x, 2.0) def testSimpleCaseNCHWVar(self): layer = group_norm.GroupNorm( groups=5, create_scale=True, create_offset=True, scale_init=initializers.Constant(0.5), offset_init=initializers.Constant(2.0), data_format="NCHW") inputs = tf.ones([2, 10, 3, 3]) outputs = layer(inputs).numpy() for x in np.nditer(outputs): self.assertEqual(x, 2.0) def testDataFormatAgnosticVar(self): c_last_layer = group_norm.GroupNorm( groups=5, create_scale=True, create_offset=True) c_first_layer = group_norm.GroupNorm( groups=5, create_scale=True, create_offset=True, data_format="NCHW") inputs = tf.random.uniform([3, 4, 4, 10], 0, 10) c_last_output = c_last_layer(inputs) inputs = tf.transpose(inputs, [0, 3, 1, 2]) c_first_output = c_first_layer(inputs) c_first_output = tf.transpose(c_first_output, [0, 2, 3, 1]) self.assertAllClose(c_last_output.numpy(), c_first_output.numpy()) def testSimpleCaseTensor(self): layer = group_norm.GroupNorm( groups=5, create_scale=False, create_offset=False) inputs = tf.ones([2, 3, 3, 10]) scale = tf.constant(0.5, shape=(10,)) offset = tf.constant(2.0, shape=(10,)) outputs = layer(inputs, scale, offset).numpy() for x in np.nditer(outputs): self.assertEqual(x, 2.0) def testSimpleCaseNCHWTensor(self): layer = group_norm.GroupNorm( groups=5, data_format="NCHW", create_scale=False, create_offset=False) inputs = tf.ones([2, 10, 3, 3]) scale = tf.constant(0.5, shape=(10, 1, 1)) offset = tf.constant(2.0, shape=(10, 1, 1)) outputs = layer(inputs, scale, offset).numpy() for x in np.nditer(outputs): self.assertEqual(x, 2.0) def testDataFormatAgnosticTensor(self): c_last = group_norm.GroupNorm( groups=5, create_scale=False, create_offset=False) c_first = group_norm.GroupNorm( groups=5, data_format="NCHW", create_scale=False, create_offset=False) inputs = tf.random.uniform([3, 4, 4, 10], 0, 10) scale = tf.random.normal((10,), mean=1.0) offset = tf.random.normal((10,)) c_last_output = c_last(inputs, scale, offset) inputs = tf.transpose(inputs, [0, 3, 1, 2]) scale = tf.reshape(scale, (10, 1, 1)) offset = tf.reshape(offset, (10, 1, 1)) c_first_output = c_first(inputs, scale, offset) c_first_output = tf.transpose(c_first_output, [0, 2, 3, 1]) self.assertAllClose(c_last_output, c_first_output, rtol=1e-5) @parameterized.parameters("NHW", "HWC", "channel_last") def testInvalidDataFormat(self, data_format): with self.assertRaisesRegex( ValueError, "Unable to extract channel information from '{}'.".format(data_format)): group_norm.GroupNorm( groups=5, data_format=data_format, create_scale=False, create_offset=False) @parameterized.parameters("NCHW", "NCW", "channels_first") def testValidDataFormatChannelsFirst(self, data_format): test = group_norm.GroupNorm( groups=5, data_format=data_format, create_scale=False, create_offset=False) self.assertEqual(test._channel_index, 1) @parameterized.parameters("NHWC", "NWC", "channels_last") def testValidDataFormatChannelsLast(self, data_format): test = group_norm.GroupNorm( groups=5, data_format=data_format, create_scale=False, create_offset=False) self.assertEqual(test._channel_index, -1) @parameterized.named_parameters(("String", "foo"), ("ListString", ["foo"])) def testInvalidAxis(self, axis): with self.assertRaisesRegex( ValueError, "`axis` should be an int, slice or iterable of ints."): group_norm.GroupNorm( groups=5, axis=axis, create_scale=False, create_offset=False) def testNoScaleAndInitProvided(self): with self.assertRaisesRegex( ValueError, "Cannot set `scale_init` if `create_scale=False`."): group_norm.GroupNorm( groups=5, create_scale=False, create_offset=True, scale_init=initializers.Ones()) def testNoOffsetBetaInitProvided(self): with self.assertRaisesRegex( ValueError, "Cannot set `offset_init` if `create_offset=False`."): group_norm.GroupNorm( groups=5, create_scale=True, create_offset=False, offset_init=initializers.Zeros()) def testCreateScaleAndScaleProvided(self): layer = group_norm.GroupNorm( groups=5, create_scale=True, create_offset=False) with self.assertRaisesRegex( ValueError, "Cannot pass `scale` at call time if `create_scale=True`."): layer(tf.ones([2, 3, 5]), scale=tf.ones([4])) def testCreateOffsetAndOffsetProvided(self): layer = group_norm.GroupNorm( groups=5, create_offset=True, create_scale=False) with self.assertRaisesRegex( ValueError, "Cannot pass `offset` at call time if `create_offset=True`."): layer(tf.ones([2, 3, 5]), offset=tf.ones([4])) def testSliceAxis(self): slice_layer = group_norm.GroupNorm( groups=5, create_scale=False, create_offset=False) axis_layer = group_norm.GroupNorm( groups=5, create_scale=False, create_offset=False) inputs = tf.random.uniform([3, 4, 4, 5], 0, 10) scale = tf.random.normal((5,), mean=1.0) offset = tf.random.normal((5,)) slice_outputs = slice_layer(inputs, scale, offset) axis_outputs = axis_layer(inputs, scale, offset) self.assertAllEqual(slice_outputs.numpy(), axis_outputs.numpy()) def testRankChanges(self): layer = group_norm.GroupNorm( groups=5, create_scale=False, create_offset=False) inputs = tf.ones([2, 3, 3, 5]) scale = tf.constant(0.5, shape=(5,)) offset = tf.constant(2.0, shape=(5,)) layer(inputs, scale, offset) with self.assertRaisesRegex( ValueError, "The rank of the inputs cannot change between calls, the original"): layer(tf.ones([2, 3, 3, 4, 5]), scale, offset) @parameterized.named_parameters(("Small", (2, 4, 4)), ("Bigger", (2, 3, 8))) def testIncompatibleGroupsAndTensor(self, shape): layer = group_norm.GroupNorm( groups=5, create_scale=False, create_offset=False) inputs = tf.ones(shape) with self.assertRaisesRegex( ValueError, "The number of channels must be divisible by the number of groups"): layer(inputs) def testWorksWithFunction(self): layer = group_norm.GroupNorm( groups=5, create_scale=False, create_offset=False) function_layer = tf.function(layer) inputs = tf.ones([2, 3, 3, 10]) scale = tf.constant(0.5, shape=(10,)) offset = tf.constant(2.0, shape=(10,)) outputs = layer(inputs, scale, offset) function_outputs = function_layer(inputs, scale, offset) self.assertAllEqual(outputs.numpy(), function_outputs.numpy()) def testBatchSizeAgnostic(self): layer = group_norm.GroupNorm( groups=5, create_scale=False, create_offset=False) inputs_spec = tf.TensorSpec([None, 3, 3, 10], dtype=tf.float32) params_spec = tf.TensorSpec([None], dtype=tf.float32) function_layer = tf.function(layer).get_concrete_function( inputs_spec, params_spec, params_spec) scale = tf.constant(0.5, shape=(10,)) offset = tf.constant(2.0, shape=(10,)) outputs = function_layer(tf.ones([2, 3, 3, 10]), scale, offset) self.assertEqual(outputs.shape, [2, 3, 3, 10]) for x in np.nditer(outputs): self.assertEqual(x, 2.0) scale = tf.constant(0.5, shape=(10,)) offset = tf.constant(2.0, shape=(10,)) outputs = function_layer(tf.ones([3, 3, 3, 10]), scale, offset) self.assertEqual(outputs.shape, [3, 3, 3, 10]) for x in np.nditer(outputs): self.assertEqual(x, 2.0) def test5DDataFormatAgnostic(self): c_last_layer = group_norm.GroupNorm( groups=5, create_scale=False, create_offset=False) c_first_layer = group_norm.GroupNorm( groups=5, create_scale=False, create_offset=False, data_format="NCDHW") inputs = tf.random.uniform([3, 4, 4, 4, 10], 0, 10) scale = tf.random.normal((10,), mean=1.0) offset = tf.random.normal((10,)) c_last_output = c_last_layer(inputs, scale, offset) inputs = tf.transpose(inputs, [0, 4, 1, 2, 3]) scale = tf.reshape(scale, [-1, 1, 1, 1]) offset = tf.reshape(offset, [-1, 1, 1, 1]) c_first_output = c_first_layer(inputs, scale, offset) c_first_output = tf.transpose(c_first_output, [0, 2, 3, 4, 1]) self.assertAllClose( c_last_output.numpy(), c_first_output.numpy(), atol=1e-5, rtol=1e-5) def test3DDataFormatAgnostic(self): c_last_layer = group_norm.GroupNorm( groups=5, create_scale=False, create_offset=False) c_first_layer = group_norm.GroupNorm( groups=5, create_scale=False, create_offset=False, data_format="NCW") inputs = tf.random.uniform([3, 4, 10], 0, 10) scale = tf.random.normal((10,), mean=1.0) offset = tf.random.normal((10,)) c_last_output = c_last_layer(inputs, scale, offset) inputs = tf.transpose(inputs, [0, 2, 1]) scale = tf.reshape(scale, [-1, 1]) offset = tf.reshape(offset, [-1, 1]) c_first_output = c_first_layer(inputs, scale, offset) c_first_output = tf.transpose(c_first_output, [0, 2, 1]) self.assertAllClose( c_last_output.numpy(), c_first_output.numpy(), atol=1e-5, rtol=1e-5) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/initializers.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Initializers for Sonnet.""" import abc import collections from typing import Iterable, Mapping, Optional, Union import numpy as np from sonnet.src import types import tensorflow as tf class Initializer(abc.ABC): """Initializer base class, all initializers must implement a call method.""" @abc.abstractmethod def __call__(self, shape: types.ShapeLike, dtype: tf.DType) -> tf.Tensor: """Returns a tensor of the given ``shape`` and ``dtype``.""" pass class Zeros(Initializer): """Initializer that generates tensors initialized to 0.""" def __call__(self, shape: types.ShapeLike, dtype: tf.DType) -> tf.Tensor: dtype = _as_numerical_dtype(dtype) return tf.zeros(shape, dtype) class Ones(Initializer): """Initializer that generates tensors initialized to 1.""" def __call__(self, shape: types.ShapeLike, dtype: tf.DType) -> tf.Tensor: dtype = _as_numerical_dtype(dtype) return tf.ones(shape, dtype) class Constant(Initializer): """Initializer that generates tensors initialized to the given value.""" def __init__(self, value: Union[float, int]): if not np.isscalar(value): raise TypeError("Invalid type for value: {} (expected scalar).".format( type(value))) self.value = value def __call__(self, shape: types.ShapeLike, dtype: tf.DType) -> tf.Tensor: dtype = _as_numerical_dtype(dtype) value = tf.convert_to_tensor(self.value, dtype) return tf.fill(value=value, dims=shape) class RandomUniform(Initializer): """Initializer that generates tensors with a uniform distribution. The generated values follow a uniform distribution in the range ``[minval, maxval)``. """ def __init__(self, minval: types.FloatLike = 0, maxval: types.FloatLike = 1, seed: Optional[int] = None): """Constructs a random uniform initializer. Args: minval: A python scalar or a scalar tensor. Lower bound of the range of random values to generate. Defaults to ``0``. maxval: A python scalar or a scalar tensor. Upper bound of the range of random values to generate. Defaults to ``1``. seed: The seed used in the generation of random numbers. """ self.minval = minval self.maxval = maxval self.seed = seed def __call__(self, shape: types.ShapeLike, dtype: tf.DType): dtype = _as_numerical_dtype(dtype) return tf.random.uniform( shape=shape, minval=self.minval, maxval=self.maxval, dtype=dtype, seed=self.seed) class RandomNormal(Initializer): """Initializer that generates tensors with a normal distribution.""" def __init__(self, mean: types.FloatLike = 0.0, stddev: types.FloatLike = 1.0, seed: Optional[int] = None): """Constructs a random normal initializer. Args: mean: A python scalar or a scalar tensor. Mean of the random values to generate. stddev: A python scalar or a scalar tensor. Standard deviation of the random values to generate. seed: The seed used in the generation of random numbers. """ self.mean = mean self.stddev = stddev self.seed = seed def __call__(self, shape: types.ShapeLike, dtype: tf.DType) -> tf.Tensor: dtype = _as_floating_dtype(dtype) return tf.random.normal( shape=shape, mean=self.mean, stddev=self.stddev, dtype=dtype, seed=self.seed) class TruncatedNormal(Initializer): """Initializer that generates a truncated normal distribution. These values follow a normal distribution except that values more than two standard deviations from the mean are discarded and re-drawn. This is the recommended initializer for neural network weights and filters. """ def __init__(self, mean: types.FloatLike = 0.0, stddev: types.FloatLike = 1.0, seed: Optional[int] = None): """Constructs a truncated normal initializer. Args: mean: A python scalar or a scalar tensor. Mean of the random values to generate. stddev: A python scalar or a scalar tensor. Standard deviation of the random values to generate. seed: The seed used in the generation of random numbers. """ self.mean = mean self.stddev = stddev self.seed = seed def __call__(self, shape: types.ShapeLike, dtype: tf.DType): dtype = _as_floating_dtype(dtype) return tf.random.truncated_normal( shape=shape, mean=self.mean, stddev=self.stddev, dtype=dtype, seed=self.seed) class Identity(Initializer): """Initializer that generates the identity matrix. Constructs a 2D identity matrix or batches of these. """ def __init__(self, gain: float = 1.0): """Constructs an identity initializer. Args: gain: Multiplicative factor to apply to the identity matrix. """ self.gain = gain def __call__(self, shape: types.ShapeLike, dtype: tf.DType) -> tf.Tensor: dtype = _as_numerical_dtype(dtype) rank = shape.shape[0] if isinstance(shape, tf.Tensor) else len(shape) if rank < 2: raise ValueError("The tensor to initialize must be " "at least two-dimensional") elif rank == 2: initializer = tf.eye(num_rows=shape[0], num_columns=shape[1], dtype=dtype) else: # rank > 2 initializer = tf.eye( num_rows=shape[-2], num_columns=shape[-1], batch_shape=shape[:-2], dtype=dtype) return self.gain * initializer class Orthogonal(Initializer): """Initializer that generates an orthogonal matrix. NOTE: Does not support 1D tensors. The implementation is based on :cite:`saxe2013exact`. If the shape of the tensor to initialize is two-dimensional, it is initialized with an orthogonal matrix obtained from the QR decomposition of a matrix of random numbers drawn from a normal distribution. If the matrix has fewer rows than columns then the output will have orthogonal rows. Otherwise, the output will have orthogonal columns. If the shape of the tensor to initialize is more than two-dimensional, a matrix of shape ``(shape[0] * ... * shape[n - 2], shape[n - 1])`` is initialized, where ``n`` is the length of the shape vector. The matrix is subsequently reshaped to give a tensor of the desired shape. """ def __init__(self, gain: float = 1.0, seed: Optional[int] = None): """Constructs an orthogonal initializer. Args: gain: Multiplicative factor to apply to the orthogonal matrix seed: ``int``, the seed used in the generation of random numbers. """ self.gain = gain self.seed = seed def __call__(self, shape: types.ShapeLike, dtype: tf.DType) -> tf.Tensor: dtype = _as_floating_dtype(dtype) if len(shape) < 2: raise ValueError("The tensor to initialize must be " "at least two-dimensional") # Flatten the input shape with the last dimension remaining # its original shape so it works for conv2d num_rows = 1 for dim in shape[:-1]: num_rows *= dim num_cols = shape[-1] flat_shape = [ tf.maximum(num_cols, num_rows), tf.minimum(num_cols, num_rows) ] # Generate a random matrix a = tf.random.normal(flat_shape, dtype=dtype, seed=self.seed) # Compute the qr factorization q, r = tf.linalg.qr(a, full_matrices=False) # Make Q uniform d = tf.linalg.tensor_diag_part(r) q *= tf.sign(d) if num_rows < num_cols: q = tf.linalg.matrix_transpose(q) return self.gain * tf.reshape(q, shape) class VarianceScaling(Initializer): """Initializer capable of adapting its scale to the shape of weights tensors. With ``distribution="truncated_normal" or "normal"``, samples are drawn from a distribution with a mean of zero and a standard deviation (after truncation, if used) ``stddev = sqrt(scale / n)`` where ``n`` is: - Number of input units in the weight tensor, if ``mode = fan_in``. - Number of output units, if ``mode = fan_out``. - Average of the numbers of input and output units, if ``mode = fan_avg``. Note that for transposed convolution the mode selected should be reversed. For number of input units use ``fan_out`` and for number of output units ``fan_in``. With ``distribution=uniform``, samples are drawn from a uniform distribution within ``[-limit, limit]``, with ``limit = sqrt(3 * scale / n)``. The variance scaling initializer can be configured to generate other standard initializers using the scale, mode and distribution arguments. Here are some example configurations: ============== ============================================================== Name Parameters ============== ============================================================== glorot_uniform scale=1.0, mode=``fan_avg``, distribution=``uniform`` glorot_normal scale=1.0, mode=``fan_avg``, distribution=``truncated_normal`` lecun_uniform scale=1.0, mode=``fan_in``, distribution=``uniform`` lecun_normal scale=1.0, mode=``fan_in``, distribution=``truncated_normal`` he_uniform scale=2.0, mode=``fan_in``, distribution=``uniform`` he_normal scale=2.0, mode=``fan_in``, distribution=``truncated_normal`` ============== ============================================================== """ def __init__(self, scale: float = 1.0, mode: str = "fan_in", distribution: str = "truncated_normal", seed: Optional[int] = None): """Constructs a variance scaling initalizer. Args: scale: Scaling factor (positive ``float``). mode: One of ``fan_in``, ``fan_out``, ``fan_avg``. distribution: Random distribution to use. One of ``truncated_normal``, ``untruncated_normal`` and ``uniform``. seed: ``int``, the seed used in the generation of random numbers. Raises: ValueError: In case of an invalid value for the ``scale``, ``mode`` or ``distribution`` arguments. """ if scale <= 0.: raise ValueError("`scale` must be positive float.") if mode not in {"fan_in", "fan_out", "fan_avg"}: raise ValueError("Invalid `mode` argument:", mode) distribution = distribution.lower() if distribution not in {"uniform", "truncated_normal", "normal"}: raise ValueError("Invalid `distribution` argument:", distribution) self.scale = scale self.mode = mode self.distribution = distribution self.seed = seed def __call__(self, shape: types.ShapeLike, dtype: tf.DType) -> tf.Tensor: dtype = _as_floating_dtype(dtype) scale = self.scale fan_in, fan_out = _compute_fans(shape) fan_in = tf.cast(fan_in, dtype) fan_out = tf.cast(fan_out, dtype) if self.mode == "fan_in": scale /= tf.maximum(1., fan_in) elif self.mode == "fan_out": scale /= tf.maximum(1., fan_out) else: scale /= tf.maximum(1., (fan_in + fan_out) / 2.) if self.distribution == "truncated_normal": # constant from scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.) distribution_stddev = .87962566103423978 stddev = tf.sqrt(scale) / distribution_stddev return tf.random.truncated_normal( shape=shape, mean=0.0, stddev=stddev, dtype=dtype, seed=self.seed) elif self.distribution == "normal": stddev = tf.sqrt(scale) return tf.random.normal( shape=shape, mean=0.0, stddev=stddev, dtype=dtype, seed=self.seed) else: # self.distribution == "uniform" limit = tf.sqrt(3.0 * scale) return tf.random.uniform( shape=shape, minval=-limit, maxval=limit, dtype=dtype, seed=self.seed) def check_initializers(initializers: Mapping[str, Initializer], expected_keys: Iterable[str]): """Checks a dictionary of initializers only contains the given keys.""" if initializers is None: return {} if not isinstance(initializers, collections.abc.Mapping): raise TypeError("Initializers must be a dict-like object.") extra_keys = frozenset(initializers) - frozenset(expected_keys) if extra_keys: raise KeyError("Invalid initializer keys {}, initializers can only " "be provided for {}".format( ", ".join(map(repr, extra_keys)), ", ".join(map(repr, expected_keys)))) return initializers def _compute_fans(shape: types.ShapeLike): """Computes the number of input and output units for a weight shape. Args: shape: Integer shape tuple or `tf.TensorShape`. Returns: A tuple of scalars `(fan_in, fan_out)`. """ if len(shape) < 1: # Just to avoid errors for constants. fan_in = fan_out = 1 elif len(shape) == 1: fan_in = fan_out = shape[0] elif len(shape) == 2: fan_in = shape[0] fan_out = shape[1] else: # Assuming convolution kernels (2D, 3D, or more). # kernel shape: (..., input_depth, depth) receptive_field_size = 1. for dim in shape[:-2]: receptive_field_size *= dim fan_in = shape[-2] * receptive_field_size fan_out = shape[-1] * receptive_field_size return fan_in, fan_out def _as_floating_dtype(dtype: tf.DType) -> tf.DType: dtype = tf.as_dtype(dtype) if dtype.is_floating: return dtype raise ValueError("Expected floating point type, got {}".format(dtype)) def _as_numerical_dtype(dtype: tf.DType) -> tf.DType: dtype = tf.as_dtype(dtype) if dtype.is_floating or dtype.is_integer: return dtype raise ValueError( "Expected integer or floating point type, got {}".format(dtype)) ================================================ FILE: sonnet/src/initializers_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.initializers.""" import itertools from absl.testing import parameterized import numpy as np from sonnet.src import initializers from sonnet.src import test_utils import tensorflow as tf class InitializersTest(test_utils.TestCase, parameterized.TestCase): def assertDifferentInitializerValues(self, init, shape=None, dtype=tf.float32): if shape is None: shape = (100,) t1 = self.evaluate(init(shape, dtype)) t2 = self.evaluate(init(shape, dtype)) self.assertEqual(t1.shape, shape) self.assertEqual(t2.shape, shape) self.assertFalse(np.allclose(t1, t2, rtol=1e-15, atol=1e-15)) def assertRange(self, init, shape, target_mean=None, target_std=None, target_max=None, target_min=None, dtype=tf.float32): output = self.evaluate(init(shape, dtype)) self.assertEqual(output.shape, shape) lim = 4e-2 if target_std is not None: self.assertNear(output.std(), target_std, err=lim) if target_mean is not None: self.assertNear(output.mean(), target_mean, err=lim) if target_max is not None: self.assertNear(output.max(), target_max, err=lim) if target_min is not None: self.assertNear(output.min(), target_min, err=lim) class ConstantInitializersTest(InitializersTest): @parameterized.parameters(tf.float32, tf.int32) def testZeros(self, dtype): self.assertRange( initializers.Zeros(), shape=(4, 5), target_mean=0., target_max=0., dtype=dtype) @parameterized.parameters(tf.float32, tf.int32) def testOnes(self, dtype): self.assertRange( initializers.Ones(), shape=(4, 5), target_mean=1., target_max=1., dtype=dtype) @parameterized.named_parameters( ("Tensor", lambda: tf.constant([1.0, 2.0, 3.0]), "Tensor"), ("Variable", lambda: tf.Variable([3.0, 2.0, 1.0]), "Variable"), ("List", lambda: [], "list"), ("Tuple", lambda: (), "tuple")) def testConstantInvalidValue(self, value, value_type): with self.assertRaisesRegex( TypeError, r"Invalid type for value: .*{}.*".format(value_type)): initializers.Constant(value()) @parameterized.parameters((42, tf.float32), (42.0, tf.float32), (42, tf.int32)) def testConstantValidValue(self, value, dtype): self.assertRange( initializers.Constant(value), shape=(4, 5), target_mean=42., target_max=42., dtype=dtype) @parameterized.parameters(initializers.Zeros, initializers.Ones) def testInvalidDataType(self, initializer): init = initializer() with self.assertRaisesRegex( ValueError, r"Expected integer or floating point type, got "): init([1], dtype=tf.string) def testInvalidDataTypeConstant(self): init = initializers.Constant(0) with self.assertRaisesRegex( ValueError, r"Expected integer or floating point type, got "): init([1], dtype=tf.string) def testTFFunction(self): init = initializers.Constant(2) f = tf.function(lambda t: init(tf.shape(t), t.dtype)) expected = init([7, 4], tf.float32) x = f(tf.zeros([7, 4])) self.assertAllEqual(expected, x) def testBatchAgnostic(self): init = initializers.Constant(2) spec = tf.TensorSpec(shape=[None, None]) f = tf.function(lambda t: init(tf.shape(t), t.dtype)) f = f.get_concrete_function(spec) expected = init([7, 4], tf.float32) x = f(tf.ones([7, 4])) self.assertAllEqual(expected, x) class RandomUniformInitializerTest(InitializersTest): def testRangeInitializer(self): shape = (16, 8, 128) self.assertRange( initializers.RandomUniform(minval=-1., maxval=1., seed=124.), shape, target_mean=0., target_max=1, target_min=-1) @parameterized.parameters(tf.float32, tf.int32) def testDifferentInitializer(self, dtype): init = initializers.RandomUniform(0, 10) self.assertDifferentInitializerValues(init, dtype=dtype) def testInvalidDataType(self): init = initializers.RandomUniform() with self.assertRaisesRegex( ValueError, r"Expected integer or floating point type, got "): init([1], dtype=tf.string) def testTFFunction(self): init = initializers.RandomUniform(seed=42) f = tf.function(lambda t: init(tf.shape(t), t.dtype)) expected = init([7, 4], tf.float32) x = f(tf.zeros([7, 4])) self.assertEqual(x.shape, [7, 4]) if self.primary_device != "TPU": # Seeds don't work as expected on TPU self.assertAllEqual(expected, x) def testBatchAgnostic(self): init = initializers.RandomUniform(seed=42) spec = tf.TensorSpec(shape=[None, None]) f = tf.function(lambda t: init(tf.shape(t), t.dtype)) f = f.get_concrete_function(spec) expected = init([7, 4], tf.float32) x = f(tf.ones([7, 4])) self.assertEqual(x.shape, [7, 4]) if self.primary_device != "TPU": # Seeds don't work as expected on TPU self.assertAllEqual(expected, x) class RandomNormalInitializerTest(InitializersTest): def testRangeInitializer(self): self.assertRange( initializers.RandomNormal(mean=0, stddev=1, seed=153), shape=(16, 8, 128), target_mean=0., target_std=1) def testDifferentInitializer(self): init = initializers.RandomNormal(0.0, 1.0) self.assertDifferentInitializerValues(init) @parameterized.parameters(tf.int32, tf.string) def testInvalidDataType(self, dtype): init = initializers.RandomNormal(0.0, 1.0) with self.assertRaisesRegex(ValueError, r"Expected floating point type, got "): init([1], dtype=dtype) def testTFFunction(self): init = initializers.RandomNormal(seed=42) f = tf.function(lambda t: init(tf.shape(t), t.dtype)) expected = init([7, 4], tf.float32) x = f(tf.zeros([7, 4])) self.assertEqual(x.shape, [7, 4]) if self.primary_device != "TPU": # Seeds don't work as expected on TPU self.assertAllEqual(expected, x) def testBatchAgnostic(self): init = initializers.RandomNormal(seed=42) spec = tf.TensorSpec(shape=[None, None]) f = tf.function(lambda t: init(tf.shape(t), t.dtype)) f = f.get_concrete_function(spec) expected = init([7, 4], tf.float32) x = f(tf.ones([7, 4])) self.assertEqual(x.shape, [7, 4]) if self.primary_device != "TPU": # Seeds don't work as expected on TPU self.assertAllEqual(expected, x) class TruncatedNormalInitializerTest(InitializersTest): def testRangeInitializer(self): self.assertRange( initializers.TruncatedNormal(mean=0, stddev=1, seed=126), shape=(16, 8, 128), target_mean=0., target_max=2, target_min=-2) def testDifferentInitializer(self): init = initializers.TruncatedNormal(0.0, 1.0) self.assertDifferentInitializerValues(init) @parameterized.parameters(tf.int32, tf.string) def testInvalidDataType(self, dtype): init = initializers.TruncatedNormal(0.0, 1.0) with self.assertRaisesRegex(ValueError, r"Expected floating point type, got "): init([1], dtype=dtype) def testTFFunction(self): init = initializers.TruncatedNormal(seed=42) f = tf.function(lambda t: init(tf.shape(t), t.dtype)) expected = init([7, 4], tf.float32) x = f(tf.zeros([7, 4])) self.assertEqual(x.shape, [7, 4]) if self.primary_device != "TPU": # Seeds don't work as expected on TPU self.assertAllEqual(expected, x) def testBatchAgnostic(self): init = initializers.TruncatedNormal(seed=42) spec = tf.TensorSpec(shape=[None, None]) f = tf.function(lambda t: init(tf.shape(t), t.dtype)) f = f.get_concrete_function(spec) expected = init([7, 4], tf.float32) x = f(tf.ones([7, 4])) self.assertEqual(x.shape, [7, 4]) if self.primary_device != "TPU": # Seeds don't work as expected on TPU self.assertAllEqual(expected, x) class IdentityInitializerTest(InitializersTest): @parameterized.parameters( *itertools.product([(4, 5), (3, 3), (3, 4, 5), (6, 2, 3, 3)], [3, 1], [tf.float32, tf.int32])) def testRange(self, shape, gain, dtype): if self.primary_device == "GPU" and dtype == tf.int32: self.skipTest("tf.int32 not supported on GPU") self.assertRange( initializers.Identity(gain), shape=shape, target_mean=gain / shape[-1], target_max=gain, dtype=dtype) def testInvalidDataType(self): init = initializers.Identity() with self.assertRaisesRegex( ValueError, r"Expected integer or floating point type, got "): init([1, 2], dtype=tf.string) @parameterized.parameters(tf.float32, tf.int32) def testInvalidShape(self, dtype): init = initializers.Identity() with self.assertRaisesRegex( ValueError, "The tensor to initialize must be at least two-dimensional"): init([1], dtype=dtype) def testTFFunction(self): init = initializers.Identity() f = tf.function(lambda t: init(tf.shape(t), t.dtype)) expected = init([4, 4], tf.float32) x = f(tf.ones([4, 4])) self.assertAllEqual(expected, x) def testTFFunction4D(self): init = initializers.Identity() f = tf.function(lambda t: init(tf.shape(t), t.dtype)) expected = init([4, 4, 3, 2], tf.float32) x = f(tf.ones([4, 4, 3, 2])) self.assertAllEqual(expected, x) def testBatchAgnostic(self): init = initializers.Identity() spec = tf.TensorSpec(shape=[None, None]) f = tf.function(lambda t: init(tf.shape(t), t.dtype)) f = f.get_concrete_function(spec) expected = init([7, 4], tf.float32) x = f(tf.ones([7, 4])) self.assertAllEqual(expected, x) class OrthogonalInitializerTest(InitializersTest): def testRangeInitializer(self): self.assertRange( initializers.Orthogonal(seed=123), shape=(20, 20), target_mean=0.) def testDuplicatedInitializer(self): init = initializers.Orthogonal() self.assertDifferentInitializerValues(init, (10, 10)) @parameterized.parameters(tf.int32, tf.string) def testInvalidDataType(self, dtype): init = initializers.Orthogonal() with self.assertRaisesRegex(ValueError, r"Expected floating point type, got "): init([1, 2], dtype=dtype) def testInvalidShape(self): init = initializers.Orthogonal() with self.assertRaisesRegex( ValueError, "The tensor to initialize must be at least two-dimensional"): init([1], tf.float32) @parameterized.named_parameters( ("Square", (10, 10)), ("3DSquare", (100, 5, 5)), ("3DRectangle", (10, 9, 8)), ("TallRectangle", (50, 40)), ("WideRectangle", (40, 50))) def testShapesValues(self, shape): init = initializers.Orthogonal() tol = 1e-5 t = self.evaluate(init(shape, tf.float32)) self.assertAllEqual(tuple(shape), t.shape) # Check orthogonality by computing the inner product t = t.reshape((np.prod(t.shape[:-1]), t.shape[-1])) if t.shape[0] > t.shape[1]: self.assertAllClose( np.dot(t.T, t), np.eye(t.shape[1]), rtol=tol, atol=tol) else: self.assertAllClose( np.dot(t, t.T), np.eye(t.shape[0]), rtol=tol, atol=tol) def testTFFunctionSimple(self): init = initializers.Orthogonal(seed=42) f = tf.function(init) x = f([4, 4], tf.float32) self.assertAllEqual(x.shape, [4, 4]) def testTFFunction(self): if self.primary_device == "TPU": self.skipTest("Dynamic slice not supported on TPU") init = initializers.Orthogonal(seed=42) f = tf.function(lambda t: init(tf.shape(t), t.dtype)) expected = init([4, 4], tf.float32) x = f(tf.ones([4, 4])) self.assertAllEqual(expected, x) def testBatchAgnostic(self): if self.primary_device == "TPU": self.skipTest("Dynamic slice not supported on TPU") init = initializers.Orthogonal(seed=42) spec = tf.TensorSpec(shape=[None, None]) f = tf.function(lambda t: init(tf.shape(t), t.dtype)) f = f.get_concrete_function(spec) expected = init([7, 4], tf.float32) x = f(tf.ones([7, 4])) self.assertAllEqual(expected, x) class VarianceScalingInitializerTest(InitializersTest): def testTruncatedNormalDistribution(self): shape = (100, 100) init = initializers.VarianceScaling(distribution="truncated_normal") self.assertRange( init, shape=shape, target_mean=0., target_std=1. / np.sqrt(shape[0])) def testNormalDistribution(self): shape = (100, 100) init = initializers.VarianceScaling(distribution="normal") self.assertRange( init, shape=shape, target_mean=0., target_std=1. / np.sqrt(shape[0])) def testUniformDistribution(self): shape = (100, 100) init = initializers.VarianceScaling(distribution="uniform") self.assertRange( init, shape=shape, target_mean=0., target_std=1. / np.sqrt(shape[0])) def testGlorotUniform(self): shape = (5, 6, 4, 2) fan_in, fan_out = initializers._compute_fans(shape) std = np.sqrt(2. / (fan_in + fan_out)) self.assertRange( initializers.VarianceScaling( scale=1.0, mode="fan_avg", distribution="uniform", seed=123), shape, target_mean=0., target_std=std) def test_GlorotNormal(self): shape = (5, 6, 4, 2) fan_in, fan_out = initializers._compute_fans(shape) std = np.sqrt(2. / (fan_in + fan_out)) self.assertRange( initializers.VarianceScaling( scale=1.0, mode="fan_avg", distribution="truncated_normal", seed=123), shape, target_mean=0., target_std=std) def testLecunUniform(self): shape = (5, 6, 4, 2) fan_in, _ = initializers._compute_fans(shape) std = np.sqrt(1. / fan_in) self.assertRange( initializers.VarianceScaling( scale=1.0, mode="fan_in", distribution="uniform", seed=123), shape, target_mean=0., target_std=std) def testLecunNormal(self): shape = (5, 6, 4, 2) fan_in, _ = initializers._compute_fans(shape) std = np.sqrt(1. / fan_in) self.assertRange( initializers.VarianceScaling( scale=1.0, mode="fan_in", distribution="truncated_normal", seed=123), shape, target_mean=0., target_std=std) def testHeUniform(self): shape = (5, 6, 4, 2) fan_in, _ = initializers._compute_fans(shape) std = np.sqrt(2. / fan_in) self.assertRange( initializers.VarianceScaling( scale=2.0, mode="fan_in", distribution="uniform", seed=123), shape, target_mean=0., target_std=std) def testHeNormal(self): shape = (5, 6, 4, 2) fan_in, _ = initializers._compute_fans(shape) std = np.sqrt(2. / fan_in) self.assertRange( initializers.VarianceScaling( scale=2.0, mode="fan_in", distribution="truncated_normal", seed=123), shape, target_mean=0., target_std=std) @parameterized.parameters( itertools.product(["fan_in", "fan_out", "fan_avg"], ["uniform", "truncated_normal", "normal"])) def testMixedShape(self, mode, distribution): init = initializers.VarianceScaling(mode=mode, distribution=distribution) tf.random.set_seed(42) x = init([tf.constant(4), 2], tf.float32) tf.random.set_seed(42) expected = init([4, 2], tf.float32) self.assertEqual(x.shape, [4, 2]) if self.primary_device != "TPU": # Seeds don't work as expected on TPU self.assertAllEqual(expected, x) @parameterized.parameters( itertools.product(["fan_in", "fan_out", "fan_avg"], ["uniform", "truncated_normal", "normal"])) def testWithTFFunction(self, mode, distribution): init = initializers.VarianceScaling( mode=mode, distribution=distribution, seed=42) f = tf.function(lambda t: init(tf.shape(t), t.dtype)) x = f(tf.zeros([4, 2])) expected = init([4, 2], tf.float32) self.assertEqual(x.shape, [4, 2]) if self.primary_device != "TPU": # Seeds don't work as expected on TPU self.assertAllClose(expected, x) @parameterized.parameters( itertools.product(["fan_in", "fan_out", "fan_avg"], ["uniform", "truncated_normal", "normal"])) def testBatchAgnostic(self, mode, distribution): init = initializers.VarianceScaling( mode=mode, distribution=distribution, seed=42) spec = tf.TensorSpec(shape=[None, None]) f = tf.function(lambda t: init(tf.shape(t), t.dtype)) f = f.get_concrete_function(spec) expected = init([7, 4], tf.float32) x = f(tf.ones([7, 4])) self.assertEqual(x.shape, [7, 4]) if self.primary_device != "TPU": # Seeds don't work as expected on TPU self.assertAllClose(expected, x) @parameterized.parameters(tf.int32, tf.string) def testInvalidDataType(self, dtype): init = initializers.VarianceScaling() with self.assertRaisesRegex(ValueError, r"Expected floating point type, got "): init([1, 2], dtype=dtype) def testCheckInitializersInvalidType(self): with self.assertRaisesRegex(TypeError, "Initializers must be a dict-like object."): initializers.check_initializers([1, 2, 3], ("a")) def testCheckInitalizersEmpty(self): a = initializers.check_initializers(None, ("b")) self.assertEqual(a, {}) @parameterized.named_parameters(("Tuple", ("a", "b")), ("List", ["a", "b"]), ("Set", {"a", "b"})) def testCheckInitalizersValid(self, keys): initializers.check_initializers({ "a": lambda x, y: 0, "b": lambda x, y: 1 }, keys) def testCheckInitalizersInvalid(self): with self.assertRaisesRegex( KeyError, r"Invalid initializer keys 'a', initializers can only be provided for"): initializers.check_initializers({ "a": lambda x, y: 0, "b": lambda x, y: 1 }, ("b")) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/leaky_clip_by_value.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Clipping operation with customized gradients.""" from typing import Optional import tensorflow as tf @tf.custom_gradient def leaky_clip_by_value(t: tf.Tensor, clip_value_min: tf.Tensor, clip_value_max: tf.Tensor, name: Optional[str] = None): """Clips tensor values to a specified min and max. The gradient is set to zero when tensor values are already out of bound and gradient-descent will push them even further away from the valid range. If gradient-descent pushes the values towards the valid range, the gradient will pass through without change. Note that this is assuming a gradient flow for minimization. For maximization, flip the gradient before it back-propagates to this op. Args: t: A Tensor. clip_value_min: A 0-D (scalar) Tensor, or a Tensor with the same shape as t. The minimum value to clip by. clip_value_max: A 0-D (scalar) Tensor, or a Tensor with the same shape as t. The maximum value to clip by. name: A name for the operation (optional). Returns: A clipped Tensor. Raises: ValueError: If the clip tensors would trigger array broadcasting that would make the returned tensor larger than the input. """ clip_t = tf.clip_by_value(t, clip_value_min, clip_value_max, name=name) def grad(dy): """Custom gradient.""" zeros = tf.zeros_like(dy) condition = tf.logical_or( tf.logical_and(t < clip_value_min, dy > 0), tf.logical_and(t > clip_value_max, dy < 0), ) dy = tf.where(condition, zeros, dy) return dy, None, None return clip_t, grad ================================================ FILE: sonnet/src/leaky_clip_by_value_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.leaky_clip_by_value.""" from absl.testing import parameterized from sonnet.src import leaky_clip_by_value from sonnet.src import test_utils import tensorflow as tf class LeakyClipByValueTest(test_utils.TestCase, parameterized.TestCase): def test_leaky_clip_by_value_forward(self): t = tf.Variable([1.0, 2.0, 3.0]) # Test when min/max are scalar values. clip_min = [1.5] clip_max = [2.5] clip_t = leaky_clip_by_value.leaky_clip_by_value(t, clip_min, clip_max) self.assertAllEqual(clip_t.numpy(), [1.5, 2.0, 2.5]) # Test when min/max are of same sizes as t. clip_min_array = [0.5, 2.5, 2.5] clip_max_array = [1.5, 3.0, 3.5] clip_t_2 = leaky_clip_by_value.leaky_clip_by_value(t, clip_min_array, clip_max_array) self.assertAllEqual(clip_t_2.numpy(), [1.0, 2.5, 3.0]) @parameterized.parameters([ (0.5, lambda x: x, [1.0]), (1.5, lambda x: x, [1.0]), (1.5, lambda x: -x, [0.0]), (-.5, lambda x: x, [0.0]), (-.5, lambda x: -x, [-1.0]), ]) def test_leaky_clip_by_value_backward(self, init, fn, expected_grad): t = tf.Variable([init]) max_val = 1.0 min_val = 0.0 with tf.GradientTape() as tape: clip_t = leaky_clip_by_value.leaky_clip_by_value(t, min_val, max_val) f = fn(clip_t) grad = tape.gradient(f, t) clip_t_value = clip_t.numpy() self.assertAllEqual(grad.numpy(), expected_grad) self.assertGreaterEqual(clip_t_value, min_val) self.assertLessEqual(clip_t_value, max_val) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/linear.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Linear module.""" import math from typing import Optional from sonnet.src import base from sonnet.src import initializers from sonnet.src import once from sonnet.src import utils import tensorflow as tf class Linear(base.Module): """Linear module, optionally including bias.""" def __init__(self, output_size: int, with_bias: bool = True, w_init: Optional[initializers.Initializer] = None, b_init: Optional[initializers.Initializer] = None, name: Optional[str] = None): """Constructs a `Linear` module. Args: output_size: Output dimensionality. with_bias: Whether to include bias parameters. Default `True`. w_init: Optional initializer for the weights. By default the weights are initialized truncated random normal values with a standard deviation of `1 / sqrt(input_feature_size)`, which is commonly used when the inputs are zero centered (see https://arxiv.org/abs/1502.03167v3). b_init: Optional initializer for the bias. By default the bias is initialized to zero. name: Name of the module. """ super().__init__(name=name) self.output_size = output_size self.with_bias = with_bias self.w_init = w_init if with_bias: self.b_init = b_init if b_init is not None else initializers.Zeros() elif b_init is not None: raise ValueError("When not using a bias the b_init must be None.") @once.once def _initialize(self, inputs: tf.Tensor): """Constructs parameters used by this module.""" utils.assert_minimum_rank(inputs, 2) input_size = inputs.shape[-1] if input_size is None: # Can happen inside an @tf.function. raise ValueError("Input size must be specified at module build time.") self.input_size = input_size if self.w_init is None: # See https://arxiv.org/abs/1502.03167v3. stddev = 1 / math.sqrt(self.input_size) self.w_init = initializers.TruncatedNormal(stddev=stddev) self.w = tf.Variable( self.w_init([self.input_size, self.output_size], inputs.dtype), name="w") if self.with_bias: self.b = tf.Variable( self.b_init([self.output_size], inputs.dtype), name="b") def __call__(self, inputs: tf.Tensor) -> tf.Tensor: self._initialize(inputs) outputs = tf.matmul(inputs, self.w) if self.with_bias: outputs = tf.add(outputs, self.b) return outputs ================================================ FILE: sonnet/src/linear_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.linear.""" from absl.testing import parameterized import numpy as np from sonnet.src import linear from sonnet.src import test_utils import tensorflow as tf class LinearTest(test_utils.TestCase, parameterized.TestCase): def testInitW(self): my_initializer = lambda shape, dtype: None mod = linear.Linear(1, w_init=my_initializer) self.assertIs(mod.w_init, my_initializer) def testInitB(self): my_initializer = lambda shape, dtype: None mod = linear.Linear(1, b_init=my_initializer) self.assertIs(mod.b_init, my_initializer) def testInitializerKeysInvalidWithoutBias(self): with self.assertRaisesRegex(ValueError, "b_init must be None"): linear.Linear(1, with_bias=False, b_init=tf.zeros_initializer()) def testParametersCreatedOnce(self): mod = linear.Linear(1) mod(tf.constant([[1.]])) w, b = mod.w, mod.b mod(tf.constant([[1.]])) self.assertIs(mod.w, w) self.assertIs(mod.b, b) def testParameterShape(self): batch_size = 1 input_size = 2 output_size = 3 mod = linear.Linear(output_size) mod(tf.ones([batch_size, input_size])) self.assertEqual(mod.w.shape.as_list(), [input_size, output_size]) self.assertEqual(mod.b.shape.as_list(), [output_size]) @parameterized.parameters([tf.float16, tf.float32, tf.int32]) def testParameterDtype(self, dtype): if dtype == tf.int32 and self.primary_device in ("GPU", "TPU"): self.skipTest("int32 not supported on %s" % self.primary_device) elif self.primary_device == "TPU" and dtype == tf.float16: dtype = tf.bfloat16 mod = linear.Linear(1, w_init=tf.zeros_initializer()) out = mod(tf.ones([1, 1], dtype=dtype)) self.assertEqual(out.dtype, dtype) self.assertEqual(mod.w.dtype, dtype) self.assertEqual(mod.b.dtype, dtype) def testBiasZeroInitialized(self): mod = linear.Linear(1) mod(tf.constant([[1.]])) self.assertEqual(mod.b.numpy(), [0.]) def testCall(self): batch_size = 1 input_size = 2 output_size = 3 def numpy_linear(): w = np.ndarray([input_size, output_size], dtype=np.float32) w.fill(2.) b = np.ndarray([output_size], dtype=np.float32) b.fill(3.) i = np.ones([batch_size, input_size], dtype=np.float32) return np.matmul(i, w) + b l = linear.Linear( output_size, w_init=tf.constant_initializer(2.), b_init=tf.constant_initializer(3.)) tf_output = l(tf.ones([batch_size, input_size])) self.assertAllEqual(tf_output, numpy_linear()) def testCallMultiBatch(self): l = linear.Linear(5) input_tensor = tf.random.uniform([1, 2, 3, 4]) tf_output = l(input_tensor) w_np = l.w.numpy() b_np = l.b.numpy() input_tensor_np = input_tensor.numpy() np_output = np.matmul(input_tensor_np, w_np) + b_np # TPU uses bfloat16 internally, so larger deviations are expected. self.assertAllClose(tf_output, np_output, atol=1e-2, rtol=5e-2) @parameterized.parameters(True, False) def testFunction(self, with_bias): linear_1 = linear.Linear( 3, with_bias=with_bias, w_init=tf.ones_initializer()) linear_2 = linear.Linear( 3, with_bias=with_bias, w_init=tf.ones_initializer()) defun_linear = tf.function(linear_2) iterations = 5 for _ in range(iterations): x = tf.random.uniform([1, 5]) y1 = linear_1(x) y2 = defun_linear(x) self.assertAllClose(self.evaluate(y1), self.evaluate(y2), atol=1e-4) def testUnknownBatchSize(self): x = tf.TensorSpec([None, 4], dtype=tf.float32) l = linear.Linear(3) defun_linear = tf.function(l) defun_linear.get_concrete_function(x) out = defun_linear(tf.ones([2, 4])) expected_out = l(tf.ones([2, 4])) self.assertEqual(out.shape, [2, 3]) self.assertAllEqual(self.evaluate(expected_out), self.evaluate(out)) out = defun_linear(tf.ones([4, 4])) self.assertEqual(out.shape, [4, 3]) def testUnknownInputSize(self): x = tf.TensorSpec([None, None], dtype=tf.float32) l = linear.Linear(3) defun_linear = tf.function(l) with self.assertRaisesRegex( ValueError, "Input size must be specified at module build time."): defun_linear.get_concrete_function(x) def testMultiBatchOutputDimensions(self): x = tf.TensorSpec([None, None, None, 2], dtype=tf.float32) l = linear.Linear(7) defun_linear = tf.function(l) defun_linear.get_concrete_function(x) out = defun_linear(tf.ones([1, 5, 3, 2])) expected_out = l(tf.ones([1, 5, 3, 2])) self.assertEqual(out.shape, [1, 5, 3, 7]) self.assertAllEqual(self.evaluate(expected_out), self.evaluate(out)) out = defun_linear(tf.ones([2, 4, 5, 2])) self.assertEqual(out.shape, [2, 4, 5, 7]) @parameterized.named_parameters(("1D", [1]),) def testIncorrectDims(self, shape): l = linear.Linear(3) with self.assertRaisesRegex(ValueError, "Shape .* must have rank >= 2"): l(tf.ones(shape)) def testInputSize(self): batch_size = 1 input_size = 2 output_size = 3 mod = linear.Linear(output_size) mod(tf.ones([batch_size, input_size])) self.assertEqual(mod.input_size, input_size) def testOutputSize(self): mod = linear.Linear(1) self.assertEqual(mod.output_size, 1) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/metrics.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Base class for general metrics within Sonnet.""" import abc from typing import Optional from sonnet.src import base from sonnet.src import once import tensorflow as tf class Metric(base.Module, metaclass=abc.ABCMeta): """Metric base class.""" @abc.abstractmethod def initialize(self, value): """Creates any input dependent variables or state.""" @abc.abstractmethod def update(self, value): """Accumulates values.""" @abc.abstractproperty def value(self): """Returns the current value of the metric.""" @abc.abstractmethod def reset(self): """Resets the metric.""" def __call__(self, value): """Updates the metric and returns the new value.""" self.update(value) return self.value class Sum(Metric): """Calculates the element-wise sum of the given values.""" def __init__(self, name: Optional[str] = None): super().__init__(name=name) self.sum = None @once.once def initialize(self, value: tf.Tensor): """See base class.""" self.sum = tf.Variable(tf.zeros_like(value), trainable=False, name="sum") def update(self, value: tf.Tensor): """See base class.""" self.initialize(value) self._checked_sum.assign_add(value) @property def _checked_sum(self): if self.sum is None: raise ValueError("Metric is not initialized. Call `initialize` first.") return self.sum @property def value(self) -> tf.Tensor: """See base class.""" return tf.convert_to_tensor(self.sum) def reset(self): """See base class.""" if self.sum is None: raise ValueError("Metric is not initialized. Call `initialize` first.") self.sum.assign(tf.zeros_like(self.sum)) class Mean(Metric): """Calculates the element-wise mean of the given values.""" def __init__(self, name: Optional[str] = None): super().__init__(name=name) self.sum = None self.count = tf.Variable(0, dtype=tf.int64, trainable=False, name="count") @once.once def initialize(self, value: tf.Tensor): """See base class.""" self.sum = tf.Variable(tf.zeros_like(value), trainable=False, name="sum") def update(self, value: tf.Tensor): """See base class.""" self.initialize(value) self._checked_sum.assign_add(value) self.count.assign_add(1) @property def _checked_sum(self) -> tf.Variable: if self.sum is None: raise ValueError("Metric is not initialized. Call `initialize` first.") return self.sum @property def value(self) -> tf.Tensor: """See base class.""" # TODO(cjfj): Assert summed type is floating-point? return self._checked_sum / tf.cast( self.count, dtype=self._checked_sum.dtype ) def reset(self): self._checked_sum.assign(tf.zeros_like(self._checked_sum)) self.count.assign(0) ================================================ FILE: sonnet/src/metrics_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.metrics.""" from sonnet.src import metrics from sonnet.src import test_utils import tensorflow as tf class SumTest(test_utils.TestCase): def testSimple(self): acc = metrics.Sum() self.assertAllEqual([2., 3.], acc(tf.constant([2., 3.]))) self.assertAllEqual([6., 8.], acc(tf.constant([4., 5.]))) def testInitialize(self): acc = metrics.Sum() acc.initialize(tf.constant([1., 2.])) self.assertAllEqual([0., 0.], acc.value) def testReset(self): acc = metrics.Sum() self.assertAllEqual([2., 3.], acc(tf.constant([2., 3.]))) self.assertAllEqual([6., 8.], acc(tf.constant([4., 5.]))) acc.reset() self.assertAllEqual([7., 8.], acc(tf.constant([7., 8.]))) class MeanTest(test_utils.TestCase): def testSimple(self): mean = metrics.Mean() self.assertAllEqual([2., 3.], mean(tf.constant([2., 3.]))) self.assertAllEqual([3., 4.], mean(tf.constant([4., 5.]))) def testInitialize(self): mean = metrics.Mean() mean.initialize(tf.constant([1., 2.])) self.assertAllEqual([1., 2.], mean(tf.constant([1., 2.]))) def testReset(self): mean = metrics.Mean() self.assertAllEqual([2., 3.], mean(tf.constant([2., 3.]))) self.assertAllEqual([3., 4.], mean(tf.constant([4., 5.]))) mean.reset() self.assertAllEqual([7., 8.], mean(tf.constant([7., 8.]))) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/mixed_precision.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Mixed Precision Decorator for Sonnet 2.""" import contextlib import uuid from sonnet.src import custom_getter from sonnet.src import utils import tensorflow as tf import tree # TODO(loreno): Make this a thread local variable _mixed_precision_mode = None _MP_SEEN_PROPERTY = '_mp_seen' def enable(dtype): """Set the mixed precision mode. Args: dtype: type to cast to. """ global _mixed_precision_mode _mixed_precision_mode = dtype def disable(): """Disable mixed precision training.""" enable(None) def _get_mixed_precision_mode(): return _mixed_precision_mode # TODO(loreno): Consider casting non-tensor/variable inputs def _maybe_cast_element(x, dtype): if isinstance(x, (tf.Tensor, tf.Variable)) and x.dtype.is_floating: x = tf.cast(x, dtype) return x def _maybe_cast_structure(x, dtype: tf.DType): return tree.map_structure(lambda x: _maybe_cast_element(x, dtype), x) def _cast_call(f, new_dtype, args, kwargs): """Runs the function with all tensor/variable arguments casted.""" # TODO(loreno): Implement more granular casting, not all Tensors/Variables args = _maybe_cast_structure(args, new_dtype) kwargs = _maybe_cast_structure(kwargs, new_dtype) # TODO(loreno): Remove float32 hardcode and replace with original dtype with custom_getter.custom_variable_getter( lambda x: _maybe_cast_structure(x, new_dtype)): ret = f(*args, **kwargs) return _maybe_cast_structure(ret, tf.float32) def modes(valid_types): """Decorate a function to cast inputs/outputs to different precision. >>> support_modes = snt.mixed_precision.modes([tf.float32, tf.float16]) >>> snt.Linear.__call__ = support_modes(snt.Linear.__call__) >>> mod = snt.Linear(10) >>> snt.mixed_precision.enable(tf.float16) >>> y = mod(tf.ones([1, 1])) # First call will be done in F32. >>> y = mod(tf.ones([1, 1])) # MatMul/Add will be done in F16. >>> snt.mixed_precision.disable() Args: valid_types: Collection of types that the function being decorated is legal to run in. Returns: A decorator that will cast the inputs and outputs of the decorated function according to the global mixed precision policy and the functions eligibility for mixed precision. """ mp_id = uuid.uuid4() @utils.decorator def _wrapper(f, instance, args, kwargs): """Decorator to cast inputs and outputs for mixed precision. Args: f: function to handle mixed precision casting for. instance: instance of f. args: positional arguments to f. kwargs: keyword arguments to f. Returns: A wrapped version of `f` that casts input Variables and Tensors to the global mixed_precision_mode dtype if that dtype is legal for this function as determined by `valid_types`. """ new_dtype = _get_mixed_precision_mode() if new_dtype is None or new_dtype not in valid_types: # TODO(loreno): consider throwing an error or doing nothing if input dtype # doesn't match any valid types return f(*args, **kwargs) if instance is None: if not _wrapper.seen_none: # TODO(loreno): Make this thread safe res = f(*args, **kwargs) _wrapper.seen_none = True return res return _cast_call(f, new_dtype, args, kwargs) else: seen = getattr(instance, _MP_SEEN_PROPERTY, None) if seen is None: seen = set() # TODO(loreno): use a weakrefset to address instances that define slots setattr(instance, _MP_SEEN_PROPERTY, seen) if mp_id not in seen: res = f(*args, **kwargs) seen.add(mp_id) return res return _cast_call(f, new_dtype, args, kwargs) _wrapper.seen_none = False return _wrapper @contextlib.contextmanager def scope(dtype: tf.DType): """Temporarily set the global mixed precision type to dtype. The global type is reset to its original value when the context is exited.:: snt.mixed_precision.enable(tf.float32) support_modes = snt.mixed_precision.modes([tf.float32, tf.float16]) snt.Linear.__call__ = support_modes(snt.Linear.__call__) mod = snt.Linear(10) with snt.mixed_precision.scope(tf.float16): y = mod(tf.ones([1, 1])) # First call will be done in F32. y = mod(tf.ones([1, 1])) # MatMul/Add will be done in F16. y = mod(tf.ones([1, 1])) # Outside the scope will be done in F32. Args: dtype: type to set the mixed precision mode to. Yields: Nothing. This is required for contextlib.contextmanager. """ # TODO(petebu) Make this a doctest once python2 is deprecated old_mode = _get_mixed_precision_mode() enable(dtype) try: yield finally: enable(old_mode) ================================================ FILE: sonnet/src/mixed_precision_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for Mixed Precision.""" from absl.testing import parameterized from sonnet.src import base from sonnet.src import mixed_precision from sonnet.src import test_utils import tensorflow as tf import tree class DummyVar(base.Module, test_utils.TestCase): def __init__(self, x): super().__init__() test_utils.TestCase.__init__(self) self.x = x def check_type(self, _, dtype): # TODO(loreno): handle dictionaries with non-sortable keys and change # this test to assertEqual once that works self.assertTrue(self.x.dtype == dtype) # pylint: disable=g-generic-assert return self.x def check_type_structure(self, _, dtype): # pylint: disable=g-generic-assert tree.map_structure(lambda y: self.assertTrue(y.dtype == dtype), self.x) return self.x def runTest(self): pass class DummyInput(test_utils.TestCase): def __init__(self, _): super().__init__() test_utils.TestCase.__init__(self) def check_type(self, x, dtype): self.assertEqual(x.dtype, dtype) return x def check_type_structure(self, x, dtype): tree.map_structure(lambda y: self.assertEqual(y.dtype, dtype), x) return x def runTest(self): pass @parameterized.parameters(DummyVar, DummyInput) class MixedPrecisionClassTest(test_utils.TestCase): def test_float16_mode_variable_eligible_class(self, test_class): mixed_precision.enable(tf.float32) x = tf.Variable([[1., 9.], [5., 0.]]) d = test_class(x) d.check_type = mixed_precision.modes([tf.float32, tf.float16])(d.check_type) mixed_precision.enable(tf.float16) # First call to forward fn always runs in full precision. self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) # Subsequent calls run in mixed precision. self.assertEqual(d.check_type(x, tf.float16).dtype, tf.float32) def test_float16_mode_disable_class(self, test_class): mixed_precision.enable(tf.float32) x = tf.Variable([[1., 9.], [5., 0.]]) d = test_class(x) d.check_type = mixed_precision.modes([tf.float32, tf.float16])(d.check_type) mixed_precision.enable(tf.float16) self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) self.assertEqual(d.check_type(x, tf.float16).dtype, tf.float32) mixed_precision.disable() self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) def test_float16_mode_nested_eligible_class(self, test_class): mixed_precision.enable(tf.float32) # TODO(loreno): test nested combo of tensor and Variables once the custom # variable getter can cast tensors. x = tf.Variable([[1., 9.], [5., 0.]]) y = tf.Variable([[1., 9.], [8., 9.]]) z = (x, y) d = test_class(z) d.check_type_structure = mixed_precision.modes([tf.float32, tf.float16])( d.check_type_structure) self.assertTrue(tree.is_nested(z)) mixed_precision.enable(tf.float16) first_run = d.check_type_structure(z, tf.float32) self.assertEqual(first_run[0].dtype, tf.float32) self.assertEqual(first_run[1].dtype, tf.float32) second_run = d.check_type_structure(z, tf.float16) self.assertEqual(second_run[0].dtype, tf.float32) self.assertEqual(second_run[1].dtype, tf.float32) def test_float16_mode_eligible_multiple_instances_class(self, test_class): mixed_precision.enable(tf.float32) x = tf.Variable([[1., 9.], [5., 0.]]) d = test_class(x) d.check_type = mixed_precision.modes([tf.float32, tf.float16])(d.check_type) d2 = test_class(x) d2.check_type = mixed_precision.modes([tf.float32, tf.float16])( d2.check_type) mixed_precision.enable(tf.float16) self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) self.assertEqual(d.check_type(x, tf.float16).dtype, tf.float32) self.assertEqual(d2.check_type(x, tf.float32).dtype, tf.float32) self.assertEqual(d2.check_type(x, tf.float16).dtype, tf.float32) def test_float16_mode_ineligible_multiple_instances_class(self, test_class): mixed_precision.enable(tf.float32) x = tf.Variable([[1., 9.], [5., 0.]]) d = test_class(x) d.check_type = mixed_precision.modes([tf.float32, tf.bfloat16])( d.check_type) d2 = test_class(x) d2.check_type = mixed_precision.modes([tf.float32, tf.bfloat16])( d2.check_type) mixed_precision.enable(tf.float16) self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) self.assertEqual(d2.check_type(x, tf.float32).dtype, tf.float32) self.assertEqual(d2.check_type(x, tf.float32).dtype, tf.float32) def test_float16_mode_multiple_instances_different_eligibility_class( self, test_class): mixed_precision.enable(tf.float32) x = tf.Variable([[1., 9.], [5., 0.]]) d = test_class(x) d.check_type = mixed_precision.modes([tf.float32, tf.bfloat16])( d.check_type) d2 = test_class(x) d2.check_type = mixed_precision.modes([tf.float32, tf.float16])( d2.check_type) mixed_precision.enable(tf.float16) self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) self.assertEqual(d2.check_type(x, tf.float32).dtype, tf.float32) self.assertEqual(d2.check_type(x, tf.float16).dtype, tf.float32) def test_bfloat16_input_float16_mode_eligible_class(self, test_class): mixed_precision.enable(tf.float32) x = tf.Variable([[1., 9.], [5., 0.]], dtype=tf.bfloat16) d = test_class(x) d.check_type = mixed_precision.modes([tf.float32, tf.float16])(d.check_type) mixed_precision.enable(tf.float16) self.assertEqual(d.check_type(x, tf.bfloat16).dtype, tf.bfloat16) self.assertEqual(d.check_type(x, tf.float16).dtype, tf.float32) def test_float16_input_float32_mode_eligible_class(self, test_class): if self.primary_device == 'TPU': self.skipTest('float16 not supported on TPU') mixed_precision.enable(tf.float32) x = tf.Variable([[1., 9.], [5., 0.]], dtype=tf.float16) d = test_class(x) d.check_type = mixed_precision.modes([tf.float32, tf.float16])(d.check_type) self.assertEqual(d.check_type(x, tf.float16).dtype, tf.float16) self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) def test_function_create_module_eligible(self, test_class): mixed_precision.enable(tf.float16) @mixed_precision.modes([tf.float32, tf.float16]) def model(): x = tf.Variable([[1., 9.], [8., 9.]]) d = test_class(x) d.check_type = mixed_precision.modes([tf.float32, tf.float16])( d.check_type) self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) self.assertEqual(d.check_type(x, tf.float16).dtype, tf.float32) model() def test_function_create_module_ineligible(self, test_class): mixed_precision.enable(tf.float16) @mixed_precision.modes([tf.float32, tf.float16]) def model(): x = tf.Variable([[1., 9.], [8., 9.]]) d = test_class(x) d.check_type = mixed_precision.modes([tf.float32, tf.bfloat16])( d.check_type) self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) model() def test_function_create_module_not_decorated(self, test_class): mixed_precision.enable(tf.float16) @mixed_precision.modes([tf.float32, tf.float16]) def model(): x = tf.Variable([[1., 9.], [8., 9.]]) d = test_class(x) self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) model() def test_scoping_option(self, test_class): mixed_precision.enable(tf.float32) x = tf.Variable([[1., 9.], [8., 9.]]) d = test_class(x) d.check_type = mixed_precision.modes([tf.float32, tf.float16])(d.check_type) with mixed_precision.scope(tf.float16): self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) self.assertEqual(d.check_type(x, tf.float16).dtype, tf.float32) self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) def test_scoping_disable(self, test_class): mixed_precision.enable(tf.float32) x = tf.Variable([[1., 9.], [8., 9.]]) d = test_class(x) d.check_type = mixed_precision.modes([tf.float32, tf.float16])(d.check_type) with mixed_precision.scope(tf.float16): self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) self.assertEqual(d.check_type(x, tf.float16).dtype, tf.float32) mixed_precision.disable() self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) def test_nested_scoping(self, test_class): mixed_precision.enable(tf.float32) x = tf.Variable([[1., 9.], [8., 9.]]) d = test_class(x) d.check_type = mixed_precision.modes([tf.float32, tf.float16])(d.check_type) with mixed_precision.scope(tf.float16): self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) self.assertEqual(d.check_type(x, tf.float16).dtype, tf.float32) with mixed_precision.scope(tf.float32): self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) with mixed_precision.scope(tf.float16): self.assertEqual(d.check_type(x, tf.float16).dtype, tf.float32) self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) class MixedPrecisionTest(test_utils.TestCase): def test_float16_mode_eligible_func(self): mixed_precision.enable(tf.float32) self.assertEqual(mixed_precision._get_mixed_precision_mode(), tf.float32) @mixed_precision.modes([tf.float32, tf.float16]) def check_type(x, expected_dtype): self.assertEqual(x.dtype, expected_dtype) return x mixed_precision.enable(tf.float16) x = tf.Variable([[1., 3], [5., 7.]]) self.assertEqual(x.dtype, tf.float32) self.assertEqual(check_type(x, tf.float32).dtype, tf.float32) self.assertEqual(check_type(x, tf.float16).dtype, tf.float32) def test_float32_mode_eligible_func(self): mixed_precision.enable(tf.float32) self.assertEqual(mixed_precision._get_mixed_precision_mode(), tf.float32) @mixed_precision.modes([tf.float32, tf.float16]) def fwd_func(x): self.assertEqual(x.dtype, tf.float32) return x x = tf.Variable([[1., 3], [5., 7.]]) self.assertEqual(x.dtype, tf.float32) self.assertEqual(fwd_func(x).dtype, tf.float32) self.assertEqual(fwd_func(x).dtype, tf.float32) def test_float16_mode_ineligible_func(self): mixed_precision.enable(tf.float32) @mixed_precision.modes([tf.float32, tf.bfloat16]) def fwd_func(x): self.assertEqual(x.dtype, tf.float32) return x x = tf.Variable([[1., 3], [5., 7.]]) self.assertEqual(x.dtype, tf.float32) mixed_precision.enable(tf.float16) self.assertEqual(fwd_func(x).dtype, tf.float32) self.assertEqual(fwd_func(x).dtype, tf.float32) def test_dont_cast_non_floats_func(self): mixed_precision.enable(tf.float32) @mixed_precision.modes([tf.float32, tf.float16]) def fwd_func(x): self.assertTrue(x.dtype.is_integer) return x x = tf.Variable([[1, 9], [8, 9]]) self.assertTrue(x.dtype.is_integer) mixed_precision.enable(tf.float16) self.assertTrue(fwd_func(x).dtype.is_integer) self.assertTrue(fwd_func(x).dtype.is_integer) def test_non_tensor_variable_input_no_cast_func(self): mixed_precision.enable(tf.float32) @mixed_precision.modes([tf.float32, tf.float16]) def fwd_func(x): self.assertEqual(type(x[0][0]), float) return x x = [[1., 3], [5., 7.]] self.assertEqual(type(x[0][0]), float) mixed_precision.enable(tf.float16) self.assertEqual(type(fwd_func(x)[0][0]), float) self.assertEqual(type(fwd_func(x)[0][0]), float) def test_float16_mode_enabled_call_function(self): mixed_precision.enable(tf.float32) class DummyCall(base.Module, test_utils.TestCase): def __init__(self): super().__init__() test_utils.TestCase.__init__(self) self.y = tf.Variable([[1., 3], [5., 7.]]) @mixed_precision.modes([tf.float16, tf.float32]) def __call__(self, x, dtype): # pylint: disable=g-generic-assert self.assertTrue(self.y.dtype == dtype) self.assertTrue(x.dtype == dtype) return x def runTest(self): pass x = tf.Variable([[1., 3], [5., 7.]]) self.assertEqual(x.dtype, tf.float32) d = DummyCall() mixed_precision.enable(tf.float16) self.assertEqual(d(x, tf.float32).dtype, tf.float32) self.assertEqual(d(x, tf.float16).dtype, tf.float32) # TODO(loreno): Run this test against custom variable getters once they can # handle and cast tensors def test_float16_mode_tensor_eligible_class(self): mixed_precision.enable(tf.float32) x = tf.constant([[1., 9.], [5., 0.]]) d = DummyInput(x) d.check_type = mixed_precision.modes([tf.float32, tf.float16])(d.check_type) mixed_precision.enable(tf.float16) self.assertEqual(d.check_type(x, tf.float32).dtype, tf.float32) self.assertEqual(d.check_type(x, tf.float16).dtype, tf.float32) if __name__ == '__main__': tf.test.main() ================================================ FILE: sonnet/src/moving_averages.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Exponential moving average for Sonnet.""" from typing import Optional, cast from sonnet.src import metrics from sonnet.src import once from sonnet.src import types import tensorflow as tf class ExponentialMovingAverage(metrics.Metric): """Maintains an exponential moving average for a value. Note this module uses debiasing by default. If you don't want this please use an alternative implementation. This module keeps track of a hidden exponential moving average that is initialized as a vector of zeros which is then normalized to give the average. This gives us a moving average which isn't biased towards either zero or the initial value. Reference (https://arxiv.org/pdf/1412.6980.pdf) Initially: hidden_0 = 0 Then iteratively: hidden_i = (hidden_{i-1} - value) * (1 - decay) average_i = hidden_i / (1 - decay^i) Attributes: average: Variable holding average. Note that this is None until the first value is passed. """ def __init__(self, decay: types.FloatLike, name: Optional[str] = None): """Creates a debiased moving average module. Args: decay: The decay to use. Note values close to 1 result in a slow decay whereas values close to 0 result in faster decay, tracking the input values more closely. name: Name of the module. """ super().__init__(name=name) self._decay = decay self._counter = tf.Variable( 0, trainable=False, dtype=tf.int64, name="counter") self._hidden: tf.Variable = cast(tf.Variable, None) self.average: tf.Variable = cast(tf.Variable, None) def update(self, value: tf.Tensor): """Applies EMA to the value given.""" self.initialize(value) self._counter.assign_add(1) value = tf.convert_to_tensor(value) counter = tf.cast(self._counter, value.dtype) self._hidden.assign_sub((self._hidden - value) * (1 - self._decay)) self.average.assign((self._hidden / (1. - tf.pow(self._decay, counter)))) @property def value(self) -> tf.Tensor: """Returns the current EMA.""" return self.average.read_value() def reset(self): """Resets the EMA.""" self._counter.assign(tf.zeros_like(self._counter)) if self._hidden is not None: self._hidden.assign(tf.zeros_like(self._hidden)) if self.average is not None: self.average.assign(tf.zeros_like(self.average)) @once.once def initialize(self, value: tf.Tensor): self._hidden = tf.Variable( tf.zeros_like(value), trainable=False, name="hidden") self.average = tf.Variable( tf.zeros_like(value), trainable=False, name="average") ================================================ FILE: sonnet/src/moving_averages_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.moving_averages.""" from absl.testing import parameterized from sonnet.src import moving_averages from sonnet.src import test_utils import tensorflow as tf class ExponentialMovingAverageTest(test_utils.TestCase, parameterized.TestCase): def testCall(self): ema = moving_averages.ExponentialMovingAverage(0.50) self.assertAllClose(ema(3.0).numpy(), 3.0) self.assertAllClose(ema(6.0).numpy(), 5.0) def testUpdateAndValue(self): ema = moving_averages.ExponentialMovingAverage(0.50) ema.update(3.0) self.assertAllClose(ema.value.numpy(), 3.0, atol=1e-3, rtol=1e-5) ema.update(6.0) self.assertAllClose(ema.value.numpy(), 5.0, atol=1e-3, rtol=1e-5) def testReset(self): ema = moving_averages.ExponentialMovingAverage(0.90) self.assertAllClose(ema(3.0).numpy(), 3.0, atol=1e-3, rtol=1e-5) ema.reset() self.assertEqual(ema.value.shape, ()) self.assertEqual(ema.value.numpy(), 0.0) self.assertAllClose(ema(3.0).numpy(), 3.0, atol=1e-3, rtol=1e-5) def testResetVector(self): ema = moving_averages.ExponentialMovingAverage(0.90) random_input = tf.random.normal((1, 5)) ema(random_input) ema.reset() self.assertEqual(ema.value.shape, (1, 5)) self.assertAllClose(ema.value.numpy(), tf.zeros_like(random_input)) self.assertEqual(ema._counter.dtype, tf.int64) def testValueEqualsLatestUpdate(self): ema = moving_averages.ExponentialMovingAverage(0.50) self.assertAllClose(ema(3.0).numpy(), 3.0, atol=1e-3, rtol=1e-5) self.assertAllClose(ema.value.numpy(), 3.0, atol=1e-3, rtol=1e-5) self.assertAllClose(ema(6.0).numpy(), 5.0, atol=1e-3, rtol=1e-5) self.assertAllClose(ema.value.numpy(), 5.0, atol=1e-3, rtol=1e-5) @parameterized.parameters(True, False) def testWithTFFunction(self, autograph): ema_1 = moving_averages.ExponentialMovingAverage(0.95) ema_2 = moving_averages.ExponentialMovingAverage(0.95) ema_func = tf.function(ema_2, autograph=autograph) for _ in range(10): x = tf.random.uniform((), 0, 10) self.assertAllClose( ema_1(x).numpy(), ema_func(x).numpy(), atol=1e-3, rtol=1e-5) @parameterized.parameters(True, False) def testResetWithTFFunction(self, autograph): ema = moving_averages.ExponentialMovingAverage(0.90) ema_func = tf.function(ema, autograph=autograph) self.assertAllClose(ema_func(3.0).numpy(), 3.0, atol=1e-3, rtol=1e-5) ema.reset() self.assertEqual(ema.value.numpy(), 0.0) self.assertAllClose(ema_func(3.0).numpy(), 3.0, atol=1e-3, rtol=1e-5) @parameterized.named_parameters(("2D", [2, 2]), ("3D", [1, 1, 3])) def testAlternativeShape(self, shape): ema = moving_averages.ExponentialMovingAverage(0.90) value = tf.random.uniform(shape) result = ema(value) self.assertEqual(value.shape, result.shape) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/nets/BUILD ================================================ load("//sonnet/src:build_defs.bzl", "snt_py_library", "snt_py_test") package(default_visibility = ["//sonnet:__subpackages__", "//docs/ext:__subpackages__", "//examples:__subpackages__"]) licenses(["notice"]) snt_py_library( name = "mlp", srcs = ["mlp.py"], deps = [ "//sonnet/src:base", "//sonnet/src:initializers", "//sonnet/src:linear", # pip: tensorflow ], ) snt_py_test( name = "mlp_test", srcs = ["mlp_test.py"], deps = [ ":mlp", # pip: absl/testing:parameterized "//sonnet/src:test_utils", # pip: tensorflow ], ) snt_py_library( name = "cifar10_convnet", srcs = ["cifar10_convnet.py"], deps = [ "//sonnet/src:base", "//sonnet/src:batch_norm", "//sonnet/src:conv", "//sonnet/src:initializers", "//sonnet/src:linear", "//sonnet/src:types", # pip: tensorflow ], ) snt_py_test( name = "cifar10_convnet_test", timeout = "long", srcs = ["cifar10_convnet_test.py"], deps = [ ":cifar10_convnet", # pip: absl/testing:parameterized # pip: numpy "//sonnet/src:test_utils", # pip: tensorflow ], ) snt_py_library( name = "vqvae", srcs = ["vqvae.py"], deps = [ "//sonnet/src:base", "//sonnet/src:initializers", "//sonnet/src:moving_averages", "//sonnet/src:types", # pip: tensorflow ], ) snt_py_test( name = "vqvae_test", srcs = ["vqvae_test.py"], deps = [ ":vqvae", # pip: absl/testing:parameterized # pip: numpy "//sonnet/src:test_utils", # pip: tensorflow # pip: tree ], ) snt_py_library( name = "resnet", srcs = ["resnet.py"], deps = [ "//sonnet/src:base", "//sonnet/src:batch_norm", "//sonnet/src:conv", "//sonnet/src:initializers", "//sonnet/src:linear", "//sonnet/src:pad", # pip: tensorflow ], ) snt_py_test( name = "resnet_test", srcs = ["resnet_test.py"], deps = [ ":resnet", # pip: absl/testing:parameterized "//sonnet/src:test_utils", # pip: tensorflow ], ) ================================================ FILE: sonnet/src/nets/__init__.py ================================================ # Copyright 2021 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ ================================================ FILE: sonnet/src/nets/cifar10_convnet.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Convnet module for Cifar10 classification.""" from typing import Mapping, Optional, Sequence, Union from sonnet.src import base from sonnet.src import batch_norm from sonnet.src import conv from sonnet.src import initializers from sonnet.src import linear from sonnet.src import types import tensorflow as tf class Cifar10ConvNet(base.Module): """Convolutional network designed for Cifar10. Approximately equivalent to "VGG, minus max pooling, plus BatchNorm". For best results the input data should be scaled to be between -1 and 1 when using the standard initializers. """ def __init__(self, num_classes: int = 10, w_init: Optional[initializers.Initializer] = None, b_init: Optional[initializers.Initializer] = None, data_format: str = 'NHWC', output_channels: Sequence[int] = ( 64, 64, 128, 128, 128, 256, 256, 256, 512, 512, 512, ), strides: Sequence[int] = (1, 1, 2, 1, 1, 2, 1, 1, 2, 1, 1), name: Optional[str] = None): super().__init__(name=name) self._num_classes = num_classes self._data_format = data_format if len(strides) != len(output_channels): raise ValueError( 'The length of `output_channels` and `strides` must be equal.') self._output_channels = output_channels self._strides = strides self._num_layers = len(self._output_channels) self._kernel_shapes = [[3, 3]] * self._num_layers # All kernels are 3x3. self._w_init = w_init self._b_init = b_init self._conv_modules = list( conv.Conv2D( # pylint: disable=g-complex-comprehension output_channels=self._output_channels[i], kernel_shape=self._kernel_shapes[i], stride=self._strides[i], w_init=self._w_init, b_init=self._b_init, data_format=self._data_format, name='conv_2d_{}'.format(i)) for i in range(self._num_layers)) self._bn_modules = list( batch_norm.BatchNorm( # pylint: disable=g-complex-comprehension create_offset=True, create_scale=False, decay_rate=0.999, data_format=self._data_format, name='batch_norm_{}'.format(i)) for i in range(self._num_layers)) self._logits_module = linear.Linear( self._num_classes, w_init=self._w_init, b_init=self._b_init, name='logits') def __call__( self, inputs: tf.Tensor, is_training: types.BoolLike, test_local_stats: bool = True ) -> Mapping[str, Union[tf.Tensor, Sequence[tf.Tensor]]]: """Connects the module to some inputs. Args: inputs: A Tensor of size [batch_size, input_height, input_width, input_channels], representing a batch of input images. is_training: Boolean to indicate to `snt.BatchNorm` if we are currently training. test_local_stats: Boolean to indicate to `snt.BatchNorm` if batch normalization should use local batch statistics at test time. By default `True`. Returns: A dictionary containing two items: - logits: The output logits of the network, this will be of size [batch_size, num_classes] - activations: A list of `tf.Tensor`, the feature activations of the module. The order of the activations is preserved in the output list. The activations in the output list are those computed after the activation function is applied, if one is applied at that layer. """ activations = [] net = inputs for conv_layer, bn_layer in zip(self._conv_modules, self._bn_modules): net = conv_layer(net) net = bn_layer( net, is_training=is_training, test_local_stats=test_local_stats) net = tf.nn.relu(net) activations.append(net) flat_output = tf.reduce_mean( net, axis=[1, 2], keepdims=False, name='avg_pool') activations.append(flat_output) logits = self._logits_module(flat_output) return {'logits': logits, 'activations': activations} ================================================ FILE: sonnet/src/nets/cifar10_convnet_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.nets.cifar10_convnet.""" from absl.testing import parameterized import numpy as np from sonnet.src import test_utils from sonnet.src.nets import cifar10_convnet import tensorflow as tf class ModelTest(parameterized.TestCase, test_utils.TestCase): def testModelCreation(self): convnet = cifar10_convnet.Cifar10ConvNet() self.assertLen(convnet.submodules, 45) def testFailedModelCreation(self): with self.assertRaisesRegex( ValueError, 'The length of `output_channels` and `strides` must be equal.'): cifar10_convnet.Cifar10ConvNet(strides=(1, 2, 3), output_channels=(1,)) @parameterized.parameters({'batch_size': 1}, {'batch_size': 4}, {'batch_size': 128}) def testModelForwards(self, batch_size): image_batch = tf.constant( np.random.randn(batch_size, 24, 24, 3), dtype=tf.float32) convnet = cifar10_convnet.Cifar10ConvNet() output = convnet(image_batch, is_training=True) self.assertLen(convnet.variables, 112) self.assertEqual(output['logits'].shape, [batch_size, 10]) # One intermediate activation per conv layer, plus one after the global # mean pooling, before the linear. self.assertLen(output['activations'], 12) @parameterized.parameters({'batch_size': 1}, {'batch_size': 4}, {'batch_size': 128}) def testModelForwardsFunction(self, batch_size): image_batch = tf.constant( np.random.randn(batch_size, 24, 24, 3), dtype=tf.float32) convnet = cifar10_convnet.Cifar10ConvNet() convnet_function = tf.function(convnet) output = convnet_function(image_batch, is_training=True) self.assertLen(convnet.variables, 112) self.assertEqual(output['logits'].shape, [batch_size, 10]) # One intermediate activation per conv layer, plus one after the global # mean pooling, before the linear. self.assertLen(output['activations'], 12) def testDifferentSizedImages(self): # Due to global average pooling, different sized images should work fine # as long they are above some minimum size. convnet = cifar10_convnet.Cifar10ConvNet() small_image = tf.constant(np.random.randn(4, 32, 32, 3), dtype=tf.float32) small_output = convnet(small_image, is_training=True) self.assertEqual(small_output['logits'].shape, [4, 10]) # Change height, width and batch size big_image = tf.constant(np.random.randn(12, 64, 64, 3), dtype=tf.float32) big_output = convnet(big_image, is_training=True) self.assertEqual(big_output['logits'].shape, [12, 10]) def testDefunBackProp(self): convnet = cifar10_convnet.Cifar10ConvNet() @tf.function def do_training_step(image, labels): with tf.GradientTape() as tape: logits = convnet(image, is_training=True)['logits'] loss = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( logits=logits, labels=labels)) grads = tape.gradient(loss, convnet.trainable_variables) return loss, grads image = tf.random.normal([4, 32, 32, 3]) labels = np.random.randint(low=0, high=10, size=[4], dtype=np.int64) loss, grads = do_training_step(image, labels) self.assertEqual(loss.numpy().shape, ()) for grad, var in zip(grads, convnet.trainable_variables): self.assertIsNotNone(grad) self.assertEqual(grad.numpy().shape, var.shape) if __name__ == '__main__': tf.test.main() ================================================ FILE: sonnet/src/nets/dnc/BUILD ================================================ # Description: # Differentiable Neural Computer load("//sonnet/src:build_defs.bzl", "snt_py_library", "snt_py_test") licenses(["notice"]) snt_py_library( name = "control", srcs = ["control.py"], deps = [ "//sonnet/src:linear", "//sonnet/src:recurrent", # pip: tensorflow ], ) snt_py_test( name = "control_test", srcs = ["control_test.py"], main = "control_test.py", deps = [ ":control", # pip: absl/testing:parameterized # pip: numpy "//sonnet/src:recurrent", "//sonnet/src:test_utils", # pip: tensorflow # pip: tree ], ) snt_py_library( name = "read", srcs = ["read.py"], deps = [ # pip: tensorflow ], ) snt_py_test( name = "read_test", srcs = ["read_test.py"], main = "read_test.py", deps = [ ":read", # pip: numpy "//sonnet/src:test_utils", # pip: tensorflow ], ) snt_py_library( name = "util", srcs = ["util.py"], deps = [ # pip: numpy # pip: tensorflow # pip: tree ], ) snt_py_test( name = "util_test", srcs = ["util_test.py"], main = "util_test.py", deps = [ ":util", # pip: absl/testing:parameterized # pip: numpy "//sonnet/src:linear", "//sonnet/src:test_utils", # pip: tensorflow # pip: tree ], ) snt_py_library( name = "write", srcs = ["write.py"], deps = [ # pip: tensorflow ], ) snt_py_test( name = "write_test", srcs = ["write_test.py"], main = "write_test.py", deps = [ ":write", # pip: numpy "//sonnet/src:test_utils", # pip: tensorflow ], ) ================================================ FILE: sonnet/src/nets/dnc/__init__.py ================================================ # Copyright 2021 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ ================================================ FILE: sonnet/src/nets/dnc/control.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """DNC Control Modules. These modules receive input and output parameters for the memory access module. We also alias external controllers in this module that are relevant, so they can be specified by string name in the core config. """ import sys from sonnet.src import linear from sonnet.src import recurrent import tensorflow as tf def get_controller_ctor(controller_name): """Returns the constructor for a givn controller name.""" if controller_name == 'LSTM': return recurrent.LSTM elif controller_name == 'GRU': return recurrent.GRU else: # References for other controllers can be added here return getattr(sys.modules[__name__], controller_name) class FeedForward(recurrent.RNNCore): """FeedForward controller module. Single feedforward linear layer, wrapped as an RNN core for convenience. There is no computation performed on the state. y <- activation(linear(x)) s_t+1 <- s_t """ def __init__(self, hidden_size, activation=tf.nn.tanh, dtype=tf.float32, name=None): """Initializes the FeedForward Module. Args: hidden_size: number of hidden units in linear layer. activation: op for output activations. dtype: datatype of inputs to accept, defaults to tf.float32. name: module name (default 'feed_forward'). """ super().__init__(name=name) self.linear = linear.Linear(hidden_size) self.dtype = dtype self._activation = activation def __call__(self, inputs, prev_state): """Connects the FeedForward controller to the graph. Args: inputs: 2D Tensor [batch_size, input_size] input_size needs to be specified at construction time. prev_state: dummy state, 2D tensor of size [batch_size, 1] Returns: output: 2D Tensor [batch_size, hidden_size]. next_state: the same dummy state passed in as an argument. """ output = self.linear(inputs) if self._activation is not None: output = self._activation(output) return output, prev_state def initial_state(self, batch_size): return tf.zeros([batch_size, 1], dtype=self.dtype) def deep_core(control_name, control_config, num_layers=1, skip_connections=True, name=None): """Constructs a deep control module. Args: control_name: Name of control module (e.g. "LSTM"). control_config: Dictionary containing the configuration for the modules. num_layers: Number of layers. skip_connections: Boolean that indicates whether to use skip connections. name: module name. Returns: Deep control module. """ control_class = get_controller_ctor(control_name) cores = [ control_class(name='{}_{}'.format(control_name, i), **control_config) for i in range(num_layers) ] if skip_connections: return recurrent.deep_rnn_with_skip_connections(cores, name=name) else: return recurrent.DeepRNN(cores, name=name) ================================================ FILE: sonnet/src/nets/dnc/control_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.nets.dnc.control.""" from absl.testing import parameterized import numpy as np from sonnet.src import recurrent from sonnet.src import test_utils from sonnet.src.nets.dnc import control import tensorflow as tf import tree class CoreTest(test_utils.TestCase, parameterized.TestCase): @parameterized.parameters({'constructor': recurrent.LSTM}, {'constructor': recurrent.GRU}) def testShape(self, constructor): batch_size = 2 hidden_size = 4 input_size = 3 inputs = tf.random.uniform([batch_size, input_size]) rnn = constructor(hidden_size) prev_state = rnn.initial_state(batch_size=batch_size) output, next_state = rnn(inputs, prev_state) tree.map_structure(lambda t1, t2: self.assertEqual(t1.shape, t2.shape), prev_state, next_state) self.assertShapeEqual(np.zeros([batch_size, hidden_size]), output) class FeedForwardTest(test_utils.TestCase): def testShape(self): batch_size = 2 hidden_size = 4 inputs = tf.random.uniform(shape=[batch_size, hidden_size]) rnn = control.FeedForward(hidden_size) prev_state = rnn.initial_state(batch_size=batch_size) output, next_state = rnn(inputs, prev_state) output_shape = np.ndarray((batch_size, hidden_size)) state_shape = np.ndarray((batch_size, 1)) self.assertShapeEqual(output_shape, output) self.assertShapeEqual(state_shape, next_state) def testValues(self): batch_size = 2 hidden_size = 4 input_size = 8 inputs = tf.random.uniform([batch_size, input_size]) rnn = control.FeedForward(hidden_size, activation=tf.identity) prev_state = rnn.initial_state(batch_size=batch_size) output, next_state = rnn(inputs, prev_state) weight, bias = rnn.linear.w, rnn.linear.b expected_output = np.dot(inputs.numpy(), weight.numpy()) + bias.numpy() self.assertAllClose(output.numpy(), expected_output, atol=1e-2) # State should remain at dummy value. self.assertAllClose(prev_state.numpy(), next_state.numpy(), atol=5e-3) class DeepCore(test_utils.TestCase, parameterized.TestCase): @parameterized.parameters({ 'control_name': 'LSTM', 'num_layers': 1 }, { 'control_name': 'LSTM', 'num_layers': 2 }, { 'control_name': 'GRU', 'num_layers': 1 }, { 'control_name': 'GRU', 'num_layers': 2 }) def testShape(self, control_name, num_layers): batch_size = 5 input_size = 3 hidden_size = 7 control_config = {'hidden_size': hidden_size} inputs = tf.random.uniform([batch_size, input_size]) rnn = control.deep_core( num_layers=num_layers, control_name=control_name, control_config=control_config) prev_state = rnn.initial_state(batch_size=batch_size) output, next_state = rnn(inputs, prev_state) # The deep_core concatenates the outputs of the individual cores. output_shape = np.ndarray((batch_size, num_layers * hidden_size)) self.assertShapeEqual(output_shape, output) tree.map_structure(lambda t1, t2: self.assertEqual(t1.shape, t2.shape), prev_state, next_state) if __name__ == '__main__': tf.test.main() ================================================ FILE: sonnet/src/nets/dnc/read.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Read modules.""" import tensorflow as tf def read(memory, weights, squash_op=tf.nn.tanh, squash_before_access=True, squash_after_access=False): """Read from the NTM memory. Args: memory: 3D Tensor [batch_size, memory_size, word_size]. weights: 3D Tensor [batch_size, num_reads, memory_size]. squash_op: op to perform squashing of memory or read word. squash_before_access: squash memory before read, default True. squash_after_access: squash read word, default False. Returns: 3D Tensor [batch_size, num_reads, word_size]. """ with tf.name_scope("read_memory"): if squash_before_access: squash_op(weights) read_word = tf.matmul(weights, memory) if squash_after_access: read_word = squash_op(read_word) return read_word ================================================ FILE: sonnet/src/nets/dnc/read_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.nets.dnc.read.""" import numpy as np from sonnet.src import test_utils from sonnet.src.nets.dnc import read import tensorflow as tf class ReadTest(test_utils.TestCase): def testShape(self): batch_size = 4 num_reads = 2 memory_size = 5 word_size = 3 mem = tf.random.uniform([batch_size, memory_size, word_size]) weights = tf.random.uniform([batch_size, num_reads, memory_size]) values_read = read.read(mem, weights) self.assertAllEqual(values_read.shape.as_list(), [batch_size, num_reads, word_size]) def testValues(self): num_reads = 2 memory_size = 5 word_size = 3 # Random memory and weights (batch_size=1) mem = tf.random.uniform([1, memory_size, word_size]) indices = np.random.randint(0, memory_size, size=num_reads) # One-hot representation read_weights = tf.constant( np.expand_dims(np.eye(memory_size)[indices], axis=0), dtype=tf.float32) read_values = read.read(mem, read_weights, squash_op=tf.identity) self.assertAllClose( mem.numpy()[0, indices, :], read_values.numpy()[0, ...], atol=2e-3) if __name__ == '__main__': tf.test.main() ================================================ FILE: sonnet/src/nets/dnc/util.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """DNC util ops and modules.""" import numpy as np import tensorflow as tf import tree def segment_dim(inputs, dim, shapes): """Returns tuple of Tensors output from segmenting input Tensor along dim. The returned tuple of Tensors produced by 'segmenting' the Tensor along a certain dimension can be transformed to specified shapes. Example: input_tensor = tf.placeholder([2, 14, 3]) one, two = segment_dim(input_tensor, dim=1, shapes=[TensorShape([3, 3]), TensorShape([5])]) # one is a [2, 3, 3, 3] Tensor and two is a [2, 5, 3] Tensor. Args: inputs: `Tensor` to segment. dim: dimension of the Tensor to operate on. Negative numbers count back from the end of the dimensions. shapes: list of TensorShapes of the output 'segments' to produce. Returns: Tuple with resulting Tensors. Raises: ValueError: if the dim used at initialization is invalid. The valid range is (-d, d], where d is the number of dimensions of the input tensor. """ inputs_shape = inputs.shape ndims = inputs_shape.ndims dynamic_shape = tf.shape(inputs) shape_as_list = [ dynamic_shape[i] if s is None else s for i, s in enumerate(inputs_shape.as_list()) ] if dim >= ndims or dim < -ndims: message = 'Invalid dims ({:d})'.format(dim) raise ValueError(message) pre_shape = shape_as_list[:dim] if dim == -1: post_shape = [] else: post_shape = shape_as_list[(dim + 1):] slice_begin = [0] * ndims slice_size = [-1] * ndims segments = [] for shape in shapes: num_elements = shape.num_elements() slice_size[dim] = num_elements flat_slice = tf.slice(inputs, slice_begin, slice_size) final_shape = pre_shape + shape.as_list() + post_shape segments.append(tf.reshape(flat_slice, final_shape)) slice_begin[dim] += num_elements return tuple(segments) def batch_invert_permutation(permutations): """Returns batched `tf.invert_permutation` for every row in `permutations`.""" unpacked = tf.unstack(permutations) inverses = [ tf.math.invert_permutation(permutation) for permutation in unpacked ] return tf.stack(inverses) def batch_gather(values, indices): """Returns batched `tf.gather` for every row in the input.""" unpacked = zip(tf.unstack(values), tf.unstack(indices)) result = [tf.gather(value, index) for value, index in unpacked] return tf.stack(result) def one_hot(length, index): """Return an nd array of given `length` filled with 0s and a 1 at `index`.""" result = np.zeros(length) result[index] = 1 return result def apply_linear(inputs, linear_modules, activation=tf.identity): """Computes linear, allowing for tuple inputs (processed in parallel). If inputs is a tuple, the linear modules must be a tuple or list of the same length. Args: inputs: tensor or list / tuple of 2 tensors. linear_modules: sonnet module, or list / tuple of 2 sonnet modules. activation: function to call as activation, default is identity. Returns: output Tensor from one / both linear modules. """ tree.assert_same_structure(inputs, linear_modules) if isinstance(inputs, (tuple, list)): assert len(inputs) == len(linear_modules) == 2, ( 'if inputs is a list, must be length 2 and match length of linears') return apply_split_linear( linear_modules[0], linear_modules[1], inputs[0], inputs[1], activation=activation) else: return activation(linear_modules(inputs)) def apply_split_linear(lin_module_1, lin_module_2, input1, input2, activation=None): """Returns a linear output of two inputs, run independently and summed.""" output_1 = lin_module_1(input1) output_2 = lin_module_2(input2) summed_output = output_1 + output_2 if activation is not None: summed_output = activation(summed_output) return summed_output ================================================ FILE: sonnet/src/nets/dnc/util_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.nets.sdnc.util.""" from absl.testing import parameterized import numpy as np from sonnet.src import linear from sonnet.src import test_utils from sonnet.src.nets.dnc import util import tensorflow as tf import tree class SegmentDimTest(test_utils.TestCase, parameterized.TestCase): @parameterized.parameters(([2], [7]), ([], [7]), ([2], []), ([2], [7, 11]), ([2, 11], [7])) def testShape(self, initial_shape, final_shape): first_shape = tf.TensorShape([3, 3]) second_shape = tf.TensorShape([5]) segment_shapes = [first_shape, second_shape] inputs_shape = ( initial_shape + [first_shape.num_elements() + second_shape.num_elements()] + final_shape) inputs = tf.random.uniform(inputs_shape) first, second = util.segment_dim( inputs, dim=len(initial_shape), shapes=segment_shapes) self.assertAllEqual(first.shape.as_list(), initial_shape + first_shape.as_list() + final_shape) self.assertAllEqual(second.shape.as_list(), initial_shape + second_shape.as_list() + final_shape) @parameterized.parameters(([2], [7]), ([], [7]), ([2], []), ([2], [7, 11]), ([2, 11], [7])) def testShapeNegative(self, initial_shape, final_shape): first_shape = tf.TensorShape([3, 3]) second_shape = tf.TensorShape([5]) segment_shapes = [first_shape, second_shape] inputs_shape = ( initial_shape + [first_shape.num_elements() + second_shape.num_elements()] + final_shape) inputs = tf.random.uniform(inputs_shape) first, second = util.segment_dim( inputs, dim=-len(final_shape) - 1, shapes=segment_shapes) self.assertAllEqual(first.shape.as_list(), initial_shape + first_shape.as_list() + final_shape) self.assertAllEqual(second.shape.as_list(), initial_shape + second_shape.as_list() + final_shape) def testValues(self): segment_shapes = [tf.TensorShape([2]), tf.TensorShape([3])] inputs = tf.constant( np.hstack([np.zeros((5, 2)), np.ones((5, 3))]), dtype=tf.float32) first, second = util.segment_dim(inputs, dim=1, shapes=segment_shapes) self.assertAllEqual(first.numpy(), np.zeros_like(first)) self.assertAllEqual(second.numpy(), np.ones_like(second)) def testInvalidDims(self): segment_shapes = [tf.TensorShape([3]), tf.TensorShape([2])] inputs = tf.random.uniform([5, 5]) with self.assertRaisesRegex(ValueError, 'Invalid dims'): util.segment_dim(inputs, 3, segment_shapes) class BatchInvertPermutationTest(test_utils.TestCase): def testCorrectOutput(self): # Tests that the _batch_invert_permutation function correctly inverts a # batch of permutations. batch_size = 5 length = 7 permutations = np.empty([batch_size, length], dtype=int) for i in range(batch_size): permutations[i] = np.random.permutation(length) inverse = util.batch_invert_permutation(tf.constant(permutations, tf.int32)) inverse_np = inverse.numpy() for i in range(batch_size): for j in range(length): self.assertEqual(permutations[i][inverse_np[i][j]], j) class BatchGatherTest(test_utils.TestCase): def testCorrectOutput(self): values = np.array([[3, 1, 4, 1], [5, 9, 2, 6], [5, 3, 5, 7]]) indices = np.array([[1, 2, 0, 3], [3, 0, 1, 2], [0, 2, 1, 3]]) target = np.array([[1, 4, 3, 1], [6, 5, 9, 2], [5, 5, 3, 7]]) result = util.batch_gather(tf.constant(values), tf.constant(indices)) self.assertAllEqual(target, result) class LinearTest(test_utils.TestCase, parameterized.TestCase): def testLinearOutputOneModule(self): batch_size = 4 input_size = 5 output_size = 3 lin_a = linear.Linear(output_size) inputs = tf.random.uniform([batch_size, input_size]) output = util.apply_linear(inputs, lin_a, activation=tf.nn.tanh) expected_output = np.tanh( np.matmul(inputs.numpy(), lin_a.w.numpy()) + lin_a.b.numpy()) self.assertAllClose(expected_output, output.numpy(), atol=self.get_atol()) def testLinearOutputTwoModules(self): batch_size = 4 input_size_a = 5 input_size_b = 6 output_size = 3 lin_a = linear.Linear(output_size, name='lin_a') lin_b = linear.Linear(output_size, name='lin_b') input_a = tf.random.uniform([batch_size, input_size_a]) input_b = tf.random.uniform([batch_size, input_size_b]) output = util.apply_linear((input_a, input_b), (lin_a, lin_b), activation=tf.nn.relu) expected_output = np.maximum( 0, (np.matmul(input_a.numpy(), lin_a.w.numpy()) + lin_a.b.numpy() + np.matmul(input_b.numpy(), lin_b.w.numpy()) + lin_b.b.numpy())) self.assertAllClose(expected_output, output.numpy(), atol=self.get_atol()) def testDifferentOutputSizeBreaks(self): batch_size = 4 input_size = 5 output_size_a = 6 output_size_b = 3 lin_a = linear.Linear(output_size_a, name='lin_a') lin_b = linear.Linear(output_size_b, name='lin_b') input_a = tf.random.uniform([batch_size, input_size]) input_b = tf.random.uniform([batch_size, input_size]) with self.assertRaisesIncompatibleShapesError( tf.errors.InvalidArgumentError): util.apply_linear((input_a, input_b), (lin_a, lin_b)) @parameterized.parameters( { 'input_sizes': 4, 'module_hidden_sizes': (2, 3) }, { 'input_sizes': (5, 7), 'module_hidden_sizes': 10 }, ) def testNonMatchingStructureBreaks(self, input_sizes, module_hidden_sizes): batch_size = 16 inputs = tree.map_structure( lambda size: tf.random.uniform([batch_size, size]), input_sizes) modules = tree.map_structure(linear.Linear, module_hidden_sizes) with self.assertRaisesRegex(ValueError, 'don\'t have the same nested structure'): util.apply_linear(inputs, modules) @parameterized.parameters( # Even when list length matches, len must be 2 { 'input_sizes': [10] * 3, 'module_hidden_sizes': [3] * 3 }, { 'input_sizes': [1], 'module_hidden_sizes': [4] }) def testListMustBeLengthTwo(self, input_sizes, module_hidden_sizes): batch_size = 16 inputs = tree.map_structure( lambda size: tf.random.uniform([batch_size, size]), input_sizes) modules = tree.map_structure(linear.Linear, module_hidden_sizes) with self.assertRaisesRegex(AssertionError, 'must be length 2'): util.apply_linear(inputs, modules) if __name__ == '__main__': tf.test.main() ================================================ FILE: sonnet/src/nets/dnc/write.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Write modules.""" import tensorflow as tf def additive_write(memory, address, values): """Additively writes values to memory at given address. M_t = M_{t-1} + w_t a_t^T. Args: memory: 3D Tensor [batch_size, memory_size, word_size]. address: 3D Tensor [batch_size, num_writes, memory_size]. values: 3D Tensor [batch_size, num_writes, word_size]. Returns: 3D Tensor [batch_size, num_reads, word_size]. """ with tf.name_scope('write_memory'): add_matrix = tf.matmul(address, values, adjoint_a=True) return memory + add_matrix def erase(memory, address, reset_weights): """Erases rows over addressing distribution by given reset word weights. M_t(i) = M_{t-1}(i) * (1 - w_t(i) * e_t) The erase is defined as a component-wise OR over reset strengths between write heads, followed by a componentwise multiplication. The reset weights contains values in [0, 1] where 1 indicates a complete reset. The reset weights are granular over a word, allowing for part of the word to be erased. Args: memory: 3D Tensor [batch_size, memory_size, word_size]. address: 3D Tensor [batch_size, num_writes, memory_size]. reset_weights: 3D Tensor [batch_size, num_writes, word_size]. Returns: erased memory: 3D Tensor [batch_size, num_reads, word_size]. """ with tf.name_scope('erase_memory'): address = tf.expand_dims(address, 3) reset_weights = tf.expand_dims(reset_weights, 2) weighted_resets = address * reset_weights reset_gate = tf.reduce_prod(1 - weighted_resets, [1]) return memory * reset_gate def erase_rows(memory, address, reset_row_weights): """Erases rows over addressing distribution by given reset weight. The reset row weight here is uniform over the values in a word. Args: memory: 3D Tensor [batch_size, memory_size, word_size]. address: 3D Tensor [batch_size, num_writes, memory_size]. reset_row_weights: 2D Tensor [batch_size, num_writes]. Returns: 3d Tensor of memory [batch_size, memory_size, word_size]. """ with tf.name_scope('erase_rows'): # Expands reset_row_weights for broadcasted cmul with address. reset_row_weights = tf.expand_dims(reset_row_weights, -1) weighted_resets = tf.multiply(address, reset_row_weights) reset_gate = tf.reduce_prod(1 - weighted_resets, axis=[1]) # Expands reset_gate for broadcasted cmul with memory. reset_gate = tf.expand_dims(reset_gate, -1) return tf.multiply(memory, reset_gate) def erase_and_write(memory, address, reset_weights, values): """Module to erase and write in the NTM memory. Implementation is based on equations (3) and (4) from 'Neural Turing Machines' (https://arxiv.org/pdf/1410.5401.pdf) by gravesa@, gregwayne@ and danihelka@: Erase operation: M_t'(i) = M_{t-1}(i) * (1 - w_t(i) * e_t) Add operation: M_t(i) = M_t'(i) + w_t(i) * a_t where e are the reset_weights, w the write weights and a the values. Args: memory: 3D Tensor [batch_size, memory_size, word_size]. address: 3D Tensor [batch_size, num_writes, memory_size]. reset_weights: 3D Tensor [batch_size, num_writes, word_size]. values: 3D Tensor [batch_size, num_writes, word_size]. Returns: 3D Tensor [batch_size, num_reads, word_size]. """ memory = erase(memory, address, reset_weights) memory = additive_write(memory, address, values) return memory ================================================ FILE: sonnet/src/nets/dnc/write_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.nets.dnc.write.""" import numpy as np from sonnet.src import test_utils from sonnet.src.nets.dnc import write import tensorflow as tf class EraseRowsTest(test_utils.TestCase): def testShape(self): batch_size = 16 num_writes = 2 memory_size = 5 word_size = 3 mem = tf.random.uniform([batch_size, memory_size, word_size]) write_address = tf.random.uniform([batch_size, num_writes, memory_size]) reset_row_weights = tf.random.uniform([batch_size, num_writes]) eraser = write.erase_rows(mem, write_address, reset_row_weights) self.assertAllEqual(eraser.shape.as_list(), [batch_size, memory_size, word_size]) def testValues(self): num_writes = 2 memory_size = 5 word_size = 3 # Random memory, weights and values (batch_size=1) mem = tf.random.uniform((1, memory_size, word_size)) mem_np = mem.numpy() # Non-repeated indices in [0, memory_size) perm = np.random.permutation(memory_size) indices_np = perm[:num_writes] excluded_indices_np = perm[num_writes:] # One-hot representation write_address = tf.constant( np.expand_dims(np.eye(memory_size)[indices_np], axis=0), dtype=tf.float32) reset_row_weights = tf.ones((1, num_writes)) erased_mem = write.erase_rows(mem, write_address, reset_row_weights) not_erased_mem = write.erase_rows(mem, write_address, reset_row_weights * 0) erased_mem_np = erased_mem.numpy() # Rows specified in indices should have been erased. self.assertAllClose( erased_mem_np[0, indices_np, :], np.zeros((num_writes, word_size)), atol=2e-3) # Other rows should not have been erased. self.assertAllClose( erased_mem_np[0, excluded_indices_np, :], mem_np[0, excluded_indices_np, :], atol=2e-3) # Write with reset weights zero'd out and nothing should change. self.assertAllEqual(not_erased_mem.numpy(), mem_np) class EraseTest(test_utils.TestCase): def testShape(self): batch_size = 1 num_writes = 2 memory_size = 5 word_size = 3 mem = tf.random.uniform([batch_size, memory_size, word_size]) write_address = tf.random.uniform([batch_size, num_writes, memory_size]) reset_weights = tf.random.uniform([batch_size, num_writes, word_size]) writer = write.erase(mem, write_address, reset_weights) self.assertTrue(writer.shape.as_list(), [batch_size, memory_size, word_size]) def testValues(self): num_writes = 2 memory_size = 5 word_size = 3 # Random memory, weights and values (batch_size=1) mem = tf.random.uniform([1, memory_size, word_size]) mem_np = mem.numpy() # Non-repeated indices in [0, memory_size) perm = np.random.permutation(memory_size) indices = perm[:num_writes] excluded_indices = perm[num_writes:] # One-hot representation write_address = tf.constant( np.expand_dims(np.eye(memory_size)[indices], axis=0), dtype=tf.float32) reset_weights = tf.ones([1, num_writes, word_size]) erased_mem = write.erase(mem, write_address, reset_weights) not_erased_mem = write.erase(mem, write_address, reset_weights * 0.) erased_mem_np = erased_mem.numpy() not_erased_mem_np = not_erased_mem.numpy() # Rows specified in indices should have been erased. self.assertAllClose( erased_mem_np[0, indices, :], np.zeros((num_writes, word_size)), atol=2e-3) # Other rows should not have been erased. self.assertAllClose( erased_mem_np[0, excluded_indices, :], mem_np[0, excluded_indices, :], atol=2e-3) # Write with reset weights zero'd out and nothing should change. self.assertAllEqual(not_erased_mem_np, mem_np) class EraseAndWriteTest(test_utils.TestCase): def testShape(self): batch_size = 4 num_writes = 2 memory_size = 5 word_size = 3 mem = tf.random.uniform([batch_size, memory_size, word_size]) write_address = tf.random.uniform([batch_size, num_writes, memory_size]) reset_weights = tf.random.uniform([batch_size, num_writes, word_size]) values = tf.random.uniform([batch_size, num_writes, word_size]) writer = write.erase_and_write(mem, write_address, reset_weights, values) self.assertTrue(writer.shape.as_list(), [batch_size, memory_size, word_size]) def testValues(self): batch_size = 4 num_writes = 2 memory_size = 5 word_size = 3 # Random memory, weights and values (batch_size=1) mem = tf.random.uniform((batch_size, memory_size, word_size)) # Non-repeated indices in [0, memory_size) indices = np.random.permutation(memory_size)[:num_writes] # One-hot representation write_address = tf.constant( np.tile(np.eye(memory_size)[indices], [batch_size, 1, 1]), dtype=tf.float32) reset_weights = tf.ones((batch_size, num_writes, word_size), 1) write_values = tf.random.uniform([batch_size, num_writes, word_size]) written_mem = write.erase_and_write(mem, write_address, reset_weights, write_values) self.assertAllClose( written_mem.numpy()[0, indices, :], write_values.numpy()[0], atol=2e-3) class AdditiveWriteTest(test_utils.TestCase): def testShape(self): batch_size = 4 num_writes = 2 memory_size = 5 word_size = 3 mem = tf.random.uniform([batch_size, memory_size, word_size]) write_address = tf.random.uniform([batch_size, num_writes, memory_size]) values = tf.random.uniform([batch_size, num_writes, word_size]) writer = write.additive_write(mem, write_address, values) self.assertAllEqual(writer.shape.as_list(), [batch_size, memory_size, word_size]) def testValues(self): num_writes = 2 memory_size = 5 word_size = 3 # Random memory, address and values (batch_size=1) mem = tf.random.uniform([1, memory_size, word_size]) mem_np = mem.numpy() # Non-repeated indices in [0, memory_size) indices = np.random.permutation(memory_size)[:num_writes] # One-hot representation write_address = tf.constant( np.expand_dims(np.eye(memory_size)[indices], axis=0), dtype=tf.float32) write_values = tf.random.uniform([1, num_writes, word_size]) write_values_np = write_values.numpy() written_mem = write.additive_write(mem, write_address, write_values) not_written_mem = write.additive_write(mem, write_address * 0, write_values) written_mem_np = written_mem.numpy() not_written_mem_np = not_written_mem.numpy() # Check values have been correctly written self.assertAllClose( written_mem.numpy()[0, indices, :], write_values_np[0] + mem_np[0, indices, :], atol=2e-3) # Check all other values in the memory are still what they started as written_mem_copy = written_mem_np.copy() written_mem_copy[0, indices, :] -= write_values_np[0] self.assertAllClose(written_mem_copy, mem_np, atol=2e-3) self.assertAllClose(not_written_mem_np, mem_np, atol=2e-3) if __name__ == '__main__': tf.test.main() ================================================ FILE: sonnet/src/nets/mlp.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """A minimal interface mlp module.""" from typing import Callable, Iterable, Optional from sonnet.src import base from sonnet.src import initializers from sonnet.src import linear import tensorflow as tf class MLP(base.Module): """A multi-layer perceptron module.""" def __init__(self, output_sizes: Iterable[int], w_init: Optional[initializers.Initializer] = None, b_init: Optional[initializers.Initializer] = None, with_bias: bool = True, activation: Callable[[tf.Tensor], tf.Tensor] = tf.nn.relu, dropout_rate=None, activate_final: bool = False, name: Optional[str] = None): """Constructs an MLP. Args: output_sizes: Sequence of layer sizes. w_init: Initializer for Linear weights. b_init: Initializer for Linear bias. Must be `None` if `with_bias` is `False`. with_bias: Whether or not to apply a bias in each layer. activation: Activation function to apply between linear layers. Defaults to ReLU. dropout_rate: Dropout rate to apply, a rate of `None` (the default) or `0` means no dropout will be applied. activate_final: Whether or not to activate the final layer of the MLP. name: Optional name for this module. Raises: ValueError: If with_bias is False and b_init is not None. """ if not with_bias and b_init is not None: raise ValueError("When with_bias=False b_init must not be set.") super().__init__(name=name) self._with_bias = with_bias self._w_init = w_init self._b_init = b_init self._activation = activation self._activate_final = activate_final self._dropout_rate = dropout_rate self._layers = [] for index, output_size in enumerate(output_sizes): self._layers.append( linear.Linear( output_size=output_size, w_init=w_init, b_init=b_init, with_bias=with_bias, name="linear_%d" % index)) def __call__(self, inputs: tf.Tensor, is_training=None) -> tf.Tensor: """Connects the module to some inputs. Args: inputs: A Tensor of shape `[batch_size, input_size]`. is_training: A bool indicating if we are currently training. Defaults to `None`. Required if using dropout. Returns: output: The output of the model of size `[batch_size, output_size]`. """ use_dropout = self._dropout_rate not in (None, 0) if use_dropout and is_training is None: raise ValueError( "The `is_training` argument is required when dropout is used.") elif not use_dropout and is_training is not None: raise ValueError( "The `is_training` argument should only be used with dropout.") num_layers = len(self._layers) for i, layer in enumerate(self._layers): inputs = layer(inputs) if i < (num_layers - 1) or self._activate_final: # Only perform dropout if we are activating the output. if use_dropout and is_training: inputs = tf.nn.dropout(inputs, rate=self._dropout_rate) inputs = self._activation(inputs) return inputs def reverse(self, activate_final: Optional[bool] = None, name: Optional[str] = None) -> "MLP": """Returns a new MLP which is the layer-wise reverse of this MLP. NOTE: Since computing the reverse of an MLP requires knowing the input size of each linear layer this method will fail if the module has not been called at least once. See `snt.Deferred` as a possible solution to this problem. The contract of reverse is that the reversed module will accept the output of the parent module as input and produce an output which is the input size of the parent. >>> mlp = snt.nets.MLP([1, 2, 3]) >>> y = mlp(tf.ones([1, 2])) >>> rev = mlp.reverse() >>> rev(y) Args: activate_final: Whether the final layer of the MLP should be activated. name: Optional name for the new module. The default name will be the name of the current module prefixed with ``"reversed_"``. Returns: An MLP instance which is the reverse of the current instance. Note these instances do not share weights and, apart from being symmetric to each other, are not coupled in any way. """ if activate_final is None: activate_final = self._activate_final if name is None: name = self.name + "_reversed" return MLP( output_sizes=(layer.input_size for layer in reversed(self.submodules)), w_init=self._w_init, b_init=self._b_init, with_bias=self._with_bias, activation=self._activation, dropout_rate=self._dropout_rate, activate_final=activate_final, name=name) ================================================ FILE: sonnet/src/nets/mlp_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.nets.mlp.""" import itertools from absl.testing import parameterized from sonnet.src import test_utils from sonnet.src.nets import mlp import tensorflow as tf class MLPTest(test_utils.TestCase, parameterized.TestCase): def test_b_init_when_with_bias_false(self): with self.assertRaisesRegex(ValueError, "b_init must not be set"): mlp.MLP([1], with_bias=False, b_init=object()) @parameterized.parameters(itertools.product((1, 2, 3), (0.1, 0.0, None))) def test_submodules(self, num_layers, dropout_rate): mod = mlp.MLP([1] * num_layers, dropout_rate=dropout_rate) self.assertLen(mod.submodules, num_layers) @parameterized.parameters(1, 2, 3) def test_applies_activation(self, num_layers): activation = CountingActivation() mod = mlp.MLP([1] * num_layers, activation=activation) mod(tf.ones([1, 1])) self.assertEqual(activation.count, num_layers - 1) @parameterized.parameters(1, 2, 3) def test_activate_final(self, num_layers): activation = CountingActivation() mod = mlp.MLP([1] * num_layers, activate_final=True, activation=activation) mod(tf.ones([1, 1])) self.assertEqual(activation.count, num_layers) @parameterized.parameters(1, 2, 3) def test_adds_index_to_layer_names(self, num_layers): mod = mlp.MLP([1] * num_layers) for index, linear in enumerate(mod.submodules): self.assertEqual(linear.name, "linear_%d" % index) @parameterized.parameters(False, True) def test_passes_with_bias_to_layers(self, with_bias): mod = mlp.MLP([1, 1, 1], with_bias=with_bias) for linear in mod.submodules: self.assertEqual(linear.with_bias, with_bias) def test_repeat_initializer(self): w_init = CountingInitializer() b_init = CountingInitializer() mod = mlp.MLP([1, 1, 1], w_init=w_init, b_init=b_init) mod(tf.ones([1, 1])) self.assertEqual(w_init.count, 3) self.assertEqual(b_init.count, 3) def test_default_name(self): mod = mlp.MLP([1]) self.assertEqual(mod.name, "mlp") def test_custom_name(self): mod = mlp.MLP([1], name="foobar") self.assertEqual(mod.name, "foobar") def test_reverse_default_name(self): mod = reversed_mlp() self.assertEqual(mod.name, "mlp_reversed") def test_reverse_custom_name(self): mod = reversed_mlp(name="foobar") self.assertEqual(mod.name, "foobar_reversed") def test_reverse_override_name(self): mod = mlp.MLP([2, 3, 4]) mod(tf.ones([1, 1])) rev = mod.reverse(name="foobar") self.assertEqual(rev.name, "foobar") def test_reverse(self): mod = reversed_mlp() self.assertEqual([l.output_size for l in mod.submodules], [3, 2, 1]) @parameterized.parameters(True, False) def test_reverse_passed_with_bias(self, with_bias): mod = reversed_mlp(with_bias=with_bias) for linear in mod.submodules: self.assertEqual(linear.with_bias, with_bias) def test_reverse_w_init(self): w_init = CountingInitializer() mod = reversed_mlp(w_init=w_init) for linear in mod.submodules: self.assertIs(linear.w_init, w_init) def test_reverse_b_init(self): b_init = CountingInitializer() mod = reversed_mlp(b_init=b_init) for linear in mod.submodules: self.assertIs(linear.b_init, b_init) def test_reverse_activation(self): activation = CountingActivation() mod = reversed_mlp(activation=activation) activation.count = 0 mod(tf.ones([1, 1])) self.assertEqual(activation.count, 2) def test_dropout_requires_is_training(self): mod = mlp.MLP([1, 1], dropout_rate=0.5) with self.assertRaisesRegex(ValueError, "is_training.* is required"): mod(tf.ones([1, 1])) @parameterized.parameters(False, True) def test_no_dropout_rejects_is_training(self, is_training): mod = mlp.MLP([1, 1]) with self.assertRaisesRegex(ValueError, "is_training.*only.*with dropout"): mod(tf.ones([1, 1]), is_training=is_training) @parameterized.parameters(False, True) def test_reverse_activate_final(self, activate_final): activation = CountingActivation() mod = reversed_mlp(activation=activation, activate_final=activate_final) activation.count = 0 mod(tf.ones([1, 1])) self.assertEqual(activation.count, 3 if activate_final else 2) @parameterized.parameters(itertools.product((False, True), (False, True))) def test_applies_activation_with_dropout(self, use_dropout, is_training): activation = CountingActivation() mod = mlp.MLP([1, 1, 1], dropout_rate=(0.5 if use_dropout else None), activation=activation) mod(tf.ones([1, 1]), is_training=(is_training if use_dropout else None)) self.assertEqual(activation.count, 2) def test_repr(self): mod = mlp.MLP([1, 2, 3]) for index, linear in enumerate(mod.submodules): self.assertEqual( repr(linear), "Linear(output_size={}, name='linear_{}')".format(index + 1, index)) def reversed_mlp(**kwargs): mod = mlp.MLP([2, 3, 4], **kwargs) mod(tf.ones([1, 1])) return mod.reverse() class CountingActivation: def __init__(self): self.count = 0 def __call__(self, x): self.count += 1 return x class CountingInitializer: def __init__(self): self.count = 0 def __call__(self, shape, dtype=tf.float32): self.count += 1 return tf.ones(shape, dtype=dtype) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/nets/resnet.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """ResNet model for Sonnet.""" from typing import Mapping, Optional, Sequence, Union from sonnet.src import base from sonnet.src import batch_norm from sonnet.src import conv from sonnet.src import initializers from sonnet.src import linear from sonnet.src import pad import tensorflow as tf class BottleNeckBlockV1(base.Module): """Bottleneck Block for a ResNet implementation.""" def __init__(self, channels: int, stride: Union[int, Sequence[int]], use_projection: bool, bn_config: Mapping[str, float], name: Optional[str] = None): super().__init__(name=name) self._channels = channels self._stride = stride self._use_projection = use_projection self._bn_config = bn_config batchnorm_args = {"create_scale": True, "create_offset": True} batchnorm_args.update(bn_config) if self._use_projection: self._proj_conv = conv.Conv2D( output_channels=channels, kernel_shape=1, stride=stride, with_bias=False, padding=pad.same, name="shortcut_conv") self._proj_batchnorm = batch_norm.BatchNorm( name="shortcut_batchnorm", **batchnorm_args) self._layers = [] conv_0 = conv.Conv2D( output_channels=channels // 4, kernel_shape=1, stride=1, with_bias=False, padding=pad.same, name="conv_0") self._layers.append( [conv_0, batch_norm.BatchNorm(name="batchnorm_0", **batchnorm_args)]) conv_1 = conv.Conv2D( output_channels=channels // 4, kernel_shape=3, stride=stride, with_bias=False, padding=pad.same, name="conv_1") self._layers.append( [conv_1, batch_norm.BatchNorm(name="batchnorm_1", **batchnorm_args)]) conv_2 = conv.Conv2D( output_channels=channels, kernel_shape=1, stride=1, with_bias=False, padding=pad.same, name="conv_2") batchnorm_2 = batch_norm.BatchNorm( name="batchnorm_2", scale_init=initializers.Zeros(), **batchnorm_args) self._layers.append([conv_2, batchnorm_2]) def __call__(self, inputs, is_training): if self._use_projection: shortcut = self._proj_conv(inputs) shortcut = self._proj_batchnorm(shortcut, is_training=is_training) else: shortcut = inputs net = inputs for i, [conv_layer, batchnorm_layer] in enumerate(self._layers): net = conv_layer(net) net = batchnorm_layer(net, is_training=is_training) net = tf.nn.relu(net) if i < 2 else net # Don't apply relu on last layer return tf.nn.relu(net + shortcut) class BottleNeckBlockV2(base.Module): """Bottleneck Block for a Resnet implementation.""" def __init__(self, channels: int, stride: Union[int, Sequence[int]], use_projection: bool, bn_config: Mapping[str, float], name: Optional[str] = None): super().__init__(name=name) self._channels = channels self._stride = stride self._use_projection = use_projection self._bn_config = bn_config batchnorm_args = {"create_scale": True, "create_offset": True} batchnorm_args.update(bn_config) if self._use_projection: self._proj_conv = conv.Conv2D( output_channels=channels, kernel_shape=1, stride=stride, with_bias=False, padding=pad.same, name="shortcut_conv") self._conv_0 = conv.Conv2D( output_channels=channels // 4, kernel_shape=1, stride=1, with_bias=False, padding=pad.same, name="conv_0") self._bn_0 = batch_norm.BatchNorm(name="batchnorm_0", **batchnorm_args) self._conv_1 = conv.Conv2D( output_channels=channels // 4, kernel_shape=3, stride=stride, with_bias=False, padding=pad.same, name="conv_1") self._bn_1 = batch_norm.BatchNorm(name="batchnorm_1", **batchnorm_args) self._conv_2 = conv.Conv2D( output_channels=channels, kernel_shape=1, stride=1, with_bias=False, padding=pad.same, name="conv_2") # NOTE: Some implementations of ResNet50 v2 suggest initializing gamma/scale # here to zeros. self._bn_2 = batch_norm.BatchNorm(name="batchnorm_2", **batchnorm_args) def __call__(self, inputs, is_training): net = inputs shortcut = inputs for i, (conv_i, bn_i) in enumerate(((self._conv_0, self._bn_0), (self._conv_1, self._bn_1), (self._conv_2, self._bn_2))): net = bn_i(net, is_training=is_training) net = tf.nn.relu(net) if i == 0 and self._use_projection: shortcut = self._proj_conv(net) net = conv_i(net) return net + shortcut class BlockGroup(base.Module): """Higher level block for ResNet implementation.""" def __init__(self, channels: int, num_blocks: int, stride: Union[int, Sequence[int]], bn_config: Mapping[str, float], resnet_v2: bool = False, name: Optional[str] = None): super().__init__(name=name) self._channels = channels self._num_blocks = num_blocks self._stride = stride self._bn_config = bn_config if resnet_v2: bottle_neck_block = BottleNeckBlockV2 else: bottle_neck_block = BottleNeckBlockV1 self._blocks = [] for id_block in range(num_blocks): self._blocks.append( bottle_neck_block( channels=channels, stride=stride if id_block == 0 else 1, use_projection=(id_block == 0), bn_config=bn_config, name="block_%d" % (id_block))) def __call__(self, inputs, is_training): net = inputs for block in self._blocks: net = block(net, is_training=is_training) return net class ResNet(base.Module): """ResNet model.""" def __init__(self, blocks_per_group_list: Sequence[int], num_classes: int, bn_config: Optional[Mapping[str, float]] = None, resnet_v2: bool = False, channels_per_group_list: Sequence[int] = (256, 512, 1024, 2048), name: Optional[str] = None): """Constructs a ResNet model. Args: blocks_per_group_list: A sequence of length 4 that indicates the number of blocks created in each group. num_classes: The number of classes to classify the inputs into. bn_config: A dictionary of two elements, `decay_rate` and `eps` to be passed on to the `BatchNorm` layers. By default the `decay_rate` is `0.9` and `eps` is `1e-5`. resnet_v2: Whether to use the v1 or v2 ResNet implementation. Defaults to False. channels_per_group_list: A sequence of length 4 that indicates the number of channels used for each block in each group. name: Name of the module. """ super().__init__(name=name) if bn_config is None: bn_config = {"decay_rate": 0.9, "eps": 1e-5} self._bn_config = bn_config self._resnet_v2 = resnet_v2 # Number of blocks in each group for ResNet. if len(blocks_per_group_list) != 4: raise ValueError( "`blocks_per_group_list` must be of length 4 not {}".format( len(blocks_per_group_list))) self._blocks_per_group_list = blocks_per_group_list # Number of channels in each group for ResNet. if len(channels_per_group_list) != 4: raise ValueError( "`channels_per_group_list` must be of length 4 not {}".format( len(channels_per_group_list))) self._channels_per_group_list = channels_per_group_list self._initial_conv = conv.Conv2D( output_channels=64, kernel_shape=7, stride=2, with_bias=False, padding=pad.same, name="initial_conv") if not self._resnet_v2: self._initial_batchnorm = batch_norm.BatchNorm( create_scale=True, create_offset=True, name="initial_batchnorm", **bn_config) self._block_groups = [] strides = [1, 2, 2, 2] for i in range(4): self._block_groups.append( BlockGroup( channels=self._channels_per_group_list[i], num_blocks=self._blocks_per_group_list[i], stride=strides[i], bn_config=bn_config, resnet_v2=resnet_v2, name="block_group_%d" % (i))) if self._resnet_v2: self._final_batchnorm = batch_norm.BatchNorm( create_scale=True, create_offset=True, name="final_batchnorm", **bn_config) self._logits = linear.Linear( output_size=num_classes, w_init=initializers.Zeros(), name="logits") def __call__(self, inputs, is_training): net = inputs net = self._initial_conv(net) if not self._resnet_v2: net = self._initial_batchnorm(net, is_training=is_training) net = tf.nn.relu(net) net = tf.nn.max_pool2d( net, ksize=3, strides=2, padding="SAME", name="initial_max_pool") for block_group in self._block_groups: net = block_group(net, is_training) if self._resnet_v2: net = self._final_batchnorm(net, is_training=is_training) net = tf.nn.relu(net) net = tf.reduce_mean(net, axis=[1, 2], name="final_avg_pool") return self._logits(net) class ResNet50(ResNet): """ResNet50 module.""" def __init__(self, num_classes: int, bn_config: Optional[Mapping[str, float]] = None, resnet_v2: bool = False, name: Optional[str] = None): """Constructs a ResNet model. Args: num_classes: The number of classes to classify the inputs into. bn_config: A dictionary of two elements, `decay_rate` and `eps` to be passed on to the `BatchNorm` layers. resnet_v2: Whether to use the v1 or v2 ResNet implementation. Defaults to False. name: Name of the module. """ super().__init__([3, 4, 6, 3], num_classes=num_classes, bn_config=bn_config, resnet_v2=resnet_v2, name=name) ================================================ FILE: sonnet/src/nets/resnet_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.nets.resnet.""" from absl.testing import parameterized from sonnet.src import test_utils from sonnet.src.nets import resnet import tensorflow as tf class ResnetTest(test_utils.TestCase, parameterized.TestCase): @parameterized.parameters(True, False) def test_simple(self, resnet_v2): image = tf.random.normal([2, 64, 64, 3]) model = resnet.ResNet([1, 1, 1, 1], 10, resnet_v2=resnet_v2) logits = model(image, is_training=True) self.assertIsNotNone(logits) self.assertEqual(logits.shape, [2, 10]) @parameterized.parameters(True, False) def test_tf_function(self, resnet_v2): image = tf.random.normal([2, 64, 64, 3]) model = resnet.ResNet( [1, 1, 1, 1], 10, resnet_v2=resnet_v2, ) f = tf.function(model) logits = f(image, is_training=True) self.assertIsNotNone(logits) self.assertEqual(logits.shape, [2, 10]) self.assertAllEqual(model(image, is_training=True).numpy(), logits.numpy()) @parameterized.parameters(3, 5) def test_error_incorrect_args_block_list(self, list_length): block_list = [i for i in range(list_length)] with self.assertRaisesRegex( ValueError, "blocks_per_group_list` must be of length 4 not {}".format( list_length)): resnet.ResNet(block_list, 10, {"decay_rate": 0.9, "eps": 1e-5}) @parameterized.parameters(3, 5) def test_error_incorrect_args_channel_list(self, list_length): channel_list = [i for i in range(list_length)] with self.assertRaisesRegex( ValueError, "channels_per_group_list` must be of length 4 not {}".format( list_length)): resnet.ResNet([1, 1, 1, 1], 10, {"decay_rate": 0.9, "eps": 1e-5}, channels_per_group_list=channel_list) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/nets/vqvae.py ================================================ # Copyright 2018 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Sonnet implementation of VQ-VAE https://arxiv.org/abs/1711.00937.""" from sonnet.src import base from sonnet.src import initializers from sonnet.src import moving_averages from sonnet.src import types import tensorflow as tf class VectorQuantizer(base.Module): """Sonnet module representing the VQ-VAE layer. Implements the algorithm presented in 'Neural Discrete Representation Learning' by van den Oord et al. https://arxiv.org/abs/1711.00937 Input any tensor to be quantized. Last dimension will be used as space in which to quantize. All other dimensions will be flattened and will be seen as different examples to quantize. The output tensor will have the same shape as the input. For example a tensor with shape [16, 32, 32, 64] will be reshaped into [16384, 64] and all 16384 vectors (each of 64 dimensions) will be quantized independently. Attributes: embedding_dim: integer representing the dimensionality of the tensors in the quantized space. Inputs to the modules must be in this format as well. num_embeddings: integer, the number of vectors in the quantized space. commitment_cost: scalar which controls the weighting of the loss terms (see equation 4 in the paper - this variable is Beta). """ def __init__(self, embedding_dim: int, num_embeddings: int, commitment_cost: types.FloatLike, dtype: tf.DType = tf.float32, name: str = 'vector_quantizer'): """Initializes a VQ-VAE module. Args: embedding_dim: dimensionality of the tensors in the quantized space. Inputs to the modules must be in this format as well. num_embeddings: number of vectors in the quantized space. commitment_cost: scalar which controls the weighting of the loss terms (see equation 4 in the paper - this variable is Beta). dtype: dtype for the embeddings variable, defaults to tf.float32. name: name of the module. """ super().__init__(name=name) self.embedding_dim = embedding_dim self.num_embeddings = num_embeddings self.commitment_cost = commitment_cost embedding_shape = [embedding_dim, num_embeddings] initializer = initializers.VarianceScaling(distribution='uniform') self.embeddings = tf.Variable( initializer(embedding_shape, dtype), name='embeddings') def __call__(self, inputs, is_training): """Connects the module to some inputs. Args: inputs: Tensor, final dimension must be equal to embedding_dim. All other leading dimensions will be flattened and treated as a large batch. is_training: boolean, whether this connection is to training data. Returns: dict containing the following keys and values: quantize: Tensor containing the quantized version of the input. loss: Tensor containing the loss to optimize. perplexity: Tensor containing the perplexity of the encodings. encodings: Tensor containing the discrete encodings, ie which element of the quantized space each input element was mapped to. encoding_indices: Tensor containing the discrete encoding indices, ie which element of the quantized space each input element was mapped to. """ flat_inputs = tf.reshape(inputs, [-1, self.embedding_dim]) distances = ( tf.reduce_sum(flat_inputs**2, 1, keepdims=True) - 2 * tf.matmul(flat_inputs, self.embeddings) + tf.reduce_sum(self.embeddings**2, 0, keepdims=True)) encoding_indices = tf.argmax(-distances, 1) encodings = tf.one_hot(encoding_indices, self.num_embeddings, dtype=distances.dtype) # NB: if your code crashes with a reshape error on the line below about a # Tensor containing the wrong number of values, then the most likely cause # is that the input passed in does not have a final dimension equal to # self.embedding_dim. Ideally we would catch this with an Assert but that # creates various other problems related to device placement / TPUs. encoding_indices = tf.reshape(encoding_indices, tf.shape(inputs)[:-1]) quantized = self.quantize(encoding_indices) e_latent_loss = tf.reduce_mean((tf.stop_gradient(quantized) - inputs)**2) q_latent_loss = tf.reduce_mean((quantized - tf.stop_gradient(inputs))**2) loss = q_latent_loss + self.commitment_cost * e_latent_loss # Straight Through Estimator quantized = inputs + tf.stop_gradient(quantized - inputs) avg_probs = tf.reduce_mean(encodings, 0) perplexity = tf.exp(-tf.reduce_sum(avg_probs * tf.math.log(avg_probs + 1e-10))) return { 'quantize': quantized, 'loss': loss, 'perplexity': perplexity, 'encodings': encodings, 'encoding_indices': encoding_indices, 'distances': distances, } def quantize(self, encoding_indices): """Returns embedding tensor for a batch of indices.""" w = tf.transpose(self.embeddings, [1, 0]) # TODO(mareynolds) in V1 we had a validate_indices kwarg, this is no longer # supported in V2. Are we missing anything here? return tf.nn.embedding_lookup(w, encoding_indices) class VectorQuantizerEMA(base.Module): """Sonnet module representing the VQ-VAE layer. Implements a slightly modified version of the algorithm presented in 'Neural Discrete Representation Learning' by van den Oord et al. https://arxiv.org/abs/1711.00937 The difference between VectorQuantizerEMA and VectorQuantizer is that this module uses exponential moving averages to update the embedding vectors instead of an auxiliary loss. This has the advantage that the embedding updates are independent of the choice of optimizer (SGD, RMSProp, Adam, K-Fac, ...) used for the encoder, decoder and other parts of the architecture. For most experiments the EMA version trains faster than the non-EMA version. Input any tensor to be quantized. Last dimension will be used as space in which to quantize. All other dimensions will be flattened and will be seen as different examples to quantize. The output tensor will have the same shape as the input. For example a tensor with shape [16, 32, 32, 64] will be reshaped into [16384, 64] and all 16384 vectors (each of 64 dimensions) will be quantized independently. Attributes: embedding_dim: integer representing the dimensionality of the tensors in the quantized space. Inputs to the modules must be in this format as well. num_embeddings: integer, the number of vectors in the quantized space. commitment_cost: scalar which controls the weighting of the loss terms (see equation 4 in the paper). decay: float, decay for the moving averages. epsilon: small float constant to avoid numerical instability. """ def __init__(self, embedding_dim, num_embeddings, commitment_cost, decay, epsilon=1e-5, dtype=tf.float32, name='vector_quantizer_ema'): """Initializes a VQ-VAE EMA module. Args: embedding_dim: integer representing the dimensionality of the tensors in the quantized space. Inputs to the modules must be in this format as well. num_embeddings: integer, the number of vectors in the quantized space. commitment_cost: scalar which controls the weighting of the loss terms (see equation 4 in the paper - this variable is Beta). decay: float between 0 and 1, controls the speed of the Exponential Moving Averages. epsilon: small constant to aid numerical stability, default 1e-5. dtype: dtype for the embeddings variable, defaults to tf.float32. name: name of the module. """ super().__init__(name=name) self.embedding_dim = embedding_dim self.num_embeddings = num_embeddings if not 0 <= decay <= 1: raise ValueError('decay must be in range [0, 1]') self.decay = decay self.commitment_cost = commitment_cost self.epsilon = epsilon embedding_shape = [embedding_dim, num_embeddings] initializer = initializers.VarianceScaling(distribution='uniform') self.embeddings = tf.Variable( initializer(embedding_shape, dtype), trainable=False, name='embeddings') self.ema_cluster_size = moving_averages.ExponentialMovingAverage( decay=self.decay, name='ema_cluster_size') self.ema_cluster_size.initialize(tf.zeros([num_embeddings], dtype=dtype)) self.ema_dw = moving_averages.ExponentialMovingAverage( decay=self.decay, name='ema_dw') self.ema_dw.initialize(self.embeddings) def __call__(self, inputs, is_training): """Connects the module to some inputs. Args: inputs: Tensor, final dimension must be equal to embedding_dim. All other leading dimensions will be flattened and treated as a large batch. is_training: boolean, whether this connection is to training data. When this is set to False, the internal moving average statistics will not be updated. Returns: dict containing the following keys and values: quantize: Tensor containing the quantized version of the input. loss: Tensor containing the loss to optimize. perplexity: Tensor containing the perplexity of the encodings. encodings: Tensor containing the discrete encodings, ie which element of the quantized space each input element was mapped to. encoding_indices: Tensor containing the discrete encoding indices, ie which element of the quantized space each input element was mapped to. """ flat_inputs = tf.reshape(inputs, [-1, self.embedding_dim]) distances = ( tf.reduce_sum(flat_inputs**2, 1, keepdims=True) - 2 * tf.matmul(flat_inputs, self.embeddings) + tf.reduce_sum(self.embeddings**2, 0, keepdims=True)) encoding_indices = tf.argmax(-distances, 1) encodings = tf.one_hot(encoding_indices, self.num_embeddings, dtype=distances.dtype) # NB: if your code crashes with a reshape error on the line below about a # Tensor containing the wrong number of values, then the most likely cause # is that the input passed in does not have a final dimension equal to # self.embedding_dim. Ideally we would catch this with an Assert but that # creates various other problems related to device placement / TPUs. encoding_indices = tf.reshape(encoding_indices, tf.shape(inputs)[:-1]) quantized = self.quantize(encoding_indices) e_latent_loss = tf.reduce_mean((tf.stop_gradient(quantized) - inputs)**2) if is_training: updated_ema_cluster_size = self.ema_cluster_size( tf.reduce_sum(encodings, axis=0)) dw = tf.matmul(flat_inputs, encodings, transpose_a=True) updated_ema_dw = self.ema_dw(dw) n = tf.reduce_sum(updated_ema_cluster_size) updated_ema_cluster_size = ((updated_ema_cluster_size + self.epsilon) / (n + self.num_embeddings * self.epsilon) * n) normalised_updated_ema_w = ( updated_ema_dw / tf.reshape(updated_ema_cluster_size, [1, -1])) self.embeddings.assign(normalised_updated_ema_w) loss = self.commitment_cost * e_latent_loss else: loss = self.commitment_cost * e_latent_loss # Straight Through Estimator quantized = inputs + tf.stop_gradient(quantized - inputs) avg_probs = tf.reduce_mean(encodings, 0) perplexity = tf.exp(-tf.reduce_sum(avg_probs * tf.math.log(avg_probs + 1e-10))) return { 'quantize': quantized, 'loss': loss, 'perplexity': perplexity, 'encodings': encodings, 'encoding_indices': encoding_indices, 'distances': distances, } def quantize(self, encoding_indices): """Returns embedding tensor for a batch of indices.""" w = tf.transpose(self.embeddings, [1, 0]) return tf.nn.embedding_lookup(w, encoding_indices) ================================================ FILE: sonnet/src/nets/vqvae_test.py ================================================ # Copyright 2018 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.nets.vqvae.""" from absl.testing import parameterized import numpy as np from sonnet.src import test_utils from sonnet.src.nets import vqvae import tensorflow as tf import tree class VqvaeTest(parameterized.TestCase, test_utils.TestCase): @parameterized.parameters((vqvae.VectorQuantizer, { 'embedding_dim': 4, 'num_embeddings': 8, 'commitment_cost': 0.25 }), (vqvae.VectorQuantizerEMA, { 'embedding_dim': 6, 'num_embeddings': 13, 'commitment_cost': 0.5, 'decay': 0.1 })) def testConstruct(self, constructor, kwargs): vqvae_module = constructor(**kwargs) # Batch of input vectors to quantize inputs_np = np.random.randn(100, kwargs['embedding_dim']).astype(np.float32) inputs = tf.constant(inputs_np) # Set is_training to False, otherwise for the EMA case just evaluating the # forward pass will change the embeddings, meaning that some of our computed # closest embeddings will be incorrect. vq_output = vqvae_module(inputs, is_training=False) # Output shape is correct self.assertEqual(vq_output['quantize'].shape, inputs.shape) vq_output_np = tree.map_structure(lambda t: t.numpy(), vq_output) embeddings_np = vqvae_module.embeddings.numpy() self.assertEqual(embeddings_np.shape, (kwargs['embedding_dim'], kwargs['num_embeddings'])) # Check that each input was assigned to the embedding it is closest to. distances = ((inputs_np**2).sum(axis=1, keepdims=True) - 2 * np.dot(inputs_np, embeddings_np) + (embeddings_np**2).sum(axis=0, keepdims=True)) closest_index = np.argmax(-distances, axis=1) # On TPU, distances can be different by ~1% due to precision. This can cause # the distanc to the closest embedding to flip, leading to a difference # in the encoding indices tensor. First we check that the continuous # distances are reasonably close, and then we only allow N differences in # the encodings. For batch of 100, N == 3 seems okay (passed 1000x tests). self.assertAllClose(distances, vq_output_np['distances'], atol=4e-2) num_differences_in_encodings = (closest_index != vq_output_np['encoding_indices']).sum() num_differences_allowed = 3 self.assertLessEqual(num_differences_in_encodings, num_differences_allowed) @parameterized.parameters((vqvae.VectorQuantizer, { 'embedding_dim': 4, 'num_embeddings': 8, 'commitment_cost': 0.25 }), (vqvae.VectorQuantizerEMA, { 'embedding_dim': 6, 'num_embeddings': 13, 'commitment_cost': 0.5, 'decay': 0.1 })) def testShapeChecking(self, constructor, kwargs): vqvae_module = constructor(**kwargs) wrong_shape_input = np.random.randn(100, kwargs['embedding_dim'] * 2) with self.assertRaisesRegex(tf.errors.InvalidArgumentError, 'but the requested shape has'): vqvae_module( tf.constant(wrong_shape_input.astype(np.float32)), is_training=False) @parameterized.parameters((vqvae.VectorQuantizer, { 'embedding_dim': 4, 'num_embeddings': 8, 'commitment_cost': 0.25 }), (vqvae.VectorQuantizerEMA, { 'embedding_dim': 6, 'num_embeddings': 13, 'commitment_cost': 0.5, 'decay': 0.1 })) def testNoneBatch(self, constructor, kwargs): """Check that vqvae can be built on input with a None batch dimension.""" vqvae_module = constructor(**kwargs) inputs = tf.zeros([0, 5, 5, kwargs['embedding_dim']]) vqvae_module(inputs, is_training=False) @parameterized.parameters({'use_tf_function': True, 'dtype': tf.float32}, {'use_tf_function': True, 'dtype': tf.float64}, {'use_tf_function': False, 'dtype': tf.float32}, {'use_tf_function': False, 'dtype': tf.float64}) def testEmaUpdating(self, use_tf_function, dtype): if self.primary_device == 'TPU' and dtype == tf.float64: self.skipTest('F64 not supported by TPU') embedding_dim = 6 np_dtype = np.float64 if dtype is tf.float64 else np.float32 decay = np.array(0.1, dtype=np_dtype) vqvae_module = vqvae.VectorQuantizerEMA( embedding_dim=embedding_dim, num_embeddings=7, commitment_cost=0.5, decay=decay, dtype=dtype) if use_tf_function: vqvae_module = tf.function(vqvae_module) batch_size = 16 prev_embeddings = vqvae_module.embeddings.numpy() # Embeddings should change with every forwards pass if is_training == True. for _ in range(10): inputs = tf.random.normal([batch_size, embedding_dim], dtype=dtype) vqvae_module(inputs, is_training=True) current_embeddings = vqvae_module.embeddings.numpy() self.assertFalse((prev_embeddings == current_embeddings).all()) prev_embeddings = current_embeddings # Forward passes with is_training == False don't change anything for _ in range(10): inputs = tf.random.normal([batch_size, embedding_dim], dtype=dtype) vqvae_module(inputs, is_training=False) current_embeddings = vqvae_module.embeddings.numpy() self.assertTrue((current_embeddings == prev_embeddings).all()) def testEmbeddingsNotTrainable(self): # NOTE: EMA embeddings are updated during the forward pass and not as part # of the optimizer step. model = vqvae.VectorQuantizerEMA( embedding_dim=6, num_embeddings=13, commitment_cost=0.5, decay=0.1) self.assertFalse(model.embeddings.trainable) if __name__ == '__main__': tf.test.main() ================================================ FILE: sonnet/src/once.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Utility to run functions and methods once.""" import uuid from sonnet.src import utils _ONCE_PROPERTY = "_snt_once" def _check_no_output(output): if output is not None: raise ValueError("@snt.once decorated functions cannot return values") def once(f): """Decorator which ensures a wrapped method is only ever run once. >>> @snt.once ... def f(): ... print('Hello, world!') >>> f() Hello, world! >>> f() >>> f() If `f` is a method then it will be evaluated once per instance: >>> class MyObject: ... @snt.once ... def f(self): ... print('Hello, world!') >>> o = MyObject() >>> o.f() Hello, world! >>> o.f() >>> o2 = MyObject() >>> o2.f() Hello, world! >>> o.f() >>> o2.f() If an error is raised during execution of `f` it will be raised to the user. Next time the method is run, it will be treated as not having run before. Args: f: A function to wrap which should only be called once. Returns: Wrapped version of `f` which will only evaluate `f` the first time it is called. """ # TODO(tomhennigan) Perhaps some more human friendly identifier? once_id = uuid.uuid4() @utils.decorator def wrapper(wrapped, instance, args, kwargs): """Decorator which ensures a wrapped method is only ever run once.""" if instance is None: # NOTE: We can't use the weakset since you can't weakref None. if not wrapper.seen_none: _check_no_output(wrapped(*args, **kwargs)) wrapper.seen_none = True return # Get or set the `seen` set for this object. seen = getattr(instance, _ONCE_PROPERTY, None) if seen is None: seen = set() setattr(instance, _ONCE_PROPERTY, seen) if once_id not in seen: _check_no_output(wrapped(*args, **kwargs)) seen.add(once_id) wrapper.seen_none = False decorated = wrapper(f) # pylint: disable=no-value-for-parameter,assignment-from-none decorated.__snt_once_wrapped__ = f return decorated ================================================ FILE: sonnet/src/once_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.once.""" import pickle from absl.testing import absltest from absl.testing import parameterized from sonnet.src import once class OnceTest(parameterized.TestCase): def test_runs_once(self): r = [] @once.once def f(): r.append(None) for _ in range(3): f() self.assertEqual(r, [None]) def test_always_returns_none(self): f = once.once(lambda: "Hello, world!") with self.assertRaisesRegex(ValueError, "snt.once .* cannot return"): f() def test_does_not_cache_on_error(self): @once.once def f(): raise ValueError with self.assertRaises(ValueError): f() with self.assertRaises(ValueError): f() def test_method(self): o1 = Counter() o2 = Counter() for _ in range(10): o1.increment() o2.increment() self.assertEqual(o1.call_count, 1) self.assertEqual(o2.call_count, 1) def test_method_does_not_cache_on_error(self): class Dummy: @once.once def f(self): raise ValueError o = Dummy() with self.assertRaises(ValueError): o.f() with self.assertRaises(ValueError): o.f() def test_pickle_method_before_evaluation(self): c1 = Counter() c2 = pickle.loads(pickle.dumps(c1)) c1.increment() self.assertEqual(c1.call_count, 1) self.assertEqual(c2.call_count, 0) c2.increment() self.assertEqual(c1.call_count, 1) self.assertEqual(c2.call_count, 1) def test_pickle_method_already_evaluated(self): c1 = Counter() c1.increment() self.assertEqual(c1.call_count, 1) c2 = pickle.loads(pickle.dumps(c1)) self.assertEqual(c2.call_count, 1) c2.increment() self.assertEqual(c2.call_count, 1) def test_inline(self): r = [] f = once.once(lambda: r.append(None)) for _ in range(10): f() self.assertEqual(r, [None]) @parameterized.named_parameters( ("lambda", lambda: lambda: None), ("function", lambda: nop), ("method", lambda: NoOpCallable().nop), ("special_method", lambda: NoOpCallable().__call__), ("object", lambda: NoOpCallable())) # pylint: disable=unnecessary-lambda def test_adds_property(self, factory): f = factory() self.assertIs(once.once(f).__snt_once_wrapped__, f) def nop(): pass class NoOpCallable: def nop(self): pass def __call__(self): pass class Counter: call_count = 0 @once.once def increment(self): self.call_count += 1 if __name__ == "__main__": absltest.main() ================================================ FILE: sonnet/src/optimizers/BUILD ================================================ load("//sonnet/src:build_defs.bzl", "snt_py_library", "snt_py_test") package(default_visibility = ["//sonnet:__subpackages__", "//docs/ext:__subpackages__", "//examples:__subpackages__"]) licenses(["notice"]) snt_py_library( name = "optimizer_tests", testonly = 1, srcs = ["optimizer_tests.py"], deps = [ # pip: absl/testing:parameterized # pip: numpy "//sonnet/src:base", "//sonnet/src:test_utils", # pip: tensorflow # pip: tree ], ) snt_py_library( name = "adam", srcs = ["adam.py"], deps = [ ":optimizer_utils", "//sonnet/src:base", "//sonnet/src:once", "//sonnet/src:types", "//sonnet/src:utils", # pip: tensorflow ], ) snt_py_test( name = "adam_test", srcs = ["adam_test.py"], shard_count = 10, deps = [ ":adam", ":optimizer_tests", "//sonnet/src:test_utils", # pip: tensorflow ], ) snt_py_library( name = "momentum", srcs = ["momentum.py"], deps = [ ":optimizer_utils", "//sonnet/src:base", "//sonnet/src:once", "//sonnet/src:types", "//sonnet/src:utils", # pip: tensorflow ], ) snt_py_test( name = "momentum_test", srcs = ["momentum_test.py"], shard_count = 10, deps = [ ":momentum", ":optimizer_tests", "//sonnet/src:test_utils", # pip: tensorflow ], ) snt_py_library( name = "rmsprop", srcs = ["rmsprop.py"], deps = [ ":optimizer_utils", "//sonnet/src:base", "//sonnet/src:once", "//sonnet/src:types", "//sonnet/src:utils", # pip: tensorflow ], ) snt_py_test( name = "rmsprop_test", srcs = ["rmsprop_test.py"], shard_count = 10, deps = [ ":optimizer_tests", ":rmsprop", "//sonnet/src:test_utils", # pip: tensorflow ], ) snt_py_library( name = "sgd", srcs = ["sgd.py"], deps = [ ":optimizer_utils", "//sonnet/src:base", "//sonnet/src:types", # pip: tensorflow ], ) snt_py_test( name = "sgd_test", srcs = ["sgd_test.py"], shard_count = 10, deps = [ ":optimizer_tests", ":sgd", # pip: tensorflow ], ) snt_py_library( name = "optimizer_utils", srcs = ["optimizer_utils.py"], deps = [ "//sonnet/src:types", "//sonnet/src/distribute:replicator", # pip: tensorflow ], ) ================================================ FILE: sonnet/src/optimizers/__init__.py ================================================ # Copyright 2021 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ ================================================ FILE: sonnet/src/optimizers/adam.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Adaptive Moment Estimation (Adam) module.""" from typing import Optional, Sequence, Union from sonnet.src import base from sonnet.src import once from sonnet.src import types from sonnet.src import utils from sonnet.src.optimizers import optimizer_utils import tensorflow as tf def adam_update(g, alpha, beta_1, beta_2, epsilon, t, m, v): """Implements 'Algorithm 1' from :cite:`kingma2014adam`.""" m = beta_1 * m + (1. - beta_1) * g # Biased first moment estimate. v = beta_2 * v + (1. - beta_2) * g * g # Biased second raw moment estimate. m_hat = m / (1. - tf.pow(beta_1, t)) # Bias corrected 1st moment estimate. v_hat = v / (1. - tf.pow(beta_2, t)) # Bias corrected 2nd moment estimate. update = alpha * m_hat / (tf.sqrt(v_hat) + epsilon) return update, m, v class Adam(base.Optimizer): """Adaptive Moment Estimation (Adam) optimizer. Adam is an algorithm for first-order gradient-based optimization of stochastic objective functions, based on adaptive estimates of lower-order moments. See :cite:`kingma2014adam` for more details. Note: default parameter values have been taken from the paper. Attributes: learning_rate: Step size (``alpha`` in the paper). beta1: Exponential decay rate for first moment estimate. beta2: Exponential decay rate for second moment estimate. epsilon: Small value to avoid zero denominator. step: Step count. m: Biased first moment estimate (a list with one value per parameter). v: Biased second raw moment estimate (a list with one value per parameter). """ def __init__(self, learning_rate: Union[types.FloatLike, tf.Variable] = 0.001, beta1: Union[types.FloatLike, tf.Variable] = 0.9, beta2: Union[types.FloatLike, tf.Variable] = 0.999, epsilon: Union[types.FloatLike, tf.Variable] = 1e-8, name: Optional[str] = None): """Constructs an `Adam` module. Args: learning_rate: Step size (``alpha`` in the paper). beta1: Exponential decay rate for first moment estimate. beta2: Exponential decay rate for second moment estimate. epsilon: Small value to avoid zero denominator. name: Name of the module. """ super().__init__(name=name) self.learning_rate = learning_rate self.beta1 = beta1 self.beta2 = beta2 self.epsilon = epsilon # TODO(petebu): Consider allowing the user to pass in a step. self.step = tf.Variable(0, trainable=False, name="t", dtype=tf.int64) self.m = [] self.v = [] @once.once def _initialize(self, parameters: Sequence[tf.Variable]): """First and second order moments are initialized to zero.""" zero_var = lambda p: utils.variable_like(p, trainable=False) with tf.name_scope("m"): self.m.extend(zero_var(p) for p in parameters) with tf.name_scope("v"): self.v.extend(zero_var(p) for p in parameters) def apply(self, updates: Sequence[types.ParameterUpdate], parameters: Sequence[tf.Variable]): r"""Applies updates to parameters. Applies the Adam update rule for each update, parameter pair: .. math:: \begin{array}{ll} m_t = \beta_1 \cdot m_{t-1} + (1 - \beta_1) \cdot update \\ v_t = \beta_2 \cdot v_{t-1} + (1 - \beta_2) \cdot update^2 \\ \hat{m}_t = m_t / (1 - \beta_1^t) \\ \hat{v}_t = v_t / (1 - \beta_2^t) \\ delta = \alpha \cdot \hat{m}_t / (\sqrt{\hat{v}_t} + \epsilon) \\ param_t = param_{t-1} - delta \\ \end{array} Args: updates: A list of updates to apply to parameters. Updates are often gradients as returned by :tf:`GradientTape.gradient`. parameters: A list of parameters. Raises: ValueError: If `updates` and `parameters` are empty, have different lengths, or have inconsistent types. """ optimizer_utils.check_distribution_strategy() optimizer_utils.check_updates_parameters(updates, parameters) self._initialize(parameters) self.step.assign_add(1) for update, param, m_var, v_var in zip(updates, parameters, self.m, self.v): if update is None: continue optimizer_utils.check_same_dtype(update, param) learning_rate = tf.cast(self.learning_rate, update.dtype) beta_1 = tf.cast(self.beta1, update.dtype) beta_2 = tf.cast(self.beta2, update.dtype) epsilon = tf.cast(self.epsilon, update.dtype) step = tf.cast(self.step, update.dtype) if isinstance(update, tf.IndexedSlices): # Sparse read our state. update, indices = optimizer_utils.deduplicate_indexed_slices(update) m = m_var.sparse_read(indices) v = v_var.sparse_read(indices) # Compute and apply a sparse update to our parameter and state. update, m, v = adam_update( g=update, alpha=learning_rate, beta_1=beta_1, beta_2=beta_2, epsilon=epsilon, t=step, m=m, v=v) param.scatter_sub(tf.IndexedSlices(update, indices)) m_var.scatter_update(tf.IndexedSlices(m, indices)) v_var.scatter_update(tf.IndexedSlices(v, indices)) else: # Compute and apply a dense update to our parameter and state. update, m, v = adam_update( g=update, alpha=learning_rate, beta_1=beta_1, beta_2=beta_2, epsilon=epsilon, t=step, m=m_var, v=v_var) param.assign_sub(update) m_var.assign(m) v_var.assign(v) ================================================ FILE: sonnet/src/optimizers/adam_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.adam.""" from sonnet.src import test_utils from sonnet.src.optimizers import adam from sonnet.src.optimizers import optimizer_tests import tensorflow as tf CONFIGS = optimizer_tests.named_product(learning_rate=(0.1, 0.01, 0.001), beta_1=(0.9, 0.99, 0.999), beta_2=(0.9, 0.99, 0.999), epsilon=(1e-8,), seed=(28, 2, 90)) class ComparisonTest(optimizer_tests.AbstractFuzzTest): """Ensures Sonnet optimizers have equivalent behavior to TensorFlow.""" def _make_tf(self, learning_rate, beta_1, beta_2, epsilon): optimizer = tf.optimizers.Adam(learning_rate=learning_rate, beta_1=beta_1, beta_2=beta_2, epsilon=epsilon) return lambda g, p: optimizer.apply_gradients(zip(g, p)) def _make_snt(self, learning_rate, beta_1, beta_2, epsilon): optimizer = adam.Adam(learning_rate=learning_rate, beta1=beta_1, beta2=beta_2, epsilon=epsilon) return optimizer.apply @test_utils.combined_named_parameters(CONFIGS) def testComparingSonnetAndTensorFlow(self, config): seed = config.pop("seed") self.assertParametersRemainClose(seed, config) class AdamTest(optimizer_tests.OptimizerTestBase): def make_optimizer(self, **kwargs): if "learning_rate" not in kwargs: kwargs["learning_rate"] = 0.001 return adam.Adam(**kwargs) def testDense(self): parameters = [tf.Variable([1., 2.]), tf.Variable([3., 4.])] updates = [tf.constant([5., 5.]), tf.constant([3., 3.])] optimizer = self.make_optimizer(learning_rate=0.001) # Step 1 of Adam optimizer.apply(updates, parameters) self.assertAllClose([[0.999, 1.999], [2.999, 3.999]], [x.numpy() for x in parameters]) # Step 2 of Adam optimizer.apply(updates, parameters) self.assertAllClose([[0.998, 1.998], [2.998, 3.998]], [x.numpy() for x in parameters]) # Step 3 of Adam optimizer.apply(updates, parameters) self.assertAllClose([[0.997, 1.997], [2.997, 3.997]], [x.numpy() for x in parameters]) def testSparse(self): if self.primary_device in ("GPU", "TPU"): self.skipTest("IndexedSlices not supported on {}.".format( self.primary_device)) parameters = [tf.Variable([[1.], [2.]]), tf.Variable([[3.], [4.]])] tf_parameters = [tf.Variable([[1.], [2.]]), tf.Variable([[3.], [4.]])] updates = [ tf.IndexedSlices( tf.constant([0.1], shape=[1, 1]), tf.constant([0]), tf.constant([2, 1])), tf.IndexedSlices( tf.constant([0.01], shape=[1, 1]), tf.constant([1]), tf.constant([2, 1])) ] optimizer = self.make_optimizer(learning_rate=0.001) # Compare against TF optimizer. tf_optimizer = tf.optimizers.Adam(learning_rate=0.001) # Step 1 of Adam optimizer.apply(updates, parameters) self.assertAllClose([[0.999], [2.0]], parameters[0].numpy()) self.assertAllClose([[3.0], [3.999]], parameters[1].numpy()) tf_optimizer.apply_gradients(zip(updates, tf_parameters)) self.assertAllClose(tf_parameters[0].numpy(), parameters[0].numpy()) self.assertAllClose(tf_parameters[1].numpy(), parameters[1].numpy()) # Step 2 of Adam optimizer.apply(updates, parameters) self.assertAllClose([[0.998], [2.0]], parameters[0].numpy()) self.assertAllClose([[3.0], [3.998]], parameters[1].numpy()) tf_optimizer.apply_gradients(zip(updates, tf_parameters)) self.assertAllClose(tf_parameters[0].numpy(), parameters[0].numpy()) self.assertAllClose(tf_parameters[1].numpy(), parameters[1].numpy()) # Step 3 of Adam optimizer.apply(updates, parameters) self.assertAllClose([[0.997], [2.0]], parameters[0].numpy()) self.assertAllClose([[3.0], [3.997]], parameters[1].numpy()) tf_optimizer.apply_gradients(zip(updates, tf_parameters)) self.assertAllClose(tf_parameters[0].numpy(), parameters[0].numpy()) self.assertAllClose(tf_parameters[1].numpy(), parameters[1].numpy()) def testVariableHyperParams(self): parameters = [tf.Variable([1., 2.]), tf.Variable([3., 4.])] updates = [tf.constant([5., 5.]), tf.constant([3., 3.])] learning_rate = tf.Variable(0.001) optimizer = self.make_optimizer(learning_rate=learning_rate) optimizer.apply(updates, parameters) self.assertAllClose([[0.999, 1.999], [2.999, 3.999]], [x.numpy() for x in parameters]) learning_rate.assign(0.1) self.assertAlmostEqual(0.1, optimizer.learning_rate.numpy()) optimizer.apply(updates, parameters) self.assertAllClose([[0.899, 1.899], [2.899, 3.899]], [x.numpy() for x in parameters], rtol=1e-4) def testHyperParamDTypeConversion(self): parameters = [tf.Variable([1., 2.]), tf.Variable([3., 4.])] updates = [tf.constant([5., 5.]), tf.constant([3., 3.])] dtype = tf.float32 if self.primary_device == "TPU" else tf.float64 learning_rate = tf.Variable(0.001, dtype=dtype) beta1 = tf.Variable(0.9, dtype=dtype) beta2 = tf.Variable(0.999, dtype=dtype) epsilon = tf.Variable(1e-8, dtype=dtype) optimizer = self.make_optimizer( learning_rate=learning_rate, beta1=beta1, beta2=beta2, epsilon=epsilon) optimizer.apply(updates, parameters) self.assertAllClose([[0.999, 1.999], [2.999, 3.999]], [x.numpy() for x in parameters], rtol=1e-4) def testAuxVariablesColocatedWithOriginal(self): optimizer = self.make_optimizer(learning_rate=0.001) with tf.device("CPU:0"): var = tf.Variable(1.0) optimizer.apply([tf.constant(0.1)], [var]) self.assertEqual(optimizer.m[0].device, var.device) self.assertEqual(optimizer.v[0].device, var.device) class ReferenceAdamTest(optimizer_tests.OptimizerTestBase): def make_optimizer(self, **kwargs): if "learning_rate" not in kwargs: kwargs["learning_rate"] = 0.001 return optimizer_tests.WrappedTFOptimizer(tf.optimizers.Adam(**kwargs)) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/optimizers/momentum.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """SGD with Momentum module.""" from typing import Optional, Sequence, Union from sonnet.src import base from sonnet.src import once from sonnet.src import types from sonnet.src import utils from sonnet.src.optimizers import optimizer_utils import tensorflow as tf def momentum_update(update, learning_rate, mu, momentum, use_nesterov): """Computes a momentum update for a single parameter.""" momentum = (mu * momentum) + update if use_nesterov: update = learning_rate * ((mu * momentum) + update) else: update = learning_rate * momentum return update, momentum class Momentum(base.Optimizer): """SGD with Momentum module. Attributes: learning_rate: Learning rate. momentum: Momentum scalar. use_nesterov: `True` if using Nesterov momentum. accumulated_momentum: Accumulated momentum for each parameter. """ def __init__(self, learning_rate: Union[types.FloatLike, tf.Variable], momentum: Union[types.FloatLike, tf.Variable], use_nesterov: bool = False, name: Optional[str] = None): """Constructs a `Momentum` module. Args: learning_rate: Learning rate. momentum: Momentum scalar. use_nesterov: Whether to use Nesterov momentum. name: Name of the module. """ super().__init__(name) self.learning_rate = learning_rate self.momentum = momentum # TODO(petebu) Reconsider name. self.use_nesterov = use_nesterov self.accumulated_momentum = [] # TODO(petebu) Reconsider name. @once.once def _initialize(self, parameters): with tf.name_scope("accumulated_momentum"): self.accumulated_momentum.extend( utils.variable_like(p, trainable=False) for p in parameters) def apply(self, updates: Sequence[types.ParameterUpdate], parameters: Sequence[tf.Variable]): """Applies updates to parameters. By default it applies the momentum update rule for each update, parameter pair: accum_t <- momentum * accum_{t-1} + update parameter <- parameter - learning_rate * accum_t And when using Nesterov momentum (`use_nesterov=True`) it applies: accum_t <- momentum * accum_{t-1} + update parameter <- parameter - (learning_rate * update + learning_rate * momentum * accum_t) Args: updates: A list of updates to apply to parameters. Updates are often gradients as returned by `tf.GradientTape.gradient`. parameters: A list of parameters. A parameter is a `tf.Variable`. Raises: ValueError: If `updates` and `parameters` are empty, have different lengths, or have inconsistent types. """ optimizer_utils.check_distribution_strategy() optimizer_utils.check_updates_parameters(updates, parameters) self._initialize(parameters) for update, param, momentum_var in zip(updates, parameters, self.accumulated_momentum): if update is None: continue optimizer_utils.check_same_dtype(update, param) learning_rate = tf.cast(self.learning_rate, update.dtype) mu = tf.cast(self.momentum, update.dtype) if isinstance(update, tf.IndexedSlices): # Sparse read our state. update, indices = optimizer_utils.deduplicate_indexed_slices(update) momentum = momentum_var.sparse_read(indices) # Compute and apply a sparse update to our parameter and state. update, momentum = momentum_update(update, learning_rate, mu, momentum, self.use_nesterov) momentum_var.scatter_update(tf.IndexedSlices(momentum, indices)) param.scatter_sub(tf.IndexedSlices(update, indices)) else: # Compute and apply a dense update. update, momentum = momentum_update(update, learning_rate, mu, momentum_var, self.use_nesterov) momentum_var.assign(momentum) param.assign_sub(update) ================================================ FILE: sonnet/src/optimizers/momentum_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.momentum.""" from sonnet.src import test_utils from sonnet.src.optimizers import momentum as momentum_lib from sonnet.src.optimizers import optimizer_tests import tensorflow as tf CONFIGS = optimizer_tests.named_product(learning_rate=(0.1, 0.01, 0.001), momentum=(0.9, 0.5, 0.2), use_nesterov=(True, False), seed=(28, 2, 90)) class ComparisonTest(optimizer_tests.AbstractFuzzTest): """Ensures Sonnet optimizers have equivalent behavior to TensorFlow.""" def _make_tf(self, learning_rate, momentum, use_nesterov): optimizer = tf.optimizers.SGD(learning_rate=learning_rate, momentum=momentum, nesterov=use_nesterov) return lambda g, p: optimizer.apply_gradients(zip(g, p)) def _make_snt(self, learning_rate, momentum, use_nesterov): optimizer = momentum_lib.Momentum(learning_rate=learning_rate, momentum=momentum, use_nesterov=use_nesterov) return optimizer.apply @test_utils.combined_named_parameters(CONFIGS) def testComparingSonnetAndTensorFlow(self, config): seed = config.pop("seed") self.assertParametersRemainClose(seed, config) class MomentumTest(optimizer_tests.OptimizerTestBase): def make_optimizer(self, **kwargs): if "learning_rate" not in kwargs: kwargs["learning_rate"] = 0.1 if "momentum" not in kwargs: kwargs["momentum"] = 0.9 return momentum_lib.Momentum(**kwargs) def testDense(self): parameters = [tf.Variable([1., 2.]), tf.Variable([3., 4.])] updates = [tf.constant([5., 5.]), tf.constant([3., 3.])] optimizer = self.make_optimizer(learning_rate=0.1, momentum=0.9) # Step 1 of Momentum optimizer.apply(updates, parameters) self.assertAllClose([[0.5, 1.5], [2.7, 3.7]], [x.numpy() for x in parameters]) # Step 2 of Momentum optimizer.apply(updates, parameters) self.assertAllClose([[-0.45, 0.55], [2.13, 3.13]], [x.numpy() for x in parameters]) # Step 3 of Momentum optimizer.apply(updates, parameters) self.assertAllClose([[-1.805, -0.805], [1.317, 2.317]], [x.numpy() for x in parameters]) def testDenseNesterov(self): parameters = [tf.Variable([1., 2.]), tf.Variable([3., 4.])] updates = [tf.constant([5., 5.]), tf.constant([3., 3.])] optimizer = self.make_optimizer( learning_rate=0.1, momentum=0.9, use_nesterov=True) # Step 1 of Momentum optimizer.apply(updates, parameters) self.assertAllClose([[0.05, 1.05], [2.43, 3.43]], [x.numpy() for x in parameters]) # Step 2 of Momentum optimizer.apply(updates, parameters) self.assertAllClose([[-1.305, -0.305], [1.617, 2.617]], [x.numpy() for x in parameters]) # Step 3 of Momentum optimizer.apply(updates, parameters) self.assertAllClose([[-3.0245, -2.0245], [0.5853, 1.5853]], [x.numpy() for x in parameters]) def testSparse(self): if self.primary_device in ("GPU", "TPU"): self.skipTest("IndexedSlices not supported on {}.".format( self.primary_device)) parameters = [tf.Variable([[1.], [2.]]), tf.Variable([[3.], [4.]])] updates = [ tf.IndexedSlices( tf.constant([0.1], shape=[1, 1]), tf.constant([0]), tf.constant([2, 1])), tf.IndexedSlices( tf.constant([0.01], shape=[1, 1]), tf.constant([1]), tf.constant([2, 1])) ] optimizer = self.make_optimizer(learning_rate=3., momentum=0.9) # Step 1 of Momentum optimizer.apply(updates, parameters) self.assertAllClose([[1.0 - 3.0 * 0.1], [2.0]], parameters[0].numpy()) self.assertAllClose([[3.0], [4.0 - 3.0 * 0.01]], parameters[1].numpy()) # Step 2 of Momentum optimizer.apply(updates, parameters) self.assertAllClose([[0.7 - 3.0 * 0.19], [2.0]], parameters[0].numpy()) self.assertAllClose([[3.0], [3.97 - 3.0 * 0.019]], parameters[1].numpy()) # Step 3 of Momentum optimizer.apply(updates, parameters) self.assertAllClose([[0.13 - 3.0 * 0.271], [2.0]], parameters[0].numpy()) self.assertAllClose([[3.0], [3.913 - 3.0 * 0.0271]], parameters[1].numpy()) def testSparseNesterov(self): if self.primary_device in ("GPU", "TPU"): self.skipTest("IndexedSlices not supported on {}.".format( self.primary_device)) parameters = [tf.Variable([[1.], [2.]]), tf.Variable([[3.], [4.]])] updates = [ tf.IndexedSlices( tf.constant([0.1], shape=[1, 1]), tf.constant([0]), tf.constant([2, 1])), tf.IndexedSlices( tf.constant([0.01], shape=[1, 1]), tf.constant([1]), tf.constant([2, 1])) ] optimizer = self.make_optimizer( learning_rate=3., momentum=0.9, use_nesterov=True) # Step 1 of Momentum optimizer.apply(updates, parameters) self.assertAllClose([[0.43], [2.0]], parameters[0].numpy()) self.assertAllClose([[3.0], [3.943]], parameters[1].numpy()) # Step 2 of Momentum optimizer.apply(updates, parameters) self.assertAllClose([[-0.383], [2.0]], parameters[0].numpy()) self.assertAllClose([[3.0], [3.8617]], parameters[1].numpy()) # Step 3 of Momentum optimizer.apply(updates, parameters) self.assertAllClose([[-1.4147], [2.0]], parameters[0].numpy()) self.assertAllClose([[3.0], [3.75853]], parameters[1].numpy()) def testVariableHyperParams(self): parameters = [tf.Variable([1., 2.]), tf.Variable([3., 4.])] updates = [tf.constant([5., 5.]), tf.constant([3., 3.])] learning_rate = tf.Variable(0.1) momentum_coeff = tf.Variable(0.9) optimizer = self.make_optimizer( learning_rate=learning_rate, momentum=momentum_coeff) if optimizer_tests.is_tf_optimizer(optimizer): self.skipTest("TF SGD optimizer doesn't support variable learning rate.") optimizer.apply(updates, parameters) self.assertAllClose([[0.5, 1.5], [2.7, 3.7]], [x.numpy() for x in parameters]) learning_rate.assign(0.01) momentum_coeff.assign(0.09) self.assertAlmostEqual(0.01, optimizer.learning_rate.numpy()) self.assertAlmostEqual(0.09, optimizer.momentum.numpy()) optimizer.apply(updates, parameters) self.assertAllClose([[0.4455, 1.4455], [2.6673, 3.6673]], [x.numpy() for x in parameters]) def testHyperParamDTypeConversion(self): parameters = [tf.Variable([1., 2.]), tf.Variable([3., 4.])] updates = [tf.constant([5., 5.]), tf.constant([3., 3.])] dtype = tf.float32 if self.primary_device == "TPU" else tf.float64 learning_rate = tf.Variable(0.1, dtype=dtype) momentum_coeff = tf.Variable(0.9, dtype=dtype) optimizer = self.make_optimizer( learning_rate=learning_rate, momentum=momentum_coeff) optimizer.apply(updates, parameters) self.assertAllClose([[0.5, 1.5], [2.7, 3.7]], [x.numpy() for x in parameters]) def testAuxVariablesColocatedWithOriginal(self): optimizer = self.make_optimizer(learning_rate=0.1, momentum=0.9) if optimizer_tests.is_tf_optimizer(optimizer): self.skipTest("TF slot variables are in a different location.") with tf.device("CPU:0"): var = tf.Variable(1.0) optimizer.apply([tf.constant(0.1)], [var]) self.assertEqual(optimizer.accumulated_momentum[0].device, var.device) class ReferenceMomentumTest(MomentumTest): def make_optimizer(self, **kwargs): if "learning_rate" not in kwargs: kwargs["learning_rate"] = 0.1 if "momentum" not in kwargs: kwargs["momentum"] = 0.9 if "use_nesterov" in kwargs: kwargs["nesterov"] = kwargs["use_nesterov"] del kwargs["use_nesterov"] if hasattr(tf.keras.optimizers, "legacy"): return optimizer_tests.WrappedTFOptimizer( tf.keras.optimizers.legacy.SGD(**kwargs)) return optimizer_tests.WrappedTFOptimizer(tf.keras.optimizers.SGD(**kwargs)) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/optimizers/optimizer_tests.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Common tests for Sonnet optimizers.""" import itertools from absl.testing import parameterized import numpy as np from sonnet.src import base from sonnet.src import test_utils import tensorflow as tf import tree class WrappedTFOptimizer(base.Optimizer): """Wraps a TF optimizer in the Sonnet API.""" wrapped = None def __init__(self, optimizer: tf.optimizers.Optimizer): super().__init__() self.wrapped = optimizer def __getattr__(self, name): return getattr(self.wrapped, name) def apply(self, updates, params): self.wrapped.apply_gradients(zip(updates, params)) def is_tf_optimizer(optimizer): return isinstance(optimizer, WrappedTFOptimizer) class OptimizerTestBase(test_utils.TestCase): """Common tests for Sonnet optimizers.""" def make_optimizer(self, *args, **kwargs): raise NotImplementedError() def testNoneUpdate(self): parameters = [tf.Variable(1.), tf.Variable(2.)] updates = [None, tf.constant(3.)] optimizer = self.make_optimizer() optimizer.apply(updates, parameters) self.assertAllClose(1., parameters[0].numpy()) def testDifferentLengthUpdatesParams(self): optimizer = self.make_optimizer() if is_tf_optimizer(optimizer): self.skipTest("TF optimizers don't check the lenghs of params/updates.") parameters = [tf.Variable([1., 2.]), tf.Variable([3., 4.])] updates = [tf.constant([5., 5.])] with self.assertRaisesRegex( ValueError, "`updates` and `parameters` must be the same length."): optimizer.apply(updates, parameters) def testEmptyParams(self): optimizer = self.make_optimizer() if is_tf_optimizer(optimizer): self.skipTest("TF optimizers don't error on empty params.") with self.assertRaisesRegex(ValueError, "`parameters` cannot be empty."): optimizer.apply([], []) def testAllUpdatesNone(self): parameters = [tf.Variable(1.), tf.Variable(2.)] updates = [None, None] optimizer = self.make_optimizer() if is_tf_optimizer(optimizer): msg = "No gradients provided for any variable" else: msg = "No updates provided for any parameter" with self.assertRaisesRegex(ValueError, msg): optimizer.apply(updates, parameters) def testInconsistentDTypes(self): optimizer = self.make_optimizer() if is_tf_optimizer(optimizer): self.skipTest("TF optimizers raise a cryptic error message here.") parameters = [tf.Variable([1., 2.], name="param0")] updates = [tf.constant([5, 5])] with self.assertRaisesRegex( ValueError, "DType of .* is not equal to that of parameter .*param0.*"): optimizer.apply(updates, parameters) def testUnsuppportedStrategyError(self): strategy = tf.distribute.MirroredStrategy() with strategy.scope(): parameters = [tf.Variable(1.0)] updates = [tf.constant(0.1)] optimizer = self.make_optimizer() if is_tf_optimizer(optimizer): self.skipTest("TF optimizers aren't restricted to Sonnet strategies.") with self.assertRaisesRegex( ValueError, "Sonnet optimizers are not compatible with `MirroredStrategy`"): strategy.run(lambda: optimizer.apply(updates, parameters)) # NOTE: Avoiding ABCMeta because of metaclass conflict. class AbstractFuzzTest(test_utils.TestCase, parameterized.TestCase): """Tests TF and Sonnet run concurrently produce equivalent output.""" def _make_tf(self, learning_rate, momentum, use_nesterov): raise NotImplementedError() def _make_snt(self, learning_rate, momentum, use_nesterov): raise NotImplementedError() def assertParametersRemainClose(self, seed, config, num_steps=100, atol=1e-4): tf_opt = self._make_tf(**config) snt_opt = self._make_snt(**config) # TODO(tomhennigan) Add sparse data. data = _generate_dense_data(seed, num_steps) tf_params = _apply_optimizer(data, tf_opt) snt_params = _apply_optimizer(data, snt_opt) assert tf_params and len(tf_params) == len(snt_params) for step, (tf_param, snt_param) in enumerate(zip(tf_params, snt_params)): msg = "TF and Sonnet diverged at step {}".format(step) for tf_p, snt_p in zip(tf_param, snt_param): self.assertEqual(tf_p.shape, snt_p.shape) self.assertEqual(tf_p.dtype, snt_p.dtype) self.assertAllClose(tf_p, snt_p, atol=atol, msg=msg) def _generate_dense_data(seed, num_steps): """Generates deterministic random parameters and gradients.""" # Use numpy random since it is deterministic (unlike TF). np.random.seed(seed=seed) params = [ np.random.normal(size=(10, 10, 10)).astype(np.float32), np.random.normal(size=(10, 10)).astype(np.float32), np.random.normal(size=(10,)).astype(np.float32), ] per_step_grads = [] for _ in range(num_steps): per_step_grads.append([ np.random.normal(size=(10, 10, 10)).astype(np.float32), np.random.normal(size=(10, 10)).astype(np.float32), np.random.normal(size=(10,)).astype(np.float32), ]) return params, per_step_grads def _apply_optimizer(data, apply_fn): params, per_step_grads = data params = [tf.Variable(p, name="rank{}".format(len(p.shape))) for p in params] per_step_grads = tree.map_structure(tf.convert_to_tensor, per_step_grads) param_vals = [] assert per_step_grads for grads in per_step_grads: apply_fn(grads, params) param_vals.append([p.numpy() for p in params]) return param_vals def named_product(**config): keys = list(config.keys()) values = list(config.values()) configs = [] for val in itertools.product(*values): config = dict(zip(keys, val)) name = ",".join("{}={}".format(k, v) for k, v in config.items()) configs.append((name, config)) return configs ================================================ FILE: sonnet/src/optimizers/optimizer_utils.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Utils for Sonnet optimizers.""" from typing import Sequence from sonnet.src import types from sonnet.src.distribute import replicator import tensorflow as tf # Sonnet only supports a subset of distribution strategies since it makes use of # a simplified update model and replica local variables. # TODO(cjfj,petebu,tomhennigan) Add async parameter server strategy when needed. # TODO(cjfj,petebu,tomhennigan) Add sync multi-worker GPU strategy when needed. _SUPPORTED_STRATEGIES = ( tf.distribute.OneDeviceStrategy, replicator.Replicator, replicator.TpuReplicator, ) def check_distribution_strategy(): if tf.distribute.has_strategy(): strategy = tf.distribute.get_strategy() if not isinstance(strategy, _SUPPORTED_STRATEGIES): raise ValueError("Sonnet optimizers are not compatible with `{}`. " "Please use one of `{}` instead.".format( strategy.__class__.__name__, "`, `".join( s.__name__ for s in _SUPPORTED_STRATEGIES))) def check_updates_parameters(updates: Sequence[types.ParameterUpdate], parameters: Sequence[tf.Variable]): if len(updates) != len(parameters): raise ValueError("`updates` and `parameters` must be the same length.") if not parameters: raise ValueError("`parameters` cannot be empty.") if all(x is None for x in updates): raise ValueError("No updates provided for any parameter.") def check_same_dtype(update: types.ParameterUpdate, parameter: tf.Variable): if update.dtype != parameter.dtype: raise ValueError( "DType of update {!r} is not equal to that of parameter {!r}".format( update, parameter)) def deduplicate_indexed_slices(indexed_slice: tf.IndexedSlices): """Sums `values` associated with any non-unique `indices`. Args: indexed_slice: An indexed slice with potentially duplicated indices. Returns: A tuple of (`summed_values`, `unique_indices`) where `unique_indices` is a de-duplicated version of `indices` and `summed_values` contains the sum of `values` slices associated with each unique index. """ values, indices = indexed_slice.values, indexed_slice.indices unique_indices, new_index_positions = tf.unique(indices) summed_values = tf.math.unsorted_segment_sum(values, new_index_positions, tf.shape(unique_indices)[0]) return summed_values, unique_indices ================================================ FILE: sonnet/src/optimizers/rmsprop.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """RMSProp module.""" import itertools from typing import Optional, Sequence, Union from sonnet.src import base from sonnet.src import once from sonnet.src import types from sonnet.src import utils from sonnet.src.optimizers import optimizer_utils import tensorflow as tf def rmsprop_update(update, decay, learning_rate, epsilon, mu, mom, ms, mg): """Computes a single RMSProp update.""" ms = tf.square(update) * (1. - decay) + ms * decay if mg is not None: # centered mg = update * (1. - decay) + mg * decay denominator = ms - tf.square(mg) + epsilon else: denominator = ms + epsilon mom = (mu * mom) + (learning_rate * update * tf.math.rsqrt(denominator)) return mom, ms, mg class RMSProp(base.Optimizer): """RMSProp module. See: http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf Maintain a moving (discounted) average of the square of updates. Divides each update by the root of this average. ms <- decay * ms + (1-decay) * update^2 mom <- momentum * mom + learning_rate * update / sqrt(ms + epsilon) parameter <- parameter - mom This implementation of `RMSprop` uses plain momentum, not Nesterov momentum. The centered version additionally maintains a moving average of the gradients, and uses that average to estimate the variance: mg <- decay * mg + (1-decay) * update ms <- decay * ms + (1-decay) * update^2 mom <- momentum * mom + learning_rate * update / sqrt(ms - mg^2 + epsilon) parameter <- parameter - mom Attributes: learning_rate: Learning rate. decay: Learning rate decay over each update. momentum: Momentum scalar. epsilon: Small value to avoid zero denominator. centered: `True` if centered. mom: Accumulated mom for each parameter. ms: Accumulated ms for each parameter. mg: Accumulated mg for each parameter. """ def __init__(self, learning_rate: Union[types.FloatLike, tf.Variable], decay: Union[types.FloatLike, tf.Variable] = 0.9, momentum: Union[types.FloatLike, tf.Variable] = 0.0, epsilon: Union[types.FloatLike, tf.Variable] = 1e-10, centered: bool = False, name: Optional[str] = None): """Constructs an `RMSProp` module. Args: learning_rate: Learning rate. decay: Learning rate decay over each update. momentum: Momentum scalar. epsilon: Small value to avoid zero denominator. centered: If True, gradients are normalized by the estimated variance of the gradient; if False, by the uncentered second moment. Setting this to True may help with training, but is slightly more expensive in terms of computation and memory. Defaults to False. name: Name for this module. """ super().__init__(name) self.learning_rate = learning_rate self.decay = decay self.momentum = momentum self.epsilon = epsilon self.centered = centered self.mom = [] self.ms = [] self.mg = [] @once.once def _initialize(self, parameters: Sequence[tf.Variable]): zero_var = lambda p: utils.variable_like(p, trainable=False) with tf.name_scope("momentum"): self.mom.extend(zero_var(p) for p in parameters) with tf.name_scope("rms"): self.ms.extend(zero_var(p) for p in parameters) if self.centered: with tf.name_scope("mg"): self.mg.extend(zero_var(p) for p in parameters) def apply(self, updates: Sequence[types.ParameterUpdate], parameters: Sequence[tf.Variable]): """Applies updates to parameters. Args: updates: A list of updates to apply to parameters. Updates are often gradients as returned by `tf.GradientTape.gradient`. parameters: A list of parameters. Raises: ValueError: If `updates` and `parameters` are empty, have different lengths, or have inconsistent types. """ optimizer_utils.check_distribution_strategy() optimizer_utils.check_updates_parameters(updates, parameters) self._initialize(parameters) for update, parameter, mom_var, ms_var, mg_var in itertools.zip_longest( updates, parameters, self.mom, self.ms, self.mg): if update is None: continue optimizer_utils.check_same_dtype(update, parameter) learning_rate = tf.cast(self.learning_rate, update.dtype) decay = tf.cast(self.decay, update.dtype) mu = tf.cast(self.momentum, update.dtype) epsilon = tf.cast(self.epsilon, update.dtype) if isinstance(update, tf.IndexedSlices): # Sparse read our state. update, indices = optimizer_utils.deduplicate_indexed_slices(update) ms = ms_var.sparse_read(indices) mg = mg_var.sparse_read(indices) if self.centered else None mom = mom_var.sparse_read(indices) # Compute and apply a sparse update to our parameter and state. mom, ms, mg = rmsprop_update(update, decay, learning_rate, epsilon, mu, mom, ms, mg) parameter.scatter_sub(tf.IndexedSlices(mom, indices)) mom_var.scatter_update(tf.IndexedSlices(mom, indices)) ms_var.scatter_update(tf.IndexedSlices(ms, indices)) if self.centered: mg_var.scatter_update(tf.IndexedSlices(mg, indices)) else: # Compute and apply a dense update to our parameters and state. mom, ms, mg = rmsprop_update(update, decay, learning_rate, epsilon, mu, mom_var, ms_var, mg_var) parameter.assign_sub(mom) mom_var.assign(mom) ms_var.assign(ms) if self.centered: mg_var.assign(mg) ================================================ FILE: sonnet/src/optimizers/rmsprop_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.rmsprop.""" from sonnet.src import test_utils from sonnet.src.optimizers import optimizer_tests from sonnet.src.optimizers import rmsprop import tensorflow as tf CONFIGS = optimizer_tests.named_product(learning_rate=(0.01, 0.001), decay=(0.8, 0.9), momentum=(0.0, 0.5), epsilon=(1e-7, 1e-8), centered=(False, True), seed=(28, 2, 90)) class ComparisonTest(optimizer_tests.AbstractFuzzTest): """Ensures Sonnet optimizers have equivalent behavior to TensorFlow.""" def _make_tf(self, learning_rate, decay, momentum, epsilon, centered): optimizer = tf.optimizers.RMSprop(learning_rate=learning_rate, rho=decay, momentum=momentum, epsilon=epsilon, centered=centered) return lambda g, p: optimizer.apply_gradients(zip(g, p)) def _make_snt(self, learning_rate, decay, momentum, epsilon, centered): optimizer = rmsprop.RMSProp(learning_rate=learning_rate, decay=decay, momentum=momentum, epsilon=epsilon, centered=centered) return optimizer.apply @test_utils.combined_named_parameters(CONFIGS) def testComparingSonnetAndTensorFlow(self, config): seed = config.pop("seed") self.assertParametersRemainClose(seed, config, atol=1e-2) class RMSPropTest(optimizer_tests.OptimizerTestBase): def make_optimizer(self, **kwargs): if "learning_rate" not in kwargs: kwargs["learning_rate"] = 0.1 return rmsprop.RMSProp(**kwargs) def testDense(self): parameters = [tf.Variable([1., 2.]), tf.Variable([3., 4.])] updates = [tf.constant([5., 5.]), tf.constant([3., 3.])] optimizer = self.make_optimizer(learning_rate=0.1) # Step 1 of RMSProp optimizer.apply(updates, parameters) self.assertAllClose([[0.683772, 1.683772], [2.683772, 3.683772]], [x.numpy() for x in parameters]) # Step 2 of RMSProp optimizer.apply(updates, parameters) self.assertAllClose([[0.454357, 1.454357], [2.454357, 3.454357]], [x.numpy() for x in parameters]) # Step 3 of RMSProp optimizer.apply(updates, parameters) self.assertAllClose([[0.262262, 1.262262], [2.262262, 3.262262]], [x.numpy() for x in parameters]) def testDenseCentered(self): parameters = [tf.Variable([1., 2.]), tf.Variable([3., 4.])] updates = [tf.constant([5., 5.]), tf.constant([3., 3.])] optimizer = self.make_optimizer(learning_rate=0.1, centered=True) # Step 1 of RMSProp optimizer.apply(updates, parameters) self.assertAllClose([[0.666667, 1.666667], [2.666667, 3.666667]], [x.numpy() for x in parameters]) # Step 2 of RMSProp optimizer.apply(updates, parameters) self.assertAllClose([[0.41176, 1.41176], [2.41176, 3.41176]], [x.numpy() for x in parameters]) # Step 3 of RMSProp optimizer.apply(updates, parameters) self.assertAllClose([[0.186776, 1.186776], [2.186776, 3.186776]], [x.numpy() for x in parameters]) def testSparse(self): if self.primary_device in ("GPU", "TPU"): self.skipTest("IndexedSlices not supported on {}.".format( self.primary_device)) parameters = [tf.Variable([[1.], [2.]]), tf.Variable([[3.], [4.]])] updates = [ tf.IndexedSlices( tf.constant([0.1], shape=[1, 1]), tf.constant([0]), tf.constant([2, 1])), tf.IndexedSlices( tf.constant([0.01], shape=[1, 1]), tf.constant([1]), tf.constant([2, 1])) ] optimizer = self.make_optimizer(learning_rate=3.) # Step 1 of RMSProp optimizer.apply(updates, parameters) self.assertAllClose([[-8.486831], [2.0]], parameters[0].numpy(), rtol=1e-4) self.assertAllClose([[3.0], [-5.486784]], parameters[1].numpy(), rtol=1e-4) # Step 2 of RMSProp optimizer.apply(updates, parameters) self.assertAllClose([[-15.369301], [2.0]], parameters[0].numpy(), rtol=1e-4) self.assertAllClose([[3.0], [-12.369237]], parameters[1].numpy(), rtol=1e-4) # Step 3 of RMSProp optimizer.apply(updates, parameters) self.assertAllClose([[-21.132141], [2.0]], parameters[0].numpy(), rtol=1e-4) self.assertAllClose([[3.0], [-18.132067]], parameters[1].numpy(), rtol=1e-4) def testSparseCentered(self): if self.primary_device in ("GPU", "TPU"): self.skipTest("IndexedSlices not supported on {}.".format( self.primary_device)) parameters = [tf.Variable([[1.], [2.]]), tf.Variable([[3.], [4.]])] updates = [ tf.IndexedSlices( tf.constant([0.1], shape=[1, 1]), tf.constant([0]), tf.constant([2, 1])), tf.IndexedSlices( tf.constant([0.01], shape=[1, 1]), tf.constant([1]), tf.constant([2, 1])) ] optimizer = self.make_optimizer(learning_rate=3., centered=True) # Step 1 of RMSProp optimizer.apply(updates, parameters) self.assertAllClose([[-8.999999], [2.0]], parameters[0].numpy(), rtol=1e-4) self.assertAllClose([[3.0], [-5.999944]], parameters[1].numpy(), rtol=1e-4) # Step 2 of RMSProp optimizer.apply(updates, parameters) self.assertAllClose([[-16.64719], [2.0]], parameters[0].numpy(), rtol=1e-4) self.assertAllClose([[3.0], [-13.647109]], parameters[1].numpy(), rtol=1e-4) # Step 3 of RMSProp optimizer.apply(updates, parameters) self.assertAllClose([[-23.396709], [2.0]], parameters[0].numpy(), rtol=1e-4) self.assertAllClose([[3.0], [-20.39661]], parameters[1].numpy(), rtol=1e-4) def testVariableHyperParams(self): parameters = [tf.Variable([1., 2.]), tf.Variable([3., 4.])] updates = [tf.constant([5., 5.]), tf.constant([3., 3.])] learning_rate = tf.Variable(0.1) optimizer = self.make_optimizer(learning_rate=learning_rate) optimizer.apply(updates, parameters) self.assertAllClose([[0.683772, 1.683772], [2.683772, 3.683772]], [x.numpy() for x in parameters]) learning_rate.assign(0.01) self.assertAlmostEqual(0.01, optimizer.learning_rate.numpy()) optimizer.apply(updates, parameters) self.assertAllClose([[0.660831, 1.660831], [2.660831, 3.660831]], [x.numpy() for x in parameters]) def testHyperParamDTypeConversion(self): parameters = [tf.Variable([1., 2.]), tf.Variable([3., 4.])] updates = [tf.constant([5., 5.]), tf.constant([3., 3.])] dtype = tf.float32 if self.primary_device == "TPU" else tf.float64 learning_rate = tf.Variable(0.1, dtype=dtype) decay = tf.Variable(0.9, dtype=dtype) momentum = tf.Variable(0.0, dtype=dtype) epsilon = tf.Variable(1e-7, dtype=dtype) optimizer = self.make_optimizer( learning_rate=learning_rate, decay=decay, momentum=momentum, epsilon=epsilon) if optimizer_tests.is_tf_optimizer(optimizer): self.skipTest("TF optimizers don't support automatic casting.") optimizer.apply(updates, parameters) self.assertAllClose([[0.683772, 1.683772], [2.683772, 3.683772]], [x.numpy() for x in parameters]) def testAuxVariablesColocatedWithOriginal(self): optimizer = self.make_optimizer(learning_rate=0.1) if optimizer_tests.is_tf_optimizer(optimizer): self.skipTest("Aux vars are in a different location for TF optimizers.") with tf.device("CPU:0"): var = tf.Variable(1.0) optimizer.apply([tf.constant(0.1)], [var]) self.assertEqual(optimizer.mom[0].device, var.device) self.assertEqual(optimizer.ms[0].device, var.device) class ReferenceRMSPropTest(RMSPropTest): def make_optimizer(self, **kwargs): if "learning_rate" not in kwargs: kwargs["learning_rate"] = 0.1 kwargs["rho"] = kwargs.pop("decay", 0.9) if hasattr(tf.keras.optimizers, "legacy"): return optimizer_tests.WrappedTFOptimizer( tf.keras.optimizers.legacy.RMSprop(**kwargs)) return optimizer_tests.WrappedTFOptimizer( tf.keras.optimizers.RMSprop(**kwargs)) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/optimizers/sgd.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Stochastic Gradient Descent module.""" from typing import Optional, Sequence, Union from sonnet.src import base from sonnet.src import types from sonnet.src.optimizers import optimizer_utils import tensorflow as tf class SGD(base.Optimizer): """Stochastic Gradient Descent (SGD) module. Attributes: learning_rate: Learning rate. """ def __init__(self, learning_rate: Union[types.FloatLike, tf.Variable], name: Optional[str] = None): """Constructs an `SGD` module. Args: learning_rate: Learning rate. name: Name of the module. """ super().__init__(name) self.learning_rate = learning_rate def apply(self, updates: Sequence[types.ParameterUpdate], parameters: Sequence[tf.Variable]): """Applies updates to parameters. Args: updates: A list of updates to apply to parameters. Updates are often gradients as returned by `tf.GradientTape.gradient`. parameters: A list of parameters. Raises: ValueError: If `updates` and `parameters` are empty, have different lengths, or have inconsistent types. """ optimizer_utils.check_distribution_strategy() optimizer_utils.check_updates_parameters(updates, parameters) for update, parameter in zip(updates, parameters): if update is not None: optimizer_utils.check_same_dtype(update, parameter) learning_rate = tf.cast(self.learning_rate, update.dtype) if isinstance(update, tf.IndexedSlices): parameter.scatter_sub( tf.IndexedSlices(update.values * learning_rate, update.indices)) else: parameter.assign_sub(update * learning_rate) ================================================ FILE: sonnet/src/optimizers/sgd_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.sgd.""" from sonnet.src.optimizers import optimizer_tests from sonnet.src.optimizers import sgd import tensorflow as tf class SGDTest(optimizer_tests.OptimizerTestBase): def make_optimizer(self, *args, **kwargs): if "learning_rate" not in kwargs: kwargs["learning_rate"] = 3. return sgd.SGD(*args, **kwargs) def testDense(self): parameters = [tf.Variable([1., 2.]), tf.Variable([3., 4.])] updates = [tf.constant([5., 5.]), tf.constant([3., 3.])] optimizer = self.make_optimizer(learning_rate=3.) optimizer.apply(updates, parameters) self.assertAllClose([[-14., -13.], [-6., -5.]], [x.numpy() for x in parameters]) def testSparse(self): if self.primary_device == "TPU": self.skipTest("IndexedSlices not supported on TPU.") parameters = [tf.Variable([[1.], [2.]]), tf.Variable([[3.], [4.]])] updates = [ tf.IndexedSlices( tf.constant([0.1], shape=[1, 1]), tf.constant([0]), tf.constant([2, 1])), tf.IndexedSlices( tf.constant([0.01], shape=[1, 1]), tf.constant([1]), tf.constant([2, 1])) ] optimizer = self.make_optimizer(learning_rate=3.) optimizer.apply(updates, parameters) self.assertAllClose([[1.0 - 3.0 * 0.1], [2.0]], parameters[0].numpy()) self.assertAllClose([[3.0], [4.0 - 3.0 * 0.01]], parameters[1].numpy()) def testVariableLearningRate(self): parameters = [tf.Variable([1., 2.]), tf.Variable([3., 4.])] updates = [tf.constant([5., 5.]), tf.constant([3., 3.])] learning_rate = tf.Variable(3.) optimizer = self.make_optimizer(learning_rate=learning_rate) optimizer.apply(updates, parameters) self.assertAllClose([[-14., -13.], [-6., -5.]], [x.numpy() for x in parameters]) learning_rate.assign_sub(1.) self.assertEqual(2., optimizer.learning_rate.numpy()) optimizer.apply(updates, parameters) self.assertAllClose([[-24., -23.], [-12., -11.]], [x.numpy() for x in parameters]) def testLearningRateDTypeConversion(self): parameters = [tf.Variable([1., 2.]), tf.Variable([3., 4.])] updates = [tf.constant([5., 5.]), tf.constant([3., 3.])] dtype = tf.int32 if self.primary_device == "TPU" else tf.int64 learning_rate = tf.Variable(3, dtype=dtype) optimizer = self.make_optimizer(learning_rate=learning_rate) optimizer.apply(updates, parameters) self.assertAllClose([[-14., -13.], [-6., -5.]], [x.numpy() for x in parameters]) class ReferenceSGDTest(SGDTest): def make_optimizer(self, *args, **kwargs): if "learning_rate" not in kwargs: kwargs["learning_rate"] = 3. if hasattr(tf.keras.optimizers, "legacy"): return optimizer_tests.WrappedTFOptimizer( tf.keras.optimizers.legacy.SGD(**kwargs)) return optimizer_tests.WrappedTFOptimizer(tf.keras.optimizers.SGD(**kwargs)) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/pad.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Padding module for Sonnet.""" from typing import Callable, Sequence, Union from sonnet.src import utils Padding = Callable[[int], Sequence[int]] Paddings = Union[Padding, Sequence[Padding]] def valid(effective_kernel_size: int): # pylint: disable=unused-argument """No padding.""" return [0, 0] def same(effective_kernel_size: int): """Pads such that the output size matches input size for stride=1.""" return [(effective_kernel_size - 1) // 2, effective_kernel_size // 2] def full(effective_kernel_size: int): """Maximal padding whilst not convolving over just padded elements.""" return [effective_kernel_size - 1, effective_kernel_size - 1] def causal(effective_kernel_size: int): """Pre-padding such that output has no dependence on the future.""" return [effective_kernel_size - 1, 0] def reverse_causal(effective_kernel_size: int): """Post-padding such that output has no dependence on the past.""" return [0, effective_kernel_size - 1] def create( padding: Paddings, kernel: Union[int, Sequence[int]], rate: Union[int, Sequence[int]], n: int, channel_index: int, ): """Generates the padding required for a given padding algorithm. Args: padding: callable or list of callables of length n. The callables take an integer representing the effective kernel size (kernel size when the rate is 1) and return a list of two integers representing the padding before and padding after for that dimension. kernel: int or list of ints of length n. The size of the kernel for each dimension. If it is an int it will be replicated for the non channel and batch dimensions. rate: int or list of ints of length n. The dilation rate for each dimension. If it is an int it will be replicated for the non channel and batch dimensions. n: the number of spatial dimensions. channel_index: the channel position of the input to which the padding will be applied. Returns: A list of length n+2 containing the padding for each element. These are of the form [pad_before, pad_after]. """ # The effective kernel size includes any holes/gaps introduced by the # dilation rate. It's equal to kernel_size when rate == 1. effective_kernel_size = map( lambda kernel, rate: (kernel - 1) * rate + 1, utils.replicate(kernel, n, "kernel"), utils.replicate(rate, n, "rate")) paddings = map( lambda x, y: x(y), utils.replicate(padding, n, "padding"), effective_kernel_size) if channel_index == 1: # N, C, ... paddings = [[0, 0], [0, 0]] + list(paddings) else: # channel_index == -1 N, ..., C paddings = [[0, 0]] + list(paddings) + [[0, 0]] return paddings ================================================ FILE: sonnet/src/pad_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.pad.""" from absl.testing import parameterized from sonnet.src import pad from sonnet.src import test_utils import tensorflow as tf class PadTest(test_utils.TestCase, parameterized.TestCase): def test_padding_2d(self): a = pad.create([pad.causal, pad.full], [3], [1, 1], 2, -1) self.assertEqual(a, [[0, 0], [2, 0], [2, 2], [0, 0]]) def test_padding_1d(self): a = pad.create(pad.full, 3, 1, 1, 1) self.assertEqual(a, [[0, 0], [0, 0], [2, 2]]) def test_padding_3d(self): a = pad.create([pad.causal, pad.full, pad.full], [3, 2, 3], [1], 3, -1) self.assertEqual(a, [[0, 0], [2, 0], [1, 1], [2, 2], [0, 0]]) @parameterized.parameters((2, [2, 2]), (3, [4, 4, 4, 4]), ([2, 2], 3), ([4, 4, 4, 4], 3)) def test_padding_incorrect_input(self, kernel_size, rate): with self.assertRaisesRegex( TypeError, r"must be a scalar or sequence of length 1 or sequence of length 3."): pad.create(pad.full, kernel_size, rate, 3, -1) def test_padding_valid(self): a = pad.create(pad.valid, 4, 3, 2, -1) self.assertEqual(a, [[0, 0], [0, 0], [0, 0], [0, 0]]) def test_padding_same(self): a = pad.create(pad.same, 4, 3, 2, -1) self.assertEqual(a, [[0, 0], [4, 5], [4, 5], [0, 0]]) def test_padding_full(self): a = pad.create(pad.full, 4, 3, 2, -1) self.assertEqual(a, [[0, 0], [9, 9], [9, 9], [0, 0]]) def test_padding_causal(self): a = pad.create(pad.causal, 4, 3, 2, -1) self.assertEqual(a, [[0, 0], [9, 0], [9, 0], [0, 0]]) def test_padding_reverse_causal(self): a = pad.create(pad.reverse_causal, 4, 3, 2, -1) self.assertEqual(a, [[0, 0], [0, 9], [0, 9], [0, 0]]) @parameterized.parameters((1, 1, 1), (3, 1, 1), (1, 3, 1), (1, 1, 3), (3, 3, 1), (3, 1, 3), (1, 3, 3), (3, 3, 3)) def test_same_padding(self, kernel_size, stride, rate): a = tf.random.normal([2, 4, 3]) k = tf.random.normal([kernel_size, 3, 4]) padding = pad.create(pad.same, kernel_size, rate, 1, -1) a_padded = tf.pad(a, padding) y1 = tf.nn.conv1d( a_padded, k, stride=stride, dilations=rate, padding="VALID") y2 = tf.nn.conv1d(a, k, stride=stride, dilations=rate, padding="SAME") self.assertEqual(y1.shape, y2.shape) self.assertAllClose(y1.numpy(), y2.numpy()) @parameterized.parameters((1, 1, 1), (3, 1, 1), (1, 3, 1), (1, 1, 3), (3, 3, 1), (3, 1, 3), (1, 3, 3), (3, 3, 3)) def test_valid_padding(self, kernel_size, stride, rate): a = tf.random.normal([2, 8, 3]) k = tf.random.normal([kernel_size, 3, 4]) padding = pad.create(pad.valid, kernel_size, rate, 1, -1) a_padded = tf.pad(a, padding) y1 = tf.nn.conv1d( a_padded, k, stride=stride, dilations=rate, padding="VALID") y2 = tf.nn.conv1d(a, k, stride=stride, dilations=rate, padding="VALID") self.assertAllEqual(y1.numpy(), y2.numpy()) if __name__ == "__main__": tf.test.main() ================================================ FILE: sonnet/src/parallel_linear.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Parallel linear module.""" import math from typing import Optional from sonnet.src import base from sonnet.src import initializers from sonnet.src import once from sonnet.src import utils import tensorflow as tf class ParallelLinears(base.Module): """Parallel linear. This is equivalent to n separate linears applied in parallel to n inputs. It takes an input of shape [num_linears, batch_size, input_size] and returns an output of shape [num_linears, batch_size, output_size]. It uses a single batched matmul which is more efficient than stacking separate snt.Linear layers. This is implemented using `num_linear`s first to avoid the need for transposes in order to make it efficient when stacking these. """ def __init__(self, output_size: int, with_bias: bool = True, w_init: Optional[initializers.Initializer] = None, b_init: Optional[initializers.Initializer] = None, name: Optional[str] = None): """Constructs a `ParallelLinear` module. Args: output_size: Output dimensionality. with_bias: Whether to include bias parameters. Default `True`. w_init: Optional initializer for the weights. By default the weights are initialized truncated random normal values with a standard deviation of `1 / sqrt(input_feature_size)`, which is commonly used when the inputs are zero centered (see https://arxiv.org/abs/1502.03167v3). b_init: Optional initializer for the bias. By default the bias is initialized to zero. name: Name of the module. """ super().__init__(name=name) self.output_size = output_size self.with_bias = with_bias self.w_init = w_init if with_bias: self.b_init = b_init if b_init is not None else initializers.Zeros() elif b_init is not None: raise ValueError("When not using a bias the b_init must be None.") @once.once def _initialize(self, inputs: tf.Tensor): """Constructs parameters used by this module.""" utils.assert_rank(inputs, 3) self.input_size = inputs.shape[2] if self.input_size is None: # Can happen inside an @tf.function. raise ValueError("Input size must be specified at module build time.") num_linears = inputs.shape[0] if num_linears is None: # Can happen inside an @tf.function. raise ValueError( "The number of linears must be specified at module build time.") if self.w_init is None: # See https://arxiv.org/abs/1502.03167v3. stddev = 1. / math.sqrt(self.input_size) self.w_init = initializers.TruncatedNormal(stddev=stddev) self.w = tf.Variable( self.w_init([num_linears, self.input_size, self.output_size], inputs.dtype), name="w") if self.with_bias: self.b = tf.Variable( self.b_init([num_linears, 1, self.output_size], inputs.dtype), name="b") def __call__(self, inputs: tf.Tensor) -> tf.Tensor: self._initialize(inputs) outputs = tf.matmul(inputs, self.w) if self.with_bias: outputs = tf.add(outputs, self.b) return outputs ================================================ FILE: sonnet/src/parallel_linear_test.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Tests for sonnet.v2.src.parallel_linear.""" from sonnet.src import linear from sonnet.src import parallel_linear from sonnet.src import test_utils import tensorflow as tf class ParallelLinearTest(test_utils.TestCase): def test_output_size_correct(self): layer = parallel_linear.ParallelLinears(3) outputs = layer(tf.ones([4, 2, 6])) self.assertEqual(outputs.shape, [4, 2, 3]) def test_behaves_same_as_stacked_linears(self): w_init = tf.random.normal((3, 5, 7)) b_init = tf.random.normal((3, 1, 7)) inputs = tf.random.normal((3, 2, 5)) parallel = parallel_linear.ParallelLinears( 7, w_init=lambda s, d: w_init, b_init=lambda s, d: b_init) parallel_outputs = parallel(inputs) stacked_outputs = [] for i in range(3): layer = linear.Linear( 7, w_init=lambda s, d, i=i: w_init[i], b_init=lambda s, d, i=i: b_init[i]) stacked_outputs.append(layer(inputs[i])) stacked_outputs = tf.stack(stacked_outputs, axis=0) self.assertAllClose(parallel_outputs.numpy(), stacked_outputs.numpy()) if __name__ == '__main__': tf.test.main() ================================================ FILE: sonnet/src/recurrent.py ================================================ # Copyright 2019 The Sonnet Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Recurrent Neural Network cores.""" import abc import collections import functools from typing import Optional, Sequence, Tuple, Union import uuid from sonnet.src import base from sonnet.src import conv from sonnet.src import initializers from sonnet.src import linear from sonnet.src import once from sonnet.src import types from sonnet.src import utils import tensorflow.compat.v1 as tf1 import tensorflow as tf import tree # pylint: disable=g-direct-tensorflow-import # Required for specializing `UnrolledLSTM` per device. from tensorflow.python.eager import context as context_lib # pylint: enable=g-direct-tensorflow-import class RNNCore(base.Module, metaclass=abc.ABCMeta): """Base class for Recurrent Neural Network cores. This class defines the basic functionality that every core should implement: :meth:`initial_state`, used to construct an example of the core state; and :meth:`__call__` which applies the core parameterized by a previous state to an input. Cores are typically used with :func:`dynamic_unroll` and :func:`static_unroll` to iteratively construct an output sequence from the given input sequence. """ @abc.abstractmethod def __call__(self, inputs: types.TensorNest, prev_state): """Performs one step of an RNN. Args: inputs: An arbitrarily nested structure of shape [B, ...] where B is the batch size. prev_state: Previous core state. Returns: A tuple with two elements: * **outputs** - An arbitrarily nested structure of shape [B, ...]. Dimensions following the batch size could be different from that of `inputs`. * **next_state** - Next core state, must be of the same shape as the previous one. """ @abc.abstractmethod def initial_state(self, batch_size: types.IntegerLike, **kwargs): """Constructs an initial state for this core. Args: batch_size: An int or an integral scalar tensor representing batch size. **kwargs: Optional keyword arguments. Returns: Arbitrarily nested initial state for this core. """ class UnrolledRNN(base.Module, metaclass=abc.ABCMeta): """Base class for unrolled Recurrent Neural Networks. This class is a generalization of :class:`RNNCore` which operates on an input sequence as opposed to a single time step. """ @abc.abstractmethod def __call__(self, input_sequence: types.TensorNest, initial_state: types.TensorNest): """Apply this RNN to the input sequence. Args: input_sequence: An arbitrarily nested structure of shape ``[T, B, ...]`` where ``T`` is the number of time steps and B is the batch size. initial_state: Initial RNN state. Returns: A tuple with two elements: * **output_sequence** - An arbitrarily nested structure of tensors of shape ``[T, B, ...]``. Dimensions following the batch size could be different from that of the ``input_sequence``. * **final_state** - Final RNN state, must be of the same shape as the initial one. """ @abc.abstractmethod def initial_state(self, batch_size: types.IntegerLike, **kwargs): """Construct an initial state for this RNN. Args: batch_size: An int or an integral scalar tensor representing batch size. **kwargs: Optional keyword arguments. Returns: Arbitrarily nested initial state for this RNN. """ class TrainableState(base.Module): """Trainable state for an :class:`RNNCore`. The state can be constructed manually from a nest of initial values:: >>> state = snt.TrainableState((tf.zeros([16]), tf.zeros([16]))) or automatically for a given :class:`RNNCore`:: >>> core = snt.LSTM(hidden_size=16) >>> state = snt.TrainableState.for_core(core) """ @classmethod def for_core(cls, core: RNNCore, mask: Optional[types.TensorNest] = None, name: Optional[str] = None): """Constructs a trainable state for a given :class:`RNNCore`. Args: core: An :class:`RNNCore` to construct the state for. mask: Optional boolean mask of the same structure as the initial state of `core` specifying which components should be trainable. If not given, the whole state is considered trainable. name: Name of the module. Returns: A `TrainableState`. """ initial_values = tree.map_structure(lambda s: tf.squeeze(s, axis=0), core.initial_state(batch_size=1)) return cls(initial_values, mask, name) def __init__(self, initial_values: types.TensorNest, mask: Optional[types.TensorNest] = None, name: Optional[str] = None): """Constructs a trainable state from initial values. Args: initial_values: Arbitrarily nested initial values for the state. mask: Optional boolean mask of the same structure as ``initial_values`` specifying which components should be trainable. If not given, the whole state is considered trainable. name: Name of the module. """ super().__init__(name) flat_initial_values = tree.flatten_with_path(initial_values) if mask is None: flat_mask = [True] * len(flat_initial_values) else: tree.assert_same_structure(initial_values, mask) flat_mask = tree.flatten(mask) flat_template = [] for (path, initial_value), trainable in zip(flat_initial_values, flat_mask): # `"state"` is only used if initial_values is not nested. name = "/".join(map(str, path)) or "state" flat_template.append( tf.Variable( tf.expand_dims(initial_value, axis=0), trainable=trainable, name=name)) self._template = tree.unflatten_as(initial_values, flat_template) def __call__(self, batch_size: int) -> types.TensorNest: """Returns a trainable state for the given batch size.""" return tree.map_structure( lambda s: tf.tile(s, [batch_size] + [1] * (s.shape.rank - 1)), self._template) def static_unroll( core: RNNCore, input_sequence: types.TensorNest, # time-major. initial_state: types.TensorNest, sequence_length: Optional[types.IntegerLike] = None ) -> Tuple[types.TensorNest, types.TensorNest]: """Performs a static unroll of an RNN. >>> core = snt.LSTM(hidden_size=16) >>> batch_size = 3 >>> input_sequence = tf.random.uniform([1, batch_size, 2]) >>> output_sequence, final_state = snt.static_unroll( ... core, ... input_sequence, ... core.initial_state(batch_size)) An *unroll* corresponds to calling the core on each element of the input sequence in a loop, carrying the state through:: state = initial_state for t in range(len(input_sequence)): outputs, state = core(input_sequence[t], state) A *static* unroll replaces a loop with its body repeated multiple times when executed inside :tf:`function`:: state = initial_state outputs0, state = core(input_sequence[0], state) outputs1, state = core(input_sequence[1], state) outputs2, state = core(input_sequence[2], state) ... See :func:`dynamic_unroll` for a loop-preserving unroll function. Args: core: An :class:`RNNCore` to unroll. input_sequence: An arbitrarily nested structure of tensors of shape ``[T, B, ...]`` where ``T`` is the number of time steps, and ``B`` is the batch size. initial_state: An initial state of the given core. sequence_length: An optional tensor of shape ``[B]`` specifying the lengths of sequences within the (padded) batch. Returns: A tuple with two elements: * **output_sequence** - An arbitrarily nested structure of tensors of shape ``[T, B, ...]``. Dimensions following the batch size could be different from that of the ``input_sequence``. * **final_state** - Core state at time step ``T``. Raises: ValueError: If ``input_sequence`` is empty or its leading dimension is not known statically. """ num_steps, input_tas = _unstack_input_sequence(input_sequence) if not isinstance(num_steps, int): raise ValueError( "input_sequence must have a statically known number of time steps") outputs = None state = initial_state output_accs = None for t in range(num_steps): outputs, state = _rnn_step( core, input_tas, sequence_length, t, prev_outputs=outputs, prev_state=state) if t == 0: output_accs = tree.map_structure(lambda o: _ListWrapper([o]), outputs) else: tree.map_structure(lambda acc, o: acc.data.append(o), output_accs, outputs) output_sequence = tree.map_structure(lambda acc: tf.stack(acc.data), output_accs) return output_sequence, state class _ListWrapper: """A wrapper hiding a list from `nest`. This allows to use `tree.map_structure` without recursing into the wrapped list. """ __slots__ = ["data"] def __init__(self, data): self.data = data # TODO(slebedev): core can be core_fn: Callable[[I, S], Tuple[O, S]]. # TODO(slebedev): explain sequence_length with ASCII art? @utils.smart_autograph def dynamic_unroll( core, input_sequence, # time-major. initial_state, sequence_length=None, parallel_iterations=1, swap_memory=False): """Performs a dynamic unroll of an RNN. >>> core = snt.LSTM(hidden_size=16) >>> batch_size = 3 >>> input_sequence = tf.random.uniform([1, batch_size, 2]) >>> output_sequence, final_state = snt.dynamic_unroll( ... core, ... input_sequence, ... core.initial_state(batch_size)) An *unroll* corresponds to calling the core on each element of the input sequence in a loop, carrying the state through:: state = initial_state for t in range(len(input_sequence)): outputs, state = core(input_sequence[t], state) A *dynamic* unroll preserves the loop structure when executed within :tf:`function`. See :func:`static_unroll` for an unroll function which replaces a loop with its body repeated multiple times. Args: core: An :class:`RNNCore` to unroll. input_sequence: An arbitrarily nested structure of tensors of shape ``[T, B, ...]`` where ``T`` is the number of time steps, and ``B`` is the batch size. initial_state: initial state of the given core. sequence_length: An optional tensor of shape ``[B]`` specifying the lengths of sequences within the (padded) batch. parallel_iterations: An optional ``int`` specifying the number of iterations to run in parallel. Those operations which do not have any temporal dependency and can be run in parallel, will be. This parameter trades off time for space. Values >> 1 use more memory but take less time, while smaller values use less memory but computations take longer. Defaults to 1. swap_memory: Transparently swap the tensors produced in forward inference but needed for back prop from GPU to CPU. This allows training RNNs which would typically not fit on a single GPU, with very minimal (or no) performance penalty. Defaults to False. Returns: A tuple with two elements: * **output_sequence** - An arbitrarily nested structure of tensors of shape ``[T, B, ...]``. Dimensions following the batch size could be different from that of the ``input_sequence``. * **final_state** - Core state at time step ``T``. Raises: ValueError: If ``input_sequence`` is empty. """ num_steps, input_tas = _unstack_input_sequence(input_sequence) # Unroll the first time step separately to infer outputs structure. outputs, state = _rnn_step( core, input_tas, sequence_length, t=0, prev_outputs=None, prev_state=initial_state) output_tas = tree.map_structure( lambda o: tf.TensorArray(o.dtype, num_steps).write(0, o), outputs) # AutoGraph converts a for loop over `tf.range` to `tf.while_loop`. # `maximum_iterations` are needed to backprop through the loop on TPU. for t in tf.range(1, num_steps): tf.autograph.experimental.set_loop_options( parallel_iterations=parallel_iterations, swap_memory=swap_memory, maximum_iterations=num_steps - 1) outputs, state = _rnn_step( core, input_tas, sequence_length, t, prev_outputs=outputs, prev_state=state) output_tas = tree.map_structure( lambda ta, o, _t=t: ta.write(_t, o), output_tas, outputs) output_sequence = tree.map_structure(tf.TensorArray.stack, output_tas) return output_sequence, state def _unstack_input_sequence(input_sequence): r"""Unstacks the input sequence into a nest of :tf:`TensorArray`\ s. This allows to traverse the input sequence using :tf:`TensorArray.read` instead of a slice, avoiding O(sliced tensor) slice gradient computation during the backwards pass. Args: input_sequence: See :func:`dynamic_unroll` or :func:`static_unroll`. Returns: num_steps: Number of steps in the input sequence. input_tas: An arbitrarily nested structure of :tf:`TensorArray`\ s of size ``num_steps``. Raises: ValueError: If tensors in ``input_sequence`` have inconsistent number of steps or the number of steps is 0. """ flat_input_sequence = tree.flatten(input_sequence) all_num_steps = {i.shape[0] for i in flat_input_sequence} if len(all_num_steps) > 1: raise ValueError( "input_sequence tensors must have consistent number of time steps") [num_steps] = all_num_steps if num_steps == 0: raise ValueError("input_sequence must have at least a single time step") elif num_steps is None: # Number of steps is not known statically, fall back to dynamic shape. num_steps = tf.shape(flat_input_sequence[0])[0] # TODO(b/141910613): uncomment when the bug is fixed. # for i in flat_input_sequence[1:]: # tf.debugging.assert_equal( # tf.shape(i)[0], num_steps, # "input_sequence tensors must have consistent number of time steps") input_tas = tree.map_structure( lambda i: tf.TensorArray(i.dtype, num_steps).unstack(i), input_sequence) return num_steps, input_tas def _safe_where(condition, x, y): # pylint: disable=g-doc-args """`tf.where` which allows scalar inputs.""" if x.shape.rank == 0: # This is to match the `tf.nn.*_rnn` behavior. In general, we might # want to branch on `tf.reduce_all(condition)`. return y # TODO(tomhennigan) Broadcasting with SelectV2 is currently broken. return tf1.where(condition, x, y) def _rnn_step(core, input_tas, sequence_length, t, prev_outputs, prev_state): """Performs a single RNN step optionally accounting for variable length.""" outputs, state = core( tree.map_structure(lambda i: i.read(t), input_tas), prev_state) if prev_outputs is None: assert t == 0 prev_outputs = tree.map_structure(tf.zeros_like, outputs) # TODO(slebedev): do not go into this block if t < min_len. if sequence_length is not None: # Selectively propagate outputs/state to the not-yet-finished # sequences. maybe_propagate = functools.partial(_safe_where, t >= sequence_length) outputs = tree.map_structure(maybe_propagate, prev_outputs, outputs) state = tree.map_structure(maybe_propagate, prev_state, state) return outputs, state class VanillaRNN(RNNCore): """Basic fully-connected RNN core. Given :math:`x_t` and the previous hidden state :math:`h_{t-1}` the core computes .. math:: h_t = w_i x_t + w_h h_{t-1} + b Attributes: input_to_hidden: Input-to-hidden weights :math:`w_i`, a tensor of shape ``[hidden_size, hidden_size]``. hidden_to_hidden: Hidden-to-hidden weights :math:`w_i`, a tensor of shape ``[input_size, hidden_size]``. b: bias, a tensor or shape ``[hidden_size]``. """ def __init__(self, hidden_size: int, activation: types.ActivationFn = tf.tanh, w_i_init: Optional[initializers.Initializer] = None, w_h_init: Optional[initializers.Initializer] = None, b_init: Optional[initializers.Initializer] = None, dtype: tf.DType = tf.float32, name: Optional[str] = None): """Constructs a vanilla RNN core. Args: hidden_size: Hidden layer size. activation: Activation function to use. Defaults to ``tf.tanh``. w_i_init: Optional initializer for the input-to-hidden weights. Defaults to :class:`~initializers.TruncatedNormal` with a standard deviation of ``1 / sqrt(input_size)``. w_h_init: Optional initializer for the hidden-to-hidden weights. Defaults to :class:`~initializers.TruncatedNormal` with a standard deviation of ``1 / sqrt(hidden_size)``. b_init: Optional initializer for the bias. Defaults to :class:`~initializers.Zeros`. dtype: Optional :tf:`DType` of the core's variables. Defaults to ``tf.float32``. name: Name of the module. """ super().__init__(name) self._hidden_size = hidden_size self._activation = activation self._b_init = b_init or initializers.Zeros() self._dtype = dtype self._input_to_hidden = linear.Linear( hidden_size, with_bias=False, w_init=w_i_init, name="input_to_hidden") self._hidden_to_hidden = linear.Linear( hidden_size, with_bias=False, w_init=w_h_init, name="hidden_to_hidden") @property def input_to_hidden(self) -> tf.Variable: return self._input_to_hidden.w @property def hidden_to_hidden(self) -> tf.Variable: return self._hidden_to_hidden.w def __call__(self, inputs: types.TensorNest, prev_state: types.TensorNest) -> Tuple[tf.Tensor, tf.Tensor]: """See base class.""" self._initialize(inputs) outputs = self._activation( self._input_to_hidden(inputs) + self._hidden_to_hidden(prev_state) + self._b) # For VanillaRNN, the next state of the RNN is the same as the outputs. return outputs, outputs def initial_state(self, batch_size: int) -> tf.Tensor: """See base class.""" return tf.zeros(shape=[batch_size, self._hidden_size], dtype=self._dtype) @once.once def _initialize(self, inputs: tf.Tensor): dtype = _check_inputs_dtype(inputs, self._dtype) self._b = tf.Variable(self._b_init([self._hidden_size], dtype), name="b") class _LegacyDeepRNN(RNNCore): """Sonnet 1 compatible :class:`DeepRNN` implementation. This class is not intended to be used directly. Refer to :class:`DeepRNN` and ``deep_rnn_with_*_connections``. """ def __init__(self, layers, skip_connections, concat_final_output_if_skip=True, name: Optional[str] = None): r"""Constructs a ``DeepRNN``. Args: layers: A list of :class:`RNNCore`\ s or callables. skip_connections: See :func:`deep_rnn_with_skip_connections`. concat_final_output_if_skip: See :func:`deep_rnn_with_skip_connections`. name: Name of the module. """ super().__init__(name) self._layers = layers if layers is not None else [] self._skip_connections = skip_connections self._concat_final_output_if_skip = concat_final_output_if_skip def __call__(self, inputs, prev_state): """See base class.""" current_inputs = inputs outputs = [] next_states = [] recurrent_idx = 0 concat = lambda *args: tf.concat(args, axis=-1) for idx, layer in enumerate(self._layers): if self._skip_connections and idx > 0: current_inputs = tree.map_structure(concat, inputs, current_inputs) if isinstance(layer, RNNCore): current_inputs, next_state = layer(current_inputs, prev_state[recurrent_idx]) next_states.append(next_state) recurrent_idx += 1 else: current_inputs = layer(current_inputs) if self._skip_connections: outputs.append(current_inputs) if self._skip_connections and self._concat_final_output_if_skip: outputs = tree.map_structure(concat, *outputs) else: outputs = current_inputs return outputs, tuple(next_states) def initial_state(self, batch_size, **kwargs): """See base class.""" return tuple( layer.initial_state(batch_size, **kwargs) for layer in self._layers if isinstance(layer, RNNCore)) class DeepRNN(_LegacyDeepRNN): r"""Linear chain of :class:`RNNCore`\ s or callables. The core takes ``(input, prev_state)`` as input and passes the input through each internal module in the order they were presented, using elements from ``prev_state`` as necessary for internal RNN cores. >>> deep_rnn = snt.DeepRNN([ ... snt.LSTM(hidden_size=16), ... snt.LSTM(hidden_size=16), ... ]) Note that the state of a ``DeepRNN`` is always a tuple, which will contain the same number of elements as there are internal RNN cores. If no internal modules are RNN cores, the state of the ``DeepRNN`` as a whole is an empty tuple. Wrapping non-recurrent modules into a ``DeepRNN`` can be useful to produce something API compatible with a "real" recurrent module, simplifying code that handles the cores. """ # TODO(slebedev): currently called `layers` to be in-sync with `Sequential`. def __init__(self, layers, name: Optional[str] = None): super().__init__(layers, skip_connections=False, name=name) def deep_rnn_with_skip_connections( layers: Sequence[RNNCore], concat_final_output: bool = True, name: str = "deep_rnn_with_skip_connections") -> RNNCore: r"""Constructs a :class:`DeepRNN` with skip connections. Skip connections alter the dependency structure within a :class:`DeepRNN`. Specifically, input to the i-th layer (i > 0) is given by a concatenation of the core's inputs and the outputs of the (i-1)-th layer. :: outputs0, ... = layers[0](inputs, ...) outputs1, ... = layers[1](tf.concat([inputs, outputs0], axis=1], ...) outputs2, ... = layers[2](tf.concat([inputs, outputs1], axis=1], ...) ... This allows the layers to learn decoupled features. Args: layers: A list of :class:`RNNCore`\ s. concat_final_output: If enabled (default), the outputs of the core is a concatenation of the outputs of all intermediate layers; otherwise, only the outputs of the final layer, i.e. that of ``layers[-1]``, are returned. name: Name of the module. Returns: A :class:`DeepRNN` with skip connections. Raises: ValueError: If any of the layers is not an :class:`RNNCore`. """ if not all(isinstance(l, RNNCore) for l in layers): raise ValueError("deep_rnn_with_skip_connections requires all layers to be " "instances of RNNCore") return _LegacyDeepRNN( layers, skip_connections=True, concat_final_output_if_skip=concat_final_output, name=name) class _ResidualWrapper(RNNCore): """Residual connection wrapper for a base :class:`RNNCore`. The output of the wrapper is the sum of the outputs of the base core with its inputs. """ def __init__(self, base_core: RNNCore): super().__init__(name=base_core.name + "_residual") self._base_core = base_core def __call__(self, inputs: types.TensorNest, prev_state: types.TensorNest): """See base class.""" outputs, next_state = self._base_core(inputs, prev_state) residual = tree.map_structure(lambda i, o: i + o, inputs, outputs) return residual, next_state def initial_state(self, batch_size, **kwargs): return self._base_core.initial_state(batch_size, **kwargs) def deep_rnn_with_residual_connections( layers: Sequence[RNNCore], name: str = "deep_rnn_with_residual_connections") -> RNNCore: r"""Constructs a :class:`DeepRNN` with residual connections. Residual connections alter the dependency structure in a :class:`DeepRNN`. Specifically, the input to the i-th intermediate layer is a sum of the original core's inputs and the outputs of all the preceding layers (